@akanjs/devkit 3.1.0-rc.1 → 3.1.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/akanConfig/akanConfig.test.ts +51 -5
- package/akanConfig/akanConfig.ts +31 -12
- package/akanConfig/akanWorkspaceConfig.test.ts +6 -6
- package/akanConfig/akanWorkspaceConfig.ts +1 -1
- package/applicationBuildRunner.test.ts +33 -1
- package/applicationBuildRunner.ts +4 -5
- package/cloud/cloudApi.ts +22 -15
- package/cloud/cloudHttpError.test.ts +32 -0
- package/cloud/cloudHttpError.ts +54 -0
- package/cloud/cloudUpdates.ts +138 -0
- package/cloud/globalConfig.test.ts +17 -0
- package/cloud/globalConfig.ts +8 -2
- package/cloud/index.ts +3 -0
- package/cloud/pipelineSecrets.ts +114 -0
- package/cloud/updatePackUpload.test.ts +100 -45
- package/cloud/updatePackUpload.ts +38 -18
- package/commandDecorators/argMeta.ts +11 -2
- package/commandDecorators/command.ts +40 -6
- package/commandDecorators/commandBuilder.ts +6 -1
- package/commandDecorators/commandDecorators.test.ts +205 -4
- package/commandDecorators/exitCodeError.ts +15 -0
- package/commandDecorators/index.ts +1 -0
- package/executors.test.ts +46 -0
- package/executors.ts +26 -23
- package/frontendBuild/csrArtifactBuilder.ts +18 -1
- package/incrementalBuilder/buildBatch.proc.ts +12 -7
- package/incrementalBuilder/incrementalBuilder.proc.test.ts +63 -0
- package/incrementalBuilder/incrementalBuilder.proc.ts +7 -5
- package/mobile/desktopServerStage.test.ts +13 -0
- package/mobile/desktopServerStage.ts +5 -7
- package/mobile/index.ts +2 -0
- package/mobile/nativeApi.test.ts +26 -0
- package/mobile/nativeApi.ts +5 -2
- package/mobile/nativeApp.test.ts +135 -33
- package/mobile/nativeApp.ts +109 -73
- package/mobile/nativeConfig.ts +2 -1
- package/mobile/nativePublicEnv.test.ts +131 -0
- package/mobile/nativePublicEnv.ts +116 -0
- package/mobile/serverBundleSecrets.test.ts +79 -0
- package/mobile/serverBundleSecrets.ts +60 -16
- package/mobile/updatePackFolder.test.ts +98 -0
- package/mobile/updatePackFolder.ts +131 -0
- package/mobile/updateUploader.test.ts +114 -22
- package/mobile/updateUploader.ts +75 -8
- package/package.json +2 -2
- package/pipeline/PipelineCompiler.test.ts +157 -1
- package/pipeline/PipelineCompiler.ts +80 -46
- package/pipeline/PipelineDefinition.test.ts +9 -0
- package/pipeline/PipelineDefinition.ts +4 -3
- package/pipeline/PipelineDryRun.test.ts +27 -0
- package/pipeline/PipelineDryRun.ts +2 -2
- package/pipeline/PipelineInputReader.ts +157 -253
- package/pipeline/PipelineRules.ts +92 -0
- package/pipeline/index.ts +1 -0
- package/qualityScanner.test.ts +5 -3
- package/slicePlanner.ts +5 -3
- package/subspace.test.ts +90 -19
- package/subspace.ts +52 -500
- package/subspaceClone.ts +361 -0
- package/subspaceConfig.ts +1 -2
- package/subspaceReport.ts +162 -0
- package/transforms/serverEnvPlugin.test.ts +76 -0
- package/transforms/serverEnvPlugin.ts +9 -25
- package/transforms/serverEnvProfile.test.ts +189 -0
- package/transforms/serverEnvProfile.ts +324 -60
- package/typecheck/projectTypecheck.ts +6 -5
|
@@ -8,21 +8,20 @@ interface SecretValue {
|
|
|
8
8
|
value: string;
|
|
9
9
|
}
|
|
10
10
|
|
|
11
|
-
//* A
|
|
12
|
-
//* workspace's server env files hold is looked for in the server the app carries, whatever put it there.
|
|
11
|
+
//* A release leaves the building, signed or not, and an update goes to a feed anyone reads: before either, every secret
|
|
12
|
+
//* value the workspace's server env files hold is looked for in the server the app carries, whatever put it there.
|
|
13
13
|
export class ServerBundleSecrets {
|
|
14
14
|
static readonly #largestFile = 64 * 1024 * 1024;
|
|
15
|
+
static readonly #printer = new Bun.Transpiler({ loader: "js", target: "bun" });
|
|
16
|
+
static readonly #notSource = new Set(["node_modules", "env", "private", "public", "dist"]);
|
|
15
17
|
|
|
16
18
|
constructor(
|
|
17
19
|
readonly workspaceRoot: string,
|
|
18
20
|
readonly appDir: string,
|
|
19
21
|
) {}
|
|
20
22
|
|
|
21
|
-
/** The app's `env/env.server.<name>.ts` and every lib's, but the type and example files. */
|
|
22
23
|
async envFiles(): Promise<string[]> {
|
|
23
|
-
const
|
|
24
|
-
const libs = await readdir(libsDir).catch(() => [] as string[]);
|
|
25
|
-
const dirs = [path.join(this.appDir, "env"), ...libs.map((lib) => path.join(libsDir, lib, "env"))];
|
|
24
|
+
const dirs = [path.join(this.appDir, "env"), ...(await this.#libDirs()).map((lib) => path.join(lib, "env"))];
|
|
26
25
|
const files = await Promise.all(
|
|
27
26
|
dirs.map(async (dir) =>
|
|
28
27
|
(await readdir(dir).catch(() => [] as string[]))
|
|
@@ -45,27 +44,59 @@ export class ServerBundleSecrets {
|
|
|
45
44
|
unread.push(ServerBundleSecrets.#relative(this.workspaceRoot, file));
|
|
46
45
|
}
|
|
47
46
|
}
|
|
48
|
-
|
|
47
|
+
if (!secrets.length) return unread;
|
|
48
|
+
const sought = secrets.map((secret) => ({ secret, needles: ServerBundleSecrets.#needles(secret.value) }));
|
|
49
|
+
const found: { bundled: string; secret: SecretValue }[] = [];
|
|
49
50
|
for (const bundled of await ServerBundleSecrets.#files(bundleDir)) {
|
|
50
51
|
const text = await readFile(bundled, "latin1");
|
|
51
|
-
for (const secret of
|
|
52
|
-
if (
|
|
53
|
-
found.push(
|
|
54
|
-
`${ServerBundleSecrets.#relative(bundleDir, bundled)} holds ${ServerBundleSecrets.#relative(this.workspaceRoot, secret.file)} ${secret.path}`,
|
|
55
|
-
);
|
|
52
|
+
for (const { secret, needles } of sought)
|
|
53
|
+
if (needles.some((needle) => text.includes(needle))) found.push({ bundled, secret });
|
|
56
54
|
}
|
|
57
|
-
|
|
55
|
+
const coded = found.length ? await this.#coded(found.map(({ secret }) => secret.value)) : new Set<string>();
|
|
56
|
+
const leaks = found
|
|
57
|
+
.filter(({ secret }) => !coded.has(secret.value))
|
|
58
|
+
.map(
|
|
59
|
+
({ bundled, secret }) =>
|
|
60
|
+
`${ServerBundleSecrets.#relative(bundleDir, bundled)} holds ${ServerBundleSecrets.#relative(this.workspaceRoot, secret.file)} ${secret.path}`,
|
|
61
|
+
);
|
|
62
|
+
if (leaks.length)
|
|
58
63
|
throw new Error(
|
|
59
|
-
`The server this app carries holds secret values
|
|
64
|
+
`The server this app carries holds secret values of the workspace's env files, so the release stops here:\n ${[...new Set(leaks)].join("\n ")}\nBuild its server with a profile (native.desktop.server.profile), whose bundle leaves the env files' secrets out. A value a profile build still holds sits in code or a file the server carries: take it out there.`,
|
|
60
65
|
);
|
|
61
66
|
return unread;
|
|
62
67
|
}
|
|
63
68
|
|
|
64
|
-
//? A
|
|
69
|
+
//? A short or plain value the app's or a lib's own code holds as well (a default password) is that code's, which
|
|
70
|
+
//? the bundle carries with or without the env file; a long mixed one is a secret copied into the code.
|
|
71
|
+
async #coded(values: string[]): Promise<Set<string>> {
|
|
72
|
+
const plain = [...new Set(values)].filter((value) => !ServerEnvProfile.distinctive(value));
|
|
73
|
+
const coded = new Set<string>();
|
|
74
|
+
if (!plain.length) return coded;
|
|
75
|
+
for (const dir of [this.appDir, ...(await this.#libDirs())])
|
|
76
|
+
for (const file of await ServerBundleSecrets.#sources(dir)) {
|
|
77
|
+
const text = await readFile(file, "utf8");
|
|
78
|
+
for (const value of plain) if (text.includes(value)) coded.add(value);
|
|
79
|
+
}
|
|
80
|
+
return coded;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async #libDirs() {
|
|
84
|
+
const libsDir = path.join(this.workspaceRoot, "libs");
|
|
85
|
+
return (await readdir(libsDir).catch(() => [] as string[])).map((lib) => path.join(libsDir, lib));
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
//? A multi-line key sits in a bundle as a JS string, escaped; one long line of it is enough to recognise. Bun prints
|
|
89
|
+
//? a string's non-ASCII characters as `\xNN` / `\uNNNN`, and the bundle is read byte for byte, so as UTF-8.
|
|
65
90
|
static #needles(value: string): string[] {
|
|
66
91
|
const escaped = JSON.stringify(value).slice(1, -1);
|
|
67
92
|
const line = value.split(/\r?\n/).find((part) => part.length >= 32);
|
|
68
|
-
|
|
93
|
+
const forms = [value, escaped, ServerBundleSecrets.#printed(value), ...(line ? [line] : [])];
|
|
94
|
+
return [...new Set(forms.map((form) => Buffer.from(form, "utf8").toString("latin1")))];
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
static #printed(value: string) {
|
|
98
|
+
const code = ServerBundleSecrets.#printer.transformSync(`export default ${JSON.stringify(value)};`);
|
|
99
|
+
return /^export default (["'`])([\s\S]*)\1;?\s*$/.exec(code)?.[2] ?? value;
|
|
69
100
|
}
|
|
70
101
|
|
|
71
102
|
static #relative(from: string, to: string) {
|
|
@@ -82,4 +113,17 @@ export class ServerBundleSecrets {
|
|
|
82
113
|
}
|
|
83
114
|
return files;
|
|
84
115
|
}
|
|
116
|
+
|
|
117
|
+
static async #sources(dir: string): Promise<string[]> {
|
|
118
|
+
const files: string[] = [];
|
|
119
|
+
for (const entry of await readdir(dir, { withFileTypes: true }).catch(() => [])) {
|
|
120
|
+
if (entry.name.startsWith(".")) continue;
|
|
121
|
+
const file = path.join(dir, entry.name);
|
|
122
|
+
if (entry.isDirectory()) {
|
|
123
|
+
if (!ServerBundleSecrets.#notSource.has(entry.name)) files.push(...(await ServerBundleSecrets.#sources(file)));
|
|
124
|
+
} else if (/\.(?:[cm]?[jt]sx?|json)$/.test(entry.name) && !/\.(?:test|spec)\.[cm]?[jt]sx?$/.test(entry.name))
|
|
125
|
+
files.push(file);
|
|
126
|
+
}
|
|
127
|
+
return files;
|
|
128
|
+
}
|
|
85
129
|
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { tempDirs, writeText } from "../testHelpers";
|
|
4
|
+
import { UpdatePackFolder } from "./updatePackFolder";
|
|
5
|
+
|
|
6
|
+
const makeTempRoot = tempDirs("akan-update-pack-");
|
|
7
|
+
|
|
8
|
+
describe("a pack read back from its folder", () => {
|
|
9
|
+
test("takes the folder as the pack's place, and refuses what is not a takeable pack", async () => {
|
|
10
|
+
const root = await makeTempRoot();
|
|
11
|
+
const pack = {
|
|
12
|
+
schema: 1,
|
|
13
|
+
app: "lu",
|
|
14
|
+
target: "default",
|
|
15
|
+
platform: "android",
|
|
16
|
+
env: "main",
|
|
17
|
+
channel: "main",
|
|
18
|
+
appId: "com.lu",
|
|
19
|
+
appVersion: "1.0.0",
|
|
20
|
+
nativeApi: "0123456789abcdef",
|
|
21
|
+
webHash: "f".repeat(64),
|
|
22
|
+
files: 1,
|
|
23
|
+
size: 13,
|
|
24
|
+
out: "/elsewhere/updates/android",
|
|
25
|
+
};
|
|
26
|
+
const moved = path.join(root, "moved");
|
|
27
|
+
await writeText(path.join(moved, "pack.json"), JSON.stringify(pack));
|
|
28
|
+
expect(await UpdatePackFolder.read(moved)).toEqual({ ...pack, out: moved } as never);
|
|
29
|
+
|
|
30
|
+
await expect(UpdatePackFolder.read(path.join(root, "empty"))).rejects.toThrow("holds no pack.json");
|
|
31
|
+
const later = path.join(root, "later");
|
|
32
|
+
await writeText(path.join(later, "pack.json"), JSON.stringify({ ...pack, schema: 2 }));
|
|
33
|
+
await expect(UpdatePackFolder.read(later)).rejects.toThrow("schema 2");
|
|
34
|
+
const refused = path.join(root, "refused");
|
|
35
|
+
const compat = { against: "store/bundle.json", compatible: false, problems: ["deepLinks differ"] };
|
|
36
|
+
await writeText(path.join(refused, "pack.json"), JSON.stringify({ ...pack, compat }));
|
|
37
|
+
expect((await UpdatePackFolder.read(refused)).compat).toEqual(compat);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("is written without the folder it was packed to, which holds this machine's paths", async () => {
|
|
41
|
+
const out = path.join(await makeTempRoot(), "updates", "android");
|
|
42
|
+
const pack = { schema: 1, app: "lu", target: "default", platform: "android", files: 1, out } as const;
|
|
43
|
+
|
|
44
|
+
await UpdatePackFolder.write(pack as never);
|
|
45
|
+
|
|
46
|
+
const written = await Bun.file(path.join(out, "pack.json")).text();
|
|
47
|
+
expect(written).not.toContain(out);
|
|
48
|
+
expect(JSON.parse(written)).toEqual({ schema: 1, app: "lu", target: "default", platform: "android", files: 1 });
|
|
49
|
+
expect((await UpdatePackFolder.read(out)).out).toBe(out);
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
describe("a pack's verdict on the store build it was compared with", () => {
|
|
54
|
+
test("is none without a comparison or for one it runs in, and exit 2 with what differs otherwise", () => {
|
|
55
|
+
const out = "/work/updates/android";
|
|
56
|
+
expect(UpdatePackFolder.storeBuildVerdict({ out })).toBeNull();
|
|
57
|
+
const against = "store/bundle.json";
|
|
58
|
+
expect(UpdatePackFolder.storeBuildVerdict({ out, compat: { against, compatible: true, problems: [] } })).toBeNull();
|
|
59
|
+
|
|
60
|
+
const verdict = UpdatePackFolder.storeBuildVerdict(
|
|
61
|
+
{ out, compat: { against, compatible: false, problems: ["deepLinks differ", "camera added"] } },
|
|
62
|
+
"Run `akan release-android lu`, then register that build.",
|
|
63
|
+
);
|
|
64
|
+
expect(verdict?.exitCode).toBe(2);
|
|
65
|
+
expect(verdict?.message).toBe(
|
|
66
|
+
`The pack in ${out} needs a new store build; it differs from the one ${against} came from:\n- deepLinks differ\n- camera added\nRun \`akan release-android lu\`, then register that build.`,
|
|
67
|
+
);
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
describe("a pack's archive", () => {
|
|
72
|
+
test("holds the files its template lists and the records there, never another pack's files", async () => {
|
|
73
|
+
const out = path.join(await makeTempRoot(), "updates", "android");
|
|
74
|
+
const listed = "a".repeat(64);
|
|
75
|
+
await writeText(path.join(out, "manifest.template.json"), JSON.stringify({ files: [{ sha256: listed }] }));
|
|
76
|
+
await writeText(path.join(out, "bundle.json"), "{}");
|
|
77
|
+
await writeText(path.join(out, "pack.json"), "{}");
|
|
78
|
+
await writeText(path.join(out, "files", listed), "listed");
|
|
79
|
+
await writeText(path.join(out, "files", "b".repeat(64)), "an earlier pack's");
|
|
80
|
+
|
|
81
|
+
const entries = await UpdatePackFolder.withArchive({ out } as never, async (archive) =>
|
|
82
|
+
Bun.spawnSync(["tar", "-tzf", archive]).stdout.toString().split(/\r?\n/).filter(Boolean).sort(),
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
expect(entries).toEqual(["./bundle.json", `./files/${listed}`, "./manifest.template.json", "./pack.json"]);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test("refuses a template that names a file by anything but its sha256", async () => {
|
|
89
|
+
const out = path.join(await makeTempRoot(), "pack");
|
|
90
|
+
await writeText(path.join(out, "manifest.template.json"), JSON.stringify({ files: [{ sha256: "../pack.json" }] }));
|
|
91
|
+
await expect(UpdatePackFolder.withArchive({ out } as never, async () => null)).rejects.toThrow(
|
|
92
|
+
"lists a file whose sha256 is not 64 hex digits",
|
|
93
|
+
);
|
|
94
|
+
await expect(
|
|
95
|
+
UpdatePackFolder.withArchive({ out: path.join(out, "none") } as never, async () => null),
|
|
96
|
+
).rejects.toThrow("holds no manifest.template.json");
|
|
97
|
+
});
|
|
98
|
+
});
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { mkdtemp, rm } from "node:fs/promises";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import type { BundleInfo, UpdateManifest } from "@akanjs/native/api";
|
|
5
|
+
import type { NativeEnv } from "akanjs";
|
|
6
|
+
import { ExitCodeError } from "../commandDecorators/exitCodeError";
|
|
7
|
+
import type { MobilePlatform } from "./mobileTarget";
|
|
8
|
+
|
|
9
|
+
export interface NativeUpdatePackCompat {
|
|
10
|
+
/** The store build's bundle.json the pack was compared with. */
|
|
11
|
+
against: string;
|
|
12
|
+
compatible: boolean;
|
|
13
|
+
problems: string[];
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** `<out>/pack.json`: what a signer needs of a pack, read instead of the log. */
|
|
17
|
+
export interface NativeUpdatePack {
|
|
18
|
+
schema: 1;
|
|
19
|
+
app: string;
|
|
20
|
+
target: string;
|
|
21
|
+
platform: MobilePlatform;
|
|
22
|
+
env: NativeEnv;
|
|
23
|
+
channel: string;
|
|
24
|
+
appId: string;
|
|
25
|
+
appVersion: string;
|
|
26
|
+
nativeApi: string;
|
|
27
|
+
/** bundle.json `web.hash`: the web files as served, the part of a bundle id that names its content. */
|
|
28
|
+
webHash: string;
|
|
29
|
+
/** bundle.json `web.publicEnvHash`: the AKAN_PUBLIC_* the page inlined, which the store build must share. */
|
|
30
|
+
publicEnvHash?: string;
|
|
31
|
+
files: number;
|
|
32
|
+
size: number;
|
|
33
|
+
/** The folder it was read from or packed to; pack.json leaves it out. */
|
|
34
|
+
out: string;
|
|
35
|
+
compat?: NativeUpdatePackCompat;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
//* The folder `akan pack-update` writes for a signer or Akan Cloud: the native build API writes `files/<sha256>`,
|
|
39
|
+
//* bundle.json and manifest.template.json, and akan writes pack.json and compat.json beside them.
|
|
40
|
+
export class UpdatePackFolder {
|
|
41
|
+
static readonly packFile = "pack.json";
|
|
42
|
+
static readonly compatFile = "compat.json";
|
|
43
|
+
static readonly bundleFile = "bundle.json";
|
|
44
|
+
static readonly templateFile = "manifest.template.json";
|
|
45
|
+
static readonly filesDir = "files";
|
|
46
|
+
static readonly #records = [
|
|
47
|
+
UpdatePackFolder.templateFile,
|
|
48
|
+
UpdatePackFolder.bundleFile,
|
|
49
|
+
UpdatePackFolder.packFile,
|
|
50
|
+
UpdatePackFolder.compatFile,
|
|
51
|
+
];
|
|
52
|
+
static readonly #sha256 = /^[0-9a-f]{64}$/;
|
|
53
|
+
|
|
54
|
+
/** akan's own records: a pack that fails must not leave the previous pack's behind for whoever reads the exit code. */
|
|
55
|
+
static async clear(dir: string) {
|
|
56
|
+
await Promise.all(
|
|
57
|
+
[UpdatePackFolder.packFile, UpdatePackFolder.compatFile].map((name) => rm(path.join(dir, name), { force: true })),
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** A pack read back from its folder wherever it was moved. */
|
|
62
|
+
static async read(dir: string): Promise<NativeUpdatePack> {
|
|
63
|
+
const file = Bun.file(path.join(dir, UpdatePackFolder.packFile));
|
|
64
|
+
if (!(await file.exists()))
|
|
65
|
+
throw new Error(`${dir} holds no ${UpdatePackFolder.packFile}: give the folder \`akan pack-update\` wrote.`);
|
|
66
|
+
const pack = (await file.json()) as NativeUpdatePack;
|
|
67
|
+
if (pack.schema !== 1)
|
|
68
|
+
throw new Error(`${dir}/${UpdatePackFolder.packFile} is schema ${pack.schema}, and this akan reads schema 1.`);
|
|
69
|
+
return { ...pack, out: path.resolve(dir) };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
//? Without `out`: the record travels in the pack's archive, and the folder is wherever it is read from.
|
|
73
|
+
static async write({ out, ...record }: NativeUpdatePack) {
|
|
74
|
+
await Bun.write(path.join(out, UpdatePackFolder.packFile), `${JSON.stringify(record, null, 2)}\n`);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
static async writeCompat(pack: NativeUpdatePack, compat: NativeUpdatePackCompat) {
|
|
78
|
+
pack.compat = compat;
|
|
79
|
+
await Bun.write(path.join(pack.out, UpdatePackFolder.compatFile), `${JSON.stringify(compat, null, 2)}\n`);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** The bundle.json a store build and a pack both carry. */
|
|
83
|
+
static async readBundle(file: string): Promise<BundleInfo> {
|
|
84
|
+
return (await Bun.file(file).json()) as BundleInfo;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
//* Exit 2, not a thrown 1: a CI tells "this needs a new store build" apart from a pack that broke.
|
|
88
|
+
static storeBuildVerdict({ out, compat }: Pick<NativeUpdatePack, "out" | "compat">, then?: string) {
|
|
89
|
+
if (!compat || compat.compatible) return null;
|
|
90
|
+
return new ExitCodeError(
|
|
91
|
+
`The pack in ${out} needs a new store build; it differs from the one ${compat.against} came from:\n- ${compat.problems.join("\n- ")}${then ? `\n${then}` : ""}`,
|
|
92
|
+
2,
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** The pack as the tgz Akan Cloud takes: the files its manifest template lists, and the records beside them. */
|
|
97
|
+
static async withArchive<T>(pack: NativeUpdatePack, use: (archive: string) => Promise<T>): Promise<T> {
|
|
98
|
+
const { templateFile, filesDir } = UpdatePackFolder;
|
|
99
|
+
const template = Bun.file(path.join(pack.out, templateFile));
|
|
100
|
+
if (!(await template.exists()))
|
|
101
|
+
throw new Error(`${pack.out} holds no ${templateFile}: give the folder \`akan pack-update\` wrote.`);
|
|
102
|
+
const { files = [] } = (await template.json()) as Partial<UpdateManifest>;
|
|
103
|
+
const hashes = [...new Set(files.map((file) => file.sha256 as unknown))];
|
|
104
|
+
if (!hashes.every((hash): hash is string => typeof hash === "string" && UpdatePackFolder.#sha256.test(hash)))
|
|
105
|
+
throw new Error(`${pack.out}/${templateFile} lists a file whose sha256 is not 64 hex digits.`);
|
|
106
|
+
const present = await Promise.all(
|
|
107
|
+
UpdatePackFolder.#records.map((name) => Bun.file(path.join(pack.out, name)).exists()),
|
|
108
|
+
);
|
|
109
|
+
const entries = [
|
|
110
|
+
...UpdatePackFolder.#records.filter((_, idx) => present[idx]),
|
|
111
|
+
...hashes.map((hash) => `${filesDir}/${hash}`),
|
|
112
|
+
];
|
|
113
|
+
const dir = await mkdtemp(path.join(os.tmpdir(), "akan-pack-"));
|
|
114
|
+
try {
|
|
115
|
+
//? files/ keeps every earlier pack's files (content-addressed), so the folder itself is not the pack; the names
|
|
116
|
+
//? go on stdin, not argv, because a Windows command line holds 32 KiB.
|
|
117
|
+
// A relative archive name, so GNU tar never reads a Windows drive letter (`C:\…`) as a remote `host:file` spec.
|
|
118
|
+
const tar = Bun.spawn(["tar", "-czf", "pack.tgz", "-C", pack.out, "-T", "-"], {
|
|
119
|
+
cwd: dir,
|
|
120
|
+
stdin: new Blob([entries.map((entry) => `./${entry}\n`).join("")]),
|
|
121
|
+
stdout: "ignore",
|
|
122
|
+
stderr: "pipe",
|
|
123
|
+
});
|
|
124
|
+
const [code, stderr] = await Promise.all([tar.exited, new Response(tar.stderr).text()]);
|
|
125
|
+
if (code !== 0) throw new Error(`tar of ${pack.out} failed: ${stderr.trim()}`);
|
|
126
|
+
return await use(path.join(dir, "pack.tgz"));
|
|
127
|
+
} finally {
|
|
128
|
+
await rm(dir, { recursive: true, force: true });
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
@@ -16,34 +16,126 @@ const fakeBucket = (held: string[] = []) => {
|
|
|
16
16
|
return { bucket, writes };
|
|
17
17
|
};
|
|
18
18
|
|
|
19
|
+
const sha = (char: string) => char.repeat(64);
|
|
20
|
+
|
|
21
|
+
const inUpdatesDir = async (files: Record<string, string>, run: (updatesDir: string) => Promise<void>) => {
|
|
22
|
+
const updatesDir = await mkdtemp(path.join(tmpdir(), "akan-update-upload-"));
|
|
23
|
+
try {
|
|
24
|
+
for (const [file, content] of Object.entries(files)) {
|
|
25
|
+
await mkdir(path.dirname(path.join(updatesDir, file)), { recursive: true });
|
|
26
|
+
await writeFile(path.join(updatesDir, file), content);
|
|
27
|
+
}
|
|
28
|
+
await run(updatesDir);
|
|
29
|
+
} finally {
|
|
30
|
+
await rm(updatesDir, { recursive: true, force: true });
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
const envelope = (manifest: object) =>
|
|
35
|
+
JSON.stringify({
|
|
36
|
+
manifest: Buffer.from(JSON.stringify(manifest)).toString("base64"),
|
|
37
|
+
keyId: "0011223344556677",
|
|
38
|
+
signature: "c2lnbmF0dXJl",
|
|
39
|
+
});
|
|
40
|
+
|
|
19
41
|
describe("UpdateUploader", () => {
|
|
20
42
|
test("reads s3://bucket/prefix and refuses anything else", () => {
|
|
21
|
-
expect(UpdateUploader.parseTarget("s3://releases/
|
|
43
|
+
expect(UpdateUploader.parseTarget("s3://releases/acme/kiosk/")).toEqual({
|
|
22
44
|
bucket: "releases",
|
|
23
|
-
prefix: "
|
|
45
|
+
prefix: "acme/kiosk",
|
|
24
46
|
});
|
|
25
47
|
expect(UpdateUploader.parseTarget("s3://releases")).toEqual({ bucket: "releases", prefix: "" });
|
|
26
|
-
expect(() => UpdateUploader.parseTarget("https://releases.example/
|
|
48
|
+
expect(() => UpdateUploader.parseTarget("https://releases.example/acme")).toThrow("s3://<bucket>/<prefix>");
|
|
27
49
|
});
|
|
28
50
|
|
|
29
|
-
test("uploads
|
|
30
|
-
const
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
51
|
+
test("uploads a desktop release's archives the bucket lacks first, then the signature before the manifest", async () => {
|
|
52
|
+
const manifest = {
|
|
53
|
+
bundle: "1.2.0-2000",
|
|
54
|
+
archive: { url: `app/${sha("b")}.tar.gz` },
|
|
55
|
+
patches: [{ url: `app/${sha("a")}-${sha("b")}.delta.gz` }],
|
|
56
|
+
};
|
|
57
|
+
await inUpdatesDir(
|
|
58
|
+
{
|
|
59
|
+
[`windows-x64/app/${sha("a")}.tar.gz`]: "previous release",
|
|
60
|
+
[`windows-x64/app/${sha("b")}.tar.gz`]: "tar",
|
|
61
|
+
[`windows-x64/app/${sha("a")}-${sha("b")}.delta.gz`]: "delta",
|
|
62
|
+
"windows-x64/main.json": JSON.stringify(manifest),
|
|
63
|
+
"windows-x64/main.json.sig": "sig",
|
|
64
|
+
"windows-x64/main.json.tmp": "{}",
|
|
65
|
+
"windows-x64/develop.json": JSON.stringify({ ...manifest, bundle: "1.1.0-1000" }),
|
|
66
|
+
"windows-x64/develop.json.sig": "sig",
|
|
67
|
+
},
|
|
68
|
+
async (updatesDir) => {
|
|
69
|
+
const { bucket, writes } = fakeBucket([`acme/windows-x64/app/${sha("a")}-${sha("b")}.delta.gz`]);
|
|
70
|
+
const release = { dir: path.join(updatesDir, "windows-x64"), channel: "main", bundle: "1.2.0-2000" };
|
|
71
|
+
|
|
72
|
+
expect(await new UpdateUploader(bucket, "acme").upload(updatesDir, release)).toEqual({
|
|
73
|
+
uploaded: 3,
|
|
74
|
+
skipped: 1,
|
|
75
|
+
});
|
|
76
|
+
expect(writes).toEqual([
|
|
77
|
+
{ key: `acme/windows-x64/app/${sha("b")}.tar.gz` },
|
|
78
|
+
{ key: "acme/windows-x64/main.json.sig", type: "text/plain" },
|
|
79
|
+
{ key: "acme/windows-x64/main.json", type: "application/json" },
|
|
80
|
+
]);
|
|
81
|
+
},
|
|
82
|
+
);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test("uploads a phone release's own files and envelope, never pack records or another manifest beside them", async () => {
|
|
86
|
+
await inUpdatesDir(
|
|
87
|
+
{
|
|
88
|
+
"android/pack.json": JSON.stringify({ out: "/Users/someone/work/dist/native/demo/default/updates/android" }),
|
|
89
|
+
"android/manifest.template.json": "{}",
|
|
90
|
+
"android/bundle.json": "{}",
|
|
91
|
+
"android/compat.json": "{}",
|
|
92
|
+
[`android/files/${sha("a")}`]: "index.html",
|
|
93
|
+
[`android/files/${sha("b")}`]: "env.runtime.json",
|
|
94
|
+
[`android/files/${sha("c")}`]: "an earlier pack's file",
|
|
95
|
+
"android/main/api2.json": envelope({
|
|
96
|
+
bundle: "3000-aaaaaaaa",
|
|
97
|
+
files: [{ sha256: sha("a") }, { sha256: sha("b") }, { sha256: sha("a") }],
|
|
98
|
+
}),
|
|
99
|
+
"android/main/api1.json": envelope({ bundle: "1000-cccccccc", files: [{ sha256: sha("c") }] }),
|
|
100
|
+
"android/develop/api2.json": envelope({ bundle: "2000-bbbbbbbb", files: [{ sha256: sha("b") }] }),
|
|
101
|
+
},
|
|
102
|
+
async (updatesDir) => {
|
|
103
|
+
const { bucket, writes } = fakeBucket();
|
|
104
|
+
const release = { dir: path.join(updatesDir, "android"), channel: "main", bundle: "3000-aaaaaaaa" };
|
|
105
|
+
|
|
106
|
+
expect(await new UpdateUploader(bucket, "").upload(updatesDir, release)).toEqual({ uploaded: 3, skipped: 0 });
|
|
107
|
+
expect(writes).toEqual([
|
|
108
|
+
{ key: `android/files/${sha("a")}` },
|
|
109
|
+
{ key: `android/files/${sha("b")}` },
|
|
110
|
+
{ key: "android/main/api2.json", type: "application/json" },
|
|
111
|
+
]);
|
|
112
|
+
},
|
|
113
|
+
);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
test("uploads nothing when the release is not there or names a file outside its folder", async () => {
|
|
117
|
+
await inUpdatesDir(
|
|
118
|
+
{
|
|
119
|
+
"linux-x64/main.json": JSON.stringify({ bundle: "1.0.0-10", archive: { url: "../../.ssh/id_ed25519" } }),
|
|
120
|
+
"linux-x64/main.json.sig": "sig",
|
|
121
|
+
"ios/main/api1.json": envelope({ bundle: "10-aaaaaaaa", files: [{ sha256: "../pack.json" }] }),
|
|
122
|
+
},
|
|
123
|
+
async (updatesDir) => {
|
|
124
|
+
const { bucket, writes } = fakeBucket();
|
|
125
|
+
const uploader = new UpdateUploader(bucket, "acme");
|
|
126
|
+
const linux = path.join(updatesDir, "linux-x64");
|
|
127
|
+
|
|
128
|
+
await expect(uploader.upload(updatesDir, { dir: linux, channel: "main", bundle: "1.0.0-9" })).rejects.toThrow(
|
|
129
|
+
"holds no main manifest of release 1.0.0-9",
|
|
130
|
+
);
|
|
131
|
+
await expect(uploader.upload(updatesDir, { dir: linux, channel: "main", bundle: "1.0.0-10" })).rejects.toThrow(
|
|
132
|
+
"which is not inside that folder",
|
|
133
|
+
);
|
|
134
|
+
await expect(
|
|
135
|
+
uploader.upload(updatesDir, { dir: path.join(updatesDir, "ios"), channel: "main", bundle: "10-aaaaaaaa" }),
|
|
136
|
+
).rejects.toThrow("not 64 hex digits");
|
|
137
|
+
expect(writes).toEqual([]);
|
|
138
|
+
},
|
|
139
|
+
);
|
|
48
140
|
});
|
|
49
141
|
});
|
package/mobile/updateUploader.ts
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
|
+
import { readdir } from "node:fs/promises";
|
|
1
2
|
import path from "node:path";
|
|
3
|
+
import type { UpdateEnvelope, UpdateManifest } from "@akanjs/native/api";
|
|
4
|
+
import { isRecord } from "akanjs/common";
|
|
5
|
+
import { UpdatePackFolder } from "./updatePackFolder";
|
|
2
6
|
|
|
3
7
|
/** What the uploader needs of an S3 client; Bun.S3Client is one. */
|
|
4
8
|
export interface UpdateBucket {
|
|
@@ -11,6 +15,15 @@ export interface UpdateUploadResult {
|
|
|
11
15
|
skipped: number;
|
|
12
16
|
}
|
|
13
17
|
|
|
18
|
+
/** The release one `publish-update` wrote, as it reports it. */
|
|
19
|
+
export interface PublishedUpdate {
|
|
20
|
+
/** `<updates>/<os>-<arch>` for a desktop app, `<updates>/<platform>` for a phone's web bundle. */
|
|
21
|
+
dir: string;
|
|
22
|
+
channel: string;
|
|
23
|
+
/** Its manifest's `bundle`, which tells it from the other manifests of the channel there. */
|
|
24
|
+
bundle: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
14
27
|
//* A manifest names files that must already be there, so they go first and the manifests last, each signature before
|
|
15
28
|
//* its manifest as `publish-update` writes them. A release file never changes once published (its name carries its
|
|
16
29
|
//* hash), so one the bucket holds is skipped; a manifest is always written.
|
|
@@ -34,6 +47,64 @@ export class UpdateUploader {
|
|
|
34
47
|
};
|
|
35
48
|
}
|
|
36
49
|
|
|
50
|
+
//* Never the whole folder: pack-update's records sit there too, and so do manifests of other channels and native
|
|
51
|
+
//* APIs, which a later publish from another machine may have replaced.
|
|
52
|
+
static async releaseFiles(updatesDir: string, { dir, channel, bundle }: PublishedUpdate): Promise<string[]> {
|
|
53
|
+
const keyOf = (file: unknown) => UpdateUploader.#keyOf(updatesDir, dir, file);
|
|
54
|
+
const desktop = await UpdateUploader.#read<UpdateManifest>(path.join(dir, `${channel}.json`));
|
|
55
|
+
if (desktop?.bundle === bundle) {
|
|
56
|
+
const archive = desktop.archive?.url;
|
|
57
|
+
const patches = UpdateUploader.#field(desktop.patches, "url");
|
|
58
|
+
return [archive, ...patches, `${channel}.json.sig`, `${channel}.json`].map(keyOf);
|
|
59
|
+
}
|
|
60
|
+
const names = await readdir(path.join(dir, channel)).catch(() => [] as string[]);
|
|
61
|
+
for (const name of names.filter((name) => name.endsWith(".json"))) {
|
|
62
|
+
const envelope = await UpdateUploader.#read<UpdateEnvelope>(path.join(dir, channel, name));
|
|
63
|
+
const manifest =
|
|
64
|
+
typeof envelope?.manifest === "string"
|
|
65
|
+
? UpdateUploader.#parse<UpdateManifest>(Buffer.from(envelope.manifest, "base64").toString("utf8"))
|
|
66
|
+
: null;
|
|
67
|
+
if (manifest?.bundle !== bundle) continue;
|
|
68
|
+
const hashes = [...new Set(UpdateUploader.#field(manifest.files, "sha256"))];
|
|
69
|
+
if (!hashes.every((hash): hash is string => typeof hash === "string" && /^[0-9a-f]{64}$/.test(hash)))
|
|
70
|
+
throw new Error(
|
|
71
|
+
`${path.join(dir, channel, name)} lists a file whose sha256 is not 64 hex digits; nothing was uploaded.`,
|
|
72
|
+
);
|
|
73
|
+
return [...hashes.map((hash) => `${UpdatePackFolder.filesDir}/${hash}`), `${channel}/${name}`].map(keyOf);
|
|
74
|
+
}
|
|
75
|
+
throw new Error(`${dir} holds no ${channel} manifest of release ${bundle}; nothing was uploaded.`);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
static #keyOf(updatesDir: string, dir: string, file: unknown) {
|
|
79
|
+
const resolved = typeof file === "string" ? path.resolve(dir, file) : "";
|
|
80
|
+
const key = path.relative(updatesDir, resolved);
|
|
81
|
+
if (!resolved.startsWith(`${path.resolve(dir)}${path.sep}`) || key.startsWith("..") || path.isAbsolute(key))
|
|
82
|
+
throw new Error(
|
|
83
|
+
`The release in ${dir} names ${typeof file === "string" ? JSON.stringify(file) : "a file with no path"}, which is not inside that folder; nothing was uploaded.`,
|
|
84
|
+
);
|
|
85
|
+
return key.split(path.sep).join("/");
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
static #field(list: unknown, key: string): unknown[] {
|
|
89
|
+
return Array.isArray(list) ? list.map((item) => (isRecord(item) ? item[key] : undefined)) : [];
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
static async #read<T>(file: string) {
|
|
93
|
+
const text = await Bun.file(file)
|
|
94
|
+
.text()
|
|
95
|
+
.catch(() => "");
|
|
96
|
+
return UpdateUploader.#parse<T>(text);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
static #parse<T>(text: string): Partial<T> | null {
|
|
100
|
+
try {
|
|
101
|
+
const value: unknown = JSON.parse(text);
|
|
102
|
+
return isRecord(value) ? (value as Partial<T>) : null;
|
|
103
|
+
} catch {
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
37
108
|
constructor(
|
|
38
109
|
readonly bucket: UpdateBucket,
|
|
39
110
|
readonly prefix: string,
|
|
@@ -44,12 +115,8 @@ export class UpdateUploader {
|
|
|
44
115
|
return new UpdateUploader(new Bun.S3Client({ bucket }), prefix);
|
|
45
116
|
}
|
|
46
117
|
|
|
47
|
-
async upload(
|
|
48
|
-
const
|
|
49
|
-
for await (const file of new Bun.Glob("**/*").scan({ cwd: dir, onlyFiles: true })) {
|
|
50
|
-
if (!file.endsWith(".tmp")) files.push(file.split(path.sep).join("/"));
|
|
51
|
-
}
|
|
52
|
-
const { releases, manifests } = UpdateUploader.order(files);
|
|
118
|
+
async upload(updatesDir: string, release: PublishedUpdate): Promise<UpdateUploadResult> {
|
|
119
|
+
const { releases, manifests } = UpdateUploader.order(await UpdateUploader.releaseFiles(updatesDir, release));
|
|
53
120
|
const keyOf = (file: string) => (this.prefix ? `${this.prefix}/${file}` : file);
|
|
54
121
|
let uploaded = 0;
|
|
55
122
|
let skipped = 0;
|
|
@@ -58,12 +125,12 @@ export class UpdateUploader {
|
|
|
58
125
|
skipped++;
|
|
59
126
|
continue;
|
|
60
127
|
}
|
|
61
|
-
await this.bucket.write(keyOf(file), Bun.file(path.join(
|
|
128
|
+
await this.bucket.write(keyOf(file), Bun.file(path.join(updatesDir, file)));
|
|
62
129
|
uploaded++;
|
|
63
130
|
}
|
|
64
131
|
for (const file of manifests) {
|
|
65
132
|
const type = file.endsWith(".json") ? "application/json" : "text/plain";
|
|
66
|
-
await this.bucket.write(keyOf(file), Bun.file(path.join(
|
|
133
|
+
await this.bucket.write(keyOf(file), Bun.file(path.join(updatesDir, file)), { type });
|
|
67
134
|
uploaded++;
|
|
68
135
|
}
|
|
69
136
|
return { uploaded, skipped };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@akanjs/devkit",
|
|
3
|
-
"version": "3.1.0-rc.
|
|
3
|
+
"version": "3.1.0-rc.2",
|
|
4
4
|
"sourceType": "module",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"publishConfig": {
|
|
@@ -48,7 +48,7 @@
|
|
|
48
48
|
"@earendil-works/pi-tui": "0.86.1",
|
|
49
49
|
"@inquirer/prompts": "^8.4.3",
|
|
50
50
|
"@tailwindcss/node": "^4.3.0",
|
|
51
|
-
"akanjs": "3.1.0-rc.
|
|
51
|
+
"akanjs": "3.1.0-rc.2",
|
|
52
52
|
"chalk": "^5.6.2",
|
|
53
53
|
"commander": "^14.0.3",
|
|
54
54
|
"dayjs": "^1.11.20",
|