@akanjs/devkit 3.1.0-rc.1 → 3.1.0-rc.3

package/mobile/updateUploader.test.ts

@@ -16,34 +16,126 @@ const fakeBucket = (held: string[] = []) => {
   return { bucket, writes };
 };
 
+const sha = (char: string) => char.repeat(64);
+
+const inUpdatesDir = async (files: Record<string, string>, run: (updatesDir: string) => Promise<void>) => {
+  const updatesDir = await mkdtemp(path.join(tmpdir(), "akan-update-upload-"));
+  try {
+    for (const [file, content] of Object.entries(files)) {
+      await mkdir(path.dirname(path.join(updatesDir, file)), { recursive: true });
+      await writeFile(path.join(updatesDir, file), content);
+    }
+    await run(updatesDir);
+  } finally {
+    await rm(updatesDir, { recursive: true, force: true });
+  }
+};
+
+const envelope = (manifest: object) =>
+  JSON.stringify({
+    manifest: Buffer.from(JSON.stringify(manifest)).toString("base64"),
+    keyId: "0011223344556677",
+    signature: "c2lnbmF0dXJl",
+  });
+
 describe("UpdateUploader", () => {
   test("reads s3://bucket/prefix and refuses anything else", () => {
-    expect(UpdateUploader.parseTarget("s3://releases/dcms/kiosk/")).toEqual({
+    expect(UpdateUploader.parseTarget("s3://releases/acme/kiosk/")).toEqual({
       bucket: "releases",
-      prefix: "dcms/kiosk",
+      prefix: "acme/kiosk",
     });
     expect(UpdateUploader.parseTarget("s3://releases")).toEqual({ bucket: "releases", prefix: "" });
-    expect(() => UpdateUploader.parseTarget("https://releases.example/dcms")).toThrow("s3://<bucket>/<prefix>");
+    expect(() => UpdateUploader.parseTarget("https://releases.example/acme")).toThrow("s3://<bucket>/<prefix>");
   });
 
-  test("uploads what the bucket lacks first, then each signature before its manifest", async () => {
-    const dir = await mkdtemp(path.join(tmpdir(), "akan-update-upload-"));
-    try {
-      await mkdir(path.join(dir, "windows-x64", "files"), { recursive: true });
-      for (const file of ["windows-x64/files/2000-aa.tar.gz", "windows-x64/files/1000-bb.tar.gz"])
-        await writeFile(path.join(dir, file), "tar");
-      for (const file of ["windows-x64/main.json", "windows-x64/main.json.sig", "windows-x64/main.json.tmp"])
-        await writeFile(path.join(dir, file), "{}");
-      const { bucket, writes } = fakeBucket(["dcms/windows-x64/files/1000-bb.tar.gz"]);
-
-      expect(await new UpdateUploader(bucket, "dcms").upload(dir)).toEqual({ uploaded: 3, skipped: 1 });
-      expect(writes).toEqual([
-        { key: "dcms/windows-x64/files/2000-aa.tar.gz" },
-        { key: "dcms/windows-x64/main.json.sig", type: "text/plain" },
-        { key: "dcms/windows-x64/main.json", type: "application/json" },
-      ]);
-    } finally {
-      await rm(dir, { recursive: true, force: true });
-    }
+  test("uploads a desktop release's archives the bucket lacks first, then the signature before the manifest", async () => {
+    const manifest = {
+      bundle: "1.2.0-2000",
+      archive: { url: `app/${sha("b")}.tar.gz` },
+      patches: [{ url: `app/${sha("a")}-${sha("b")}.delta.gz` }],
+    };
+    await inUpdatesDir(
+      {
+        [`windows-x64/app/${sha("a")}.tar.gz`]: "previous release",
+        [`windows-x64/app/${sha("b")}.tar.gz`]: "tar",
+        [`windows-x64/app/${sha("a")}-${sha("b")}.delta.gz`]: "delta",
+        "windows-x64/main.json": JSON.stringify(manifest),
+        "windows-x64/main.json.sig": "sig",
+        "windows-x64/main.json.tmp": "{}",
+        "windows-x64/develop.json": JSON.stringify({ ...manifest, bundle: "1.1.0-1000" }),
+        "windows-x64/develop.json.sig": "sig",
+      },
+      async (updatesDir) => {
+        const { bucket, writes } = fakeBucket([`acme/windows-x64/app/${sha("a")}-${sha("b")}.delta.gz`]);
+        const release = { dir: path.join(updatesDir, "windows-x64"), channel: "main", bundle: "1.2.0-2000" };
+
+        expect(await new UpdateUploader(bucket, "acme").upload(updatesDir, release)).toEqual({
+          uploaded: 3,
+          skipped: 1,
+        });
+        expect(writes).toEqual([
+          { key: `acme/windows-x64/app/${sha("b")}.tar.gz` },
+          { key: "acme/windows-x64/main.json.sig", type: "text/plain" },
+          { key: "acme/windows-x64/main.json", type: "application/json" },
+        ]);
+      },
+    );
+  });
+
+  test("uploads a phone release's own files and envelope, never pack records or another manifest beside them", async () => {
+    await inUpdatesDir(
+      {
+        "android/pack.json": JSON.stringify({ out: "/Users/someone/work/dist/native/demo/default/updates/android" }),
+        "android/manifest.template.json": "{}",
+        "android/bundle.json": "{}",
+        "android/compat.json": "{}",
+        [`android/files/${sha("a")}`]: "index.html",
+        [`android/files/${sha("b")}`]: "env.runtime.json",
+        [`android/files/${sha("c")}`]: "an earlier pack's file",
+        "android/main/api2.json": envelope({
+          bundle: "3000-aaaaaaaa",
+          files: [{ sha256: sha("a") }, { sha256: sha("b") }, { sha256: sha("a") }],
+        }),
+        "android/main/api1.json": envelope({ bundle: "1000-cccccccc", files: [{ sha256: sha("c") }] }),
+        "android/develop/api2.json": envelope({ bundle: "2000-bbbbbbbb", files: [{ sha256: sha("b") }] }),
+      },
+      async (updatesDir) => {
+        const { bucket, writes } = fakeBucket();
+        const release = { dir: path.join(updatesDir, "android"), channel: "main", bundle: "3000-aaaaaaaa" };
+
+        expect(await new UpdateUploader(bucket, "").upload(updatesDir, release)).toEqual({ uploaded: 3, skipped: 0 });
+        expect(writes).toEqual([
+          { key: `android/files/${sha("a")}` },
+          { key: `android/files/${sha("b")}` },
+          { key: "android/main/api2.json", type: "application/json" },
+        ]);
+      },
+    );
+  });
+
+  test("uploads nothing when the release is not there or names a file outside its folder", async () => {
+    await inUpdatesDir(
+      {
+        "linux-x64/main.json": JSON.stringify({ bundle: "1.0.0-10", archive: { url: "../../.ssh/id_ed25519" } }),
+        "linux-x64/main.json.sig": "sig",
+        "ios/main/api1.json": envelope({ bundle: "10-aaaaaaaa", files: [{ sha256: "../pack.json" }] }),
+      },
+      async (updatesDir) => {
+        const { bucket, writes } = fakeBucket();
+        const uploader = new UpdateUploader(bucket, "acme");
+        const linux = path.join(updatesDir, "linux-x64");
+
+        await expect(uploader.upload(updatesDir, { dir: linux, channel: "main", bundle: "1.0.0-9" })).rejects.toThrow(
+          "holds no main manifest of release 1.0.0-9",
+        );
+        await expect(uploader.upload(updatesDir, { dir: linux, channel: "main", bundle: "1.0.0-10" })).rejects.toThrow(
+          "which is not inside that folder",
+        );
+        await expect(
+          uploader.upload(updatesDir, { dir: path.join(updatesDir, "ios"), channel: "main", bundle: "10-aaaaaaaa" }),
+        ).rejects.toThrow("not 64 hex digits");
+        expect(writes).toEqual([]);
+      },
+    );
   });
 });

package/mobile/updateUploader.ts

@@ -1,4 +1,8 @@
+import { readdir } from "node:fs/promises";
 import path from "node:path";
+import type { UpdateEnvelope, UpdateManifest } from "@akanjs/native/api";
+import { isRecord } from "akanjs/common";
+import { UpdatePackFolder } from "./updatePackFolder";
 
 /** What the uploader needs of an S3 client; Bun.S3Client is one. */
 export interface UpdateBucket {
@@ -11,6 +15,15 @@ export interface UpdateUploadResult {
   skipped: number;
 }
 
+/** The release one `publish-update` wrote, as it reports it. */
+export interface PublishedUpdate {
+  /** `<updates>/<os>-<arch>` for a desktop app, `<updates>/<platform>` for a phone's web bundle. */
+  dir: string;
+  channel: string;
+  /** Its manifest's `bundle`, which tells it from the other manifests of the channel there. */
+  bundle: string;
+}
+
 //* A manifest names files that must already be there, so they go first and the manifests last, each signature before
 //* its manifest as `publish-update` writes them. A release file never changes once published (its name carries its
 //* hash), so one the bucket holds is skipped; a manifest is always written.
@@ -34,6 +47,64 @@ export class UpdateUploader {
     };
   }
 
+  //* Never the whole folder: pack-update's records sit there too, and so do manifests of other channels and native
+  //* APIs, which a later publish from another machine may have replaced.
+  static async releaseFiles(updatesDir: string, { dir, channel, bundle }: PublishedUpdate): Promise<string[]> {
+    const keyOf = (file: unknown) => UpdateUploader.#keyOf(updatesDir, dir, file);
+    const desktop = await UpdateUploader.#read<UpdateManifest>(path.join(dir, `${channel}.json`));
+    if (desktop?.bundle === bundle) {
+      const archive = desktop.archive?.url;
+      const patches = UpdateUploader.#field(desktop.patches, "url");
+      return [archive, ...patches, `${channel}.json.sig`, `${channel}.json`].map(keyOf);
+    }
+    const names = await readdir(path.join(dir, channel)).catch(() => [] as string[]);
+    for (const name of names.filter((name) => name.endsWith(".json"))) {
+      const envelope = await UpdateUploader.#read<UpdateEnvelope>(path.join(dir, channel, name));
+      const manifest =
+        typeof envelope?.manifest === "string"
+          ? UpdateUploader.#parse<UpdateManifest>(Buffer.from(envelope.manifest, "base64").toString("utf8"))
+          : null;
+      if (manifest?.bundle !== bundle) continue;
+      const hashes = [...new Set(UpdateUploader.#field(manifest.files, "sha256"))];
+      if (!hashes.every((hash): hash is string => typeof hash === "string" && /^[0-9a-f]{64}$/.test(hash)))
+        throw new Error(
+          `${path.join(dir, channel, name)} lists a file whose sha256 is not 64 hex digits; nothing was uploaded.`,
+        );
+      return [...hashes.map((hash) => `${UpdatePackFolder.filesDir}/${hash}`), `${channel}/${name}`].map(keyOf);
+    }
+    throw new Error(`${dir} holds no ${channel} manifest of release ${bundle}; nothing was uploaded.`);
+  }
+
+  static #keyOf(updatesDir: string, dir: string, file: unknown) {
+    const resolved = typeof file === "string" ? path.resolve(dir, file) : "";
+    const key = path.relative(updatesDir, resolved);
+    if (!resolved.startsWith(`${path.resolve(dir)}${path.sep}`) || key.startsWith("..") || path.isAbsolute(key))
+      throw new Error(
+        `The release in ${dir} names ${typeof file === "string" ? JSON.stringify(file) : "a file with no path"}, which is not inside that folder; nothing was uploaded.`,
+      );
+    return key.split(path.sep).join("/");
+  }
+
+  static #field(list: unknown, key: string): unknown[] {
+    return Array.isArray(list) ? list.map((item) => (isRecord(item) ? item[key] : undefined)) : [];
+  }
+
+  static async #read<T>(file: string) {
+    const text = await Bun.file(file)
+      .text()
+      .catch(() => "");
+    return UpdateUploader.#parse<T>(text);
+  }
+
+  static #parse<T>(text: string): Partial<T> | null {
+    try {
+      const value: unknown = JSON.parse(text);
+      return isRecord(value) ? (value as Partial<T>) : null;
+    } catch {
+      return null;
+    }
+  }
+
   constructor(
     readonly bucket: UpdateBucket,
     readonly prefix: string,
@@ -44,12 +115,8 @@ export class UpdateUploader {
     return new UpdateUploader(new Bun.S3Client({ bucket }), prefix);
   }
 
-  async upload(dir: string): Promise<UpdateUploadResult> {
-    const files: string[] = [];
-    for await (const file of new Bun.Glob("**/*").scan({ cwd: dir, onlyFiles: true })) {
-      if (!file.endsWith(".tmp")) files.push(file.split(path.sep).join("/"));
-    }
-    const { releases, manifests } = UpdateUploader.order(files);
+  async upload(updatesDir: string, release: PublishedUpdate): Promise<UpdateUploadResult> {
+    const { releases, manifests } = UpdateUploader.order(await UpdateUploader.releaseFiles(updatesDir, release));
     const keyOf = (file: string) => (this.prefix ? `${this.prefix}/${file}` : file);
     let uploaded = 0;
     let skipped = 0;
@@ -58,12 +125,12 @@ export class UpdateUploader {
         skipped++;
         continue;
       }
-      await this.bucket.write(keyOf(file), Bun.file(path.join(dir, file)));
+      await this.bucket.write(keyOf(file), Bun.file(path.join(updatesDir, file)));
       uploaded++;
     }
     for (const file of manifests) {
       const type = file.endsWith(".json") ? "application/json" : "text/plain";
-      await this.bucket.write(keyOf(file), Bun.file(path.join(dir, file)), { type });
+      await this.bucket.write(keyOf(file), Bun.file(path.join(updatesDir, file)), { type });
       uploaded++;
     }
     return { uploaded, skipped };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@akanjs/devkit",
-  "version": "3.1.0-rc.1",
+  "version": "3.1.0-rc.3",
   "sourceType": "module",
   "type": "module",
   "publishConfig": {
@@ -48,7 +48,7 @@
     "@earendil-works/pi-tui": "0.86.1",
     "@inquirer/prompts": "^8.4.3",
     "@tailwindcss/node": "^4.3.0",
-    "akanjs": "3.1.0-rc.1",
+    "akanjs": "3.1.0-rc.3",
     "chalk": "^5.6.2",
     "commander": "^14.0.3",
     "dayjs": "^1.11.20",

package/pipeline/PipelineCompiler.test.ts

@@ -337,7 +337,28 @@ describe("PipelineCompiler", () => {
     ]);
     const e2e = plan.phases.at(-1);
     expect(e2e).toMatchObject({ id: "e2e", timeoutSeconds: 3600, milestone: "finish" });
-    expect(e2e?.stages.map((stage) => [stage.id, stage.name, stage.app])).toEqual([["e2e:api", "api", "api"]]);
+    expect(e2e?.stages.map((stage) => [stage.id, stage.name, stage.app])).toEqual([["e2e.apps.api", "api", "api"]]);
+  });
+
+  test("a matrix copy's id never collides with an app's own stage or a post step", () => {
+    const stageIds = (plan: PipelinePlan) => plan.phases.flatMap((phase) => phase.stages.map((stage) => stage.id));
+    const plan = planOf(
+      compile({
+        workspace: { after: [{ name: "Web", matrix: "apps", sh: "bun run e2e $AKAN_CI_APP" }] },
+        apps: { web: { stages: [{ use: "build" }, { name: "API", sh: "bun run test:api" }] } },
+      }),
+    );
+    expect(stageIds(plan)).toEqual(expect.arrayContaining(["web:api", "web.apps.api", "web.apps.web"]));
+    expect(new Set(stageIds(plan)).size).toBe(stageIds(plan).length);
+    const post = planOf(
+      compile({
+        ctx: context({ apps: ["success"] }),
+        workspaceApps: ["success"],
+        workspace: { after: [{ name: "Post", matrix: "apps", sh: "x" }], post: { success: ["y"] } },
+      }),
+    );
+    expect(stageIds(post)).toContain("post.apps.success");
+    expect(post.post.success.map((step) => step.id)).toEqual(["post.success/1"]);
   });
 
   test("env is workspace, then app, then stage, then step; secrets are each step's union", () => {
@@ -571,6 +592,10 @@ describe("PipelineCompiler", () => {
       expect(codes(appResult({ stages: [{ name: "x", sh: "x", secrets: ["lower"] }] }))).toContain(
         "pipeline.secret.name",
       );
+      const pasted = appResult({ stages: [{ name: "x", sh: "x", secrets: ["NPM_TOKEN", "npm_live_value"] }] });
+      const secretName = pasted.diagnostics.find((diagnostic) => diagnostic.code === "pipeline.secret.name");
+      expect(secretName?.path).toEndWith(".secrets[1]");
+      expect(secretName?.message).not.toContain("npm_live_value");
       expect(codes(appResult({ stages: [{ name: "x", steps: [{ sh: "x", cwd: "../up" }] }] }))).toContain(
         "pipeline.cwd.outside",
       );
@@ -604,6 +629,30 @@ describe("PipelineCompiler", () => {
       );
     });
 
+    test("a parallel branch may not take the id of a post list's steps", () => {
+      const refused = compile({
+        workspace: { before: [{ name: "Post", parallel: [{ name: "Success", sh: "x" }] }], post: { success: ["y"] } },
+      });
+      expect(refused.ok).toBe(false);
+      expect(refused.diagnostics).toEqual([
+        expect.objectContaining({
+          code: "pipeline.id.reserved",
+          file: "akan.config.ts",
+          path: "pipeline.before[0].parallel[0].name",
+        }),
+      ]);
+      expect(
+        compile({
+          workspace: { after: [{ id: "post", name: "Checks", parallel: [{ id: "always", name: "A", sh: "x" }] }] },
+        }).diagnostics,
+      ).toEqual([expect.objectContaining({ code: "pipeline.id.reserved", path: "pipeline.after[0].parallel[0].id" })]);
+      const otherBranch = compile({
+        workspace: { before: [{ name: "Post", parallel: [{ name: "Lint", sh: "x" }] }], post: { success: ["y"] } },
+      });
+      expect(otherBranch.ok).toBe(true);
+      expect(compile({ workspace: { after: [{ name: "Post", sh: "x" }], post: { success: ["y"] } } }).ok).toBe(true);
+    });
+
     test("loading and sampling", () => {
       const thrown = appResult(() => {
         throw new Error("boom");
@@ -620,6 +669,113 @@ describe("PipelineCompiler", () => {
       expect(codes(sampled)).toContain("pipeline.function.sampled");
     });
 
+    test("a function returning nothing, an async one, and a falsy pipeline are errors, never a default", async () => {
+      const refused = (result: PipelinePlanResult, file: string) => {
+        expect(result.ok).toBe(false);
+        expect(result.diagnostics).toContainEqual(
+          expect.objectContaining({ severity: "error", code: "pipeline.schema.invalid", file, path: "pipeline" }),
+        );
+        return result.diagnostics.map((diagnostic) => diagnostic.message).join("\n");
+      };
+      const onMainOnly = ((ctx: { branch: string }) =>
+        ctx.branch === "main" ? { stages: [{ use: "build" }] } : undefined) as unknown as AppPipelineConfig;
+      expect(appResult(onMainOnly).ok).toBe(true);
+      const develop = compile({ ctx: context({ branch: "develop", apps: ["web"] }), apps: { web: onMainOnly } });
+      expect(refused(develop, "apps/web/akan.config.ts")).toContain("returned nothing");
+      expect(refused(compile({ workspace: (() => undefined) as never }), "akan.config.ts")).toContain(
+        "returned nothing",
+      );
+      expect(refused(compile({ workspace: (async () => ({ before: [] })) as never }), "akan.config.ts")).toContain(
+        "Promise",
+      );
+      const rejecting = (async () => {
+        throw new Error("boom");
+      }) as unknown as AppPipelineConfig;
+      expect(refused(appResult(rejecting), "apps/web/akan.config.ts")).toContain("Promise");
+      await Bun.sleep(1);
+      expect(refused(appResult(false as never), "apps/web/akan.config.ts")).toContain("must be an object");
+      expect(refused(compile({ workspace: 0 as never }), "akan.config.ts")).toContain("must be an object");
+    });
+
+    test("a plan larger than an executor reads, as written, is refused", () => {
+      const steps = Array.from({ length: 20 }, () => "x".repeat(60_000));
+      const result = appResult({ stages: [{ use: "build" }, { name: "Big", steps }] });
+      expect(result.ok).toBe(false);
+      expect(result.diagnostics).toContainEqual(
+        expect.objectContaining({ code: "pipeline.limit.exceeded", file: null, path: null }),
+      );
+    });
+
+    test("conditions", () => {
+      expect(appResult({ stages: [{ use: "build", when: { trigger: [] } }] }).diagnostics).toContainEqual(
+        expect.objectContaining({ code: "pipeline.schema.invalid", path: "pipeline.stages[0].when.trigger" }),
+      );
+      const branch = Array.from({ length: 6 }, (_, idx) => `release/${String(idx).repeat(240)}`);
+      const plan = planOf(
+        compile({
+          ctx: context({ branch: "develop", env: "develop", apps: ["web"] }),
+          apps: { web: { stages: [{ use: "build", when: { branch } }] } },
+        }),
+      );
+      const reason = plan.phases[0]?.stages[0]?.skip?.reason ?? "";
+      expect(reason.startsWith('branch "develop" does not match "release/')).toBe(true);
+      expect(reason.endsWith("…")).toBe(true);
+      expect(reason.length).toBeLessThanOrEqual(1024);
+    });
+
+    test("a value the plan would refuse is reported where it was written", () => {
+      expect(
+        appResult({ stages: [{ use: "build" }, { name: "x", sh: "x", env: { BIG: "x".repeat(64 * 1024 + 1) } }] })
+          .diagnostics,
+      ).toContainEqual(
+        expect.objectContaining({ code: "pipeline.limit.exceeded", path: "pipeline.stages[1].env.BIG" }),
+      );
+      const unset = compile({ workspace: { before: [{ name: "Checks", parallel: undefined as never }] } });
+      expect(unset.ok).toBe(false);
+      expect(unset.diagnostics).toEqual([
+        expect.objectContaining({
+          code: "pipeline.schema.invalid",
+          file: "akan.config.ts",
+          path: "pipeline.before[0].parallel",
+        }),
+      ]);
+    });
+
+    test("a key set to undefined is the key left out", () => {
+      const stage = { name: "Test", sh: "bun test", use: undefined, parallel: undefined, matrix: undefined };
+      const plan = planOf(
+        compile({
+          workspace: {
+            before: [stage as never, { name: "Checks", parallel: [{ name: "Lint", sh: "x", use: undefined }] }],
+          },
+          apps: { web: { stages: [{ use: "build", targets: undefined } as never, stage as never] } },
+        }),
+      );
+      expect(phaseIds(plan)).toEqual(["test", "checks", "build", "after-build", "docker", "deploy"]);
+      expect(appResult({ stages: [{ use: undefined } as never] }).diagnostics).toContainEqual(
+        expect.objectContaining({ code: "pipeline.schema.invalid", path: "pipeline.stages[0].use" }),
+      );
+    });
+
+    test("a value JSON cannot hold is reported, never thrown", () => {
+      const cyclic: Record<string, unknown> = {};
+      cyclic.self = cyclic;
+      const result = appResult({ stages: [{ use: "build", timeout: 10n as never }, { use: cyclic as never }] });
+      expect(result.ok).toBe(false);
+      expect(result.diagnostics.map((diagnostic) => [diagnostic.code, diagnostic.message.split(" ")[0]])).toEqual([
+        ["pipeline.duration.invalid", "10n"],
+        ["pipeline.schema.invalid", "[object"],
+      ]);
+    });
+
+    test("a long first line names its step without splitting a surrogate pair", () => {
+      const plan = planOf(
+        appResult({ stages: [{ use: "build" }, { name: "Emoji", sh: `${"a".repeat(78)}${"😀".repeat(5)}` }] }),
+      );
+      const name = plan.phases.find((phase) => phase.id === "after-build")?.stages[0]?.steps[0]?.name;
+      expect(name).toBe(`${"a".repeat(78)}…`);
+    });
+
     test("milestones that move backwards are a warning", () => {
       const result = appResult({
         stages: [{ use: "build" }, { use: "docker" }, { name: "Late", sh: "x", milestone: "verify" }],