@intentius/chant 0.124.35 → 0.124.36

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@intentius/chant",
-  "version": "0.124.35",
+  "version": "0.124.36",
   "description": "Declarative infrastructure-as-code toolkit — TypeScript on Node.js",
   "license": "Apache-2.0",
   "homepage": "https://intentius.io/chant",

package/src/cli/handlers/release-publish-targets.e2e.test.ts

@@ -160,6 +160,9 @@ describe("chant release publish with targets", () => {
         ["publish", join(dir, "packages/cli"), "--tag", "next", "--access", "public", "--provenance"],
         ["dist-tag", "add", "@acme/core@1.0.0", "latest"],
         ["dist-tag", "add", "@acme/cli@1.0.0", "latest"],
+        // #4244: the reveal waits until the registry serves each version it published.
+        ["view", "@acme/core@1.0.0", "version"],
+        ["view", "@acme/cli@1.0.0", "version"],
         // #4049: in CI, what was published is installed in an empty directory and its signatures and attestations audited.
         ["install", "--ignore-scripts", "--no-audit", "--no-fund"],
         ["audit", "signatures"],
@@ -212,7 +215,7 @@ describe("chant release publish with targets", () => {
       const moved = await chant(dir, ["release", "publish", "v1", "--yes"], { ...path, ...secrets, ...oidc });
       expect(moved.code).toBe(1);
       expect(moved.stderr).toMatch(/\[release-head-mismatch\]/);
-      expect(calls()).toHaveLength(8);
+      expect(calls()).toHaveLength(10);
     },
     240_000,
   );

package/src/lifecycle/__fixtures__/publish-fakes.ts

@@ -155,6 +155,11 @@ export function seedFakeNpmRegistry(log: string, registry: FakeNpmRegistry): voi
  *   its registry at that version, and installs nothing.
  * - `audit signatures` (#4049) prints what npm prints for verified
  *   signatures and attestations, and exits 1 when `$FAKE_NPM_AUDIT_FAIL` is set.
+ * - `$FAKE_NPM_LAG` (#4244), `<name>=<n>[,<name>=<n>]`: after a publish of
+ *   that name, the next `n` views of the version find nothing, as a registry
+ *   that accepted a publish but does not serve it yet.
+ * - `$FAKE_NPM_INSTALL_ETARGET` (#4244), `<n>`: the first `n` installs fail
+ *   with ETARGET.
  */
 export function writeFakeNpm(binDir: string, log: string): string {
   mkdirSync(binDir, { recursive: true });
@@ -186,6 +191,10 @@ if (args[0] === "publish") {
   const p = reg.packages[name] || (reg.packages[name] = { versions: [], distTags: {} });
   if (p.versions.includes(version)) { console.error("npm error 403 You cannot publish over the previously published versions: " + version); process.exit(1); }
   p.versions.push(version);
+  for (const pair of (process.env.FAKE_NPM_LAG || "").split(",").filter(Boolean)) {
+    const eq = pair.lastIndexOf("=");
+    if (pair.slice(0, eq) === name) (p.lag || (p.lag = {}))[version] = Number(pair.slice(eq + 1));
+  }
   p.distTags[flag("--tag") || "latest"] = version;
   if (!p.distTags.latest) p.distTags.latest = version;
   save();
@@ -194,7 +203,8 @@ if (args[0] === "publish") {
   const [name, version] = split(args[1]);
   const p = reg.packages[name];
   if (!p) { console.error("npm error 404 Not Found - " + name); process.exit(1); }
-  if (p.versions.includes(version)) console.log(version);
+  if (p.lag && p.lag[version] > 0) { p.lag[version] -= 1; save(); }
+  else if (p.versions.includes(version)) console.log(version);
 } else if (args[0] === "dist-tag" && args[1] === "add") {
   const [name, version] = split(args[2]);
   if (process.env.FAKE_NPM_FAIL_TAG && name.includes(process.env.FAKE_NPM_FAIL_TAG)) { console.error("npm error 403 Forbidden - no permission to change dist-tags"); process.exit(1); }
@@ -204,6 +214,10 @@ if (args[0] === "publish") {
   save();
   console.log("+" + args[3] + ": " + args[2]);
 } else if (args[0] === "install") {
+  if (Number(process.env.FAKE_NPM_INSTALL_ETARGET || 0) > (reg.installs || 0)) {
+    reg.installs = (reg.installs || 0) + 1; save();
+    console.error("npm error code ETARGET"); console.error("npm error notarget No matching version found for a package"); process.exit(1);
+  }
   const deps = JSON.parse(fs.readFileSync("package.json", "utf-8")).dependencies || {};
   for (const [name, version] of Object.entries(deps)) {
     const p = reg.packages[name];

package/src/lifecycle/publish-declaration.ts

@@ -60,6 +60,12 @@ export const NpmPublishSchema = z
      * token rather than failing the release. Unset or empty, there is no retry.
      */
     tokenEnv: envName.optional(),
+    /**
+     * Seconds the reveal waits for the registry to serve every version it
+     * published before installing them to audit (#4244). A registry accepts a
+     * publish before it serves it. Default 600.
+     */
+    registryWait: z.number().int().min(0).optional(),
   })
   .strict();
 

package/src/lifecycle/publish-npm.ts

@@ -160,8 +160,17 @@ function run(bin: string, args: string[], ctx: PublishContext, cwd: string = ctx
   });
 }
 
+/** Seconds the reveal waits for the registry to serve what it published, by default (#4244). */
+export const NPM_REGISTRY_WAIT_SECONDS = 600;
+/** Between polls of the registry while it waits. */
+export const NPM_REGISTRY_POLL_MS = 10_000;
+/** npm's errors for a version the registry does not serve yet. */
+const NPM_NOT_SERVED = /\b(ETARGET|ERESOLVE|notarget)\b|No matching version found/;
+
 export function npmTarget(decl: NpmPublishDeclaration, deps: PublishDeps): PublishTarget {
   const bin = deps.npm ?? "npm";
+  const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
+  const now = deps.now ?? Date.now;
   const registry = decl.registry ?? "the registry npm's config names";
   const shown = (tag: string) => `${registry}, dist-tag ${tag}`;
   const revealTag = (ctx: PublishContext) => ctx.tag ?? "latest";
@@ -216,15 +225,50 @@ export function npmTarget(decl: NpmPublishDeclaration, deps: PublishDeps): Publi
    * `npm audit signatures` there (#4049), which checks each package's
    * registry signature and provenance attestation.
    */
+  /**
+   * Wait until the registry serves every version (#4244): npm accepts a
+   * publish minutes before `npm view` and `npm install` find it, so an
+   * install straight after fails with ETARGET or ERESOLVE.
+   */
+  const served = async (ctx: PublishContext, pkgs: NpmPackage[]): Promise<void> => {
+    const wait = (decl.registryWait ?? NPM_REGISTRY_WAIT_SECONDS) * 1000;
+    const start = now();
+    let missing = pkgs;
+    for (;;) {
+      const still: NpmPackage[] = [];
+      for (const pkg of missing) if (!(await onRegistry(ctx, pkg))) still.push(pkg);
+      missing = still;
+      if (missing.length === 0) return;
+      const names = missing.map((p) => `${p.name}@${p.version}`).join(", ");
+      if (now() - start >= wait) {
+        throw new PublishTargetError(
+          "publish-npm-not-served",
+          `the registry did not serve ${names} within ${Math.round(wait / 1000)}s of the publish, so nothing was audited`,
+          "Each package was published. Once npm serves them, run the publish again: it publishes nothing twice and audits again. `registryWait` sets the wait.",
+        );
+      }
+      ctx.log(`waiting for the registry to serve ${names}`);
+      await sleep(NPM_REGISTRY_POLL_MS);
+    }
+  };
   const audit = async (ctx: PublishContext, pkgs: NpmPackage[]): Promise<void> => {
+    await served(ctx, pkgs);
     const dir = mkdtempSync(join(tmpdir(), "chant-npm-audit-"));
     try {
       const dependencies = Object.fromEntries(pkgs.map((p) => [p.name, p.version]));
       writeFileSync(join(dir, "package.json"), `${JSON.stringify({ name: "chant-publish-audit", version: "0.0.0", private: true, dependencies }, null, 2)}\n`);
       for (const args of [npmAuditInstallArgs(decl), npmAuditSignaturesArgs(decl)]) {
         ctx.log(`npm ${args.join(" ")}`);
-        const result = await npm(ctx, args, dir);
+        let result = await npm(ctx, args, dir);
         for (const line of result.output) ctx.log(`  ${line}`);
+        // `npm view` can answer before every mirror `npm install` reads does: wait once more, then install again.
+        if (result.code !== 0 && args[0] === "install" && result.output.some((l) => NPM_NOT_SERVED.test(l))) {
+          ctx.log("the registry does not serve every version to an install yet; waiting, then installing again");
+          await sleep(NPM_REGISTRY_POLL_MS);
+          await served(ctx, pkgs);
+          result = await npm(ctx, args, dir);
+          for (const line of result.output) ctx.log(`  ${line}`);
+        }
         if (result.code !== 0) {
           throw new PublishTargetError(
             "publish-npm-audit-failed",

package/src/lifecycle/publish-targets.test.ts

@@ -136,6 +136,10 @@ describe("npm", () => {
       ["dist-tag", "add", "@acme/core@1.4.0", "latest"],
       ["dist-tag", "add", "@acme/cli@1.4.0", "latest"],
       ["dist-tag", "add", "@acme/kit@1.4.0", "latest"],
+      // #4244: it first asks the registry for each version it published.
+      ["view", "@acme/core@1.4.0", "version"],
+      ["view", "@acme/cli@1.4.0", "version"],
+      ["view", "@acme/kit@1.4.0", "version"],
       // #4049: with an OIDC token, what was published is installed in an empty directory and its signatures audited.
       ["install", "--ignore-scripts", "--no-audit", "--no-fund"],
       ["audit", "signatures"],
@@ -179,8 +183,8 @@ describe("npm", () => {
       ["publish", join(dir, "packages/core"), "--tag", "latest", "--access", "public", "--provenance"],
       ["publish", join(dir, "kit.tgz"), "--tag", "latest", "--access", "public", "--provenance"],
     ]);
-    // The reveal only audits.
-    expect(calls(log).slice(before).map((c) => c.args[0])).toEqual(["install", "audit"]);
+    // The reveal only audits, once the registry serves both (#4244).
+    expect(calls(log).slice(before).map((c) => c.args[0])).toEqual(["view", "view", "install", "audit"]);
     expect(done.items.map((i) => i.destination)).toEqual(Array(2).fill("the registry npm's config names, dist-tag latest"));
     expect(readFakeNpmRegistry(log).packages["@acme/core"]!.distTags.next).toBeUndefined();
   });
@@ -481,6 +485,65 @@ describe("SBOMs and provenance (#4049)", () => {
     expect(readFileSync(log, "utf-8")).toContain(`"args":["audit","signatures"]`);
   });
 
+  describe("npm: the reveal waits for the registry to serve what it published (#4244)", () => {
+    function lagged(env: Record<string, string>, registryWait?: number) {
+      const dir = temp();
+      writePackage(join(dir, "core"), "@acme/core", "1.0.0");
+      writePackage(join(dir, "cli"), "@acme/cli", "1.0.0");
+      const log = join(dir, "npm.log");
+      const npm = writeFakeNpm(join(dir, "bin"), log);
+      let clock = 0;
+      const sleeps: number[] = [];
+      const deps = { npm, now: () => clock, sleep: async (ms: number) => { sleeps.push(ms); clock += ms; } };
+      const ctx = context(dir, { env: { PATH: process.env.PATH, ...env }, provenance: OIDC });
+      const d = decl({ type: "npm", packages: ["core", "cli"], stage: false, ...(registryWait !== undefined ? { registryWait } : {}) });
+      return { log, deps, ctx, d, sleeps };
+    }
+    const installs = (log: string) => readFileSync(log, "utf-8").split("\n").filter((l) => l.includes(`"args":["install"`)).length;
+
+    test("every version served at once: no wait, one install", async () => {
+      const { log, deps, ctx, d, sleeps } = lagged({});
+      const [target] = await createPublishTargets([d], deps);
+      await target!.reveal(ctx, await target!.stage(ctx));
+      expect(sleeps).toEqual([]);
+      expect(installs(log)).toBe(1);
+    });
+
+    test("one version served late: it waits, names it, then installs once", async () => {
+      const { log, deps, ctx, d, sleeps } = lagged({ FAKE_NPM_LAG: "@acme/cli=2" });
+      const [target] = await createPublishTargets([d], deps);
+      await target!.reveal(ctx, await target!.stage(ctx));
+      expect(sleeps).toEqual([10_000, 10_000]);
+      expect(ctx.lines).toContain("waiting for the registry to serve @acme/cli@1.0.0");
+      expect(installs(log)).toBe(1);
+    });
+
+    test("a version never served: the wait ends with an error naming it, before any install", async () => {
+      const { log, deps, ctx, d } = lagged({ FAKE_NPM_LAG: "@acme/core=1000" }, 30);
+      const [target] = await createPublishTargets([d], deps);
+      const err = await refused(target!.reveal(ctx, await target!.stage(ctx)));
+      expect(err.code).toBe("publish-npm-not-served");
+      expect(err.message).toBe("the registry did not serve @acme/core@1.0.0 within 30s of the publish, so nothing was audited");
+      expect(installs(log)).toBe(0);
+    });
+
+    test("an install that fails with ETARGET waits and installs once more", async () => {
+      const { log, deps, ctx, d, sleeps } = lagged({ FAKE_NPM_INSTALL_ETARGET: "1" });
+      const [target] = await createPublishTargets([d], deps);
+      await target!.reveal(ctx, await target!.stage(ctx));
+      expect(installs(log)).toBe(2);
+      expect(sleeps).toEqual([10_000]);
+    });
+
+    test("a second ETARGET is the audit's failure, not a third install", async () => {
+      const { log, deps, ctx, d } = lagged({ FAKE_NPM_INSTALL_ETARGET: "2" });
+      const [target] = await createPublishTargets([d], deps);
+      const err = await refused(target!.reveal(ctx, await target!.stage(ctx)));
+      expect(err.code).toBe("publish-npm-audit-failed");
+      expect(installs(log)).toBe(2);
+    });
+  });
+
   test("OCI: the provenance statement carries the cut's signature, says a workstation's provenance is the signature, and is the same bytes every run; a declared SBOM is pushed as it is", async () => {
     const reg = await startFakeRegistry({ username: "robot", password: "pw-attach", token: "tok-attach" });
     try {

package/src/lifecycle/release-publish.ts

@@ -129,6 +129,10 @@ export interface PublishDeps {
   fetch?: PublishFetch;
   /** The npm executable. Default: `npm` on `PATH`. */
   npm?: string;
+  /** How a target waits between polls of a registry (#4244). Default: a timer. */
+  sleep?: (ms: number) => Promise<void>;
+  /** The clock a target's wait reads (#4244). Default: `Date.now`. */
+  now?: () => number;
 }
 
 /** The subset of the WHATWG `fetch` the targets use. */