@theappagency/valinor 10.18.0 → 10.19.0

package/README.md

@@ -24,11 +24,30 @@ GitHub runner's default Node works without a `setup-node` step.
 **Authentication.** The downloads endpoint can require a credential. In GitHub Actions the launcher mints one
 itself: give the job `permissions: id-token: write` and it requests a GitHub OIDC token for the audience
 `https://downloads.theappagency.com`, so the repository stores no secret. Elsewhere, set
-`VALINOR_DOWNLOADS_TOKEN`. With neither, the launcher sends no credential, which works for as long as the
-endpoint does not require one. An HTTP 401 or 403 is never retried; the error names the fix. A
+`VALINOR_DOWNLOADS_TOKEN`. With neither, the launcher sends no credential, which works only from GitHub
+Actions and allow-listed networks. An HTTP 401 or 403 is never retried; the error names the fix. A
 `pull_request` run from a fork cannot mint an OIDC token, so it needs `VALINOR_DOWNLOADS_TOKEN` once the
 endpoint requires authentication.
 
+**Getting a downloads token.** Ask whoever runs your Valinor downloads endpoint for a personal token: each
+person gets their own, so one person's access can be withdrawn without touching anyone else's. Keep it in
+your secret store and export it as `VALINOR_DOWNLOADS_TOKEN` in the shell that runs Valinor; never commit
+it.
+
+**Which 403 you got.** A 403 is named by who sent it. "The endpoint verified the GitHub OIDC token, but this
+repository's owner is not authorized" comes from the endpoint itself, for a CI token from an organization it
+does not serve. "HTTP 403 from Cloudflare's firewall in front of the downloads endpoint" means the request
+never reached the endpoint: without a credential, set `VALINOR_DOWNLOADS_TOKEN`; with one, retry from
+another network and report the Ray ID the message gives.
+
+**A failed download is a hard stop.** Valinor does not run, and nothing runs in its place: there is no
+switch, variable or flag that skips the download, and no other cached, local or `npx` copy stands in.
+A cache hit runs offline only after the kept tarball re-hashes to the SHA-256 recorded when it was first
+verified; a cache entry that fails that check is refused, and the error says to delete it and run again
+with network access. Installs live under `verified/` in the cache. An entry an older launcher wrote (which
+kept no tarball) is never run, moved or deleted: the version is downloaded and verified once into
+`verified/`. Concurrent runs never replace an install another run is using.
+
 A download that fails for a transient reason (a dropped connection, a stalled transfer, a timeout, or an
 HTTP 408, 425, 429 or 5xx) is retried with jittered backoff: 3 attempts per file within a 2-minute budget by default. A 404
 and a checksum mismatch are never retried. When `review-trigger-run` still cannot be fetched after the

package/lib/launcher.js

@@ -10,8 +10,9 @@
 //   • One credential per run, only when the endpoint asks for one: a GitHub Actions OIDC token minted
 //     for the downloads audience, else VALINOR_DOWNLOADS_TOKEN, else none (see downloadsAuthorization).
 //   • Fail closed on integrity: a checksum mismatch DELETES the download and refuses to run.
-//   • Offline-friendly: an exactly-pinned, already-cached version runs with NO network at all;
-//     only a dist-tag (`latest`) needs the manifest.
+//   • Offline-friendly: an exactly-pinned, already-cached version runs with NO network at all, once its
+//     kept tarball re-hashes to the digest recorded at install; only a dist-tag (`latest`) needs the manifest.
+//   • A failed download is a HARD STOP: nothing (no cached, local or npx copy) runs in the bundle's place.
 //   • Loud, actionable degradation: an unreachable endpoint names the URL it tried AND the local
 //     cache state (what versions could still run offline) — never a bare stack trace.
 import { execFileSync, spawnSync } from "node:child_process";
@@ -19,7 +20,7 @@ import { createHash, randomBytes } from "node:crypto";
 import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { setTimeout as sleep } from "node:timers/promises";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 
 /**
@@ -82,11 +83,15 @@ export function requestedVersion(env = process.env) {
   return own === "0.0.0" ? "latest" : own;
 }
 
-/** The versions present in the local cache (complete installs only), newest-ish last. Never throws. */
+/**
+ * The versions present in the local cache (complete installs that kept their verified tarball, so they can run
+ * offline), newest-ish last. Never throws. It only feeds diagnostics: each one is still re-verified before it runs.
+ */
 export function cachedVersions(root) {
   try {
-    return readdirSync(root)
-      .filter((name) => SEMVER_RE.test(name) && existsSync(join(root, name, ".complete")))
+    const verified = join(root, VERIFIED_CACHE_DIR);
+    return readdirSync(verified)
+      .filter((name) => SEMVER_RE.test(name) && existsSync(join(verified, name, ".complete")) && existsSync(join(verified, name, "valinor.tgz")))
       .sort();
   } catch (e) {
     // ENOENT only = "no cache dir yet" — a genuinely empty cache. Anything else (EACCES, EIO, …)
@@ -190,20 +195,95 @@ function onceAuthorization(resolveAuthorization) {
   return () => (pending ??= resolveAuthorization());
 }
 
+/**
+ * The first words of the downloads Worker's OWN 403 body (sites/downloads/src/worker.js `refuse`): the only 403 the
+ * endpoint itself sends, for a verified GitHub OIDC token whose repository owner is not on its allowlist.
+ */
+const OWNER_REFUSAL_PREFIX = "forbidden: the GitHub OIDC token is valid";
+
+/** The doc that says how an operator gets a downloads token: this package's own README, which ships with it. */
+const DOWNLOADS_TOKEN_DOC = "the \"Authentication\" section of the @theappagency/valinor README";
+
+/** Said after every refusal: a failed download is a hard stop, and nothing else runs in Valinor's place. */
+const HARD_STOP = "Valinor did not run, and nothing runs in its place: no cached, local or npx copy is used instead.";
+
+/** The title Cloudflare's own block page carries, near the top of its HTML. */
+const CLOUDFLARE_BLOCK_PAGE_RE = /<title>[^<]*\|\s*Cloudflare\s*<\/title>/i;
+
+/** At most this much of a refusal body is read: enough for the Worker's one-line answer and a block page's title. */
+export const REFUSAL_BODY_PREFIX_BYTES = 4096;
+
+/**
+ * Who answered a 403, from its headers and at most {@link REFUSAL_BODY_PREFIX_BYTES} of its body. `owner` is the
+ * Worker's own plain-text OIDC-owner refusal. `edge` is Cloudflare's firewall in front of it, and needs a
+ * Cloudflare-specific signal: a `cf-mitigated` header, or a `cf-ray` header on Cloudflare's HTML block page. The request
+ * never reached the Worker. Anything else (`other`, an HTML page from a corporate proxy included) is a 403 from
+ * something in front of the endpoint that is not named.
+ */
+export function classify403(headers, body) {
+  const type = String(headers?.get?.("content-type") ?? "").toLowerCase();
+  const text = String(body ?? "");
+  if (type.startsWith("text/plain") && text.startsWith(OWNER_REFUSAL_PREFIX)) return "owner";
+  if (headers?.get?.("cf-mitigated")) return "edge";
+  if (headers?.get?.("cf-ray") && type.startsWith("text/html") && CLOUDFLARE_BLOCK_PAGE_RE.test(text)) return "edge";
+  return "other";
+}
+
+/**
+ * Read at most `max` bytes of a response body as text, then cancel the rest, so a large refusal page is never held
+ * in memory. A body that cannot be read yields what was read so far.
+ */
+async function readBodyPrefix(res, max = REFUSAL_BODY_PREFIX_BYTES) {
+  const reader = res.body?.getReader?.();
+  if (!reader) return "";
+  const chunks = [];
+  let size = 0;
+  try {
+    while (size < max) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      chunks.push(value);
+      size += value.byteLength;
+    }
+  } catch {
+    // keep what was read
+  } finally {
+    await reader.cancel().catch(() => {});
+  }
+  return new TextDecoder().decode(Buffer.concat(chunks).subarray(0, max));
+}
+
 /** The refusal for a 401/403 — an auth answer is never an outage, so it names the fix, not the network. */
-function authRefusal(url, status, sentCredential) {
+function authRefusal(url, status, sentCredential, headers, body) {
   const fix =
     "In GitHub Actions, add `permissions: id-token: write` to this job so the launcher can mint an OIDC token " +
     "(`valinor refresh-workflows --apply` adds it to Valinor's own workflows); elsewhere, set VALINOR_DOWNLOADS_TOKEN.";
-  const e =
-    status === 403
-      ? fetchError(`${url} returned HTTP 403: the endpoint verified the credential, but this repository's owner is not authorized to download Valinor.`, false)
-      : fetchError(
-          sentCredential
-            ? `${url} returned HTTP 401: the downloads endpoint refused the credential the launcher sent (expired or not valid for it). ${fix}`
-            : `${url} returned HTTP 401: the downloads endpoint requires authentication and this run sent no credential. ${fix}`,
-          false,
-        );
+  let message;
+  if (status === 403) {
+    const who = classify403(headers, body);
+    const ray = headers?.get?.("cf-ray");
+    const rayNote = ray ? ` (Cloudflare Ray ID ${ray})` : "";
+    if (who === "owner") {
+      message = `${url} returned HTTP 403: the endpoint verified the GitHub OIDC token, but this repository's owner is not authorized to download Valinor.`;
+    } else if (!sentCredential) {
+      const where = who === "edge" ? `Cloudflare's firewall in front of the downloads endpoint${rayNote}` : "something between this machine and the downloads endpoint (a proxy or firewall)";
+      message =
+        `${url} returned HTTP 403 from ${where}; the endpoint itself never answers 403 to a request without a credential. ` +
+        "This request carried none, and a download without a credential is admitted only from GitHub Actions and allow-listed networks. " +
+        `Set VALINOR_DOWNLOADS_TOKEN to your own downloads token (how to get one: ${DOWNLOADS_TOKEN_DOC}); ` +
+        "in GitHub Actions, add `permissions: id-token: write` to the job instead.";
+    } else {
+      const where = who === "edge" ? `Cloudflare's firewall in front of the downloads endpoint${rayNote}` : "something between this machine and the downloads endpoint (a proxy or firewall)";
+      message =
+        `${url} returned HTTP 403 from ${where}, not from the endpoint, although the request carried a credential. ` +
+        `The credential was never checked. Retry from another network, or report the refusal${ray ? ` with Ray ID ${ray}` : ""}.`;
+    }
+  } else {
+    message = sentCredential
+      ? `${url} returned HTTP 401: the downloads endpoint refused the credential the launcher sent (expired or not valid for it). ${fix}`
+      : `${url} returned HTTP 401: the downloads endpoint requires authentication and this run sent no credential. ${fix}`;
+  }
+  const e = fetchError(`${message}\n${HARD_STOP}`, false);
   e.auth = true;
   return e;
 }
@@ -304,7 +384,10 @@ async function fetchBody(url, read, fetchImpl, policy, cancel, authorization) {
       last = fetchError(`could not reach ${url} (${describeFailure(e, timeoutMs, "waiting for a response")})`, true, e);
     }
     if (res && (res.status === 401 || res.status === 403)) {
-      last = authRefusal(url, res.status, credential !== null);
+      // A 403's body says WHO refused (the Worker's one-line answer, or a block page's title), so a bounded prefix of
+      // it is read, unless a `cf-mitigated` header already says so; every other body is cancelled unread.
+      const body = res.status === 403 && !res.headers.get("cf-mitigated") ? await readBodyPrefix(res) : "";
+      last = authRefusal(url, res.status, credential !== null, res.headers, body);
       await res.body?.cancel().catch(() => {});
     } else if (res && !res.ok) {
       last = fetchError(`${url} returned HTTP ${res.status}`, isTransientStatus(res.status));
@@ -337,18 +420,96 @@ export async function fetchManifest(base, fetchImpl, policy = fetchPolicy(), aut
   return fetchBody(`${base}/manifest.json`, (res) => res.json(), fetchImpl, policy, undefined, authorization);
 }
 
+/** A cached version that failed its check: terminal, never retried, and never given the offline hint. */
+function cacheRefusal(versionDir, why) {
+  const e = new Error(
+    `the cached Valinor at ${versionDir} failed verification: ${why}. Refusing to run it. ` +
+      `Delete ${versionDir} and run again with network access, so the launcher downloads and verifies it afresh.\n${HARD_STOP}`,
+  );
+  e.transient = false;
+  e.integrity = true;
+  return e;
+}
+
+const DIGEST_RE = /^[0-9a-f]{64}$/;
+
+/**
+ * Check a cached version before it runs. The cache keeps the VERIFIED tarball beside the extracted package, and
+ * `.complete` records the SHA-256 that SHA256SUMS gave for it at install. A hit re-hashes the kept tarball against
+ * that digest on every run, so a `.complete` marker on its own is never enough: a bare or malformed marker, or a
+ * tarball whose bytes are not the recorded ones (or a missing one), is refused. Returns { state: "absent" } (no
+ * marker: an ordinary cache miss) or { state: "verified" }, and throws {@link cacheRefusal} otherwise. This check
+ * cannot stop someone who writes a matching tarball AND digest into the cache, but anyone able to do that can run
+ * `node` on any file already; it closes the bare-marker shortcut.
+ */
+export function verifyCachedVersion(versionDir) {
+  const marker = join(versionDir, ".complete");
+  if (!existsSync(marker)) return { state: "absent" };
+  let recorded;
+  try {
+    recorded = readFileSync(marker, "utf8").trim();
+  } catch (e) {
+    throw cacheRefusal(versionDir, `its .complete marker could not be read (${e?.code ?? e})`);
+  }
+  if (!DIGEST_RE.test(recorded)) throw cacheRefusal(versionDir, "its .complete marker records no valid SHA-256 digest");
+  const kept = join(versionDir, "valinor.tgz");
+  if (!existsSync(kept)) throw cacheRefusal(versionDir, "it has no kept tarball to re-hash");
+  let actual;
+  try {
+    actual = createHash("sha256").update(readFileSync(kept)).digest("hex");
+  } catch (e) {
+    throw cacheRefusal(versionDir, `its kept tarball could not be read (${e?.code ?? e})`);
+  }
+  if (actual !== recorded) throw cacheRefusal(versionDir, `its kept tarball hashes to ${actual}, not the ${recorded} recorded when it was verified`);
+  return { state: "verified" };
+}
+
+/**
+ * Where this launcher keeps its installs: `<cache root>/verified/<version>/`, apart from the `<cache root>/<version>/`
+ * entries older launchers wrote (which kept no tarball, so cannot be re-verified). An older entry is never read, moved
+ * or deleted here — an older launcher may be running from it — so it is simply not a hit: the version is downloaded
+ * and verified once into this directory. Not a semver name, so an older launcher's cache listing never sees it.
+ */
+export const VERIFIED_CACHE_DIR = "verified";
+
+/** The directory this launcher installs `version` into (see {@link VERIFIED_CACHE_DIR}). */
+export function versionCacheDir(root, version) {
+  return join(root, VERIFIED_CACHE_DIR, version);
+}
+
 /**
  * Ensure `version` is present + verified in the cache; returns the cache dir for that version.
- * Cache HIT (a completed prior install) is served with NO network I/O. On a miss: download the
- * tarball AND its SHA256SUMS, recompute the digest, and FAIL CLOSED on mismatch (the partial
- * download is deleted; nothing executable is left behind). The verified tarball is extracted into
- * a temp dir and atomically renamed into place, so a crashed install can never half-populate a
- * version dir that a later run would trust.
+ * Cache HIT (a completed prior install whose kept tarball still matches its recorded digest, see
+ * {@link verifyCachedVersion}) is served with NO network I/O. On a miss: download the tarball AND its
+ * SHA256SUMS, recompute the digest, and FAIL CLOSED on mismatch (the partial download is deleted; nothing
+ * executable is left behind). The verified tarball is kept and extracted into a fresh staging dir that is
+ * renamed into place. A rename never replaces an existing install, so a version dir another process is running
+ * from is never moved or deleted: a run that loses the race uses the winner's install once it verifies.
  */
 export async function ensureVersion(base, version, root, fetchImpl, policy = fetchPolicy(), authorization) {
-  const versionDir = join(root, version);
-  if (existsSync(join(versionDir, ".complete"))) return versionDir; // cache hit — zero network.
+  const versionDir = versionCacheDir(root, version);
+  if (verifyCachedVersion(versionDir).state === "verified") return versionDir; // cache hit — zero network.
+  const legacyDir = join(root, version);
+  if (!existsSync(join(legacyDir, ".complete"))) return installVersion(base, version, versionDir, fetchImpl, policy, authorization);
+  try {
+    return await installVersion(base, version, versionDir, fetchImpl, policy, authorization);
+  } catch (e) {
+    if (e?.auth === true || e?.integrity === true || String(e?.message).includes("SHA-256 MISMATCH")) throw e;
+    const msg = e instanceof Error ? e.message : String(e);
+    const wrapped = new Error(
+      `the cached Valinor at ${legacyDir} was written by an older launcher that did not keep the verified tarball, so it is never ` +
+        `run; this version is downloaded and verified once into ${versionDir}, and that download failed: ${msg}\n` +
+        `Run again with network access (outside GitHub Actions, with VALINOR_DOWNLOADS_TOKEN set).\n${HARD_STOP}`,
+      { cause: e },
+    );
+    wrapped.transient = e?.transient === true;
+    wrapped.integrity = true;
+    throw wrapped;
+  }
+}
 
+/** Download, verify and install `version` into `versionDir`. */
+async function installVersion(base, version, versionDir, fetchImpl, policy, authorization) {
   const tarballUrl = `${base}/v/${version}/valinor.tgz`;
   // Either file failing for good fails the install, so it cancels the other's retry loop at once.
   const siblings = new AbortController();
@@ -370,20 +531,28 @@ export async function ensureVersion(base, version, root, fetchImpl, policy = fet
     );
   }
 
-  const stagingDir = join(root, `.tmp-${version}-${randomBytes(6).toString("hex")}`);
-  mkdirSync(stagingDir, { recursive: true });
+  const parent = dirname(versionDir);
+  mkdirSync(parent, { recursive: true });
+  const stagingDir = join(parent, `.tmp-${version}-${randomBytes(6).toString("hex")}`);
+  mkdirSync(stagingDir);
   try {
     const tgzPath = join(stagingDir, "valinor.tgz");
     writeFileSync(tgzPath, tarball);
     // `tar` ships with every supported platform (incl. Windows 10+'s bsdtar); using it (rather than
-    // npm) keeps the install registry-free and auth-free by construction.
+    // npm) keeps the install registry-free and auth-free by construction. The tarball stays: every later
+    // cache hit re-hashes it against the digest recorded below.
     execFileSync("tar", ["-xzf", tgzPath, "-C", stagingDir]);
-    rmSync(tgzPath);
     writeFileSync(join(stagingDir, ".complete"), `${expected}\n`);
     try {
+      // A directory rename never replaces a non-empty directory, so an install another process is running from
+      // is never moved or deleted under it.
       renameSync(stagingDir, versionDir);
     } catch (e) {
-      if (existsSync(join(versionDir, ".complete"))) return versionDir; // lost a concurrent race — theirs is verified too.
+      // Lost a concurrent race: theirs is used only if it verifies like any other cache hit.
+      if (verifyCachedVersion(versionDir).state === "verified") {
+        rmSync(stagingDir, { recursive: true, force: true });
+        return versionDir;
+      }
       throw e;
     }
   } catch (e) {
@@ -446,7 +615,7 @@ export async function run(args, env = process.env) {
   } catch (e) {
     const msg = e instanceof Error ? e.message : String(e);
     // Integrity and authentication failures are terminal as-is; reachability failures get the offline hint.
-    throw withHint(e, msg.includes("SHA-256 MISMATCH") || e?.auth === true ? msg : `${msg}\n${offlineHint(base, root)}`);
+    throw withHint(e, msg.includes("SHA-256 MISMATCH") || e?.auth === true || e?.integrity === true ? msg : `${msg}\n${offlineHint(base, root)}`);
   }
   return spawnCli(cachedBinPath(versionDir), args);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@theappagency/valinor",
-  "version": "10.18.0",
+  "version": "10.19.0",
   "description": "Public launcher for the Valinor CLI — fetches, SHA-256-verifies, caches, and runs the proprietary Valinor bundle from downloads.valinorci.com. Zero setup, zero secrets, zero npm auth.",
   "repository": {
     "type": "git",