@futurelastic/colab-handbook 1.14.0-rc.2 → 1.14.0-rc.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/audit/audit.mjs CHANGED
@@ -41,6 +41,19 @@
41
41
  // # (colab batch-stats --json) — see "Batch history" below
42
42
  // node audit.mjs --base <rev> # the instruction-file ratchet's base for LOCAL repos
43
43
  // # (default origin/<trunk>) — see checkInstructionFileShape
44
+ // node audit.mjs --offline # local checks only: no GitHub read at all — see "Remote cache"
45
+ // node audit.mjs --fresh # ignore the cache TTL: re-ask GitHub (still conditional)
46
+ // node audit.mjs --cache-ttl <secs> # how long a cached GitHub read is served unasked
47
+ // # (default 600; env COLAB_AUDIT_CACHE_TTL; 0 = always ask)
48
+ //
49
+ // Remote cache (#593): the GitHub reads (repository metadata, labels, and a slug target's files,
50
+ // branches and tags) are cached across runs, one file per repo under COLAB_AUDIT_CACHE_DIR, else
51
+ // ${COLAB_HOME:-~/.colab}/cache/audit when that home already exists (the audit never creates it).
52
+ // Within the TTL a read costs no request; past it each page is re-asked with If-None-Match, and a
53
+ // 304 costs no quota. A rate limit or no response serves the cached copy, marked stale. --json
54
+ // states per repo where its remote facts came from (`remote`). --offline is the cheap frequent
55
+ // pass: local checks only, remote-only (slug) entries skipped, and it refuses --identity,
56
+ // --batch-history and --fresh, which all need the network.
44
57
  //
45
58
  // Identity (--identity): metadata never passes through git, so the pre-commit identity scan
46
59
  // (templates/pre-commit-identity) structurally cannot see it, and it is the first thing a
@@ -112,6 +125,8 @@ const shipBatch = require("../tools/lib/ship-batch.js");
112
125
  const ciProfile = require("../tools/lib/ci-profile.js"); // #559
113
126
  const thresholdsLib = require("../tools/lib/thresholds.js"); // #560
114
127
  const batchHistory = require("../tools/lib/batch-history.js"); // #556
128
+ // #593: every GitHub REST read goes through one on-disk, per-repo cache (ETag + TTL) — see the module.
129
+ const auditCache = require("../tools/lib/audit-cache.js");
115
130
  // #383: where migrations live — the one rule `colab ship`'s gate and `release cut` read; the audit
116
131
  // validates the `migrations:` declaration against it and reports a `*/migrations/` dir it misses.
117
132
  const migrationPaths = require("../tools/lib/migration-paths.js");
@@ -155,13 +170,20 @@ const COLAB_HOME = process.env.COLAB_HOME || join(homedir(), ".colab");
155
170
 
156
171
  function parseArgs(argv) {
157
172
  // config === null means "resolve from the precedence chain"; a string means explicit.
158
- const opts = { config: null, locals: [], slugs: [], json: false, quiet: false, help: false, identity: false, batchHistory: false };
173
+ const opts = { config: null, locals: [], slugs: [], json: false, quiet: false, help: false, identity: false, batchHistory: false, offline: false, fresh: false, cacheTtl: null };
159
174
  for (let i = 0; i < argv.length; i++) {
160
175
  const a = argv[i];
161
176
  if (a === "--json") opts.json = true;
162
177
  else if (a === "--quiet" || a === "-q") opts.quiet = true;
163
178
  else if (a === "--identity") opts.identity = true;
164
179
  else if (a === "--batch-history") opts.batchHistory = true;
180
+ else if (a === "--offline" || a === "--no-remote") opts.offline = true;
181
+ else if (a === "--fresh") opts.fresh = true;
182
+ else if (a === "--cache-ttl") {
183
+ const p = argv[++i];
184
+ if (p === undefined) die("--cache-ttl needs a number of seconds");
185
+ opts.cacheTtl = p;
186
+ }
165
187
  else if (a === "--base") {
166
188
  const p = argv[++i];
167
189
  if (!p) die("--base needs a rev");
@@ -515,31 +537,30 @@ function runGh(args) {
515
537
  }
516
538
 
517
539
  // A slug is fetched read-only through the API — nothing is cloned, nothing is written
518
- // into anyone's working tree.
540
+ // into anyone's working tree. Every REST read below goes through `remoteCache` (#593), set up
541
+ // once per run in runAudit; the `--jq` filters these used to pass are applied here in JS,
542
+ // because a cached page is the raw API body.
543
+ let remoteCache = auditCache.create({ dir: null, ttlSeconds: 0 });
544
+
519
545
  function readRemoteFile(slug, path) {
520
- try {
521
- return runGh(["api", `repos/${slug}/contents/${path}`, "-H", "Accept: application/vnd.github.raw"]);
522
- } catch {
523
- return null;
524
- }
546
+ const r = remoteCache.get(slug, `repos/${slug}/contents/${path}`, { accept: "application/vnd.github.raw" });
547
+ return r.status === "ok" ? r.text : null;
525
548
  }
526
549
 
527
550
  function listRemoteDir(slug, path) {
551
+ const r = remoteCache.get(slug, `repos/${slug}/contents/${path}`);
552
+ if (r.status !== "ok") return [];
528
553
  try {
529
- const out = runGh(["api", `repos/${slug}/contents/${path}`, "--jq", ".[].name"]);
530
- return out.split("\n").map((s) => s.trim()).filter(Boolean);
554
+ const arr = JSON.parse(r.text);
555
+ return Array.isArray(arr) ? arr.map((e) => String(e.name || "").trim()).filter(Boolean) : [];
531
556
  } catch {
532
557
  return [];
533
558
  }
534
559
  }
535
560
 
536
561
  function listRemoteBranches(slug) {
537
- try {
538
- const out = runGh(["api", `repos/${slug}/branches`, "--paginate", "--jq", ".[].name"]);
539
- return out.split("\n").map((s) => s.trim()).filter(Boolean);
540
- } catch {
541
- return null; // null = could not determine, distinct from "no branches"
542
- }
562
+ const r = remoteCache.get(slug, `repos/${slug}/branches`, { paginate: true });
563
+ return r.status === "ok" ? r.items.map((b) => String(b.name)) : null; // null = could not determine, distinct from "no branches"
543
564
  }
544
565
 
545
566
  // Same null/[] contract as listRemoteBranches: null = could not determine (no remote, no
@@ -547,33 +568,22 @@ function listRemoteBranches(slug) {
547
568
  // falsifiers (#137) — a version-shaped tag is evidence against a declared "nothing consumes
548
569
  // this" claim, whether the source is local or remote.
549
570
  function listRemoteTags(slug) {
550
- try {
551
- const out = runGh(["api", `repos/${slug}/tags`, "--paginate", "--jq", ".[].name"]);
552
- return out.split("\n").map((s) => s.trim()).filter(Boolean);
553
- } catch {
554
- return null;
555
- }
571
+ const r = remoteCache.get(slug, `repos/${slug}/tags`, { paginate: true });
572
+ return r.status === "ok" ? r.items.map((t) => String(t.name)) : null;
556
573
  }
557
574
 
558
575
  // Labels live on GitHub, not in the working tree, so this is the audit's one check that
559
576
  // needs the tracker. `null` means "could not determine" — no remote, no auth, gh absent,
560
- // API error — and the caller stays SILENT on it rather than warning: unlike branches
577
+ // API error, --offline — and the caller stays SILENT on it rather than warning: unlike branches
561
578
  // (every git repo has them), labels are a GitHub-only concept and a remote-less or
562
579
  // offline audit legitimately cannot see them. A warn per offline repo would be noise,
563
580
  // and we cannot assert a label is missing when we could not read the set at all.
564
- // Each entry is { name, description } (#364) — one JSON object per line from `--jq`, so a
565
- // paginated read never has to splice arrays. A line that does not parse makes the whole read
566
- // null: a half-read set would report the unread half as "missing".
581
+ // Each entry is { name, description } (#364). Any page that does not parse makes the whole
582
+ // read null: a half-read set would report the unread half as "missing".
567
583
  function listRemoteLabels(slug) {
568
- try {
569
- const out = runGh(["api", `repos/${slug}/labels`, "--paginate", "--jq", ".[] | {name, description}"]);
570
- return out.split("\n").map((s) => s.trim()).filter(Boolean).map((line) => {
571
- const l = JSON.parse(line);
572
- return { name: String(l.name), description: typeof l.description === "string" ? l.description : "" };
573
- });
574
- } catch {
575
- return null;
576
- }
584
+ const r = remoteCache.get(slug, `repos/${slug}/labels`, { paginate: true });
585
+ if (r.status !== "ok") return null;
586
+ return r.items.map((l) => ({ name: String(l.name), description: typeof l.description === "string" ? l.description : "" }));
577
587
  }
578
588
 
579
589
  // The repository's own METADATA — name, description, homepage, topics, visibility — which
@@ -596,6 +606,7 @@ function listRemoteLabels(slug) {
596
606
  // One issue's state, labels and comments (#441 — a release.final-grant's decision issue), or null
597
607
  // when it could not be read. null is never "no decision": the caller fails closed on it.
598
608
  function readRemoteIssue(slug, num) {
609
+ if (remoteCache.offline) return null; // --offline: no tracker read (the caller skips the check)
599
610
  try {
600
611
  return JSON.parse(runGh(["issue", "view", String(num), "--repo", slug, "--json", "state,labels,comments"]));
601
612
  } catch {
@@ -609,11 +620,11 @@ const remoteMetadataCache = new Map();
609
620
  function readRemoteMetadata(slug) {
610
621
  if (remoteMetadataCache.has(slug)) return remoteMetadataCache.get(slug);
611
622
  let r;
612
- try {
613
- const out = runGh(["api", `repos/${slug}`]);
614
- r = { status: "ok", data: JSON.parse(out) };
615
- } catch {
616
- r = { status: "unreadable" };
623
+ const got = remoteCache.get(slug, `repos/${slug}`);
624
+ if (got.status === "offline") r = { status: "offline" };
625
+ else if (got.status !== "ok") r = { status: "unreadable" };
626
+ else {
627
+ try { r = { status: "ok", data: JSON.parse(got.text) }; } catch { r = { status: "unreadable" }; }
617
628
  }
618
629
  remoteMetadataCache.set(slug, r);
619
630
  return r;
@@ -1123,6 +1134,16 @@ function renderDuration(age, minDays = DURATION_MIN_DAYS) {
1123
1134
  }
1124
1135
 
1125
1136
  function auditRepo(target, ctx) {
1137
+ remoteCache.begin();
1138
+ const info = auditRepoInner(target, ctx);
1139
+ // #593: where this repo's remote facts came from — cache, revalidated (304), fetched, stale —
1140
+ // and how old the oldest is. Absent when the repo made no GitHub read at all.
1141
+ const remote = remoteCache.summary();
1142
+ if (remote) info.remote = remote;
1143
+ return info;
1144
+ }
1145
+
1146
+ function auditRepoInner(target, ctx) {
1126
1147
  const src = makeSource(target);
1127
1148
  // #522: the branch list is read by the single-trunk rule (tier B / exposure none|released) AND the
1128
1149
  // trunk-exists check below — one read, so a remote target costs one API listing, not three.
@@ -1572,7 +1593,11 @@ function auditRepo(target, ctx) {
1572
1593
  // not been reopened. Unreadable fails closed: an automatic production deploy nobody can
1573
1594
  // trace to a human's choice is exactly what the grant exists to rule out.
1574
1595
  const grant = rel.effective.finalGrant;
1575
- if (grant) {
1596
+ if (grant && ctx.offline) {
1597
+ // #593: --offline reads no tracker. Recorded, not judged — the run header says once that
1598
+ // remote facts were not read, so this row is never mistaken for a verified grant.
1599
+ info.releaseFinalGrant = { issue: grant.issue, ok: null, checked: false, reason: "--offline" };
1600
+ } else if (grant) {
1576
1601
  const v = releasePolicy.finalGrantVerdict(src.issue ? src.issue(grant.issue) : null, { issue: grant.issue, trust: trustHumansLib.parseTrustHumans(cfg) });
1577
1602
  info.releaseFinalGrant = { issue: grant.issue, ok: v.ok, ruledBy: v.ruledBy };
1578
1603
  if (!v.ok) {
@@ -2900,6 +2925,9 @@ function checkPrivateNpm(src, workflows, fail, warn) {
2900
2925
  const assetSurface = npmGuard.releaseAssetUploads(workflows, reader.readFile).length > 0;
2901
2926
  if (!npmSurface && !assetSurface) return;
2902
2927
  const meta = src.metadata ? src.metadata() : { status: "unreadable" };
2928
+ // #593: --offline reads no visibility by request; the run header already says so once, and a
2929
+ // "could not be read" advisory on every offline pass would mark every such repo non-clean.
2930
+ if (meta.status === "offline") return;
2903
2931
  let visibility = null;
2904
2932
  if (meta.status === "ok" && meta.data) {
2905
2933
  visibility = meta.data.visibility || (meta.data.private === true ? "private" : meta.data.private === false ? "public" : null);
@@ -3251,6 +3279,10 @@ function report(results, opts, ctx) {
3251
3279
  ? { scanned: true, vocabulary: ctx.identity.path, source: ctx.identity.source, terms: ctx.identity.count }
3252
3280
  : { scanned: false, reason: "not requested (--identity)" },
3253
3281
  batchHistory: ctx.batchHistory ? { read: true } : { read: false, reason: "not requested (--batch-history)" },
3282
+ // #593: whether GitHub was read at all this run, and the cache it was read through.
3283
+ remote: ctx.offline
3284
+ ? { read: false, reason: "--offline" }
3285
+ : { read: true, cache: ctx.cache.dir, persistent: !!ctx.cache.dir, ttlSeconds: ctx.cache.ttlSeconds, fresh: ctx.cache.fresh },
3254
3286
  results,
3255
3287
  }, null, 2));
3256
3288
  return;
@@ -3272,6 +3304,10 @@ function report(results, opts, ctx) {
3272
3304
  console.log(ctx.batchHistory
3273
3305
  ? "batch: batch landing history read per local repo (colab batch-stats); advisories only against declared thresholds"
3274
3306
  : "batch: batch landing history NOT read (pass --batch-history)");
3307
+ // #593: same once-per-run statement for the remote reads.
3308
+ console.log(ctx.offline
3309
+ ? "remote: GitHub NOT read (--offline) — labels, visibility and grants unchecked; remote-only entries skipped"
3310
+ : `remote: GitHub read through ${ctx.cache.dir ? `the cache at ${ctx.cache.dir}` : "an in-memory cache (no colab home to persist it)"}, TTL ${ctx.cache.ttlSeconds}s${ctx.cache.fresh ? ", --fresh" : ""}`);
3275
3311
  console.log("");
3276
3312
 
3277
3313
  const shown = opts.quiet ? results.filter((r) => !r.clean) : results;
@@ -3287,7 +3323,8 @@ function report(results, opts, ctx) {
3287
3323
  // The handbook's own row says WHY it carries no stamp findings. Without this it
3288
3324
  // would read as clean-by-luck, which is indistinguishable from a check that
3289
3325
  // quietly stopped working.
3290
- if (r.self) lines.push("⌂ handbook source — every rule applies except its own stamps (it has nothing to copy from)");
3326
+ if (r.skipped === "offline") lines.push("– skipped (--offline: a remote-only entry has nothing local to check)");
3327
+ else if (r.self) lines.push("⌂ handbook source — every rule applies except its own stamps (it has nothing to copy from)");
3291
3328
  else if (r.adoption === "local") lines.push(`⌂ adopted locally, not committed — descriptor hidden by .git/info/exclude; a repo the fleet does not own (CONVENTIONS.md §9)${r.clean ? " ✓" : ""}`);
3292
3329
  else if (r.clean) lines.push("✓");
3293
3330
  r.findings.forEach((f) => lines.push(`${f.level === "fail" ? "⚠" : "·"} ${f.text}`));
@@ -3304,8 +3341,9 @@ function report(results, opts, ctx) {
3304
3341
 
3305
3342
  const failed = results.filter((r) => !r.ok).length;
3306
3343
  const warned = results.filter((r) => r.ok && !r.clean).length;
3344
+ const skipped = results.filter((r) => r.skipped).length;
3307
3345
  console.log("");
3308
- console.log(`${results.length} repo(s): ${results.length - failed - warned} clean, ${warned} with advisories, ${failed} with problems.`);
3346
+ console.log(`${results.length} repo(s): ${results.length - failed - warned - skipped} clean, ${warned} with advisories, ${failed} with problems${skipped ? `, ${skipped} skipped` : ""}.`);
3309
3347
  }
3310
3348
 
3311
3349
  /**
@@ -3353,13 +3391,33 @@ function runAudit(opts) {
3353
3391
  const targets = loadTargets(opts);
3354
3392
  if (!targets.length) die("nothing to audit — add entries to the repo list or pass --local <path>");
3355
3393
 
3394
+ if (opts.offline) {
3395
+ // Each of these is a read of the network the operator asked for; an offline run that quietly
3396
+ // skipped it would report clean over a check that never ran.
3397
+ if (opts.identity) die("--offline and --identity conflict: the identity scan reads repository metadata from GitHub");
3398
+ if (opts.batchHistory) die("--offline and --batch-history conflict: batch history reads CI runs from GitHub");
3399
+ if (opts.fresh) die("--offline and --fresh conflict: --fresh asks GitHub, --offline never does");
3400
+ }
3401
+ const ttlSeconds = auditCache.resolveTtlSeconds(opts.cacheTtl);
3402
+ if (ttlSeconds === null) die(`--cache-ttl / COLAB_AUDIT_CACHE_TTL must be a whole number of seconds (got "${opts.cacheTtl ?? process.env.COLAB_AUDIT_CACHE_TTL}")`);
3403
+ remoteCache = auditCache.create({
3404
+ dir: auditCache.resolveDir(process.env, COLAB_HOME), ttlSeconds, fresh: opts.fresh, offline: opts.offline,
3405
+ });
3406
+
3356
3407
  const ctx = { handbook: handbookInfo(HANDBOOK_ROOT), templateNames: templateNames(HANDBOOK_ROOT) };
3408
+ ctx.offline = !!opts.offline;
3409
+ ctx.cache = { dir: remoteCache.dir, ttlSeconds, fresh: !!opts.fresh };
3357
3410
  ctx.identity = opts.identity ? loadIdentityVocabulary() : null;
3358
3411
  ctx.batchHistory = !!opts.batchHistory;
3359
3412
  ctx.base = opts.base || null;
3360
3413
 
3361
3414
  const results = [];
3362
3415
  for (const t of targets) {
3416
+ if (ctx.offline && t.kind !== "local") {
3417
+ // A slug target has nothing local to check: every fact about it is a GitHub read.
3418
+ results.push({ repo: t.label, kind: t.kind, tier: null, ok: true, clean: true, skipped: "offline", remote: { mode: "offline" }, findings: [] });
3419
+ continue;
3420
+ }
3363
3421
  try {
3364
3422
  results.push(auditRepo(t, ctx));
3365
3423
  } catch (err) {
package/package.json CHANGED
@@ -33,5 +33,5 @@
33
33
  "engines": {
34
34
  "node": ">=18"
35
35
  },
36
- "version": "1.14.0-rc.2"
36
+ "version": "1.14.0-rc.4"
37
37
  }
package/tools/colab CHANGED
@@ -11819,12 +11819,14 @@ function ciGrantPayload(ciResult) {
11819
11819
  * on its own.
11820
11820
  *
11821
11821
  * Measures, in order:
11822
- * 1. containment — `git merge-base --is-ancestor <redSha> <branch>`, translated to a bool.
11822
+ * 1. containment — cure-diff's `cureContainment`: `git merge-base --is-ancestor <redSha> <ref>`
11823
+ * against the ref ship lands (local branch, else `<remote>/<branch>` — #589), translated to
11824
+ * true/false, or null ("could not resolve") when the ref or the red sha does not resolve.
11823
11825
  * 2. evidence — a completed, successful CI run for `branch`'s CURRENT head, via the identical
11824
11826
  * `ghRunForSha` the raw CI check and `shipCiGrants` both use.
11825
11827
  * 3. anti-stacking — `computeAntiStacking`, the SAME function `cmdCiGrantCreate` calls, so a cure
11826
11828
  * and a human grant can never independently disagree about whether trunk is mid-stack.
11827
- * 4. the diff signals — tools/lib/cure-diff.js's `cureDiffSignals` over `<trunk>...<branch>`
11829
+ * 4. the diff signals — tools/lib/cure-diff.js's `cureDiffSignals` over `<trunk>...<ref>` (same ref)
11828
11830
  * (merge-base-relative, the tree a squash lands), `--no-renames` so a workflow file or
11829
11831
  * manifest MOVED away still counts (#297 — the old `--name-only` read listed only a rename's
11830
11832
  * new path). It yields `workflowsTouched` (condition 4) and `manifestScriptsTouched`
@@ -11860,11 +11862,17 @@ function shipCiCure(repoAbs, gh, target, trunk, redSha, branch) {
11860
11862
  if (!gh) {
11861
11863
  return { ok: false, reason: `gh not usable (${ghUnusableWhy(repoAbs)}) — the cure rule needs a measured CI run`, redSha, evidenceSha: null };
11862
11864
  }
11863
- const containsRedSha = git.git(['merge-base', '--is-ancestor', redSha, branch], repoAbs).ok;
11865
+ // #589: containment and the diff read the ref ship lands — the local branch, else its remote-tracking
11866
+ // ref (a --handoff land has no local branch). The bare name failed there and read as "not an
11867
+ // ancestor"; an unresolvable ref now refuses as "could not resolve", never as "does not contain".
11868
+ // The CI read keeps the NAME: ghRunForSha resolves the remote head itself (ls-remote).
11869
+ const containment = cureDiff.cureContainment(git, repoAbs, redSha, branch);
11870
+ const containsRedSha = containment.contains;
11871
+ const branchRef = containment.ref || branch;
11864
11872
  const evRun = git.ghRunForSha(repoAbs, branch);
11865
11873
  const evidence = !evRun ? null : { ok: evRun.status === 'completed' && evRun.conclusion === 'success', sha: evRun.sha };
11866
11874
  const stacking = computeAntiStacking(repoAbs, trunk, true /* trunkIsRed: shipCiCure is only ever reached from a HUMAN_GATED raw verdict */);
11867
- const diff = cureDiff.cureDiffSignals(git, repoAbs, trunk, branch);
11875
+ const diff = cureDiff.cureDiffSignals(git, repoAbs, trunk, branchRef);
11868
11876
  // Measured only past containment + 2a: a verdict that already refuses on either never reads it,
11869
11877
  // so a branch that cannot cure pays no job-level `gh` calls for it.
11870
11878
  const jobEvidence = (containsRedSha && evidence && evidence.ok)
@@ -11876,7 +11884,7 @@ function shipCiCure(repoAbs, gh, target, trunk, redSha, branch) {
11876
11884
  const dispatchable = jobEvidence && evidence ? dispatchableWorkflows(repoAbs, evidence.sha) : [];
11877
11885
  // #477: a stacked refusal may be lifted by progress — measure the prior red set only then.
11878
11886
  const priorRedJobs = jobEvidence && stacking && stacking.stacked ? priorExemptionRedJobs(repoAbs, trunk, stacking) : null;
11879
- const verdict = ciCure.cureVerdict({ containsRedSha, evidence, redSha, stacking, jobEvidence, priorRedJobs,
11887
+ const verdict = ciCure.cureVerdict({ containsRedSha, containmentReason: containment.reason, evidence, redSha, stacking, jobEvidence, priorRedJobs,
11880
11888
  workflowsTouched: diff.workflowsTouched,
11881
11889
  manifestScriptsTouched: diff.manifestScriptsTouched, manifestPaths: diff.manifestPaths,
11882
11890
  manifestScriptsAddOnly: diff.manifestScriptsAddOnly,
@@ -0,0 +1,277 @@
1
+ 'use strict';
2
+ /**
3
+ * The audit's GitHub REST reads, cached across runs (#593).
4
+ *
5
+ * Why: the audit reads GitHub for every repo on every run (`repos/<r>`, `repos/<r>/labels`, and
6
+ * for a slug target its files, branches and tags). A host that runs it often — a dashboard
7
+ * rebuilding its overview — made it the largest consumer of the account's REST quota (measured
8
+ * ≈10,000 calls/h on one machine against a 5,000/h limit), stalling everything else that machine
9
+ * does with GitHub. Repo settings and label sets change rarely.
10
+ *
11
+ * How:
12
+ * - one JSON file per repo, keyed by API path (+ Accept), holding each PAGE's body and ETag;
13
+ * - within the TTL (default 600 s) an entry is served with no request at all;
14
+ * - past it, every page is re-asked with `If-None-Match` — a 304 costs no quota and re-arms the
15
+ * TTL; a 200 replaces the page;
16
+ * - `fresh` skips the TTL (every read goes to the server, still conditional — a 304 is the
17
+ * server saying "unchanged", which is as fresh as a 200);
18
+ * - a 404 is cached as "missing" for the TTL (it has no ETag, so re-asking it past the TTL costs one call);
19
+ * - on a rate limit or no response at all, an existing entry is served and labelled `stale`;
20
+ * any other error (404, 403, unparseable) is an error, never a stale answer — a repo that
21
+ * was deleted must not keep reporting its old label set.
22
+ *
23
+ * Where: `COLAB_AUDIT_CACHE_DIR`, else `<COLAB_HOME>/cache/audit` — but only when COLAB_HOME
24
+ * already EXISTS. The audit never creates the operator's colab home; without it the cache lives
25
+ * in memory for one run (each read still made at most once per run).
26
+ *
27
+ * Partitioned by WHO answers: the `gh` binary's resolved path, GH_HOST and a hash of any token in
28
+ * the environment. Two tokens can see different repos, and a test's fake `gh` must never read —
29
+ * or write — the operator's real entries.
30
+ *
31
+ * Provenance: every read is logged against the repo currently being audited (`begin()`), and
32
+ * `summary()` reports how many reads came from cache, were revalidated, fetched, or stale, and
33
+ * the age of the oldest data served — what `--json` shows per repo.
34
+ */
35
+
36
+ const fs = require('fs');
37
+ const path = require('path');
38
+ const crypto = require('crypto');
39
+ const { spawnSync } = require('child_process');
40
+ const { parseHttp } = require('./ci-wait');
41
+
42
+ const DEFAULT_TTL_SECONDS = 600;
43
+ const PER_PAGE = 100;
44
+ const MAX_PAGES = 50; // 5,000 labels/tags/branches — a runaway Link chain must not loop forever
45
+ const PRUNE_AFTER_MS = 7 * 24 * 3600 * 1000;
46
+ const FORMAT = 1;
47
+
48
+ /** TTL in seconds from a flag value, else COLAB_AUDIT_CACHE_TTL, else the default. null = invalid. */
49
+ function resolveTtlSeconds(flag, env = process.env) {
50
+ const raw = flag !== undefined && flag !== null ? flag : env.COLAB_AUDIT_CACHE_TTL;
51
+ if (raw === undefined || raw === null || raw === '') return DEFAULT_TTL_SECONDS;
52
+ const s = String(raw).trim();
53
+ if (!/^\d+$/.test(s)) return null;
54
+ return Number(s);
55
+ }
56
+
57
+ /** The cache directory, or null when there is nowhere we are allowed to write. */
58
+ function resolveDir(env = process.env, colabHome) {
59
+ if (env.COLAB_AUDIT_CACHE_DIR) return path.resolve(env.COLAB_AUDIT_CACHE_DIR);
60
+ if (colabHome && fs.existsSync(colabHome)) return path.join(colabHome, 'cache', 'audit');
61
+ return null;
62
+ }
63
+
64
+ /** First executable `gh` on PATH, resolved — the transport half of the partition key. */
65
+ function ghOnPath(env = process.env) {
66
+ for (const d of String(env.PATH || '').split(path.delimiter)) {
67
+ if (!d) continue;
68
+ const p = path.join(d, 'gh');
69
+ try {
70
+ fs.accessSync(p, fs.constants.X_OK);
71
+ return fs.realpathSync(p);
72
+ } catch (_) { /* next */ }
73
+ }
74
+ return null;
75
+ }
76
+
77
+ function partitionKey(env = process.env) {
78
+ const token = env.GH_TOKEN || env.GITHUB_TOKEN || env.GH_ENTERPRISE_TOKEN || '';
79
+ const parts = [ghOnPath(env) || 'no-gh', env.GH_HOST || 'github.com',
80
+ token ? crypto.createHash('sha256').update(token).digest('hex') : 'ambient'];
81
+ return crypto.createHash('sha256').update(parts.join('\n')).digest('hex').slice(0, 16);
82
+ }
83
+
84
+ /**
85
+ * One GET through `gh api -i`. The path stays the first argument after `api` (callers and fakes
86
+ * match on it). Returns { kind: 'ok'|'not-modified'|'rate-limited'|'unreachable'|'error',
87
+ * etag, next, body }.
88
+ */
89
+ function ghFetch(apiPath, { etag, accept } = {}) {
90
+ const args = ['api', apiPath, '-i'];
91
+ if (etag) args.push('-H', `If-None-Match: ${etag}`);
92
+ if (accept) args.push('-H', `Accept: ${accept}`);
93
+ const r = spawnSync('gh', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 64 * 1024 * 1024 });
94
+ if (r.error) return { kind: 'unreachable' };
95
+ return classify(r.stdout, r.stderr, r.status);
96
+ }
97
+
98
+ function classify(stdout, stderr, exitCode) {
99
+ const http = parseHttp(stdout);
100
+ const status = http.status;
101
+ const h = http.headers || {};
102
+ const rateText = /rate limit/i.test(String(stderr || '') + (status && status >= 400 ? http.body : ''));
103
+ if (status === 429 || (status === 403 && (h['x-ratelimit-remaining'] === '0' || rateText)) || (status === null && rateText)) {
104
+ return { kind: 'rate-limited' };
105
+ }
106
+ if (status === 304) return { kind: 'not-modified', etag: h.etag || null };
107
+ const next = /rel="next"/.test(h.link || '');
108
+ if (status !== null && status >= 200 && status < 300) return { kind: 'ok', etag: h.etag || null, next, linkKnown: true, body: http.body };
109
+ // No status line: a `gh` that does not print headers. Exit 0 is an answer without an ETag.
110
+ if (status === null && exitCode === 0) return { kind: 'ok', etag: null, next: false, linkKnown: false, body: String(stdout || '') };
111
+ if (status === null) return { kind: 'unreachable' };
112
+ return { kind: 'error', status };
113
+ }
114
+
115
+ function withPage(apiPath, page) {
116
+ return `${apiPath}${apiPath.includes('?') ? '&' : '?'}per_page=${PER_PAGE}&page=${page}`;
117
+ }
118
+
119
+ /**
120
+ * create({ dir, ttlSeconds, fresh, offline, now, fetch, env })
121
+ * get(slug, apiPath, { accept, paginate }) → { status: 'ok', text } | { status: 'ok', items }
122
+ * | { status: 'offline' } | { status: 'error' }
123
+ * begin() — start logging reads for the next repo; summary() — that repo's provenance.
124
+ */
125
+ function create(opts = {}) {
126
+ const env = opts.env || process.env;
127
+ const now = opts.now || Date.now;
128
+ const fetch = opts.fetch || ghFetch;
129
+ const ttlMs = (opts.ttlSeconds === undefined ? DEFAULT_TTL_SECONDS : opts.ttlSeconds) * 1000;
130
+ const fresh = !!opts.fresh;
131
+ const offline = !!opts.offline;
132
+ const dir = opts.dir ? path.join(opts.dir, opts.partition || partitionKey(env)) : null;
133
+
134
+ const files = new Map(); // slug → { entries }
135
+ const doneThisRun = new Set(); // slug|key validated or fetched in this process
136
+ let log = [];
137
+ let pruned = false;
138
+
139
+ const fileFor = (slug) => (dir ? path.join(dir, `${slug.replace(/[^A-Za-z0-9._-]/g, '__')}.json`) : null);
140
+
141
+ function load(slug) {
142
+ if (files.has(slug)) return files.get(slug);
143
+ let data = { format: FORMAT, slug, entries: {} };
144
+ const f = fileFor(slug);
145
+ if (f) {
146
+ try {
147
+ const parsed = JSON.parse(fs.readFileSync(f, 'utf8'));
148
+ if (parsed && parsed.format === FORMAT && parsed.entries && typeof parsed.entries === 'object') data = parsed;
149
+ } catch (_) { /* absent or corrupt: start empty, the next write repairs it */ }
150
+ }
151
+ files.set(slug, data);
152
+ return data;
153
+ }
154
+
155
+ function save(slug) {
156
+ const f = fileFor(slug);
157
+ if (!f) return;
158
+ try {
159
+ fs.mkdirSync(dir, { recursive: true });
160
+ const tmp = `${f}.${process.pid}.tmp`;
161
+ fs.writeFileSync(tmp, JSON.stringify(files.get(slug)));
162
+ fs.renameSync(tmp, f); // atomic: a concurrent audit reads the old file or the new one
163
+ } catch (_) { /* a cache that cannot be written is a slower run, never a failed one */ }
164
+ if (!pruned) { pruned = true; prune(); }
165
+ }
166
+
167
+ function prune() {
168
+ const root = path.dirname(dir);
169
+ const cutoff = now() - PRUNE_AFTER_MS;
170
+ let parts;
171
+ try { parts = fs.readdirSync(root); } catch (_) { return; }
172
+ for (const p of parts) {
173
+ const d = path.join(root, p);
174
+ let names;
175
+ try { names = fs.readdirSync(d); } catch (_) { continue; }
176
+ for (const n of names) {
177
+ const fp = path.join(d, n);
178
+ try { if (fs.statSync(fp).mtimeMs < cutoff) fs.unlinkSync(fp); } catch (_) { /* raced */ }
179
+ }
180
+ try { if (!fs.readdirSync(d).length) fs.rmdirSync(d); } catch (_) { /* not empty / raced */ }
181
+ }
182
+ }
183
+
184
+ function record(source, ageMs) { log.push({ source, ageSeconds: Math.max(0, Math.round(ageMs / 1000)) }); }
185
+
186
+ function answer(entry, paginate) {
187
+ if (entry.missing) return { status: 'error' };
188
+ if (!paginate) return { status: 'ok', text: entry.pages[0].body };
189
+ const items = [];
190
+ for (const p of entry.pages) {
191
+ let arr;
192
+ try { arr = JSON.parse(p.body); } catch (_) { return { status: 'error' }; }
193
+ if (!Array.isArray(arr)) return { status: 'error' };
194
+ items.push(...arr);
195
+ }
196
+ return { status: 'ok', items };
197
+ }
198
+
199
+ function get(slug, apiPath, { accept = null, paginate = false } = {}) {
200
+ if (offline) { record('offline', 0); return { status: 'offline' }; }
201
+ const key = accept ? `${apiPath}|${accept}` : apiPath;
202
+ const data = load(slug);
203
+ const entry = data.entries[key];
204
+ const runKey = `${slug}|${key}`;
205
+
206
+ if (entry && (doneThisRun.has(runKey) || (!fresh && now() - entry.at < ttlMs))) {
207
+ record(doneThisRun.has(runKey) ? 'run' : 'cache', now() - entry.at);
208
+ return answer(entry, paginate);
209
+ }
210
+
211
+ const pages = [];
212
+ let allNotModified = true;
213
+ for (let i = 0; i < (paginate ? MAX_PAGES : 1); i++) {
214
+ const cached = entry && !entry.missing && entry.pages[i];
215
+ const r = fetch(paginate ? withPage(apiPath, i + 1) : apiPath, { etag: cached ? cached.etag : null, accept });
216
+ let page;
217
+ if (r.kind === 'not-modified' && cached) page = cached;
218
+ else if (r.kind === 'ok') {
219
+ allNotModified = false;
220
+ let next = !!r.next;
221
+ if (paginate && !r.linkKnown) { // no headers to follow: a full page means there may be more
222
+ try { const a = JSON.parse(r.body); next = Array.isArray(a) && a.length >= PER_PAGE; } catch (_) { next = false; }
223
+ }
224
+ page = { etag: r.etag || null, next, body: r.body };
225
+ } else {
226
+ if (entry && (r.kind === 'rate-limited' || r.kind === 'unreachable')) {
227
+ record('stale', now() - entry.at);
228
+ return { ...answer(entry, paginate), stale: true };
229
+ }
230
+ if (r.kind === 'error' && r.status === 404 && i === 0) {
231
+ // An absent file is an answer too, and the audit asks for several on every run (a
232
+ // runbook, a tool, a workflow dir). Cached for the TTL like any other; past it, asked
233
+ // again — a 404 carries no ETag, so that one costs a call.
234
+ data.entries[key] = { at: now(), missing: true, pages: [] };
235
+ doneThisRun.add(runKey);
236
+ save(slug);
237
+ record('fetched', 0);
238
+ return { status: 'error' };
239
+ }
240
+ record('error', 0);
241
+ return { status: 'error' };
242
+ }
243
+ pages.push(page);
244
+ if (!paginate || !page.next) break;
245
+ }
246
+
247
+ data.entries[key] = { at: now(), pages };
248
+ doneThisRun.add(runKey);
249
+ save(slug);
250
+ record(allNotModified ? 'revalidated' : 'fetched', 0);
251
+ return answer(data.entries[key], paginate);
252
+ }
253
+
254
+ function begin() { log = []; }
255
+
256
+ /** This repo's remote provenance since begin(); null when it made no GitHub read at all. */
257
+ function summary() {
258
+ if (offline) return { mode: 'offline' };
259
+ if (!log.length) return null;
260
+ const count = (s) => log.filter((e) => e.source === s).length;
261
+ const served = log.filter((e) => e.source !== 'error');
262
+ return {
263
+ mode: 'live',
264
+ reads: log.length,
265
+ cache: count('cache') + count('run'),
266
+ revalidated: count('revalidated'),
267
+ fetched: count('fetched'),
268
+ stale: count('stale'),
269
+ errors: count('error'),
270
+ oldestAgeSeconds: served.length ? Math.max(...served.map((e) => e.ageSeconds)) : null,
271
+ };
272
+ }
273
+
274
+ return { get, begin, summary, dir, ttlSeconds: ttlMs / 1000, fresh, offline };
275
+ }
276
+
277
+ module.exports = { create, ghFetch, classify, resolveTtlSeconds, resolveDir, partitionKey, DEFAULT_TTL_SECONDS, PER_PAGE };
@@ -820,6 +820,8 @@ function stackingWithProgress(stacking, priorRedJobs, jobEvidence) {
820
820
  * `containsRedSha` — bool, whether the branch's tree contains the red trunk sha as an ancestor
821
821
  * (the caller's `git merge-base --is-ancestor <redSha> <branch>`, translated to a bool — never
822
822
  * re-derived here, this module does not touch git).
823
+ * `null` (#589) means the caller could NOT measure it — the ref did not resolve, or git failed —
824
+ * and refuses with `containmentReason` ("could not resolve …"), never as "does not contain".
823
825
  *
824
826
  * `evidence` — `{ok, sha}` for a completed, successful CI run measured on the branch's CURRENT
825
827
  * head, or `null` when the caller's own read of it FAILED. Passing `evidence: null` means "the
@@ -886,7 +888,14 @@ function stackingWithProgress(stacking, priorRedJobs, jobEvidence) {
886
888
  */
887
889
  function cureVerdict({ containsRedSha, evidence, redSha, stacking, workflowsTouched, jobEvidence,
888
890
  manifestScriptsTouched, manifestPaths, manifestScriptsAddOnly, pythonManifestTouched, pythonManifestPaths,
889
- pythonPinOnly, dryRunCapable, dispatchable, branchRuns, priorRedJobs }) {
891
+ pythonPinOnly, dryRunCapable, dispatchable, branchRuns, priorRedJobs, containmentReason }) {
892
+ // #589: `null` = containment could not be MEASURED (an unresolvable ref, a failed git read) — a
893
+ // different fact from "not an ancestor", whose remedy (rebase onto the red) would be wrong advice.
894
+ if (containsRedSha === null) {
895
+ return { ok: false,
896
+ reason: `${containmentReason || `could not resolve the branch or trunk's red head \`${redSha}\``} — ` +
897
+ 'containment was not measured, so this cannot prove a cure (an unmeasured read is never a cure)' };
898
+ }
890
899
  if (!containsRedSha) {
891
900
  return { ok: false,
892
901
  reason: `branch does not contain trunk's current red head \`${redSha}\` as an ancestor — ` +
@@ -358,6 +358,32 @@ function sideText(git, repo, sha) {
358
358
  return r.ok ? r.stdout : null;
359
359
  }
360
360
 
361
+ /**
362
+ * The cure rule's containment read (#281 condition 1), against the ref `ship` actually lands (#589).
363
+ *
364
+ * A `--handoff` land usually has no local branch — only `<remote>/<branch>` — so asking git about
365
+ * the bare name failed, and that failure read as "not an ancestor": a real cure refused until a
366
+ * local ref was made by hand. The ref is resolved here exactly as `shipRefFor` does (local branch,
367
+ * else its remote-tracking ref), and the three outcomes are kept apart:
368
+ * `{ref, contains: true}` · `{ref, contains: false}` (git exit 1 — genuinely not an ancestor) ·
369
+ * `{ref, contains: null, reason}` — the branch or the red sha could not be resolved, or git failed.
370
+ * `null` refuses in `cureVerdict` with the "could not resolve" reason, never "does not contain".
371
+ * Never throws.
372
+ */
373
+ function cureContainment(git, repo, redSha, branch, remote) {
374
+ const refs = git.branchRefs(repo, branch, ...(remote ? [remote] : []));
375
+ const ref = refs.localSha || refs.remoteSha;
376
+ if (!ref) {
377
+ return { ref: null, contains: null,
378
+ reason: `could not resolve ${branch} — no local branch and no remote-tracking ref (fetch it first)` };
379
+ }
380
+ const r = git.git(['merge-base', '--is-ancestor', redSha, ref], repo);
381
+ if (r.ok) return { ref, contains: true, reason: '' };
382
+ if (r.code === 1) return { ref, contains: false, reason: '' };
383
+ return { ref, contains: null,
384
+ reason: `could not resolve trunk's red head \`${redSha}\` against ${branch} (${ref.slice(0, 7)}): ${(r.stderr || '').split('\n')[0] || `git exit ${r.code}`}` };
385
+ }
386
+
361
387
  /**
362
388
  * The diff signals for one branch against trunk, over the merge-base range (`trunk...branch` — the
363
389
  * same tree a squash lands). Returns `{ ok, reason, workflowsTouched, manifestScriptsTouched,
@@ -452,5 +478,5 @@ function pythonSignal(git, repo, trunk, branch, entries) {
452
478
  return { touched: paths.length > 0, paths, reason: '' };
453
479
  }
454
480
 
455
- module.exports = { cureDiffSignals, scriptsBlockDiffers, scriptsAddition, pythonPinChange, parseRequirement,
481
+ module.exports = { cureDiffSignals, cureContainment, scriptsBlockDiffers, scriptsAddition, pythonPinChange, parseRequirement,
456
482
  isPythonManifest, pythonIncludes, ABSENT };