@futurelastic/colab-handbook 1.14.0-rc.3 → 1.14.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/audit/audit.mjs +100 -42
- package/package.json +1 -1
- package/tools/lib/audit-cache.js +277 -0
package/audit/audit.mjs
CHANGED
|
@@ -41,6 +41,19 @@
|
|
|
41
41
|
// # (colab batch-stats --json) — see "Batch history" below
|
|
42
42
|
// node audit.mjs --base <rev> # the instruction-file ratchet's base for LOCAL repos
|
|
43
43
|
// # (default origin/<trunk>) — see checkInstructionFileShape
|
|
44
|
+
// node audit.mjs --offline # local checks only: no GitHub read at all — see "Remote cache"
|
|
45
|
+
// node audit.mjs --fresh # ignore the cache TTL: re-ask GitHub (still conditional)
|
|
46
|
+
// node audit.mjs --cache-ttl <secs> # how long a cached GitHub read is served unasked
|
|
47
|
+
// # (default 600; env COLAB_AUDIT_CACHE_TTL; 0 = always ask)
|
|
48
|
+
//
|
|
49
|
+
// Remote cache (#593): the GitHub reads (repository metadata, labels, and a slug target's files,
|
|
50
|
+
// branches and tags) are cached across runs, one file per repo under COLAB_AUDIT_CACHE_DIR, else
|
|
51
|
+
// ${COLAB_HOME:-~/.colab}/cache/audit when that home already exists (the audit never creates it).
|
|
52
|
+
// Within the TTL a read costs no request; past it each page is re-asked with If-None-Match, and a
|
|
53
|
+
// 304 costs no quota. A rate limit or no response serves the cached copy, marked stale. --json
|
|
54
|
+
// states per repo where its remote facts came from (`remote`). --offline is the cheap frequent
|
|
55
|
+
// pass: local checks only, remote-only (slug) entries skipped, and it refuses --identity,
|
|
56
|
+
// --batch-history and --fresh, which all need the network.
|
|
44
57
|
//
|
|
45
58
|
// Identity (--identity): metadata never passes through git, so the pre-commit identity scan
|
|
46
59
|
// (templates/pre-commit-identity) structurally cannot see it, and it is the first thing a
|
|
@@ -112,6 +125,8 @@ const shipBatch = require("../tools/lib/ship-batch.js");
|
|
|
112
125
|
const ciProfile = require("../tools/lib/ci-profile.js"); // #559
|
|
113
126
|
const thresholdsLib = require("../tools/lib/thresholds.js"); // #560
|
|
114
127
|
const batchHistory = require("../tools/lib/batch-history.js"); // #556
|
|
128
|
+
// #593: every GitHub REST read goes through one on-disk, per-repo cache (ETag + TTL) — see the module.
|
|
129
|
+
const auditCache = require("../tools/lib/audit-cache.js");
|
|
115
130
|
// #383: where migrations live — the one rule `colab ship`'s gate and `release cut` read; the audit
|
|
116
131
|
// validates the `migrations:` declaration against it and reports a `*/migrations/` dir it misses.
|
|
117
132
|
const migrationPaths = require("../tools/lib/migration-paths.js");
|
|
@@ -155,13 +170,20 @@ const COLAB_HOME = process.env.COLAB_HOME || join(homedir(), ".colab");
|
|
|
155
170
|
|
|
156
171
|
function parseArgs(argv) {
|
|
157
172
|
// config === null means "resolve from the precedence chain"; a string means explicit.
|
|
158
|
-
const opts = { config: null, locals: [], slugs: [], json: false, quiet: false, help: false, identity: false, batchHistory: false };
|
|
173
|
+
const opts = { config: null, locals: [], slugs: [], json: false, quiet: false, help: false, identity: false, batchHistory: false, offline: false, fresh: false, cacheTtl: null };
|
|
159
174
|
for (let i = 0; i < argv.length; i++) {
|
|
160
175
|
const a = argv[i];
|
|
161
176
|
if (a === "--json") opts.json = true;
|
|
162
177
|
else if (a === "--quiet" || a === "-q") opts.quiet = true;
|
|
163
178
|
else if (a === "--identity") opts.identity = true;
|
|
164
179
|
else if (a === "--batch-history") opts.batchHistory = true;
|
|
180
|
+
else if (a === "--offline" || a === "--no-remote") opts.offline = true;
|
|
181
|
+
else if (a === "--fresh") opts.fresh = true;
|
|
182
|
+
else if (a === "--cache-ttl") {
|
|
183
|
+
const p = argv[++i];
|
|
184
|
+
if (p === undefined) die("--cache-ttl needs a number of seconds");
|
|
185
|
+
opts.cacheTtl = p;
|
|
186
|
+
}
|
|
165
187
|
else if (a === "--base") {
|
|
166
188
|
const p = argv[++i];
|
|
167
189
|
if (!p) die("--base needs a rev");
|
|
@@ -515,31 +537,30 @@ function runGh(args) {
|
|
|
515
537
|
}
|
|
516
538
|
|
|
517
539
|
// A slug is fetched read-only through the API — nothing is cloned, nothing is written
|
|
518
|
-
// into anyone's working tree.
|
|
540
|
+
// into anyone's working tree. Every REST read below goes through `remoteCache` (#593), set up
|
|
541
|
+
// once per run in runAudit; the `--jq` filters these used to pass are applied here in JS,
|
|
542
|
+
// because a cached page is the raw API body.
|
|
543
|
+
let remoteCache = auditCache.create({ dir: null, ttlSeconds: 0 });
|
|
544
|
+
|
|
519
545
|
function readRemoteFile(slug, path) {
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
} catch {
|
|
523
|
-
return null;
|
|
524
|
-
}
|
|
546
|
+
const r = remoteCache.get(slug, `repos/${slug}/contents/${path}`, { accept: "application/vnd.github.raw" });
|
|
547
|
+
return r.status === "ok" ? r.text : null;
|
|
525
548
|
}
|
|
526
549
|
|
|
527
550
|
function listRemoteDir(slug, path) {
|
|
551
|
+
const r = remoteCache.get(slug, `repos/${slug}/contents/${path}`);
|
|
552
|
+
if (r.status !== "ok") return [];
|
|
528
553
|
try {
|
|
529
|
-
const
|
|
530
|
-
return
|
|
554
|
+
const arr = JSON.parse(r.text);
|
|
555
|
+
return Array.isArray(arr) ? arr.map((e) => String(e.name || "").trim()).filter(Boolean) : [];
|
|
531
556
|
} catch {
|
|
532
557
|
return [];
|
|
533
558
|
}
|
|
534
559
|
}
|
|
535
560
|
|
|
536
561
|
function listRemoteBranches(slug) {
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
return out.split("\n").map((s) => s.trim()).filter(Boolean);
|
|
540
|
-
} catch {
|
|
541
|
-
return null; // null = could not determine, distinct from "no branches"
|
|
542
|
-
}
|
|
562
|
+
const r = remoteCache.get(slug, `repos/${slug}/branches`, { paginate: true });
|
|
563
|
+
return r.status === "ok" ? r.items.map((b) => String(b.name)) : null; // null = could not determine, distinct from "no branches"
|
|
543
564
|
}
|
|
544
565
|
|
|
545
566
|
// Same null/[] contract as listRemoteBranches: null = could not determine (no remote, no
|
|
@@ -547,33 +568,22 @@ function listRemoteBranches(slug) {
|
|
|
547
568
|
// falsifiers (#137) — a version-shaped tag is evidence against a declared "nothing consumes
|
|
548
569
|
// this" claim, whether the source is local or remote.
|
|
549
570
|
function listRemoteTags(slug) {
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
return out.split("\n").map((s) => s.trim()).filter(Boolean);
|
|
553
|
-
} catch {
|
|
554
|
-
return null;
|
|
555
|
-
}
|
|
571
|
+
const r = remoteCache.get(slug, `repos/${slug}/tags`, { paginate: true });
|
|
572
|
+
return r.status === "ok" ? r.items.map((t) => String(t.name)) : null;
|
|
556
573
|
}
|
|
557
574
|
|
|
558
575
|
// Labels live on GitHub, not in the working tree, so this is the audit's one check that
|
|
559
576
|
// needs the tracker. `null` means "could not determine" — no remote, no auth, gh absent,
|
|
560
|
-
// API error — and the caller stays SILENT on it rather than warning: unlike branches
|
|
577
|
+
// API error, --offline — and the caller stays SILENT on it rather than warning: unlike branches
|
|
561
578
|
// (every git repo has them), labels are a GitHub-only concept and a remote-less or
|
|
562
579
|
// offline audit legitimately cannot see them. A warn per offline repo would be noise,
|
|
563
580
|
// and we cannot assert a label is missing when we could not read the set at all.
|
|
564
|
-
// Each entry is { name, description } (#364)
|
|
565
|
-
//
|
|
566
|
-
// null: a half-read set would report the unread half as "missing".
|
|
581
|
+
// Each entry is { name, description } (#364). Any page that does not parse makes the whole
|
|
582
|
+
// read null: a half-read set would report the unread half as "missing".
|
|
567
583
|
function listRemoteLabels(slug) {
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
const l = JSON.parse(line);
|
|
572
|
-
return { name: String(l.name), description: typeof l.description === "string" ? l.description : "" };
|
|
573
|
-
});
|
|
574
|
-
} catch {
|
|
575
|
-
return null;
|
|
576
|
-
}
|
|
584
|
+
const r = remoteCache.get(slug, `repos/${slug}/labels`, { paginate: true });
|
|
585
|
+
if (r.status !== "ok") return null;
|
|
586
|
+
return r.items.map((l) => ({ name: String(l.name), description: typeof l.description === "string" ? l.description : "" }));
|
|
577
587
|
}
|
|
578
588
|
|
|
579
589
|
// The repository's own METADATA — name, description, homepage, topics, visibility — which
|
|
@@ -596,6 +606,7 @@ function listRemoteLabels(slug) {
|
|
|
596
606
|
// One issue's state, labels and comments (#441 — a release.final-grant's decision issue), or null
|
|
597
607
|
// when it could not be read. null is never "no decision": the caller fails closed on it.
|
|
598
608
|
function readRemoteIssue(slug, num) {
|
|
609
|
+
if (remoteCache.offline) return null; // --offline: no tracker read (the caller skips the check)
|
|
599
610
|
try {
|
|
600
611
|
return JSON.parse(runGh(["issue", "view", String(num), "--repo", slug, "--json", "state,labels,comments"]));
|
|
601
612
|
} catch {
|
|
@@ -609,11 +620,11 @@ const remoteMetadataCache = new Map();
|
|
|
609
620
|
function readRemoteMetadata(slug) {
|
|
610
621
|
if (remoteMetadataCache.has(slug)) return remoteMetadataCache.get(slug);
|
|
611
622
|
let r;
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
r = { status: "unreadable" };
|
|
623
|
+
const got = remoteCache.get(slug, `repos/${slug}`);
|
|
624
|
+
if (got.status === "offline") r = { status: "offline" };
|
|
625
|
+
else if (got.status !== "ok") r = { status: "unreadable" };
|
|
626
|
+
else {
|
|
627
|
+
try { r = { status: "ok", data: JSON.parse(got.text) }; } catch { r = { status: "unreadable" }; }
|
|
617
628
|
}
|
|
618
629
|
remoteMetadataCache.set(slug, r);
|
|
619
630
|
return r;
|
|
@@ -1123,6 +1134,16 @@ function renderDuration(age, minDays = DURATION_MIN_DAYS) {
|
|
|
1123
1134
|
}
|
|
1124
1135
|
|
|
1125
1136
|
function auditRepo(target, ctx) {
|
|
1137
|
+
remoteCache.begin();
|
|
1138
|
+
const info = auditRepoInner(target, ctx);
|
|
1139
|
+
// #593: where this repo's remote facts came from — cache, revalidated (304), fetched, stale —
|
|
1140
|
+
// and how old the oldest is. Absent when the repo made no GitHub read at all.
|
|
1141
|
+
const remote = remoteCache.summary();
|
|
1142
|
+
if (remote) info.remote = remote;
|
|
1143
|
+
return info;
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
function auditRepoInner(target, ctx) {
|
|
1126
1147
|
const src = makeSource(target);
|
|
1127
1148
|
// #522: the branch list is read by the single-trunk rule (tier B / exposure none|released) AND the
|
|
1128
1149
|
// trunk-exists check below — one read, so a remote target costs one API listing, not three.
|
|
@@ -1572,7 +1593,11 @@ function auditRepo(target, ctx) {
|
|
|
1572
1593
|
// not been reopened. Unreadable fails closed: an automatic production deploy nobody can
|
|
1573
1594
|
// trace to a human's choice is exactly what the grant exists to rule out.
|
|
1574
1595
|
const grant = rel.effective.finalGrant;
|
|
1575
|
-
if (grant) {
|
|
1596
|
+
if (grant && ctx.offline) {
|
|
1597
|
+
// #593: --offline reads no tracker. Recorded, not judged — the run header says once that
|
|
1598
|
+
// remote facts were not read, so this row is never mistaken for a verified grant.
|
|
1599
|
+
info.releaseFinalGrant = { issue: grant.issue, ok: null, checked: false, reason: "--offline" };
|
|
1600
|
+
} else if (grant) {
|
|
1576
1601
|
const v = releasePolicy.finalGrantVerdict(src.issue ? src.issue(grant.issue) : null, { issue: grant.issue, trust: trustHumansLib.parseTrustHumans(cfg) });
|
|
1577
1602
|
info.releaseFinalGrant = { issue: grant.issue, ok: v.ok, ruledBy: v.ruledBy };
|
|
1578
1603
|
if (!v.ok) {
|
|
@@ -2900,6 +2925,9 @@ function checkPrivateNpm(src, workflows, fail, warn) {
|
|
|
2900
2925
|
const assetSurface = npmGuard.releaseAssetUploads(workflows, reader.readFile).length > 0;
|
|
2901
2926
|
if (!npmSurface && !assetSurface) return;
|
|
2902
2927
|
const meta = src.metadata ? src.metadata() : { status: "unreadable" };
|
|
2928
|
+
// #593: --offline reads no visibility by request; the run header already says so once, and a
|
|
2929
|
+
// "could not be read" advisory on every offline pass would mark every such repo non-clean.
|
|
2930
|
+
if (meta.status === "offline") return;
|
|
2903
2931
|
let visibility = null;
|
|
2904
2932
|
if (meta.status === "ok" && meta.data) {
|
|
2905
2933
|
visibility = meta.data.visibility || (meta.data.private === true ? "private" : meta.data.private === false ? "public" : null);
|
|
@@ -3251,6 +3279,10 @@ function report(results, opts, ctx) {
|
|
|
3251
3279
|
? { scanned: true, vocabulary: ctx.identity.path, source: ctx.identity.source, terms: ctx.identity.count }
|
|
3252
3280
|
: { scanned: false, reason: "not requested (--identity)" },
|
|
3253
3281
|
batchHistory: ctx.batchHistory ? { read: true } : { read: false, reason: "not requested (--batch-history)" },
|
|
3282
|
+
// #593: whether GitHub was read at all this run, and the cache it was read through.
|
|
3283
|
+
remote: ctx.offline
|
|
3284
|
+
? { read: false, reason: "--offline" }
|
|
3285
|
+
: { read: true, cache: ctx.cache.dir, persistent: !!ctx.cache.dir, ttlSeconds: ctx.cache.ttlSeconds, fresh: ctx.cache.fresh },
|
|
3254
3286
|
results,
|
|
3255
3287
|
}, null, 2));
|
|
3256
3288
|
return;
|
|
@@ -3272,6 +3304,10 @@ function report(results, opts, ctx) {
|
|
|
3272
3304
|
console.log(ctx.batchHistory
|
|
3273
3305
|
? "batch: batch landing history read per local repo (colab batch-stats); advisories only against declared thresholds"
|
|
3274
3306
|
: "batch: batch landing history NOT read (pass --batch-history)");
|
|
3307
|
+
// #593: same once-per-run statement for the remote reads.
|
|
3308
|
+
console.log(ctx.offline
|
|
3309
|
+
? "remote: GitHub NOT read (--offline) — labels, visibility and grants unchecked; remote-only entries skipped"
|
|
3310
|
+
: `remote: GitHub read through ${ctx.cache.dir ? `the cache at ${ctx.cache.dir}` : "an in-memory cache (no colab home to persist it)"}, TTL ${ctx.cache.ttlSeconds}s${ctx.cache.fresh ? ", --fresh" : ""}`);
|
|
3275
3311
|
console.log("");
|
|
3276
3312
|
|
|
3277
3313
|
const shown = opts.quiet ? results.filter((r) => !r.clean) : results;
|
|
@@ -3287,7 +3323,8 @@ function report(results, opts, ctx) {
|
|
|
3287
3323
|
// The handbook's own row says WHY it carries no stamp findings. Without this it
|
|
3288
3324
|
// would read as clean-by-luck, which is indistinguishable from a check that
|
|
3289
3325
|
// quietly stopped working.
|
|
3290
|
-
if (r.
|
|
3326
|
+
if (r.skipped === "offline") lines.push("– skipped (--offline: a remote-only entry has nothing local to check)");
|
|
3327
|
+
else if (r.self) lines.push("⌂ handbook source — every rule applies except its own stamps (it has nothing to copy from)");
|
|
3291
3328
|
else if (r.adoption === "local") lines.push(`⌂ adopted locally, not committed — descriptor hidden by .git/info/exclude; a repo the fleet does not own (CONVENTIONS.md §9)${r.clean ? " ✓" : ""}`);
|
|
3292
3329
|
else if (r.clean) lines.push("✓");
|
|
3293
3330
|
r.findings.forEach((f) => lines.push(`${f.level === "fail" ? "⚠" : "·"} ${f.text}`));
|
|
@@ -3304,8 +3341,9 @@ function report(results, opts, ctx) {
|
|
|
3304
3341
|
|
|
3305
3342
|
const failed = results.filter((r) => !r.ok).length;
|
|
3306
3343
|
const warned = results.filter((r) => r.ok && !r.clean).length;
|
|
3344
|
+
const skipped = results.filter((r) => r.skipped).length;
|
|
3307
3345
|
console.log("");
|
|
3308
|
-
console.log(`${results.length} repo(s): ${results.length - failed - warned} clean, ${warned} with advisories, ${failed} with problems.`);
|
|
3346
|
+
console.log(`${results.length} repo(s): ${results.length - failed - warned - skipped} clean, ${warned} with advisories, ${failed} with problems${skipped ? `, ${skipped} skipped` : ""}.`);
|
|
3309
3347
|
}
|
|
3310
3348
|
|
|
3311
3349
|
/**
|
|
@@ -3353,13 +3391,33 @@ function runAudit(opts) {
|
|
|
3353
3391
|
const targets = loadTargets(opts);
|
|
3354
3392
|
if (!targets.length) die("nothing to audit — add entries to the repo list or pass --local <path>");
|
|
3355
3393
|
|
|
3394
|
+
if (opts.offline) {
|
|
3395
|
+
// Each of these is a read of the network the operator asked for; an offline run that quietly
|
|
3396
|
+
// skipped it would report clean over a check that never ran.
|
|
3397
|
+
if (opts.identity) die("--offline and --identity conflict: the identity scan reads repository metadata from GitHub");
|
|
3398
|
+
if (opts.batchHistory) die("--offline and --batch-history conflict: batch history reads CI runs from GitHub");
|
|
3399
|
+
if (opts.fresh) die("--offline and --fresh conflict: --fresh asks GitHub, --offline never does");
|
|
3400
|
+
}
|
|
3401
|
+
const ttlSeconds = auditCache.resolveTtlSeconds(opts.cacheTtl);
|
|
3402
|
+
if (ttlSeconds === null) die(`--cache-ttl / COLAB_AUDIT_CACHE_TTL must be a whole number of seconds (got "${opts.cacheTtl ?? process.env.COLAB_AUDIT_CACHE_TTL}")`);
|
|
3403
|
+
remoteCache = auditCache.create({
|
|
3404
|
+
dir: auditCache.resolveDir(process.env, COLAB_HOME), ttlSeconds, fresh: opts.fresh, offline: opts.offline,
|
|
3405
|
+
});
|
|
3406
|
+
|
|
3356
3407
|
const ctx = { handbook: handbookInfo(HANDBOOK_ROOT), templateNames: templateNames(HANDBOOK_ROOT) };
|
|
3408
|
+
ctx.offline = !!opts.offline;
|
|
3409
|
+
ctx.cache = { dir: remoteCache.dir, ttlSeconds, fresh: !!opts.fresh };
|
|
3357
3410
|
ctx.identity = opts.identity ? loadIdentityVocabulary() : null;
|
|
3358
3411
|
ctx.batchHistory = !!opts.batchHistory;
|
|
3359
3412
|
ctx.base = opts.base || null;
|
|
3360
3413
|
|
|
3361
3414
|
const results = [];
|
|
3362
3415
|
for (const t of targets) {
|
|
3416
|
+
if (ctx.offline && t.kind !== "local") {
|
|
3417
|
+
// A slug target has nothing local to check: every fact about it is a GitHub read.
|
|
3418
|
+
results.push({ repo: t.label, kind: t.kind, tier: null, ok: true, clean: true, skipped: "offline", remote: { mode: "offline" }, findings: [] });
|
|
3419
|
+
continue;
|
|
3420
|
+
}
|
|
3363
3421
|
try {
|
|
3364
3422
|
results.push(auditRepo(t, ctx));
|
|
3365
3423
|
} catch (err) {
|
package/package.json
CHANGED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* The audit's GitHub REST reads, cached across runs (#593).
|
|
4
|
+
*
|
|
5
|
+
* Why: the audit reads GitHub for every repo on every run (`repos/<r>`, `repos/<r>/labels`, and
|
|
6
|
+
* for a slug target its files, branches and tags). A host that runs it often — a dashboard
|
|
7
|
+
* rebuilding its overview — made it the largest consumer of the account's REST quota (measured
|
|
8
|
+
* ≈10,000 calls/h on one machine against a 5,000/h limit), stalling everything else that machine
|
|
9
|
+
* does with GitHub. Repo settings and label sets change rarely.
|
|
10
|
+
*
|
|
11
|
+
* How:
|
|
12
|
+
* - one JSON file per repo, keyed by API path (+ Accept), holding each PAGE's body and ETag;
|
|
13
|
+
* - within the TTL (default 600 s) an entry is served with no request at all;
|
|
14
|
+
* - past it, every page is re-asked with `If-None-Match` — a 304 costs no quota and re-arms the
|
|
15
|
+
* TTL; a 200 replaces the page;
|
|
16
|
+
* - `fresh` skips the TTL (every read goes to the server, still conditional — a 304 is the
|
|
17
|
+
* server saying "unchanged", which is as fresh as a 200);
|
|
18
|
+
* - a 404 is cached as "missing" for the TTL (it has no ETag, so re-asking it past the TTL costs one call);
|
|
19
|
+
* - on a rate limit or no response at all, an existing entry is served and labelled `stale`;
|
|
20
|
+
* any other error (404, 403, unparseable) is an error, never a stale answer — a repo that
|
|
21
|
+
* was deleted must not keep reporting its old label set.
|
|
22
|
+
*
|
|
23
|
+
* Where: `COLAB_AUDIT_CACHE_DIR`, else `<COLAB_HOME>/cache/audit` — but only when COLAB_HOME
|
|
24
|
+
* already EXISTS. The audit never creates the operator's colab home; without it the cache lives
|
|
25
|
+
* in memory for one run (each read still made at most once per run).
|
|
26
|
+
*
|
|
27
|
+
* Partitioned by WHO answers: the `gh` binary's resolved path, GH_HOST and a hash of any token in
|
|
28
|
+
* the environment. Two tokens can see different repos, and a test's fake `gh` must never read —
|
|
29
|
+
* or write — the operator's real entries.
|
|
30
|
+
*
|
|
31
|
+
* Provenance: every read is logged against the repo currently being audited (`begin()`), and
|
|
32
|
+
* `summary()` reports how many reads came from cache, were revalidated, fetched, or stale, and
|
|
33
|
+
* the age of the oldest data served — what `--json` shows per repo.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
const fs = require('fs');
|
|
37
|
+
const path = require('path');
|
|
38
|
+
const crypto = require('crypto');
|
|
39
|
+
const { spawnSync } = require('child_process');
|
|
40
|
+
const { parseHttp } = require('./ci-wait');
|
|
41
|
+
|
|
42
|
+
const DEFAULT_TTL_SECONDS = 600;
|
|
43
|
+
const PER_PAGE = 100;
|
|
44
|
+
const MAX_PAGES = 50; // 5,000 labels/tags/branches — a runaway Link chain must not loop forever
|
|
45
|
+
const PRUNE_AFTER_MS = 7 * 24 * 3600 * 1000;
|
|
46
|
+
const FORMAT = 1;
|
|
47
|
+
|
|
48
|
+
/** TTL in seconds from a flag value, else COLAB_AUDIT_CACHE_TTL, else the default. null = invalid. */
|
|
49
|
+
function resolveTtlSeconds(flag, env = process.env) {
|
|
50
|
+
const raw = flag !== undefined && flag !== null ? flag : env.COLAB_AUDIT_CACHE_TTL;
|
|
51
|
+
if (raw === undefined || raw === null || raw === '') return DEFAULT_TTL_SECONDS;
|
|
52
|
+
const s = String(raw).trim();
|
|
53
|
+
if (!/^\d+$/.test(s)) return null;
|
|
54
|
+
return Number(s);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** The cache directory, or null when there is nowhere we are allowed to write. */
|
|
58
|
+
function resolveDir(env = process.env, colabHome) {
|
|
59
|
+
if (env.COLAB_AUDIT_CACHE_DIR) return path.resolve(env.COLAB_AUDIT_CACHE_DIR);
|
|
60
|
+
if (colabHome && fs.existsSync(colabHome)) return path.join(colabHome, 'cache', 'audit');
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** First executable `gh` on PATH, resolved — the transport half of the partition key. */
|
|
65
|
+
function ghOnPath(env = process.env) {
|
|
66
|
+
for (const d of String(env.PATH || '').split(path.delimiter)) {
|
|
67
|
+
if (!d) continue;
|
|
68
|
+
const p = path.join(d, 'gh');
|
|
69
|
+
try {
|
|
70
|
+
fs.accessSync(p, fs.constants.X_OK);
|
|
71
|
+
return fs.realpathSync(p);
|
|
72
|
+
} catch (_) { /* next */ }
|
|
73
|
+
}
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function partitionKey(env = process.env) {
|
|
78
|
+
const token = env.GH_TOKEN || env.GITHUB_TOKEN || env.GH_ENTERPRISE_TOKEN || '';
|
|
79
|
+
const parts = [ghOnPath(env) || 'no-gh', env.GH_HOST || 'github.com',
|
|
80
|
+
token ? crypto.createHash('sha256').update(token).digest('hex') : 'ambient'];
|
|
81
|
+
return crypto.createHash('sha256').update(parts.join('\n')).digest('hex').slice(0, 16);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* One GET through `gh api -i`. The path stays the first argument after `api` (callers and fakes
|
|
86
|
+
* match on it). Returns { kind: 'ok'|'not-modified'|'rate-limited'|'unreachable'|'error',
|
|
87
|
+
* etag, next, body }.
|
|
88
|
+
*/
|
|
89
|
+
function ghFetch(apiPath, { etag, accept } = {}) {
|
|
90
|
+
const args = ['api', apiPath, '-i'];
|
|
91
|
+
if (etag) args.push('-H', `If-None-Match: ${etag}`);
|
|
92
|
+
if (accept) args.push('-H', `Accept: ${accept}`);
|
|
93
|
+
const r = spawnSync('gh', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 64 * 1024 * 1024 });
|
|
94
|
+
if (r.error) return { kind: 'unreachable' };
|
|
95
|
+
return classify(r.stdout, r.stderr, r.status);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function classify(stdout, stderr, exitCode) {
|
|
99
|
+
const http = parseHttp(stdout);
|
|
100
|
+
const status = http.status;
|
|
101
|
+
const h = http.headers || {};
|
|
102
|
+
const rateText = /rate limit/i.test(String(stderr || '') + (status && status >= 400 ? http.body : ''));
|
|
103
|
+
if (status === 429 || (status === 403 && (h['x-ratelimit-remaining'] === '0' || rateText)) || (status === null && rateText)) {
|
|
104
|
+
return { kind: 'rate-limited' };
|
|
105
|
+
}
|
|
106
|
+
if (status === 304) return { kind: 'not-modified', etag: h.etag || null };
|
|
107
|
+
const next = /rel="next"/.test(h.link || '');
|
|
108
|
+
if (status !== null && status >= 200 && status < 300) return { kind: 'ok', etag: h.etag || null, next, linkKnown: true, body: http.body };
|
|
109
|
+
// No status line: a `gh` that does not print headers. Exit 0 is an answer without an ETag.
|
|
110
|
+
if (status === null && exitCode === 0) return { kind: 'ok', etag: null, next: false, linkKnown: false, body: String(stdout || '') };
|
|
111
|
+
if (status === null) return { kind: 'unreachable' };
|
|
112
|
+
return { kind: 'error', status };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function withPage(apiPath, page) {
|
|
116
|
+
return `${apiPath}${apiPath.includes('?') ? '&' : '?'}per_page=${PER_PAGE}&page=${page}`;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* create({ dir, ttlSeconds, fresh, offline, now, fetch, env })
|
|
121
|
+
* get(slug, apiPath, { accept, paginate }) → { status: 'ok', text } | { status: 'ok', items }
|
|
122
|
+
* | { status: 'offline' } | { status: 'error' }
|
|
123
|
+
* begin() — start logging reads for the next repo; summary() — that repo's provenance.
|
|
124
|
+
*/
|
|
125
|
+
function create(opts = {}) {
|
|
126
|
+
const env = opts.env || process.env;
|
|
127
|
+
const now = opts.now || Date.now;
|
|
128
|
+
const fetch = opts.fetch || ghFetch;
|
|
129
|
+
const ttlMs = (opts.ttlSeconds === undefined ? DEFAULT_TTL_SECONDS : opts.ttlSeconds) * 1000;
|
|
130
|
+
const fresh = !!opts.fresh;
|
|
131
|
+
const offline = !!opts.offline;
|
|
132
|
+
const dir = opts.dir ? path.join(opts.dir, opts.partition || partitionKey(env)) : null;
|
|
133
|
+
|
|
134
|
+
const files = new Map(); // slug → { entries }
|
|
135
|
+
const doneThisRun = new Set(); // slug|key validated or fetched in this process
|
|
136
|
+
let log = [];
|
|
137
|
+
let pruned = false;
|
|
138
|
+
|
|
139
|
+
const fileFor = (slug) => (dir ? path.join(dir, `${slug.replace(/[^A-Za-z0-9._-]/g, '__')}.json`) : null);
|
|
140
|
+
|
|
141
|
+
function load(slug) {
|
|
142
|
+
if (files.has(slug)) return files.get(slug);
|
|
143
|
+
let data = { format: FORMAT, slug, entries: {} };
|
|
144
|
+
const f = fileFor(slug);
|
|
145
|
+
if (f) {
|
|
146
|
+
try {
|
|
147
|
+
const parsed = JSON.parse(fs.readFileSync(f, 'utf8'));
|
|
148
|
+
if (parsed && parsed.format === FORMAT && parsed.entries && typeof parsed.entries === 'object') data = parsed;
|
|
149
|
+
} catch (_) { /* absent or corrupt: start empty, the next write repairs it */ }
|
|
150
|
+
}
|
|
151
|
+
files.set(slug, data);
|
|
152
|
+
return data;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function save(slug) {
|
|
156
|
+
const f = fileFor(slug);
|
|
157
|
+
if (!f) return;
|
|
158
|
+
try {
|
|
159
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
160
|
+
const tmp = `${f}.${process.pid}.tmp`;
|
|
161
|
+
fs.writeFileSync(tmp, JSON.stringify(files.get(slug)));
|
|
162
|
+
fs.renameSync(tmp, f); // atomic: a concurrent audit reads the old file or the new one
|
|
163
|
+
} catch (_) { /* a cache that cannot be written is a slower run, never a failed one */ }
|
|
164
|
+
if (!pruned) { pruned = true; prune(); }
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function prune() {
|
|
168
|
+
const root = path.dirname(dir);
|
|
169
|
+
const cutoff = now() - PRUNE_AFTER_MS;
|
|
170
|
+
let parts;
|
|
171
|
+
try { parts = fs.readdirSync(root); } catch (_) { return; }
|
|
172
|
+
for (const p of parts) {
|
|
173
|
+
const d = path.join(root, p);
|
|
174
|
+
let names;
|
|
175
|
+
try { names = fs.readdirSync(d); } catch (_) { continue; }
|
|
176
|
+
for (const n of names) {
|
|
177
|
+
const fp = path.join(d, n);
|
|
178
|
+
try { if (fs.statSync(fp).mtimeMs < cutoff) fs.unlinkSync(fp); } catch (_) { /* raced */ }
|
|
179
|
+
}
|
|
180
|
+
try { if (!fs.readdirSync(d).length) fs.rmdirSync(d); } catch (_) { /* not empty / raced */ }
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function record(source, ageMs) { log.push({ source, ageSeconds: Math.max(0, Math.round(ageMs / 1000)) }); }
|
|
185
|
+
|
|
186
|
+
function answer(entry, paginate) {
|
|
187
|
+
if (entry.missing) return { status: 'error' };
|
|
188
|
+
if (!paginate) return { status: 'ok', text: entry.pages[0].body };
|
|
189
|
+
const items = [];
|
|
190
|
+
for (const p of entry.pages) {
|
|
191
|
+
let arr;
|
|
192
|
+
try { arr = JSON.parse(p.body); } catch (_) { return { status: 'error' }; }
|
|
193
|
+
if (!Array.isArray(arr)) return { status: 'error' };
|
|
194
|
+
items.push(...arr);
|
|
195
|
+
}
|
|
196
|
+
return { status: 'ok', items };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function get(slug, apiPath, { accept = null, paginate = false } = {}) {
|
|
200
|
+
if (offline) { record('offline', 0); return { status: 'offline' }; }
|
|
201
|
+
const key = accept ? `${apiPath}|${accept}` : apiPath;
|
|
202
|
+
const data = load(slug);
|
|
203
|
+
const entry = data.entries[key];
|
|
204
|
+
const runKey = `${slug}|${key}`;
|
|
205
|
+
|
|
206
|
+
if (entry && (doneThisRun.has(runKey) || (!fresh && now() - entry.at < ttlMs))) {
|
|
207
|
+
record(doneThisRun.has(runKey) ? 'run' : 'cache', now() - entry.at);
|
|
208
|
+
return answer(entry, paginate);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const pages = [];
|
|
212
|
+
let allNotModified = true;
|
|
213
|
+
for (let i = 0; i < (paginate ? MAX_PAGES : 1); i++) {
|
|
214
|
+
const cached = entry && !entry.missing && entry.pages[i];
|
|
215
|
+
const r = fetch(paginate ? withPage(apiPath, i + 1) : apiPath, { etag: cached ? cached.etag : null, accept });
|
|
216
|
+
let page;
|
|
217
|
+
if (r.kind === 'not-modified' && cached) page = cached;
|
|
218
|
+
else if (r.kind === 'ok') {
|
|
219
|
+
allNotModified = false;
|
|
220
|
+
let next = !!r.next;
|
|
221
|
+
if (paginate && !r.linkKnown) { // no headers to follow: a full page means there may be more
|
|
222
|
+
try { const a = JSON.parse(r.body); next = Array.isArray(a) && a.length >= PER_PAGE; } catch (_) { next = false; }
|
|
223
|
+
}
|
|
224
|
+
page = { etag: r.etag || null, next, body: r.body };
|
|
225
|
+
} else {
|
|
226
|
+
if (entry && (r.kind === 'rate-limited' || r.kind === 'unreachable')) {
|
|
227
|
+
record('stale', now() - entry.at);
|
|
228
|
+
return { ...answer(entry, paginate), stale: true };
|
|
229
|
+
}
|
|
230
|
+
if (r.kind === 'error' && r.status === 404 && i === 0) {
|
|
231
|
+
// An absent file is an answer too, and the audit asks for several on every run (a
|
|
232
|
+
// runbook, a tool, a workflow dir). Cached for the TTL like any other; past it, asked
|
|
233
|
+
// again — a 404 carries no ETag, so that one costs a call.
|
|
234
|
+
data.entries[key] = { at: now(), missing: true, pages: [] };
|
|
235
|
+
doneThisRun.add(runKey);
|
|
236
|
+
save(slug);
|
|
237
|
+
record('fetched', 0);
|
|
238
|
+
return { status: 'error' };
|
|
239
|
+
}
|
|
240
|
+
record('error', 0);
|
|
241
|
+
return { status: 'error' };
|
|
242
|
+
}
|
|
243
|
+
pages.push(page);
|
|
244
|
+
if (!paginate || !page.next) break;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
data.entries[key] = { at: now(), pages };
|
|
248
|
+
doneThisRun.add(runKey);
|
|
249
|
+
save(slug);
|
|
250
|
+
record(allNotModified ? 'revalidated' : 'fetched', 0);
|
|
251
|
+
return answer(data.entries[key], paginate);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function begin() { log = []; }
|
|
255
|
+
|
|
256
|
+
/** This repo's remote provenance since begin(); null when it made no GitHub read at all. */
|
|
257
|
+
function summary() {
|
|
258
|
+
if (offline) return { mode: 'offline' };
|
|
259
|
+
if (!log.length) return null;
|
|
260
|
+
const count = (s) => log.filter((e) => e.source === s).length;
|
|
261
|
+
const served = log.filter((e) => e.source !== 'error');
|
|
262
|
+
return {
|
|
263
|
+
mode: 'live',
|
|
264
|
+
reads: log.length,
|
|
265
|
+
cache: count('cache') + count('run'),
|
|
266
|
+
revalidated: count('revalidated'),
|
|
267
|
+
fetched: count('fetched'),
|
|
268
|
+
stale: count('stale'),
|
|
269
|
+
errors: count('error'),
|
|
270
|
+
oldestAgeSeconds: served.length ? Math.max(...served.map((e) => e.ageSeconds)) : null,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
return { get, begin, summary, dir, ttlSeconds: ttlMs / 1000, fresh, offline };
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
module.exports = { create, ghFetch, classify, resolveTtlSeconds, resolveDir, partitionKey, DEFAULT_TTL_SECONDS, PER_PAGE };
|