@futurelastic/colab-handbook 1.14.0-rc.11 → 1.14.0-rc.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/audit/audit.mjs +8 -5
- package/package.json +1 -1
- package/tools/colab +113 -5
- package/tools/lib/docs-only.js +76 -21
- package/tools/lib/secret-scan.js +132 -0
package/audit/audit.mjs
CHANGED
|
@@ -1497,13 +1497,16 @@ function auditRepoInner(target, ctx) {
|
|
|
1497
1497
|
if (!ifl.valid) fail(ifl.reason);
|
|
1498
1498
|
}
|
|
1499
1499
|
|
|
1500
|
-
// ---- ci-docs-skip (#570)
|
|
1501
|
-
//
|
|
1502
|
-
//
|
|
1503
|
-
//
|
|
1500
|
+
// ---- ci-docs-skip (#570, #606) ---------------------------------------------
|
|
1501
|
+
// The docs-only path list (tools/lib/docs-only.js parseCiDocsSkip). ON BY DEFAULT (#606): absent
|
|
1502
|
+
// = on, a list = on with those paths excluded (a change to one is not docs-only), `off` = off.
|
|
1503
|
+
// `colab ship` reads it to decide a docs-only change needs no CI; the CI templates' legacy guard
|
|
1504
|
+
// reads the same list. Reported as the exception list (`[]` when absent), `"off"`, or null when
|
|
1505
|
+
// invalid. An invalid value fails: every reader takes it as "not docs-only", so it silently
|
|
1506
|
+
// opts the repo out.
|
|
1504
1507
|
{
|
|
1505
1508
|
const ds = docsOnlyLib.parseCiDocsSkip(cfg);
|
|
1506
|
-
info.ciDocsSkip = ds.
|
|
1509
|
+
info.ciDocsSkip = !ds.valid ? null : ds.on ? ds.exclude : "off";
|
|
1507
1510
|
if (!ds.valid) fail(ds.reason);
|
|
1508
1511
|
}
|
|
1509
1512
|
|
package/package.json
CHANGED
package/tools/colab
CHANGED
|
@@ -70,6 +70,7 @@ const shipBatch = require('./lib/ship-batch');
|
|
|
70
70
|
const batchStats = require('./lib/batch-stats');
|
|
71
71
|
const conflictMarkers = require('./lib/conflict-markers');
|
|
72
72
|
const docsOnly = require('./lib/docs-only');
|
|
73
|
+
const secretScan = require('./lib/secret-scan'); // #606
|
|
73
74
|
const configSet = require('./lib/config-set'); // #561
|
|
74
75
|
const cureDiff = require('./lib/cure-diff');
|
|
75
76
|
const codeowners = require('./lib/codeowners');
|
|
@@ -11470,6 +11471,13 @@ function readCiVerdict(repoAbs, gh, target, trunk) {
|
|
|
11470
11471
|
via = `${trunk} (by commit — the branch-filtered read was empty)`;
|
|
11471
11472
|
}
|
|
11472
11473
|
}
|
|
11474
|
+
// #606: trunk's tip has no FINISHED run and is a docs-only commit (an adopter's CI `paths-ignore`s
|
|
11475
|
+
// docs, or the run is simply not done) — docs change no code, so trunk's CI is the verdict of the
|
|
11476
|
+
// newest trunk commit that is NOT docs-only. A finished red run at the tip is never overridden.
|
|
11477
|
+
if (target === trunk && (run.status === 'none' || run.status !== 'completed')) {
|
|
11478
|
+
const door = docsOnlyTipDoor(repoAbs, trunk, run.sha, readOpts);
|
|
11479
|
+
if (door) return door;
|
|
11480
|
+
}
|
|
11473
11481
|
const at = `${via}@${shortSha(run.sha)}`;
|
|
11474
11482
|
let verdict = classifyCiRun(repoAbs, run, at);
|
|
11475
11483
|
// #413: still no run anywhere at a trunk sha committed minutes ago — the run may not exist YET
|
|
@@ -11648,6 +11656,75 @@ function baseCiAdvisory(repoAbs, gh, base, branch) {
|
|
|
11648
11656
|
};
|
|
11649
11657
|
}
|
|
11650
11658
|
|
|
11659
|
+
/**
|
|
11660
|
+
* #606: trunk's tip is docs-only and has no finished run — read trunk CI at the newest first-parent
|
|
11661
|
+
* ancestor that is NOT docs-only (tools/lib/docs-only.js commitChanges, narrowed by trunk's
|
|
11662
|
+
* `ci-docs-skip`). Returns a readCiVerdict result, or null when the door does not apply: the repo
|
|
11663
|
+
* declared `ci-docs-skip: off` (or an invalid value), the tip is not local or not docs-only, no code
|
|
11664
|
+
* commit within DOCS_TIP_WALK commits, or that commit's runs are unreadable. A null keeps every
|
|
11665
|
+
* existing reading of the tip unchanged — the door only ever answers for a docs-only tip.
|
|
11666
|
+
*/
|
|
11667
|
+
const DOCS_TIP_WALK = 50;
|
|
11668
|
+
function docsOnlyTipDoor(repoAbs, trunk, tipSha, readOpts) {
|
|
11669
|
+
if (!tipSha) return null;
|
|
11670
|
+
const setting = docsOnly.parseCiDocsSkip(projectDoc(repoAbs));
|
|
11671
|
+
if (!setting.on) return null;
|
|
11672
|
+
const list = git.git(['rev-list', '--first-parent', `--max-count=${DOCS_TIP_WALK}`, tipSha], repoAbs);
|
|
11673
|
+
if (!list.ok) return null;
|
|
11674
|
+
let skipped = 0;
|
|
11675
|
+
let code = null;
|
|
11676
|
+
for (const sha of list.stdout.split('\n').map((x) => x.trim()).filter(Boolean)) {
|
|
11677
|
+
const c = docsOnly.commitChanges(git, repoAbs, sha, { exclude: setting.exclude });
|
|
11678
|
+
if (!c.docsOnly) { code = sha; break; }
|
|
11679
|
+
skipped++;
|
|
11680
|
+
}
|
|
11681
|
+
if (!code || skipped === 0) return null;
|
|
11682
|
+
const run = git.ghRunForCommitAnyRef(repoAbs, code, undefined, readOpts);
|
|
11683
|
+
if (!run) return null;
|
|
11684
|
+
const at = `${trunk}@${shortSha(code)}`;
|
|
11685
|
+
const v = classifyCiRun(repoAbs, run, at);
|
|
11686
|
+
const note = `docs-only tip (#606): ${trunk}@${shortSha(tipSha)}${skipped > 1 ? ` and ${skipped - 1} commit(s) below it` : ''} change only documentation, so trunk CI is read at the newest code commit — `;
|
|
11687
|
+
const verdict = { ...v, detail: note + v.detail, docsOnlyTip: { tip: tipSha, read: code, skipped } };
|
|
11688
|
+
return { run, verdict, final: verdict.ok || verdict.class !== HUMAN_GATED };
|
|
11689
|
+
}
|
|
11690
|
+
|
|
11691
|
+
/**
|
|
11692
|
+
* #606: is this branch DOCS-ONLY for CI purposes — so `colab ship` lands it without waiting for CI?
|
|
11693
|
+
* tools/lib/docs-only.js shipDocsOnly (the #345 rule, narrowed by `ci-docs-skip`, off by `off`),
|
|
11694
|
+
* measured on the ref ship would land. When it applies, the secret check runs too
|
|
11695
|
+
* (tools/lib/secret-scan.js) — the suite's own secret scan may never see this change.
|
|
11696
|
+
* Returns `{ applies, files, reason, scan }`; `scan` only when `applies`.
|
|
11697
|
+
*/
|
|
11698
|
+
function shipDocsOnlyCi(repoAbs, doc, target, branch) {
|
|
11699
|
+
const ref = shipRefFor(repoAbs, branch);
|
|
11700
|
+
const d = docsOnly.shipDocsOnly(git, repoAbs, target, ref, doc);
|
|
11701
|
+
if (!d.docsOnly) return { applies: false, files: d.files, reason: d.reason, scan: null };
|
|
11702
|
+
return { applies: true, files: d.files, reason: d.reason, scan: secretScan.scanBranch(git, repoAbs, target, ref) };
|
|
11703
|
+
}
|
|
11704
|
+
|
|
11705
|
+
/** #606: the 🚢 comment's evidence clause for a docs-only ship — files, and the trunk sha CI was read at. */
|
|
11706
|
+
function docsOnlyEvidence(dc, ci) {
|
|
11707
|
+
if (!dc || !dc.applies) return '';
|
|
11708
|
+
const tip = ci && ci.docsOnlyTip;
|
|
11709
|
+
const read = tip ? ` — trunk CI read at ${shortSha(tip.read)}, the newest code commit below the docs-only tip ${shortSha(tip.tip)}` : ' — trunk CI read at its tip';
|
|
11710
|
+
return ` — docs-only: CI not required (${dc.files} files)${read}; secret check clean`;
|
|
11711
|
+
}
|
|
11712
|
+
|
|
11713
|
+
/**
|
|
11714
|
+
* #606: the branch-run row for a docs-only branch, in place of #395's stale-base row. CI is not
|
|
11715
|
+
* required, so no run, a run in flight, or a run measured on an older base is fine — but a run that
|
|
11716
|
+
* EXISTS at the pushed head and finished red still refuses: ship never overrides a red verdict.
|
|
11717
|
+
*/
|
|
11718
|
+
function docsOnlyBranchRow(dc, stale) {
|
|
11719
|
+
const run = stale && stale.run;
|
|
11720
|
+
const red = !!run && run.status === 'completed' && run.conclusion !== 'success' && run.conclusion !== 'cancelled' && run.conclusion !== 'skipped';
|
|
11721
|
+
if (red) {
|
|
11722
|
+
return { ok: false, class: HUMAN_GATED,
|
|
11723
|
+
detail: `docs-only (${dc.files} files), but the branch's own run at ${shortSha(stale.headSha)} finished ${run.conclusion || 'red'} — CI is not required here, and a red verdict is still never overridden (#606)` };
|
|
11724
|
+
}
|
|
11725
|
+
return { ok: true, class: null, detail: `docs-only: CI not required (${dc.files} files) — the branch's own run is not awaited (#606; ${stale ? stale.verdict : 'unread'})` };
|
|
11726
|
+
}
|
|
11727
|
+
|
|
11651
11728
|
/**
|
|
11652
11729
|
* #395: does the branch's CI verdict cover the CURRENT `base`? A PRECONDITION, unlike
|
|
11653
11730
|
* `baseCiAdvisory` above: serial ship syncs the base in locally (B0) and squashes without the synced
|
|
@@ -11676,7 +11753,8 @@ function shipStaleBaseCheck(repoAbs, gh, base, branch) {
|
|
|
11676
11753
|
if (contains === false && git.git(['diff', '--quiet', `${baseTip}...${headSha}`], repoAbs).code === 0) contains = 'no-change';
|
|
11677
11754
|
}
|
|
11678
11755
|
if (headSha && gh) run = git.ghRunForCommit(repoAbs, branch, headSha);
|
|
11679
|
-
|
|
11756
|
+
// #606: `run`/`headSha` are additive — the docs-only row reads whether that run finished red.
|
|
11757
|
+
return { ...staleBase.staleBaseVerdict({ headSha, run, baseTip, contains, behind, base, ghUsable: !!gh }), run, headSha };
|
|
11680
11758
|
}
|
|
11681
11759
|
|
|
11682
11760
|
/**
|
|
@@ -12481,7 +12559,14 @@ function cmdShipDryJson(opts) {
|
|
|
12481
12559
|
// read is exempt: the batch grades the COMBINED head (trunk + every member), which is exactly the
|
|
12482
12560
|
// re-run this row demands of a serial land, so a member behind trunk is the ordinary case there.
|
|
12483
12561
|
const staleJson = shipStaleBaseCheck(repoAbs, gh, target, sess.branch);
|
|
12484
|
-
|
|
12562
|
+
// #606: a docs-only branch needs no CI of its own — its row replaces the stale-base one, and the
|
|
12563
|
+
// secret check stands in for the suite's secret scan. Same producers as the prose path.
|
|
12564
|
+
const docsCiJson = shipDocsOnlyCi(repoAbs, doc, target, sess.branch);
|
|
12565
|
+
if (docsCiJson.applies) {
|
|
12566
|
+
const row = docsOnlyBranchRow(docsCiJson, staleJson);
|
|
12567
|
+
checks.push({ name: 'branch CI (docs-only, #606)', ok: row.ok, class: row.class, verdict: staleJson.verdict, detail: row.detail });
|
|
12568
|
+
checks.push({ name: 'docs-only secret check (#606)', ok: docsCiJson.scan.ok, class: docsCiJson.scan.ok ? null : HUMAN_GATED, detail: docsCiJson.scan.reason });
|
|
12569
|
+
} else if (opts._batchMember) {
|
|
12485
12570
|
checks.push({ name: 'branch run contains current base (#395)', ok: true, class: null, verdict: staleJson.verdict,
|
|
12486
12571
|
detail: `batch member — the combined run grades it (${staleJson.detail})` });
|
|
12487
12572
|
} else {
|
|
@@ -13662,7 +13747,18 @@ function cmdShip(argv) {
|
|
|
13662
13747
|
// #395: the branch's run must have been measured on a head that contains the base's CURRENT tip —
|
|
13663
13748
|
// B0 below syncs locally and never re-runs CI, so this is the only place a stale verdict is caught.
|
|
13664
13749
|
const staleResult = shipStaleBaseCheck(repoAbs, gh, target, sess.branch);
|
|
13665
|
-
|
|
13750
|
+
// #606: a docs-only branch needs no CI of its own — this row replaces the stale-base one (a red
|
|
13751
|
+
// run that exists still refuses), and the secret check stands in for the suite's secret scan.
|
|
13752
|
+
const docsCi = shipDocsOnlyCi(repoAbs, doc, target, sess.branch);
|
|
13753
|
+
let docsRowOk = true;
|
|
13754
|
+
if (docsCi.applies) {
|
|
13755
|
+
const row = docsOnlyBranchRow(docsCi, staleResult);
|
|
13756
|
+
checks.push([row.ok ? '✓' : '✗', 'branch CI (docs-only, #606)', row.detail]);
|
|
13757
|
+
checks.push([docsCi.scan.ok ? '✓' : '✗', 'docs-only secret check (#606)', docsCi.scan.reason]);
|
|
13758
|
+
docsRowOk = row.ok && docsCi.scan.ok;
|
|
13759
|
+
} else {
|
|
13760
|
+
checks.push([staleResult.ok ? '✓' : '✗', 'branch run contains current base (#395)', staleResult.detail]);
|
|
13761
|
+
}
|
|
13666
13762
|
|
|
13667
13763
|
// #350: the core-path review. A 'pending' verdict is not a failed precondition — it is the pause,
|
|
13668
13764
|
// taken below once every real precondition has passed, so it is the ONLY thing between the branch
|
|
@@ -13688,7 +13784,7 @@ function cmdShip(argv) {
|
|
|
13688
13784
|
// issue can be reopened, but the false claim stays in trunk's immutable history, and GitHub will
|
|
13689
13785
|
// have closed it the instant trunk was pushed. #263: gateOk joins the same list — an unticked
|
|
13690
13786
|
// checklist with no declared remainder is exactly as unrecoverable once `Closes #N` ships.
|
|
13691
|
-
const preOk = ciOk && migOk && treeOk && corr.ok && gateOk && syncResult.ok && staleResult.ok && coreOk && recOk;
|
|
13787
|
+
const preOk = ciOk && migOk && treeOk && corr.ok && gateOk && syncResult.ok && (docsCi.applies ? docsRowOk : staleResult.ok) && coreOk && recOk;
|
|
13692
13788
|
|
|
13693
13789
|
// ---- report the table ----
|
|
13694
13790
|
print(`ship ${shortRepo(repoAbs)} · branch ${sess.branch} → ${target}` +
|
|
@@ -14053,6 +14149,18 @@ function cmdShip(argv) {
|
|
|
14053
14149
|
return 1;
|
|
14054
14150
|
}
|
|
14055
14151
|
}
|
|
14152
|
+
// #606: the same for the docs-only CI path — B0 may have made the change something CI must see,
|
|
14153
|
+
// or added a line the secret check has not read. Re-measured on the synced branch.
|
|
14154
|
+
if (docsCi.applies) {
|
|
14155
|
+
const again = docsOnly.shipDocsOnly(git, repoAbs, mergeRef, sess.branch, doc);
|
|
14156
|
+
const rescan = again.docsOnly ? secretScan.scanBranch(git, repoAbs, mergeRef, sess.branch) : null;
|
|
14157
|
+
if (!again.docsOnly || !rescan.ok) {
|
|
14158
|
+
cleanupEphemeral();
|
|
14159
|
+
process.stderr.write(`✗ B1: the docs-only CI path (#606) no longer holds after B0 sync — ${again.docsOnly ? rescan.reason : again.reason}. Nothing pushed.\n`);
|
|
14160
|
+
process.stderr.write(` Push the synced branch and re-run once its CI run completes (\`git push origin ${sess.branch}\`, then \`colab ship\` again) — or remove the finding first if the secret check refused.\n`);
|
|
14161
|
+
return 1;
|
|
14162
|
+
}
|
|
14163
|
+
}
|
|
14056
14164
|
if (autonomyGate.via === 'tuning') {
|
|
14057
14165
|
const again = configSet.branchTuning(git, repoAbs, mergeRef, sess.branch);
|
|
14058
14166
|
if (!again.tuning) {
|
|
@@ -14489,7 +14597,7 @@ function cmdShip(argv) {
|
|
|
14489
14597
|
// same block, and re-assigning the collector here would silently discard them.
|
|
14490
14598
|
shipLandedTail({ repoAbs, sess, target, trunk, gh, sha, closeIssues, refsIssues, refsSet, integration, opts,
|
|
14491
14599
|
skipPrs: reviewPrHandled ? [reviewPrHandled] : [],
|
|
14492
|
-
evidenceExtra: (opts.humanDoorEvidence || '') + handoffEvidence(sess.handoff) }); // #525, #578
|
|
14600
|
+
evidenceExtra: (opts.humanDoorEvidence || '') + docsOnlyEvidence(docsCi, ciResult) + handoffEvidence(sess.handoff) }); // #525, #578, #606
|
|
14493
14601
|
|
|
14494
14602
|
print('');
|
|
14495
14603
|
print(`✓ Shipped ${sess.branch} → ${target} @ ${sha}${sess.issues.length ? ' (' + sess.issues.map((n) => '#' + n).join(' ') + ')' : ''}`);
|
package/tools/lib/docs-only.js
CHANGED
|
@@ -8,15 +8,33 @@
|
|
|
8
8
|
* caller. Nothing here takes a list, a flag, an env var or a project.yml field. The allowlist and the
|
|
9
9
|
* exclusions below are constants, so widening them is a handbook change reviewed in a commit.
|
|
10
10
|
*
|
|
11
|
-
* Every doubt resolves to "not docs-only": an empty diff, a git read that failed, a binary
|
|
12
|
-
*
|
|
13
|
-
* approved.
|
|
11
|
+
* Every doubt resolves to "not docs-only": an empty diff, a git read that failed, a binary (other
|
|
12
|
+
* than a design artifact under `docs/`, below), a symlink, a submodule pointer. A false "no" costs a
|
|
13
|
+
* human click. A false "yes" merges code nobody approved.
|
|
14
|
+
*
|
|
15
|
+
* #606: the same rule also decides when `colab ship` lands a change WITHOUT WAITING FOR CI
|
|
16
|
+
* (shipDocsOnly, below — on by default, narrowed by `ci-docs-skip`). So widening anything here
|
|
17
|
+
* widens BOTH at once — the binary design-artifact rule below did exactly that: a PNG mockup under
|
|
18
|
+
* `docs/design/` is now docs-only for the autonomy exception too (CONVENTIONS.md §2).
|
|
14
19
|
*/
|
|
15
20
|
|
|
16
21
|
/** Extensions that count as documentation. Compared exactly (`README.MD` does not match). */
|
|
17
22
|
const DOC_EXTENSIONS = Object.freeze(['.md', '.mdx', '.txt']);
|
|
18
23
|
/** A path under this top-level directory counts as documentation whatever its extension. */
|
|
19
24
|
const DOCS_DIR = 'docs/';
|
|
25
|
+
/**
|
|
26
|
+
* #606: a BINARY under `docs/` with one of these extensions is a design artifact (a mockup, a
|
|
27
|
+
* diagram, a printed spec) and counts as documentation. Case-insensitive (`Mock.PNG` matches). Any
|
|
28
|
+
* other binary under `docs/`, and every binary outside it, is still not docs-only.
|
|
29
|
+
*/
|
|
30
|
+
const DOCS_BINARY_EXTENSIONS = Object.freeze(['.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.pdf']);
|
|
31
|
+
function docsBinaryOk(p) {
|
|
32
|
+
const s = String(p);
|
|
33
|
+
if (!s.startsWith(DOCS_DIR)) return false;
|
|
34
|
+
const base = s.slice(s.lastIndexOf('/') + 1);
|
|
35
|
+
const dot = base.lastIndexOf('.');
|
|
36
|
+
return dot > 0 && DOCS_BINARY_EXTENSIONS.includes(base.slice(dot).toLowerCase());
|
|
37
|
+
}
|
|
20
38
|
/** Rules and agent instructions: never documentation, at any depth, even though they are `.md`. */
|
|
21
39
|
const NEVER_BASENAMES = Object.freeze(['CLAUDE.md', 'CLAUDE.local.md', 'AGENTS.md']);
|
|
22
40
|
/**
|
|
@@ -77,15 +95,18 @@ function pathReason(p) {
|
|
|
77
95
|
* Returns `{ docsOnly, files, offenders: [{ path, reason }], reason }`. `files` counts distinct paths.
|
|
78
96
|
* `reason` is one line for a human, set on both outcomes.
|
|
79
97
|
*/
|
|
80
|
-
function classify(entries) {
|
|
98
|
+
function classify(entries, { exclude = [] } = {}) {
|
|
81
99
|
const paths = new Map();
|
|
82
100
|
for (const e of entries || []) {
|
|
83
101
|
const why = [];
|
|
84
102
|
const nameWhy = pathReason(e.path);
|
|
85
103
|
if (nameWhy) why.push(nameWhy);
|
|
104
|
+
// #606: a path the repo's suite reads, listed in `ci-docs-skip` — documentation, but it needs CI.
|
|
105
|
+
const listed = excludedBy(e.path, exclude);
|
|
106
|
+
if (!nameWhy && listed) why.push(`listed in ci-docs-skip (${listed}) — the suite reads it`);
|
|
86
107
|
if (e.oldMode === MODE_SYMLINK || e.newMode === MODE_SYMLINK) why.push('symlink change');
|
|
87
108
|
if (e.oldMode === MODE_GITLINK || e.newMode === MODE_GITLINK) why.push('submodule change');
|
|
88
|
-
if (e.binary) why.push('binary change');
|
|
109
|
+
if (e.binary && !docsBinaryOk(e.path)) why.push('binary change');
|
|
89
110
|
const prev = paths.get(e.path) || [];
|
|
90
111
|
paths.set(e.path, [...new Set([...prev, ...why])]);
|
|
91
112
|
}
|
|
@@ -101,6 +122,12 @@ function classify(entries) {
|
|
|
101
122
|
return { docsOnly: true, files, offenders: [], reason: `docs-only (${files} files)` };
|
|
102
123
|
}
|
|
103
124
|
|
|
125
|
+
/** The `ci-docs-skip` member `p` is, or sits under — a path prefix, never a string prefix. */
|
|
126
|
+
function excludedBy(p, exclude) {
|
|
127
|
+
const s = String(p);
|
|
128
|
+
return (exclude || []).find((x) => s === x || s.startsWith(`${x}/`)) || null;
|
|
129
|
+
}
|
|
130
|
+
|
|
104
131
|
/**
|
|
105
132
|
* Parse `git diff --raw -z --no-renames --no-abbrev` output into
|
|
106
133
|
* `[{ path, oldMode, newMode, oldSha, newSha, status }]`. The sha/status fields are additive
|
|
@@ -151,10 +178,30 @@ function readEntries(git, repo, cmd, range) {
|
|
|
151
178
|
* The branch shape: what `branch` changes relative to its merge base with `base` — the same three-dot
|
|
152
179
|
* diff the squash would land.
|
|
153
180
|
*/
|
|
154
|
-
function branchChanges(git, repo, base, branch) {
|
|
181
|
+
function branchChanges(git, repo, base, branch, opts = {}) {
|
|
155
182
|
const entries = readEntries(git, repo, 'diff', [`${base}...${branch}`, '--']);
|
|
156
183
|
if (!entries) return { docsOnly: false, files: 0, offenders: [], reason: `could not read the diff ${base}...${branch}` };
|
|
157
|
-
return classify(entries);
|
|
184
|
+
return classify(entries, opts);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** One commit's own change (against its parent(s), `diff-tree -m`) — the shape a trunk squash has. */
|
|
188
|
+
function commitChanges(git, repo, sha, opts = {}) {
|
|
189
|
+
const entries = readEntries(git, repo, 'diff-tree', [sha]);
|
|
190
|
+
if (!entries) return { docsOnly: false, files: 0, offenders: [], reason: `could not read commit ${String(sha).slice(0, 7)}` };
|
|
191
|
+
return classify(entries, opts);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* #606: does `colab ship` treat this branch as needing no CI? The SAME classification as the
|
|
196
|
+
* autonomy exception (#345), narrowed by the repo's `ci-docs-skip` list — a path the suite reads is
|
|
197
|
+
* documentation but still needs CI — and switched off entirely by `ci-docs-skip: off`. An invalid
|
|
198
|
+
* value reads as off (every reader takes doubt as "CI required"); the audit fails it.
|
|
199
|
+
* Returns the classify() shape plus `{ setting }` (parseCiDocsSkip's verdict).
|
|
200
|
+
*/
|
|
201
|
+
function shipDocsOnly(git, repo, base, branch, doc) {
|
|
202
|
+
const setting = parseCiDocsSkip(doc);
|
|
203
|
+
if (!setting.on) return { docsOnly: false, files: 0, offenders: [], reason: setting.reason, setting };
|
|
204
|
+
return { ...branchChanges(git, repo, base, branch, { exclude: setting.exclude }), setting };
|
|
158
205
|
}
|
|
159
206
|
|
|
160
207
|
/**
|
|
@@ -179,24 +226,32 @@ function directChanges(git, repo, trunk, since) {
|
|
|
179
226
|
}
|
|
180
227
|
|
|
181
228
|
/**
|
|
182
|
-
* #570: `ci-docs-skip:` in project.yml —
|
|
183
|
-
*
|
|
184
|
-
*
|
|
229
|
+
* #570/#606: `ci-docs-skip:` in project.yml — which changes `colab ship` may land without waiting for
|
|
230
|
+
* CI (#606), and the paths the CI templates' legacy guard (#570) excludes. ON BY DEFAULT for ship:
|
|
231
|
+
* absent = on, nothing excluded. A list = on, and each member is a repo path whose changes always run
|
|
232
|
+
* the suite even though this module calls them documentation (`docs/api/` feeding a generator, a
|
|
233
|
+
* README a link checker reads). `[]` = on, nothing excluded. `off` = the one opt-out, for a repo whose
|
|
234
|
+
* suite reads most of its docs (YAML 1.1 readers turn a bare `off` into boolean false; both spell it).
|
|
185
235
|
*
|
|
186
|
-
* The CI guard parses the same
|
|
187
|
-
*
|
|
188
|
-
*
|
|
189
|
-
*
|
|
236
|
+
* The legacy CI guard parses the same list in shell, but stays opt-in until its follow-up removes it:
|
|
237
|
+
* there `off` and absent both mean "run the full suite". Both readers take anything else — an empty
|
|
238
|
+
* `ci-docs-skip:` included — as "CI required", the stricter direction. So an invalid value here is a finding:
|
|
239
|
+
* a declaration that silently opts the repo out. Members are plain paths (the guard matches `p == x`
|
|
240
|
+
* or `p` under `x/`): no glob, no `..` or `.` segment, not absolute, only `[A-Za-z0-9._/-]`.
|
|
190
241
|
*/
|
|
191
242
|
const CI_DOCS_SKIP_KEY = 'ci-docs-skip';
|
|
192
243
|
function parseCiDocsSkip(doc) {
|
|
193
244
|
const has = !!doc && Object.prototype.hasOwnProperty.call(doc, CI_DOCS_SKIP_KEY);
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
245
|
+
if (!has) {
|
|
246
|
+
return { declared: false, valid: true, on: true, exclude: [], reason: 'ci-docs-skip absent — on by default (#606): colab ship lands a docs-only change without waiting for CI' };
|
|
247
|
+
}
|
|
248
|
+
const v = doc[CI_DOCS_SKIP_KEY];
|
|
249
|
+
if (v === 'off' || v === false) {
|
|
250
|
+
return { declared: true, valid: true, on: false, exclude: null, reason: 'ci-docs-skip: off — colab ship requires CI for every change' };
|
|
197
251
|
}
|
|
198
252
|
const bad = (why) => ({ declared: true, valid: false, on: false, exclude: null,
|
|
199
|
-
reason: `ci-docs-skip ${why} — it must be a list of repo paths (\`[]\` for none)
|
|
253
|
+
reason: `ci-docs-skip ${why} — it must be a list of repo paths (\`[]\` for none) or \`off\`; every reader takes anything else as "CI required", so the declaration silently opts this repo out of the docs-only path` });
|
|
254
|
+
if (v === null || v === undefined) return bad('is empty');
|
|
200
255
|
if (!Array.isArray(v)) return bad(`is ${JSON.stringify(v)}`);
|
|
201
256
|
const exclude = [];
|
|
202
257
|
for (const m of v) {
|
|
@@ -207,11 +262,11 @@ function parseCiDocsSkip(doc) {
|
|
|
207
262
|
exclude.push(s);
|
|
208
263
|
}
|
|
209
264
|
return { declared: true, valid: true, on: true, exclude,
|
|
210
|
-
reason: exclude.length ? `ci-docs-skip on, except ${exclude.join(', ')}` : 'ci-docs-skip on — a docs-only change
|
|
265
|
+
reason: exclude.length ? `ci-docs-skip on, except ${exclude.join(', ')}` : 'ci-docs-skip on — colab ship lands a docs-only change without waiting for CI' };
|
|
211
266
|
}
|
|
212
267
|
|
|
213
268
|
module.exports = {
|
|
214
|
-
DOC_EXTENSIONS, DOCS_DIR, NEVER_BASENAMES, NEVER_DIRS, NEVER_SUBDIRS,
|
|
269
|
+
DOC_EXTENSIONS, DOCS_DIR, DOCS_BINARY_EXTENSIONS, docsBinaryOk, NEVER_BASENAMES, NEVER_DIRS, NEVER_SUBDIRS,
|
|
215
270
|
pathReason, classify, parseRaw, parseBinary, readEntries, branchChanges, directChanges,
|
|
216
|
-
CI_DOCS_SKIP_KEY, parseCiDocsSkip,
|
|
271
|
+
CI_DOCS_SKIP_KEY, parseCiDocsSkip, excludedBy, commitChanges, shipDocsOnly,
|
|
217
272
|
};
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* #606 — the secret check `colab ship` runs on a DOCS-ONLY change before it merges.
|
|
4
|
+
*
|
|
5
|
+
* A docs-only change lands without waiting for CI (tools/lib/docs-only.js shipDocsOnly), so the CI
|
|
6
|
+
* suite's own secret scan (gitleaks in the templates) may never see it. This is the cheap stand-in:
|
|
7
|
+
* the lines the change ADDS — `git diff -U0 <base>...<branch>`, text files only (git prints no lines
|
|
8
|
+
* for a binary) — matched against generic credential shapes, plus the repo's own forbidden terms
|
|
9
|
+
* when it keeps a file at TERMS_PATH. Seconds, no dependency, no network.
|
|
10
|
+
*
|
|
11
|
+
* A finding is reported as `file:line (rule)`, never with the matched text: the refusal is printed
|
|
12
|
+
* to a terminal and may be pasted onto an Issue, and a public repo's Issues are public.
|
|
13
|
+
*
|
|
14
|
+
* It is a backstop, not gitleaks: generic shapes only, added lines only. A shape it misses is
|
|
15
|
+
* missed; nothing here claims the change is clean beyond these rules.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** The repo-local forbidden-terms file (project.schema.md "ci-docs-skip"), read at the TARGET's tip. */
|
|
19
|
+
const TERMS_PATH = '.github/secret-scan-terms';
|
|
20
|
+
|
|
21
|
+
/** Generic credential shapes. `test` gets one added line. */
|
|
22
|
+
const RULES = Object.freeze([
|
|
23
|
+
{ rule: 'private key', re: /-----BEGIN ([A-Z0-9]+ )*PRIVATE KEY( BLOCK)?-----/ },
|
|
24
|
+
{ rule: 'AWS access key', re: /(^|[^A-Z0-9])(AKIA|ASIA)[A-Z0-9]{16}([^A-Z0-9]|$)/ },
|
|
25
|
+
{ rule: 'GitHub token', re: /(gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{60,})/ },
|
|
26
|
+
{ rule: 'Slack token', re: /xox[abposr]-[A-Za-z0-9-]{10,}/ },
|
|
27
|
+
{ rule: 'Google API key', re: /AIza[0-9A-Za-z_-]{35}/ },
|
|
28
|
+
{ rule: 'Stripe live key', re: /(sk|rk)_live_[0-9A-Za-z]{20,}/ },
|
|
29
|
+
{ rule: 'LLM API key', re: /sk-(ant|proj)-[A-Za-z0-9_-]{20,}/ },
|
|
30
|
+
]);
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* `password=…` and its kin. The value must hold a lower-case letter AND a digit, so a documented
|
|
34
|
+
* placeholder (`YOUR_API_KEY`, `changeme`, `$PASSWORD`, `<secret>`, `${{ secrets.X }}`) is not a
|
|
35
|
+
* finding while `password=hunter22` is.
|
|
36
|
+
*/
|
|
37
|
+
const ASSIGN = /(password|passwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token)["']?\s*[:=]\s*["']?([^"'\s<>{}$]{8,})/i;
|
|
38
|
+
function assignment(line) {
|
|
39
|
+
const m = ASSIGN.exec(line);
|
|
40
|
+
return !!m && /[0-9]/.test(m[2]) && /[a-z]/.test(m[2]);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Parse a forbidden-terms file — the format of templates/identity-vocabulary.example: one term per
|
|
45
|
+
* line, trimmed; blank and `#` lines are comments; a plain term matches case-insensitively as a
|
|
46
|
+
* substring; `re:<ERE>` is a regular expression matched against the lower-cased line; a term
|
|
47
|
+
* shorter than 3 characters is refused as a mistake. Returns `{ terms: [{line, test}], problems }`.
|
|
48
|
+
*/
|
|
49
|
+
function parseTerms(text) {
|
|
50
|
+
const terms = [];
|
|
51
|
+
const problems = [];
|
|
52
|
+
String(text || '').split(/\r?\n/).forEach((raw, i) => {
|
|
53
|
+
const t = raw.trim();
|
|
54
|
+
if (!t || t.startsWith('#')) return;
|
|
55
|
+
const line = i + 1;
|
|
56
|
+
if (t.startsWith('re:')) {
|
|
57
|
+
try {
|
|
58
|
+
const re = new RegExp(t.slice(3));
|
|
59
|
+
terms.push({ line, test: (s) => re.test(s.toLowerCase()) });
|
|
60
|
+
} catch (_) {
|
|
61
|
+
problems.push(`${TERMS_PATH} line ${line} is not a valid regular expression`);
|
|
62
|
+
}
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
if (t.length < 3) { problems.push(`${TERMS_PATH} line ${line} is shorter than 3 characters`); return; }
|
|
66
|
+
const want = t.toLowerCase();
|
|
67
|
+
terms.push({ line, test: (s) => s.toLowerCase().includes(want) });
|
|
68
|
+
});
|
|
69
|
+
return { terms, problems };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* The added lines of a `git diff -U0 --no-renames --no-color` patch: `[{ file, line, text }]`.
|
|
74
|
+
* `line` is the line number in the new file.
|
|
75
|
+
*/
|
|
76
|
+
function addedLines(patch) {
|
|
77
|
+
const out = [];
|
|
78
|
+
let file = null;
|
|
79
|
+
let ln = 0;
|
|
80
|
+
let inHunk = false;
|
|
81
|
+
for (const l of String(patch || '').split('\n')) {
|
|
82
|
+
// A file header only outside a hunk: an added line `++ x` prints as `+++ x` inside one.
|
|
83
|
+
if (l.startsWith('diff --git ')) { inHunk = false; file = null; continue; }
|
|
84
|
+
if (!inHunk && l.startsWith('+++ ')) { const f = l.slice(4); file = f === '/dev/null' ? null : f.replace(/^b\//, ''); continue; }
|
|
85
|
+
const h = /^@@ -\S+ \+(\d+)(?:,\d+)? @@/.exec(l);
|
|
86
|
+
if (h) { ln = Number(h[1]); inHunk = true; continue; }
|
|
87
|
+
if (inHunk && l.startsWith('+') && file) { out.push({ file, line: ln, text: l.slice(1) }); ln++; }
|
|
88
|
+
}
|
|
89
|
+
return out;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Pure: findings `[{ file, line, rule }]` over added lines, against RULES + assignment + `terms`. */
|
|
93
|
+
function scanLines(lines, terms = []) {
|
|
94
|
+
const findings = [];
|
|
95
|
+
for (const { file, line, text } of lines) {
|
|
96
|
+
for (const r of RULES) if (r.re.test(text)) findings.push({ file, line, rule: r.rule });
|
|
97
|
+
if (assignment(text)) findings.push({ file, line, rule: 'credential assignment' });
|
|
98
|
+
for (const t of terms) if (t.test(text)) findings.push({ file, line, rule: `forbidden term, line ${t.line} of ${TERMS_PATH}` });
|
|
99
|
+
}
|
|
100
|
+
return findings;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Scan what `branch` adds relative to its merge base with `base`, with the forbidden terms read from
|
|
105
|
+
* `base`'s tree (the branch cannot edit them and stay docs-only: `.github/` is never documentation).
|
|
106
|
+
* Returns `{ ok, findings, files, reason }`; `ok: false` with no findings means the scan could not
|
|
107
|
+
* run — a caller refuses on that too, it never reads it as clean.
|
|
108
|
+
*/
|
|
109
|
+
function scanBranch(git, repo, base, branch) {
|
|
110
|
+
const d = git.git(['diff', '-U0', '--no-renames', '--no-color', '--no-ext-diff', `${base}...${branch}`, '--'], repo);
|
|
111
|
+
if (!d.ok) return { ok: false, findings: [], reason: `could not read the diff ${base}...${branch}` };
|
|
112
|
+
let terms = [];
|
|
113
|
+
if (git.git(['cat-file', '-e', `${base}:${TERMS_PATH}`], repo).ok) {
|
|
114
|
+
const t = git.git(['show', `${base}:${TERMS_PATH}`], repo);
|
|
115
|
+
if (!t.ok) return { ok: false, findings: [], reason: `could not read ${TERMS_PATH} at ${base}` };
|
|
116
|
+
const parsed = parseTerms(t.stdout);
|
|
117
|
+
if (parsed.problems.length) return { ok: false, findings: [], reason: parsed.problems.join('; ') };
|
|
118
|
+
terms = parsed.terms;
|
|
119
|
+
}
|
|
120
|
+
const lines = addedLines(d.stdout);
|
|
121
|
+
const findings = scanLines(lines, terms);
|
|
122
|
+
const files = new Set(lines.map((l) => l.file)).size;
|
|
123
|
+
const extra = terms.length ? ` and ${terms.length} forbidden term(s)` : '';
|
|
124
|
+
if (findings.length) {
|
|
125
|
+
const shown = findings.slice(0, 5).map((f) => `${f.file}:${f.line} (${f.rule})`).join('; ');
|
|
126
|
+
const more = findings.length > 5 ? `; +${findings.length - 5} more` : '';
|
|
127
|
+
return { ok: false, findings, files, reason: `${findings.length} finding(s): ${shown}${more}` };
|
|
128
|
+
}
|
|
129
|
+
return { ok: true, findings, files, reason: `${lines.length} added line(s) in ${files} file(s) clean against the credential shapes${extra}` };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
module.exports = { TERMS_PATH, RULES, parseTerms, addedLines, scanLines, scanBranch, assignment };
|