@futurelastic/colab-handbook 1.14.0-rc.8 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/README.vi.md +19 -0
- package/audit/audit.mjs +77 -16
- package/package.json +1 -1
- package/templates/README.md +64 -2
- package/templates/ci-laravel-heavy.yml +370 -0
- package/templates/ci-laravel.yml +27 -218
- package/templates/ci-node-heavy.yml +293 -0
- package/templates/ci-node.yml +23 -202
- package/templates/ci-python-heavy.yml +302 -0
- package/templates/ci-python.yml +26 -207
- package/templates/release-auto.yml +70 -1
- package/templates/release-tag.yml +20 -3
- package/tools/colab +475 -38
- package/tools/lib/adopt.js +1 -1
- package/tools/lib/disposition.js +27 -10
- package/tools/lib/docs-only.js +76 -21
- package/tools/lib/git.js +6 -1
- package/tools/lib/instruction-file.js +26 -0
- package/tools/lib/instruction-gate.js +31 -2
- package/tools/lib/integration-profile.js +161 -0
- package/tools/lib/release-cut.js +10 -0
- package/tools/lib/release-finalize.js +9 -2
- package/tools/lib/secret-scan.js +132 -0
- package/tools/lib/ship-batch.js +71 -2
- package/tools/lib/stamp.js +16 -1
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* #606 — the secret check `colab ship` runs on a DOCS-ONLY change before it merges.
|
|
4
|
+
*
|
|
5
|
+
* A docs-only change lands without waiting for CI (tools/lib/docs-only.js shipDocsOnly), so the CI
|
|
6
|
+
* suite's own secret scan (gitleaks in the templates) may never see it. This is the cheap stand-in:
|
|
7
|
+
* the lines the change ADDS — `git diff -U0 <base>...<branch>`, text files only (git prints no lines
|
|
8
|
+
* for a binary) — matched against generic credential shapes, plus the repo's own forbidden terms
|
|
9
|
+
* when it keeps a file at TERMS_PATH. Seconds, no dependency, no network.
|
|
10
|
+
*
|
|
11
|
+
* A finding is reported as `file:line (rule)`, never with the matched text: the refusal is printed
|
|
12
|
+
* to a terminal and may be pasted onto an Issue, and a public repo's Issues are public.
|
|
13
|
+
*
|
|
14
|
+
* It is a backstop, not gitleaks: generic shapes only, added lines only. A shape it misses is
|
|
15
|
+
* missed; nothing here claims the change is clean beyond these rules.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** The repo-local forbidden-terms file (project.schema.md "ci-docs-skip"), read at the TARGET's tip. */
|
|
19
|
+
const TERMS_PATH = '.github/secret-scan-terms';
|
|
20
|
+
|
|
21
|
+
/** Generic credential shapes. `test` gets one added line. */
|
|
22
|
+
const RULES = Object.freeze([
|
|
23
|
+
{ rule: 'private key', re: /-----BEGIN ([A-Z0-9]+ )*PRIVATE KEY( BLOCK)?-----/ },
|
|
24
|
+
{ rule: 'AWS access key', re: /(^|[^A-Z0-9])(AKIA|ASIA)[A-Z0-9]{16}([^A-Z0-9]|$)/ },
|
|
25
|
+
{ rule: 'GitHub token', re: /(gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{60,})/ },
|
|
26
|
+
{ rule: 'Slack token', re: /xox[abposr]-[A-Za-z0-9-]{10,}/ },
|
|
27
|
+
{ rule: 'Google API key', re: /AIza[0-9A-Za-z_-]{35}/ },
|
|
28
|
+
{ rule: 'Stripe live key', re: /(sk|rk)_live_[0-9A-Za-z]{20,}/ },
|
|
29
|
+
{ rule: 'LLM API key', re: /sk-(ant|proj)-[A-Za-z0-9_-]{20,}/ },
|
|
30
|
+
]);
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* `password=…` and its kin. The value must hold a lower-case letter AND a digit, so a documented
|
|
34
|
+
* placeholder (`YOUR_API_KEY`, `changeme`, `$PASSWORD`, `<secret>`, `${{ secrets.X }}`) is not a
|
|
35
|
+
* finding while `password=hunter22` is.
|
|
36
|
+
*/
|
|
37
|
+
const ASSIGN = /(password|passwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token)["']?\s*[:=]\s*["']?([^"'\s<>{}$]{8,})/i;
|
|
38
|
+
function assignment(line) {
|
|
39
|
+
const m = ASSIGN.exec(line);
|
|
40
|
+
return !!m && /[0-9]/.test(m[2]) && /[a-z]/.test(m[2]);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Parse a forbidden-terms file — the format of templates/identity-vocabulary.example: one term per
|
|
45
|
+
* line, trimmed; blank and `#` lines are comments; a plain term matches case-insensitively as a
|
|
46
|
+
* substring; `re:<ERE>` is a regular expression matched against the lower-cased line; a term
|
|
47
|
+
* shorter than 3 characters is refused as a mistake. Returns `{ terms: [{line, test}], problems }`.
|
|
48
|
+
*/
|
|
49
|
+
function parseTerms(text) {
|
|
50
|
+
const terms = [];
|
|
51
|
+
const problems = [];
|
|
52
|
+
String(text || '').split(/\r?\n/).forEach((raw, i) => {
|
|
53
|
+
const t = raw.trim();
|
|
54
|
+
if (!t || t.startsWith('#')) return;
|
|
55
|
+
const line = i + 1;
|
|
56
|
+
if (t.startsWith('re:')) {
|
|
57
|
+
try {
|
|
58
|
+
const re = new RegExp(t.slice(3));
|
|
59
|
+
terms.push({ line, test: (s) => re.test(s.toLowerCase()) });
|
|
60
|
+
} catch (_) {
|
|
61
|
+
problems.push(`${TERMS_PATH} line ${line} is not a valid regular expression`);
|
|
62
|
+
}
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
if (t.length < 3) { problems.push(`${TERMS_PATH} line ${line} is shorter than 3 characters`); return; }
|
|
66
|
+
const want = t.toLowerCase();
|
|
67
|
+
terms.push({ line, test: (s) => s.toLowerCase().includes(want) });
|
|
68
|
+
});
|
|
69
|
+
return { terms, problems };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* The added lines of a `git diff -U0 --no-renames --no-color` patch: `[{ file, line, text }]`.
|
|
74
|
+
* `line` is the line number in the new file.
|
|
75
|
+
*/
|
|
76
|
+
function addedLines(patch) {
|
|
77
|
+
const out = [];
|
|
78
|
+
let file = null;
|
|
79
|
+
let ln = 0;
|
|
80
|
+
let inHunk = false;
|
|
81
|
+
for (const l of String(patch || '').split('\n')) {
|
|
82
|
+
// A file header only outside a hunk: an added line `++ x` prints as `+++ x` inside one.
|
|
83
|
+
if (l.startsWith('diff --git ')) { inHunk = false; file = null; continue; }
|
|
84
|
+
if (!inHunk && l.startsWith('+++ ')) { const f = l.slice(4); file = f === '/dev/null' ? null : f.replace(/^b\//, ''); continue; }
|
|
85
|
+
const h = /^@@ -\S+ \+(\d+)(?:,\d+)? @@/.exec(l);
|
|
86
|
+
if (h) { ln = Number(h[1]); inHunk = true; continue; }
|
|
87
|
+
if (inHunk && l.startsWith('+') && file) { out.push({ file, line: ln, text: l.slice(1) }); ln++; }
|
|
88
|
+
}
|
|
89
|
+
return out;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Pure: findings `[{ file, line, rule }]` over added lines, against RULES + assignment + `terms`. */
|
|
93
|
+
function scanLines(lines, terms = []) {
|
|
94
|
+
const findings = [];
|
|
95
|
+
for (const { file, line, text } of lines) {
|
|
96
|
+
for (const r of RULES) if (r.re.test(text)) findings.push({ file, line, rule: r.rule });
|
|
97
|
+
if (assignment(text)) findings.push({ file, line, rule: 'credential assignment' });
|
|
98
|
+
for (const t of terms) if (t.test(text)) findings.push({ file, line, rule: `forbidden term, line ${t.line} of ${TERMS_PATH}` });
|
|
99
|
+
}
|
|
100
|
+
return findings;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Scan what `branch` adds relative to its merge base with `base`, with the forbidden terms read from
|
|
105
|
+
* `base`'s tree (the branch cannot edit them and stay docs-only: `.github/` is never documentation).
|
|
106
|
+
* Returns `{ ok, findings, files, reason }`; `ok: false` with no findings means the scan could not
|
|
107
|
+
* run — a caller refuses on that too, it never reads it as clean.
|
|
108
|
+
*/
|
|
109
|
+
function scanBranch(git, repo, base, branch) {
|
|
110
|
+
const d = git.git(['diff', '-U0', '--no-renames', '--no-color', '--no-ext-diff', `${base}...${branch}`, '--'], repo);
|
|
111
|
+
if (!d.ok) return { ok: false, findings: [], reason: `could not read the diff ${base}...${branch}` };
|
|
112
|
+
let terms = [];
|
|
113
|
+
if (git.git(['cat-file', '-e', `${base}:${TERMS_PATH}`], repo).ok) {
|
|
114
|
+
const t = git.git(['show', `${base}:${TERMS_PATH}`], repo);
|
|
115
|
+
if (!t.ok) return { ok: false, findings: [], reason: `could not read ${TERMS_PATH} at ${base}` };
|
|
116
|
+
const parsed = parseTerms(t.stdout);
|
|
117
|
+
if (parsed.problems.length) return { ok: false, findings: [], reason: parsed.problems.join('; ') };
|
|
118
|
+
terms = parsed.terms;
|
|
119
|
+
}
|
|
120
|
+
const lines = addedLines(d.stdout);
|
|
121
|
+
const findings = scanLines(lines, terms);
|
|
122
|
+
const files = new Set(lines.map((l) => l.file)).size;
|
|
123
|
+
const extra = terms.length ? ` and ${terms.length} forbidden term(s)` : '';
|
|
124
|
+
if (findings.length) {
|
|
125
|
+
const shown = findings.slice(0, 5).map((f) => `${f.file}:${f.line} (${f.rule})`).join('; ');
|
|
126
|
+
const more = findings.length > 5 ? `; +${findings.length - 5} more` : '';
|
|
127
|
+
return { ok: false, findings, files, reason: `${findings.length} finding(s): ${shown}${more}` };
|
|
128
|
+
}
|
|
129
|
+
return { ok: true, findings, files, reason: `${lines.length} added line(s) in ${files} file(s) clean against the credential shapes${extra}` };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
module.exports = { TERMS_PATH, RULES, parseTerms, addedLines, scanLines, scanBranch, assignment };
|
package/tools/lib/ship-batch.js
CHANGED
|
@@ -513,7 +513,10 @@ function laneHold({ batches, trunkNow, nowMs, graceSec = LANE_GRACE_SEC } = {})
|
|
|
513
513
|
const fresh = (atMs) => Number.isFinite(atMs) && Number.isFinite(nowMs) && (nowMs - atMs) < graceSec * 1000;
|
|
514
514
|
const mins = (atMs) => `${Math.max(0, Math.round((nowMs - atMs) / 60000))}m ago`;
|
|
515
515
|
for (const b of (Array.isArray(batches) ? batches : [])) {
|
|
516
|
-
|
|
516
|
+
// #605: a chained batch (`rooted`, from batchChains) is part of the one holder its root is —
|
|
517
|
+
// built on another batch's head, so it reaches trunk only through that chain. Without the flag,
|
|
518
|
+
// only a batch built on trunk's tip counts, exactly as before.
|
|
519
|
+
if (!b || !b.base7 || !(b.rooted === undefined ? now.startsWith(b.base7) : b.rooted)) continue;
|
|
517
520
|
const v = b.verdict || { state: 'pending', why: 'the run list could not be read' };
|
|
518
521
|
if (v.state === 'red') { released.push({ ref: b.ref, why: `declined — ${v.why}` }); continue; }
|
|
519
522
|
if (v.state === 'pending') return { held: true, ref: b.ref, why: `combined run in flight (${v.why})` };
|
|
@@ -532,6 +535,72 @@ function laneHold({ batches, trunkNow, nowMs, graceSec = LANE_GRACE_SEC } = {})
|
|
|
532
535
|
return { held: false, released };
|
|
533
536
|
}
|
|
534
537
|
|
|
538
|
+
/**
|
|
539
|
+
* #605: speculative chains. A batch built with `--onto <ship-batch ref>` sits on that ref's HEAD, and
|
|
540
|
+
* its name already says so: `ship-batch/<parent-head-sha7>`. So the whole chain is read from
|
|
541
|
+
* `ls-remote` alone — a ref's parent is the ref whose head sha starts with its base7, and a ref is
|
|
542
|
+
* ROOTED when following parents ends at a ref built on trunk's tip. No chain state is kept anywhere.
|
|
543
|
+
*
|
|
544
|
+
* In: refs `[{ ref, base7, sha }]` (every ship-batch/* on the remote), `trunkNow`.
|
|
545
|
+
* Out: Map ref → `{ ref, base7, sha, parentRef, rootRef, rootSha, depth, rooted, why }`:
|
|
546
|
+
* - parentRef null for a batch built on trunk's tip, else the ref it was built on (null too
|
|
547
|
+
* when that ref is gone — then `rooted` is false);
|
|
548
|
+
* - rootSha trunk's tip when rooted, else null;
|
|
549
|
+
* - depth 1 at trunk's tip, 2 for its child, …;
|
|
550
|
+
* - why for an unrooted ref, the reason it is (the sweep deletes it, and says this).
|
|
551
|
+
* A chain deeper than MAX_CHAIN_DEPTH is not rooted: every link pins trunk for one more combined run,
|
|
552
|
+
* and a wrong guess low in the chain throws away every run above it.
|
|
553
|
+
*/
|
|
554
|
+
const MAX_CHAIN_DEPTH = 3;
|
|
555
|
+
|
|
556
|
+
function batchChains({ refs, trunkNow } = {}) {
|
|
557
|
+
const now = String(trunkNow || '');
|
|
558
|
+
const all = (Array.isArray(refs) ? refs : []).filter((r) => r && r.ref && r.base7 && r.sha);
|
|
559
|
+
const parentOf = (r) => all.find((p) => p !== r && String(p.sha).startsWith(r.base7)) || null;
|
|
560
|
+
const out = new Map();
|
|
561
|
+
for (const r of all) {
|
|
562
|
+
const atTip = !!now && now.startsWith(r.base7);
|
|
563
|
+
const parent = atTip ? null : parentOf(r);
|
|
564
|
+
let cur = r, depth = 1, why = null, root = null;
|
|
565
|
+
const seen = new Set([r.ref]);
|
|
566
|
+
while (true) {
|
|
567
|
+
if (now && now.startsWith(cur.base7)) { root = cur; break; }
|
|
568
|
+
const p = parentOf(cur);
|
|
569
|
+
if (!p) {
|
|
570
|
+
why = cur === r ? `its base ${r.base7} is neither trunk's tip nor the head of another batch ref — trunk moved, or its parent was rebuilt or deleted`
|
|
571
|
+
: `its chain breaks at ${cur.ref}, whose base ${cur.base7} is neither trunk's tip nor another batch's head`;
|
|
572
|
+
break;
|
|
573
|
+
}
|
|
574
|
+
if (seen.has(p.ref)) { why = `its chain loops at ${p.ref}`; break; }
|
|
575
|
+
seen.add(p.ref);
|
|
576
|
+
cur = p;
|
|
577
|
+
depth += 1;
|
|
578
|
+
if (depth > MAX_CHAIN_DEPTH) { why = `its chain is deeper than ${MAX_CHAIN_DEPTH}`; break; }
|
|
579
|
+
}
|
|
580
|
+
out.set(r.ref, {
|
|
581
|
+
ref: r.ref, base7: r.base7, sha: r.sha, parentRef: parent ? parent.ref : null,
|
|
582
|
+
rootRef: root ? root.ref : null, rootSha: root ? now : null, depth, rooted: !!root, why,
|
|
583
|
+
});
|
|
584
|
+
}
|
|
585
|
+
return out;
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
/**
|
|
589
|
+
* #605: what a chained batch (built `--onto` a parent) does when it is ready to land but its parent
|
|
590
|
+
* has not landed yet — trunk's tip is not its base. Facts: the parent's chain entry (null = gone)
|
|
591
|
+
* and the parent's combined verdict.
|
|
592
|
+
* wait the parent is rooted and its run is in flight, not yet run, or green → exit 3
|
|
593
|
+
* decline the parent is gone, unrooted, or red → nothing of this chain can land in order: delete
|
|
594
|
+
* this batch and decline it (exit 4)
|
|
595
|
+
*/
|
|
596
|
+
function chainedLandStep({ parent, parentVerdict } = {}) {
|
|
597
|
+
if (!parent) return { step: 'decline', why: 'its parent batch is gone — discarded, or rebuilt on another head' };
|
|
598
|
+
if (!parent.rooted) return { step: 'decline', why: `its parent ${parent.ref} is no longer rooted at trunk's tip (${parent.why})` };
|
|
599
|
+
const v = parentVerdict || { state: 'pending', why: 'the run list could not be read' };
|
|
600
|
+
if (v.state === 'red') return { step: 'decline', why: `its parent ${parent.ref} is red (${v.why}) — it will not land at the sha this batch was built on` };
|
|
601
|
+
return { step: 'wait', why: `its parent ${parent.ref} has not landed yet (${v.state === 'green' ? 'green, waiting for its lander' : (v.why || v.state)})` };
|
|
602
|
+
}
|
|
603
|
+
|
|
535
604
|
/** #581: when a batch's combined run settled — the newest `updatedAt` among its finished rows, else null. */
|
|
536
605
|
function settledAt(rows) {
|
|
537
606
|
const t = (Array.isArray(rows) ? rows : []).filter((r) => r && r.status === 'completed')
|
|
@@ -599,5 +668,5 @@ module.exports = {
|
|
|
599
668
|
droppedTrailer, parseDroppedTrailers, appendTrailers,
|
|
600
669
|
branchCiClass, memberEligibility, selectMembers, wiring, combinedVerdict, nextStep, foreignBatchStep,
|
|
601
670
|
batchGreenCoversTrunk, evidenceSuffix, serialLine, notStaged, landPushFailure,
|
|
602
|
-
LANE_GRACE_SEC, laneHold, settledAt,
|
|
671
|
+
LANE_GRACE_SEC, laneHold, settledAt, MAX_CHAIN_DEPTH, batchChains, chainedLandStep,
|
|
603
672
|
};
|
package/tools/lib/stamp.js
CHANGED
|
@@ -475,6 +475,9 @@ const WORKFLOW_FINGERPRINTS = [
|
|
|
475
475
|
{ marker: 'CI (Node) — TEMPLATE. Copy me into your repo', kind: 'text', template: 'ci-node' },
|
|
476
476
|
{ marker: 'CI (Laravel) — TEMPLATE. Copy me into your repo', kind: 'text', template: 'ci-laravel' },
|
|
477
477
|
{ marker: 'CI (Python) — TEMPLATE. Copy me into your repo', kind: 'text', template: 'ci-python' },
|
|
478
|
+
{ marker: 'CI heavy lane (Node) — TEMPLATE. Copy me into your repo', kind: 'text', template: 'ci-node-heavy' },
|
|
479
|
+
{ marker: 'CI heavy lane (Laravel) — TEMPLATE. Copy me into your repo', kind: 'text', template: 'ci-laravel-heavy' },
|
|
480
|
+
{ marker: 'CI heavy lane (Python) — TEMPLATE. Copy me into your repo', kind: 'text', template: 'ci-python-heavy' },
|
|
478
481
|
{ marker: 'Release (tag) — TEMPLATE. Copy me into your repo', kind: 'text', template: 'release-tag' },
|
|
479
482
|
{ marker: 'Release (auto) — TEMPLATE. Copy me into your repo', kind: 'text', template: 'release-auto' },
|
|
480
483
|
{ marker: 'Deploy (SSH/rsync to shared hosting) — TEMPLATE. Copy me into your repo', kind: 'text', template: 'deploy-xserver' },
|
|
@@ -557,6 +560,18 @@ function workflowProvenance(text, stem, tmplNames) {
|
|
|
557
560
|
return { origin: 'none', template: null, evidence: [] };
|
|
558
561
|
}
|
|
559
562
|
|
|
563
|
+
/**
|
|
564
|
+
* Is this provenance a CI template copy (#607)? Its named template is a `ci-*` one, or — when the
|
|
565
|
+
* evidence names several (a heavy-lane copy carries the light lane's step names too) — every
|
|
566
|
+
* attributed marker it matched belongs to a `ci-*` template. A CI copy's drift is information.
|
|
567
|
+
*/
|
|
568
|
+
function isCiProvenance(prov) {
|
|
569
|
+
if (!prov || prov.origin !== 'derived') return false;
|
|
570
|
+
if (prov.template) return /^ci-/.test(prov.template);
|
|
571
|
+
const owners = WORKFLOW_FINGERPRINTS.filter((fp) => fp.template && prov.evidence.some((e) => e === fp.marker || e.includes(fp.marker) || fp.marker.includes(e))).map((fp) => fp.template);
|
|
572
|
+
return owners.length > 0 && owners.every((n) => /^ci-/.test(n));
|
|
573
|
+
}
|
|
574
|
+
|
|
560
575
|
/**
|
|
561
576
|
* The single source of the advice both tools print, so the two can never drift apart on the one
|
|
562
577
|
* thing that matters here: whether to suggest asserting provenance.
|
|
@@ -745,7 +760,7 @@ module.exports = {
|
|
|
745
760
|
AXES, axesPredating,
|
|
746
761
|
WRITES_VETO_MARKER, writesRulingKnownAt,
|
|
747
762
|
stampLine, stampCommentPrefix, insertStamp, parseWorkflowStamp, parseClaudeStamp,
|
|
748
|
-
fingerprintHits, workflowProvenance, unstampedFinding,
|
|
763
|
+
fingerprintHits, workflowProvenance, unstampedFinding, isCiProvenance,
|
|
749
764
|
looksLikeHandbookWorkflow, looksLikeHandbookClaude,
|
|
750
765
|
cmpParts, cmpSemver, sameContent, classifyStamped,
|
|
751
766
|
};
|