@futurelastic/colab-handbook 1.16.0-rc.2 → 1.16.0-rc.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,184 @@
1
+ # Publish downloads — TEMPLATE. Copy me into your repo at .github/workflows/publish-downloads.yml.
2
+ #
3
+ # This is a starting point, not a dependency. Once you copy it, YOU OWN THIS FILE.
4
+ # Edit it freely; nothing calls this file remotely and nothing will update it for you.
5
+ #
6
+ # EDIT POINTS are marked `# EDIT:` — walk them once after copying.
7
+ #
8
+ # It needs its script in your repo (copy-and-own as well), at the path PUBLISH_SCRIPT names below:
9
+ # colab template publish-downloads.mjs --dest .github/downloads/publish-downloads.mjs
10
+ #
11
+ # For: a repo whose FINAL releases carry files people install on their own machines (a .dmg, a .deb, an
12
+ # .msi, a tarball …), published to a self-hosted downloads site — one folder per app, the newest finals kept
13
+ # by the site (colab-handbook #648; docs/downloads.md has the whole contract). What is published comes from
14
+ # the repo's `.github/project.yml` `downloads:` block plus a manifest.json the site's portal lists the app
15
+ # from; the script's header documents both.
16
+ #
17
+ # WHAT IT DOES for a final tag vX.Y.Z whose files are on its GitHub Release: uploads them with rsync over SSH
18
+ # in the order that keeps a half-finished run harmless — the version folder (with a combined SHA256SUMS last
19
+ # inside it, written only after every file read back over HTTPS against it), then the installers with the
20
+ # site address and the manifest's `requires` stamped in (each required app checked on the site first, #653),
21
+ # then manifest.json, then latest.json LAST — and reads it all back. A published
22
+ # version is immutable: nothing is ever replaced, and a build whose SHA256SUMS differs from the published
23
+ # one is refused. A re-run of a complete version verifies and sends nothing. NEVER a candidate (-rc.N).
24
+ #
25
+ # WHEN IT RUNS.
26
+ # - Chained: your release/build workflow calls this one as its LAST job for a final tag, after the jobs
27
+ # that attach the files to the Release, and only when none of them failed:
28
+ # downloads:
29
+ # needs: [build-macos, build-linux] # the jobs that upload the Release's files
30
+ # if: ${{ !contains(github.ref_name, '-') }} # finals only
31
+ # uses: ./.github/workflows/publish-downloads.yml
32
+ # with: { tag: '${{ github.ref_name }}' }
33
+ # secrets: inherit
34
+ # (A tag pushed or a Release published with GITHUB_TOKEN starts no other workflow, so a chained call is
35
+ # the reliable trigger — not `on: release`.) Nothing is published from an incomplete Release: the script
36
+ # names what is missing and uploads nothing.
37
+ # - By hand (Actions → Run workflow) with a tag: retries a failed publish, finishes a partial one (only the
38
+ # missing files are sent), re-checks a complete one. `dry_run` stages and lists — it uploads nothing.
39
+ # The project.yml, manifest, installers and script are read from THIS run's checkout: the tag's commit when
40
+ # chained, the branch you dispatch from when run by hand.
41
+ #
42
+ # THE RUNNER. The read-back goes over HTTPS to the site, which may be reachable only on a private network —
43
+ # a GitHub-hosted runner then cannot read anything back. `runs-on` is an input: a bare label or a JSON label
44
+ # list of a runner that can reach the site. If the site's name does not resolve there, the run fails saying
45
+ # so, before anything is sent. The runner needs `gh`, `rsync` and `ssh`; Node is set up by the job.
46
+ #
47
+ # CONFIGURATION — per repo, never shared between repos:
48
+ # secrets DOWNLOADS_SSH_KEY · DOWNLOADS_SSH_HOST · DOWNLOADS_SSH_PORT · DOWNLOADS_SSH_USER ·
49
+ # DOWNLOADS_SSH_KNOWN_HOSTS — an upload account restricted to this app's folder, that cannot delete.
50
+ # The older name DOWNLOADS_KNOWN_HOSTS is still read when DOWNLOADS_SSH_KNOWN_HOSTS is unset, so a
51
+ # repo set up before the rename keeps publishing; rename the secret when convenient.
52
+ # variable DOWNLOADS_URL — the site's root address (https). The script appends /<app>; no host is ever
53
+ # written in the repository.
54
+ #
55
+ # THIS FILE DOES NOT CHANGE YOUR REPO'S EXPOSURE (OR LEGACY TIER): it publishes files a final already made.
56
+ name: Publish downloads
57
+
58
+ on:
59
+ workflow_call:
60
+ inputs:
61
+ tag:
62
+ type: string
63
+ required: true
64
+ dry_run:
65
+ type: boolean
66
+ default: false
67
+ runs-on:
68
+ type: string
69
+ default: '["self-hosted","linux"]' # EDIT: a runner that can reach the downloads site
70
+ reserved:
71
+ type: string
72
+ default: '' # EDIT: names the site keeps for itself, refused as an app slug (space-separated)
73
+ workflow_dispatch:
74
+ inputs:
75
+ tag:
76
+ description: 'Existing FINAL tag to publish, e.g. v1.4.0 (a candidate is refused)'
77
+ type: string
78
+ required: true
79
+ dry_run:
80
+ description: 'Dry run: stage and list what would be sent — nothing is uploaded'
81
+ type: boolean
82
+ default: false
83
+ runs-on:
84
+ description: 'Runner: a bare label or a JSON label list of a runner that can reach the site'
85
+ type: string
86
+ default: '["self-hosted","linux"]' # EDIT: keep in step with the workflow_call default
87
+ reserved:
88
+ description: 'Names the site reserves for itself, refused as an app slug (space-separated)'
89
+ type: string
90
+ default: '' # EDIT: keep in step with the workflow_call default
91
+
92
+ permissions: {}
93
+
94
+ concurrency:
95
+ group: publish-downloads
96
+ cancel-in-progress: false
97
+
98
+ env:
99
+ PUBLISH_SCRIPT: .github/downloads/publish-downloads.mjs # EDIT: where you copied the script
100
+
101
+ jobs:
102
+ publish:
103
+ name: Publish ${{ inputs.tag }}
104
+ runs-on: ${{ startsWith(inputs.runs-on, '[') && fromJSON(inputs.runs-on) || inputs.runs-on }}
105
+ timeout-minutes: 45 # EDIT: above your own p99 — large installers over a slow link take a while
106
+ permissions:
107
+ contents: read
108
+ env:
109
+ GH_TOKEN: ${{ github.token }}
110
+ TAG: ${{ inputs.tag }}
111
+ DRY_RUN: ${{ inputs.dry_run }}
112
+ DOWNLOADS_URL: ${{ vars.DOWNLOADS_URL }}
113
+ DOWNLOADS_RESERVED: ${{ inputs.reserved }}
114
+ DOWNLOADS_SSH_HOST: ${{ secrets.DOWNLOADS_SSH_HOST }}
115
+ DOWNLOADS_SSH_PORT: ${{ secrets.DOWNLOADS_SSH_PORT }}
116
+ DOWNLOADS_SSH_USER: ${{ secrets.DOWNLOADS_SSH_USER }}
117
+ steps:
118
+ - name: A final tag only
119
+ run: |
120
+ set -euo pipefail
121
+ [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo "::error::$TAG is not a final tag (vX.Y.Z): only finals are published"; exit 1; }
122
+
123
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
124
+ with:
125
+ persist-credentials: false
126
+
127
+ - name: Setup Node
128
+ uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
129
+ with:
130
+ node-version: '22' # EDIT: any Node >= 18 runs the script
131
+
132
+ - name: The tools the run needs
133
+ run: |
134
+ set -euo pipefail
135
+ need="gh"; [ "$DRY_RUN" = "true" ] || need="gh rsync ssh"
136
+ for t in $need; do command -v "$t" >/dev/null || { echo "::error::$t is not on this runner — pick one that has it (the runs-on input)"; exit 1; }; done
137
+ node "$PUBLISH_SCRIPT" check
138
+
139
+ - name: Download the Release's files
140
+ id: assets
141
+ run: |
142
+ set -euo pipefail
143
+ DIR="$RUNNER_TEMP/assets"; rm -rf "$DIR"; mkdir -p "$DIR"
144
+ gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json assets -q '.assets[].name' > "$RUNNER_TEMP/release-assets.txt"
145
+ while IFS= read -r name; do
146
+ if grep -qxF -- "$name" "$RUNNER_TEMP/release-assets.txt"; then
147
+ gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --pattern "$name" --dir "$DIR"
148
+ fi
149
+ done < <(node "$PUBLISH_SCRIPT" names "$TAG")
150
+ ls -l "$DIR"
151
+ echo "dir=$DIR" >> "$GITHUB_OUTPUT"
152
+
153
+ - name: Write the upload key (this run only)
154
+ if: inputs.dry_run != true
155
+ env:
156
+ KEY: ${{ secrets.DOWNLOADS_SSH_KEY }}
157
+ KNOWN_HOSTS: ${{ secrets.DOWNLOADS_SSH_KNOWN_HOSTS || secrets.DOWNLOADS_KNOWN_HOSTS }}
158
+ run: |
159
+ set -euo pipefail
160
+ umask 077
161
+ [ -n "$KEY" ] && [ -n "$KNOWN_HOSTS" ] || { echo "::error::the downloads upload secrets are not set on this repo"; exit 1; }
162
+ mkdir -p "$RUNNER_TEMP/downloads-ssh"
163
+ printf '%s\n' "$KEY" > "$RUNNER_TEMP/downloads-ssh/key"
164
+ printf '%s\n' "$KNOWN_HOSTS" > "$RUNNER_TEMP/downloads-ssh/known_hosts"
165
+ chmod 600 "$RUNNER_TEMP/downloads-ssh/key" "$RUNNER_TEMP/downloads-ssh/known_hosts"
166
+
167
+ - name: Publish
168
+ env:
169
+ ASSETS: ${{ steps.assets.outputs.dir }}
170
+ DOWNLOADS_KEY_FILE: ${{ runner.temp }}/downloads-ssh/key
171
+ DOWNLOADS_KNOWN_HOSTS_FILE: ${{ runner.temp }}/downloads-ssh/known_hosts
172
+ run: |
173
+ set -euo pipefail
174
+ FLAG=""; [ "$DRY_RUN" = "true" ] && FLAG="--dry-run"
175
+ # The script's own status is kept: a `{ …; echo; } | tee` group would end on the echo and go green.
176
+ status=0
177
+ { echo "### Downloads → $TAG"; echo; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
178
+ node "$PUBLISH_SCRIPT" publish "$TAG" "$ASSETS" $FLAG 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" || status=$?
179
+ echo '```' >> "$GITHUB_STEP_SUMMARY"
180
+ exit "$status"
181
+
182
+ - name: Remove the upload key
183
+ if: always()
184
+ run: rm -rf "$RUNNER_TEMP/downloads-ssh"
package/tools/colab CHANGED
@@ -4797,7 +4797,8 @@ CI grant merges into a repo whose own test suite is known-failing:
4797
4797
  "Red" here (#155) is the SAME verdict \`ship\`'s trunk-CI-green check uses — an ordinary
4798
4798
  still-running run is NOT red (nothing for a grant to exempt; \`ship\` would clear it on its own),
4799
4799
  but a WEDGED run (queued with zero jobs, or stuck past a generous age backstop — the shape a
4800
- billing lockout or dead runner pool leaves) reclassifies to red, because \`ship\` cannot clear
4800
+ billing lockout or dead runner pool leaves; or every job completed and the forge never concluded
4801
+ the run, which cancel and rerun cannot move, #650) reclassifies to red, because \`ship\` cannot clear
4801
4802
  that on its own either and this is the sanctioned door for exactly that case. Before #155 the
4802
4803
  two commands computed this independently and disagreed: a grant could be minted for a state
4803
4804
  \`ship\` would never consult it for, reading live in \`--list\` while doing nothing.
@@ -11515,8 +11516,9 @@ function resolveCloseRefsSplit(repoAbs, gh, issues, refsFromFlag, declaredHolds
11515
11516
  if (gh) {
11516
11517
  for (const num of issues) {
11517
11518
  if (!refsFromFlag.has(num)) continue;
11518
- const info = git.ghIssueView(repoAbs, num, ['body', 'labels']);
11519
+ const info = git.ghIssueView(repoAbs, num, ['body', 'labels', 'state']);
11519
11520
  if (!info) continue; // read failed — a reminder never fires on a read we could not attempt
11521
+ if (String(info.state || '').toUpperCase() === 'CLOSED') continue; // #647: nothing re-takes a closed issue
11520
11522
  const f = refsBrake.refsBrakeFinding({ issue: num, body: info.body, labels: info.labels, declaredHolds });
11521
11523
  if (f) brakeFindings.push(f);
11522
11524
  }
@@ -11788,7 +11790,16 @@ function treeGreenVerdict(repoAbs, run, verdict) {
11788
11790
  */
11789
11791
  function classifyCiRun(repoAbs, run, at) {
11790
11792
  const v = classifyCiRunRaw(repoAbs, run, at);
11791
- return { ...v, detail: v.detail + excludedRunsNote(run) + supersededRunsNote(run) + verifyRuns.setAsideNote(run && run.setAside) + verifyRuns.dispatchCountedNote(run && run.dispatchCounted) };
11793
+ return { ...v, detail: v.detail + excludedRunsNote(run) + supersededRunsNote(run) + verifyRuns.setAsideNote(run && run.setAside) + verifyRuns.dispatchCountedNote(run && run.dispatchCounted) + readViaNote(run) };
11794
+ }
11795
+
11796
+ /**
11797
+ * #651: the runs were read from the unfiltered `actions/runs` listing because the forge's filtered
11798
+ * queries (`head_sha=` / `branch=`) answered nothing. The verdict is judged as usual. The note says
11799
+ * how it was read, so nobody takes an empty filter index for missing CI. '' otherwise.
11800
+ */
11801
+ function readViaNote(run) {
11802
+ return run && run.readVia === 'unfiltered-listing' ? ' — read via unfiltered listing (forge filter index empty, #651)' : '';
11792
11803
  }
11793
11804
 
11794
11805
  /**
@@ -11835,12 +11846,15 @@ function classifyCiRunRaw(repoAbs, run, at) {
11835
11846
  }
11836
11847
  // Non-completed, non-none: running/queued/etc. Ordinarily SELF_CLEARING — but #155's wedged
11837
11848
  // shape (queued, zero jobs, unmoving for hours) can never clear on its own, so check for it.
11838
- const jobCount = run.databaseId != null ? git.ghRunJobCount(repoAbs, run.databaseId) : null;
11849
+ // #650: one `gh run view --json jobs` — the same single call the job count used to cost — so the
11850
+ // jobs' statuses can tell a run the forge never concluded (every job done) from #155's shapes.
11851
+ const jobs = run.databaseId != null ? git.ghRunJobs(repoAbs, run.databaseId) : null;
11852
+ const jobCount = Array.isArray(jobs) ? jobs.length : null;
11839
11853
  // #559: wedge age and zero-jobs floor follow the repo's measured CI (cached profile; bootstrap = 6h / 2m).
11840
11854
  const th = ciProfileFor(repoAbs).th;
11841
- const wedge = ciVerdict.wedgedVerdict({ status: run.status, createdAt: run.createdAt, jobCount }, { wedgeAgeSec: th.wedgeAgeSec, zeroJobsFloorSec: th.zeroJobsFloorSec });
11855
+ const wedge = ciVerdict.wedgedVerdict({ status: run.status, createdAt: run.createdAt, jobCount, jobs }, { wedgeAgeSec: th.wedgeAgeSec, zeroJobsFloorSec: th.zeroJobsFloorSec });
11842
11856
  if (wedge.wedged) {
11843
- return { ok: false, class: HUMAN_GATED, detail: `run for ${at}: status=${run.status} — WEDGED (${wedge.reason}); a human must look, not retry` };
11857
+ return { ok: false, class: HUMAN_GATED, wedge: wedge.shape, detail: `run for ${at}: status=${run.status} — WEDGED (${wedge.reason}); ${ciVerdict.wedgeExit(wedge)}` };
11844
11858
  }
11845
11859
  return { ok: false, class: SELF_CLEARING, detail: `run for ${at}: status=${run.status} — still running, retry later` };
11846
11860
  }
@@ -12729,9 +12743,7 @@ function cmdShipDryJson(opts) {
12729
12743
 
12730
12744
  // #74/#263: same close-gate split the prose path runs, reported via the additive fields above —
12731
12745
  // AND, since #263, folded into `checks` as its own precondition row (below).
12732
- const refsFromFlagJson = new Set(
12733
- String(opts.refs || '').split(',').map((s) => s.trim().replace(/^#/, '')).filter(Boolean).map(Number),
12734
- );
12746
+ const refsFromFlagJson = shipRefsFromFlag(opts, sess); // #647: + the hand-off's closed-before issues
12735
12747
  const split = resolveCloseRefsSplit(repoAbs, gh, sess.issues, refsFromFlagJson, descriptorHolds(doc));
12736
12748
  closeIssues = split.closeIssues; refsIssues = split.refsIssues; checklistFindings = split.gateFindings;
12737
12749
  refsBrakeFindings = split.brakeFindings;
@@ -13746,27 +13758,48 @@ function applyShipHandoff(repoAbs, sess, opts, trunk) {
13746
13758
  syncChain = shipHandoff.syncMergeChain((a) => git.git(a, repoAbs), refs.remoteSha, `refs/remotes/${remote}/${trunk}`);
13747
13759
  }
13748
13760
  const claimsByIssue = {};
13761
+ const issueStates = {}; // #647: an issue closed before the hand-off needs no claim — read state too
13749
13762
  if (u && comment) {
13750
13763
  for (const n of shipguard.branchIssueNumbers(sess.branch)) {
13751
- const info = git.ghIssueView(repoAbs, n, ['comments']);
13764
+ const info = git.ghIssueView(repoAbs, n, ['comments', 'state', 'closedAt']);
13752
13765
  claimsByIssue[n] = info ? liveClaimComments(info.comments || []).map((c) => ({ ...c })) : null;
13766
+ if (info) issueStates[n] = { state: info.state || '', closedAt: info.closedAt || '' };
13753
13767
  }
13754
13768
  }
13755
- const v = shipHandoff.handoffVerdict({ url: opts.handoff, comment, branch: sess.branch, refs, claimsByIssue, syncChain });
13769
+ const v = shipHandoff.handoffVerdict({ url: opts.handoff, comment, branch: sess.branch, refs, claimsByIssue, syncChain,
13770
+ issueStates, refNumbers: refsFlagNumbers(opts.refs) });
13756
13771
  if (!v.ok) return v;
13757
13772
  sess.issues = v.issues;
13758
13773
  sess.handoff = v;
13774
+ // #647: the closed-before-hand-off issues — Refs in the squash, no 🚢/🔖 comment in the tail.
13775
+ sess.handoffRefs = new Set(v.refIssues || []);
13759
13776
  // #372: the implementer is the executor that handed off — name it, never leave it reading as the shipper.
13760
- const ex = (claimsByIssue[v.issues[0]] || []).find((c) => c.branch === sess.branch && c.login === v.executor.login);
13777
+ const claimed = v.issues.find((n) => !sess.handoffRefs.has(n));
13778
+ const ex = (claimsByIssue[claimed] || []).find((c) => c.branch === sess.branch && c.login === v.executor.login);
13761
13779
  if (ex && (ex.session || ex.sessionName)) sess.holder = { session: ex.session || '', sessionName: ex.sessionName || '' };
13762
13780
  return v;
13763
13781
  }
13764
13782
 
13783
+ /** The `--refs` value as issue numbers (`12,#13` → [12, 13]). */
13784
+ function refsFlagNumbers(refs) {
13785
+ return String(refs || '').split(',').map((x) => x.trim().replace(/^#/, '')).filter(Boolean).map(Number);
13786
+ }
13787
+
13788
+ /** #647: --refs ∪ the hand-off's closed-before issues — the set resolveCloseRefsSplit writes as Refs. */
13789
+ function shipRefsFromFlag(opts, sess) {
13790
+ const set = new Set(refsFlagNumbers(opts.refs));
13791
+ for (const n of (sess && sess.handoffRefs) || []) set.add(n);
13792
+ return set;
13793
+ }
13794
+
13765
13795
  function handoffDetail(v) {
13766
13796
  const head = v.syncMerges
13767
13797
  ? `${shortSha(v.handedOff)}, under ${v.syncMerges} clean base-sync merge${v.syncMerges === 1 ? '' : 's'} to the remote head ${shortSha(v.sha)} (#579)`
13768
13798
  : `${shortSha(v.sha)} = the remote head`;
13769
- return `${v.url} hands off ${head}; executor ${v.executor.login}'s live claim on ${v.issues.map((n) => '#' + n).join(' ')}`;
13799
+ const refd = new Set(v.refIssues || []);
13800
+ const closed = v.issues.filter((n) => !refd.has(n));
13801
+ const tail = refd.size ? `; ${[...refd].map((n) => '#' + n).join(' ')} closed before the hand-off — Refs only, no claim needed (#647)` : '';
13802
+ return `${v.url} hands off ${head}; executor ${v.executor.login}'s live claim on ${closed.map((n) => '#' + n).join(' ')}${tail}`;
13770
13803
  }
13771
13804
 
13772
13805
  /** The clause the 🚢 comment carries for a hand-off land — the executor still holds its claim. */
@@ -13922,9 +13955,7 @@ function cmdShip(argv) {
13922
13955
  // resolveCloseRefsSplit (#48, #74): --refs, the tracking label, and #74's close gate (an
13923
13956
  // unticked ## Plan checklist with no declared Remainder: #M). The claim is still released
13924
13957
  // unconditionally either way (B3, unchanged) — only Closes→Refs changes.
13925
- const refsFromFlag = new Set(
13926
- String(opts.refs || '').split(',').map((s) => s.trim().replace(/^#/, '')).filter(Boolean).map(Number),
13927
- );
13958
+ const refsFromFlag = shipRefsFromFlag(opts, sess); // #647: + the hand-off's closed-before issues
13928
13959
  for (const n of refsFromFlag) {
13929
13960
  if (!sess.issues.includes(n)) warn(`--refs #${n} is not a claimed issue on this branch — ignored`);
13930
13961
  }
@@ -14985,6 +15016,8 @@ function shipLandedTail({ repoAbs, sess, target, trunk, gh, sha, closeIssues, re
14985
15016
  const closedByShip = []; // the issues this ship actually closed — for (i2) below (#371)
14986
15017
  if (gh) {
14987
15018
  for (const num of sess.issues) {
15019
+ // #647: closed before the hand-off — history the branch name carries, not work this ship did.
15020
+ if (sess.handoffRefs && sess.handoffRefs.has(num)) continue;
14988
15021
  const isRef = refsSet.has(num);
14989
15022
  const info = git.ghIssueView(repoAbs, num, ['state', 'labels']);
14990
15023
  // A closed issue must have auto-closed; a ref'd (tracking) issue must have STAYED open. The
@@ -15834,6 +15867,14 @@ function shipBatchLand(repoAbs, doc, trunk, remote, base, existing, verdict, opt
15834
15867
  const sess = sessions[i];
15835
15868
  const closeIssues = sess.issues.filter((n) => closed.has(n));
15836
15869
  const refsIssues = sess.issues.filter((n) => !closed.has(n));
15870
+ // #647: a hand-off member's Refs issue that is already CLOSED is one the build exempted as
15871
+ // closed before the hand-off (an open one needed a claim) — history, so no 🔖 comment on it.
15872
+ if (sess.handoff && gh && refsIssues.length) {
15873
+ sess.handoffRefs = new Set(refsIssues.filter((n) => {
15874
+ const info = git.ghIssueView(repoAbs, n, ['state']);
15875
+ return info && String(info.state || '').toUpperCase() === 'CLOSED';
15876
+ }));
15877
+ }
15837
15878
  shipBatchMemberTail({ repoAbs, doc, trunk, sess, sha: shortSha(c.sha), closeIssues, refsIssues, suffix, opts, gh });
15838
15879
  });
15839
15880
 
@@ -15904,7 +15945,8 @@ The first output word is one verdict:
15904
15945
  GREEN every judged workflow's newest run at the sha succeeded — ship's row would pass
15905
15946
  RED a judged run completed and did not succeed — trunk is red
15906
15947
  PENDING a judged run is still in flight (or the sha is minutes old with no run yet)
15907
- WEDGED a run that will never move on its own (zero jobs / hours unmoving, #155)
15948
+ WEDGED a run that will never move on its own (zero jobs / hours unmoving, #155; every job
15949
+ completed but the forge never concluded the run, #650 — the detail names the exits)
15908
15950
  NONE no repo-owned run exists at the sha — CI not wired or never started
15909
15951
  UNREADABLE gh unusable or the read failed — never "green"
15910
15952
 
@@ -15916,14 +15958,17 @@ passed an identical tree (#493) names that run in its detail ("relied on <url>")
15916
15958
  head is local and its tree differs from trunk's, the read is RED: trunk is untested.
15917
15959
 
15918
15960
  Cost (#495): one REST read of the runs at the sha (\`actions/runs?head_sha=\`), plus one
15919
- check-runs read on a GREEN. No auth probe in front of it — the read is the probe, and an
15920
- UNREADABLE explains which credential/remote condition failed. The verdict is cached per
15961
+ check-runs read on a GREEN. An EMPTY filtered answer is re-read from the unfiltered listing
15962
+ (#651: the forge's filter index was measured empty for hours while the runs existed) — at most 3
15963
+ pages, usually 1 — and a verdict found that way says "read via unfiltered listing". No auth
15964
+ probe in front of it — the read is the probe, and an UNREADABLE explains which credential/remote condition failed. The verdict is cached per
15921
15965
  trunk+sha for ${TRUNK_CI_TTL_DEFAULT_SEC}s in the repo's git dir, shared by every session on the
15922
15966
  checkout (COLAB_TRUNK_CI_TTL=<sec> overrides, 0 disables; --fresh skips the cache for one read).
15923
15967
  To WAIT for CI, use \`colab ci-wait\` — never a loop around this command.
15924
15968
 
15925
15969
  --fresh ignore a cached verdict (still refreshes the cache)
15926
- --json { verdict, sha, ok, class, detail, trunk[, cited: [<run url>…] on a #493 skip][, cachedAgeSec] }
15970
+ --json { verdict, sha, ok, class, detail, trunk[, wedge: zero-jobs|unconcluded|age on a WEDGED (#650)]
15971
+ [, cited: [<run url>…] on a #493 skip][, cachedAgeSec] }
15927
15972
  Exit 0 whenever a verdict was read (whatever it is); 2 on UNREADABLE.`;
15928
15973
 
15929
15974
  /** Map readCiVerdict's result to the closed trunk-ci vocabulary. Pure — exported for tests. */
@@ -15964,6 +16009,7 @@ function cmdTrunkCi(argv) {
15964
16009
  let detail = r.verdict.detail;
15965
16010
  if (verdict === 'UNREADABLE' && gh && !git.ghState().ok) detail += ` (${ghUnusableWhy(repoAbs)})`;
15966
16011
  out = { verdict, trunk, sha, ok: r.verdict.ok, class: r.verdict.class || null, detail,
16012
+ ...(r.verdict.wedge ? { wedge: r.verdict.wedge } : {}),
15967
16013
  ...(r.verdict.treeGreen ? { cited: r.verdict.treeGreen } : {}),
15968
16014
  ...(r.run && Array.isArray(r.run.setAside) && r.run.setAside.length ? { setAside: r.run.setAside } : {}),
15969
16015
  ...(r.run && Array.isArray(r.run.dispatchCounted) && r.run.dispatchCounted.length ? { dispatchCounted: r.run.dispatchCounted } : {}) };
@@ -37,6 +37,20 @@
37
37
  * materialization window was not re-measured empirically here. Costs nothing against the signal's
38
38
  * actual target: a billing lockout or dead runner pool leaves a run at zero jobs for far longer than
39
39
  * two minutes, so the floor never masks a real wedge, only the moment right after creation.
40
+ *
41
+ * Amendment (#650): a THIRD shape, told apart from both of #155's — every job reached `completed`,
42
+ * yet the forge never concluded the run (run and check suite stay `in_progress`, `updated_at`
43
+ * frozen). Measured during a forge incident that delayed Actions result processing: the skipped
44
+ * jobs' check runs were written ~6 minutes after the job they `needs:`, and from then on the REST
45
+ * API refused every exit at once — cancel/force-cancel 409 "not in progress", every rerun and the
46
+ * check-suite rerequest 403 "already running". Nothing CI can still do will move it; only the forge
47
+ * (or a human) can. Before this the shape had no signal of its own: the jobs are non-empty, so it
48
+ * read "still running, retry later" until the 6h age backstop, then a WEDGED that named no exit.
49
+ * Signal: the job list is non-empty, every job is `completed`, and the newest job timestamp is past
50
+ * UNCONCLUDED_FLOOR_MINUTES. Every job needs a readable timestamp — a missing one never manufactures
51
+ * a positive (same posture as the two signals above). Its verdict is still HUMAN_GATED, never green,
52
+ * even when every job succeeded: the forge has not said the run passed (epic #652, fail closed).
53
+ * `wedgeExit` below names the exits that work for each shape, so the verdict is actionable.
40
54
  */
41
55
 
42
56
  // Generous on purpose (see doc comment above) — an ordinary run finishes in minutes to tens of
@@ -50,6 +64,11 @@ const WEDGE_AGE_HOURS = 6;
50
64
  // hours, not minutes).
51
65
  const ZERO_JOBS_AGE_FLOOR_MINUTES = 2;
52
66
 
67
+ // #650: how long after its newest job finished a run may still be waiting for the forge to conclude
68
+ // it. Normally that takes seconds; the measured incident wrote check runs ~6 minutes late, so 15
69
+ // minutes is more than twice the worst lag seen before the read says "the forge, not CI".
70
+ const UNCONCLUDED_FLOOR_MINUTES = 15;
71
+
53
72
  function hoursSince(iso, nowMs) {
54
73
  if (!iso) return null;
55
74
  const t = Date.parse(iso);
@@ -59,7 +78,41 @@ function hoursSince(iso, nowMs) {
59
78
  }
60
79
 
61
80
  /**
62
- * `run` is the `{status, conclusion, sha, createdAt, jobCount}` shape — `createdAt`/`jobCount` may be
81
+ * #650: did every job of a non-completed run finish, long enough ago that the forge, not CI, is what
82
+ * is holding it? `jobs` is tools/lib/git.js `ghRunJobs`'s shape (`[{name, status, conclusion,
83
+ * startedAt, completedAt}]`), or null when unread. Returns `{ unconcluded, minutes, failed, counts }`:
84
+ * `minutes` since the newest job timestamp (null when not measurable), `failed` the names of jobs
85
+ * that concluded anything but success/skipped/neutral, `counts` conclusion → number of jobs.
86
+ * Never a positive from a missing signal: null/empty jobs, any job not `completed`, or any job with
87
+ * no readable timestamp all read `unconcluded: false`.
88
+ */
89
+ function unconcludedJobs(jobs, opts = {}) {
90
+ const no = { unconcluded: false, minutes: null, failed: [], counts: {} };
91
+ if (!Array.isArray(jobs) || jobs.length === 0) return no;
92
+ if (!jobs.every((j) => j && j.status === 'completed')) return no;
93
+ let newest = -Infinity;
94
+ for (const j of jobs) {
95
+ // The measured incident wrote a skipped job's started_at AFTER its completed_at — so take the
96
+ // later of the two, per job, and the latest over all jobs.
97
+ const ts = [j.startedAt, j.completedAt].map((x) => (x ? Date.parse(x) : NaN)).filter((t) => !Number.isNaN(t));
98
+ if (!ts.length) return no;
99
+ newest = Math.max(newest, ...ts);
100
+ }
101
+ const now = opts.nowMs != null ? opts.nowMs : Date.now();
102
+ const minutes = (now - newest) / 60000;
103
+ const floorMin = Number.isFinite(opts.floorSec) ? opts.floorSec / 60 : UNCONCLUDED_FLOOR_MINUTES;
104
+ const counts = {};
105
+ const failed = [];
106
+ for (const j of jobs) {
107
+ const c = j.conclusion || 'none';
108
+ counts[c] = (counts[c] || 0) + 1;
109
+ if (!['success', 'skipped', 'neutral'].includes(c)) failed.push(`${j.name || 'unnamed'} (${c})`);
110
+ }
111
+ return { unconcluded: minutes >= floorMin, minutes, failed, counts, floorMin };
112
+ }
113
+
114
+ /**
115
+ * `run` is the `{status, conclusion, sha, createdAt, jobCount, jobs}` shape — `createdAt`/`jobCount` may be
63
116
  * null when not measured (job count is fetched lazily by the caller, only for a run already known to
64
117
  * be non-completed; see tools/lib/git.js `ghRunJobCount`'s doc comment for why).
65
118
  *
@@ -67,10 +120,15 @@ function hoursSince(iso, nowMs) {
67
120
  * never wedged by definition (it finished, however it finished), and `'none'` means no run exists at
68
121
  * all, a different and already-handled case (HUMAN_GATED — "CI is not wired for this ref").
69
122
  *
70
- * Returns `{ wedged, reason }`. `reason` is null when `wedged` is false.
123
+ * `jobs` (#650) is optional — `ghRunJobs`'s list, read by the same lazy call; when present it is what
124
+ * tells the forge-stuck shape (every job completed, run never concluded) from #155's two.
125
+ *
126
+ * Returns `{ wedged, reason, shape }`. `reason` and `shape` are null when `wedged` is false; `shape` is
127
+ * `'zero-jobs'` | `'unconcluded'` | `'age'` (which signal fired). An `'unconcluded'` verdict also
128
+ * carries `failed` — the jobs that concluded red, if any.
71
129
  */
72
130
  function wedgedVerdict(run, opts = {}) {
73
- if (!run || run.status === 'completed' || run.status === 'none') return { wedged: false, reason: null };
131
+ if (!run || run.status === 'completed' || run.status === 'none') return { wedged: false, reason: null, shape: null };
74
132
  const age = hoursSince(run.createdAt, opts.nowMs);
75
133
  // #559: both bounds may come from the repo's measured CI (tools/lib/ci-profile.js); absent → today's.
76
134
  const floorMin = Number.isFinite(opts.zeroJobsFloorSec) ? opts.zeroJobsFloorSec / 60 : ZERO_JOBS_AGE_FLOOR_MINUTES;
@@ -82,15 +140,48 @@ function wedgedVerdict(run, opts = {}) {
82
140
  if (age !== null && age >= floorHours) {
83
141
  return {
84
142
  wedged: true,
143
+ shape: 'zero-jobs',
85
144
  reason: `zero jobs — no runner has ever picked it up (unmoving for ${age.toFixed(2)}h, past the ${fmtMin(floorMin)} floor)`,
86
145
  };
87
146
  }
88
- return { wedged: false, reason: null };
147
+ return { wedged: false, reason: null, shape: null };
148
+ }
149
+ // #650: checked BEFORE the age backstop — a run in this shape is old by the time anyone looks, and
150
+ // the backstop's reason ("status=in_progress for 7h") would hide the one fact that decides the exit.
151
+ const u = unconcludedJobs(run.jobs, { nowMs: opts.nowMs, floorSec: opts.unconcludedFloorSec });
152
+ if (u.unconcluded) {
153
+ const tally = Object.keys(u.counts).sort().map((c) => `${u.counts[c]} ${c}`).join(', ');
154
+ return {
155
+ wedged: true,
156
+ shape: 'unconcluded',
157
+ failed: u.failed,
158
+ reason: `every job completed (${tally}) ${u.minutes.toFixed(0)}m ago, but the forge never concluded the run — past the ${fmtMin(u.floorMin)} floor`,
159
+ };
89
160
  }
90
161
  if (age !== null && age >= wedgeHours) {
91
- return { wedged: true, reason: `status=${run.status} for ${age.toFixed(1)}h (>= ${fmtHours(wedgeHours)} backstop)` };
162
+ return { wedged: true, shape: 'age', reason: `status=${run.status} for ${age.toFixed(1)}h (>= ${fmtHours(wedgeHours)} backstop)` };
163
+ }
164
+ return { wedged: false, reason: null, shape: null };
165
+ }
166
+
167
+ /**
168
+ * The exit a WEDGED verdict names (#650) — the tail of the detail line after the reason. #155's two
169
+ * shapes keep their words: a human must look (billing, a runner pool, a dead runner). The forge-stuck
170
+ * shape names what works and what was measured not to, so nobody retries an exit the forge refuses.
171
+ * The ci-grant authority line does not move: no exit here lets an agent's own run stand in for
172
+ * trunk's (epic #652).
173
+ */
174
+ function wedgeExit(wedge) {
175
+ if (!wedge || wedge.shape !== 'unconcluded') return 'a human must look, not retry';
176
+ if (wedge.failed && wedge.failed.length) {
177
+ return `red in substance — ${wedge.failed.join(', ')} did not succeed, whatever the run's status says; ` +
178
+ "fix trunk (the fix's own push gets a new run), and only the branch that IS that fix may take a human ci-grant";
92
179
  }
93
- return { wedged: false, reason: null };
180
+ return 'nothing CI can do moves it, and nothing here makes it green. Works: wait for the forge to conclude the run ' +
181
+ '(re-read with `colab trunk-ci --fresh`), or a human `colab ci-grant <issue> --branch <branch>` per candidate. ' +
182
+ 'Does not work: cancel / force-cancel (409 "not in progress"), rerun / rerun-failed-jobs / check-suite rerequest ' +
183
+ '(403 "already running"), a workflow_dispatch at this sha (the lost-push rescue skips it while this push run exists, #567). ' +
184
+ "Deleting the run cannot be undone — the owner's call";
94
185
  }
95
186
 
96
187
  const fmtMin = (m) => `${Number.isInteger(m) ? m : m.toFixed(1)}m`;
@@ -118,4 +209,4 @@ function emptyReadVerdict(committedAtMs, opts = {}) {
118
209
  return { fresh: minutes >= 0 && minutes < graceMinutes, minutes, graceMinutes };
119
210
  }
120
211
 
121
- module.exports = { WEDGE_AGE_HOURS, ZERO_JOBS_AGE_FLOOR_MINUTES, EMPTY_READ_GRACE_MINUTES, wedgedVerdict, emptyReadVerdict };
212
+ module.exports = { WEDGE_AGE_HOURS, ZERO_JOBS_AGE_FLOOR_MINUTES, UNCONCLUDED_FLOOR_MINUTES, EMPTY_READ_GRACE_MINUTES, wedgedVerdict, unconcludedJobs, wedgeExit, emptyReadVerdict };