@futurelastic/colab-handbook 1.20.4-rc.3 → 1.20.4-rc.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -33,5 +33,5 @@
33
33
  "engines": {
34
34
  "node": ">=18"
35
35
  },
36
- "version": "1.20.4-rc.3"
36
+ "version": "1.20.4-rc.5"
37
37
  }
@@ -32,6 +32,9 @@
32
32
  // DOWNLOADS_KEY_FILE the private key file the workflow wrote from DOWNLOADS_SSH_KEY
33
33
  // DOWNLOADS_KNOWN_HOSTS_FILE the known-hosts file the workflow wrote from DOWNLOADS_SSH_KNOWN_HOSTS
34
34
  // DOWNLOADS_RESERVED names the site keeps for itself, refused as an app slug (space/comma list)
35
+ // DOWNLOADS_READ_HEADERS optional (#714): a site that requires sign-in for every file. One `Name: value`
36
+ // HTTP header per line, sent on every read of the site (the requires check, the
37
+ // read-back); never printed. Unset, the site is read with no credential
35
38
  //
36
39
  // THE APPS IT REQUIRES (#653) come from manifest.json's optional `requires`: [{"slug": "<app>", "min": "X.Y.Z"}],
37
40
  // other apps on the same site that its installers install first. Before anything is sent, every slug must have a final
@@ -570,6 +573,29 @@ function runRsync({ rsync, sshArgs, args, log }) {
570
573
  if (r.status !== 0) throw new Error(`rsync failed (${r.status ?? r.signal}): ${(r.stderr || r.stdout || '').trim()}`)
571
574
  }
572
575
 
576
+ /**
577
+ * DOWNLOADS_READ_HEADERS (#714) as a headers object: one `Name: value` per line, blank lines ignored. A line that is not
578
+ * a header throws naming its line number only — the value is a credential and is never echoed.
579
+ */
580
+ export function readHeaders(text) {
581
+ const out = {}
582
+ const lines = String(text ?? '').split(/\r?\n/)
583
+ for (let i = 0; i < lines.length; i++) {
584
+ const line = lines[i].trim()
585
+ if (!line) continue
586
+ const m = /^([A-Za-z0-9!#$%&'*+.^_`|~-]+):[ \t]*(\S.*)$/.exec(line)
587
+ if (!m) throw new Error(`DOWNLOADS_READ_HEADERS: line ${i + 1} is not a "Name: value" header (the value is not shown); nothing was published`)
588
+ out[m[1].toLowerCase()] = m[2]
589
+ }
590
+ return out
591
+ }
592
+
593
+ /** `fetchImpl` with `headers` added to every request — how every read of the site carries DOWNLOADS_READ_HEADERS. */
594
+ export function withHeaders(fetchImpl, headers) {
595
+ if (!Object.keys(headers).length) return fetchImpl
596
+ return (url, init = {}) => fetchImpl(url, { ...init, headers: { ...headers, ...(init.headers ?? {}) } })
597
+ }
598
+
573
599
  async function get(fetchImpl, url, attempts = 3) {
574
600
  let last
575
601
  for (let i = 0; i < attempts; i++) {
@@ -589,6 +615,8 @@ async function get(fetchImpl, url, attempts = 3) {
589
615
  async function readBytes(fetchImpl, url) {
590
616
  const res = await get(fetchImpl, url)
591
617
  if (res.status === 404) return null
618
+ if (res.status === 401 || res.status === 403)
619
+ throw new Error(`${url} answered ${res.status} — the site requires sign-in: set or check the DOWNLOADS_READ_HEADERS secret (docs/downloads.md)`)
592
620
  if (!res.ok) throw new Error(`${url} answered ${res.status}`)
593
621
  return Buffer.from(await res.arrayBuffer())
594
622
  }
@@ -826,12 +854,13 @@ async function main(argv, env) {
826
854
  const dryRun = rest.includes('--dry-run')
827
855
  const assetsDir = resolve(rest.filter((a) => !a.startsWith('--'))[0] ?? '')
828
856
  const baseUrl = env.DOWNLOADS_URL
857
+ const fetchImpl = withHeaders(fetch, readHeaders(env.DOWNLOADS_READ_HEADERS))
829
858
  const outDir = mkdtempSync(join(tmpdir(), 'publish-downloads-'))
830
859
  const staged = await stage({ tag, assetsDir, outDir, baseUrl, root, config, manifest })
831
860
  console.log(`Staged ${config.app} ${staged.version}: ${[...Object.keys(staged.files), ...staged.images].join(', ')}`)
832
861
  await assertResolves(baseUrl)
833
862
  const target = dryRun ? { dest: 'DRY-RUN:', sshArgs: undefined } : sshCommand(env)
834
- const result = await publish({ outDir, app: config.app, ...target, baseUrl, dryRun })
863
+ const result = await publish({ outDir, app: config.app, ...target, baseUrl, dryRun, fetchImpl })
835
864
  if (dryRun) console.log(`Dry run of ${result.version} done — nothing was uploaded`)
836
865
  else
837
866
  console.log(
@@ -58,6 +58,10 @@
58
58
  # repo set up before the rename keeps publishing; rename the secret when convenient.
59
59
  # variable DOWNLOADS_URL — the site's root address (https). The script appends /<app>; no host is ever
60
60
  # written in the repository.
61
+ # secret DOWNLOADS_READ_HEADERS — optional (#714), only for a site that requires sign-in for every file: one
62
+ # `Name: value` HTTP header per line (e.g. `Authorization: Bearer …`, or an access proxy's two
63
+ # service-token headers), sent on every read of the site. The same secret release-auto.yml's downloads
64
+ # sweep reads latest.json with. Unset, the site is read with no credential.
61
65
  #
62
66
  # THIS FILE DOES NOT CHANGE YOUR REPO'S EXPOSURE (OR LEGACY TIER): it publishes files a final already made.
63
67
  name: Publish downloads
@@ -174,6 +178,7 @@ jobs:
174
178
  - name: Publish
175
179
  env:
176
180
  ASSETS: ${{ steps.assets.outputs.dir }}
181
+ DOWNLOADS_READ_HEADERS: ${{ secrets.DOWNLOADS_READ_HEADERS }}
177
182
  DOWNLOADS_KEY_FILE: ${{ runner.temp }}/downloads-ssh/key
178
183
  DOWNLOADS_KNOWN_HOSTS_FILE: ${{ runner.temp }}/downloads-ssh/known_hosts
179
184
  run: |
@@ -145,7 +145,11 @@
145
145
  # one (a complete version sends nothing), and when the site cannot be read from this job's runner —
146
146
  # a warning then, never a red run; the next daily run retries. The site is read at
147
147
  # vars.DOWNLOADS_URL/<downloads.app>/latest.json FROM THE RELEASE JOB'S RUNNER: a site only a private
148
- # network reaches needs the release job on a runner inside it (RUNNERS, below). The wiring:
148
+ # network reaches needs the release job on a runner inside it (RUNNERS, below). A site that requires
149
+ # sign-in (#714) is read with the optional DOWNLOADS_READ_HEADERS secret — one `Name: value` header
150
+ # per line, the same secret publish-downloads.yml reads with; a 401/403 is not the transient warning
151
+ # but an `::error title=Downloads not shipped::` plus a failing `downloads/<app>` commit status on
152
+ # the final, naming it (still exit 0, for the reason "A REFUSAL IS NOT A FAILURE" gives). The wiring:
149
153
  #
150
154
  # build-macos: # EDIT: one job per platform you build
151
155
  # needs: release
@@ -362,6 +366,7 @@ jobs:
362
366
  contents: write # push the tags (and, with promotion: main-loop, the promotion), publish the GitHub Release
363
367
  issues: write # finalize opens / comments on / closes the "release: vX.Y.Z" tracking issue
364
368
  actions: write # read the runs at a commit (ci-green, full-suite, trunk-green); dispatch CI after a promotion
369
+ statuses: write # the downloads sweep marks a final its site refused to read (401/403) failed (#714)
365
370
  # What the npm job below needs (#433) — empty / publish=false unless this run tagged something (or
366
371
  # the daily sweep found a final npm lacks, #612) AND .github/project.yml opts in with release.npm.
367
372
  outputs:
@@ -823,7 +828,8 @@ jobs:
823
828
  # when the site's latest.json does not name it yet (absent → never published, older → behind) — a
824
829
  # final a person tagged (early, #627, or `final: human`) is never tagged by a run, so without this
825
830
  # sweep it never reached the site. The site at that version or newer, no `downloads:` block, no
826
- # DOWNLOADS_URL, or an unreadable site: nothing is handed on, and the step stays green.
831
+ # DOWNLOADS_URL, or an unreadable site: nothing is handed on, and the step stays green — a 401/403
832
+ # (#714) too, but as an error annotation and a failing commit status on the final, not a warning.
827
833
  - name: Read the downloads target [publish]
828
834
  id: downloads-target
829
835
  if: env.DRY_RUN != 'true' && (steps.cut.outputs.tag != '' || steps.finalize.outputs.tag != '' || (github.event_name == 'schedule' && env.RETRY_RUN != 'true') || github.event_name == 'workflow_dispatch')
@@ -831,6 +837,8 @@ jobs:
831
837
  TAGS: ${{ steps.cut.outputs.tag }} ${{ steps.finalize.outputs.tag }}
832
838
  SWEEP: ${{ (github.event_name == 'schedule' && env.RETRY_RUN != 'true') || github.event_name == 'workflow_dispatch' }}
833
839
  DOWNLOADS_URL: ${{ vars.DOWNLOADS_URL }}
840
+ # #714: a site that requires sign-in for every file. Optional; unset, the read sends no credential.
841
+ DOWNLOADS_READ_HEADERS: ${{ secrets.DOWNLOADS_READ_HEADERS }}
834
842
  run: |
835
843
  set -uo pipefail
836
844
  # The `downloads:` block's `app:` — the same key publish-downloads.mjs reads. No block, no target.
@@ -871,7 +879,20 @@ jobs:
871
879
  { echo "tag=$FINAL"; echo "swept=$FINAL"; } >> "$GITHUB_OUTPUT"
872
880
  echo "downloads sweep: $1 — handing $FINAL to the build and downloads jobs (#669)" | tee -a "$GITHUB_STEP_SUMMARY"
873
881
  }
874
- if BODY="$(retry_transient curl -fsSL "$URL" 2> "$ERR")"; then
882
+ # #714: DOWNLOADS_READ_HEADERS — one `Name: value` header per line (docs/downloads.md, "A site
883
+ # that requires sign-in") — goes to curl from a file, never on its command line, and is never printed.
884
+ AUTH=()
885
+ if [ -n "${DOWNLOADS_READ_HEADERS:-}" ]; then
886
+ HDRS="$RUNNER_TEMP/downloads-read-headers"
887
+ trap 'rm -f "$HDRS"' EXIT
888
+ ( umask 077; printf '%s\n' "$DOWNLOADS_READ_HEADERS" | tr -d '\r' | sed '/^[[:space:]]*$/d' > "$HDRS" )
889
+ if grep -qvE '^[A-Za-z0-9!#$%&*+.^_`|~-]+:[[:space:]]*[^[:space:]]' "$HDRS"; then
890
+ echo "::error title=Downloads not shipped::the DOWNLOADS_READ_HEADERS secret has a line that is not a 'Name: value' header (the value is not shown) — $FINAL is not handed on"
891
+ exit 0 # a standing refusal, not a red run (the 401/403 branch below says why)
892
+ fi
893
+ AUTH=(-H "@$HDRS")
894
+ fi
895
+ if BODY="$(retry_transient curl -fsSL ${AUTH[@]+"${AUTH[@]}"} "$URL" 2> "$ERR")"; then
875
896
  SITE="$(jq -r '.version // empty' <<< "$BODY" 2> /dev/null || true)"
876
897
  if [ -z "$SITE" ]; then
877
898
  echo "::warning::downloads sweep: $URL names no version — $FINAL is not handed on in this run"
@@ -884,6 +905,29 @@ jobs:
884
905
  fi
885
906
  elif grep -q 'error: 404' "$ERR"; then
886
907
  hand_on "$FINAL is final on main and the site has no latest.json for $APP"
908
+ elif CODE="$(grep -oE 'error: 40[13]' "$ERR" | head -n 1 | grep -oE '40[13]')"; then
909
+ # #714: 401/403 is a refusal no later run clears by itself, so it is not the warning below: a
910
+ # final whose files would never ship. Visible, but exit 0 — the #467 rule (header, "A REFUSAL
911
+ # IS NOT A FAILURE"): a red run would make main's head not-green and refuse the very ship
912
+ # that fixes it. So: an error annotation, a summary banner, and a failing commit status
913
+ # `downloads/<app>` on the final's own commit. The next daily run re-reads and re-posts.
914
+ if [ -n "${DOWNLOADS_READ_HEADERS:-}" ]; then
915
+ WHY="the site refused the DOWNLOADS_READ_HEADERS credential — check or replace that secret"
916
+ else
917
+ WHY="the site requires sign-in and no DOWNLOADS_READ_HEADERS secret is set on this repo"
918
+ fi
919
+ MSG="$URL answered $CODE, so $FINAL is final on main but its files never ship: $WHY (docs/downloads.md, \"A site that requires sign-in\")"
920
+ echo "::error title=Downloads not shipped::downloads sweep: $MSG"
921
+ { echo "#### ⚠ Downloads not shipped — $FINAL"; echo; echo "$MSG"; } >> "$GITHUB_STEP_SUMMARY"
922
+ SHA="$(git rev-list -n 1 "$FINAL")"
923
+ # github.com only: a forge reads CI from commit statuses (#681), where a failing one would be the
924
+ # red main this branch exists to avoid. There the annotation stands alone.
925
+ if [ "${GITHUB_SERVER_URL:-https://github.com}" != "https://github.com" ]; then
926
+ :
927
+ elif ! retry_transient gh api --silent -X POST "repos/$GITHUB_REPOSITORY/statuses/$SHA" -f state=failure \
928
+ -f context="downloads/$APP" -f description="$FINAL not on the site: latest.json answered $CODE" > /dev/null; then
929
+ echo "::warning::downloads sweep: could not post the failing downloads/$APP status on $FINAL — the annotation above stands"
930
+ fi
887
931
  else
888
932
  echo "::warning::downloads sweep: could not read $URL from this runner ($(tail -n 1 "$ERR" 2> /dev/null)) — $FINAL is not handed on in this run; the next daily run retries"
889
933
  fi
package/tools/colab CHANGED
@@ -13514,8 +13514,9 @@ function shipMode(repoAbs, target, branch, commits) {
13514
13514
  * this ship actually closed, read its NATIVE parent and let lib/container-close.js decide; close
13515
13515
  * the parent with an evidence comment only on `close`, then ask the same question of ITS parent
13516
13516
  * (an epic of epics empties from the bottom up). Every other verdict leaves the parent exactly as
13517
- * it is; the ones a human should see (`not-container`, `unfiled-items`, `unknown`) are printed as
13518
- * findings. Best-effort like every post-merge write: a failure here warns, never fails the ship —
13517
+ * it is; the ones a human should see (`not-container`, `unfiled-items`, `done-when`, `unknown`)
13518
+ * are printed as findings. `done-when` (#713) also gets a one-line proposal comment on the epic —
13519
+ * every filed child is closed, but the body's `Done when` acceptance is the coordinator's to read. Best-effort like every post-merge write: a failure here warns, never fails the ship —
13519
13520
  * the merge already landed. A lagging `subIssuesSummary` reads as `children-open` and is caught by
13520
13521
  * the next ship of a sibling or by code-sweep §5, never by guessing.
13521
13522
  */
@@ -13540,6 +13541,10 @@ function shipCloseSpentContainers(repoAbs, closedIssues, { sha, target } = {}) {
13540
13541
  if (c.verdict !== 'close') {
13541
13542
  const finding = containerClose.containerFinding({ ...c, number: c.number === null ? pnum : c.number });
13542
13543
  if (finding) warn(`container ${finding}`);
13544
+ if (c.verdict === 'done-when') {
13545
+ const pc = git.ghIssueComment(repoAbs, pnum, containerClose.containerProposalComment({ child, sha, total: c.total, target }));
13546
+ if (!pc.ok) warnDeferred(`could not post the done-when proposal on #${pnum} — it stays open either way`);
13547
+ }
13543
13548
  break;
13544
13549
  }
13545
13550
  const body = containerClose.containerCloseComment({ child, sha, total: c.total, target });
@@ -20,6 +20,12 @@
20
20
  * - its body holds no unticked checklist line. An unticked `- [ ]` on an epic is an item
21
21
  * somebody listed and nobody filed as a sub-issue yet — the "two phases whose issues are not
22
22
  * written yet" B2c's rule was protecting. It is reported, never closed over;
23
+ * - its body states no `Done when` acceptance (#713). `file-issues` tells a filer to put an
24
+ * epic's end state in a `Done when:` line, never a body checklist — so on an epic written to
25
+ * that convention the unticked check above sees nothing, and a phase epic filed with only its
26
+ * design child (the code children to be filed once the design is approved) read as spent the
27
+ * moment that one child shipped. "Every filed child closed" says nothing about whether the
28
+ * stated end state holds; that reading is the coordinator's, so ship proposes and never closes;
23
29
  * - it is not a release tracking record (`<!-- colab:release …` body) — `colab release
24
30
  * finalize` owns that record's lifecycle, not a child's ship.
25
31
  *
@@ -36,6 +42,8 @@
36
42
  * children-open at least one sub-issue is still open — the normal state of a live epic
37
43
  * not-container every child closed, but no `epic` label — report for a human
38
44
  * unfiled-items every child closed, but the body still lists unticked items — report
45
+ * done-when every child closed, but the body states `Done when` acceptance — ship posts
46
+ * a one-line proposal for the coordinator to check it and close (#713)
39
47
  *
40
48
  * Fail toward keeping open, the same asymmetry `group-labels.js` states for its own delete: a
41
49
  * container closed early buries unwritten work behind a closed issue nobody browses, while one
@@ -52,6 +60,14 @@ const RELEASE_RECORD_RE = /^\s*<!--\s*colab:release\b/;
52
60
  /** An UNTICKED GitHub checklist line anywhere in the body — `- [ ] …` / `* [ ] …`. */
53
61
  const UNTICKED_RE = /^\s*[-*]\s*\[ \]\s*(.+)$/;
54
62
 
63
+ /**
64
+ * A `Done when` acceptance statement at the start of a line (#713): a heading (`## Done when`), a
65
+ * label line (`Done when:`, `**Done when:**`, `- Done when: …`), any case, `Done-when` too. Only at
66
+ * a line start, so prose that merely mentions the phrase mid-sentence does not count; a blockquote
67
+ * (`> Done when`) is a quotation, not this epic's own acceptance.
68
+ */
69
+ const DONE_WHEN_RE = /^\s*(?:#{1,6}\s+)?(?:[-*+]\s+)?(?:\*\*|__)?done[ -]when\b/i;
70
+
55
71
  function labelNames(labels) {
56
72
  return (labels || []).map((l) => (l && typeof l === 'object' ? l.name : l)).map((n) => String(n));
57
73
  }
@@ -70,6 +86,16 @@ function untickedItems(body) {
70
86
  return out;
71
87
  }
72
88
 
89
+ /** Does the body state `Done when` acceptance outside fenced code? (#713) */
90
+ function hasDoneWhen(body) {
91
+ let fenced = false;
92
+ for (const line of String(body || '').split('\n')) {
93
+ if (/^\s*(```|~~~)/.test(line)) { fenced = !fenced; continue; }
94
+ if (!fenced && DONE_WHEN_RE.test(line)) return true;
95
+ }
96
+ return false;
97
+ }
98
+
73
99
  /**
74
100
  * Classify ONE parent issue after a child of it closed.
75
101
  *
@@ -99,6 +125,7 @@ function classifyContainer(parent) {
99
125
  if (!labelNames(parent.labels).includes(EPIC_LABEL)) return { ...out, verdict: 'not-container' };
100
126
  const unticked = untickedItems(parent.body);
101
127
  if (unticked.length) return { ...out, unticked, verdict: 'unfiled-items' };
128
+ if (hasDoneWhen(parent.body)) return { ...out, verdict: 'done-when' };
102
129
  return { ...out, verdict: 'close' };
103
130
  }
104
131
 
@@ -120,6 +147,16 @@ function containerCloseComment({ child, sha, total, target }) {
120
147
  return `📦 Closed by colab ship — ${via}${at}; all ${total} sub-issue(s) are closed and the body lists no unticked item (#371).`;
121
148
  }
122
149
 
150
+ /**
151
+ * The proposal comment posted on an epic whose verdict is `done-when` (#713) — the epic stays OPEN.
152
+ * One line, issue-specific only: which child emptied it, and what the coordinator checks next.
153
+ */
154
+ function containerProposalComment({ child, sha, total, target }) {
155
+ const via = child ? `its last open sub-issue #${child} shipped` : 'its last open sub-issue closed';
156
+ const at = sha ? ` to ${target || 'trunk'} at ${sha}` : '';
157
+ return `📋 All ${total} sub-issue(s) closed — ${via}${at}. Left open: check \`Done when\` and close if met, or file the remaining work as sub-issues (#713).`;
158
+ }
159
+
123
160
  /** One report line for a verdict that is NOT `close` but still worth a human's eye; null otherwise. */
124
161
  function containerFinding(c) {
125
162
  const n = c.number !== null ? `#${c.number}` : 'parent';
@@ -128,6 +165,8 @@ function containerFinding(c) {
128
165
  return `${n}: all ${c.total} sub-issue(s) closed, but it is not labelled \`epic\` — close it by hand if it is a container`;
129
166
  case 'unfiled-items':
130
167
  return `${n}: all ${c.total} sub-issue(s) closed, but ${c.unticked.length} unticked item(s) remain in its body — file them as sub-issues or tick them, then close`;
168
+ case 'done-when':
169
+ return `${n}: all ${c.total} sub-issue(s) closed, but its body states \`Done when\` — left open for the coordinator to check it`;
131
170
  case 'unknown':
132
171
  return `${n}: could not read the parent — container close skipped`;
133
172
  default:
@@ -173,7 +212,8 @@ function parentRelationFinding(rel) {
173
212
  }
174
213
 
175
214
  module.exports = {
176
- EPIC_LABEL, RELEASE_RECORD_RE,
215
+ EPIC_LABEL, RELEASE_RECORD_RE, DONE_WHEN_RE,
177
216
  issueRepoSlug, parentRelation, parentRelationFinding,
178
- untickedItems, classifyContainer, containerDeliveryLabels, containerCloseComment, containerFinding,
217
+ untickedItems, hasDoneWhen, classifyContainer, containerDeliveryLabels, containerCloseComment,
218
+ containerProposalComment, containerFinding,
179
219
  };