@futurelastic/colab-handbook 1.20.4-rc.4 → 1.20.4-rc.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -33,5 +33,5 @@
33
33
  "engines": {
34
34
  "node": ">=18"
35
35
  },
36
- "version": "1.20.4-rc.4"
36
+ "version": "1.20.4-rc.5"
37
37
  }
@@ -32,6 +32,9 @@
32
32
  // DOWNLOADS_KEY_FILE the private key file the workflow wrote from DOWNLOADS_SSH_KEY
33
33
  // DOWNLOADS_KNOWN_HOSTS_FILE the known-hosts file the workflow wrote from DOWNLOADS_SSH_KNOWN_HOSTS
34
34
  // DOWNLOADS_RESERVED names the site keeps for itself, refused as an app slug (space/comma list)
35
+ // DOWNLOADS_READ_HEADERS optional (#714): a site that requires sign-in for every file. One `Name: value`
36
+ // HTTP header per line, sent on every read of the site (the requires check, the
37
+ // read-back); never printed. Unset, the site is read with no credential
35
38
  //
36
39
  // THE APPS IT REQUIRES (#653) come from manifest.json's optional `requires`: [{"slug": "<app>", "min": "X.Y.Z"}],
37
40
  // other apps on the same site that its installers install first. Before anything is sent, every slug must have a final
@@ -570,6 +573,29 @@ function runRsync({ rsync, sshArgs, args, log }) {
570
573
  if (r.status !== 0) throw new Error(`rsync failed (${r.status ?? r.signal}): ${(r.stderr || r.stdout || '').trim()}`)
571
574
  }
572
575
 
576
+ /**
577
+ * DOWNLOADS_READ_HEADERS (#714) as a headers object: one `Name: value` per line, blank lines ignored. A line that is not
578
+ * a header throws naming its line number only — the value is a credential and is never echoed.
579
+ */
580
+ export function readHeaders(text) {
581
+ const out = {}
582
+ const lines = String(text ?? '').split(/\r?\n/)
583
+ for (let i = 0; i < lines.length; i++) {
584
+ const line = lines[i].trim()
585
+ if (!line) continue
586
+ const m = /^([A-Za-z0-9!#$%&'*+.^_`|~-]+):[ \t]*(\S.*)$/.exec(line)
587
+ if (!m) throw new Error(`DOWNLOADS_READ_HEADERS: line ${i + 1} is not a "Name: value" header (the value is not shown); nothing was published`)
588
+ out[m[1].toLowerCase()] = m[2]
589
+ }
590
+ return out
591
+ }
592
+
593
+ /** `fetchImpl` with `headers` added to every request — how every read of the site carries DOWNLOADS_READ_HEADERS. */
594
+ export function withHeaders(fetchImpl, headers) {
595
+ if (!Object.keys(headers).length) return fetchImpl
596
+ return (url, init = {}) => fetchImpl(url, { ...init, headers: { ...headers, ...(init.headers ?? {}) } })
597
+ }
598
+
573
599
  async function get(fetchImpl, url, attempts = 3) {
574
600
  let last
575
601
  for (let i = 0; i < attempts; i++) {
@@ -589,6 +615,8 @@ async function get(fetchImpl, url, attempts = 3) {
589
615
  async function readBytes(fetchImpl, url) {
590
616
  const res = await get(fetchImpl, url)
591
617
  if (res.status === 404) return null
618
+ if (res.status === 401 || res.status === 403)
619
+ throw new Error(`${url} answered ${res.status} — the site requires sign-in: set or check the DOWNLOADS_READ_HEADERS secret (docs/downloads.md)`)
592
620
  if (!res.ok) throw new Error(`${url} answered ${res.status}`)
593
621
  return Buffer.from(await res.arrayBuffer())
594
622
  }
@@ -826,12 +854,13 @@ async function main(argv, env) {
826
854
  const dryRun = rest.includes('--dry-run')
827
855
  const assetsDir = resolve(rest.filter((a) => !a.startsWith('--'))[0] ?? '')
828
856
  const baseUrl = env.DOWNLOADS_URL
857
+ const fetchImpl = withHeaders(fetch, readHeaders(env.DOWNLOADS_READ_HEADERS))
829
858
  const outDir = mkdtempSync(join(tmpdir(), 'publish-downloads-'))
830
859
  const staged = await stage({ tag, assetsDir, outDir, baseUrl, root, config, manifest })
831
860
  console.log(`Staged ${config.app} ${staged.version}: ${[...Object.keys(staged.files), ...staged.images].join(', ')}`)
832
861
  await assertResolves(baseUrl)
833
862
  const target = dryRun ? { dest: 'DRY-RUN:', sshArgs: undefined } : sshCommand(env)
834
- const result = await publish({ outDir, app: config.app, ...target, baseUrl, dryRun })
863
+ const result = await publish({ outDir, app: config.app, ...target, baseUrl, dryRun, fetchImpl })
835
864
  if (dryRun) console.log(`Dry run of ${result.version} done — nothing was uploaded`)
836
865
  else
837
866
  console.log(
@@ -58,6 +58,10 @@
58
58
  # repo set up before the rename keeps publishing; rename the secret when convenient.
59
59
  # variable DOWNLOADS_URL — the site's root address (https). The script appends /<app>; no host is ever
60
60
  # written in the repository.
61
+ # secret DOWNLOADS_READ_HEADERS — optional (#714), only for a site that requires sign-in for every file: one
62
+ # `Name: value` HTTP header per line (e.g. `Authorization: Bearer …`, or an access proxy's two
63
+ # service-token headers), sent on every read of the site. The same secret release-auto.yml's downloads
64
+ # sweep reads latest.json with. Unset, the site is read with no credential.
61
65
  #
62
66
  # THIS FILE DOES NOT CHANGE YOUR REPO'S EXPOSURE (OR LEGACY TIER): it publishes files a final already made.
63
67
  name: Publish downloads
@@ -174,6 +178,7 @@ jobs:
174
178
  - name: Publish
175
179
  env:
176
180
  ASSETS: ${{ steps.assets.outputs.dir }}
181
+ DOWNLOADS_READ_HEADERS: ${{ secrets.DOWNLOADS_READ_HEADERS }}
177
182
  DOWNLOADS_KEY_FILE: ${{ runner.temp }}/downloads-ssh/key
178
183
  DOWNLOADS_KNOWN_HOSTS_FILE: ${{ runner.temp }}/downloads-ssh/known_hosts
179
184
  run: |
@@ -145,7 +145,11 @@
145
145
  # one (a complete version sends nothing), and when the site cannot be read from this job's runner —
146
146
  # a warning then, never a red run; the next daily run retries. The site is read at
147
147
  # vars.DOWNLOADS_URL/<downloads.app>/latest.json FROM THE RELEASE JOB'S RUNNER: a site only a private
148
- # network reaches needs the release job on a runner inside it (RUNNERS, below). The wiring:
148
+ # network reaches needs the release job on a runner inside it (RUNNERS, below). A site that requires
149
+ # sign-in (#714) is read with the optional DOWNLOADS_READ_HEADERS secret — one `Name: value` header
150
+ # per line, the same secret publish-downloads.yml reads with; a 401/403 is not the transient warning
151
+ # but an `::error title=Downloads not shipped::` plus a failing `downloads/<app>` commit status on
152
+ # the final, naming it (still exit 0, for the reason "A REFUSAL IS NOT A FAILURE" gives). The wiring:
149
153
  #
150
154
  # build-macos: # EDIT: one job per platform you build
151
155
  # needs: release
@@ -362,6 +366,7 @@ jobs:
362
366
  contents: write # push the tags (and, with promotion: main-loop, the promotion), publish the GitHub Release
363
367
  issues: write # finalize opens / comments on / closes the "release: vX.Y.Z" tracking issue
364
368
  actions: write # read the runs at a commit (ci-green, full-suite, trunk-green); dispatch CI after a promotion
369
+ statuses: write # the downloads sweep marks a final its site refused to read (401/403) failed (#714)
365
370
  # What the npm job below needs (#433) — empty / publish=false unless this run tagged something (or
366
371
  # the daily sweep found a final npm lacks, #612) AND .github/project.yml opts in with release.npm.
367
372
  outputs:
@@ -823,7 +828,8 @@ jobs:
823
828
  # when the site's latest.json does not name it yet (absent → never published, older → behind) — a
824
829
  # final a person tagged (early, #627, or `final: human`) is never tagged by a run, so without this
825
830
  # sweep it never reached the site. The site at that version or newer, no `downloads:` block, no
826
- # DOWNLOADS_URL, or an unreadable site: nothing is handed on, and the step stays green.
831
+ # DOWNLOADS_URL, or an unreadable site: nothing is handed on, and the step stays green — a 401/403
832
+ # (#714) too, but as an error annotation and a failing commit status on the final, not a warning.
827
833
  - name: Read the downloads target [publish]
828
834
  id: downloads-target
829
835
  if: env.DRY_RUN != 'true' && (steps.cut.outputs.tag != '' || steps.finalize.outputs.tag != '' || (github.event_name == 'schedule' && env.RETRY_RUN != 'true') || github.event_name == 'workflow_dispatch')
@@ -831,6 +837,8 @@ jobs:
831
837
  TAGS: ${{ steps.cut.outputs.tag }} ${{ steps.finalize.outputs.tag }}
832
838
  SWEEP: ${{ (github.event_name == 'schedule' && env.RETRY_RUN != 'true') || github.event_name == 'workflow_dispatch' }}
833
839
  DOWNLOADS_URL: ${{ vars.DOWNLOADS_URL }}
840
+ # #714: a site that requires sign-in for every file. Optional; unset, the read sends no credential.
841
+ DOWNLOADS_READ_HEADERS: ${{ secrets.DOWNLOADS_READ_HEADERS }}
834
842
  run: |
835
843
  set -uo pipefail
836
844
  # The `downloads:` block's `app:` — the same key publish-downloads.mjs reads. No block, no target.
@@ -871,7 +879,20 @@ jobs:
871
879
  { echo "tag=$FINAL"; echo "swept=$FINAL"; } >> "$GITHUB_OUTPUT"
872
880
  echo "downloads sweep: $1 — handing $FINAL to the build and downloads jobs (#669)" | tee -a "$GITHUB_STEP_SUMMARY"
873
881
  }
874
- if BODY="$(retry_transient curl -fsSL "$URL" 2> "$ERR")"; then
882
+ # #714: DOWNLOADS_READ_HEADERS — one `Name: value` header per line (docs/downloads.md, "A site
883
+ # that requires sign-in") — goes to curl from a file, never on its command line, and is never printed.
884
+ AUTH=()
885
+ if [ -n "${DOWNLOADS_READ_HEADERS:-}" ]; then
886
+ HDRS="$RUNNER_TEMP/downloads-read-headers"
887
+ trap 'rm -f "$HDRS"' EXIT
888
+ ( umask 077; printf '%s\n' "$DOWNLOADS_READ_HEADERS" | tr -d '\r' | sed '/^[[:space:]]*$/d' > "$HDRS" )
889
+ if grep -qvE '^[A-Za-z0-9!#$%&*+.^_`|~-]+:[[:space:]]*[^[:space:]]' "$HDRS"; then
890
+ echo "::error title=Downloads not shipped::the DOWNLOADS_READ_HEADERS secret has a line that is not a 'Name: value' header (the value is not shown) — $FINAL is not handed on"
891
+ exit 0 # a standing refusal, not a red run (the 401/403 branch below says why)
892
+ fi
893
+ AUTH=(-H "@$HDRS")
894
+ fi
895
+ if BODY="$(retry_transient curl -fsSL ${AUTH[@]+"${AUTH[@]}"} "$URL" 2> "$ERR")"; then
875
896
  SITE="$(jq -r '.version // empty' <<< "$BODY" 2> /dev/null || true)"
876
897
  if [ -z "$SITE" ]; then
877
898
  echo "::warning::downloads sweep: $URL names no version — $FINAL is not handed on in this run"
@@ -884,6 +905,29 @@ jobs:
884
905
  fi
885
906
  elif grep -q 'error: 404' "$ERR"; then
886
907
  hand_on "$FINAL is final on main and the site has no latest.json for $APP"
908
+ elif CODE="$(grep -oE 'error: 40[13]' "$ERR" | head -n 1 | grep -oE '40[13]')"; then
909
+ # #714: 401/403 is a refusal no later run clears by itself, so it is not the warning below: a
910
+ # final whose files would never ship. Visible, but exit 0 — the #467 rule (header, "A REFUSAL
911
+ # IS NOT A FAILURE"): a red run would make main's head not-green and refuse the very ship
912
+ # that fixes it. So: an error annotation, a summary banner, and a failing commit status
913
+ # `downloads/<app>` on the final's own commit. The next daily run re-reads and re-posts.
914
+ if [ -n "${DOWNLOADS_READ_HEADERS:-}" ]; then
915
+ WHY="the site refused the DOWNLOADS_READ_HEADERS credential — check or replace that secret"
916
+ else
917
+ WHY="the site requires sign-in and no DOWNLOADS_READ_HEADERS secret is set on this repo"
918
+ fi
919
+ MSG="$URL answered $CODE, so $FINAL is final on main but its files never ship: $WHY (docs/downloads.md, \"A site that requires sign-in\")"
920
+ echo "::error title=Downloads not shipped::downloads sweep: $MSG"
921
+ { echo "#### ⚠ Downloads not shipped — $FINAL"; echo; echo "$MSG"; } >> "$GITHUB_STEP_SUMMARY"
922
+ SHA="$(git rev-list -n 1 "$FINAL")"
923
+ # github.com only: a forge reads CI from commit statuses (#681), where a failing one would be the
924
+ # red main this branch exists to avoid. There the annotation stands alone.
925
+ if [ "${GITHUB_SERVER_URL:-https://github.com}" != "https://github.com" ]; then
926
+ :
927
+ elif ! retry_transient gh api --silent -X POST "repos/$GITHUB_REPOSITORY/statuses/$SHA" -f state=failure \
928
+ -f context="downloads/$APP" -f description="$FINAL not on the site: latest.json answered $CODE" > /dev/null; then
929
+ echo "::warning::downloads sweep: could not post the failing downloads/$APP status on $FINAL — the annotation above stands"
930
+ fi
887
931
  else
888
932
  echo "::warning::downloads sweep: could not read $URL from this runner ($(tail -n 1 "$ERR" 2> /dev/null)) — $FINAL is not handed on in this run; the next daily run retries"
889
933
  fi