@futurelastic/colab-handbook 1.12.0 → 1.13.0-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +248 -213
  2. package/README.vi.md +252 -186
  3. package/audit/audit.mjs +193 -10
  4. package/package.json +7 -1
  5. package/templates/README.md +30 -2
  6. package/templates/ci-laravel.yml +102 -21
  7. package/templates/ci-node.yml +61 -3
  8. package/templates/ci-python.yml +52 -3
  9. package/templates/deploy-adapter-portainer.mjs +206 -0
  10. package/templates/deploy-container-run.mjs +368 -0
  11. package/templates/deploy-container.yml +219 -0
  12. package/templates/dist-refs.yml +6 -0
  13. package/templates/docs-lint.mjs +1 -1
  14. package/templates/npx-launcher.mjs +118 -20
  15. package/templates/npx-service.mjs +828 -0
  16. package/templates/release-auto.yml +341 -32
  17. package/tools/colab +1902 -278
  18. package/tools/lib/adopt.js +373 -14
  19. package/tools/lib/branch-name.js +29 -1
  20. package/tools/lib/ci-cure.js +522 -31
  21. package/tools/lib/ci-grant.js +289 -22
  22. package/tools/lib/ci-wait.js +223 -0
  23. package/tools/lib/claim-comments.js +13 -64
  24. package/tools/lib/claim-identity.js +42 -7
  25. package/tools/lib/codec/claim.d.ts +41 -0
  26. package/tools/lib/codec/claim.js +153 -0
  27. package/tools/lib/codec/decision.d.ts +29 -0
  28. package/tools/lib/codec/decision.js +81 -0
  29. package/tools/lib/codec/grants.d.ts +82 -0
  30. package/tools/lib/codec/grants.js +268 -0
  31. package/tools/lib/codec/hold.d.ts +26 -0
  32. package/tools/lib/codec/hold.js +152 -0
  33. package/tools/lib/codec/index.d.ts +21 -0
  34. package/tools/lib/codec/index.js +32 -0
  35. package/tools/lib/codec/labels.d.ts +65 -0
  36. package/tools/lib/codec/labels.js +723 -0
  37. package/tools/lib/codec/markers.d.ts +35 -0
  38. package/tools/lib/codec/markers.js +138 -0
  39. package/tools/lib/codec/release.d.ts +17 -0
  40. package/tools/lib/codec/release.js +69 -0
  41. package/tools/lib/codec/samples.json +778 -0
  42. package/tools/lib/codeowners.js +68 -4
  43. package/tools/lib/container-close.js +38 -0
  44. package/tools/lib/cure-diff.js +198 -9
  45. package/tools/lib/decision-record.js +15 -39
  46. package/tools/lib/docs-only.js +15 -1
  47. package/tools/lib/exposure-shape.js +42 -8
  48. package/tools/lib/git.js +324 -42
  49. package/tools/lib/group-labels.js +65 -0
  50. package/tools/lib/install-check.js +14 -8
  51. package/tools/lib/install-route.js +139 -0
  52. package/tools/lib/instruction-file.js +1 -0
  53. package/tools/lib/labels.js +22 -647
  54. package/tools/lib/local-adopt.js +8 -4
  55. package/tools/lib/migration-grant.js +206 -104
  56. package/tools/lib/place.js +2 -2
  57. package/tools/lib/release-cut.js +199 -25
  58. package/tools/lib/release-finalize.js +111 -23
  59. package/tools/lib/release-policy.js +130 -22
  60. package/tools/lib/release-runner.js +125 -0
  61. package/tools/lib/release-tag.js +3 -1
  62. package/tools/lib/scratch-dirs.js +118 -0
  63. package/tools/lib/ship-batch.js +36 -1
  64. package/tools/lib/ship-human-door.js +76 -0
  65. package/tools/lib/stamp.js +7 -0
  66. package/tools/lib/tracker-fake.js +195 -0
  67. package/tools/lib/tracker-github.js +106 -0
  68. package/tools/lib/tracker-scenarios.json +206 -0
  69. package/tools/lib/tracker.js +64 -0
  70. package/tools/lib/tree-green.js +88 -0
  71. package/tools/lib/util.js +17 -1
  72. package/tools/lib/verify-runs.js +104 -0
  73. package/tools/lib/wake.js +8 -90
  74. package/tools/lib/workflow-triggers.js +48 -0
@@ -45,8 +45,8 @@
45
45
  # The last two bullets fire ONLY when rows exist — on an empty table every one of
46
46
  # them passes on both engines. Set ROUNDTRIP_SEEDER below, or they go unchecked.
47
47
  #
48
- # Not gated on RUN_TESTS: a bad migration is cheapest to fix on the branch that adds
49
- # it, so this runs on every push, not only on main.
48
+ # Not gated on RUN_TESTS: a bad migration is cheapest to fix on the branch that adds or
49
+ # modifies it, so this runs on every push, not only on main.
50
50
 
51
51
  name: CI
52
52
 
@@ -119,28 +119,76 @@ jobs:
119
119
  # Why not simply `if: !github.event.created` on every job: a run whose jobs are ALL
120
120
  # skipped concludes `skipped`, which `colab ship` reads as not-green (CONVENTIONS.md
121
121
  # §4, "Branch CI"). This job is the one that ends green.
122
+ #
123
+ # Already tested at this TREE? (#493) — the squash merge.
124
+ # A ship squash-merges a branch whose green run already contained the current base, so
125
+ # the trunk commit's TREE is byte-identical to the tree that branch run passed — only the
126
+ # sha is new. On a non-creating push to trunk this job therefore also asks: does THIS
127
+ # workflow have a green `push` run, on another ref of this repo, whose commit has THIS
128
+ # exact tree? It confirms the tree through the git object, never only the run listing.
129
+ # Yes → the suite is skipped and the run cites the run it relied on (a
130
+ # `tree-already-green` notice that `colab ship` / `colab trunk-ci` read back). Equal
131
+ # trees cover every committed byte, workflows and lockfiles included — NOT runner images,
132
+ # unpinned downloads, secrets or `github.ref`-conditional steps; a repo whose suite
133
+ # depends on those declares `tree-reuse: off` in .github/project.yml. Trunk only (the
134
+ # descriptor's `trunk:`, read at this sha), never a release branch, never a dispatch;
135
+ # any doubt (no descriptor, opt-out, an API error, no exact match) runs the full suite.
136
+ # Never put a trunk-only job (publish, deploy, release) behind this gate.
122
137
  dedupe:
123
- name: Already tested at this sha?
124
- if: github.event_name == 'push' && github.event.created
138
+ name: Already tested at this sha or tree?
139
+ # EDIT: the trunk refs here mirror `concurrency`'s list above (#493).
140
+ if: github.event_name == 'push' && (github.event.created || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev')
125
141
  runs-on: ubuntu-latest # EDIT: the same label as the jobs below; it needs `gh` (GitHub-hosted images have it).
126
142
  permissions:
127
143
  actions: read
144
+ contents: read
128
145
  outputs:
129
146
  tested: ${{ steps.check.outputs.tested }}
147
+ cited: ${{ steps.check.outputs.cited }}
130
148
  steps:
131
- - name: Look for a green run of this workflow at this sha
149
+ - name: Look for a green run of this workflow at this sha or tree
132
150
  id: check
133
151
  env:
134
152
  GH_TOKEN: ${{ github.token }}
135
153
  REPO: ${{ github.repository }}
136
154
  RUN_ID: ${{ github.run_id }}
137
155
  SHA: ${{ github.sha }}
156
+ CREATED: ${{ github.event.created }}
157
+ REF_NAME: ${{ github.ref_name }}
138
158
  run: |
139
159
  tested=false
160
+ cited=
140
161
  if ! command -v gh >/dev/null 2>&1; then
141
162
  echo "::notice::gh is not installed on this runner — running the full suite (#418)."
142
163
  elif ! WFID="$(gh api "repos/$REPO/actions/runs/$RUN_ID" --jq .workflow_id 2>/dev/null)" || [ -z "$WFID" ]; then
143
164
  echo "::notice::Could not read this run's workflow id — running the full suite (#418)."
165
+ elif [ "${CREATED:-}" = "false" ]; then
166
+ # A non-creating push: trunk tree mode (#493). Every branch below but one runs the suite.
167
+ if ! DESC="$(gh api "repos/$REPO/contents/.github/project.yml?ref=$SHA" -H 'Accept: application/vnd.github.raw' 2>/dev/null)" || [ -z "$DESC" ]; then
168
+ echo "::notice::Could not read .github/project.yml at $SHA — running the full suite (#493)."
169
+ elif grep -Eq '^tree-reuse[[:space:]]*:' <<< "$DESC"; then
170
+ echo "::notice::tree-reuse is declared in .github/project.yml — running the full suite (#493)."
171
+ elif [ "$(printf '%s\n' "$DESC" | sed -nE "s/^trunk:[[:space:]]*[\"']?([A-Za-z0-9._/-]+).*/\\1/p" | head -n 1)" != "$REF_NAME" ]; then
172
+ echo "::notice::$REF_NAME is not this repo's trunk — running the full suite (#493)."
173
+ elif ! OURS="$(gh api "repos/$REPO/git/commits/$SHA" --jq .tree.sha 2>/dev/null)" || [ -z "$OURS" ]; then
174
+ echo "::notice::Could not read the tree of $SHA — running the full suite (#493)."
175
+ elif ! CAND="$(gh api "repos/$REPO/actions/workflows/$WFID/runs?event=push&status=success&per_page=50" \
176
+ --jq "first(.workflow_runs[] | select(.id != $RUN_ID and .head_branch != \"$REF_NAME\" and .head_repository.full_name == \"$REPO\" and .head_commit.tree_id == \"$OURS\") | \"\\(.head_sha) \\(.html_url) \\(.head_branch)\") // empty" 2>/dev/null)"; then
177
+ echo "::notice::Could not list this workflow's green runs — running the full suite (#493)."
178
+ elif [ -z "$CAND" ]; then
179
+ echo "No green run of this workflow on another ref has tree $OURS — running the full suite."
180
+ else
181
+ read -r CSHA URL BR <<< "$CAND"
182
+ if ! CTREE="$(gh api "repos/$REPO/git/commits/$CSHA" --jq .tree.sha 2>/dev/null)" || [ "$CTREE" != "$OURS" ]; then
183
+ echo "::notice::Could not confirm the tree of $CSHA — running the full suite (#493)."
184
+ else
185
+ tested=true
186
+ cited="$URL"
187
+ echo "::notice title=tree-already-green::$URL tree=$OURS head=$CSHA branch=$BR"
188
+ echo "Tree $OURS already passed this workflow at $CSHA on $BR ($URL) — skipping the suite (#493)."
189
+ echo "Already tested: tree \`$OURS\` passed this workflow on \`$BR\` — [the run relied on]($URL); suite skipped (#493)." >> "$GITHUB_STEP_SUMMARY"
190
+ fi
191
+ fi
144
192
  elif ! N="$(gh api "repos/$REPO/actions/workflows/$WFID/runs?head_sha=$SHA&status=success&per_page=20" \
145
193
  --jq "[.workflow_runs[] | select(.id != $RUN_ID and .event != \"pull_request\")] | length" 2>/dev/null)"; then
146
194
  echo "::notice::Could not list this workflow's runs at $SHA — running the full suite (#418)."
@@ -152,6 +200,7 @@ jobs:
152
200
  echo "No green run of this workflow at $SHA yet — running the full suite."
153
201
  fi
154
202
  echo "tested=$tested" >> "$GITHUB_OUTPUT"
203
+ [ -z "$cited" ] || echo "cited=$cited" >> "$GITHUB_OUTPUT"
155
204
 
156
205
  gitleaks:
157
206
  name: Secret scan (gitleaks)
@@ -412,14 +461,19 @@ jobs:
412
461
  # Migration round-trip — see the header for what each engine catches.
413
462
  #
414
463
  # 1. migrate from zero (schema snapshot A)
415
- # 2. migrate:rollback the migrations THIS change adds — every down() among them
416
- # runs (all migrations when the change adds none)
464
+ # 2. migrate:rollback back to the OLDEST migration this change adds OR modifies —
465
+ # every down() it touched runs, plus the untouched ones after
466
+ # it (all migrations when the change touches none)
417
467
  # 3. seed only if ROUNDTRIP_SEEDER is set, and only when the change
418
- # adds migrations: rows land at the PRE-change schema
419
- # 4. migrate again — the new migrations land on those rows (snapshot B)
468
+ # touches migrations: rows land at the PRE-change schema
469
+ # 4. migrate again — the touched migrations land on those rows (snapshot B)
420
470
  # 5. compare B must equal A — a down() that does not exactly undo its
421
471
  # up() leaves a schema that differs on the second pass
422
472
  #
473
+ # MODIFIED counts as touched, not only added (#507): a branch that only repairs an
474
+ # existing down() must see that down() run, and a branch that adds one migration and
475
+ # repairs an older one must roll back past the older one, not just the newest step.
476
+ #
423
477
  # Steps 2-4 are the shape of a real deploy: new migrations landing on tables that
424
478
  # already hold rows written before they existed. That is why the seed sits BETWEEN
425
479
  # the rollback and the second migrate — rows seeded after the new migrations would
@@ -484,7 +538,7 @@ jobs:
484
538
  steps:
485
539
  - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
486
540
  with:
487
- # Full history: step 3 counts the migrations this change adds against trunk.
541
+ # Full history: the count step diffs the migrations this change touches against trunk.
488
542
  fetch-depth: 0
489
543
  persist-credentials: false
490
544
 
@@ -531,7 +585,7 @@ jobs:
531
585
  - name: Discover packages
532
586
  run: php artisan package:discover --ansi
533
587
 
534
- - name: Count the migrations this change adds
588
+ - name: Find the migrations this change adds or modifies
535
589
  env:
536
590
  DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
537
591
  run: |
@@ -548,15 +602,24 @@ jobs:
548
602
  base="$(git rev-parse -q --verify 'HEAD^' || true)"
549
603
  fi
550
604
  fi
551
- steps=0
605
+ # Added, Modified, and Renamed (a rename is a new name to the migrator) — not
606
+ # added only (#507): a branch that only repairs an existing down() got a green
607
+ # round-trip that never ran it.
608
+ touched=""
552
609
  if [ -n "$base" ]; then
553
- steps="$(git diff --name-only --diff-filter=A "$base" HEAD -- "$MIGRATIONS_PATH" | grep -c '\.php$' || true)"
610
+ touched="$(git diff --name-status --diff-filter=AMR "$base" HEAD -- "$MIGRATIONS_PATH" | awk -F'\t' '$NF ~ /\.php$/ { print substr($1,1,1) "\t" $NF }')"
554
611
  fi
555
- echo "ROLLBACK_STEPS=$steps" >> "$GITHUB_ENV"
556
- if [ "$steps" -gt 0 ]; then
557
- echo "This change adds $steps migration(s) (vs ${base_ref}); step 3 rolls back exactly those."
612
+ # The migrator orders by file name, so the oldest touched one is the smallest
613
+ # base name; step 2 rolls back to it and everything after it.
614
+ from="$(printf '%s\n' "$touched" | awk -F'\t' 'NF { n=$2; sub(/.*\//, "", n); sub(/\.php$/, "", n); print n }' | LC_ALL=C sort | head -1)"
615
+ echo "ROUNDTRIP_FROM=$from" >> "$GITHUB_ENV"
616
+ if [ -n "$from" ]; then
617
+ added="$(printf '%s\n' "$touched" | grep -c '^A' || true)"
618
+ modified="$(printf '%s\n' "$touched" | grep -c '^[MR]' || true)"
619
+ echo "This change touches migrations (vs ${base_ref}): $added added, $modified modified. Step 2 rolls back to the oldest, $from, so every down() listed below runs:"
620
+ printf '%s\n' "$touched" | sed 's/^A\t/ added /; s/^[MR]\t/ modified /'
558
621
  else
559
- echo "This change adds no migrations (vs ${base_ref:-nothing}); step 3 rolls back all of them."
622
+ echo "This change adds or modifies no migrations (vs ${base_ref:-nothing}); step 2 rolls back all of them."
560
623
  fi
561
624
 
562
625
  - name: Write the schema snapshot script
@@ -588,6 +651,18 @@ jobs:
588
651
  ksort($out);
589
652
  echo json_encode($out, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), "\n";
590
653
  PHP
654
+ # How many migrations step 2 must roll back to reach ROUNDTRIP_FROM: the rows of
655
+ # the migrations table at or after it. Read from the table, not by counting
656
+ # files, so migrations a package registers from another path are counted too.
657
+ cat > "$RUNNER_TEMP/rollback-steps.php" <<'PHP'
658
+ <?php
659
+ require getcwd() . '/vendor/autoload.php';
660
+ $app = require getcwd() . '/bootstrap/app.php';
661
+ $app->make(Illuminate\Contracts\Console\Kernel::class)->bootstrap();
662
+ $m = config('database.migrations');
663
+ $table = is_array($m) ? ($m['table'] ?? 'migrations') : ($m ?: 'migrations');
664
+ echo Illuminate\Support\Facades\DB::table($table)->where('migration', '>=', getenv('ROUNDTRIP_FROM'))->count(), "\n";
665
+ PHP
591
666
 
592
667
  - name: 1. migrate from zero
593
668
  run: |
@@ -596,16 +671,22 @@ jobs:
596
671
 
597
672
  - name: 2. roll back
598
673
  run: |
599
- if [ "$ROLLBACK_STEPS" -gt 0 ]; then
600
- php artisan migrate:rollback --step="$ROLLBACK_STEPS" --force
674
+ set -euo pipefail
675
+ steps=0
676
+ if [ -n "$ROUNDTRIP_FROM" ]; then
677
+ steps="$(php "$RUNNER_TEMP/rollback-steps.php")"
678
+ fi
679
+ if [ "$steps" -gt 0 ]; then
680
+ echo "Rolling back $steps migration(s), down to $ROUNDTRIP_FROM."
681
+ php artisan migrate:rollback --step="$steps" --force
601
682
  else
602
683
  # Everything ran in one batch at step 1, so this rolls all of it back.
603
684
  php artisan migrate:rollback --force
604
685
  fi
605
686
 
606
687
  - name: 3. seed rows at the pre-change schema
607
- # No new migrations = nothing lands on the rows, and the schema is empty anyway.
608
- if: env.ROUNDTRIP_SEEDER != '' && env.ROLLBACK_STEPS != '0'
688
+ # No touched migrations = nothing lands on the rows, and the schema is empty anyway.
689
+ if: env.ROUNDTRIP_SEEDER != '' && env.ROUNDTRIP_FROM != ''
609
690
  run: php artisan db:seed --class="$ROUNDTRIP_SEEDER" --force
610
691
 
611
692
  - name: 4. migrate again
@@ -84,28 +84,76 @@ jobs:
84
84
  # Why not simply `if: !github.event.created` on every job: a run whose jobs are ALL
85
85
  # skipped concludes `skipped`, which `colab ship` reads as not-green (CONVENTIONS.md
86
86
  # §4, "Branch CI"). This job is the one that ends green.
87
+ #
88
+ # Already tested at this TREE? (#493) — the squash merge.
89
+ # A ship squash-merges a branch whose green run already contained the current base, so
90
+ # the trunk commit's TREE is byte-identical to the tree that branch run passed — only the
91
+ # sha is new. On a non-creating push to trunk this job therefore also asks: does THIS
92
+ # workflow have a green `push` run, on another ref of this repo, whose commit has THIS
93
+ # exact tree? It confirms the tree through the git object, never only the run listing.
94
+ # Yes → the suite is skipped and the run cites the run it relied on (a
95
+ # `tree-already-green` notice that `colab ship` / `colab trunk-ci` read back). Equal
96
+ # trees cover every committed byte, workflows and lockfiles included — NOT runner images,
97
+ # unpinned downloads, secrets or `github.ref`-conditional steps; a repo whose suite
98
+ # depends on those declares `tree-reuse: off` in .github/project.yml. Trunk only (the
99
+ # descriptor's `trunk:`, read at this sha), never a release branch, never a dispatch;
100
+ # any doubt (no descriptor, opt-out, an API error, no exact match) runs the full suite.
101
+ # Never put a trunk-only job (publish, deploy, release) behind this gate.
87
102
  dedupe:
88
- name: Already tested at this sha?
89
- if: github.event_name == 'push' && github.event.created
103
+ name: Already tested at this sha or tree?
104
+ # EDIT: the trunk refs here mirror `concurrency`'s list above (#493).
105
+ if: github.event_name == 'push' && (github.event.created || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev')
90
106
  runs-on: ubuntu-latest # EDIT: the same label as the jobs below; it needs `gh` (GitHub-hosted images have it).
91
107
  permissions:
92
108
  actions: read
109
+ contents: read
93
110
  outputs:
94
111
  tested: ${{ steps.check.outputs.tested }}
112
+ cited: ${{ steps.check.outputs.cited }}
95
113
  steps:
96
- - name: Look for a green run of this workflow at this sha
114
+ - name: Look for a green run of this workflow at this sha or tree
97
115
  id: check
98
116
  env:
99
117
  GH_TOKEN: ${{ github.token }}
100
118
  REPO: ${{ github.repository }}
101
119
  RUN_ID: ${{ github.run_id }}
102
120
  SHA: ${{ github.sha }}
121
+ CREATED: ${{ github.event.created }}
122
+ REF_NAME: ${{ github.ref_name }}
103
123
  run: |
104
124
  tested=false
125
+ cited=
105
126
  if ! command -v gh >/dev/null 2>&1; then
106
127
  echo "::notice::gh is not installed on this runner — running the full suite (#418)."
107
128
  elif ! WFID="$(gh api "repos/$REPO/actions/runs/$RUN_ID" --jq .workflow_id 2>/dev/null)" || [ -z "$WFID" ]; then
108
129
  echo "::notice::Could not read this run's workflow id — running the full suite (#418)."
130
+ elif [ "${CREATED:-}" = "false" ]; then
131
+ # A non-creating push: trunk tree mode (#493). Every branch below but one runs the suite.
132
+ if ! DESC="$(gh api "repos/$REPO/contents/.github/project.yml?ref=$SHA" -H 'Accept: application/vnd.github.raw' 2>/dev/null)" || [ -z "$DESC" ]; then
133
+ echo "::notice::Could not read .github/project.yml at $SHA — running the full suite (#493)."
134
+ elif grep -Eq '^tree-reuse[[:space:]]*:' <<< "$DESC"; then
135
+ echo "::notice::tree-reuse is declared in .github/project.yml — running the full suite (#493)."
136
+ elif [ "$(printf '%s\n' "$DESC" | sed -nE "s/^trunk:[[:space:]]*[\"']?([A-Za-z0-9._/-]+).*/\\1/p" | head -n 1)" != "$REF_NAME" ]; then
137
+ echo "::notice::$REF_NAME is not this repo's trunk — running the full suite (#493)."
138
+ elif ! OURS="$(gh api "repos/$REPO/git/commits/$SHA" --jq .tree.sha 2>/dev/null)" || [ -z "$OURS" ]; then
139
+ echo "::notice::Could not read the tree of $SHA — running the full suite (#493)."
140
+ elif ! CAND="$(gh api "repos/$REPO/actions/workflows/$WFID/runs?event=push&status=success&per_page=50" \
141
+ --jq "first(.workflow_runs[] | select(.id != $RUN_ID and .head_branch != \"$REF_NAME\" and .head_repository.full_name == \"$REPO\" and .head_commit.tree_id == \"$OURS\") | \"\\(.head_sha) \\(.html_url) \\(.head_branch)\") // empty" 2>/dev/null)"; then
142
+ echo "::notice::Could not list this workflow's green runs — running the full suite (#493)."
143
+ elif [ -z "$CAND" ]; then
144
+ echo "No green run of this workflow on another ref has tree $OURS — running the full suite."
145
+ else
146
+ read -r CSHA URL BR <<< "$CAND"
147
+ if ! CTREE="$(gh api "repos/$REPO/git/commits/$CSHA" --jq .tree.sha 2>/dev/null)" || [ "$CTREE" != "$OURS" ]; then
148
+ echo "::notice::Could not confirm the tree of $CSHA — running the full suite (#493)."
149
+ else
150
+ tested=true
151
+ cited="$URL"
152
+ echo "::notice title=tree-already-green::$URL tree=$OURS head=$CSHA branch=$BR"
153
+ echo "Tree $OURS already passed this workflow at $CSHA on $BR ($URL) — skipping the suite (#493)."
154
+ echo "Already tested: tree \`$OURS\` passed this workflow on \`$BR\` — [the run relied on]($URL); suite skipped (#493)." >> "$GITHUB_STEP_SUMMARY"
155
+ fi
156
+ fi
109
157
  elif ! N="$(gh api "repos/$REPO/actions/workflows/$WFID/runs?head_sha=$SHA&status=success&per_page=20" \
110
158
  --jq "[.workflow_runs[] | select(.id != $RUN_ID and .event != \"pull_request\")] | length" 2>/dev/null)"; then
111
159
  echo "::notice::Could not list this workflow's runs at $SHA — running the full suite (#418)."
@@ -117,6 +165,7 @@ jobs:
117
165
  echo "No green run of this workflow at $SHA yet — running the full suite."
118
166
  fi
119
167
  echo "tested=$tested" >> "$GITHUB_OUTPUT"
168
+ [ -z "$cited" ] || echo "cited=$cited" >> "$GITHUB_OUTPUT"
120
169
 
121
170
  gitleaks:
122
171
  name: Secret scan (gitleaks)
@@ -361,6 +410,15 @@ jobs:
361
410
  if: steps.scripts.outputs.lint == 'true'
362
411
  run: npm run lint
363
412
 
413
+ # EDIT: cap your test runner's workers in CI, in its own config (#478). vitest,
414
+ # jest, node:test and playwright start one worker per CPU the job can SEE. On a
415
+ # self-hosted pool slot that is often the whole host's count, while the slot
416
+ # caps only memory. A host upgrade then adds workers until the slot is
417
+ # throttled. There is no OOM, the tests time out, and trunk goes red with no
418
+ # code change. Use a fixed number sized to the slot's memory, e.g. vitest
419
+ # `maxWorkers: process.env.CI ? 4 : undefined`, jest `--maxWorkers=4`,
420
+ # node:test `--test-concurrency=4`. GitHub Actions sets CI=true. Why and how
421
+ # to size it: CONVENTIONS.md §7, "Self-hosted runners".
364
422
  - name: Test
365
423
  if: steps.scripts.outputs.test == 'true'
366
424
  run: npm test
@@ -98,28 +98,76 @@ jobs:
98
98
  # Why not simply `if: !github.event.created` on every job: a run whose jobs are ALL
99
99
  # skipped concludes `skipped`, which `colab ship` reads as not-green (CONVENTIONS.md
100
100
  # §4, "Branch CI"). This job is the one that ends green.
101
+ #
102
+ # Already tested at this TREE? (#493) — the squash merge.
103
+ # A ship squash-merges a branch whose green run already contained the current base, so
104
+ # the trunk commit's TREE is byte-identical to the tree that branch run passed — only the
105
+ # sha is new. On a non-creating push to trunk this job therefore also asks: does THIS
106
+ # workflow have a green `push` run, on another ref of this repo, whose commit has THIS
107
+ # exact tree? It confirms the tree through the git object, never only the run listing.
108
+ # Yes → the suite is skipped and the run cites the run it relied on (a
109
+ # `tree-already-green` notice that `colab ship` / `colab trunk-ci` read back). Equal
110
+ # trees cover every committed byte, workflows and lockfiles included — NOT runner images,
111
+ # unpinned downloads, secrets or `github.ref`-conditional steps; a repo whose suite
112
+ # depends on those declares `tree-reuse: off` in .github/project.yml. Trunk only (the
113
+ # descriptor's `trunk:`, read at this sha), never a release branch, never a dispatch;
114
+ # any doubt (no descriptor, opt-out, an API error, no exact match) runs the full suite.
115
+ # Never put a trunk-only job (publish, deploy, release) behind this gate.
101
116
  dedupe:
102
- name: Already tested at this sha?
103
- if: github.event_name == 'push' && github.event.created
117
+ name: Already tested at this sha or tree?
118
+ # EDIT: the trunk refs here mirror `concurrency`'s list above (#493).
119
+ if: github.event_name == 'push' && (github.event.created || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev')
104
120
  runs-on: ubuntu-latest # EDIT: the same label as the jobs below; it needs `gh` (GitHub-hosted images have it).
105
121
  permissions:
106
122
  actions: read
123
+ contents: read
107
124
  outputs:
108
125
  tested: ${{ steps.check.outputs.tested }}
126
+ cited: ${{ steps.check.outputs.cited }}
109
127
  steps:
110
- - name: Look for a green run of this workflow at this sha
128
+ - name: Look for a green run of this workflow at this sha or tree
111
129
  id: check
112
130
  env:
113
131
  GH_TOKEN: ${{ github.token }}
114
132
  REPO: ${{ github.repository }}
115
133
  RUN_ID: ${{ github.run_id }}
116
134
  SHA: ${{ github.sha }}
135
+ CREATED: ${{ github.event.created }}
136
+ REF_NAME: ${{ github.ref_name }}
117
137
  run: |
118
138
  tested=false
139
+ cited=
119
140
  if ! command -v gh >/dev/null 2>&1; then
120
141
  echo "::notice::gh is not installed on this runner — running the full suite (#418)."
121
142
  elif ! WFID="$(gh api "repos/$REPO/actions/runs/$RUN_ID" --jq .workflow_id 2>/dev/null)" || [ -z "$WFID" ]; then
122
143
  echo "::notice::Could not read this run's workflow id — running the full suite (#418)."
144
+ elif [ "${CREATED:-}" = "false" ]; then
145
+ # A non-creating push: trunk tree mode (#493). Every branch below but one runs the suite.
146
+ if ! DESC="$(gh api "repos/$REPO/contents/.github/project.yml?ref=$SHA" -H 'Accept: application/vnd.github.raw' 2>/dev/null)" || [ -z "$DESC" ]; then
147
+ echo "::notice::Could not read .github/project.yml at $SHA — running the full suite (#493)."
148
+ elif grep -Eq '^tree-reuse[[:space:]]*:' <<< "$DESC"; then
149
+ echo "::notice::tree-reuse is declared in .github/project.yml — running the full suite (#493)."
150
+ elif [ "$(printf '%s\n' "$DESC" | sed -nE "s/^trunk:[[:space:]]*[\"']?([A-Za-z0-9._/-]+).*/\\1/p" | head -n 1)" != "$REF_NAME" ]; then
151
+ echo "::notice::$REF_NAME is not this repo's trunk — running the full suite (#493)."
152
+ elif ! OURS="$(gh api "repos/$REPO/git/commits/$SHA" --jq .tree.sha 2>/dev/null)" || [ -z "$OURS" ]; then
153
+ echo "::notice::Could not read the tree of $SHA — running the full suite (#493)."
154
+ elif ! CAND="$(gh api "repos/$REPO/actions/workflows/$WFID/runs?event=push&status=success&per_page=50" \
155
+ --jq "first(.workflow_runs[] | select(.id != $RUN_ID and .head_branch != \"$REF_NAME\" and .head_repository.full_name == \"$REPO\" and .head_commit.tree_id == \"$OURS\") | \"\\(.head_sha) \\(.html_url) \\(.head_branch)\") // empty" 2>/dev/null)"; then
156
+ echo "::notice::Could not list this workflow's green runs — running the full suite (#493)."
157
+ elif [ -z "$CAND" ]; then
158
+ echo "No green run of this workflow on another ref has tree $OURS — running the full suite."
159
+ else
160
+ read -r CSHA URL BR <<< "$CAND"
161
+ if ! CTREE="$(gh api "repos/$REPO/git/commits/$CSHA" --jq .tree.sha 2>/dev/null)" || [ "$CTREE" != "$OURS" ]; then
162
+ echo "::notice::Could not confirm the tree of $CSHA — running the full suite (#493)."
163
+ else
164
+ tested=true
165
+ cited="$URL"
166
+ echo "::notice title=tree-already-green::$URL tree=$OURS head=$CSHA branch=$BR"
167
+ echo "Tree $OURS already passed this workflow at $CSHA on $BR ($URL) — skipping the suite (#493)."
168
+ echo "Already tested: tree \`$OURS\` passed this workflow on \`$BR\` — [the run relied on]($URL); suite skipped (#493)." >> "$GITHUB_STEP_SUMMARY"
169
+ fi
170
+ fi
123
171
  elif ! N="$(gh api "repos/$REPO/actions/workflows/$WFID/runs?head_sha=$SHA&status=success&per_page=20" \
124
172
  --jq "[.workflow_runs[] | select(.id != $RUN_ID and .event != \"pull_request\")] | length" 2>/dev/null)"; then
125
173
  echo "::notice::Could not list this workflow's runs at $SHA — running the full suite (#418)."
@@ -131,6 +179,7 @@ jobs:
131
179
  echo "No green run of this workflow at $SHA yet — running the full suite."
132
180
  fi
133
181
  echo "tested=$tested" >> "$GITHUB_OUTPUT"
182
+ [ -z "$cited" ] || echo "cited=$cited" >> "$GITHUB_OUTPUT"
134
183
 
135
184
  gitleaks:
136
185
  name: Secret scan (gitleaks)
@@ -0,0 +1,206 @@
1
+ #!/usr/bin/env node
2
+ // deploy adapter (Portainer) — TEMPLATE. Copy me into your repo next to deploy-container-run.mjs
3
+ // (`colab template deploy-adapter-portainer.mjs --dest .github/deploy/deploy-adapter-portainer.mjs`)
4
+ // and OWN the copy, exactly like the workflow templates (templates/README.md). Nothing here calls
5
+ // back to the handbook, and nothing here updates itself.
6
+ //
7
+ // WHAT IT IS. One platform adapter behind templates/deploy-container.yml (colab-handbook #452). The
8
+ // driver (deploy-container-run.mjs) builds the images and decides WHAT runs; an adapter only tells
9
+ // ONE platform "run exactly this tag" and proves that it did. A second platform is a second file
10
+ // exporting the same five names (`name`, `configFromEnv`, `snapshot`, `deploy`, `restore`) — the
11
+ // consuming workflow then changes only DEPLOY_ADAPTER, never its steps.
12
+ //
13
+ // TOPOLOGY. Every server runs its own Portainer: no central panel, no Portainer Agents. So the
14
+ // whole per-app configuration is THAT host's Portainer URL + endpointId + stack id + one key:
15
+ //
16
+ // PORTAINER_URL https://portainer.app.example.com — a DNS name, never an address
17
+ // PORTAINER_ENDPOINT_ID the environment (endpoint) id the stack lives in
18
+ // PORTAINER_STACK_ID the stack's id
19
+ // PORTAINER_STACK_KIND git (default) | file
20
+ // PORTAINER_COMPOSE_FILE file stacks only: the compose file sent as the stack content
21
+ // PORTAINER_API_KEY the secret — an access token of a NON-ADMIN user that owns only this
22
+ // app's stack. Never an admin key: a leaked deploy key must reach one app.
23
+ //
24
+ // MEASURED (Portainer CE 2.45.1, git stacks):
25
+ // - `PUT /api/stacks/{id}/git/redeploy` honours `RepositoryReferenceName: refs/tags/vX.Y.Z`.
26
+ // - 200 only means ACCEPTED: the redeploy is asynchronous. `Status` is 3 while deploying and 1
27
+ // once active; `GitConfig.ConfigHash` then holds the commit actually deployed.
28
+ // - 409 = a deploy is already in progress (busy: wait, retry). 500 = a hard failure, returned
29
+ // synchronously, and nothing changed (e.g. a tag that does not exist).
30
+ // - `Env` sent on redeploy REPLACES the stack's whole list — so this sends the full current list
31
+ // with IMAGE_TAG replaced, never only the changed key.
32
+ // - EVERY REDEPLOY RECREATES THE CONTAINERS, even when the compose file did not change: expect a
33
+ // short restart on every deploy, including a rollback.
34
+ // - A call can return 200 while the environment refused the container (e.g. a bind mount blocked
35
+ // for a non-admin user). So success is read from what RUNS — the running containers' image —
36
+ // never from the HTTP status.
37
+ // NOT MEASURED (assumptions, each a place to check first if your deploy misbehaves): file-stack
38
+ // `PUT /api/stacks/{id}` being asynchronous too (this polls either way); a non-admin, stack-owning
39
+ // key being allowed to list its stack's containers through the Docker proxy; private-repository
40
+ // auth on redeploy (this reuses a stored git credential id, and refuses otherwise).
41
+
42
+ export const name = 'portainer';
43
+
44
+ export class AdapterError extends Error {
45
+ /** kind: 'hard' (refused synchronously, nothing changed) | 'busy' (a deploy never freed up) |
46
+ * 'refused' (accepted, but what runs is not what was asked) | 'config'. */
47
+ constructor(kind, message, { changed = false } = {}) {
48
+ super(message);
49
+ this.kind = kind;
50
+ this.changed = changed;
51
+ }
52
+ }
53
+
54
+ const STATUS_ACTIVE = 1;
55
+ const STATUS_DEPLOYING = 3;
56
+
57
+ /** The adapter's configuration from env; throws naming every missing key. */
58
+ export function configFromEnv(env) {
59
+ const need = ['PORTAINER_URL', 'PORTAINER_ENDPOINT_ID', 'PORTAINER_STACK_ID', 'PORTAINER_API_KEY'];
60
+ const missing = need.filter((k) => !String(env[k] || '').trim());
61
+ if (missing.length) throw new AdapterError('config', `portainer adapter: missing ${missing.join(', ')}`);
62
+ const kind = String(env.PORTAINER_STACK_KIND || 'git').trim();
63
+ if (!['git', 'file'].includes(kind)) throw new AdapterError('config', `PORTAINER_STACK_KIND is "${kind}", expected git or file`);
64
+ return {
65
+ url: String(env.PORTAINER_URL).trim().replace(/\/+$/, ''),
66
+ endpointId: String(env.PORTAINER_ENDPOINT_ID).trim(),
67
+ stackId: String(env.PORTAINER_STACK_ID).trim(),
68
+ kind,
69
+ composeFile: String(env.PORTAINER_COMPOSE_FILE || 'docker-compose.yml').trim(),
70
+ apiKey: String(env.PORTAINER_API_KEY).trim(),
71
+ };
72
+ }
73
+
74
+ /** The full env list with every `kv` key replaced in place or appended — never a partial list. */
75
+ export function mergeEnv(list, kv) {
76
+ const out = (list || []).map((e) => ({ name: e.name, value: e.value }));
77
+ for (const [k, v] of Object.entries(kv)) {
78
+ const at = out.findIndex((e) => e.name === k);
79
+ if (at >= 0) out[at] = { name: k, value: v };
80
+ else out.push({ name: k, value: v });
81
+ }
82
+ return out;
83
+ }
84
+
85
+ async function call(cfg, io, method, path, body) {
86
+ const res = await io.fetch(`${cfg.url}${path}`, {
87
+ method,
88
+ headers: { 'X-API-Key': cfg.apiKey, ...(body === undefined ? {} : { 'Content-Type': 'application/json' }) },
89
+ body: body === undefined ? undefined : JSON.stringify(body),
90
+ });
91
+ const text = await res.text();
92
+ let json = null;
93
+ try { json = text ? JSON.parse(text) : null; } catch { json = null; }
94
+ return { status: res.status, json, text };
95
+ }
96
+
97
+ async function getStack(cfg, io) {
98
+ const r = await call(cfg, io, 'GET', `/api/stacks/${cfg.stackId}`);
99
+ if (r.status !== 200 || !r.json) throw new AdapterError('hard', `GET /api/stacks/${cfg.stackId} answered ${r.status}: ${r.text.slice(0, 200)}`);
100
+ return r.json;
101
+ }
102
+
103
+ /** What the stack runs now — the rollback target and the base of every Env sent. */
104
+ export async function snapshot(cfg, io) {
105
+ const s = await getStack(cfg, io);
106
+ const snap = {
107
+ kind: cfg.kind,
108
+ name: s.Name,
109
+ ref: (s.GitConfig && s.GitConfig.ReferenceName) || null,
110
+ configHash: (s.GitConfig && s.GitConfig.ConfigHash) || null,
111
+ gitCredentialId: (s.GitConfig && s.GitConfig.Authentication && s.GitConfig.Authentication.GitCredentialID) || 0,
112
+ privateRepo: Boolean(s.GitConfig && s.GitConfig.Authentication),
113
+ env: (s.Env || []).map((e) => ({ name: e.name, value: e.value })),
114
+ fileContent: null,
115
+ };
116
+ if (cfg.kind === 'file') {
117
+ const f = await call(cfg, io, 'GET', `/api/stacks/${cfg.stackId}/file`);
118
+ if (f.status !== 200 || !f.json) throw new AdapterError('hard', `GET /api/stacks/${cfg.stackId}/file answered ${f.status}`);
119
+ snap.fileContent = f.json.StackFileContent;
120
+ }
121
+ return snap;
122
+ }
123
+
124
+ function requestFor(cfg, snap, { ref, env, fileContent }) {
125
+ const q = `?endpointId=${encodeURIComponent(cfg.endpointId)}`;
126
+ if (cfg.kind === 'git') {
127
+ const body = { RepositoryReferenceName: ref, Env: env, PullImage: true, Prune: true, RepositoryAuthentication: false };
128
+ if (snap.privateRepo) {
129
+ if (!snap.gitCredentialId) {
130
+ throw new AdapterError('hard', 'the stack pulls a private repository without a stored git credential — unsupported here (unmeasured); store a git credential in Portainer for the stack');
131
+ }
132
+ body.RepositoryAuthentication = true;
133
+ body.RepositoryGitCredentialID = snap.gitCredentialId;
134
+ }
135
+ return { method: 'PUT', path: `/api/stacks/${cfg.stackId}/git/redeploy${q}`, body };
136
+ }
137
+ return { method: 'PUT', path: `/api/stacks/${cfg.stackId}${q}`, body: { StackFileContent: fileContent, Env: env, PullImage: true, Prune: true } };
138
+ }
139
+
140
+ /** Send one deploy request: 409 waited out and retried, any other non-200 a hard failure. */
141
+ async function send(cfg, snap, req, io) {
142
+ const t = io.timing;
143
+ const busyUntil = io.now() + t.busyTimeoutS * 1000;
144
+ for (;;) {
145
+ const r = await call(cfg, io, req.method, req.path, req.body);
146
+ if (r.status === 200) return;
147
+ if (r.status === 409) {
148
+ if (io.now() >= busyUntil) throw new AdapterError('busy', `the stack stayed busy (409) for ${t.busyTimeoutS}s — nothing was changed by this run`);
149
+ io.log(`portainer: a deploy is already in progress (409) — waiting for it, then retrying`);
150
+ await waitWhileDeploying(cfg, io, busyUntil);
151
+ continue;
152
+ }
153
+ throw new AdapterError('hard', `${req.method} ${req.path.split('?')[0]} answered ${r.status}: ${r.text.slice(0, 300)} — nothing changed`);
154
+ }
155
+ }
156
+
157
+ async function waitWhileDeploying(cfg, io, until) {
158
+ for (;;) {
159
+ const s = await getStack(cfg, io);
160
+ if (s.Status !== STATUS_DEPLOYING) return s;
161
+ if (io.now() >= until) return s;
162
+ await io.sleep(io.timing.pollMs);
163
+ }
164
+ }
165
+
166
+ /**
167
+ * Accepted is not deployed: wait for the stack to leave "deploying", require it active, require the
168
+ * git commit asked for, then require every listed image running at `tag`.
169
+ */
170
+ async function settle(cfg, { tag, tagSha, images }, io) {
171
+ const until = io.now() + io.timing.deployTimeoutS * 1000;
172
+ const s = await waitWhileDeploying(cfg, io, until);
173
+ if (s.Status === STATUS_DEPLOYING) throw new AdapterError('refused', `the stack was still deploying after ${io.timing.deployTimeoutS}s`, { changed: true });
174
+ if (s.Status !== STATUS_ACTIVE) throw new AdapterError('refused', `the stack settled with Status ${s.Status}, not active (1)`, { changed: true });
175
+ if (cfg.kind === 'git' && tagSha) {
176
+ const got = s.GitConfig && s.GitConfig.ConfigHash;
177
+ if (got !== tagSha) throw new AdapterError('refused', `the stack runs commit ${got || '(none)'}, not ${tag}'s ${tagSha}`, { changed: true });
178
+ }
179
+ if (!images || !images.length) return;
180
+ const filters = encodeURIComponent(JSON.stringify({ label: [`com.docker.compose.project=${s.Name}`] }));
181
+ const r = await call(cfg, io, 'GET', `/api/endpoints/${encodeURIComponent(cfg.endpointId)}/docker/containers/json?all=1&filters=${filters}`);
182
+ if (r.status !== 200 || !Array.isArray(r.json)) {
183
+ throw new AdapterError('refused', `could not list the stack's containers (${r.status}) — success is read from what runs, so this fails`, { changed: true });
184
+ }
185
+ const notRunning = images.filter((ref) => !r.json.some((c) => c.Image === `${ref}:${tag}` && c.State === 'running'));
186
+ if (notRunning.length) {
187
+ const seen = r.json.map((c) => `${c.Image} (${c.State})`).join(', ') || 'no containers';
188
+ throw new AdapterError('refused', `accepted, but not running ${notRunning.map((i) => `${i}:${tag}`).join(', ')} — found ${seen}`, { changed: true });
189
+ }
190
+ }
191
+
192
+ /** Run exactly `target.tag`: every listed image in one call, then prove it runs. */
193
+ export async function deploy(cfg, target, io, snap) {
194
+ const base = snap || (await snapshot(cfg, io));
195
+ const env = mergeEnv(base.env, { IMAGE_TAG: target.tag, ...(target.env || {}) });
196
+ const req = requestFor(cfg, base, { ref: `refs/tags/${target.tag}`, env, fileContent: target.fileContent });
197
+ await send(cfg, base, req, io);
198
+ await settle(cfg, target, io);
199
+ }
200
+
201
+ /** Put back exactly what `snap` recorded — its ref, its whole Env, its file content. */
202
+ export async function restore(cfg, snap, io, { tag, images } = {}) {
203
+ const req = requestFor(cfg, snap, { ref: snap.ref, env: snap.env, fileContent: snap.fileContent });
204
+ await send(cfg, snap, req, io);
205
+ await settle(cfg, { tag, tagSha: snap.configHash, images: tag ? images : [] }, io);
206
+ }