elanous 0.2.18 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/pwa/out/404/index.html +1 -1
- package/apps/pwa/out/404.html +1 -1
- package/apps/pwa/out/_next/static/chunks/{1307-30dff22c9e1f3073.js → 1307-77fad80cf54477c6.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/1612-f60c0c93ec773f81.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2116-6429c188eb846ab8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/224-e7be8f762fe74986.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2489-ff55a35b202c0f94.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2549-89c518a4c8ed4d00.js +1 -0
- package/apps/pwa/out/_next/static/chunks/3857-995ea9f5ef33d71e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4144-fad095b48a1862ee.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5755-ea7ed88eb956f47b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5977-94ed6f78e9814b2b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7748-6e51bfe927019f5c.js +25 -0
- package/apps/pwa/out/_next/static/chunks/8401-35e94e9091f5eb8d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9247-c1753a059e7bac08.js +1 -0
- package/apps/pwa/out/_next/static/chunks/a9f2b54d-33c0689b70ed8cf3.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/approvals/{page-d92f7ee3994e84c7.js → page-c855c24de7db74fe.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/autopilot/{page-4733c967b56a4277.js → page-85b316bb4235980b.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/board/{page-d22fd849cc22f104.js → page-9b2b26a33cb2cd42.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/botlab/{page-bb742f995c6769e7.js → page-c9955fde67b1bcb5.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/bots/{page-02b073a2ac793b83.js → page-7a852956c74015a3.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/ceo/page-c5a5c621be0e04ae.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/chat/{page-8a9b3594bc23289b.js → page-4ba4abe096b6c2a8.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/consult/page-575d4a8b4f7becb2.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/control/{page-308111abadf35670.js → page-f5d56ff5f45ad164.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/decisions/page-2260e9eef0357421.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/design-check/{page-5d97b2901e0d1ea6.js → page-7c058069c5e7fa63.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/editor/page-dc5da0b8a2edb8a4.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/exec/{page-8c3d8e63ae2e1c98.js → page-466bf0ee16ad1432.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/field/{page-9b72a07d424701c6.js → page-a23bc71713bcc08f.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/inside/page-ed4a6f799f005d09.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/intake/{page-a572f58cd3ca5864.js → page-9cd565005b93aca6.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/layout-55f0f40c6db95f15.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/live/{page-be5ea965a07fbc41.js → page-474cb5888ea0e8f3.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/loops/page-35bfec2e631d3eae.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/market/{page-fa244bd146a00ae8.js → page-9ca21f9ad3208a61.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/missions/[id]/{page-44d56d6391965a6a.js → page-f9ccb1b1adbea14d.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/morning/{page-83a700d4c754d802.js → page-ec102c9b8b691e37.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/{not-found-7b3170f374f77b9d.js → not-found-a9c2cfa8dda922a5.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/observatory/page-6da35144878feb6d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/checklist/page-f6055495e9ed7ea8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/release/page-25204694fa9984a5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/seats/page-a60e035f151d84d3.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/outputs/page-189c587f36874538.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/page-c56634bde705a65d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/reflection/{page-1467047104141297.js → page-db03ae3589a81c9e.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/scheduler/page-4a6463b97f0ad94f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/sessions/{page-102310930791197a.js → page-75bffecc9aa1004e.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/settings/devices/page-a8d2fe0c669e0de4.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/settings/{page-6ed0b73bae8351d8.js → page-31d5b875760bb62d.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/setup/done/page-c26fa10aac6c21c4.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/setup/{page-43336055451f6e37.js → page-68793c6e10331d66.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/share/{page-3cc47dca0922ab9a.js → page-46d5201d2207a511.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/showroom/{page-a77ee355a2304ff0.js → page-d81c07a7b4f618ae.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/tasks/{page-b17cd46cf0e9b8cf.js → page-be1c4e8697f22360.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/term/page-cf0a4b4abead71ea.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/today/page-a4717e2c4bf7a28f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/trace/{page-7bf54c9bb9274af8.js → page-0fd895fab9b62c1b.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/vault/page-8b6c1454a8e8f306.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workflows/chat-ui/page-c6e4364d9de8f717.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workflows/page-0acc8e198f7b0dfb.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workspace/{page-134da468538baf48.js → page-34882c84f84021c5.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/worktrees/page-0506a1e0e4bdb2f5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{webpack-c2782b330447ffb6.js → webpack-c260571513bf43c2.js} +1 -1
- package/apps/pwa/out/_next/static/css/{e99e3cff94ced78a.css → 599369d853c61df7.css} +1 -1
- package/apps/pwa/out/_next/static/css/c23d959d3ee15e57.css +1 -0
- package/apps/pwa/out/_next/static/css/d3427996da11e480.css +1 -0
- package/apps/pwa/out/approvals/index.html +1 -1
- package/apps/pwa/out/approvals/index.txt +14 -14
- package/apps/pwa/out/autopilot/index.html +1 -1
- package/apps/pwa/out/autopilot/index.txt +14 -14
- package/apps/pwa/out/board/index.html +1 -1
- package/apps/pwa/out/board/index.txt +14 -14
- package/apps/pwa/out/botlab/index.html +1 -1
- package/apps/pwa/out/botlab/index.txt +14 -14
- package/apps/pwa/out/bots/index.html +1 -1
- package/apps/pwa/out/bots/index.txt +14 -14
- package/apps/pwa/out/ceo/index.html +1 -1
- package/apps/pwa/out/ceo/index.txt +14 -14
- package/apps/pwa/out/chat/index.html +1 -1
- package/apps/pwa/out/chat/index.txt +14 -14
- package/apps/pwa/out/consult/index.html +1 -1
- package/apps/pwa/out/consult/index.txt +14 -14
- package/apps/pwa/out/control/index.html +1 -1
- package/apps/pwa/out/control/index.txt +14 -14
- package/apps/pwa/out/decisions/index.html +1 -1
- package/apps/pwa/out/decisions/index.txt +14 -14
- package/apps/pwa/out/design-check/index.html +1 -1
- package/apps/pwa/out/design-check/index.txt +14 -14
- package/apps/pwa/out/editor/index.html +1 -1
- package/apps/pwa/out/editor/index.txt +16 -15
- package/apps/pwa/out/exec/index.html +1 -1
- package/apps/pwa/out/exec/index.txt +14 -14
- package/apps/pwa/out/field/index.html +1 -1
- package/apps/pwa/out/field/index.txt +14 -14
- package/apps/pwa/out/healthz +1 -1
- package/apps/pwa/out/index.html +1 -1
- package/apps/pwa/out/index.txt +14 -14
- package/apps/pwa/out/inside/index.html +1 -1
- package/apps/pwa/out/inside/index.txt +16 -15
- package/apps/pwa/out/intake/index.html +1 -1
- package/apps/pwa/out/intake/index.txt +14 -14
- package/apps/pwa/out/live/index.html +1 -1
- package/apps/pwa/out/live/index.txt +14 -14
- package/apps/pwa/out/loops/index.html +1 -1
- package/apps/pwa/out/loops/index.txt +24 -23
- package/apps/pwa/out/market/index.html +1 -1
- package/apps/pwa/out/market/index.txt +14 -14
- package/apps/pwa/out/missions/placeholder/index.html +1 -1
- package/apps/pwa/out/missions/placeholder/index.txt +14 -14
- package/apps/pwa/out/morning/index.html +1 -1
- package/apps/pwa/out/morning/index.txt +14 -14
- package/apps/pwa/out/observatory/index.html +1 -1
- package/apps/pwa/out/observatory/index.txt +14 -14
- package/apps/pwa/out/ops/checklist/index.html +1 -1
- package/apps/pwa/out/ops/checklist/index.txt +14 -14
- package/apps/pwa/out/ops/release/index.html +1 -1
- package/apps/pwa/out/ops/release/index.txt +14 -14
- package/apps/pwa/out/ops/seats/index.html +1 -1
- package/apps/pwa/out/ops/seats/index.txt +14 -14
- package/apps/pwa/out/outputs/index.html +1 -1
- package/apps/pwa/out/outputs/index.txt +14 -14
- package/apps/pwa/out/reflection/index.html +1 -1
- package/apps/pwa/out/reflection/index.txt +15 -15
- package/apps/pwa/out/scheduler/index.html +1 -1
- package/apps/pwa/out/scheduler/index.txt +14 -14
- package/apps/pwa/out/sessions/index.html +1 -1
- package/apps/pwa/out/sessions/index.txt +14 -14
- package/apps/pwa/out/settings/devices/index.html +1 -1
- package/apps/pwa/out/settings/devices/index.txt +14 -14
- package/apps/pwa/out/settings/index.html +1 -1
- package/apps/pwa/out/settings/index.txt +14 -14
- package/apps/pwa/out/setup/done/index.html +1 -1
- package/apps/pwa/out/setup/done/index.txt +14 -14
- package/apps/pwa/out/setup/index.html +1 -1
- package/apps/pwa/out/setup/index.txt +14 -14
- package/apps/pwa/out/share/index.html +1 -1
- package/apps/pwa/out/share/index.txt +14 -14
- package/apps/pwa/out/showroom/index.html +1 -1
- package/apps/pwa/out/showroom/index.txt +14 -14
- package/apps/pwa/out/tasks/index.html +1 -1
- package/apps/pwa/out/tasks/index.txt +14 -14
- package/apps/pwa/out/term/index.html +1 -1
- package/apps/pwa/out/term/index.txt +14 -14
- package/apps/pwa/out/today/index.html +1 -1
- package/apps/pwa/out/today/index.txt +14 -14
- package/apps/pwa/out/trace/index.html +1 -1
- package/apps/pwa/out/trace/index.txt +14 -14
- package/apps/pwa/out/vault/index.html +1 -1
- package/apps/pwa/out/vault/index.txt +14 -14
- package/apps/pwa/out/workflows/chat-ui/index.html +1 -1
- package/apps/pwa/out/workflows/chat-ui/index.txt +14 -14
- package/apps/pwa/out/workflows/index.html +1 -1
- package/apps/pwa/out/workflows/index.txt +14 -14
- package/apps/pwa/out/workspace/index.html +1 -1
- package/apps/pwa/out/workspace/index.txt +14 -14
- package/apps/pwa/out/worktrees/index.html +1 -1
- package/apps/pwa/out/worktrees/index.txt +14 -14
- package/graphs/demo/editor-recipes.yaml +22 -0
- package/graphs/ops/guardian.yaml +26 -0
- package/graphs/ops/recipes.yaml +3 -0
- package/graphs/retro/daily.yaml +25 -0
- package/graphs/retro/recipes.yaml +24 -0
- package/package.json +1 -1
- package/scripts/ask-marker-check.ts +3 -1
- package/scripts/coord-channel-watch.sh +2 -2
- package/scripts/discord-seat-probe.ts +3 -1
- package/scripts/land-global-checks.ts +89 -0
- package/scripts/lib/pch-regress-map.ts +24 -0
- package/scripts/morning-report.ts +18 -4
- package/scripts/pch-regress.ts +88 -0
- package/scripts/release-loop/gate-diff.ts +18 -0
- package/scripts/release-loop/gate-node.ts +628 -65
- package/scripts/release-loop/gate-partial.ts +106 -0
- package/scripts/release-loop/gate-timing.ts +229 -0
- package/scripts/release-loop/preflight.ts +6 -4
- package/scripts/release-loop/publish-window.ts +141 -0
- package/scripts/release-loop/release-docs.ts +1 -4
- package/scripts/release-loop/release-universe.ts +38 -0
- package/scripts/release-loop/resume-release.ts +67 -5
- package/scripts/release-loop/shard-plan.ts +65 -0
- package/scripts/release-loop/unattended-release.ts +12 -2
- package/scripts/release-story/draft.ts +5 -1
- package/scripts/retro/shadow-compare.ts +71 -0
- package/scripts/rhythm/daily.ts +6 -1
- package/scripts/video/scene-table.ts +159 -0
- package/src/acp/core-turn-bridge.ts +39 -2
- package/src/agent-mission/driver.ts +34 -6
- package/src/autopilot/build/llm-conflict-merge.ts +132 -7
- package/src/autopilot/build/merge-intent.ts +135 -7
- package/src/away/presence.ts +40 -0
- package/src/away/release-watch.ts +272 -0
- package/src/chat/index.ts +2 -1
- package/src/chat/short-question.ts +3 -40
- package/src/cli/away-cli.ts +68 -0
- package/src/cli/config-drift.ts +513 -0
- package/src/cli/guardian-cli.ts +49 -0
- package/src/cli/intake-cli.ts +13 -7
- package/src/cli/log-event-names.ts +5 -0
- package/src/cli/pod-cli.ts +5 -1
- package/src/cli/pr-cli.ts +20 -0
- package/src/cli/pty-drive-cli.ts +14 -3
- package/src/cli/release-cli.ts +52 -12
- package/src/cli/retro-cli.ts +78 -0
- package/src/cli/setup-cli.ts +31 -1
- package/src/cli/tasks-cli.ts +193 -14
- package/src/context-bus/context-now-public.ts +1 -1
- package/src/context-bus/context-now-surfaces.ts +16 -6
- package/src/context-bus/context-now.ts +97 -3
- package/src/core-turn/short-question.ts +100 -0
- package/src/dashboard/dashboard-status-lines.ts +24 -4
- package/src/dashboard/index.ts +50 -5
- package/src/dashboard/input/slash-executor.ts +6 -2
- package/src/dashboard/slash-runtime/dashboard-handlers.ts +68 -40
- package/src/dashboard/slash-runtime/decide-slash.ts +60 -0
- package/src/dashboard/slash-runtime/help-from-registry.ts +62 -23
- package/src/dashboard/slash-runtime/help-groups.ts +44 -0
- package/src/dashboard/slash-runtime/model-choices.ts +125 -0
- package/src/domains/guardian-tick.ts +138 -0
- package/src/domains/outbound-alert.ts +44 -13
- package/src/domains/outbound-log-probe.ts +6 -1
- package/src/domains/schedule-registry.ts +23 -0
- package/src/domains/telegram-kind-route.ts +39 -0
- package/src/domains/watch-default-scheduler.ts +95 -0
- package/src/esc-abort-gate.ts +44 -0
- package/src/graph-runner/graph-run-api.ts +205 -0
- package/src/graph-runner/runner.ts +36 -6
- package/src/harness/harness-cli-command.ts +420 -51
- package/src/harness/harness-queue.ts +315 -25
- package/src/harness/harness-salvage-cli.ts +140 -0
- package/src/harness/harness-worktree-add.ts +68 -0
- package/src/harness/ledger-write-guard.ts +6 -16
- package/src/hq/fence-outcomes.ts +63 -0
- package/src/hq/fence-wrapper.ts +33 -0
- package/src/hq/hq.ts +17 -2
- package/src/index.ts +118 -6
- package/src/instance/test-write-guard-spy-first.fixture.ts +13 -0
- package/src/instance/test-write-guard.ts +200 -0
- package/src/intake-plane/digest.ts +39 -15
- package/src/intake-plane/harness-target.ts +4 -3
- package/src/intake-plane/intake-sources.ts +1 -0
- package/src/intake-plane/items.ts +1 -0
- package/src/loops/orchestrator/degradation.ts +57 -0
- package/src/loops/orchestrator/finish-rate.ts +33 -3
- package/src/loops/orchestrator/tick.ts +166 -45
- package/src/maturity/feature-maturity.ts +6 -4
- package/src/maturity/tui-slash-maturity.ts +1 -0
- package/src/mss/logging/log-store.ts +3 -0
- package/src/nexus/api/graph-kinds.ts +10 -4
- package/src/nexus/api/graphs-api.ts +178 -6
- package/src/nexus/api/http-server.ts +54 -8
- package/src/nexus/api/loop-edges.ts +292 -17
- package/src/nexus/api/ops-api.ts +112 -29
- package/src/nexus/api/ops-board-route.ts +48 -0
- package/src/nexus/index.ts +7 -0
- package/src/onboarding/entry-hints.ts +5 -0
- package/src/onboarding.ts +24 -6
- package/src/release-loop/checklist.ts +2 -0
- package/src/release-loop/feature-store.ts +68 -18
- package/src/release-loop/gate-shards.ts +158 -0
- package/src/release-loop/light-rc.ts +173 -0
- package/src/release-loop/placement.ts +51 -17
- package/src/release-loop/release-schedule.ts +6 -2
- package/src/release-loop/rubric.ts +57 -1
- package/src/retro/daily.ts +372 -0
- package/src/seat-loop/op-decide.ts +173 -0
- package/src/seat-loop/seat-loop.ts +60 -3
- package/src/seat-loop/seat-report.ts +58 -3
- package/src/self-dev/ask-launch-flow.ts +5 -4
- package/src/self-dev/ask-launch-io.ts +49 -1
- package/src/self-dev/ask-path-candidates.ts +40 -2
- package/src/self-dev/dev-cli.ts +36 -0
- package/src/self-dev/draft-sweep.ts +371 -16
- package/src/self-dev/draft-triage-rules.ts +19 -4
- package/src/self-dev/orchestrate.ts +1 -0
- package/src/self-dev/pr-preview.ts +247 -0
- package/src/self-dev/shard-run.ts +23 -7
- package/src/self-implement/blocked-draft-policy.ts +101 -0
- package/src/self-implement/execution-clarification.ts +11 -0
- package/src/self-implement/gate-cli.ts +29 -8
- package/src/self-implement/gate-env-retry.ts +83 -0
- package/src/self-implement/goal-author-clarification.ts +88 -13
- package/src/self-implement/goal-clarification-escalation.ts +2 -0
- package/src/self-implement/orchestrator.ts +220 -6
- package/src/self-implement/run-stop.ts +123 -0
- package/src/self-implement/run-ttl.ts +258 -0
- package/src/self-implement/salvage-branches.ts +254 -0
- package/src/self-implement/salvage-retention.ts +123 -0
- package/src/self-implement/seams.ts +4 -0
- package/src/self-implement/self-implement-cli.ts +2 -0
- package/src/self-implement/sibling-resync.ts +223 -0
- package/src/self-implement/stop-autoheal.ts +204 -0
- package/src/task-agent/actions.ts +63 -2
- package/src/task-agent/board.ts +132 -0
- package/src/task-agent/card-evidence.ts +292 -0
- package/src/task-agent/completion-evidence.ts +126 -0
- package/src/task-agent/judge.ts +25 -0
- package/src/task-agent/live-moves.ts +276 -0
- package/src/task-agent/mission.ts +33 -7
- package/src/task-agent/shadow.ts +192 -15
- package/src/task-agent/task-hand.ts +124 -17
- package/src/task-orchestrator/surfaces/pod-command-job.ts +82 -8
- package/src/task-orchestrator/surfaces/pod-failure-reason.ts +4 -0
- package/src/task-orchestrator/surfaces/pod-install-slots.ts +89 -0
- package/src/task-orchestrator/surfaces/pod-lease.ts +233 -21
- package/src/task-orchestrator/surfaces/pod-pool.ts +155 -34
- package/src/task-orchestrator/surfaces/self-implement-pod.ts +21 -5
- package/src/telegram-away-command.ts +38 -0
- package/src/telegram-commands.ts +52 -0
- package/src/telegram-report-log-probe.ts +21 -0
- package/src/telegram-report.ts +27 -3
- package/src/testing/real-cli-spawn.ts +106 -0
- package/src/tui-client/dashboard-session.ts +1 -0
- package/src/user-config.ts +190 -5
- package/src/version/packed-revision.json +1 -1
- package/apps/pwa/out/_next/static/chunks/224-798346143007ff65.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3857-4f936cdb30322aa6.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4144-a0041c8dc3be57ac.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4554-5761612531d0146b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5977-a11f37254196b659.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5994-ae92c0de046ab9ba.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8401-8615a660c84cdb03.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8868-10a3a4440fb0ccd8.js +0 -25
- package/apps/pwa/out/_next/static/chunks/9247-269e4464cc4b491c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/a9f2b54d-7865aba0f4d2ee49.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ceo/page-60c3da708f9a0ccc.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/consult/page-a0eb02638b1e6f31.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/decisions/page-6eaf7bac37830c21.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/editor/page-c39554c321576982.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/inside/page-bd5d586429167fea.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/layout-59b7834d497c2255.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/loops/page-c130b63862437814.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/observatory/page-5f23d0c66a3b4793.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/checklist/page-a3c58f5c72a5f3f9.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/release/page-a2c4625596f8d69c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/seats/page-0e9099531d239f11.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/outputs/page-e8e1ecaea7cd71d2.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/page-a1b7ffed710a93e0.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/scheduler/page-2da6f9b8a38c6978.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/settings/devices/page-f331e38fefc00393.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/setup/done/page-44d5f767491692bf.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/term/page-61553851aa606146.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/today/page-a1f585e154e61c59.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/vault/page-5150ef17b5383a57.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/workflows/chat-ui/page-b9d8ce70792c1672.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/workflows/page-a5a3de231f2cc7d5.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/worktrees/page-31ac4afa2dcd915c.js +0 -1
- package/apps/pwa/out/_next/static/css/7fa358fec7da7c47.css +0 -1
- /package/apps/pwa/out/_next/static/{P6YTVgzuZz5oTZhY4Din_ → Bam1YziK2iH9bN4K0utGk}/_buildManifest.js +0 -0
- /package/apps/pwa/out/_next/static/{P6YTVgzuZz5oTZhY4Din_ → Bam1YziK2iH9bN4K0utGk}/_ssgManifest.js +0 -0
|
@@ -1,29 +1,34 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import { spawn, spawnSync } from 'node:child_process';
|
|
3
|
-
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { appendFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
|
-
import { randomUUID } from 'node:crypto';
|
|
5
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
6
6
|
import { dirname, join, resolve } from 'node:path';
|
|
7
7
|
import { debug } from '../../src/debug/log.js';
|
|
8
|
+
import { registerStandaloneLogSink } from '../../src/domains/standalone-log-sink.js';
|
|
8
9
|
import { landingFreezeMessage, readLandingFreeze } from '../../src/release-loop/landing-freeze.js';
|
|
9
10
|
import { effectiveInstanceRoot, releaseLedgerRoot } from '../../src/instance/resolve.js';
|
|
10
11
|
import { getElanousConfigDirOverride } from '../../src/elanous-config-dir.js';
|
|
11
|
-
import { diffFailures, junitFailures, parseFailures } from './gate-diff';
|
|
12
|
+
import { diffFailures, junitFailures, parseFailures, parseTimedOutFailures } from './gate-diff';
|
|
12
13
|
import { loadEnvKnownFailures, splitKnownEnv } from './env-known-failures';
|
|
13
14
|
import { baselineFromCutLogs } from './baseline-from-logs';
|
|
14
|
-
import {
|
|
15
|
+
import { mergePartialFailures, readPartialPlan, testFileOfId } from './gate-partial';
|
|
16
|
+
import { logPlanWalltime, planDurations, planShards, readFileMemory } from './shard-plan';
|
|
15
17
|
import { deriveCdpTestPatterns } from '../test-deterministic';
|
|
16
18
|
import { emitNodeResult, readGraphContext } from './node-verdict.js';
|
|
17
|
-
import { runPodCommand, type RunPodCommandOptions, type PodCommandResult } from '../../src/task-orchestrator/surfaces/pod-command-job.js';
|
|
19
|
+
import { runPodCommand, parsePodCpu, resolvePodCpu, type RunPodCommandOptions, type PodCommandResult, type PodCpu } from '../../src/task-orchestrator/surfaces/pod-command-job.js';
|
|
18
20
|
import { POD_BUN_CACHE_HOST_PATH, parseInstallSeconds, podBunCacheVolume } from '../../src/task-orchestrator/surfaces/pod-bun-cache.js';
|
|
19
21
|
import { PodPoolScheduler, parsePodPool, checkPodPool } from '../../src/task-orchestrator/surfaces/pod-pool.js';
|
|
22
|
+
import { POD_INSTALL_SLOTS_DEFAULT, POD_INSTALL_SLOTS_HOST_PATH, installSlotScript } from '../../src/task-orchestrator/surfaces/pod-install-slots.js';
|
|
23
|
+
import { gateShardsPath, writeGateShards, type GateShard } from '../../src/release-loop/gate-shards.js';
|
|
20
24
|
import { defaultKubectl } from '../../src/task-orchestrator/surfaces/self-implement-pod.js';
|
|
21
25
|
|
|
22
26
|
interface CommandResult { rc: number; output: string; passedIds?: string[]; stalledEnv?: Array<{ file: string; reason: 'no-output' }>; timedOut?: boolean }
|
|
23
27
|
export interface GateRunner {
|
|
24
28
|
command(cmd: string, args: string[], cwd: string, limitMs?: number): Promise<CommandResult>;
|
|
25
29
|
localCommand(cmd: string, args: string[], cwd: string, limitMs?: number): Promise<CommandResult>;
|
|
26
|
-
|
|
30
|
+
/** `only` (GATE-PARTIAL) restricts the sweep to these test files; absent = every test file. */
|
|
31
|
+
sweep(tree: string, logDir?: string, pod?: GateOptions['pod'], only?: readonly string[]): Promise<CommandResult>;
|
|
27
32
|
add(tree: string, commit: string): Promise<void>;
|
|
28
33
|
remove(tree: string): Promise<void>;
|
|
29
34
|
snapshot(tree: string, commit: string): Promise<void>;
|
|
@@ -42,24 +47,47 @@ export interface GateOptions {
|
|
|
42
47
|
instanceRoot?: string;
|
|
43
48
|
ledgerRoot?: string;
|
|
44
49
|
repo?: string;
|
|
45
|
-
|
|
50
|
+
/** freshShards — GATE-SHARD-RESUME off: every shard starts a Pod even when this logDir already holds its result. */
|
|
51
|
+
pod?: { pool: string; shards?: number; shardTimeoutSeconds?: number; durationSource?: string; bunCache?: string; freshShards?: boolean;
|
|
52
|
+
/** GATE-SKIP-KNOWN-RETRY — failure ids the previous release already had (and env-known ones): a shard that fails only
|
|
53
|
+
* with these is not re-run in isolation; the verdict still classifies them against the baseline as before. */
|
|
54
|
+
knownFailures?: readonly string[];
|
|
55
|
+
/** GATE-SPEED A3① — shard Pod CPU request/limit (`release.loop.gatePodCpu`); omitted = request 1 / limit 4. */
|
|
56
|
+
cpu?: PodCpu;
|
|
57
|
+
/** GATE-INSTALL-CACHE — concurrent `bun install`s per node (`release.loop.gatePodInstallSlots`); omitted = 4 · 0 = off. */
|
|
58
|
+
installSlots?: number;
|
|
59
|
+
/** GATE-MEM-ADMIT — seconds a shard waits for pool admission (memory headroom) before it launches anyway with
|
|
60
|
+
* `admission-timeout` (`release.loop.gatePodAdmissionWaitSeconds`); omitted = {@link GATE_ADMISSION_WAIT_SECONDS_DEFAULT} · 0 = off. */
|
|
61
|
+
admissionWaitSeconds?: number;
|
|
62
|
+
/** GATE-MEM-ADMIT — an injected admission (tests · embedders); config never sets it. Absent = the gate's own pool's. */
|
|
63
|
+
admission?: GateAdmission;
|
|
64
|
+
/** GATE-LIVE-OBS — the shard state file (`gateShardsPath(version)`) this sweep keeps current; absent = not written. */
|
|
65
|
+
shardsFile?: { path: string; version: string } };
|
|
46
66
|
}
|
|
47
67
|
export interface GateResult {
|
|
48
68
|
outcome: 'ok' | 'regression' | 'error';
|
|
49
69
|
commit: string;
|
|
50
70
|
introduced: string[];
|
|
51
71
|
preexisting: number;
|
|
52
|
-
|
|
72
|
+
/** null = not counted (GATE-SPEED A4: the swept baseline ran only the cut's failing files). */
|
|
73
|
+
fixed: number | null;
|
|
53
74
|
knownEnv: number;
|
|
54
75
|
knownEnvCleared: string[];
|
|
55
76
|
stalledEnv: Array<{ file: string; reason: 'no-output' | 'isolated-timeout'; local: 'passed' | 'failed' | 'timeout' }>;
|
|
56
77
|
baselineSource?: 'ledger' | 'instance' | 'cut-logs' | 'swept';
|
|
57
78
|
/** GATE-BASELINE-CACHE — isolated baseline files answered from the ledger (hits) or run now (misses). */
|
|
58
79
|
baselineCache?: { hits: number; misses: number };
|
|
80
|
+
/** GATE-SPEED A4 — a swept baseline ran only the cut's failing files that exist at the previous release. */
|
|
81
|
+
baselineDeferred?: { cutFiles: number; swept: number };
|
|
59
82
|
durationMs: number;
|
|
60
83
|
error?: string;
|
|
61
|
-
stalledShards?: Array<{ shard: number; files: string[]; reason: 'incomplete' | 'no-output' | 'job-failed' | 'unattributed'; lastFile?: string; summaryFailures?: number; namedFailures?: number; detail?: string }>;
|
|
84
|
+
stalledShards?: Array<{ shard: number; files: string[]; reason: 'incomplete' | 'no-output' | 'job-failed' | 'unattributed' | 'timeout'; lastFile?: string; summaryFailures?: number; namedFailures?: number; detail?: string }>;
|
|
62
85
|
partialSummary?: { pass: number; fail: number; errors: number; ran: number; files: number };
|
|
86
|
+
/** GATE-PARTIAL — this verdict re-ran only these files and carried the prior gate's results for the rest. */
|
|
87
|
+
partial?: { priorCommit: string; files: number };
|
|
88
|
+
/** GATE-INTRO-RECHECK — new failures that were only the «timed out» shape and passed one isolated re-run with a
|
|
89
|
+
* {@link GATE_TIMEOUT_RECHECK_MS} per-test limit: a warning (flaky under load), not an introduced regression. */
|
|
90
|
+
loadFlaky?: string[];
|
|
63
91
|
}
|
|
64
92
|
|
|
65
93
|
const sha = /^[0-9a-f]{7,40}$/i;
|
|
@@ -116,6 +144,65 @@ const lastMatch = (output: string, pattern: RegExp): RegExpExecArray | null => {
|
|
|
116
144
|
const summaryCount = (output: string, label: string) => Number(lastMatch(output, new RegExp(`(?:^|\\n)\\s*(\\d+) ${label}s?\\s*(?=\\n|$)`))?.[1] ?? NaN);
|
|
117
145
|
/** bun reads a bare positional as a substring filter (`test` matches every *.test.ts); `./` makes it a path. */
|
|
118
146
|
const asPath = (path: string) => path.startsWith('./') || path.startsWith('/') ? path : `./${path}`;
|
|
147
|
+
/** A Pod path (`~/repo/src/…` · `../..~/repo/src/…`) as a repository-relative one. */
|
|
148
|
+
const podRelative = (path: string) => path.replace(/^(?:\.\.\/)*\/?home\/ubuntu\/repo\//, '');
|
|
149
|
+
/** `file > [error]` ids of bun's «Unhandled error between tests» blocks, or undefined when one cannot be attributed. */
|
|
150
|
+
function unhandledErrorIds(output: string): string[] | undefined {
|
|
151
|
+
const errors: string[] = [];
|
|
152
|
+
let file: string | undefined;
|
|
153
|
+
for (const raw of output.split(/\r?\n/)) {
|
|
154
|
+
const header = /^(?:\.\/)?((?:[\w.-]+\/)+[\w.-]+\.test\.tsx?):/.exec(raw.trim());
|
|
155
|
+
if (header) file = header[1];
|
|
156
|
+
if (/^# Unhandled error between tests/.test(raw.trim())) {
|
|
157
|
+
if (!file) return undefined;
|
|
158
|
+
errors.push(`${file} > [error]`);
|
|
159
|
+
file = undefined;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return errors;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* GATE-SKIP-KNOWN-RETRY / GATE-ISOLATE-FAILED-FILES — the failing ids of one Pod shard, but only when every failure the
|
|
166
|
+
* summary counts is named (junit first, else the console `(fail)` lines) and every unhandled error is attributed.
|
|
167
|
+
* Anything less returns undefined, and the shard keeps the old retry path.
|
|
168
|
+
*/
|
|
169
|
+
export function shardFailureIds(clean: string, junit: string | undefined, fails: number, errors: number): { failures: string[]; errors: string[] } | undefined {
|
|
170
|
+
if (!Number.isFinite(fails)) return undefined;
|
|
171
|
+
const stripped = stripPodRepoRoot(clean);
|
|
172
|
+
let failures: string[];
|
|
173
|
+
try { failures = junit !== undefined ? [...new Set(junitFailures(junit).map(podRelative))].sort() : parseFailures(stripped); }
|
|
174
|
+
catch { return undefined; }
|
|
175
|
+
if (failures.length !== fails) return undefined;
|
|
176
|
+
const errorIds = unhandledErrorIds(stripped);
|
|
177
|
+
if (!errorIds || errorIds.length !== (Number.isNaN(errors) ? 0 : errors)) return undefined;
|
|
178
|
+
return { failures, errors: [...new Set(errorIds)].sort() };
|
|
179
|
+
}
|
|
180
|
+
/** The console form `failuresOf` reads back, for failures named from junit (same shape as `pod-shard-junit-attribution`). */
|
|
181
|
+
const failureOutput = (failures: readonly string[], errors: readonly string[]) => [
|
|
182
|
+
...failures.map((id) => { const cut = id.indexOf(' > '); return `${id.slice(0, cut)}:\n(fail) ${id.slice(cut + 3)}`; }),
|
|
183
|
+
...errors.map((id) => `${testFileOfId(id)}:\n# Unhandled error between tests`),
|
|
184
|
+
].join('\n');
|
|
185
|
+
/** Repository-relative test files with at least one passed or failed junit testcase (an empty suite is no result). */
|
|
186
|
+
function junitFiles(xml: string): Set<string> {
|
|
187
|
+
const files = new Set<string>();
|
|
188
|
+
for (const id of [...junitPassedIds(xml), ...junitFailures(xml)]) {
|
|
189
|
+
try { files.add(testFileOfId(podRelative(id))); } catch { /* an unreadable id proves nothing about its file */ }
|
|
190
|
+
}
|
|
191
|
+
return files;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* GATE-ISOLATE-FAILED-FILES — the files of a failed shard worth re-running: those with a failing test or an unhandled
|
|
195
|
+
* error, and those with no junit result at all (crashed or never reached). undefined = cannot tell (fallback: whole shard).
|
|
196
|
+
*/
|
|
197
|
+
export function retryFiles(paths: readonly string[], clean: string | undefined, junit: string | undefined, fails: number, errors: number): string[] | undefined {
|
|
198
|
+
if (junit === undefined || clean === undefined) return undefined;
|
|
199
|
+
const ids = shardFailureIds(clean, junit, fails, errors);
|
|
200
|
+
if (!ids) return undefined;
|
|
201
|
+
const failing = new Set([...ids.failures, ...ids.errors].map((id) => { try { return testFileOfId(id); } catch { return id; } }));
|
|
202
|
+
if ([...failing].some((file) => !paths.includes(file))) return undefined;
|
|
203
|
+
const reported = junitFiles(junit);
|
|
204
|
+
return paths.filter((file) => failing.has(file) || !reported.has(file));
|
|
205
|
+
}
|
|
119
206
|
export const POD_ISOLATION_MEMORY_LIMIT = '32Gi';
|
|
120
207
|
/** GT1b — a file whose measured peak is at least this runs alone at the isolation limit from the start: one such file
|
|
121
208
|
* sank a whole 203-file root shard at 16Gi (0.2.6 shard 12 · `unwired-exports` 9.9 GB) and cost two halving rounds. */
|
|
@@ -129,6 +216,138 @@ export const POD_MEMORY_SOURCE = 'docs/measurements/td1-whole-gate-mechanical-20
|
|
|
129
216
|
/** GT1 — files per retry chunk when a root shard fails (0.2.7: bundles of 54+ files OOMed at 16Gi, smaller passed). */
|
|
130
217
|
export const POD_SHARD_FILE_CAP = 27;
|
|
131
218
|
|
|
219
|
+
/** GATE-TIMEOUT-HEAL (0.2.20) — the per-shard Job deadline is the shard's planned wall-clock × 1.5, never under the
|
|
220
|
+
* configured floor (`release.loop.gatePodShardTimeoutSeconds` · default 1200 s) and never over the cap (or the floor,
|
|
221
|
+
* if that is higher). 0.2.19: the largest shard was planned at 18.9 min, ran 20–27 min (install 220–340 s under load)
|
|
222
|
+
* and 11 of 24 shards were cut at the flat 1200 s with nothing left to read. */
|
|
223
|
+
export const POD_SHARD_TIMEOUT_FLOOR_SECONDS = 1200;
|
|
224
|
+
export const POD_SHARD_TIMEOUT_CAP_SECONDS = 3600;
|
|
225
|
+
export function shardDeadlineSeconds(plannedSeconds: number, configured?: number): number {
|
|
226
|
+
const floor = configured ?? POD_SHARD_TIMEOUT_FLOOR_SECONDS;
|
|
227
|
+
const scaled = Number.isFinite(plannedSeconds) && plannedSeconds > 0 ? Math.ceil(plannedSeconds * 1.5) : 0;
|
|
228
|
+
return Math.min(Math.max(floor, POD_SHARD_TIMEOUT_CAP_SECONDS), Math.max(floor, scaled));
|
|
229
|
+
}
|
|
230
|
+
/** Seconds the Pod keeps after the in-shell soft deadline: kill grace, artifact emission, and slack for what the Job
|
|
231
|
+
* deadline counts before the shell starts and the shell cannot see (scheduling · init containers). */
|
|
232
|
+
export const POD_SHARD_EMIT_RESERVE_SECONDS = 120;
|
|
233
|
+
export const softDeadlineSeconds = (deadline: number) => deadline > 2 * POD_SHARD_EMIT_RESERVE_SECONDS
|
|
234
|
+
? deadline - POD_SHARD_EMIT_RESERVE_SECONDS : Math.max(1, Math.floor(deadline * 0.8));
|
|
235
|
+
/** GATE-TIMEOUT-HEAL ② — a shard runs as consecutive parts of about this many planned seconds (at most
|
|
236
|
+
* {@link POD_SHARD_FILE_CAP} files), each with its own log and junit, so a shard cut by its deadline still leaves the
|
|
237
|
+
* results of every part that finished (F1/F2 read them) instead of nothing. */
|
|
238
|
+
export const POD_SHARD_PART_SECONDS = 120;
|
|
239
|
+
export function shardParts(paths: readonly string[], seconds: (file: string) => number, target = POD_SHARD_PART_SECONDS): string[][] {
|
|
240
|
+
const parts: string[][] = [];
|
|
241
|
+
let current: string[] = [];
|
|
242
|
+
let sum = 0;
|
|
243
|
+
for (const file of paths) {
|
|
244
|
+
current.push(file);
|
|
245
|
+
sum += seconds(file);
|
|
246
|
+
if (sum >= target || current.length >= POD_SHARD_FILE_CAP) { parts.push(current); current = []; sum = 0; }
|
|
247
|
+
}
|
|
248
|
+
if (current.length) parts.push(current);
|
|
249
|
+
return parts;
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* The shard's Pod shell. Install once (behind the node's install slots), then run each part under the soft deadline,
|
|
253
|
+
* leaving `install.log`/`install.rc`, `part-<i>.log`/`.rc`/`.junit.xml`, and `shard.timeout` (the part index) when the
|
|
254
|
+
* deadline cut the shard. A part without `.rc` did not finish. The shell itself always exits 0 (as before).
|
|
255
|
+
*/
|
|
256
|
+
export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPatterns: readonly string[]; softSeconds: number; cachePrefix: string; installSlots: number }): string {
|
|
257
|
+
const run = o.parts.map((part, index) => {
|
|
258
|
+
const ignores = o.cdpPatterns.filter((pattern) => part.includes(pattern)).flatMap((pattern) => ['--path-ignore-patterns', pattern]);
|
|
259
|
+
return `run_part ${index} ${['bun', 'run', 'test:deterministic', ...ignores, ...part.map(asPath)].map(quote).join(' ')}`;
|
|
260
|
+
}).join(' && ');
|
|
261
|
+
return [
|
|
262
|
+
'mkdir -p "$HOME/outbox"; O="$HOME/outbox"',
|
|
263
|
+
...(o.cachePrefix ? [o.cachePrefix] : []),
|
|
264
|
+
// Seconds since this container's PID 1 started (the clone ran before this shell) — 0 when /proc cannot tell.
|
|
265
|
+
`pod_age=$(awk -v t="$(cut -d' ' -f22 /proc/1/stat 2>/dev/null)" -v hz="$(getconf CLK_TCK 2>/dev/null)" '{ if (t > 0 && hz > 0) print int($1 - t / hz); else print 0 }' /proc/uptime 2>/dev/null)`,
|
|
266
|
+
'case "$pod_age" in ""|*[!0-9]*) pod_age=0;; esac',
|
|
267
|
+
`gate_end=$(( $(date +%s) + ${Math.max(1, Math.floor(o.softSeconds))} - pod_age ))`,
|
|
268
|
+
...(o.installSlots > 0 ? [installSlotScript({ slots: o.installSlots })] : []),
|
|
269
|
+
'run_part() {',
|
|
270
|
+
' local i=$1; shift',
|
|
271
|
+
' local left=$(( gate_end - $(date +%s) ))',
|
|
272
|
+
' if [ "$left" -le 0 ]; then echo "$i" > "$O/shard.timeout"; return 1; fi',
|
|
273
|
+
' timeout -k 10 "$left" "$@" --reporter=junit --reporter-outfile="$O/part-$i.junit.xml" > "$O/part-$i.log" 2>&1',
|
|
274
|
+
' local prc=$?',
|
|
275
|
+
' if [ "$(date +%s)" -ge "$gate_end" ] && { [ "$prc" -eq 124 ] || [ "$prc" -eq 137 ] || [ "$prc" -eq 143 ]; }; then echo "$i" > "$O/shard.timeout"; return 1; fi',
|
|
276
|
+
' echo "$prc" > "$O/part-$i.rc"',
|
|
277
|
+
'}',
|
|
278
|
+
'if cd .. && cd repo; then',
|
|
279
|
+
...(o.installSlots > 0 ? [' install_slot_acquire >> "$O/install.log" 2>&1'] : []),
|
|
280
|
+
' { bun install && (cd apps/pwa && bun install); } >> "$O/install.log" 2>&1; irc=$?',
|
|
281
|
+
...(o.installSlots > 0 ? [' install_slot_release'] : []),
|
|
282
|
+
' echo "$irc" > "$O/install.rc"',
|
|
283
|
+
` if [ "$irc" -eq 0 ]; then ${run || 'true'}; fi`,
|
|
284
|
+
'else echo 5 > "$O/install.rc"; fi',
|
|
285
|
+
'true',
|
|
286
|
+
].join('\n');
|
|
287
|
+
}
|
|
288
|
+
/** Removes bun's own summary lines so one summary over several runs can follow (same rule as the sweep's merge). */
|
|
289
|
+
const stripSummary = (output: string) => output.replace(/(?:^|\n)\s*\d+ (?:pass|fail|errors?)\s*(?=\n|$)/g, '\n').replace(/Ran \d+ tests? across \d+ files?\.?/g, '');
|
|
290
|
+
export interface ShardArtifacts {
|
|
291
|
+
/** The verdict text: install log ⊕ every finished part (its summary removed) ⊕ one summary over the finished parts. */
|
|
292
|
+
output?: string;
|
|
293
|
+
junit?: string;
|
|
294
|
+
/** 0|1 when every part finished · the failing part's exit when one crashed · undefined when the deadline cut the shard. */
|
|
295
|
+
rc?: number;
|
|
296
|
+
timedOut: boolean;
|
|
297
|
+
/** Files of the parts that finished with a readable summary. */
|
|
298
|
+
finished: string[];
|
|
299
|
+
/** Raw logs of the parts that did not finish (diagnostics · last started file). */
|
|
300
|
+
unfinishedLog?: string;
|
|
301
|
+
}
|
|
302
|
+
/** Reads a shard Pod's outbox: the part layout ({@link gateShardShell}) or the older single-run one (shard.log · shard.rc · junit.xml). */
|
|
303
|
+
export function readShardArtifacts(dir: string, parts: readonly (readonly string[])[]): ShardArtifacts {
|
|
304
|
+
const read = (name: string) => { const path = join(dir, name); return existsSync(path) ? readFileSync(path, 'utf8') : undefined; };
|
|
305
|
+
const rcOf = (text: string | undefined) => { const value = text?.trim() ?? ''; return /^\d+$/.test(value) && Number.isSafeInteger(Number(value)) ? Number(value) : undefined; };
|
|
306
|
+
const installRc = rcOf(read('install.rc'));
|
|
307
|
+
const install = read('install.log');
|
|
308
|
+
if (installRc === undefined && install === undefined) {
|
|
309
|
+
const junit = read('junit.xml');
|
|
310
|
+
const rc = rcOf(read('shard.rc'));
|
|
311
|
+
return { output: read('shard.log'), ...(junit !== undefined ? { junit } : {}), ...(rc !== undefined ? { rc } : {}), timedOut: false, finished: [] };
|
|
312
|
+
}
|
|
313
|
+
const timedOut = read('shard.timeout') !== undefined;
|
|
314
|
+
if (installRc !== 0) return { output: install, ...(installRc !== undefined ? { rc: installRc } : {}), timedOut, finished: [] };
|
|
315
|
+
const good: Array<{ log: string; junit?: string; rc: 0 | 1; files: readonly string[] }> = [];
|
|
316
|
+
const bad: string[] = [];
|
|
317
|
+
let badRc: number | undefined;
|
|
318
|
+
parts.forEach((files, index) => {
|
|
319
|
+
const log = read(`part-${index}.log`);
|
|
320
|
+
const rc = rcOf(read(`part-${index}.rc`));
|
|
321
|
+
const clean = log?.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '');
|
|
322
|
+
if (clean !== undefined && (rc === 0 || rc === 1) && lastMatch(clean, /Ran (\d+) tests? across (\d+) files?/)) {
|
|
323
|
+
const junit = read(`part-${index}.junit.xml`);
|
|
324
|
+
good.push({ log: clean, rc, files, ...(junit !== undefined ? { junit } : {}) });
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
if (log !== undefined) bad.push(`# part ${index} (${rc === undefined ? 'unfinished' : `rc=${rc}`})\n${log}`);
|
|
328
|
+
// A finished part without a readable summary is not a verdict: its exit, or 2 when it claimed 0|1.
|
|
329
|
+
if (rc !== undefined && badRc === undefined) badRc = rc === 0 || rc === 1 ? 2 : rc;
|
|
330
|
+
});
|
|
331
|
+
const total = good.reduce((sum, part) => {
|
|
332
|
+
const n = (label: string) => { const value = summaryCount(part.log, label); return Number.isNaN(value) ? 0 : value; };
|
|
333
|
+
const ran = lastMatch(part.log, /Ran (\d+) tests? across (\d+) files?/)!;
|
|
334
|
+
return { pass: sum.pass + n('pass'), fail: sum.fail + n('fail'), errors: sum.errors + n('error'), ran: sum.ran + Number(ran[1]), files: sum.files + Number(ran[2]) };
|
|
335
|
+
}, { pass: 0, fail: 0, errors: 0, ran: 0, files: 0 });
|
|
336
|
+
const output = [install ?? '', ...good.map((part) => stripSummary(part.log)),
|
|
337
|
+
...(good.length ? [`${total.pass} pass\n${total.fail} fail\n${total.errors} errors\nRan ${total.ran} tests across ${total.files} files.\n`] : [])].join('\n');
|
|
338
|
+
const junits = good.flatMap((part) => part.junit !== undefined ? [part.junit] : []);
|
|
339
|
+
const complete = good.length === parts.length;
|
|
340
|
+
const rc = complete ? (good.some((part) => part.rc === 1) ? 1 : 0) : timedOut && badRc === undefined ? undefined : (badRc ?? 2);
|
|
341
|
+
return {
|
|
342
|
+
output,
|
|
343
|
+
...(junits.length ? { junit: junits.join('\n') } : {}),
|
|
344
|
+
...(rc !== undefined ? { rc } : {}),
|
|
345
|
+
timedOut: !complete && timedOut && badRc === undefined,
|
|
346
|
+
finished: good.flatMap((part) => [...part.files]),
|
|
347
|
+
...(bad.length ? { unfinishedLog: bad.join('\n') } : {}),
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
|
|
132
351
|
// review-model-ab: Pod 에서 출력 없이 멈춤 2/2(10-04 0.2.11 · 로컬 단독 4/0 · 1.7s) — GATE-STALL 수리 전까지 Pod 밖.
|
|
133
352
|
export const POD_SWEEP_INTEGRATION_ONLY: readonly string[] = ['scripts/install.test.ts', 'scripts/review-model-ab.test.ts'];
|
|
134
353
|
/** Nightly-only whole-repository cases; gate runs the fixture cases in these same files. */
|
|
@@ -150,8 +369,64 @@ const lastStartedTestFile = (output: string, paths: string[]): string | undefine
|
|
|
150
369
|
return last;
|
|
151
370
|
};
|
|
152
371
|
|
|
372
|
+
/**
|
|
373
|
+
* GATE-SHARD-RESUME (0.2.19) — the result a previous attempt left in this log directory for exactly this shard: same name,
|
|
374
|
+
* same commit, the same files in the same order and a test exit (rc 0|1) with its log. 0.2.18: the second gate stopped
|
|
375
|
+
* after 33 minutes on a tool error (unsafe test path) and the third ran all 80 minutes again although the cut commit and
|
|
376
|
+
* every finished shard were unchanged. The caller re-reads the log with the current tool code; anything else is a miss.
|
|
377
|
+
*/
|
|
378
|
+
/** GATE-SHARD-RESUME — how a previous attempt went past this shard without a verdict of its own: it re-ran in a fresh Job
|
|
379
|
+
* after no output (`-retry`), or it failed here and was split (`-c0` cap chunks · `-0` halves · `-file-0` isolation). The
|
|
380
|
+
* split shape follows from depth and size alone, so the same plan reaches the same child names (TC 09:30 review ②).
|
|
381
|
+
* `runKey` hashes where and how the Pod ran the shard (pool · command · ignores · memory · deadline · Bun cache): a fix on the Pod side
|
|
382
|
+
* makes every old result a miss instead of re-reading an old failure as introduced (TC 09:48 review). */
|
|
383
|
+
export function previousShardDescent(logDir: string, name: string, commit: string, files: readonly string[], runKey: string, parentAttempt?: string, splitChild?: string): { way: 'retry' | 'split'; attemptId: string } | undefined {
|
|
384
|
+
const meta = (file: string): { commit?: unknown; files?: unknown; rc?: unknown; runKey?: unknown; attemptId?: unknown; parentAttempt?: unknown } | undefined => {
|
|
385
|
+
try { return JSON.parse(readFileSync(join(logDir, `${file}.json`), 'utf8')); } catch { return undefined; }
|
|
386
|
+
};
|
|
387
|
+
const own = meta(name);
|
|
388
|
+
if (own?.commit !== commit || own.runKey !== runKey || !Array.isArray(own.files) || own.files.length !== files.length || own.files.some((file, index) => file !== files[index])) return undefined;
|
|
389
|
+
if (typeof own.attemptId !== 'string' || (parentAttempt !== undefined && own.parentAttempt !== parentAttempt)) return undefined;
|
|
390
|
+
const attemptId = own.attemptId;
|
|
391
|
+
// Children are checked whatever the parent's rc was: a parent can exit 0|1 and still have gone down (no output · an
|
|
392
|
+
// unreadable summary). A child counts only for this commit and only with files taken from this shard.
|
|
393
|
+
const parent = new Set(files);
|
|
394
|
+
const child = (file: string, whole: boolean) => {
|
|
395
|
+
const row = meta(file);
|
|
396
|
+
// A child counts only when the parent's latest attempt wrote it — a child left by an older attempt is not this way down.
|
|
397
|
+
return row?.commit === commit && row.parentAttempt === attemptId && Array.isArray(row.files) && row.files.length > 0 && row.files.every((path) => typeof path === 'string' && parent.has(path))
|
|
398
|
+
&& (!whole || row.files.length === files.length);
|
|
399
|
+
};
|
|
400
|
+
if (child(`${name}-retry`, true)) return { way: 'retry', attemptId };
|
|
401
|
+
// Only the first child of the split the caller takes now (`-file-0` · `-c0` · `-0` · review r3): a record of another
|
|
402
|
+
// split shape is not followed.
|
|
403
|
+
return splitChild !== undefined && child(`${name}${splitChild}`, false) ? { way: 'split', attemptId } : undefined;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
export function previousShardResult(logDir: string, name: string, commit: string, files: readonly string[], runKey: string, parentAttempt?: string): { output: string; junit?: string; rc: 0 | 1; durationMs?: number } | undefined {
|
|
407
|
+
try {
|
|
408
|
+
const meta = JSON.parse(readFileSync(join(logDir, `${name}.json`), 'utf8')) as { commit?: unknown; files?: unknown; rc?: unknown; durationMs?: unknown; runKey?: unknown; parentAttempt?: unknown; jobExitCode?: unknown; jobFailed?: unknown };
|
|
409
|
+
// Only a Job that itself exited 0 — a crashed Job can still leave rc 0 and a clean-looking log (review r6).
|
|
410
|
+
if (meta.jobExitCode !== 0 || meta.jobFailed !== false) return undefined;
|
|
411
|
+
if (meta.commit !== commit || meta.runKey !== runKey || (parentAttempt !== undefined && meta.parentAttempt !== parentAttempt) || (meta.rc !== 0 && meta.rc !== 1) || !Array.isArray(meta.files)
|
|
412
|
+
|| meta.files.length !== files.length || meta.files.some((file, index) => file !== files[index])) return undefined;
|
|
413
|
+
const logPath = join(logDir, `${name}.log`);
|
|
414
|
+
if (!existsSync(logPath)) return undefined;
|
|
415
|
+
const junitPath = join(logDir, `${name}.junit.xml`);
|
|
416
|
+
return { output: readFileSync(logPath, 'utf8'), rc: meta.rc, ...(existsSync(junitPath) ? { junit: readFileSync(junitPath, 'utf8') } : {}),
|
|
417
|
+
...(typeof meta.durationMs === 'number' ? { durationMs: meta.durationMs } : {}) };
|
|
418
|
+
} catch { return undefined; }
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
/** 0.2.18 gate: a test that changed process.cwd made bun print later test files relative to that cwd
|
|
422
|
+
* (`../..~/repo/src/…`), and the safe-path check stopped the whole gate. The Pod clone root is
|
|
423
|
+
* `~/repo`; strip it back to a repository-relative path before any id is read. */
|
|
424
|
+
export function stripPodRepoRoot(output: string): string {
|
|
425
|
+
return output.replace(/^(\s*(?:\(fail\)\s+)?)(?:\.\.\/)*(?:\/)?home\/ubuntu\/repo\//gm, '$1');
|
|
426
|
+
}
|
|
427
|
+
|
|
153
428
|
function failuresOf(run: CommandResult, label: string): SweepFailures {
|
|
154
|
-
const output = run.output.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '');
|
|
429
|
+
const output = stripPodRepoRoot(run.output.replace(/\x1b\[[0-9;]*[A-Za-z]/g, ''));
|
|
155
430
|
const ran = lastMatch(output, /Ran (\d+) tests? across ([1-9]\d*) files?/);
|
|
156
431
|
const reportedErrors = summaryCount(output, 'error');
|
|
157
432
|
const errorCount = Number.isNaN(reportedErrors) ? 0 : reportedErrors;
|
|
@@ -193,6 +468,11 @@ class StalledPodShards extends Error {
|
|
|
193
468
|
|
|
194
469
|
/** Isolated re-runs (host stall re-run · cut and baseline re-checks) are time-limited; sweeps and installs keep their full time. */
|
|
195
470
|
export const GATE_ISOLATED_TIMEOUT_MS = 300_000;
|
|
471
|
+
/** GATE-INTRO-RECHECK (0.2.19) — the per-test limit of the one re-run given to a «timed out after 5000ms» new failure.
|
|
472
|
+
* 0.2.18: both first-gate introductions were 5 s limits missed at 6.04/6.06 s, and they alone cost an 84-minute re-gate. */
|
|
473
|
+
export const GATE_TIMEOUT_RECHECK_MS = 60_000;
|
|
474
|
+
/** TEST-CENSUS intake: one row per load-flaky test, in the machine ledger (read by the test census, never by the gate). */
|
|
475
|
+
export const loadFlakyCensusPath = (ledger: string) => join(ledger, 'release', 'test-census', 'load-flaky.jsonl');
|
|
196
476
|
|
|
197
477
|
/**
|
|
198
478
|
* Runs a limited local command in its own process group. On timeout: SIGTERM the group (test-deterministic forwards it to its own
|
|
@@ -231,6 +511,54 @@ export function limitedLocalCommand(cmd: string, args: string[], cwd: string, li
|
|
|
231
511
|
});
|
|
232
512
|
}
|
|
233
513
|
|
|
514
|
+
/**
|
|
515
|
+
* GATE-MEM-ADMIT (0.2.20) — the gate's shard Jobs ask the same pool admission harness goal Pods ask
|
|
516
|
+
* (`PodPoolScheduler.acquireAdmission` · memory headroom by observed usage · `limitedBy`). 0.2.19: 8 shards sat Pending
|
|
517
|
+
* for CPU (requests 31.2/32) because harness Pods held the node and the gate counted only its own Jobs (#24698).
|
|
518
|
+
* The gate never deadlocks on it: after {@link GATE_ADMISSION_WAIT_SECONDS_DEFAULT} it launches anyway and records
|
|
519
|
+
* `admission-timeout`; an admission error launches at once (`admission-error`). Harness launches wait without a bound,
|
|
520
|
+
* so the gate keeps priority.
|
|
521
|
+
*/
|
|
522
|
+
export interface GateAdmission {
|
|
523
|
+
acquire(signal: AbortSignal): Promise<() => void>;
|
|
524
|
+
/** The pool-wide reading admission waits on (recommended · limitedBy · memorySlots); null before any reading. */
|
|
525
|
+
waitReason(): string | null;
|
|
526
|
+
}
|
|
527
|
+
export const GATE_ADMISSION_WAIT_SECONDS_DEFAULT = 300;
|
|
528
|
+
export type GateAdmissionOutcome = 'granted' | 'admission-timeout' | 'admission-error' | 'off';
|
|
529
|
+
export interface GateAdmissionResult { outcome: GateAdmissionOutcome; waitedMs: number; reason?: string; release: () => void }
|
|
530
|
+
export const gateAdmissionWaitText = (reading: string | null) => `메모리 여유 대기(admission) · ${reading ?? '측정 전'}`.slice(0, 160);
|
|
531
|
+
export function poolGateAdmission(pool: PodPoolScheduler): GateAdmission {
|
|
532
|
+
return { acquire: (signal) => pool.acquireAdmission(signal, undefined, { gate: true }), waitReason: () => pool.waitReason() };
|
|
533
|
+
}
|
|
534
|
+
export async function admitGateShard(admission: GateAdmission | undefined, o: { waitMs: number; onWait: (reason: string) => void; reasonPollMs?: number; firstReasonMs?: number }): Promise<GateAdmissionResult> {
|
|
535
|
+
const noop = () => {};
|
|
536
|
+
if (!admission || !(o.waitMs > 0)) return { outcome: 'off', waitedMs: 0, release: noop };
|
|
537
|
+
const started = Date.now();
|
|
538
|
+
const controller = new AbortController();
|
|
539
|
+
let timedOut = false;
|
|
540
|
+
let lastReason: string | undefined;
|
|
541
|
+
const tell = () => { lastReason = gateAdmissionWaitText(admission.waitReason()); o.onWait(lastReason); };
|
|
542
|
+
const deadline = setTimeout(() => { timedOut = true; controller.abort(); }, o.waitMs);
|
|
543
|
+
// A grant inside the first moment never touches the shard row (no flapping in shards.json).
|
|
544
|
+
let ticker: ReturnType<typeof setInterval> | undefined;
|
|
545
|
+
const first = setTimeout(() => { tell(); ticker = setInterval(tell, o.reasonPollMs ?? 15_000); }, o.firstReasonMs ?? 1_000);
|
|
546
|
+
try {
|
|
547
|
+
const granted = await admission.acquire(controller.signal);
|
|
548
|
+
// Released at placement and again after the Job (the launch may fail before placement): only the first counts.
|
|
549
|
+
let released = false;
|
|
550
|
+
const release = () => { if (!released) { released = true; granted(); } };
|
|
551
|
+
return { outcome: 'granted', waitedMs: Date.now() - started, ...(lastReason ? { reason: lastReason } : {}), release };
|
|
552
|
+
} catch (error) {
|
|
553
|
+
if (timedOut) return { outcome: 'admission-timeout', waitedMs: Date.now() - started, reason: lastReason ?? gateAdmissionWaitText(admission.waitReason()), release: noop };
|
|
554
|
+
return { outcome: 'admission-error', waitedMs: Date.now() - started, reason: (error instanceof Error ? error.message : String(error)).slice(0, 160), release: noop };
|
|
555
|
+
} finally {
|
|
556
|
+
clearTimeout(deadline);
|
|
557
|
+
clearTimeout(first);
|
|
558
|
+
if (ticker) clearInterval(ticker);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
|
|
234
562
|
export function createGateRunner(repo: string, remote?: string, commandOverride?: GateRunner['command'], podCommand: (options: RunPodCommandOptions) => Promise<PodCommandResult> = runPodCommand, poolOverride?: PodPoolScheduler, podLogTail?: (job: PodCommandResult) => string): GateRunner {
|
|
235
563
|
let remoteMirror: string | undefined;
|
|
236
564
|
const localCommand: GateRunner['localCommand'] = async (cmd, args, cwd, limitMs) => {
|
|
@@ -252,9 +580,10 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
252
580
|
return { rc: run.status ?? 2, output: `${run.stdout ?? ''}\n${run.stderr ?? ''}${run.error ? `\n${run.error}` : ''}`, ...(timedOut ? { timedOut } : {}) };
|
|
253
581
|
}
|
|
254
582
|
: localCommand);
|
|
255
|
-
const podSweep = async (tree: string, logDir: string | undefined, pod: NonNullable<GateOptions['pod']
|
|
583
|
+
const podSweep = async (tree: string, logDir: string | undefined, pod: NonNullable<GateOptions['pod']>, only?: readonly string[]): Promise<CommandResult> => {
|
|
256
584
|
const shardCount = pod.shards ?? 24;
|
|
257
|
-
|
|
585
|
+
// GATE-TIMEOUT-HEAL ①: the configured value is the floor; each shard's own deadline is shardDeadlineSeconds(planned).
|
|
586
|
+
const deadlineSeconds = pod.shardTimeoutSeconds ?? POD_SHARD_TIMEOUT_FLOOR_SECONDS;
|
|
258
587
|
if (!pod.pool || !Number.isSafeInteger(shardCount) || shardCount < 1
|
|
259
588
|
|| !Number.isSafeInteger(deadlineSeconds) || deadlineSeconds < 1) throw new Error('invalid pod sweep options');
|
|
260
589
|
const head = await command('git', ['rev-parse', 'HEAD'], tree);
|
|
@@ -263,7 +592,13 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
263
592
|
if (!/^[0-9a-f]{40}$/.test(commit)) throw new Error('invalid cut tree HEAD');
|
|
264
593
|
const listed = await command('git', ['ls-files', '*.test.*'], tree);
|
|
265
594
|
check(listed, 'git ls-files tests');
|
|
266
|
-
const
|
|
595
|
+
const allFiles = listed.output.split(/\r?\n/).filter((file) => /\.test\.(?:tsx?|jsx?|mts|cts)$/.test(file)).sort();
|
|
596
|
+
const onlySet = only ? new Set(only) : undefined;
|
|
597
|
+
// GATE-PARTIAL: every planned file must exist at this commit — a deleted/renamed failing file would otherwise «pass»
|
|
598
|
+
// by not running. Fail closed (gate error), never a smaller sweep than planned.
|
|
599
|
+
const missing = only?.filter((file) => !allFiles.includes(file)) ?? [];
|
|
600
|
+
if (missing.length) throw new Error(`partial plan file missing at this commit: ${missing.slice(0, 5).join(', ')}`);
|
|
601
|
+
const files = onlySet ? allFiles.filter((file) => onlySet.has(file)) : allFiles;
|
|
267
602
|
if (files.some((file) => file.startsWith('-') || file.startsWith('/') || file.startsWith('./')
|
|
268
603
|
|| file.split('/').some((part) => !part || part === '.' || part === '..') || /[\r\n]/.test(file))) throw new Error('unsafe pod test path');
|
|
269
604
|
if (new Set(files).size !== files.length) throw new Error('duplicate pod test path');
|
|
@@ -278,7 +613,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
278
613
|
const integrationOnly = files.filter((file) => POD_SWEEP_INTEGRATION_ONLY.includes(file));
|
|
279
614
|
if (integrationOnly.length) debug.log('release-loop.gate', 'pod-sweep-integration-only', { files: integrationOnly });
|
|
280
615
|
if (!assignable.length) throw new Error('sweep incomplete: no tests ran');
|
|
281
|
-
const durations
|
|
616
|
+
const { durations, wall: planWall } = planDurations(pod.durationSource);
|
|
282
617
|
const memory = readFileMemory(join(tree, POD_MEMORY_SOURCE));
|
|
283
618
|
// The TD1 measurements predate the fixture-only gate; do not isolate those five files at their old 2–13 GB peaks.
|
|
284
619
|
const fixtureOnly = new Set([...GATE_NIGHTLY_AUDITS, 'test/guardian/dispatch-surface-contract.test.ts', 'test/user-config-mcp.test.ts']);
|
|
@@ -288,6 +623,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
288
623
|
...(light.length ? planShards(light, durations, shardCount) : []),
|
|
289
624
|
...[...heavy].map((file) => ({ files: [file], plannedSeconds: durations.get(file) ?? 0 })),
|
|
290
625
|
];
|
|
626
|
+
logPlanWalltime(planWall, shards);
|
|
291
627
|
if (heavy.size) debug.log('release-loop.gate', 'pod-heavy-alone', { files: [...heavy], thresholdMb: POD_HEAVY_FILE_MB, memoryLimit: POD_ISOLATION_MEMORY_LIMIT });
|
|
292
628
|
const known = assignable.filter((file) => durations.has(file)).length;
|
|
293
629
|
debug.log('release-loop.gate', 'pod-shard-plan', {
|
|
@@ -295,6 +631,9 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
295
631
|
maxPlannedSeconds: Math.max(...shards.map((shard) => shard.plannedSeconds)),
|
|
296
632
|
source: durations.size ? pod.durationSource : 'none',
|
|
297
633
|
});
|
|
634
|
+
// GATE-SPEED A3①: shard Pod CPU (`release.loop.gatePodCpu` · default request 1 / limit 4) — shards × request must fit the pool.
|
|
635
|
+
const cpu = resolvePodCpu(pod.cpu);
|
|
636
|
+
debug.log('release-loop.gate', 'shard-resources', { shards: shards.length, cpuRequest: cpu.request, cpuLimit: cpu.limit, cpuSource: pod.cpu ? 'config' : 'default' });
|
|
298
637
|
const ready = poolOverride ? undefined : checkPodPool(parsePodPool(pod.pool), defaultKubectl);
|
|
299
638
|
if (ready && !ready.ok) throw new Error('pod command: 풀의 노드가 하나도 준비되지 않았다');
|
|
300
639
|
const poolScheduler = poolOverride ?? new PodPoolScheduler(ready!.ready);
|
|
@@ -302,7 +641,17 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
302
641
|
const envCache = process.env[POD_BUN_CACHE_HOST_PATH]?.trim();
|
|
303
642
|
const bunCache = configCache || envCache || undefined;
|
|
304
643
|
debug.log('release-loop.gate', 'pod-bun-cache', { source: configCache ? 'config' : envCache ? 'env' : 'none' });
|
|
305
|
-
const cachePrefix = bunCache ?
|
|
644
|
+
const cachePrefix = bunCache ? podBunCacheVolume(bunCache).shellPrefix : '';
|
|
645
|
+
// GATE-INSTALL-CACHE: node-wide install slots (see pod-install-slots.ts for the measured cause and the safety argument).
|
|
646
|
+
const installSlots = pod.installSlots ?? POD_INSTALL_SLOTS_DEFAULT;
|
|
647
|
+
if (!Number.isSafeInteger(installSlots) || installSlots < 0) throw new Error('invalid pod sweep options');
|
|
648
|
+
debug.log('release-loop.gate', 'pod-install-slots', { slots: installSlots, source: pod.installSlots !== undefined ? 'config' : 'default' });
|
|
649
|
+
// GATE-MEM-ADMIT: every shard Job (retries and splits too) asks pool admission before it is placed.
|
|
650
|
+
const admissionWaitSeconds = pod.admissionWaitSeconds ?? GATE_ADMISSION_WAIT_SECONDS_DEFAULT;
|
|
651
|
+
if (!Number.isSafeInteger(admissionWaitSeconds) || admissionWaitSeconds < 0) throw new Error('invalid pod sweep options');
|
|
652
|
+
// An injected pool (tests) runs without admission unless `pod.admission` is passed; the gate's own pool asks it.
|
|
653
|
+
const gateAdmission = pod.admission ?? (poolOverride ? undefined : poolGateAdmission(poolScheduler));
|
|
654
|
+
debug.log('release-loop.gate', 'pod-admission', { waitSeconds: admissionWaitSeconds, source: pod.admissionWaitSeconds !== undefined ? 'config' : 'default', enabled: !!gateAdmission && admissionWaitSeconds > 0 });
|
|
306
655
|
const jobLogTail = (job: PodCommandResult): string => {
|
|
307
656
|
if (podLogTail) return podLogTail(job);
|
|
308
657
|
for (const member of poolScheduler.members) {
|
|
@@ -315,43 +664,112 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
315
664
|
type ShardReason = NonNullable<GateResult['stalledShards']>[number]['reason'];
|
|
316
665
|
// 깊이 끝 OOM 조각은 «조각 전체»를 파일 하나씩 격리한다 — 상한 40 이면 나머지가 못 잰 채 남았다(09-30 ① 7번 조각 110 파일).
|
|
317
666
|
const isolationRemaining = shards.map((item) => item.files.length);
|
|
667
|
+
const knownFailures = new Set(pod.knownFailures ?? []);
|
|
318
668
|
const estimated = (paths: string[]) => paths.reduce((sum, file) => {
|
|
319
669
|
const value = durations.get(file);
|
|
320
670
|
return sum + (value !== undefined && Number.isFinite(value) && value >= 0 ? value : unknownSeconds);
|
|
321
671
|
}, 0);
|
|
322
672
|
const knownTimes = assignable.map((file) => durations.get(file)).filter((value): value is number => value !== undefined && Number.isFinite(value) && value >= 0).sort((a, b) => a - b);
|
|
323
673
|
const unknownSeconds = knownTimes.length ? (knownTimes[Math.floor((knownTimes.length - 1) / 2)]! + knownTimes[Math.floor(knownTimes.length / 2)]!) / 2 : 1;
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
674
|
+
// GATE-LIVE-OBS — one row per root shard in the release ledger's shards.json (UX contract · src/release-loop/gate-shards.ts).
|
|
675
|
+
// Retries and splits of a shard update its row. A write failure is logged and never stops the gate.
|
|
676
|
+
const board: GateShard[] = shards.map((item, index) => ({ id: `pod-${index}`, state: 'pending', plannedMin: Math.round(item.plannedSeconds / 6) / 10 }));
|
|
677
|
+
const flushBoard = () => {
|
|
678
|
+
if (!pod.shardsFile) return;
|
|
679
|
+
try { writeGateShards(pod.shardsFile.path, { v: 1, version: pod.shardsFile.version, updatedAt: new Date().toISOString(), shards: board }); }
|
|
680
|
+
catch (error) { debug.log('release-loop.gate', 'shards-write-failed', { path: pod.shardsFile.path, error: String(error) }); }
|
|
681
|
+
};
|
|
682
|
+
const markShard = (shard: number, patch: Partial<GateShard>, clear: Array<keyof GateShard> = []) => {
|
|
683
|
+
const row = board[shard];
|
|
684
|
+
if (!row) return;
|
|
685
|
+
const next: GateShard = { ...row, ...patch };
|
|
686
|
+
for (const key of clear) delete next[key];
|
|
687
|
+
if (JSON.stringify(next) === JSON.stringify(row)) return;
|
|
688
|
+
board[shard] = next;
|
|
689
|
+
flushBoard();
|
|
690
|
+
};
|
|
691
|
+
flushBoard();
|
|
692
|
+
/** The gate's scheduler, seen through one shard: a slot wait and the Pod start become row states. */
|
|
693
|
+
// GATE-MEM-ADMIT: `placed` releases the shard's admission once a member is chosen — from then on the Job is
|
|
694
|
+
// Pending/Running in the cluster and the next admission reading counts it, so holding the lease would count it twice.
|
|
695
|
+
const trackedPool = (shard: number, retry: boolean, placed: () => void = () => {}): PodPoolScheduler => ({
|
|
696
|
+
members: poolScheduler.members,
|
|
697
|
+
tryAcquire: async (...args: Parameters<PodPoolScheduler['tryAcquire']>) => {
|
|
698
|
+
const member = await poolScheduler.tryAcquire(...args);
|
|
699
|
+
if (member) placed();
|
|
700
|
+
if (member) markShard(shard, { state: retry ? 'retry' : 'running', startedAt: board[shard]?.startedAt ?? new Date().toISOString() }, ['waitReason']);
|
|
701
|
+
else markShard(shard, { waitReason: 'Pod 자리 대기' });
|
|
702
|
+
return member;
|
|
703
|
+
},
|
|
704
|
+
release: (member: Parameters<PodPoolScheduler['release']>[0]) => poolScheduler.release(member),
|
|
705
|
+
}) as unknown as PodPoolScheduler;
|
|
706
|
+
const runShard = async (paths: string[], shard: number, depth = 0, branch = '', retriedNoOutput = false, fresh = pod.freshShards === true, parentAttempt?: string): Promise<{ runs: ShardRun[]; stalled: NonNullable<GateResult['stalledShards']> }> => {
|
|
327
707
|
// 파일별 소요는 junit 으로 남긴다 — 콘솔 요약(판정 원천)은 그대로이고, 느린 시험 목록(K10 D4)·계층 분리(D2)의 자가 된다.
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
const
|
|
708
|
+
// GATE-TIMEOUT-HEAL: parts (each with its own junit) under a soft deadline inside this shard's own Job deadline.
|
|
709
|
+
const shardDeadline = shardDeadlineSeconds(estimated(paths), deadlineSeconds);
|
|
710
|
+
const parts = shardParts(paths, (file) => estimated([file]));
|
|
711
|
+
const memoryLimit = paths.length === 1 && (depth >= 1 || heavy.has(paths[0]!)) ? POD_ISOLATION_MEMORY_LIMIT : undefined;
|
|
712
|
+
const podShell = gateShardShell({ parts, cdpPatterns, softSeconds: softDeadlineSeconds(shardDeadline), cachePrefix, installSlots });
|
|
713
|
+
const runKey = createHash('sha256').update(JSON.stringify({ pool: pod.pool, podShell, memoryLimit: memoryLimit ?? null, deadlineSeconds: shardDeadline, bunCache: bunCache ?? null })).digest('hex');
|
|
714
|
+
let start = Date.now();
|
|
715
|
+
let admitted: GateAdmissionResult | undefined;
|
|
331
716
|
let output: string | undefined;
|
|
332
717
|
let junit: string | undefined;
|
|
333
718
|
let rc: number | undefined;
|
|
334
719
|
let jobExitCode: number | undefined;
|
|
335
720
|
let lastFile: string | undefined;
|
|
336
721
|
let jobFailed = false;
|
|
337
|
-
|
|
722
|
+
let timedOut = false;
|
|
723
|
+
let finished: string[] = [];
|
|
724
|
+
let unfinishedLog: string | undefined;
|
|
725
|
+
// The way down a previous attempt took wins over its own log: that log was not a verdict, or it would have no children.
|
|
726
|
+
// The split this shard would take if it failed here — the same rule as the split code below (isolation at depth 2 ·
|
|
727
|
+
// cap chunks for a big root shard · halves otherwise · none for one file).
|
|
728
|
+
// Isolation follows only while this shard's isolation budget still covers every file (review r7) — the budget is the
|
|
729
|
+
// shard's file count and isolations take disjoint files, so this holds unless the plan changed; if not, run the Pod.
|
|
730
|
+
const splitChild = depth === 2 && paths.length > 1 ? (isolationRemaining[shard]! >= paths.length ? '-file-0' : undefined)
|
|
731
|
+
: depth < 2 && paths.length > 1 ? (depth === 0 && paths.length > POD_SHARD_FILE_CAP ? '-c0' : '-0') : undefined;
|
|
732
|
+
const found = !fresh && logDir ? previousShardDescent(logDir, `pod-${shard}${branch}`, commit, paths, runKey, parentAttempt, splitChild) : undefined;
|
|
733
|
+
const descent = found?.way;
|
|
734
|
+
const reused = !fresh && !descent && logDir ? previousShardResult(logDir, `pod-${shard}${branch}`, commit, paths, runKey, parentAttempt) : undefined;
|
|
735
|
+
// This shard's attempt id: the recorded one when its way down is followed, a new one when a Pod runs. Children carry it.
|
|
736
|
+
const attemptId = found?.attemptId ?? randomUUID();
|
|
737
|
+
if (descent === 'retry') {
|
|
738
|
+
// «descend» is only the way down; «resumed» comes from each leaf whose log reads cleanly (review r4).
|
|
739
|
+
debug.log('release-loop.gate', 'pod-shard-descend', { shard, branch, files: paths.length, via: 'retry' });
|
|
740
|
+
return runShard(paths, shard, depth, `${branch}-retry`, true, undefined, attemptId);
|
|
741
|
+
}
|
|
742
|
+
if (descent === 'split') {
|
|
743
|
+
output = undefined;
|
|
744
|
+
debug.log('release-loop.gate', 'pod-shard-descend', { shard, branch, files: paths.length, via: 'split' });
|
|
745
|
+
} else if (reused) {
|
|
746
|
+
output = reused.output;
|
|
747
|
+
junit = reused.junit;
|
|
748
|
+
rc = reused.rc;
|
|
749
|
+
jobExitCode = 0;
|
|
750
|
+
lastFile = lastStartedTestFile(output, paths);
|
|
751
|
+
} else try {
|
|
752
|
+
admitted = await admitGateShard(gateAdmission, { waitMs: admissionWaitSeconds * 1000, onWait: (reason) => markShard(shard, { waitReason: reason }) });
|
|
753
|
+
if (admitted.outcome !== 'off') debug.log('release-loop.gate', 'shard-admission', { shard, branch, outcome: admitted.outcome, waitedMs: admitted.waitedMs, ...(admitted.reason ? { reason: admitted.reason } : {}) },
|
|
754
|
+
admitted.outcome === 'granted' ? undefined : { level: 'warn' });
|
|
755
|
+
// The admission wait is not the shard's run time: deadline/timeout judgment and the measured duration start here.
|
|
756
|
+
start = Date.now();
|
|
338
757
|
const job = await podCommand({
|
|
339
|
-
pool: pod.pool, poolScheduler, clone: true, source: { kind: 'commit', sha: commit }, deadlineSeconds,
|
|
758
|
+
pool: pod.pool, poolScheduler: trackedPool(shard, depth > 0 || branch !== '', admitted.release), clone: true, source: { kind: 'commit', sha: commit }, deadlineSeconds: shardDeadline,
|
|
340
759
|
...(bunCache ? { bunCache } : {}),
|
|
760
|
+
...(installSlots > 0 ? { installSlots: POD_INSTALL_SLOTS_HOST_PATH } : {}),
|
|
761
|
+
...(pod.cpu ? { cpu: pod.cpu } : {}),
|
|
341
762
|
// 실패 뒤 다시 도는 파일 하나짜리 Job 은 메모리 한도를 올린다 — 16Gi 에선 무거운 한 파일이 혼자서도 OOM 이었다(09-30 `unwired-exports`).
|
|
342
|
-
...(
|
|
763
|
+
...(memoryLimit ? { memoryLimit } : {}),
|
|
343
764
|
name: `gate-${randomUUID()}`,
|
|
344
|
-
command: ['bash', '-lc',
|
|
765
|
+
command: ['bash', '-lc', podShell],
|
|
345
766
|
});
|
|
346
767
|
jobExitCode = job.exitCode;
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
const rcText = existsSync(rcPath) ? readFileSync(rcPath, 'utf8').trim() : '';
|
|
353
|
-
rc = /^\d+$/.test(rcText) && Number.isSafeInteger(Number(rcText)) ? Number(rcText) : undefined;
|
|
354
|
-
lastFile = output === undefined ? undefined : lastStartedTestFile(output, paths);
|
|
768
|
+
const read = readShardArtifacts(job.artifactsDir, parts);
|
|
769
|
+
({ output, junit, rc, timedOut, finished, unfinishedLog } = read);
|
|
770
|
+
// A Job the cluster cut at its own deadline leaves no outbox at all: past the deadline with nothing is a timeout too.
|
|
771
|
+
if (!timedOut && output === undefined && job.exitCode !== 0 && Date.now() - start >= shardDeadline * 1000) timedOut = true;
|
|
772
|
+
lastFile = output === undefined && unfinishedLog === undefined ? undefined : lastStartedTestFile(`${output ?? ''}\n${unfinishedLog ?? ''}`, paths);
|
|
355
773
|
if (!lastFile && (job.exitCode !== 0 || rc === undefined || (rc !== 0 && rc !== 1))) {
|
|
356
774
|
try { lastFile = lastStartedTestFile(jobLogTail(job).split(/\r?\n/).slice(-200).join('\n'), paths); }
|
|
357
775
|
catch { /* Log retrieval is diagnostic; do not hide the stalled shard. */ }
|
|
@@ -359,18 +777,32 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
359
777
|
} catch {
|
|
360
778
|
jobFailed = true;
|
|
361
779
|
}
|
|
780
|
+
// A launch that failed before placement still gives its admission back (release is idempotent).
|
|
781
|
+
admitted?.release();
|
|
362
782
|
const durationMs = Date.now() - start;
|
|
363
|
-
|
|
783
|
+
// A reused shard keeps its files as they are — rewriting would replace the measured durationMs the next release plans by.
|
|
784
|
+
if (logDir && !reused && !descent) {
|
|
364
785
|
mkdirSync(logDir, { recursive: true });
|
|
786
|
+
// GATE-SHARD-RESUME: the json is what makes a log reusable, so it goes first and comes back last — a stop between
|
|
787
|
+
// the new log and the new json leaves no json, never the old Job's exit paired with the new log (review r8).
|
|
788
|
+
rmSync(join(logDir, `pod-${shard}${branch}.json`), { force: true });
|
|
365
789
|
const destination = join(logDir, `pod-${shard}${branch}.log`);
|
|
366
790
|
if (output !== undefined) writeFileSync(destination, output, { mode: 0o600 });
|
|
367
791
|
else rmSync(destination, { force: true });
|
|
368
|
-
writeFileSync(join(logDir, `pod-${shard}${branch}.json`), JSON.stringify({ durationMs, rc: rc ?? null, files: paths, plannedSeconds: estimated(paths), shardCount: shards.length, commit }) + '\n', { mode: 0o600 });
|
|
369
792
|
const junitDestination = join(logDir, `pod-${shard}${branch}.junit.xml`);
|
|
370
793
|
if (junit !== undefined) writeFileSync(junitDestination, junit, { mode: 0o600 });
|
|
371
794
|
else rmSync(junitDestination, { force: true });
|
|
795
|
+
const unfinishedDestination = join(logDir, `pod-${shard}${branch}.unfinished.log`);
|
|
796
|
+
if (unfinishedLog !== undefined) writeFileSync(unfinishedDestination, unfinishedLog, { mode: 0o600 });
|
|
797
|
+
else rmSync(unfinishedDestination, { force: true });
|
|
798
|
+
writeFileSync(join(logDir, `pod-${shard}${branch}.json`), JSON.stringify({ durationMs, rc: rc ?? null, ...(timedOut ? { reason: 'timeout', finishedFiles: finished.length } : {}), files: paths, plannedSeconds: estimated(paths), deadlineSeconds: shardDeadline, shardCount: shards.length, commit, runKey, attemptId, ...(parentAttempt ? { parentAttempt } : {}), jobExitCode: jobExitCode ?? null, jobFailed,
|
|
799
|
+
...(admitted && admitted.outcome !== 'off' ? { admission: admitted.outcome, admissionWaitMs: admitted.waitedMs } : {}) }) + '\n', { mode: 0o600 });
|
|
372
800
|
}
|
|
373
|
-
|
|
801
|
+
const installSeconds = parseInstallSeconds(output ?? '');
|
|
802
|
+
// GATE-TIMEOUT-HEAL ③: a cut shard is «reason: timeout» (with the files that still finished), not a bare rc=null.
|
|
803
|
+
if (!reused && !descent) debug.log('release-loop.gate', 'pod-shard', { shard, files: paths, durationMs, rc: rc ?? null, attempt: depth + 1, installSeconds, deadlineSeconds: shardDeadline, parts: parts.length,
|
|
804
|
+
...(timedOut ? { reason: 'timeout', finishedFiles: finished.length } : {}) });
|
|
805
|
+
if (!reused && !descent) markShard(shard, { ...(installSeconds !== null ? { installSec: installSeconds } : {}), ...(rc !== undefined ? { rc } : {}), ...(timedOut ? { state: 'timeout' as const } : {}) });
|
|
374
806
|
let clean = output?.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '');
|
|
375
807
|
const ran = clean && lastMatch(clean, /Ran (\d+) tests? across (\d+) files?/);
|
|
376
808
|
const passes = clean === undefined ? NaN : summaryCount(clean, 'pass');
|
|
@@ -379,7 +811,9 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
379
811
|
let reason: ShardReason | undefined;
|
|
380
812
|
let namedFailures: number | undefined;
|
|
381
813
|
let unattributedDetail: string | undefined;
|
|
382
|
-
if (
|
|
814
|
+
if (descent === 'split') reason = 'incomplete';
|
|
815
|
+
else if (timedOut) reason = 'timeout';
|
|
816
|
+
else if (jobFailed || (jobExitCode !== undefined && jobExitCode !== 0) || (rc !== undefined && rc !== 0 && rc !== 1)) reason = 'job-failed';
|
|
383
817
|
else if (!clean || rc === undefined) reason = 'no-output';
|
|
384
818
|
else if (!ran || !Number.isFinite(passes) || !Number.isFinite(fails)
|
|
385
819
|
|| Number(ran[2]) === 0 || Number(ran[2]) > paths.length
|
|
@@ -410,19 +844,56 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
410
844
|
}
|
|
411
845
|
}
|
|
412
846
|
}
|
|
413
|
-
|
|
414
|
-
|
|
847
|
+
// A reused result that the current tool cannot read cleanly is not a verdict: run that shard for real.
|
|
848
|
+
if (reused && reason) {
|
|
849
|
+
debug.log('release-loop.gate', 'pod-shard-resume-miss', { shard, branch, reason });
|
|
850
|
+
return runShard(paths, shard, depth, branch, retriedNoOutput, true, parentAttempt);
|
|
851
|
+
}
|
|
852
|
+
// GATE-TIMEOUT-HEAL ④: a Pod that died before any test ran (no outbox · jobExitCode≠0 — 0.2.19: isolated retries
|
|
853
|
+
// ending with exit 5, the source checkout) gets the same one fresh retry as «no output» instead of stalling the sweep.
|
|
854
|
+
// 137 (killed · OOM) keeps the isolation path: re-running an OOM file only re-measures the same kill.
|
|
855
|
+
const diedBeforeTests = reason === 'job-failed' && output === undefined && !jobFailed && jobExitCode !== undefined && jobExitCode !== 0
|
|
856
|
+
&& jobExitCode !== 137 && (paths.length === 1 || jobExitCode === 5);
|
|
857
|
+
if ((reason === 'no-output' || diedBeforeTests) && !retriedNoOutput) {
|
|
858
|
+
const retry = await runShard(paths, shard, depth, `${branch}-retry`, true, undefined, attemptId);
|
|
415
859
|
debug.log('release-loop.gate', 'pod-shard-retry', { shard, files: paths, reason, outcome: retry.stalled[0]?.reason ?? 'ok' });
|
|
416
860
|
return retry;
|
|
417
861
|
}
|
|
418
862
|
if (reason) {
|
|
863
|
+
// GATE-SKIP-KNOWN-RETRY — a shard whose summary is complete but whose failures could not be read back (unattributed)
|
|
864
|
+
// and whose every named failure is one the previous release already had is not re-run: re-running only re-measures
|
|
865
|
+
// known failures (0.2.18 cut: 109 retry Pods · 1,840 shard-minutes, 4x the first pass). Those failures go to the
|
|
866
|
+
// verdict as they are, so it still classifies them against the baseline (preexisting) — never silently dropped.
|
|
867
|
+
if (reason === 'unattributed' && knownFailures.size && clean && ran) {
|
|
868
|
+
const ids = shardFailureIds(clean, junit, fails, errors);
|
|
869
|
+
const all = ids ? [...ids.failures, ...ids.errors] : [];
|
|
870
|
+
if (ids && all.length && all.every((id) => knownFailures.has(id))) {
|
|
871
|
+
debug.log('release-loop.gate', 'retry-skipped-known', { shard, failures: all.length });
|
|
872
|
+
return { runs: [{ output: `${failureOutput(ids.failures, ids.errors)}\n`, rc: 1, pass: Number.isFinite(passes) ? passes : 0,
|
|
873
|
+
fail: ids.failures.length, errors: ids.errors.length, ran: Number(ran[1]), files: Number(ran[2]),
|
|
874
|
+
passedIds: junit ? junitPassedIds(junit) : [] }], stalled: [] };
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
// GATE-ISOLATE-FAILED-FILES — with this shard's junit, only the files with a failing test or an unhandled error and
|
|
878
|
+
// the files with no junit result (crashed · never reached) are re-run; the rest passed here and keep that result.
|
|
879
|
+
// The narrowed set runs under its own branch (`-n`) at the same depth, so the usual split/isolation rules still apply
|
|
880
|
+
// to it, and a resumed gate finds this shard's own log (not a `-c0`/`-0` child) and narrows the same way again.
|
|
881
|
+
const narrowed = !descent ? retryFiles(paths, clean, junit, fails, errors) : undefined;
|
|
882
|
+
if (narrowed && narrowed.length && narrowed.length < paths.length) {
|
|
883
|
+
debug.log('release-loop.gate', 'retry-narrowed', { shard, before: paths.length, after: narrowed.length });
|
|
884
|
+
const kept = new Set(paths.filter((file) => !narrowed.includes(file)));
|
|
885
|
+
const passedIds = junitPassedIds(junit!).map(podRelative).filter((id) => { try { return kept.has(testFileOfId(id)); } catch { return false; } });
|
|
886
|
+
const child = await runShard(narrowed, shard, depth, `${branch}-n`, false, undefined, attemptId);
|
|
887
|
+
return { runs: [{ output: '', rc: 0, pass: passedIds.length, fail: 0, errors: 0, ran: passedIds.length, files: kept.size, passedIds },
|
|
888
|
+
...child.runs], stalled: child.stalled };
|
|
889
|
+
}
|
|
419
890
|
// 깊이 2 에서 «이름 없는 실패»·«불완전»도 파일 단위로 가른다 — 아니면 149파일 조각의 실패 하나가 끝까지 주인 없이 남는다(09-30 G1e 7번 조각: 요약 28 · 이름 27).
|
|
420
891
|
// 로그 없이 죽은 조각(`no-output` · OOM)도 같다 — 09-30 G1f ② 4-0-0 은 150파일이 격리 0 으로 남았다.
|
|
421
892
|
if (depth === 2 && paths.length > 1) {
|
|
422
893
|
const count = Math.min(paths.length, isolationRemaining[shard]!);
|
|
423
894
|
isolationRemaining[shard]! -= count;
|
|
424
895
|
const children = await Promise.all(paths.slice(0, count).map(async (file, index) => {
|
|
425
|
-
const child = await runShard([file], shard, depth + 1, `${branch}-file-${index}
|
|
896
|
+
const child = await runShard([file], shard, depth + 1, `${branch}-file-${index}`, false, undefined, attemptId);
|
|
426
897
|
debug.log('release-loop.gate', 'pod-shard-isolate', { shard, file, outcome: child.stalled[0]?.reason ?? 'ok' });
|
|
427
898
|
return child;
|
|
428
899
|
}));
|
|
@@ -443,23 +914,34 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
443
914
|
if (depth === 0 && paths.length > POD_SHARD_FILE_CAP) {
|
|
444
915
|
const chunks = Array.from({ length: Math.ceil(paths.length / POD_SHARD_FILE_CAP) }, (_, i) => paths.slice(i * POD_SHARD_FILE_CAP, (i + 1) * POD_SHARD_FILE_CAP));
|
|
445
916
|
debug.log('release-loop.gate', 'pod-shard-cap-split', { shard, files: paths.length, chunks: chunks.length, cap: POD_SHARD_FILE_CAP, reason });
|
|
446
|
-
const children = await Promise.all(chunks.map((chunk, i) => runShard(chunk, shard, 2, `${branch}-c${i}
|
|
917
|
+
const children = await Promise.all(chunks.map((chunk, i) => runShard(chunk, shard, 2, `${branch}-c${i}`, false, undefined, attemptId)));
|
|
447
918
|
return { runs: children.flatMap((child) => child.runs), stalled: children.flatMap((child) => child.stalled) };
|
|
448
919
|
}
|
|
449
920
|
debug.log('release-loop.gate', 'pod-shard-split', { shard, depth, files: paths, reason });
|
|
450
921
|
const middle = Math.ceil(paths.length / 2);
|
|
451
922
|
const children = await Promise.all([
|
|
452
|
-
runShard(paths.slice(0, middle), shard, depth + 1, `${branch}-0
|
|
453
|
-
runShard(paths.slice(middle), shard, depth + 1, `${branch}-1
|
|
923
|
+
runShard(paths.slice(0, middle), shard, depth + 1, `${branch}-0`, false, undefined, attemptId),
|
|
924
|
+
runShard(paths.slice(middle), shard, depth + 1, `${branch}-1`, false, undefined, attemptId),
|
|
454
925
|
]);
|
|
455
926
|
return { runs: children.flatMap((child) => child.runs), stalled: children.flatMap((child) => child.stalled) };
|
|
456
927
|
}
|
|
928
|
+
if (reused) debug.log('release-loop.gate', 'pod-shard-resumed', { shard, branch, files: paths.length, rc, durationMs: reused.durationMs ?? null });
|
|
457
929
|
return { runs: [{ output: clean!, rc: rc!, pass: passes, fail: fails,
|
|
458
930
|
errors: Number.isNaN(errors) ? 0 : errors,
|
|
459
931
|
ran: Number(ran![1]), files: Number(ran![2]), passedIds: junit ? junitPassedIds(junit) : [] }], stalled: [] };
|
|
460
932
|
};
|
|
461
933
|
// 모든 조각이 끝난 뒤 판정한다 — 한 조각의 예외로 먼저 돌아가면 다른 Pod 가 도는 채로 정리가 시작된다(#22002 리뷰 R3).
|
|
462
|
-
const settled = await Promise.allSettled(shards.map((item, shard) =>
|
|
934
|
+
const settled = await Promise.allSettled(shards.map(async (item, shard) => {
|
|
935
|
+
try {
|
|
936
|
+
const done = await runShard(item.files, shard);
|
|
937
|
+
const state = !done.stalled.length ? 'done' as const : done.stalled.some((stall) => stall.reason === 'timeout') ? 'timeout' as const : 'failed' as const;
|
|
938
|
+
markShard(shard, { state, endedAt: new Date().toISOString() }, ['waitReason']);
|
|
939
|
+
return done;
|
|
940
|
+
} catch (error) {
|
|
941
|
+
markShard(shard, { state: 'failed', endedAt: new Date().toISOString() }, ['waitReason']);
|
|
942
|
+
throw error;
|
|
943
|
+
}
|
|
944
|
+
}));
|
|
463
945
|
const rejected = settled.find((result): result is PromiseRejectedResult => result.status === 'rejected');
|
|
464
946
|
if (rejected) throw rejected.reason;
|
|
465
947
|
const results = settled.flatMap((result) => result.status === 'fulfilled' ? [result.value] : []);
|
|
@@ -480,12 +962,16 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
480
962
|
localCommand: commandOverride && !remote ? commandOverride : localCommand,
|
|
481
963
|
get remoteMirror() { return remoteMirror; },
|
|
482
964
|
set remoteMirror(path) { remoteMirror = path; },
|
|
483
|
-
async sweep(tree, logDir, pod) {
|
|
484
|
-
if (pod) return podSweep(tree, logDir, pod);
|
|
965
|
+
async sweep(tree, logDir, pod, only) {
|
|
966
|
+
if (pod) return podSweep(tree, logDir, pod, only);
|
|
485
967
|
const listed = await command('git', ['ls-files', '*.test.*'], tree);
|
|
486
968
|
check(listed, 'git ls-files tests');
|
|
487
|
-
const
|
|
488
|
-
const
|
|
969
|
+
const allFiles = listed.output.split(/\r?\n/).filter((file) => /\.test\.(?:tsx?|jsx?|mts|cts)$/.test(file));
|
|
970
|
+
const onlySet = only ? new Set(only) : undefined;
|
|
971
|
+
const missing = only?.filter((file) => !allFiles.includes(file)) ?? [];
|
|
972
|
+
if (missing.length) throw new Error(`partial plan file missing at this commit: ${missing.slice(0, 5).join(', ')}`);
|
|
973
|
+
const files = onlySet ? allFiles.filter((file) => onlySet.has(file)) : allFiles;
|
|
974
|
+
const groups = onlySet ? [{ name: 'partial', paths: files }] : [
|
|
489
975
|
{ name: 'src-cli', paths: ['src/cli'] },
|
|
490
976
|
{ name: 'src-rest', paths: [...new Set(files.filter((f) => f.startsWith('src/') && !f.startsWith('src/cli/')).map((f) => f.split('/').length === 2 ? f : f.split('/').slice(0, 2).join('/')))] },
|
|
491
977
|
{ name: 'test', paths: ['test'] },
|
|
@@ -579,11 +1065,7 @@ export function baselineCommit(root: string, version: string): string {
|
|
|
579
1065
|
return commit;
|
|
580
1066
|
}
|
|
581
1067
|
|
|
582
|
-
|
|
583
|
-
const file = id.split(' > ', 1)[0]!;
|
|
584
|
-
if (!id.includes(' > ') || !/^(?:[\w.-]+\/)+[\w.-]+\.test\.tsx?$/.test(file) || file.split('/').includes('..')) throw new Error(`unsafe test path: ${file}`);
|
|
585
|
-
return file;
|
|
586
|
-
}
|
|
1068
|
+
const fileOf = testFileOfId;
|
|
587
1069
|
|
|
588
1070
|
/**
|
|
589
1071
|
* GATE-BASELINE-CACHE (0.2.18) — the isolated `bun run test:deterministic <file>` result of one test file at one commit,
|
|
@@ -716,8 +1198,28 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
716
1198
|
trees.push(cutTree);
|
|
717
1199
|
await runner.add(cutTree, opts.commit);
|
|
718
1200
|
for (const dir of [cutTree, join(cutTree, 'apps/pwa')]) check(await runner.command('bun', ['install'], dir), `bun install ${dir}`);
|
|
719
|
-
|
|
720
|
-
|
|
1201
|
+
// GATE-PARTIAL: a plan made by `release resume --from gate --partial` for exactly this commit re-runs only its files.
|
|
1202
|
+
// Its logs go to cut-partial/ — the full cut/ logs stay whole (the next release reads them as a baseline source).
|
|
1203
|
+
const partial = readPartialPlan(roots, opts.version, opts.commit);
|
|
1204
|
+
if (partial) debug.log('release-loop.gate', 'partial-regate', { version: opts.version, priorCommit: partial.priorCommit, commit: opts.commit, files: partial.files.length });
|
|
1205
|
+
// GATE-SKIP-KNOWN-RETRY — the ids the verdict will not count as introduced (the recorded baseline ⊕ env-known), read
|
|
1206
|
+
// before the sweep so a shard failing only with them skips its isolation retries. No recorded baseline = no skip.
|
|
1207
|
+
let knownFailures: string[] = [];
|
|
1208
|
+
if (opts.pod) {
|
|
1209
|
+
try {
|
|
1210
|
+
const knownSha = opts.baselineCommit ?? baselineReleaseCommit(opts.baselineVersion);
|
|
1211
|
+
const saved = cachedBaseline(opts.baselineVersion, knownSha)?.saved;
|
|
1212
|
+
const recorded = saved ? undefined : baselineFromCutLogs(join(ledger, 'release', opts.baselineVersion, 'gate-logs', 'cut'), knownSha);
|
|
1213
|
+
const trustedKnown = saved ?? (recorded?.complete ? recorded : undefined);
|
|
1214
|
+
if (trustedKnown) knownFailures = [...trustedKnown.failures, ...(trustedKnown.errors ?? []), ...loadEnvKnownFailures(repo)];
|
|
1215
|
+
} catch (error) { debug.log('release-loop.gate', 'known-failures-unavailable', { error: String(error) }); }
|
|
1216
|
+
}
|
|
1217
|
+
const cutRun = await runner.sweep(cutTree, join(root, 'release', opts.version, 'gate-logs', partial ? 'cut-partial' : 'cut'),
|
|
1218
|
+
opts.pod ? { ...opts.pod, durationSource: join(ledger, 'release', opts.baselineVersion, 'gate-logs', 'cut'), ...(knownFailures.length ? { knownFailures } : {}),
|
|
1219
|
+
// GATE-LIVE-OBS: the release ledger's shards.json (UX #24782) — never the run tree's universe; an explicit
|
|
1220
|
+
// --ledger-root (tests) keeps it in that ledger.
|
|
1221
|
+
shardsFile: { path: opts.ledgerRoot ? gateShardsPath(opts.version, opts.ledgerRoot) : gateShardsPath(opts.version), version: opts.version } } : undefined,
|
|
1222
|
+
partial?.files);
|
|
721
1223
|
const localStalled = async (run: CommandResult, tree: string, commit: string, label: string) => {
|
|
722
1224
|
const stalled = run.stalledEnv ?? [];
|
|
723
1225
|
const noCompletedShard = stalled.length && run.rc === 0
|
|
@@ -757,7 +1259,9 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
757
1259
|
debug.log('release-loop.gate', 'pod-shard-stalled-env', { files: stalled.map(({ file, reason }) => ({ file, reason })), local });
|
|
758
1260
|
return measured;
|
|
759
1261
|
};
|
|
760
|
-
const
|
|
1262
|
+
const measuredCut = await localStalled(cutRun, cutTree, opts.commit, 'cut');
|
|
1263
|
+
const cut = partial ? mergePartialFailures(partial, measuredCut) : measuredCut;
|
|
1264
|
+
if (partial) result.partial = { priorCommit: partial.priorCommit, files: partial.files.length };
|
|
761
1265
|
cutFailures = cut;
|
|
762
1266
|
const baseSha = opts.baselineCommit ?? baselineReleaseCommit(opts.baselineVersion);
|
|
763
1267
|
const cached = cachedBaseline(opts.baselineVersion, baseSha);
|
|
@@ -782,11 +1286,28 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
782
1286
|
};
|
|
783
1287
|
let baseline: SweepFailures | undefined = trusted ? { failures: trusted.failures, errors: trusted.errors ?? [] } : undefined;
|
|
784
1288
|
if (!baseline) {
|
|
785
|
-
//
|
|
786
|
-
//
|
|
787
|
-
const
|
|
788
|
-
|
|
789
|
-
|
|
1289
|
+
// GATE-SPEED A4 (RFC-release-gate-under-30min §A4) — with no recorded baseline only the files that failed at the cut
|
|
1290
|
+
// are swept at the previous release; a passing cut file needs no comparison. Files absent there stay new failures.
|
|
1291
|
+
const cutFiles = [...new Set([...cut.failures, ...cut.errors].map(fileOf))].sort();
|
|
1292
|
+
let present: string[] = [];
|
|
1293
|
+
if (cutFiles.length) {
|
|
1294
|
+
const tree = await getBaseTree();
|
|
1295
|
+
// -z: the default output quotes paths with special characters, which would read as absent.
|
|
1296
|
+
const listed = await runner.command('git', ['ls-files', '-z', '--', ...cutFiles], tree);
|
|
1297
|
+
check(listed, 'baseline ls-files');
|
|
1298
|
+
const known = new Set(listed.output.split('\0'));
|
|
1299
|
+
present = cutFiles.filter((file) => known.has(file));
|
|
1300
|
+
}
|
|
1301
|
+
result.baselineDeferred = { cutFiles: cutFiles.length, swept: present.length };
|
|
1302
|
+
debug.log('release-loop.gate', 'baseline-deferred', { version: opts.baselineVersion, commit: baseSha, cutFiles: cutFiles.length, swept: present.length });
|
|
1303
|
+
if (present.length) {
|
|
1304
|
+
// K9b — the baseline is swept on the same Pod pool as the cut, not on this host (0.2.6: an hour of local baseline
|
|
1305
|
+
// after a 23-minute Pod cut).
|
|
1306
|
+
const baseRun = await runner.sweep(baseTree!, join(root, 'release', opts.version, 'gate-logs', 'baseline'),
|
|
1307
|
+
opts.pod ? { ...opts.pod, durationSource: join(ledger, 'release', opts.baselineVersion, 'gate-logs', 'cut') } : undefined,
|
|
1308
|
+
present);
|
|
1309
|
+
baseline = await localStalled(baseRun, baseTree!, baseSha, 'baseline');
|
|
1310
|
+
} else baseline = { failures: [], errors: [] };
|
|
790
1311
|
}
|
|
791
1312
|
for (const id of [...baseline.failures, ...baseline.errors, ...cut.failures, ...cut.errors]) fileOf(id);
|
|
792
1313
|
const known = opts.pod ? loadEnvKnownFailures(repo) : new Set<string>();
|
|
@@ -798,7 +1319,7 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
798
1319
|
result.knownEnv = countedCut.knownEnv.length;
|
|
799
1320
|
result.knownEnvCleared = [...new Set(cutRun.passedIds ?? [])].filter((id) => known.has(id) && !cutIds.includes(id) && !cutRun.stalledEnv?.some(({ file }) => fileOf(id) === file)).sort();
|
|
800
1321
|
const diff = diffFailures(countedCut.counted, countedBaseline.counted);
|
|
801
|
-
result.fixed = diff.fixed.length;
|
|
1322
|
+
result.fixed = result.baselineDeferred ? null : diff.fixed.length;
|
|
802
1323
|
result.preexisting = diff.common.length;
|
|
803
1324
|
// GATE-BASELINE-CACHE — only the files whose new failures reproduce in isolation reach the baseline, and each
|
|
804
1325
|
// (commit, file, host) is run at most once across gates: this gate's cut results also seed the next gate's baseline.
|
|
@@ -829,7 +1350,36 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
829
1350
|
: failuresOf(cutCheck, `cut isolated ${file}`);
|
|
830
1351
|
if (cutCheck !== undefined) remember({ commit: opts.commit, file, host: cacheHost, failures: isolatedCut.failures, errors: isolatedCut.errors, missing: false });
|
|
831
1352
|
const reproduced = new Set([...isolatedCut.failures, ...isolatedCut.errors]);
|
|
832
|
-
|
|
1353
|
+
let candidates = diff.newFailures.filter((id) => fileOf(id) === file && reproduced.has(id));
|
|
1354
|
+
// GATE-INTRO-RECHECK — a new failure that is only bun's «timed out» shape gets one more isolated run with a generous
|
|
1355
|
+
// per-test limit, on the same runner (node-b when the gate is remote). Passing there = flaky under load (warning);
|
|
1356
|
+
// an assertion failure, a failure again, or a run that cannot be read stays introduced.
|
|
1357
|
+
const timedOut = cutCheck === undefined ? new Set<string>() : new Set(parseTimedOutFailures(stripPodRepoRoot(cutCheck.output)));
|
|
1358
|
+
const shaped = candidates.filter((id) => timedOut.has(id) && isolatedCut.failures.includes(id));
|
|
1359
|
+
if (shaped.length) {
|
|
1360
|
+
const relaxed = await runner.command('bun', ['run', 'test:deterministic', asPath(file), '--timeout', String(GATE_TIMEOUT_RECHECK_MS)], cutTree, GATE_ISOLATED_TIMEOUT_MS);
|
|
1361
|
+
let flaky: string[] = [];
|
|
1362
|
+
let recheck: 'read' | 'timeout' | 'unreadable' = 'read';
|
|
1363
|
+
if (relaxed.timedOut) recheck = 'timeout';
|
|
1364
|
+
else {
|
|
1365
|
+
try {
|
|
1366
|
+
const again = failuresOf(relaxed, `cut timeout recheck ${file}`);
|
|
1367
|
+
const still = new Set([...again.failures, ...again.errors]);
|
|
1368
|
+
flaky = again.errors.length ? [] : shaped.filter((id) => !still.has(id));
|
|
1369
|
+
} catch { recheck = 'unreadable'; }
|
|
1370
|
+
}
|
|
1371
|
+
debug.log('release-loop.gate', 'timeout-recheck', { file, shaped: shaped.length, flaky: flaky.length, recheck, remote: opts.remote ?? 'local', limitMs: GATE_TIMEOUT_RECHECK_MS });
|
|
1372
|
+
if (flaky.length) {
|
|
1373
|
+
(result.loadFlaky ??= []).push(...flaky);
|
|
1374
|
+
candidates = candidates.filter((id) => !flaky.includes(id));
|
|
1375
|
+
try {
|
|
1376
|
+
const path = loadFlakyCensusPath(ledger);
|
|
1377
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
1378
|
+
const at = new Date().toISOString();
|
|
1379
|
+
appendFileSync(path, flaky.map((id) => JSON.stringify({ at, version: opts.version, commit: opts.commit, id, host: cacheHost, limitMs: GATE_TIMEOUT_RECHECK_MS }) + '\n').join(''), { mode: 0o600 });
|
|
1380
|
+
} catch (error) { debug.log('release-loop.gate', 'load-flaky-census-write-failed', { file, error: String(error) }); }
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
833
1383
|
if (candidates.length === 0) continue;
|
|
834
1384
|
let oldFailures: Set<string>;
|
|
835
1385
|
const hit = baseCache.get(`${file}\0${cacheHost}`);
|
|
@@ -912,7 +1462,9 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
912
1462
|
export function graphGateResult(result: GateResult, graph: boolean) {
|
|
913
1463
|
return { ...result, ...(graph && result.outcome === 'regression' ? { outcome: 'fail' as const } : {}),
|
|
914
1464
|
verdict: result.outcome === 'ok' ? 'pass' as const : 'fail' as const,
|
|
915
|
-
summary: (result.
|
|
1465
|
+
summary: (result.partial ? `부분 재검 ${result.partial.files}파일(앞 게이트 ${result.partial.priorCommit.slice(0, 9)} 결과 이음) · ` : '')
|
|
1466
|
+
+ (result.outcome === 'ok' ? `새 회귀 ${result.introduced.length} · 기존 ${result.preexisting} · 고침 ${result.fixed ?? '못 셈(기준선 미루기)'}` : `게이트 ${result.outcome}: ${result.error ?? result.introduced.length + ' new regressions'}`) + ` · 환경 알려진 실패 ${result.knownEnv}`
|
|
1467
|
+
+ (result.loadFlaky?.length ? ` · ⚠ 부하 흔들림 ${result.loadFlaky.length}(시간 초과 꼴 · ${GATE_TIMEOUT_RECHECK_MS / 1000}s 재실행 통과 · 막지 않음)` : '')
|
|
916
1468
|
+ (result.stalledEnv.length ? ` · 환경 멈춤 ${result.stalledEnv.length} (Pod 밖 통과 ${result.stalledEnv.filter((item) => item.local === 'passed').length})` : '') };
|
|
917
1469
|
}
|
|
918
1470
|
|
|
@@ -936,12 +1488,19 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
936
1488
|
...(fromGraph.gatePodShards !== undefined ? { shards: Number(fromGraph.gatePodShards) } : {}),
|
|
937
1489
|
...(fromGraph.gatePodShardTimeoutSeconds !== undefined ? { shardTimeoutSeconds: Number(fromGraph.gatePodShardTimeoutSeconds) } : {}),
|
|
938
1490
|
...(typeof fromGraph.gatePodBunCache === 'string' && fromGraph.gatePodBunCache.trim() ? { bunCache: fromGraph.gatePodBunCache } : {}),
|
|
1491
|
+
...(fromGraph.gatePodInstallSlots !== undefined ? { installSlots: Number(fromGraph.gatePodInstallSlots) } : {}),
|
|
1492
|
+
...(fromGraph.gatePodAdmissionWaitSeconds !== undefined ? { admissionWaitSeconds: Number(fromGraph.gatePodAdmissionWaitSeconds) } : {}),
|
|
1493
|
+
...(fromGraph.gatePodFreshShards === true ? { freshShards: true } : {}),
|
|
1494
|
+
...((cpu) => cpu ? { cpu } : {})(parsePodCpu(fromGraph.gatePodCpu, 'release.loop.gatePodCpu')),
|
|
939
1495
|
} : undefined,
|
|
940
1496
|
};
|
|
1497
|
+
let freshShards = false;
|
|
941
1498
|
for (let i = 0; i < args.length; i++) {
|
|
942
1499
|
const arg = args[i];
|
|
943
1500
|
if (arg === '--help') return 'help';
|
|
944
1501
|
if (arg === '--json') continue;
|
|
1502
|
+
// Applied after the loop and only to a Pod gate — on its own it must not turn a local gate into a Pod one (review r5).
|
|
1503
|
+
if (arg === '--fresh-shards') { freshShards = true; continue; }
|
|
945
1504
|
const key = ({ '--commit': 'commit', '--version': 'version', '--baseline-version': 'baselineVersion', '--baseline-commit': 'baselineCommit', '--remote': 'remote', '--remote-mirror': 'remoteMirror', '--pod-pool': 'pod', '--pod-shards': 'pod', '--pod-shard-timeout-seconds': 'pod' } as Record<string, keyof GateOptions>)[arg!];
|
|
946
1505
|
if (!key) throw new Error(`unknown option: ${arg}`);
|
|
947
1506
|
const value = args[++i];
|
|
@@ -956,17 +1515,21 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
956
1515
|
else if (key === 'remote') opts.remote = value;
|
|
957
1516
|
else if (key === 'remoteMirror') opts.remoteMirror = value;
|
|
958
1517
|
}
|
|
1518
|
+
if (freshShards && opts.pod) opts.pod = { ...opts.pod, freshShards: true };
|
|
959
1519
|
return opts;
|
|
960
1520
|
}
|
|
961
1521
|
|
|
962
1522
|
if (import.meta.main) {
|
|
963
1523
|
const start = Date.now();
|
|
1524
|
+
// The graph spawns this file as its own process, which inherits no daemon sink — without this every
|
|
1525
|
+
// `release-loop.gate`/`release.gate`/`pod.pool` event stayed in the file trail and `elanous logs` read 0 rows.
|
|
1526
|
+
await registerStandaloneLogSink('release-loop');
|
|
964
1527
|
let result: GateResult;
|
|
965
1528
|
let version = '';
|
|
966
1529
|
try {
|
|
967
1530
|
const opts = parseOptions(process.argv.slice(2), process.env);
|
|
968
1531
|
if (opts === 'help') {
|
|
969
|
-
console.log('Usage: bun scripts/release-loop/gate-node.ts --commit <sha> --version <v> [--baseline-version <prev>] [--baseline-commit <sha>] [--remote <ssh-host>] [--remote-mirror <path>] [--pod-pool <pool>] [--pod-shards <n>] [--pod-shard-timeout-seconds <n>] [--json]\nWithout flags, input.commit, input.version and input.previousVersion come from the JSON file at ELANOUS_GRAPH_CONTEXT.');
|
|
1532
|
+
console.log('Usage: bun scripts/release-loop/gate-node.ts --commit <sha> --version <v> [--baseline-version <prev>] [--baseline-commit <sha>] [--remote <ssh-host>] [--remote-mirror <path>] [--pod-pool <pool>] [--pod-shards <n>] [--pod-shard-timeout-seconds <n>] [--fresh-shards] [--json]\nWithout flags, input.commit, input.version and input.previousVersion come from the JSON file at ELANOUS_GRAPH_CONTEXT.');
|
|
970
1533
|
process.exit(0);
|
|
971
1534
|
}
|
|
972
1535
|
version = opts.version;
|