elanous 0.2.23 → 0.2.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/pwa/out/404/index.html +1 -1
- package/apps/pwa/out/404.html +1 -1
- package/apps/pwa/out/_next/static/chunks/{272-b23d2e92161d11fd.js → 272-0eed0fb0aab716b6.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{3961-fbef064c213b2b18.js → 3961-aaae04ed1d32622f.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/approvals/{page-d5341ac96203a983.js → page-3b9fc39362473ae7.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/autopilot/page-f2b62c19ca68fef8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/board/{page-55c631c032dcc47e.js → page-d7602d489edf5f37.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/botlab/{page-ad3bd8084048bee4.js → page-24282bf9f90031a5.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/bots/page-8b7c5ab6b5bc8d64.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ceo/{page-5c5d0e370f495ef6.js → page-4eb84d454f8c3415.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/chat/{page-c325e21543cd48a8.js → page-b2e494bdad57fcb5.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/consult/{page-6168a7c43e6ef8f5.js → page-c0ab229673a8b0eb.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/control/page-3e2b45f3c36dbac4.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/decisions/page-c22aa511b28743ad.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/design-check/page-26345242e4049f55.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/editor/{page-25bb425afdb27541.js → page-bf88c0f558d8ef54.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/exec/{page-32d72fdef1afe11a.js → page-52fec523e5b5d3ad.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/field/page-67723dfdcde92b31.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/inside/{page-2bd9ce9c020d02d4.js → page-22725e2570f1126b.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/intake/{page-a34096245e63b9f2.js → page-1891335950d68b70.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/layout-3267af1ebdc73cf5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/live/{page-4cc66d22ca4ec502.js → page-c314b75c11234998.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/live-run/{page-83b14c729fcea837.js → page-9f9c85415f61782b.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/loops/page-c1d4276059085587.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/market/page-833aab587f26ad4c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/missions/[id]/{page-ccdb83ed40c8d724.js → page-8526c17ae38e6a50.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/morning/{page-7ad3de8fa06fdea9.js → page-f998637a8402115c.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/not-found-b7bb53828b17b335.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/observatory/page-1b1850de0257e46d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/checklist/{page-786111f30b6f0969.js → page-6f23159fedb41f67.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/release/{page-48fb31810693c357.js → page-1d809a87139dd024.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/seats/{page-4e17cce5e2ed0fee.js → page-da9e570234d481fd.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/outputs/{page-cb154acadb025414.js → page-b92caddb5c715f30.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/page-8ce3a38e210b810a.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/reflection/{page-c2dd2b19c93f90ab.js → page-57ffeeb6019440fd.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/scheduler/page-abbb417bf514630b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/sessions/{page-d3c007c8a0ddd7ee.js → page-2c2b3f29cec1efa4.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/settings/devices/{page-7837ce2aacb97b04.js → page-9369279f72fc7b51.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/settings/page-64861f14039499c3.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/setup/done/page-ddc4c93efd8ab94b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/setup/page-901a1391b81053bc.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/share/page-68d9bbaac9841a58.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/showroom/{page-119be9a0444d2482.js → page-b5c0c996f329371e.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/tasks/{page-fa806c018833e332.js → page-89881ccbcf728978.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/term/page-0e87a803b3d7165e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/today/{page-d431fbdea6ad93dd.js → page-58d4e5cb30ef4871.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/trace/page-02ce4c3b5f39e4d2.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/vault/page-e620d1bb8eadcec2.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workflows/chat-ui/{page-847c3de6ca8a4416.js → page-acf868c64d77f950.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/workflows/page-54fa605fa6dba457.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workspace/{page-74e69ba2c8e970d4.js → page-bc79f0833533fa84.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/worktrees/{page-70038f41ae7cf931.js → page-d5c4398eea03d8f3.js} +1 -1
- package/apps/pwa/out/_next/static/css/72becfd42807b50d.css +1 -0
- package/apps/pwa/out/approvals/index.html +1 -1
- package/apps/pwa/out/approvals/index.txt +14 -14
- package/apps/pwa/out/autopilot/index.html +1 -1
- package/apps/pwa/out/autopilot/index.txt +14 -14
- package/apps/pwa/out/board/index.html +1 -1
- package/apps/pwa/out/board/index.txt +14 -14
- package/apps/pwa/out/botlab/index.html +1 -1
- package/apps/pwa/out/botlab/index.txt +14 -14
- package/apps/pwa/out/bots/index.html +1 -1
- package/apps/pwa/out/bots/index.txt +14 -14
- package/apps/pwa/out/ceo/index.html +1 -1
- package/apps/pwa/out/ceo/index.txt +14 -14
- package/apps/pwa/out/chat/index.html +1 -1
- package/apps/pwa/out/chat/index.txt +14 -14
- package/apps/pwa/out/consult/index.html +1 -1
- package/apps/pwa/out/consult/index.txt +14 -14
- package/apps/pwa/out/control/index.html +1 -1
- package/apps/pwa/out/control/index.txt +14 -14
- package/apps/pwa/out/decisions/index.html +1 -1
- package/apps/pwa/out/decisions/index.txt +14 -14
- package/apps/pwa/out/design-check/index.html +1 -1
- package/apps/pwa/out/design-check/index.txt +14 -14
- package/apps/pwa/out/editor/index.html +1 -1
- package/apps/pwa/out/editor/index.txt +14 -14
- package/apps/pwa/out/exec/index.html +1 -1
- package/apps/pwa/out/exec/index.txt +14 -14
- package/apps/pwa/out/field/index.html +1 -1
- package/apps/pwa/out/field/index.txt +14 -14
- package/apps/pwa/out/healthz +1 -1
- package/apps/pwa/out/index.html +1 -1
- package/apps/pwa/out/index.txt +14 -14
- package/apps/pwa/out/inside/index.html +1 -1
- package/apps/pwa/out/inside/index.txt +14 -14
- package/apps/pwa/out/intake/index.html +1 -1
- package/apps/pwa/out/intake/index.txt +14 -14
- package/apps/pwa/out/live/index.html +1 -1
- package/apps/pwa/out/live/index.txt +14 -14
- package/apps/pwa/out/live-run/index.html +1 -1
- package/apps/pwa/out/live-run/index.txt +14 -14
- package/apps/pwa/out/loops/index.html +1 -1
- package/apps/pwa/out/loops/index.txt +14 -14
- package/apps/pwa/out/market/index.html +1 -1
- package/apps/pwa/out/market/index.txt +14 -14
- package/apps/pwa/out/missions/placeholder/index.html +1 -1
- package/apps/pwa/out/missions/placeholder/index.txt +14 -14
- package/apps/pwa/out/morning/index.html +1 -1
- package/apps/pwa/out/morning/index.txt +14 -14
- package/apps/pwa/out/observatory/index.html +1 -1
- package/apps/pwa/out/observatory/index.txt +14 -14
- package/apps/pwa/out/ops/checklist/index.html +1 -1
- package/apps/pwa/out/ops/checklist/index.txt +14 -14
- package/apps/pwa/out/ops/release/index.html +1 -1
- package/apps/pwa/out/ops/release/index.txt +14 -14
- package/apps/pwa/out/ops/seats/index.html +1 -1
- package/apps/pwa/out/ops/seats/index.txt +14 -14
- package/apps/pwa/out/outputs/index.html +1 -1
- package/apps/pwa/out/outputs/index.txt +14 -14
- package/apps/pwa/out/reflection/index.html +1 -1
- package/apps/pwa/out/reflection/index.txt +15 -15
- package/apps/pwa/out/scheduler/index.html +1 -1
- package/apps/pwa/out/scheduler/index.txt +14 -14
- package/apps/pwa/out/sessions/index.html +1 -1
- package/apps/pwa/out/sessions/index.txt +14 -14
- package/apps/pwa/out/settings/devices/index.html +1 -1
- package/apps/pwa/out/settings/devices/index.txt +14 -14
- package/apps/pwa/out/settings/index.html +1 -1
- package/apps/pwa/out/settings/index.txt +14 -14
- package/apps/pwa/out/setup/done/index.html +1 -1
- package/apps/pwa/out/setup/done/index.txt +14 -14
- package/apps/pwa/out/setup/index.html +1 -1
- package/apps/pwa/out/setup/index.txt +14 -14
- package/apps/pwa/out/share/index.html +1 -1
- package/apps/pwa/out/share/index.txt +14 -14
- package/apps/pwa/out/showroom/index.html +1 -1
- package/apps/pwa/out/showroom/index.txt +14 -14
- package/apps/pwa/out/tasks/index.html +1 -1
- package/apps/pwa/out/tasks/index.txt +14 -14
- package/apps/pwa/out/term/index.html +1 -1
- package/apps/pwa/out/term/index.txt +14 -14
- package/apps/pwa/out/today/index.html +1 -1
- package/apps/pwa/out/today/index.txt +14 -14
- package/apps/pwa/out/trace/index.html +1 -1
- package/apps/pwa/out/trace/index.txt +14 -14
- package/apps/pwa/out/vault/index.html +1 -1
- package/apps/pwa/out/vault/index.txt +14 -14
- package/apps/pwa/out/workflows/chat-ui/index.html +1 -1
- package/apps/pwa/out/workflows/chat-ui/index.txt +14 -14
- package/apps/pwa/out/workflows/index.html +1 -1
- package/apps/pwa/out/workflows/index.txt +14 -14
- package/apps/pwa/out/workspace/index.html +1 -1
- package/apps/pwa/out/workspace/index.txt +14 -14
- package/apps/pwa/out/worktrees/index.html +1 -1
- package/apps/pwa/out/worktrees/index.txt +14 -14
- package/catalog/models/openrouter/deepseek-v4-pro-0813.yaml +39 -0
- package/catalog/models/openrouter/deepseek-v4.1-flash.yaml +38 -0
- package/catalog/models/openrouter/glm-5.3-flash.yaml +35 -0
- package/catalog/providers/openrouter.yaml +2 -0
- package/graphs/implement-loop.yaml +31 -0
- package/graphs/ops/harness-gc.yaml +17 -0
- package/graphs/ops/recipes.yaml +3 -0
- package/graphs/overlays/README.md +1 -1
- package/graphs/overlays/stop-heal-ladder.yaml +105 -0
- package/package.json +1 -1
- package/scripts/add-api-key.sh +1 -1
- package/scripts/ci-typecheck-changed.ts +71 -14
- package/scripts/fleet/bedrock-pod-secret.ts +122 -0
- package/scripts/fleet/node-setup-linux.sh +97 -0
- package/scripts/fleet/node-setup.sh +11 -0
- package/scripts/fleet/zai-pod-secret.ts +86 -0
- package/scripts/land-global-checks.ts +3 -0
- package/scripts/measure-log-diet-2.ts +213 -0
- package/scripts/measure-tsc-affected-roots.ts +289 -0
- package/scripts/release-loop/gate-executor.ts +28 -0
- package/scripts/release-loop/gate-node.ts +262 -26
- package/scripts/release-loop/resume-release.ts +9 -3
- package/scripts/release-loop/resume.ts +41 -0
- package/scripts/release-loop/unattended-release.ts +22 -7
- package/scripts/tsc-baseline-cache.ts +147 -0
- package/scripts/usage-rollup.ts +58 -6
- package/src/agent/run-context.ts +10 -0
- package/src/agent-mission/driver.ts +18 -3
- package/src/agent-mission/resource-ladder.ts +8 -3
- package/src/agent-mission/review-loop.ts +8 -2
- package/src/agent-substrate/acp-implementer.ts +139 -0
- package/src/agent-substrate/pr-reviewer.ts +31 -4
- package/src/agent-substrate/self-review-cli.ts +48 -9
- package/src/autopilot/mission-critique.ts +18 -5
- package/src/autopilot/mission-feasibility-negotiate.ts +11 -10
- package/src/autopilot/mission-trace.ts +12 -1
- package/src/autopilot/system-lookback-run.ts +16 -10
- package/src/bench/bench.ts +183 -0
- package/src/budget/fetchers/grok.ts +15 -5
- package/src/budget/llm-cost.ts +37 -5
- package/src/budget/types.ts +8 -2
- package/src/budget/unified-usage.ts +88 -6
- package/src/cli/bench-cli.ts +34 -0
- package/src/cli/guardian-cli.ts +26 -1
- package/src/cli/image-cli.ts +101 -0
- package/src/cli/log-event-names.ts +4 -0
- package/src/cli/pr-cli.ts +62 -5
- package/src/cli/pr-land-overlap-evidence.ts +160 -12
- package/src/cli/release-cli.ts +18 -9
- package/src/cli/rfc-status.ts +52 -2
- package/src/cli/self-update.ts +104 -0
- package/src/cli/tasks-cli.ts +109 -4
- package/src/cli/usage-cli.ts +91 -8
- package/src/core-turn/run-goal-loop.ts +5 -4
- package/src/dashboard/ad-run-setup-bindings.ts +279 -0
- package/src/dashboard/index.ts +18 -276
- package/src/dashboard/stale-install-check.ts +11 -0
- package/src/debug/log-diet.ts +38 -7
- package/src/debug/log.ts +19 -1
- package/src/domains/release-tool.ts +39 -0
- package/src/error-report/report.ts +21 -4
- package/src/github/pr-labels.ts +1 -0
- package/src/graph-runner/graph-cli.ts +42 -1
- package/src/graph-runner/runner.ts +167 -6
- package/src/grok/usage.ts +23 -2
- package/src/grounding/ref-index.ts +253 -0
- package/src/harness/dispatch-backpressure.ts +10 -2
- package/src/harness/harness-clean.ts +199 -78
- package/src/harness/harness-cli-command.ts +122 -7
- package/src/harness/harness-gc.ts +331 -0
- package/src/harness/harness-stop.ts +4 -2
- package/src/harness/harness-substrate-default.ts +142 -7
- package/src/harness/launch-registry.ts +6 -0
- package/src/harness/worktree-branch-prefix.ts +9 -0
- package/src/hq/fence.ts +187 -0
- package/src/hq/seats.ts +38 -22
- package/src/images/image-ledger.ts +179 -0
- package/src/index.ts +93 -11
- package/src/instance/child-scope.ts +32 -1
- package/src/instance/dev-project.ts +133 -0
- package/src/instance/resolve.ts +24 -0
- package/src/instance/spawn-elanous.ts +45 -0
- package/src/intake-plane/ref-prefetch.ts +110 -0
- package/src/intelligence-map/model-catalog.ts +72 -0
- package/src/llm/bedrock-eventstream.ts +215 -0
- package/src/llm/bedrock-provider.ts +114 -41
- package/src/llm/bedrock.ts +174 -28
- package/src/llm/openrouter-byok-cost.ts +96 -0
- package/src/llm/openrouter-deepseek.ts +151 -0
- package/src/llm/openrouter-upstream-pins.config.ts +25 -0
- package/src/llm/zai-coding-plan.ts +700 -0
- package/src/llm.ts +87 -16
- package/src/loops/checker.ts +148 -9
- package/src/loops/orchestrator/resync-shadow.ts +34 -7
- package/src/maturity/feature-maturity.ts +2 -2
- package/src/model-tier/llm-tier-map.ts +9 -5
- package/src/models/prompts.ts +2 -1
- package/src/mss/logging/log-store.ts +58 -11
- package/src/nexus/api/missions.ts +73 -2
- package/src/nexus/api/ops-api.ts +18 -0
- package/src/nexus/index.ts +7 -75
- package/src/nexus/tools-kind.ts +78 -0
- package/src/oauth/codex-account-rotation.ts +39 -2
- package/src/oauth/codex-account-store.ts +323 -24
- package/src/oauth/fallback-chain.ts +53 -15
- package/src/onboarding/first-run.ts +37 -18
- package/src/onboarding/web-first.ts +20 -2
- package/src/ops/machine-heartbeat.ts +185 -0
- package/src/ops/overview-cache.ts +175 -0
- package/src/ops/overview.ts +8 -0
- package/src/prompt-cache/openai.ts +3 -0
- package/src/prompt-cache/types.ts +5 -0
- package/src/prompt-enhance/enhance.ts +23 -0
- package/src/release-loop/placement.ts +16 -1
- package/src/requirements/requirement-ledger.ts +536 -0
- package/src/retro/daily.ts +5 -5
- package/src/seat-loop/seat-loop.ts +32 -10
- package/src/self-dev/dev-cli.ts +44 -10
- package/src/self-dev/dev-pipeline.ts +7 -2
- package/src/self-dev/draft-residue.ts +14 -2
- package/src/self-dev/draft-sweep.ts +103 -0
- package/src/self-dev/graph-journey-route.ts +6 -0
- package/src/self-dev/orchestrate-cli.ts +25 -3
- package/src/self-dev/orchestrate.ts +41 -6
- package/src/self-dev/run-supervisor.ts +188 -15
- package/src/self-dev/run-types.ts +13 -0
- package/src/self-dev/supervisor-verdict-edges.ts +1 -0
- package/src/self-implement/budget-gate.ts +53 -13
- package/src/self-implement/child-context-composition.ts +139 -0
- package/src/self-implement/child-llm-preference.ts +13 -5
- package/src/self-implement/claude-acp-implement.ts +67 -0
- package/src/self-implement/flex-signals.ts +59 -0
- package/src/self-implement/frozen-merges.ts +8 -3
- package/src/self-implement/goal-author-decompose.ts +10 -2
- package/src/self-implement/goal-author.ts +24 -9
- package/src/self-implement/graph-overlay-condition.ts +28 -0
- package/src/self-implement/graph-overlay-yaml.ts +41 -7
- package/src/self-implement/graph-templates.ts +33 -1
- package/src/self-implement/graph-yaml.ts +56 -0
- package/src/self-implement/heal-intake.ts +5 -1
- package/src/self-implement/orchestrator.ts +144 -15
- package/src/self-implement/running-runs.ts +75 -5
- package/src/self-implement/seams.ts +47 -1
- package/src/self-implement/self-implement-cli.ts +9 -4
- package/src/self-implement/stop-autoheal.ts +69 -2
- package/src/session-runtime/retry-policy.ts +2 -1
- package/src/steward/launch.ts +5 -4
- package/src/steward/triage-budget.ts +62 -0
- package/src/steward/triage.ts +83 -22
- package/src/task-agent/actions.ts +62 -5
- package/src/task-agent/completion-evidence.ts +5 -0
- package/src/task-agent/cover.ts +101 -3
- package/src/task-agent/judge.ts +16 -0
- package/src/task-agent/land-order-ledger.ts +95 -0
- package/src/task-agent/live-move-history.ts +87 -1
- package/src/task-agent/live-moves.ts +350 -20
- package/src/task-agent/rejudge-on-head.ts +75 -20
- package/src/task-agent/review-prior-fix.ts +146 -0
- package/src/task-agent/run-adopt.ts +136 -0
- package/src/task-agent/shadow.ts +53 -9
- package/src/task-agent/task-hand.ts +54 -7
- package/src/task-orchestrator/surfaces/pod-account-broker.ts +38 -5
- package/src/task-orchestrator/surfaces/pod-bedrock-child.ts +74 -0
- package/src/task-orchestrator/surfaces/pod-child-default.ts +50 -0
- package/src/task-orchestrator/surfaces/pod-child-providers.ts +14 -1
- package/src/task-orchestrator/surfaces/pod-failure-reason.ts +79 -6
- package/src/task-orchestrator/surfaces/pod-lease.ts +17 -5
- package/src/task-orchestrator/surfaces/pod-ledger-collect.ts +11 -5
- package/src/task-orchestrator/surfaces/pod-pool.ts +20 -4
- package/src/task-orchestrator/surfaces/pod-zai-child.ts +101 -0
- package/src/task-orchestrator/surfaces/self-implement-pod.ts +213 -23
- package/src/task-orchestrator/surfaces/self-implement.ts +19 -1
- package/src/typecheck-ratchet.ts +1 -0
- package/src/user-config.ts +51 -11
- package/src/version/packed-revision.json +1 -1
- package/src/workflow-runtime/triggers/registry.ts +59 -0
- package/src/workflow-runtime/triggers/webhook-source.ts +35 -5
- package/apps/pwa/out/_next/static/chunks/app/autopilot/page-0160ab83c8685a12.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/bots/page-e6784759a2531031.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/control/page-31417d3604154a6d.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/decisions/page-e9606dc0bdec9305.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/design-check/page-a648bc464c9dc754.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/field/page-d27b794c8ba76bbb.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/layout-22170f4f00a15496.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/loops/page-6e2781c523dad761.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/market/page-26dd415c8f13f341.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/not-found-015c02c4fec4023e.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/observatory/page-6b0ae5b5c42f6d2f.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/page-16258e956ea93fc2.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/scheduler/page-85f35dd1dd96ae87.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/settings/page-7350ee93f4fe1033.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/setup/done/page-00723458575f2b19.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/setup/page-745d9b6c015076f8.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/share/page-736d33c7414a8a1a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/term/page-c782041b5f5199ba.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/trace/page-0c056abff82b9891.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/vault/page-242d668e4fbf72c4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/workflows/page-81dd0d8ae923ed72.js +0 -1
- package/apps/pwa/out/_next/static/css/5619d59e87eddf66.css +0 -1
- /package/apps/pwa/out/_next/static/{rfkMm4movqkqMebBrCXiR → Q-7iofoxOEv1WX18S35of}/_buildManifest.js +0 -0
- /package/apps/pwa/out/_next/static/{rfkMm4movqkqMebBrCXiR → Q-7iofoxOEv1WX18S35of}/_ssgManifest.js +0 -0
|
@@ -16,6 +16,7 @@ import { mergePartialFailures, readPartialPlan, testFileOfId } from './gate-part
|
|
|
16
16
|
import { logPlanWalltime, planDurations, planShards, readFileMemory } from './shard-plan';
|
|
17
17
|
import { deriveCdpTestPatterns } from '../test-deterministic';
|
|
18
18
|
import { emitNodeResult, readGraphContext } from './node-verdict.js';
|
|
19
|
+
import { isGateExecutor, type GateExecutor } from './gate-executor.js';
|
|
19
20
|
import { runPodCommand, parsePodCpu, resolvePodCpu, type RunPodCommandOptions, type PodCommandResult, type PodCpu } from '../../src/task-orchestrator/surfaces/pod-command-job.js';
|
|
20
21
|
import { POD_BUN_CACHE_HOST_PATH, parseInstallSeconds, podBunCacheVolume } from '../../src/task-orchestrator/surfaces/pod-bun-cache.js';
|
|
21
22
|
import { PodPoolScheduler, parsePodPool, checkPodPool } from '../../src/task-orchestrator/surfaces/pod-pool.js';
|
|
@@ -45,6 +46,8 @@ export interface GateOptions {
|
|
|
45
46
|
baselineCommit?: string;
|
|
46
47
|
remote?: string;
|
|
47
48
|
remoteMirror?: string;
|
|
49
|
+
/** GATE-HOST-EXECUTOR — `host` ignores every pod option and sweeps on `remote` (required). Absent = today's choice (pod when a pool is set). */
|
|
50
|
+
executor?: GateExecutor;
|
|
48
51
|
instanceRoot?: string;
|
|
49
52
|
ledgerRoot?: string;
|
|
50
53
|
repo?: string;
|
|
@@ -62,6 +65,16 @@ export interface GateOptions {
|
|
|
62
65
|
admissionWaitSeconds?: number;
|
|
63
66
|
/** GATE-MEM-ADMIT — an injected admission (tests · embedders); config never sets it. Absent = the gate's own pool's. */
|
|
64
67
|
admission?: GateAdmission;
|
|
68
|
+
/** GATE-POD-STALL-FAILFAST — seconds after the source checkout a shard may go without any test output (outside an
|
|
69
|
+
* install-slot wait) before it is stopped as unmeasured (`gatePodStallSeconds`); omitted = {@link GATE_POD_STALL_SECONDS_DEFAULT} · 0 = off. */
|
|
70
|
+
stallSeconds?: number;
|
|
71
|
+
/** GATE-POD-STALL-FAILFAST — injected live Pod log reader / Job stop (tests · embedders); config never sets them.
|
|
72
|
+
* Absent = kubectl on the gate's own pool; an injected pool without `liveLog` runs without the stall watch. */
|
|
73
|
+
liveLog?: (job: { name: string; context?: string }) => string | null;
|
|
74
|
+
/** false = the stop did not take; the watch asks again every poll until it does (void = taken). */
|
|
75
|
+
stopJob?: (job: { name: string; context?: string }) => boolean | void;
|
|
76
|
+
/** How often the stall watch reads the live log (ms); omitted = {@link GATE_POD_STALL_POLL_MS}. */
|
|
77
|
+
stallPollMs?: number;
|
|
65
78
|
/** GATE-LIVE-OBS — the shard state file (`gateShardsPath(version)`) this sweep keeps current; absent = not written. */
|
|
66
79
|
shardsFile?: { path: string; version: string } };
|
|
67
80
|
}
|
|
@@ -83,7 +96,9 @@ export interface GateResult {
|
|
|
83
96
|
baselineDeferred?: { cutFiles: number; swept: number };
|
|
84
97
|
durationMs: number;
|
|
85
98
|
error?: string;
|
|
86
|
-
stalledShards?: Array<{ shard: number; files: string[]; reason: 'incomplete' | 'no-output' | 'job-failed' | 'unattributed' | 'timeout'; lastFile?: string; summaryFailures?: number; namedFailures?: number; detail?: string
|
|
99
|
+
stalledShards?: Array<{ shard: number; files: string[]; reason: 'incomplete' | 'no-output' | 'job-failed' | 'exit-code' | 'unattributed' | 'timeout'; lastFile?: string; summaryFailures?: number; namedFailures?: number; detail?: string;
|
|
100
|
+
/** GATE-POD-STALL-FAILFAST — the shard (and its one fresh retry) printed no test output within the stall window: unmeasured. */
|
|
101
|
+
stall?: { seconds: number; sinceCheckoutMs: number } }>;
|
|
87
102
|
partialSummary?: { pass: number; fail: number; errors: number; ran: number; files: number };
|
|
88
103
|
/** GATE-PARTIAL — this verdict re-ran only these files and carried the prior gate's results for the rest. */
|
|
89
104
|
partial?: { priorCommit: string; files: number };
|
|
@@ -255,13 +270,14 @@ export function shardParts(paths: readonly string[], seconds: (file: string) =>
|
|
|
255
270
|
* leaving `install.log`/`install.rc`, `part-<i>.log`/`.rc`/`.junit.xml`, and `shard.timeout` (the part index) when the
|
|
256
271
|
* deadline cut the shard. A part without `.rc` did not finish. The shell itself always exits 0 (as before).
|
|
257
272
|
*/
|
|
258
|
-
export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPatterns: readonly string[]; softSeconds: number; cachePrefix: string; installSlots: number; depsDir?: string }): string {
|
|
273
|
+
export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPatterns: readonly string[]; softSeconds: number; cachePrefix: string; installSlots: number; depsDir?: string; beatSeconds?: number }): string {
|
|
259
274
|
const run = o.parts.map((part, index) => {
|
|
260
275
|
const ignores = o.cdpPatterns.filter((pattern) => part.includes(pattern)).flatMap((pattern) => ['--path-ignore-patterns', pattern]);
|
|
261
276
|
return `run_part ${index} ${['bun', 'run', 'test:deterministic', ...ignores, ...part.map(asPath)].map(quote).join(' ')}`;
|
|
262
277
|
}).join(' && ');
|
|
263
278
|
return [
|
|
264
279
|
'mkdir -p "$HOME/outbox"; O="$HOME/outbox"',
|
|
280
|
+
...(o.beatSeconds ? gateBeatScript(o.beatSeconds, o.installSlots > 0) : []),
|
|
265
281
|
...(o.cachePrefix ? [o.cachePrefix] : []),
|
|
266
282
|
// Seconds since this container's PID 1 started (the clone ran before this shell) — 0 when /proc cannot tell.
|
|
267
283
|
`pod_age=$(awk -v t="$(cut -d' ' -f22 /proc/1/stat 2>/dev/null)" -v hz="$(getconf CLK_TCK 2>/dev/null)" '{ if (t > 0 && hz > 0) print int($1 - t / hz); else print 0 }' /proc/uptime 2>/dev/null)`,
|
|
@@ -286,9 +302,109 @@ export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPa
|
|
|
286
302
|
'true',
|
|
287
303
|
].join('\n');
|
|
288
304
|
}
|
|
305
|
+
/**
|
|
306
|
+
* GATE-POD-STALL-FAILFAST (0.2.24) — the shard shell's live heartbeat on the Pod's stdout (everything else goes to the
|
|
307
|
+
* outbox, which the host sees only after the Job ends). 10-10 0.2.24 gate: the node's containerd was unresponsive, all 28
|
|
308
|
+
* shards ran to their 1697 s deadline with `finishedFiles: 0`, then retries and splits did the same for 2h+ and the node
|
|
309
|
+
* never finished. The beat lets the host see «no test output yet» while the Job runs:
|
|
310
|
+
* `ELANOUS_GATE_BEAT <seq> <slot|install|test> <0|1>` — the last field is 1 once any part log has a byte.
|
|
311
|
+
* It is observation only: the host hashes the shell without it into `runKey`.
|
|
312
|
+
*/
|
|
313
|
+
export function gateBeatScript(beatSeconds: number, installSlots: boolean): string[] {
|
|
314
|
+
return [
|
|
315
|
+
'gate_beat() {',
|
|
316
|
+
' local n=0 st t f',
|
|
317
|
+
' while :; do',
|
|
318
|
+
' n=$((n + 1)); t=0',
|
|
319
|
+
' for f in "$O"/part-*.log; do if [ -s "$f" ]; then t=1; break; fi; done',
|
|
320
|
+
' if [ -f "$O/install.rc" ]; then st=test',
|
|
321
|
+
// A deps miss with no slot line yet = waiting for an install slot (bounded by the slot script; not a stall).
|
|
322
|
+
...(installSlots ? [" elif [ -f \"$O/install.log\" ] && grep -q '^\\[gate\\] deps .* miss' \"$O/install.log\" 2>/dev/null && ! grep -q '^\\[gate\\] install-slot' \"$O/install.log\" 2>/dev/null; then st=slot"] : []),
|
|
323
|
+
' else st=install; fi',
|
|
324
|
+
' echo "ELANOUS_GATE_BEAT $n $st $t"',
|
|
325
|
+
` sleep ${Math.max(1, Math.floor(beatSeconds))} >/dev/null 2>&1 </dev/null`,
|
|
326
|
+
' done',
|
|
327
|
+
'}',
|
|
328
|
+
'gate_beat & gate_beat_pid=$!',
|
|
329
|
+
// Stopped before the shell returns, so no beat line lands between the Job's artifact lines.
|
|
330
|
+
'trap \'kill "$gate_beat_pid" 2>/dev/null; wait "$gate_beat_pid" 2>/dev/null\' EXIT',
|
|
331
|
+
];
|
|
332
|
+
}
|
|
333
|
+
export const GATE_POD_STALL_SECONDS_DEFAULT = 300;
|
|
334
|
+
export const GATE_POD_STALL_POLL_MS = 15_000;
|
|
335
|
+
/** At least ten beats per stall window, never slower than one per 30 s. */
|
|
336
|
+
export const gateStallBeatSeconds = (stallSeconds: number) => Math.min(30, Math.max(1, Math.floor(stallSeconds / 10)));
|
|
337
|
+
export interface ShardLiveLog { checkout: boolean; beat?: { seq: number; stage: 'slot' | 'install' | 'test'; testOutput: boolean } }
|
|
338
|
+
/** What the Pod's live stdout says so far: the source checkout line and the latest beat. */
|
|
339
|
+
export function readShardLiveLog(text: string): ShardLiveLog {
|
|
340
|
+
let checkout = false;
|
|
341
|
+
let beat: ShardLiveLog['beat'];
|
|
342
|
+
for (const line of text.split(/\r?\n/)) {
|
|
343
|
+
if (/^ELANOUS_POD_SOURCE (?:commit|default|pr) /.test(line)) checkout = true;
|
|
344
|
+
const match = /^ELANOUS_GATE_BEAT (\d+) (slot|install|test) ([01])\s*$/.exec(line);
|
|
345
|
+
if (match) beat = { seq: Number(match[1]), stage: match[2] as 'slot' | 'install' | 'test', testOutput: match[3] === '1' };
|
|
346
|
+
}
|
|
347
|
+
return { checkout, ...(beat ? { beat } : {}) };
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* GATE-POD-STALL-FAILFAST — watches one running shard Job. Armed once the source checkout line is seen; disarmed for good
|
|
351
|
+
* the moment any test output exists (that shard then behaves exactly as before). Stalled = `stallSeconds` since the
|
|
352
|
+
* checkout with no test output, not counting time the beat reported an install-slot wait. A silent shell (no beat at all)
|
|
353
|
+
* counts like any other stage. Only successful reads decide; a read failure never stalls a shard. `graceMs` covers the
|
|
354
|
+
* beat's report lag: output that appeared just inside the window is reported by the next beat before the stop decides.
|
|
355
|
+
*/
|
|
356
|
+
export function watchShardStall(o: { stallSeconds: number; pollMs: number; read: () => string | null; onStall: (sinceCheckoutMs: number) => void; now?: () => number; graceMs?: number;
|
|
357
|
+
/** Stops the Job; false = not taken, asked again every poll until it is or the watch ends (a stalled Job must not run to its deadline). */
|
|
358
|
+
stopJob?: () => boolean | void }): { stop(): void; stalled(): { sinceCheckoutMs: number } | undefined } {
|
|
359
|
+
const now = o.now ?? Date.now;
|
|
360
|
+
let checkoutAt: number | undefined;
|
|
361
|
+
let windowStart: number | undefined;
|
|
362
|
+
let lastSeq: number | undefined;
|
|
363
|
+
let hit: { sinceCheckoutMs: number } | undefined;
|
|
364
|
+
let done = false;
|
|
365
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
366
|
+
let ended = false;
|
|
367
|
+
const askStop = () => {
|
|
368
|
+
timer = undefined;
|
|
369
|
+
if (ended) return;
|
|
370
|
+
let taken: boolean | void = false;
|
|
371
|
+
try { taken = o.stopJob ? o.stopJob() : true; } catch { taken = false; }
|
|
372
|
+
if (taken === false && !ended) timer = setTimeout(askStop, o.pollMs);
|
|
373
|
+
};
|
|
374
|
+
const tick = () => {
|
|
375
|
+
timer = undefined;
|
|
376
|
+
if (done) return;
|
|
377
|
+
let text: string | null = null;
|
|
378
|
+
try { text = o.read(); } catch { text = null; }
|
|
379
|
+
if (done) return;
|
|
380
|
+
if (text !== null) {
|
|
381
|
+
const t = now();
|
|
382
|
+
const live = readShardLiveLog(text);
|
|
383
|
+
if (live.beat?.testOutput) { done = true; return; }
|
|
384
|
+
if (live.checkout && checkoutAt === undefined) { checkoutAt = t; windowStart = t; }
|
|
385
|
+
if (live.beat && live.beat.seq !== lastSeq) {
|
|
386
|
+
lastSeq = live.beat.seq;
|
|
387
|
+
if (live.beat.stage === 'slot' && windowStart !== undefined) windowStart = t;
|
|
388
|
+
}
|
|
389
|
+
if (checkoutAt !== undefined && windowStart !== undefined && t - windowStart >= o.stallSeconds * 1000 + (o.graceMs ?? 0)) {
|
|
390
|
+
done = true;
|
|
391
|
+
hit = { sinceCheckoutMs: t - checkoutAt };
|
|
392
|
+
o.onStall(hit.sinceCheckoutMs);
|
|
393
|
+
askStop();
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
timer = setTimeout(tick, o.pollMs);
|
|
398
|
+
};
|
|
399
|
+
timer = setTimeout(tick, o.pollMs);
|
|
400
|
+
return { stop: () => { done = true; ended = true; if (timer) clearTimeout(timer); }, stalled: () => hit };
|
|
401
|
+
}
|
|
289
402
|
/** Removes bun's own summary lines so one summary over several runs can follow (same rule as the sweep's merge). */
|
|
290
403
|
const stripSummary = (output: string) => output.replace(/(?:^|\n)\s*\d+ (?:pass|fail|errors?)\s*(?=\n|$)/g, '\n').replace(/Ran \d+ tests? across \d+ files?\.?/g, '');
|
|
291
404
|
export interface ShardArtifacts {
|
|
405
|
+
/** GATE-POD-STALL-FAILFAST (OP) — every unfinished part ended with a complete clean summary (0 fail · 0 error) but an exit
|
|
406
|
+
* other than 0/1: an exit-code anomaly (a test left process.exitCode set), with those parts' files. */
|
|
407
|
+
exitCodeOnly?: string[];
|
|
292
408
|
/** The verdict text: install log ⊕ every finished part (its summary removed) ⊕ one summary over the finished parts. */
|
|
293
409
|
output?: string;
|
|
294
410
|
junit?: string;
|
|
@@ -315,6 +431,8 @@ export function readShardArtifacts(dir: string, parts: readonly (readonly string
|
|
|
315
431
|
if (installRc !== 0) return { output: install, ...(installRc !== undefined ? { rc: installRc } : {}), timedOut, finished: [] };
|
|
316
432
|
const good: Array<{ log: string; junit?: string; rc: 0 | 1; files: readonly string[] }> = [];
|
|
317
433
|
const bad: string[] = [];
|
|
434
|
+
const exitAnomaly: string[] = [];
|
|
435
|
+
let otherBad = false;
|
|
318
436
|
let badRc: number | undefined;
|
|
319
437
|
parts.forEach((files, index) => {
|
|
320
438
|
const log = read(`part-${index}.log`);
|
|
@@ -326,6 +444,9 @@ export function readShardArtifacts(dir: string, parts: readonly (readonly string
|
|
|
326
444
|
return;
|
|
327
445
|
}
|
|
328
446
|
if (log !== undefined) bad.push(`# part ${index} (${rc === undefined ? 'unfinished' : `rc=${rc}`})\n${log}`);
|
|
447
|
+
if (clean !== undefined && rc !== undefined && rc !== 0 && rc !== 1 && Number(lastMatch(clean, /Ran (\d+) tests? across ([1-9]\d*) files?/)?.[1] ?? 0) > 0
|
|
448
|
+
&& summaryCount(clean, 'fail') === 0 && !(summaryCount(clean, 'error') > 0) && !/^# Unhandled error between tests/m.test(clean)) exitAnomaly.push(...files);
|
|
449
|
+
else otherBad = true;
|
|
329
450
|
// A finished part without a readable summary is not a verdict: its exit, or 2 when it claimed 0|1.
|
|
330
451
|
if (rc !== undefined && badRc === undefined) badRc = rc === 0 || rc === 1 ? 2 : rc;
|
|
331
452
|
});
|
|
@@ -346,6 +467,7 @@ export function readShardArtifacts(dir: string, parts: readonly (readonly string
|
|
|
346
467
|
timedOut: !complete && timedOut && badRc === undefined,
|
|
347
468
|
finished: good.flatMap((part) => [...part.files]),
|
|
348
469
|
...(bad.length ? { unfinishedLog: bad.join('\n') } : {}),
|
|
470
|
+
...(exitAnomaly.length && !otherBad ? { exitCodeOnly: exitAnomaly } : {}),
|
|
349
471
|
};
|
|
350
472
|
}
|
|
351
473
|
|
|
@@ -504,7 +626,11 @@ export function readIsolatedRun(run: CommandResult, label: string, file: string)
|
|
|
504
626
|
|
|
505
627
|
class StalledPodShards extends Error {
|
|
506
628
|
constructor(readonly stalledShards: NonNullable<GateResult['stalledShards']>, readonly partialSummary: NonNullable<GateResult['partialSummary']>) {
|
|
507
|
-
|
|
629
|
+
const unmeasured = stalledShards.filter((item) => item.stall);
|
|
630
|
+
const exitCode = stalledShards.filter((item) => item.reason === 'exit-code');
|
|
631
|
+
super(`pod sweep incomplete: stalled shards ${stalledShards.map(({ shard }) => shard).join(', ')}`
|
|
632
|
+
+ (unmeasured.length ? ` (unmeasured: no test output within ${unmeasured[0]!.stall!.seconds}s of checkout — shards ${[...new Set(unmeasured.map(({ shard }) => shard))].join(', ')})` : '')
|
|
633
|
+
+ (exitCode.length ? ` (exit-code anomaly: 0 fail but exit ≠ 0/1 — ${[...new Set(exitCode.flatMap(({ files }) => files))].join(', ')})` : ''));
|
|
508
634
|
}
|
|
509
635
|
}
|
|
510
636
|
|
|
@@ -694,6 +820,26 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
694
820
|
// An injected pool (tests) runs without admission unless `pod.admission` is passed; the gate's own pool asks it.
|
|
695
821
|
const gateAdmission = pod.admission ?? (poolOverride ? undefined : poolGateAdmission(poolScheduler));
|
|
696
822
|
debug.log('release-loop.gate', 'pod-admission', { waitSeconds: admissionWaitSeconds, source: pod.admissionWaitSeconds !== undefined ? 'config' : 'default', enabled: !!gateAdmission && admissionWaitSeconds > 0 });
|
|
823
|
+
// GATE-POD-STALL-FAILFAST: a shard with no test output within the stall window is stopped and recorded unmeasured.
|
|
824
|
+
const stallSeconds = pod.stallSeconds ?? GATE_POD_STALL_SECONDS_DEFAULT;
|
|
825
|
+
const stallPollMs = pod.stallPollMs ?? GATE_POD_STALL_POLL_MS;
|
|
826
|
+
if (!Number.isFinite(stallSeconds) || stallSeconds < 0 || !Number.isFinite(stallPollMs) || stallPollMs <= 0) throw new Error('invalid pod sweep options');
|
|
827
|
+
// An injected pool (tests) runs without the watch unless `pod.liveLog` is passed — never a real kubectl from a test.
|
|
828
|
+
const liveLog = pod.liveLog ?? (poolOverride ? undefined : (job: { name: string; context?: string }) => {
|
|
829
|
+
if (!job.context) return null;
|
|
830
|
+
const logs = defaultKubectl(['--context', job.context, '-n', 'elanous-test', 'logs', `job/${job.name}`, '-c', 'child', '--tail=400']);
|
|
831
|
+
return logs.status === 0 ? logs.stdout : null;
|
|
832
|
+
});
|
|
833
|
+
const stopJob = pod.stopJob ?? ((job: { name: string; context?: string }) => {
|
|
834
|
+
// Not `delete job`: runPodCommand would poll a missing Job forever. A 1 s deadline makes the Job fail now and
|
|
835
|
+
// runPodCommand returns through its normal path (logs · exit code · Secret cleanup).
|
|
836
|
+
if (!job.context) return false;
|
|
837
|
+
const patched = defaultKubectl(['--context', job.context, '-n', 'elanous-test', 'patch', 'job', job.name, '--type=merge', '-p', JSON.stringify({ spec: { activeDeadlineSeconds: 1 } })]);
|
|
838
|
+
if (patched.status !== 0) debug.log('release-loop.gate', 'pod-shard-stall-stop-failed', { job: job.name, error: patched.stderr.trim().slice(0, 300) }, { level: 'warn' });
|
|
839
|
+
return patched.status === 0;
|
|
840
|
+
});
|
|
841
|
+
const stallWatch = stallSeconds > 0 && !!liveLog;
|
|
842
|
+
debug.log('release-loop.gate', 'pod-stall-watch', { stallSeconds, source: pod.stallSeconds !== undefined ? 'config' : 'default', enabled: stallWatch });
|
|
697
843
|
const jobLogTail = (job: PodCommandResult): string => {
|
|
698
844
|
if (podLogTail) return podLogTail(job);
|
|
699
845
|
for (const member of poolScheduler.members) {
|
|
@@ -734,11 +880,11 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
734
880
|
/** The gate's scheduler, seen through one shard: a slot wait and the Pod start become row states. */
|
|
735
881
|
// GATE-MEM-ADMIT: `placed` releases the shard's admission once a member is chosen — from then on the Job is
|
|
736
882
|
// Pending/Running in the cluster and the next admission reading counts it, so holding the lease would count it twice.
|
|
737
|
-
const trackedPool = (shard: number, retry: boolean, placed: () => void = () => {}): PodPoolScheduler => ({
|
|
883
|
+
const trackedPool = (shard: number, retry: boolean, placed: (member: NonNullable<Awaited<ReturnType<PodPoolScheduler['tryAcquire']>>>) => void = () => {}): PodPoolScheduler => ({
|
|
738
884
|
members: poolScheduler.members,
|
|
739
885
|
tryAcquire: async (...args: Parameters<PodPoolScheduler['tryAcquire']>) => {
|
|
740
886
|
const member = await poolScheduler.tryAcquire(...args);
|
|
741
|
-
if (member) placed();
|
|
887
|
+
if (member) placed(member);
|
|
742
888
|
if (member) markShard(shard, { state: retry ? 'retry' : 'running', startedAt: board[shard]?.startedAt ?? new Date().toISOString() }, ['waitReason']);
|
|
743
889
|
else markShard(shard, { waitReason: 'Pod 자리 대기' });
|
|
744
890
|
return member;
|
|
@@ -751,8 +897,11 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
751
897
|
const shardDeadline = shardDeadlineSeconds(estimated(paths), deadlineSeconds);
|
|
752
898
|
const parts = shardParts(paths, (file) => estimated([file]));
|
|
753
899
|
const memoryLimit = paths.length === 1 && (depth >= 1 || heavy.has(paths[0]!)) ? POD_ISOLATION_MEMORY_LIMIT : undefined;
|
|
754
|
-
const
|
|
755
|
-
const
|
|
900
|
+
const shellOptions = { parts, cdpPatterns, softSeconds: softDeadlineSeconds(shardDeadline), cachePrefix, installSlots };
|
|
901
|
+
const keyShell = gateShardShell(shellOptions);
|
|
902
|
+
// The stall beat only reports; runKey keeps hashing the shell without it, so earlier results stay reusable.
|
|
903
|
+
const podShell = stallWatch ? gateShardShell({ ...shellOptions, beatSeconds: gateStallBeatSeconds(stallSeconds) }) : keyShell;
|
|
904
|
+
const runKey = createHash('sha256').update(JSON.stringify({ pool: pod.pool, podShell: keyShell, memoryLimit: memoryLimit ?? null, deadlineSeconds: shardDeadline, bunCache: bunCache ?? null })).digest('hex');
|
|
756
905
|
let start = Date.now();
|
|
757
906
|
let admitted: GateAdmissionResult | undefined;
|
|
758
907
|
let output: string | undefined;
|
|
@@ -764,6 +913,8 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
764
913
|
let timedOut = false;
|
|
765
914
|
let finished: string[] = [];
|
|
766
915
|
let unfinishedLog: string | undefined;
|
|
916
|
+
let exitCodeOnly: string[] | undefined;
|
|
917
|
+
let stall: { seconds: number; sinceCheckoutMs: number } | undefined;
|
|
767
918
|
// The way down a previous attempt took wins over its own log: that log was not a verdict, or it would have no children.
|
|
768
919
|
// The split this shard would take if it failed here — the same rule as the split code below (isolation at depth 2 ·
|
|
769
920
|
// cap chunks for a big root shard · halves otherwise · none for one file).
|
|
@@ -796,19 +947,34 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
796
947
|
admitted.outcome === 'granted' ? undefined : { level: 'warn' });
|
|
797
948
|
// The admission wait is not the shard's run time: deadline/timeout judgment and the measured duration start here.
|
|
798
949
|
start = Date.now();
|
|
950
|
+
const jobName = `gate-${randomUUID()}`;
|
|
951
|
+
// The member's context is known once the Job is placed; the live log is read (and the Job stopped) there.
|
|
952
|
+
let jobContext: string | undefined;
|
|
953
|
+
const liveJob = () => ({ name: jobName, ...(jobContext ? { context: jobContext } : {}) });
|
|
954
|
+
// Grace = two beats ⊕ one poll: test output that appeared inside the window is seen before the stop.
|
|
955
|
+
const watch = stallWatch ? watchShardStall({ stallSeconds, pollMs: stallPollMs, graceMs: 2 * gateStallBeatSeconds(stallSeconds) * 1000 + stallPollMs, read: () => liveLog!(liveJob()),
|
|
956
|
+
onStall: (sinceCheckoutMs) => debug.log('release-loop.gate', 'pod-shard-stall', { shard, branch, stallSeconds, sinceCheckoutMs }, { level: 'warn' }),
|
|
957
|
+
stopJob: () => {
|
|
958
|
+
try { return stopJob(liveJob()); }
|
|
959
|
+
catch (error) { debug.log('release-loop.gate', 'pod-shard-stall-stop-failed', { job: jobName, error: String(error) }, { level: 'warn' }); return false; }
|
|
960
|
+
} }) : undefined;
|
|
961
|
+
const placed = admitted.release;
|
|
799
962
|
const job = await podCommand({
|
|
800
|
-
pool: pod.pool, poolScheduler: trackedPool(shard, depth > 0 || branch !== '',
|
|
963
|
+
pool: pod.pool, poolScheduler: trackedPool(shard, depth > 0 || branch !== '', (member) => { jobContext = member.context; placed(); }), clone: true, source: { kind: 'commit', sha: commit }, deadlineSeconds: shardDeadline,
|
|
801
964
|
...(bunCache ? { bunCache } : {}),
|
|
802
965
|
...(installSlots > 0 ? { installSlots: POD_INSTALL_SLOTS_HOST_PATH } : {}),
|
|
803
966
|
...(pod.cpu ? { cpu: pod.cpu } : {}),
|
|
804
967
|
// 실패 뒤 다시 도는 파일 하나짜리 Job 은 메모리 한도를 올린다 — 16Gi 에선 무거운 한 파일이 혼자서도 OOM 이었다(09-30 `unwired-exports`).
|
|
805
968
|
...(memoryLimit ? { memoryLimit } : {}),
|
|
806
|
-
name:
|
|
969
|
+
name: jobName,
|
|
807
970
|
command: ['bash', '-lc', podShell],
|
|
808
|
-
});
|
|
971
|
+
}).finally(() => watch?.stop());
|
|
809
972
|
jobExitCode = job.exitCode;
|
|
810
973
|
const read = readShardArtifacts(job.artifactsDir, parts);
|
|
811
|
-
({ output, junit, rc, timedOut, finished, unfinishedLog } = read);
|
|
974
|
+
({ output, junit, rc, timedOut, finished, unfinishedLog, exitCodeOnly } = read);
|
|
975
|
+
// A stall counts only when the Job also left no verdict (a failed stop can let a late shard finish after all).
|
|
976
|
+
const stalledAt = watch?.stalled();
|
|
977
|
+
if (stalledAt && !finished.length && rc === undefined) stall = { seconds: stallSeconds, sinceCheckoutMs: stalledAt.sinceCheckoutMs };
|
|
812
978
|
// A Job the cluster cut at its own deadline leaves no outbox at all: past the deadline with nothing is a timeout too.
|
|
813
979
|
if (!timedOut && output === undefined && job.exitCode !== 0 && Date.now() - start >= shardDeadline * 1000) timedOut = true;
|
|
814
980
|
lastFile = output === undefined && unfinishedLog === undefined ? undefined : lastStartedTestFile(`${output ?? ''}\n${unfinishedLog ?? ''}`, paths);
|
|
@@ -837,13 +1003,14 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
837
1003
|
const unfinishedDestination = join(logDir, `pod-${shard}${branch}.unfinished.log`);
|
|
838
1004
|
if (unfinishedLog !== undefined) writeFileSync(unfinishedDestination, unfinishedLog, { mode: 0o600 });
|
|
839
1005
|
else rmSync(unfinishedDestination, { force: true });
|
|
840
|
-
writeFileSync(join(logDir, `pod-${shard}${branch}.json`), JSON.stringify({ durationMs, rc: rc ?? null,
|
|
1006
|
+
writeFileSync(join(logDir, `pod-${shard}${branch}.json`), JSON.stringify({ durationMs, rc: rc ?? null,
|
|
1007
|
+
...(stall ? { reason: 'stall', finishedFiles: 0, stallSeconds: stall.seconds, sinceCheckoutMs: stall.sinceCheckoutMs } : timedOut ? { reason: 'timeout', finishedFiles: finished.length } : {}), files: paths, plannedSeconds: estimated(paths), deadlineSeconds: shardDeadline, shardCount: shards.length, commit, runKey, attemptId, ...(parentAttempt ? { parentAttempt } : {}), jobExitCode: jobExitCode ?? null, jobFailed,
|
|
841
1008
|
...(admitted && admitted.outcome !== 'off' ? { admission: admitted.outcome, admissionWaitMs: admitted.waitedMs } : {}) }) + '\n', { mode: 0o600 });
|
|
842
1009
|
}
|
|
843
1010
|
const installSeconds = parseInstallSeconds(output ?? '');
|
|
844
1011
|
// GATE-TIMEOUT-HEAL ③: a cut shard is «reason: timeout» (with the files that still finished), not a bare rc=null.
|
|
845
1012
|
if (!reused && !descent) debug.log('release-loop.gate', 'pod-shard', { shard, files: paths, durationMs, rc: rc ?? null, attempt: depth + 1, installSeconds, gateDeps: parseGateDeps(output ?? ''), deadlineSeconds: shardDeadline, parts: parts.length,
|
|
846
|
-
...(timedOut ? { reason: 'timeout', finishedFiles: finished.length } : {}) });
|
|
1013
|
+
...(stall ? { reason: 'stall', finishedFiles: 0, stallSeconds: stall.seconds } : timedOut ? { reason: 'timeout', finishedFiles: finished.length } : {}) });
|
|
847
1014
|
if (!reused && !descent) markShard(shard, { ...(installSeconds !== null ? { installSec: installSeconds } : {}), ...(rc !== undefined ? { rc } : {}), ...(timedOut ? { state: 'timeout' as const } : {}) });
|
|
848
1015
|
let clean = output?.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '');
|
|
849
1016
|
const ran = clean && lastMatch(clean, /Ran (\d+) tests? across (\d+) files?/);
|
|
@@ -854,7 +1021,13 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
854
1021
|
let namedFailures: number | undefined;
|
|
855
1022
|
let unattributedDetail: string | undefined;
|
|
856
1023
|
if (descent === 'split') reason = 'incomplete';
|
|
1024
|
+
// GATE-POD-STALL-FAILFAST: stopped with no test output — the existing «no output» reason (one fresh retry).
|
|
1025
|
+
else if (stall) reason = 'no-output';
|
|
857
1026
|
else if (timedOut) reason = 'timeout';
|
|
1027
|
+
// GATE-POD-STALL-FAILFAST (OP): a complete summary with 0 fail · 0 error but an exit other than 0/1 is an exit-code
|
|
1028
|
+
// anomaly (0.2.23: a test left process.exitCode set → «0 fail · rc 3» read as a stalled shard). Same split/isolation
|
|
1029
|
+
// path as job-failed — only the label differs, so the verdict names the file instead of saying «stalled».
|
|
1030
|
+
else if (!jobFailed && exitCodeOnly?.length && rc !== undefined && rc !== 0 && rc !== 1) reason = 'exit-code';
|
|
858
1031
|
else if (jobFailed || (jobExitCode !== undefined && jobExitCode !== 0) || (rc !== undefined && rc !== 0 && rc !== 1)) reason = 'job-failed';
|
|
859
1032
|
else if (!clean || rc === undefined) reason = 'no-output';
|
|
860
1033
|
else if (!ran || !Number.isFinite(passes) || !Number.isFinite(fails)
|
|
@@ -901,6 +1074,9 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
901
1074
|
debug.log('release-loop.gate', 'pod-shard-retry', { shard, files: paths, reason, outcome: retry.stalled[0]?.reason ?? 'ok' });
|
|
902
1075
|
return retry;
|
|
903
1076
|
}
|
|
1077
|
+
// GATE-POD-STALL-FAILFAST: a stall after the fresh retry is unmeasured — no split, no isolation: smaller Pods on the
|
|
1078
|
+
// same stalled node only re-measure the stall (10-10: retries and splits repeated it for 2h+).
|
|
1079
|
+
if (stall && reason === 'no-output') return { runs: [], stalled: [{ shard, files: paths, reason, stall }] };
|
|
904
1080
|
if (reason) {
|
|
905
1081
|
// GATE-SKIP-KNOWN-RETRY — a shard whose summary is complete but whose failures could not be read back (unattributed)
|
|
906
1082
|
// and whose every named failure is one the previous release already had is not re-run: re-running only re-measures
|
|
@@ -992,7 +1168,11 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
992
1168
|
const total = measured.reduce((sum, run) => ({ pass: sum.pass + run.pass, fail: sum.fail + run.fail,
|
|
993
1169
|
errors: sum.errors + run.errors, ran: sum.ran + run.ran, files: sum.files + run.files }),
|
|
994
1170
|
{ pass: 0, fail: 0, errors: 0, ran: 0, files: 0 });
|
|
995
|
-
|
|
1171
|
+
const unmeasured = stalledShards.filter((item) => item.stall);
|
|
1172
|
+
if (unmeasured.length) debug.log('release-loop.gate', 'pod-sweep-unmeasured', { shards: [...new Set(unmeasured.map((item) => item.shard))], files: unmeasured.reduce((sum, item) => sum + item.files.length, 0),
|
|
1173
|
+
stallSeconds, measuredFiles: total.files }, { level: 'warn' });
|
|
1174
|
+
// An unmeasured (stalled) shard is never handed to the host re-run as an environment stall: the gate fails now.
|
|
1175
|
+
if (stalledShards.some((item) => item.reason !== 'no-output' || item.files.length !== 1 || item.stall)) throw new StalledPodShards(stalledShards, total);
|
|
996
1176
|
if (total.ran === 0 && total.errors === 0 && !stalledShards.length) throw new Error('sweep incomplete: no tests ran');
|
|
997
1177
|
const outputs = measured.map((run) => run.output.replace(/(?:^|\n)\s*\d+ (?:pass|fail|errors?)\s*(?=\n|$)/g, '\n').replace(/Ran \d+ tests? across \d+ files?\.?/g, ''));
|
|
998
1178
|
return { rc: total.fail + total.errors ? 1 : 0, output: outputs.join('\n') + `\n${total.pass} pass\n${total.fail} fail\n${total.errors} errors\nRan ${total.ran} tests across ${total.files} files.\n`,
|
|
@@ -1534,6 +1714,31 @@ export function graphGateResult(result: GateResult, graph: boolean) {
|
|
|
1534
1714
|
+ (result.stalledEnv.length ? ` · 환경 멈춤 ${result.stalledEnv.length} (Pod 밖 통과 ${result.stalledEnv.filter((item) => item.local === 'passed').length})` : '') };
|
|
1535
1715
|
}
|
|
1536
1716
|
|
|
1717
|
+
/** GATE-HOST-EXECUTOR — which executor produced (or will produce) this verdict, for the node output and observation. */
|
|
1718
|
+
export function gateExecutorOf(opts: Pick<GateOptions, 'pod' | 'remote'>): { executor: GateExecutor; host: string | null } {
|
|
1719
|
+
// A Pod sweep runs in shard Pods even when gateRemote is also set (that host only holds the mirror) — no SSH host judged it.
|
|
1720
|
+
return opts.pod ? { executor: 'pod', host: null } : { executor: 'host', host: opts.remote ?? 'local' };
|
|
1721
|
+
}
|
|
1722
|
+
|
|
1723
|
+
/** What was asked for, read best-effort when the options themselves were refused — so a failed gate still says which executor. */
|
|
1724
|
+
export function requestedGateExecutor(args: string[], env: NodeJS.ProcessEnv): { executor: string | null; host: string | null } {
|
|
1725
|
+
let executor: unknown;
|
|
1726
|
+
let host: unknown;
|
|
1727
|
+
try {
|
|
1728
|
+
if (env.ELANOUS_GRAPH_CONTEXT && args.every((arg) => arg === '--json')) {
|
|
1729
|
+
const input = readGraphContext(env).input as Record<string, unknown>;
|
|
1730
|
+
executor = input.gateExecutor ?? (typeof input.gatePodPool === 'string' ? 'pod' : undefined);
|
|
1731
|
+
host = input.gateRemote;
|
|
1732
|
+
}
|
|
1733
|
+
} catch { /* the refusal itself is reported as the error */ }
|
|
1734
|
+
for (let i = 0; i < args.length; i++) {
|
|
1735
|
+
if (args[i] === '--executor') executor = args[i + 1];
|
|
1736
|
+
else if (args[i] === '--remote') host = args[i + 1];
|
|
1737
|
+
}
|
|
1738
|
+
// Same rule as gateExecutorOf: a pod gate names no SSH host.
|
|
1739
|
+
return { executor: typeof executor === 'string' ? executor : null, host: executor !== 'pod' && typeof host === 'string' ? host : null };
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1537
1742
|
export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOptions | 'help' {
|
|
1538
1743
|
const context = env.ELANOUS_GRAPH_CONTEXT && args.every((arg) => arg === '--json') ? readGraphContext(env) : undefined;
|
|
1539
1744
|
const fromGraph: Record<string, unknown> = context?.input ?? {};
|
|
@@ -1541,6 +1746,9 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
1541
1746
|
if (typeof graphCommit === 'string') fromGraph.commit = graphCommit;
|
|
1542
1747
|
const baselineCommit = context?.outputs.cutoff?.baseline as { sha?: unknown } | undefined;
|
|
1543
1748
|
if (typeof baselineCommit?.sha === 'string') fromGraph.previousCommit = baselineCommit.sha;
|
|
1749
|
+
if (fromGraph.gateExecutor !== undefined && !isGateExecutor(fromGraph.gateExecutor)) throw new Error(`gateExecutor must be pod or host: ${String(fromGraph.gateExecutor)}`);
|
|
1750
|
+
// GATE-HOST-EXECUTOR: a host gate never reads the pod settings (a bad pod value must not block the fallback executor).
|
|
1751
|
+
const hostExecutor = fromGraph.gateExecutor === 'host';
|
|
1544
1752
|
const opts: GateOptions = {
|
|
1545
1753
|
commit: typeof fromGraph.commit === 'string' ? fromGraph.commit : '',
|
|
1546
1754
|
version: typeof fromGraph.version === 'string' ? fromGraph.version : '',
|
|
@@ -1549,17 +1757,19 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
1549
1757
|
baselineCommit: typeof fromGraph.previousCommit === 'string' ? fromGraph.previousCommit : undefined,
|
|
1550
1758
|
remote: typeof fromGraph.gateRemote === 'string' ? fromGraph.gateRemote : undefined,
|
|
1551
1759
|
remoteMirror: typeof fromGraph.gateRemoteMirror === 'string' ? fromGraph.gateRemoteMirror : undefined,
|
|
1552
|
-
pod: typeof fromGraph.gatePodPool === 'string' ? {
|
|
1760
|
+
pod: !hostExecutor && typeof fromGraph.gatePodPool === 'string' ? {
|
|
1553
1761
|
pool: fromGraph.gatePodPool,
|
|
1554
1762
|
...(fromGraph.gatePodShards !== undefined ? { shards: Number(fromGraph.gatePodShards) } : {}),
|
|
1555
1763
|
...(fromGraph.gatePodShardTimeoutSeconds !== undefined ? { shardTimeoutSeconds: Number(fromGraph.gatePodShardTimeoutSeconds) } : {}),
|
|
1556
1764
|
...(typeof fromGraph.gatePodBunCache === 'string' && fromGraph.gatePodBunCache.trim() ? { bunCache: fromGraph.gatePodBunCache } : {}),
|
|
1557
1765
|
...(fromGraph.gatePodInstallSlots !== undefined ? { installSlots: Number(fromGraph.gatePodInstallSlots) } : {}),
|
|
1558
1766
|
...(fromGraph.gatePodAdmissionWaitSeconds !== undefined ? { admissionWaitSeconds: Number(fromGraph.gatePodAdmissionWaitSeconds) } : {}),
|
|
1767
|
+
...(fromGraph.gatePodStallSeconds !== undefined ? { stallSeconds: Number(fromGraph.gatePodStallSeconds) } : {}),
|
|
1559
1768
|
...(fromGraph.gatePodFreshShards === true ? { freshShards: true } : {}),
|
|
1560
1769
|
...((cpu) => cpu ? { cpu } : {})(parsePodCpu(fromGraph.gatePodCpu, 'release.loop.gatePodCpu')),
|
|
1561
1770
|
} : undefined,
|
|
1562
1771
|
};
|
|
1772
|
+
if (isGateExecutor(fromGraph.gateExecutor)) opts.executor = fromGraph.gateExecutor;
|
|
1563
1773
|
let freshShards = false;
|
|
1564
1774
|
for (let i = 0; i < args.length; i++) {
|
|
1565
1775
|
const arg = args[i];
|
|
@@ -1567,6 +1777,12 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
1567
1777
|
if (arg === '--json') continue;
|
|
1568
1778
|
// Applied after the loop and only to a Pod gate — on its own it must not turn a local gate into a Pod one (review r5).
|
|
1569
1779
|
if (arg === '--fresh-shards') { freshShards = true; continue; }
|
|
1780
|
+
if (arg === '--executor') {
|
|
1781
|
+
const value = args[++i];
|
|
1782
|
+
if (!isGateExecutor(value)) throw new Error(`--executor must be pod or host: ${value ?? ''}`);
|
|
1783
|
+
opts.executor = value;
|
|
1784
|
+
continue;
|
|
1785
|
+
}
|
|
1570
1786
|
const key = ({ '--commit': 'commit', '--version': 'version', '--baseline-version': 'baselineVersion', '--baseline-commit': 'baselineCommit', '--remote': 'remote', '--remote-mirror': 'remoteMirror', '--pod-pool': 'pod', '--pod-shards': 'pod', '--pod-shard-timeout-seconds': 'pod' } as Record<string, keyof GateOptions>)[arg!];
|
|
1571
1787
|
if (!key) throw new Error(`unknown option: ${arg}`);
|
|
1572
1788
|
const value = args[++i];
|
|
@@ -1582,29 +1798,49 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
1582
1798
|
else if (key === 'remoteMirror') opts.remoteMirror = value;
|
|
1583
1799
|
}
|
|
1584
1800
|
if (freshShards && opts.pod) opts.pod = { ...opts.pod, freshShards: true };
|
|
1801
|
+
// GATE-HOST-EXECUTOR: the host executor is the existing non-pod sweep on gateRemote — a configured pool must not pull it
|
|
1802
|
+
// back into Pods, and without a host it would silently sweep on the runner's own machine.
|
|
1803
|
+
if (opts.executor === 'host') {
|
|
1804
|
+
if (!opts.remote) throw new Error('gateExecutor host requires gateRemote (the SSH host whose clean clone runs the sweep)');
|
|
1805
|
+
opts.pod = undefined;
|
|
1806
|
+
}
|
|
1807
|
+
// An explicit pod choice must not quietly become a host sweep for want of a pool (absent executor = today's behaviour).
|
|
1808
|
+
if (opts.executor === 'pod' && !opts.pod?.pool) throw new Error('gateExecutor pod requires gatePodPool');
|
|
1585
1809
|
return opts;
|
|
1586
1810
|
}
|
|
1587
1811
|
|
|
1588
|
-
|
|
1812
|
+
/** The node's run: judge, emit the graph result line, and return the process exit code (ok 0 · regression 1 · else 2).
|
|
1813
|
+
* `runner` is a seam for tests; the process entry passes none. */
|
|
1814
|
+
export async function runGateNode(argv: string[], env: NodeJS.ProcessEnv, runner?: GateRunner): Promise<number> {
|
|
1589
1815
|
const start = Date.now();
|
|
1590
|
-
// The graph spawns this file as its own process, which inherits no daemon sink — without this every
|
|
1591
|
-
// `release-loop.gate`/`release.gate`/`pod.pool` event stayed in the file trail and `elanous logs` read 0 rows.
|
|
1592
|
-
await registerStandaloneLogSink('release-loop');
|
|
1593
1816
|
let result: GateResult;
|
|
1594
1817
|
let version = '';
|
|
1818
|
+
let placement: { executor: string | null; host: string | null } | undefined;
|
|
1595
1819
|
try {
|
|
1596
|
-
const opts = parseOptions(
|
|
1820
|
+
const opts = parseOptions(argv, env);
|
|
1821
|
+
if (opts !== 'help') {
|
|
1822
|
+
placement = gateExecutorOf(opts);
|
|
1823
|
+
debug.log('release-loop.gate', 'executor', { version: opts.version, ...placement, requested: opts.executor ?? null });
|
|
1824
|
+
}
|
|
1597
1825
|
if (opts === 'help') {
|
|
1598
|
-
console.log('Usage: bun scripts/release-loop/gate-node.ts --commit <sha> --version <v> [--baseline-version <prev>] [--baseline-commit <sha>] [--remote <ssh-host>] [--remote-mirror <path>] [--pod-pool <pool>] [--pod-shards <n>] [--pod-shard-timeout-seconds <n>] [--fresh-shards] [--json]\nWithout flags, input.commit, input.version and input.previousVersion come from the JSON file at ELANOUS_GRAPH_CONTEXT.');
|
|
1599
|
-
|
|
1826
|
+
console.log('Usage: bun scripts/release-loop/gate-node.ts --commit <sha> --version <v> [--baseline-version <prev>] [--baseline-commit <sha>] [--remote <ssh-host>] [--remote-mirror <path>] [--pod-pool <pool>] [--pod-shards <n>] [--pod-shard-timeout-seconds <n>] [--fresh-shards] [--executor <pod|host>] [--json]\nWithout flags, input.commit, input.version and input.previousVersion come from the JSON file at ELANOUS_GRAPH_CONTEXT.\n--executor host (input.gateExecutor) ignores every pod option and sweeps on --remote (input.gateRemote, required).');
|
|
1827
|
+
return 0;
|
|
1600
1828
|
}
|
|
1601
1829
|
version = opts.version;
|
|
1602
|
-
result = await judgeGate(opts);
|
|
1830
|
+
result = await (runner ? judgeGate(opts, runner) : judgeGate(opts));
|
|
1603
1831
|
} catch (error) {
|
|
1604
1832
|
result = { outcome: 'error', commit: '', introduced: [], preexisting: 0, fixed: 0, knownEnv: 0, knownEnvCleared: [], stalledEnv: [], rcNonzeroNoFailures: [], durationMs: Date.now() - start, error: String(error) };
|
|
1833
|
+
placement ??= requestedGateExecutor(argv, env);
|
|
1605
1834
|
}
|
|
1606
1835
|
if (result.error) console.error(result.error);
|
|
1607
|
-
debug.log('release-loop.gate', 'result', { version, outcome: result.outcome });
|
|
1608
|
-
emitNodeResult(graphGateResult(result, !!
|
|
1609
|
-
|
|
1836
|
+
debug.log('release-loop.gate', 'result', { version, outcome: result.outcome, ...(placement ?? {}) });
|
|
1837
|
+
emitNodeResult({ ...graphGateResult(result, !!env.ELANOUS_GRAPH_CONTEXT), ...(placement ?? {}) });
|
|
1838
|
+
return result.outcome === 'ok' ? 0 : result.outcome === 'regression' ? 1 : 2;
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1841
|
+
if (import.meta.main) {
|
|
1842
|
+
// The graph spawns this file as its own process, which inherits no daemon sink — without this every
|
|
1843
|
+
// `release-loop.gate`/`release.gate`/`pod.pool` event stayed in the file trail and `elanous logs` read 0 rows.
|
|
1844
|
+
await registerStandaloneLogSink('release-loop');
|
|
1845
|
+
process.exitCode = await runGateNode(process.argv.slice(2), process.env);
|
|
1610
1846
|
}
|
|
@@ -10,7 +10,7 @@ import { lastJsonObject, runGraph, type GraphRunState } from '../../src/graph-ru
|
|
|
10
10
|
import { releaseRunUniverse } from './release-universe.js';
|
|
11
11
|
import { isReleaseVersion } from './release-version.js';
|
|
12
12
|
import { changedFilesBetween, clearPartialPlan, planPartialRegate, writePartialPlan } from './gate-partial.js';
|
|
13
|
-
import { releaseResumeAcceptedRegressions, releaseResumeWaiver, type AcceptedRegression } from './resume.js';
|
|
13
|
+
import { releaseResumeAcceptedRegressions, releaseResumeGateExecutor, releaseResumeWaiver, type AcceptedRegression } from './resume.js';
|
|
14
14
|
|
|
15
15
|
const GRAPH = join(import.meta.dir, '../../graphs/release/release-loop.yaml');
|
|
16
16
|
|
|
@@ -168,14 +168,19 @@ function savedGraphPath(statePath: string): string | undefined {
|
|
|
168
168
|
}
|
|
169
169
|
|
|
170
170
|
/** Refresh the branch tip, then restart the failed run at `from` with the saved graph snapshot. */
|
|
171
|
-
export async function resumeReleaseRun(opts: { runId: string; from: string; partial?: boolean; waive?: string; reason?: string; acceptedRegressions?: AcceptedRegression[] }, deps: ResumeReleaseDeps = {}): Promise<{ tip: TipRefresh; partial: PartialOutcome; state: GraphRunState }> {
|
|
171
|
+
export async function resumeReleaseRun(opts: { runId: string; from: string; partial?: boolean; waive?: string; reason?: string; acceptedRegressions?: AcceptedRegression[]; gateExecutor?: string; gateHost?: string }, deps: ResumeReleaseDeps = {}): Promise<{ tip: TipRefresh; partial: PartialOutcome; state: GraphRunState }> {
|
|
172
172
|
if (opts.from === 'version-release') throw new Error('--from version-release would cut again — resume at a later node');
|
|
173
173
|
// RELEASE-LEDGER-UNIVERSE: decide the universe once; the run lookup and every resumed node use it.
|
|
174
174
|
deps = { ...deps, root: releaseRunUniverse({ root: deps.root }) };
|
|
175
175
|
const statePath = join(deps.root!, 'graph-runs', 'release-loop', `${safeRunId(opts.runId)}.json`);
|
|
176
176
|
let waiverCommit: string | undefined;
|
|
177
|
-
|
|
177
|
+
if (opts.gateHost !== undefined && opts.gateExecutor === undefined) throw new Error('--gate-host requires --gate-executor host');
|
|
178
|
+
const original = opts.waive !== undefined || opts.reason !== undefined || opts.acceptedRegressions !== undefined || opts.gateExecutor !== undefined
|
|
178
179
|
? JSON.parse(readFileSync(statePath, 'utf8')) as GraphRunState : undefined;
|
|
180
|
+
// GATE-HOST-EXECUTOR: checked before the tip refresh writes anything; the runner checks it again under the resume lock.
|
|
181
|
+
const gateExecutor = original && opts.gateExecutor !== undefined
|
|
182
|
+
? releaseResumeGateExecutor(original, { from: opts.from, executor: opts.gateExecutor, host: opts.gateHost }) : undefined;
|
|
183
|
+
if (gateExecutor) debug.log('release-loop.resume', 'gate-executor', { runId: opts.runId, from: opts.from, executor: gateExecutor.set.gateExecutor, host: gateExecutor.set.gateRemote ?? null, previous: gateExecutor.previous });
|
|
179
184
|
const waiver = original && (opts.waive !== undefined || opts.reason !== undefined)
|
|
180
185
|
? releaseResumeWaiver(original, opts) : undefined;
|
|
181
186
|
const acceptedRegressions = original && opts.acceptedRegressions !== undefined && !waiver
|
|
@@ -202,6 +207,7 @@ export async function resumeReleaseRun(opts: { runId: string; from: string; part
|
|
|
202
207
|
const state = await (deps.graph ?? runGraph)(graphPath, { resumeRunId: opts.runId, fromNodeId: opts.from, pinChildUniverse: true,
|
|
203
208
|
...(waiver ? { releaseWaiver: { ...waiver, requestedRegressions: opts.acceptedRegressions ?? [] } } : {}),
|
|
204
209
|
...(acceptedRegressions ? { releaseAcceptedRegressions: { requested: opts.acceptedRegressions!, acceptedRegressions } } : {}),
|
|
210
|
+
...(gateExecutor ? { releaseGateExecutor: { executor: gateExecutor.set.gateExecutor, ...(opts.gateHost !== undefined ? { host: opts.gateHost } : {}) } } : {}),
|
|
205
211
|
deps: { root: deps.root, ...(deps.runBash ? { runBash: deps.runBash } : {}) } });
|
|
206
212
|
return { tip, partial, state };
|
|
207
213
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { GraphRunState } from '../../src/graph-runner/runner.js';
|
|
2
2
|
import { publicWaiverReason } from './known-issues-node.js';
|
|
3
|
+
import { checkGateExecutorChoice, type GateExecutor } from './gate-executor.js';
|
|
3
4
|
|
|
4
5
|
export interface AcceptedRegression { id: string; note: string }
|
|
5
6
|
|
|
@@ -51,3 +52,43 @@ export function releaseResumeWaiver(
|
|
|
51
52
|
return { nodeId: opts.waive, reason, acceptedRegressions: releaseResumeAcceptedRegressions(state,
|
|
52
53
|
[...(opts.acceptedRegressions ?? []), { id: `waive:${opts.waive}`, note: reason }]) };
|
|
53
54
|
}
|
|
55
|
+
|
|
56
|
+
/** GATE-HOST-EXECUTOR — one explicit executor switch, recorded on the run (`GraphRunState.inputOverrides`). */
|
|
57
|
+
export interface GateExecutorOverride {
|
|
58
|
+
kind: 'gate-executor';
|
|
59
|
+
at: string;
|
|
60
|
+
from: string;
|
|
61
|
+
set: { gateExecutor: GateExecutor; gateRemote?: string };
|
|
62
|
+
/** Input keys removed — switching back to pod drops the host an earlier override added (the original had none). */
|
|
63
|
+
unset?: Array<'gateRemote'>;
|
|
64
|
+
previous: { gateExecutor: unknown; gateRemote: unknown };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Switch the gate executor of a resumed release run. Not a waiver: the gate node runs again (the restart must be at or
|
|
69
|
+
* before gate) and produces a normal verdict; only where its sweep runs changes.
|
|
70
|
+
*/
|
|
71
|
+
export function releaseResumeGateExecutor(state: GraphRunState, opts: { from: string; executor: unknown; host?: unknown; at?: string }): GateExecutorOverride {
|
|
72
|
+
const input = state.input;
|
|
73
|
+
if (!input || typeof input !== 'object' || Array.isArray(input) || typeof (input as { version?: unknown }).version !== 'string') throw new Error('release run has no input');
|
|
74
|
+
const saved = input as { gateExecutor?: unknown; gateRemote?: unknown; gatePodPool?: unknown };
|
|
75
|
+
// Host executor without --gate-host: the run's own gateRemote (release.loop.gateRemote when it started), like `release run`.
|
|
76
|
+
const choice = checkGateExecutorChoice(opts.executor, opts.executor === 'host' && opts.host === undefined ? saved.gateRemote : opts.host);
|
|
77
|
+
if (choice.executor === 'pod' && (typeof saved.gatePodPool !== 'string' || !saved.gatePodPool.trim())) {
|
|
78
|
+
throw new Error('--gate-executor pod requires the saved run input gatePodPool');
|
|
79
|
+
}
|
|
80
|
+
const start = state.path.indexOf(opts.from);
|
|
81
|
+
if (start < 0) throw new Error(`--from ${opts.from} is not a node on the saved path`);
|
|
82
|
+
const gate = state.path.indexOf('gate');
|
|
83
|
+
// A restart after gate keeps the recorded verdict — switching its executor there would record a choice nothing used.
|
|
84
|
+
if (gate >= 0 && start > gate) throw new Error('--gate-executor needs a resume that re-runs gate (--from gate or earlier)');
|
|
85
|
+
// Back to pod: restore the gateRemote the run started with (an earlier host override replaced it).
|
|
86
|
+
const firstHost = choice.executor === 'pod' ? state.inputOverrides?.find((item) => item.kind === 'gate-executor' && item.set.gateRemote !== undefined) : undefined;
|
|
87
|
+
const restored = firstHost ? firstHost.previous.gateRemote : undefined;
|
|
88
|
+
return {
|
|
89
|
+
kind: 'gate-executor', at: opts.at ?? new Date().toISOString(), from: opts.from,
|
|
90
|
+
set: { gateExecutor: choice.executor, ...(choice.host ? { gateRemote: choice.host } : typeof restored === 'string' ? { gateRemote: restored } : {}) },
|
|
91
|
+
...(firstHost && typeof restored !== 'string' ? { unset: ['gateRemote' as const] } : {}),
|
|
92
|
+
previous: { gateExecutor: saved.gateExecutor ?? null, gateRemote: saved.gateRemote ?? null },
|
|
93
|
+
};
|
|
94
|
+
}
|