elanous 0.2.38 → 0.2.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -7
- package/apps/pwa/out/404/index.html +1 -1
- package/apps/pwa/out/404.html +1 -1
- package/apps/pwa/out/_next/static/{RrjXTwz-urRAbChP1bX02 → GiOLWmt4MqDZePubBjA4Q}/_buildManifest.js +1 -1
- package/apps/pwa/out/_next/static/chunks/1115.434ae2ecb0ad986d.js +149 -0
- package/apps/pwa/out/_next/static/chunks/1121.667cec0bcdbb3fdb.js +36 -0
- package/apps/pwa/out/_next/static/chunks/1178.1aa9cad0e7695427.js +1 -0
- package/apps/pwa/out/_next/static/chunks/1197-54da59d250a6e476.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{5203.51e5d8b5f1dcfe53.js → 1219.7e17964e0e7cd8e5.js} +3 -3
- package/apps/pwa/out/_next/static/chunks/1245.1cfd13b26dbd71d6.js +287 -0
- package/apps/pwa/out/_next/static/chunks/1494-f93046ff5c5a124d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/1609.2b312797599b8e45.js +1 -0
- package/apps/pwa/out/_next/static/chunks/1661.0dd330f3d89f73f3.js +1 -0
- package/apps/pwa/out/_next/static/chunks/167-0c121e4b47e0f9da.js +1 -0
- package/apps/pwa/out/_next/static/chunks/1700.cdc2bf8cc685e221.js +90 -0
- package/apps/pwa/out/_next/static/chunks/1703-78e46acad29f3cfc.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{7573.2376e13c9c21f1f8.js → 1765.679b29625c8e88d4.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/{3528.e4b8ab70d339f965.js → 1800.f7ff51dc4978a958.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/1971-7efb628c471dc5a0.js +1 -0
- package/apps/pwa/out/_next/static/chunks/1999-8de7b7bb36801913.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2073.689dcb2680113d76.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2198-557d49184104d165.js +1 -0
- package/apps/pwa/out/_next/static/chunks/221-440ce67fa9f5c839.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{8879.ac06755bcf3e44a1.js → 2263.e09b6edea33bad2d.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/2427-8ed8e98624cbd664.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2457-725c5827bc003795.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2491-2718b15a5ccb75a5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2527-5c636a43c7cc5dde.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2692-1027cb16fe4ccf49.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2743.5d3f76948be32b34.js +41 -0
- package/apps/pwa/out/_next/static/chunks/2773.0e80b9d325fb5635.js +32 -0
- package/apps/pwa/out/_next/static/chunks/2873-0f3cb56c8e3528ea.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{9fe1b809.738da85c8b17e3fb.js → 2fdf2931.b431eee4fb468bc3.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{eaad0cba.771200940c529df7.js → 2fe2c302.41b517f88d6fe502.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/3097.864e92bdaa2f4d15.js +79 -0
- package/apps/pwa/out/_next/static/chunks/3136-d6870ba63300a93e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/3173-0289113d434e83b7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/3230-49298c3b73ef2268.js +1 -0
- package/apps/pwa/out/_next/static/chunks/325-dd9208a1a0ab7ce4.js +1 -0
- package/apps/pwa/out/_next/static/chunks/3283.528f430dd043fb4e.js +74 -0
- package/apps/pwa/out/_next/static/chunks/3295.cc8cb13b8f0dae1a.js +1 -0
- package/apps/pwa/out/_next/static/chunks/3688-86e3b88abf18cb3b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/3804-7220467a5473a8df.js +1 -0
- package/apps/pwa/out/_next/static/chunks/3808.3e1e247ce7bdf889.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{b7478fef.5917716286213fcc.js → 385e4574.e9886ab32759eda3.js} +3 -3
- package/apps/pwa/out/_next/static/chunks/3927.b5473ad109da44c5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/3934-98ea94832e194796.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4133.2056bb4d78f621af.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4156.ba4de4a3664f04b4.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4297-239ca82f293faab0.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4320-dcc5a7734126f4c9.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4434.94b4c552a2d91e69.js +148 -0
- package/apps/pwa/out/_next/static/chunks/{8911.bf5e712acb25fb6c.js → 4527.a69f2894d1bc9f4a.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/4579.03e4092f73950965.js +226 -0
- package/apps/pwa/out/_next/static/chunks/{5307.d7328020227880c2.js → 4667.97e31aa38c95c475.js} +4 -4
- package/apps/pwa/out/_next/static/chunks/{6142.913b44e260a3f925.js → 4702.4a4adc1f16d7c93b.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/4815.481501c57b3568d9.js +1 -0
- package/apps/pwa/out/_next/static/chunks/487-d16e698ef14662d9.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4888-7a38bf522c9cc6fe.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5034-30e429d6bb086881.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5040-35c494e60dadd97d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/50c6ab1e.81a96e419350909d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5154.4b6a836179356f30.js +94 -0
- package/apps/pwa/out/_next/static/chunks/{8487.291666583bebe640.js → 5159.115f1c2ca471386c.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/5228-a8200df486278f2d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/530.d26346e18089640a.js +115 -0
- package/apps/pwa/out/_next/static/chunks/5379.97af106d09566f69.js +8 -0
- package/apps/pwa/out/_next/static/chunks/{89d6c04f-6b8ab341748e468a.js → 53aeb4ee-51c92f8d683f5220.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{3291.f6c1005387d514bd.js → 5471.1e611302ed3a7c53.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/5479-b6372e762719f92f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5534-699c5cbcbc96c5d7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{3086.fb14b60ce98b53d3.js → 5574.f35accf4f132b29d.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/5701.9c4e8f250312eeee.js +24 -0
- package/apps/pwa/out/_next/static/chunks/{1632.7d87f998e714bede.js → 5744.b9415e18ad623100.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{5300.677c97e0393742e6.js → 5892.2526d962339662d8.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/5937.c2df9e3447e1f667.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5941.8210cc93c9202c0a.js +286 -0
- package/apps/pwa/out/_next/static/chunks/{7161-8f22db3ec5ed005e.js → 5947-135e26537d80a6b9.js} +3 -3
- package/apps/pwa/out/_next/static/chunks/{3124-561f7f037943cde2.js → 5964-7e21164a8f2cf0ce.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/5975.4004f820ae3cecd5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6142-7ccd48795d7abd55.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6160.bf9177b8c1bfb243.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6398.71ba0690c8027555.js +27 -0
- package/apps/pwa/out/_next/static/chunks/6492.2bd5c10edc1a064e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6552.a3840aa7da35ad68.js +62 -0
- package/apps/pwa/out/_next/static/chunks/6736.1e36d3ac5fa5e41d.js +24 -0
- package/apps/pwa/out/_next/static/chunks/6874.d5cf7eb8edb2cabe.js +2 -0
- package/apps/pwa/out/_next/static/chunks/6900.3ccbe069bdd4556a.js +1 -0
- package/apps/pwa/out/_next/static/chunks/71327eb4.add50b365f94bf3a.js +122 -0
- package/apps/pwa/out/_next/static/chunks/7136.deca3e3a2ddaa374.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7139-ee9f8ed2677a40b3.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{2415.738b777642a02233.js → 7191.fdc955f7d96064a4.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/7423-e0aa5d9e2962120e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7453.afc1ea7e05d56a8c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7503-5e6a3d7fe67e4a0c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7761-476fd7bf768abd34.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7975.eafb12097a0a9e62.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{1026-44010f405704050c.js → 8050-9b364bdef8f71927.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/8152.64e253c35cccfaae.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8190.620f0231b3924bcb.js +1 -0
- package/apps/pwa/out/_next/static/chunks/83184dcd-0ee6247ff41eb3f2.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8448.45b6b1f1a99eeb35.js +104 -0
- package/apps/pwa/out/_next/static/chunks/8464-e24f10451b658aac.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8477.e4c944bb3aa5d2f3.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8710.ca385257d5ca422e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8830.627ccf294ec2465c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8880.64c445592f04acaf.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8981.c706964c8ba27aff.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{5638.107314dc479e2f37.js → 9048.212083de42d25be3.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/9053.bd0fffaff8d3f836.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9161-c66e723d7ec9d682.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9185.1ecdf9b65c0bb684.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9356.fccd5f58086b4ff5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9446.dc0406109371d329.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9506.dcf25d467a42627e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{8019.21ca554a821dad01.js → 9507.7026657a2f47eb64.js} +16 -16
- package/apps/pwa/out/_next/static/chunks/9541.eb8e2f8480c7feca.js +93 -0
- package/apps/pwa/out/_next/static/chunks/9577.ed7582186efa5482.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9579.4a8169600fff2058.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9642-ff2e039eb62e3b92.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9659-8f2323348113ad80.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9724.4c5623af2db43153.js +34 -0
- package/apps/pwa/out/_next/static/chunks/{6493.27ad77de7aac7966.js → 9834.63ab2bbc3153172e.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{3389.012c70767a872f9b.js → 9869.81a1e24ca682d5a0.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{8994d93e.33ba491f95345fc2.js → a0a01613.6d1cca322238d7fe.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{de8b930d.d94c0f42acf807fc.js → addc2b32.8c4ff8f0aefe987d.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/_not-found/page-8d9d76ea322d855c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/approvals/page-ed2c4e59f9f1fe21.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/autopilot/page-2dc99e03c5d76ca0.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/board/page-b5a3a788330e920f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/botlab/{page-866aac5ff226a7f3.js → page-14949db884bfa672.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/bots/page-ea76c85b36388a97.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ceo/page-01716d20af62d325.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/chat/page-62baf6881a64b1e9.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/consult/page-f89ba590c2af8afa.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/control/page-22f8b0f78d9637e5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/decisions/page-817f9d7e6820be80.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/design-check/page-ffb9bc0d2650249a.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/editor/page-f6d0e64873c0f629.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/exec/page-8bfa0d4ba23a209d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/field/page-f5060462249e32c7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/healthz/route-8d9d76ea322d855c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/inside/page-b2049f8b1ef79117.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/intake/page-1446c5fa93907f11.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/layout-b48a7a07c5204156.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/live/page-e974b8c4ca47614c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/live-run/page-8a919b869bd28594.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/loops/page-d31b8de6bfcd72ed.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/market/page-64765f817909db29.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/missions/[id]/page-d128c8569a36ee9b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/morning/page-b2679d357e9b9270.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/not-found-6fabec98a9e7b269.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/observatory/page-048f30149dcf1349.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/checklist/page-9113181d6b5a2725.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/release/page-40f9e19149024298.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/seats/page-1e7a755a13227611.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/outputs/page-3c8f8b67a203cf07.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/page-18032f5437215062.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/reflection/page-da3b739192fd0ee3.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/scheduler/page-d332545571bd6fd1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/sessions/page-5237aa5b3f3b8ec5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/settings/devices/page-a142dfb0331a05f5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/settings/page-d16e56c3989c147e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/setup/done/page-6b6593600869e2cd.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/setup/layout-8d9d76ea322d855c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/setup/page-29e8b8a8396b55cb.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/share/page-b3414d856a1a7e2f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/showroom/page-c3affa52b7ce961b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/tasks/page-f21dddf9f95b6fab.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/term/page-f5a30a24745bf088.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/today/page-8c8569a09b4d4e34.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/trace/page-2ce32722c0243483.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/vault/page-4767b0d9c0a98c6c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workflows/chat-ui/page-116ee1339c387a75.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workflows/page-35e8c50ca4ef7ffc.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workspace/page-52e17113356bb891.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/worktrees/page-7458ae05ed2ac1a0.js +1 -0
- package/apps/pwa/out/_next/static/chunks/b2271e54.c1de3347976e90b0.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{b7459770.6539b11860eb8a9c.js → c0c1a699.0ba79f97005965d4.js} +7 -7
- package/apps/pwa/out/_next/static/chunks/framework-ad6c9dd524ad6ef8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/main-app-2ddd1f2b44d6aa59.js +1 -0
- package/apps/pwa/out/_next/static/chunks/main-e3d82bdc358d20c5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/pages/_app-71d5aa3896e1655f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/pages/_error-03ffc54e00438546.js +1 -0
- package/apps/pwa/out/_next/static/chunks/webpack-56bb89af5a611d0b.js +1 -0
- package/apps/pwa/out/approvals/index.html +1 -1
- package/apps/pwa/out/approvals/index.txt +22 -22
- package/apps/pwa/out/autopilot/index.html +1 -1
- package/apps/pwa/out/autopilot/index.txt +22 -22
- package/apps/pwa/out/board/index.html +1 -1
- package/apps/pwa/out/board/index.txt +22 -22
- package/apps/pwa/out/botlab/index.html +1 -1
- package/apps/pwa/out/botlab/index.txt +21 -21
- package/apps/pwa/out/bots/index.html +1 -1
- package/apps/pwa/out/bots/index.txt +21 -21
- package/apps/pwa/out/ceo/index.html +1 -1
- package/apps/pwa/out/ceo/index.txt +22 -22
- package/apps/pwa/out/chat/index.html +1 -1
- package/apps/pwa/out/chat/index.txt +21 -21
- package/apps/pwa/out/consult/index.html +1 -1
- package/apps/pwa/out/consult/index.txt +22 -22
- package/apps/pwa/out/control/index.html +1 -1
- package/apps/pwa/out/control/index.txt +22 -22
- package/apps/pwa/out/decisions/index.html +1 -1
- package/apps/pwa/out/decisions/index.txt +21 -21
- package/apps/pwa/out/design-check/index.html +1 -1
- package/apps/pwa/out/design-check/index.txt +22 -22
- package/apps/pwa/out/editor/index.html +1 -1
- package/apps/pwa/out/editor/index.txt +22 -22
- package/apps/pwa/out/exec/index.html +1 -1
- package/apps/pwa/out/exec/index.txt +22 -22
- package/apps/pwa/out/field/index.html +1 -1
- package/apps/pwa/out/field/index.txt +22 -22
- package/apps/pwa/out/healthz +1 -1
- package/apps/pwa/out/index.html +1 -1
- package/apps/pwa/out/index.txt +21 -21
- package/apps/pwa/out/inside/index.html +1 -1
- package/apps/pwa/out/inside/index.txt +21 -21
- package/apps/pwa/out/intake/index.html +1 -1
- package/apps/pwa/out/intake/index.txt +22 -22
- package/apps/pwa/out/live/index.html +1 -1
- package/apps/pwa/out/live/index.txt +22 -22
- package/apps/pwa/out/live-run/index.html +1 -1
- package/apps/pwa/out/live-run/index.txt +22 -22
- package/apps/pwa/out/loops/index.html +1 -1
- package/apps/pwa/out/loops/index.txt +21 -21
- package/apps/pwa/out/market/index.html +1 -1
- package/apps/pwa/out/market/index.txt +21 -21
- package/apps/pwa/out/missions/placeholder/index.html +1 -1
- package/apps/pwa/out/missions/placeholder/index.txt +21 -21
- package/apps/pwa/out/morning/index.html +1 -1
- package/apps/pwa/out/morning/index.txt +22 -22
- package/apps/pwa/out/observatory/index.html +1 -1
- package/apps/pwa/out/observatory/index.txt +22 -22
- package/apps/pwa/out/ops/checklist/index.html +1 -1
- package/apps/pwa/out/ops/checklist/index.txt +21 -21
- package/apps/pwa/out/ops/release/index.html +1 -1
- package/apps/pwa/out/ops/release/index.txt +21 -21
- package/apps/pwa/out/ops/seats/index.html +1 -1
- package/apps/pwa/out/ops/seats/index.txt +21 -21
- package/apps/pwa/out/outputs/index.html +1 -1
- package/apps/pwa/out/outputs/index.txt +21 -21
- package/apps/pwa/out/reflection/index.html +1 -1
- package/apps/pwa/out/reflection/index.txt +22 -22
- package/apps/pwa/out/scheduler/index.html +1 -1
- package/apps/pwa/out/scheduler/index.txt +22 -22
- package/apps/pwa/out/sessions/index.html +1 -1
- package/apps/pwa/out/sessions/index.txt +21 -21
- package/apps/pwa/out/settings/devices/index.html +1 -1
- package/apps/pwa/out/settings/devices/index.txt +22 -22
- package/apps/pwa/out/settings/index.html +1 -1
- package/apps/pwa/out/settings/index.txt +22 -22
- package/apps/pwa/out/setup/done/index.html +1 -1
- package/apps/pwa/out/setup/done/index.txt +22 -22
- package/apps/pwa/out/setup/index.html +1 -1
- package/apps/pwa/out/setup/index.txt +22 -22
- package/apps/pwa/out/share/index.html +1 -1
- package/apps/pwa/out/share/index.txt +22 -22
- package/apps/pwa/out/showroom/index.html +1 -1
- package/apps/pwa/out/showroom/index.txt +21 -21
- package/apps/pwa/out/tasks/index.html +1 -1
- package/apps/pwa/out/tasks/index.txt +22 -22
- package/apps/pwa/out/term/index.html +1 -1
- package/apps/pwa/out/term/index.txt +22 -22
- package/apps/pwa/out/today/index.html +1 -1
- package/apps/pwa/out/today/index.txt +22 -22
- package/apps/pwa/out/trace/index.html +1 -1
- package/apps/pwa/out/trace/index.txt +22 -22
- package/apps/pwa/out/vault/index.html +1 -1
- package/apps/pwa/out/vault/index.txt +22 -22
- package/apps/pwa/out/workflows/chat-ui/index.html +1 -1
- package/apps/pwa/out/workflows/chat-ui/index.txt +22 -22
- package/apps/pwa/out/workflows/index.html +1 -1
- package/apps/pwa/out/workflows/index.txt +22 -22
- package/apps/pwa/out/workspace/index.html +1 -1
- package/apps/pwa/out/workspace/index.txt +22 -22
- package/apps/pwa/out/worktrees/index.html +1 -1
- package/apps/pwa/out/worktrees/index.txt +22 -22
- package/bin/elanous.mjs +6 -1
- package/catalog/models/openai/gpt-6.1-sol.yaml +16 -0
- package/catalog/models/openai/gpt-6.1-terra.yaml +16 -0
- package/graphs/demo/persona-marketer/fixtures/marketer-scenario.yaml +10 -0
- package/graphs/demo/persona-marketer/persona-marketer.yaml +25 -0
- package/graphs/demo/persona-marketer/recipes.yaml +19 -0
- package/graphs/demo/persona-teacher/lesson-pack.yaml +16 -0
- package/graphs/demo/persona-teacher/recipes.yaml +17 -0
- package/graphs/demo/persona-teacher/weekly-lesson.yaml +36 -0
- package/graphs/overlays/stop-heal-ladder.yaml +23 -2
- package/graphs/release/release-loop.yaml +3 -1
- package/graphs/units/gate/gate.yaml +20 -0
- package/graphs/units/git/git.yaml +26 -0
- package/graphs/units/implement/implement.yaml +19 -0
- package/graphs/units/landing/landing.yaml +18 -0
- package/graphs/units/review/review.yaml +20 -0
- package/graphs/units/verbs/recipes.yaml +3 -0
- package/graphs/units/verbs/research.yaml +20 -0
- package/graphs/video/explainer-line.yaml +4 -1
- package/graphs/work/research-report.yaml +11 -1
- package/package.json +1 -1
- package/scripts/dead-code/report.ts +169 -0
- package/scripts/disk-steward.ts +46 -0
- package/scripts/fleet/node-setup.sh +50 -0
- package/scripts/graph-units/verbs/research.ts +55 -0
- package/scripts/lib/morning-report-options.ts +3 -4
- package/scripts/lib/tui-regress-evidence.ts +56 -0
- package/scripts/llm-cost-ledger.ts +179 -8
- package/scripts/measure-nl-routing-corpus.ts +10 -2
- package/scripts/morning-report.ts +5 -1
- package/scripts/persona-teacher/weekly-lesson-node.ts +172 -0
- package/scripts/public-copy-lint.ts +163 -19
- package/scripts/public-export.ts +15 -7
- package/scripts/ratchets/train-side-invariants.sh +0 -1
- package/scripts/release-loop/export-check-node.ts +21 -3
- package/scripts/release-loop/gate-autoheal.ts +117 -0
- package/scripts/release-loop/gate-baseline-receipt.ts +8 -1
- package/scripts/release-loop/gate-diff.ts +22 -3
- package/scripts/release-loop/gate-node.ts +583 -64
- package/scripts/release-loop/gate-shard-dispatch.ts +128 -0
- package/scripts/release-loop/gate-shard-load-flaky.ts +256 -0
- package/scripts/release-loop/gate-shard-receipts.ts +89 -0
- package/scripts/release-loop/gate-triage-node.ts +70 -10
- package/scripts/release-loop/gate-wall.ts +5 -5
- package/scripts/release-loop/preflight.ts +6 -2
- package/scripts/release-loop/resume-release.ts +267 -20
- package/scripts/release-loop/resume.ts +48 -0
- package/scripts/release-loop/traffic-node.ts +6 -3
- package/scripts/release-loop/unattended-release.ts +4 -2
- package/scripts/release-loop/version-node.ts +7 -2
- package/scripts/release-story/draft.ts +1 -1
- package/scripts/research-bench/README.md +53 -0
- package/scripts/research-bench/questions.json +66 -0
- package/scripts/research-bench/rubric.md +33 -0
- package/scripts/research-bench/run.ts +523 -0
- package/scripts/shadow-authoring-probe.ts +20 -0
- package/scripts/tui-regress.ts +85 -10
- package/scripts/ux-sim.ts +19 -2
- package/src/acp/core-turn-bridge.ts +94 -4
- package/src/agent/autonomous-tools.ts +10 -3
- package/src/agent/monad-agent-turn.ts +4 -1
- package/src/agent/registry.ts +26 -2
- package/src/agent/shared-app-tools.ts +1 -1
- package/src/agent/surface-ux/build.ts +12 -3
- package/src/agent/surface-ux/types.ts +1 -1
- package/src/agent/task-notification.ts +75 -6
- package/src/agent/types.ts +3 -0
- package/src/agent-mission/driver.ts +29 -3
- package/src/agent-mission/mission-cli.ts +2 -1
- package/src/agent-substrate/review-context-pack.ts +365 -0
- package/src/agent-substrate/self-review-cli.ts +291 -20
- package/src/ask-user-question/index.ts +6 -0
- package/src/ask-user-question/inline-resolver.ts +49 -0
- package/src/auto-research/question-ladder-contract.ts +22 -3
- package/src/auto-research/research-turn.ts +359 -17
- package/src/autopilot/agent-loop.ts +1 -1
- package/src/autopilot/discovery/wiki-claim-proposer.ts +22 -1
- package/src/autopilot/pty-control-loop.ts +8 -1
- package/src/boot/daemon-tools/index.ts +4 -3
- package/src/boot/daemon-tools/self-implement.ts +312 -7
- package/src/boot/daemon-tools/types.ts +2 -0
- package/src/budget/llm-cost.ts +44 -5
- package/src/budget/shadow-arms.ts +52 -0
- package/src/budget/unified-usage.ts +17 -1
- package/src/capture/tui-self-report.ts +14 -0
- package/src/chat/index.ts +33 -7
- package/src/chat/pickers/state.ts +7 -1
- package/src/cli/agent-cli.ts +43 -8
- package/src/cli/intake-cli.ts +22 -1
- package/src/cli/lesson-cli.ts +19 -3
- package/src/cli/log-event-names.ts +6 -0
- package/src/cli/mcp-login.ts +8 -18
- package/src/cli/msg-cli.ts +57 -1
- package/src/cli/pod-cli.ts +17 -2
- package/src/cli/pr-land-heavy-remote.ts +15 -9
- package/src/cli/pty-takeover-cli.ts +67 -7
- package/src/cli/release-cli.ts +79 -7
- package/src/cli/repo-cli.ts +2 -1
- package/src/cli/sim-cli.ts +7 -1
- package/src/cli/tasks-cli.ts +41 -13
- package/src/codex/models.ts +30 -15
- package/src/control-plane/ledger.ts +21 -4
- package/src/core-turn/run-core-turn.ts +47 -3
- package/src/core-turn/run-goal-loop.ts +240 -10
- package/src/core-turn/types.ts +9 -0
- package/src/dashboard/acp-boot.ts +72 -3
- package/src/dashboard/chat/acp-chat.ts +2 -2
- package/src/dashboard/classic-sigquit.ts +16 -0
- package/src/dashboard/index.ts +327 -27
- package/src/dashboard/input/chat-main-interaction-opts.ts +3 -0
- package/src/dashboard/input/chat-main-plain-turn-runtime.ts +7 -2
- package/src/dashboard/input/chat-main-slash-command.ts +7 -0
- package/src/dashboard/input/dashboard-plain-turn-stream.ts +4 -0
- package/src/dashboard/input/dashboard-priority-key-route.ts +21 -5
- package/src/dashboard/input/external-input-host.ts +2 -0
- package/src/dashboard/sidetab-runtime.ts +228 -0
- package/src/dashboard/slash-runtime/ask-clarification.ts +7 -1
- package/src/dashboard/slash-runtime/dashboard-handlers.ts +20 -8
- package/src/dashboard/slash-runtime/help-from-registry.ts +2 -2
- package/src/dashboard/term-slash-runtime.ts +16 -18
- package/src/dashboard/turn-preamble.ts +14 -1
- package/src/disk-steward/judge.ts +73 -0
- package/src/disk-steward/ledger.ts +122 -0
- package/src/disk-steward/steward.ts +373 -0
- package/src/domains/guardian-tick.ts +191 -2
- package/src/domains/knowledge.ts +1 -1
- package/src/esc-abort-gate.ts +33 -1
- package/src/execution-loop/launch-gate.ts +206 -1
- package/src/graph-runner/graph-cli.ts +27 -4
- package/src/graph-runner/in-process-registry.ts +37 -0
- package/src/graph-runner/runner.ts +63 -11
- package/src/graph-wizard/generate.ts +16 -7
- package/src/graph-wizard/mission-compose.ts +201 -0
- package/src/grok/agent-search.ts +8 -2
- package/src/grok/credential.ts +5 -0
- package/src/grounding/ref-index.ts +30 -0
- package/src/grounding/ref-vector.ts +61 -0
- package/src/grounding/sources-cli.ts +11 -3
- package/src/grounding/sources.ts +15 -1
- package/src/harness/harness-cli-command.ts +305 -23
- package/src/harness/harness-incidents.ts +312 -7
- package/src/harness/harness-pod-dispatch.ts +7 -1
- package/src/harness/harness-space.ts +15 -1
- package/src/harness/progress-worker-entry.ts +59 -0
- package/src/harness/progress-worker-runtime.ts +11 -0
- package/src/index.ts +35 -5
- package/src/input/external-input-queue.ts +10 -0
- package/src/instance/dev-project.ts +25 -0
- package/src/intake-plane/collect-telegram-saved.ts +36 -1
- package/src/intake-plane/intake-to-tasks.ts +246 -3
- package/src/intelligence-map/model-alias.ts +6 -5
- package/src/intelligence-map/model-catalog.ts +59 -2
- package/src/lessons/friction-ledger.ts +154 -0
- package/src/live-supply/collector.ts +149 -0
- package/src/live-supply/probes.ts +52 -0
- package/src/live-supply/snapshot.ts +40 -0
- package/src/llm/model-defaults.ts +4 -4
- package/src/llm.ts +473 -87
- package/src/loops/orchestrator/tick.ts +56 -0
- package/src/mcp/mcp-oauth.ts +3 -1
- package/src/model-tier/llm-tier-map.ts +9 -9
- package/src/model-tier/turn-tier.ts +1 -1
- package/src/msg/msg-mirror-github.ts +80 -0
- package/src/msg/msg-mirror-policy.ts +61 -0
- package/src/msg/msg-mirror-runner.ts +219 -0
- package/src/msg/msg-mirror-state.ts +33 -0
- package/src/msg/msg-store.ts +11 -0
- package/src/nexus/api/harness-api.ts +2 -1
- package/src/output-truncation.ts +41 -12
- package/src/project/project-store.ts +48 -1
- package/src/pty/snapshot.py +132 -0
- package/src/pty-shell/pty-control-ipc.ts +29 -2
- package/src/pty-shell/pty-manifest.ts +59 -3
- package/src/pty-shell/registry.ts +2 -0
- package/src/pty-shell/tui-run-state.ts +80 -3
- package/src/release-loop/gate-host-reservation.ts +157 -4
- package/src/release-loop/gate-shard-ledger.ts +68 -0
- package/src/release-loop/merged-pr-checklist.ts +25 -2
- package/src/release-loop/rehearsal-shard-plan.ts +6 -1
- package/src/release-loop/rehearsal.ts +27 -0
- package/src/release-loop/release-schedule.ts +90 -1
- package/src/research/bloom-exemplars.ts +135 -0
- package/src/research/edge-decision.ts +124 -15
- package/src/research/gap-followup.ts +249 -0
- package/src/research/lenses/counter-evidence.ts +100 -0
- package/src/research/lenses/working-backwards.ts +89 -0
- package/src/research/plugin-discovery.ts +242 -0
- package/src/research-bridge/invoke.ts +4 -2
- package/src/research-bridge/types.ts +2 -0
- package/src/self-dev/ask-launch-flow.ts +73 -131
- package/src/self-dev/dev-cli.ts +19 -17
- package/src/self-dev/dev-pipeline.ts +8 -4
- package/src/self-dev/graph-front-nodes.ts +23 -0
- package/src/self-dev/graph-journey-route.ts +10 -2
- package/src/self-dev/launch-preflight-text.ts +2 -2
- package/src/self-dev/launch-preflight.ts +47 -11
- package/src/self-dev/orchestrate-cli.ts +1 -0
- package/src/self-dev/orchestrate.ts +15 -0
- package/src/self-dev/run-supervisor.ts +204 -3
- package/src/self-dev/run-types.ts +3 -0
- package/src/self-implement/device-first.ts +8 -2
- package/src/self-implement/doctor-visit.ts +76 -58
- package/src/self-implement/gate-artifacts.ts +16 -0
- package/src/self-implement/gate-baseline.ts +75 -4
- package/src/self-implement/gate-cli.ts +368 -30
- package/src/self-implement/gate-receipt-registry.ts +180 -0
- package/src/self-implement/gate-remote.ts +165 -24
- package/src/self-implement/goal-author-clarification.ts +205 -32
- package/src/self-implement/goal-author-decompose.ts +24 -15
- package/src/self-implement/goal-author.ts +86 -10
- package/src/self-implement/graph-stage-units.ts +172 -0
- package/src/self-implement/graph-templates.ts +162 -1
- package/src/self-implement/graph-variant.ts +37 -4
- package/src/self-implement/host-regate.ts +124 -49
- package/src/self-implement/impl-findings-feedback.ts +188 -0
- package/src/self-implement/knowhow-pack.ts +227 -10
- package/src/self-implement/off-diff-evidence.ts +124 -0
- package/src/self-implement/orchestrator.ts +320 -28
- package/src/self-implement/run-ledger.ts +110 -14
- package/src/self-implement/run-stop.ts +33 -2
- package/src/self-implement/running-runs.ts +35 -12
- package/src/self-implement/seams.ts +6 -3
- package/src/self-implement/self-implement-cli.ts +40 -6
- package/src/self-implement/stop-autoheal-operational.ts +43 -0
- package/src/self-implement/stop-autoheal.ts +15 -6
- package/src/session/chat.ts +66 -15
- package/src/session/request-size-guard.ts +123 -0
- package/src/session-runtime/provider-circuit.ts +140 -0
- package/src/shell-runner/file-engine.ts +1 -0
- package/src/shell-runner/types.ts +4 -0
- package/src/sidetab-core/model.ts +312 -0
- package/src/skills/tools/schedule-wakeup.ts +23 -1
- package/src/skills/tools/shell.ts +51 -5
- package/src/steward/directive-audit.ts +354 -0
- package/src/surface/child-process.ts +31 -0
- package/src/surface/tui-stderr-owner.ts +48 -0
- package/src/task-agent/cover.ts +30 -12
- package/src/task-agent/judge.ts +56 -0
- package/src/task-agent/parent-host.ts +14 -1
- package/src/task-agent/postland-card.ts +35 -2
- package/src/task-agent/rejudge-on-head.ts +46 -2
- package/src/task-agent/repair-cycle.ts +1 -1
- package/src/task-agent/task-hand.ts +133 -25
- package/src/task-agent/tier-split.ts +105 -21
- package/src/task-orchestrator/surfaces/pod-command-job.ts +330 -2
- package/src/task-orchestrator/surfaces/pod-config-snapshot.ts +123 -0
- package/src/task-orchestrator/surfaces/pod-failure-reason.ts +70 -2
- package/src/task-orchestrator/surfaces/pod-ledger-events.ts +29 -4
- package/src/task-orchestrator/surfaces/pod-mirror-seed.ts +11 -8
- package/src/task-orchestrator/surfaces/pod-si-warm-protocol.ts +1 -1
- package/src/task-orchestrator/surfaces/pod-source-receive.ts +74 -6
- package/src/task-orchestrator/surfaces/pod-warm-worker-pod.ts +2 -1
- package/src/task-orchestrator/surfaces/pod-warm-worker-provider.ts +56 -15
- package/src/task-orchestrator/surfaces/self-implement-pod.ts +298 -45
- package/src/task-orchestrator/surfaces/self-implement.ts +18 -0
- package/src/tool-runtime/skill-exec-runtime.ts +95 -5
- package/src/tui/entry.ts +25 -5
- package/src/tui/inline-choice.ts +188 -0
- package/src/tui/sidetab/list.ts +36 -165
- package/src/tui/sidetab/nl-tool.ts +2 -136
- package/src/tui/sidetab/peek.ts +52 -0
- package/src/tui/slash/term.ts +12 -1
- package/src/tui-client/dashboard-session.ts +10 -1
- package/src/user-config.ts +45 -9
- package/src/ux-sim/classic-tui-seat-scenarios.ts +1 -7
- package/src/ux-sim/fake-land-queue.ts +88 -0
- package/src/ux-sim/fault-needs-human-reason-repair.ts +107 -0
- package/src/ux-sim/fault-pod-child-result.ts +42 -0
- package/src/ux-sim/gate-shard-host-reroute.ts +82 -0
- package/src/ux-sim/goal-interim-trace.ts +24 -0
- package/src/ux-sim/next-tui-seat-scenarios.ts +8 -13
- package/src/ux-sim/release-sim-rig.ts +262 -0
- package/src/ux-sim/seminar-rehearsal.ts +109 -0
- package/src/ux-sim/subagent-tools-device.ts +2 -2
- package/src/ux-sim/task-orchestrator-surfaces-self-implement-pod-device.ts +4 -1
- package/src/version/packed-revision.json +1 -1
- package/src/video-pipeline/recipes/explainer.ts +54 -2
- package/src/web-search/grok.ts +32 -6
- package/src/web-search/index.ts +8 -7
- package/src/workflow-runtime/node-catalog.ts +53 -4
- package/apps/pwa/out/_next/static/chunks/1118.a8a74ea29aac3e46.js +0 -27
- package/apps/pwa/out/_next/static/chunks/1204-2aa830479adf80f4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1341-43b7d2dc77d45c9d.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1440-c3ea8882badfd9bf.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1698-9c11cbf97f13717e.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1770-3b1fc92253ef9f2f.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1923-5c1695332240246a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1938.0e6c60ee91c4d7f5.js +0 -148
- package/apps/pwa/out/_next/static/chunks/2104.70e086d5174a4736.js +0 -1
- package/apps/pwa/out/_next/static/chunks/213-20675f20673ba22a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/214.186a430da28338f1.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2277.3bd4c0ac7f1b5996.js +0 -93
- package/apps/pwa/out/_next/static/chunks/240.e09f433f71c78ff4.js +0 -104
- package/apps/pwa/out/_next/static/chunks/2417.02151393605a7cd0.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2463.cd2641a76daaa9ff.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2576.b9e9c629c4b84c52.js +0 -1
- package/apps/pwa/out/_next/static/chunks/265-c021de53f0865bd9.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2998-b1cc13956cd6a6e7.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2e4782d3.ba2f0ab2b63a53e7.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3130.626b54382b4dc379.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3221-1bb3f99a55088613.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3280.24edb1ffd03a7f3f.js +0 -24
- package/apps/pwa/out/_next/static/chunks/3288-eb3c22a1cbf2ae68.js +0 -1
- package/apps/pwa/out/_next/static/chunks/33.ef99e8374f96c6c4.js +0 -36
- package/apps/pwa/out/_next/static/chunks/3341-f09b9efbd7a82f70.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3410-d7c0f14b329d6fda.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3421.c87d6313a7d670f7.js +0 -287
- package/apps/pwa/out/_next/static/chunks/3422-83a3953f9bd67e26.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3587-002e24e368c515a9.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3644.f12a7d821ab940af.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3660.d7516588e2d875c3.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3695.b82e2610f4cd4ea9.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3858.ca1f5bc0f577eac9.js +0 -94
- package/apps/pwa/out/_next/static/chunks/3959-e6ac4a139d9d4bcc.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3f0de3de-2340c532c00395db.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4032.53f62176514ec0ea.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4199-f7c6eb40cd0322f8.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4204-ef8484cbecd5d62f.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4259-163129ca79913c4b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/428-eaa77df3d0c85969.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4432-97e400ffa1e161f5.js +0 -1
- package/apps/pwa/out/_next/static/chunks/449-aa427cf2cbb75f50.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4491-39a0fd32bbaf3840.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4588.b7cf06d382953979.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4690.70ec8060825366ab.js +0 -32
- package/apps/pwa/out/_next/static/chunks/4804.6012b5cd33c22792.js +0 -90
- package/apps/pwa/out/_next/static/chunks/4881.e4d22d730e3b9df5.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4937.ce2e340dc29cad3e.js +0 -79
- package/apps/pwa/out/_next/static/chunks/5-bb1c08d84bd71df3.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5005.de98df23b37cdf3c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5035.dc64a89cad5573cc.js +0 -74
- package/apps/pwa/out/_next/static/chunks/5041-5aada92ca215ffdd.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5207.1a224035455790fd.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5255-1b60d50eabe9f057.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5269.d0436ee6d773382a.js +0 -286
- package/apps/pwa/out/_next/static/chunks/5309.3367c4d50f52b489.js +0 -1
- package/apps/pwa/out/_next/static/chunks/552-689827d49780fa00.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5539.34cc9d60622cb69b.js +0 -8
- package/apps/pwa/out/_next/static/chunks/5568.218dea0c00ca2ec7.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5645.b7e3b055455a04ea.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5662.18cc0f30353cbd96.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5671.da24fc9dc5e98325.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5b141171.324c40fd49cb6821.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6175-d4621f6a10b0cfab.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6290.3eb53a20746b72ce.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6415-9307f3eafa90837f.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6579-3de35d00aacd71b5.js +0 -1
- package/apps/pwa/out/_next/static/chunks/680-2f324c4e039a5b67.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6845-331b3c7481da3ba3.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7000.28157393caaf5343.js +0 -62
- package/apps/pwa/out/_next/static/chunks/7311-60a7ea80d23c7521.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7401.525d0a12f7d02c06.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7438.c2cf72b66dca6aa4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7488.966b116789f3d41a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7524.ce973525d30b3d52.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7577.ea6e9ac72e035520.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7628.6b16863ac3972333.js +0 -34
- package/apps/pwa/out/_next/static/chunks/7657.8d1fb3d2f7ebe273.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7903-1435d5d2f0dd7807.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7954.4f7a8832663ee979.js +0 -115
- package/apps/pwa/out/_next/static/chunks/7978-30a91192c5dbe3c2.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8025-8795821525cf1a07.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8191.3674a00f0b565c23.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8217-604955d8885c5ef2.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8298-eed6f5f2585a63f4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8509.3a54d4e61893d7cb.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8758.ac421f726a189125.js +0 -1
- package/apps/pwa/out/_next/static/chunks/891.d6c723a51a067b36.js +0 -149
- package/apps/pwa/out/_next/static/chunks/9223.cbcc88e0e9950426.js +0 -41
- package/apps/pwa/out/_next/static/chunks/9299.24e2a7d9b6d1f91b.js +0 -226
- package/apps/pwa/out/_next/static/chunks/9317.6a9869039c231bb3.js +0 -24
- package/apps/pwa/out/_next/static/chunks/9383.e4755f275b3c11ff.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9581.da63ddf41acfad46.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9715.8c5529a0a5c41a36.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9817.3009d65f0889ef1f.js +0 -2
- package/apps/pwa/out/_next/static/chunks/app/_not-found/page-76873dd80da4f715.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/approvals/page-ed32b6ec5dea8f23.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/autopilot/page-fdd40e15babf3d25.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/board/page-384efc78e85e400b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/bots/page-d91809c5aeed319a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ceo/page-128f3522f89a81ab.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/chat/page-595961abf3b43ad1.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/consult/page-f41dccb76d8c0263.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/control/page-a3205c2e28b7d0d9.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/decisions/page-c39a80277814a3f4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/design-check/page-1ab3f98194513195.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/editor/page-a5246f7e254d77d0.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/exec/page-0f92287bdb81217d.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/field/page-5a37131ff63793ef.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/healthz/route-76873dd80da4f715.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/inside/page-ac4bd14bb2348bfd.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/intake/page-c348a3959c5b5fc6.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/layout-2addd81c0d3e2906.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/live/page-da6b765f58a3eefb.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/live-run/page-0661e40f8780c32c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/loops/page-b89af1b1c8e4875b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/market/page-da8aeb82ba751cdb.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/missions/[id]/page-cc71317b239439db.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/morning/page-bde7fc1d64ffe369.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/not-found-e2797214f713fa58.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/observatory/page-2a35bcda70defaf6.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/checklist/page-3af03deedde45032.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/release/page-d1d1210f99f620ae.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/seats/page-46d1b5500885861d.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/outputs/page-291ab19fe2acb173.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/page-7f569e533ddeecba.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/reflection/page-990129c9fee46d8f.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/scheduler/page-7737f785ebb1cdaf.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/sessions/page-9f3c73a4f7ffd2f8.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/settings/devices/page-03be7fb347f01058.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/settings/page-96b0280625e4df2b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/setup/done/page-56855545049eb56d.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/setup/layout-76873dd80da4f715.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/setup/page-79a909c9981f0338.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/share/page-a9d9a3e66459c1e0.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/showroom/page-c383cbf2013f7069.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/tasks/page-f57c6a2e54dbdca4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/term/page-6b140f77593576eb.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/today/page-6b767ae2d9d0adfc.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/trace/page-116d6198d5da0e3d.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/vault/page-b9c9b778c7501370.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/workflows/chat-ui/page-a6c78392ab8f6335.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/workflows/page-455183deb4ed0cfe.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/workspace/page-6a63e99526bd63d0.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/worktrees/page-31c22caafe788c08.js +0 -1
- package/apps/pwa/out/_next/static/chunks/bb6987f9.3647e1a23b0e3217.js +0 -122
- package/apps/pwa/out/_next/static/chunks/framework-4b12a946f352cee2.js +0 -1
- package/apps/pwa/out/_next/static/chunks/main-941155250ed2644b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/main-app-f8b6765abed6d149.js +0 -1
- package/apps/pwa/out/_next/static/chunks/pages/_app-78432194c85388f9.js +0 -1
- package/apps/pwa/out/_next/static/chunks/pages/_error-cdf7390829512b1b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/webpack-d5f0964a9c2e92d2.js +0 -1
- package/scripts/ratchets/next-tui-entry-budget.ts +0 -124
- /package/apps/pwa/out/_next/static/{RrjXTwz-urRAbChP1bX02 → GiOLWmt4MqDZePubBjA4Q}/_ssgManifest.js +0 -0
|
@@ -9,7 +9,7 @@ import { registerStandaloneLogSink } from '../../src/domains/standalone-log-sink
|
|
|
9
9
|
import { landingFreezeMessage, readLandingFreeze } from '../../src/release-loop/landing-freeze.js';
|
|
10
10
|
import { effectiveInstanceRoot, releaseLedgerRoot } from '../../src/instance/resolve.js';
|
|
11
11
|
import { getElanousConfigDirOverride } from '../../src/elanous-config-dir.js';
|
|
12
|
-
import { diffFailures, junitFailures, parseFailures, parseTimedOutFailures } from './gate-diff';
|
|
12
|
+
import { collapseGateFailureAttempts, diffFailures, GateFailureAttributionMismatch, junitFailures, parseFailureAttempts, parseFailures, parseTimedOutFailures } from './gate-diff';
|
|
13
13
|
import { loadEnvKnownFailures, splitKnownEnv } from './env-known-failures';
|
|
14
14
|
import { loadGateBatchSafe } from './gate-batch-safe';
|
|
15
15
|
import { observeGatePrewarmCutShadow, type GatePrewarmInput } from './gate-prewarm-plan.js';
|
|
@@ -43,12 +43,17 @@ import { resolveWarmPoolProfile } from '../../src/self-implement/pod-warm-pool-p
|
|
|
43
43
|
import { decideWarmReuse, type WarmReuseSwitchConfig, type WarmReuseEvidence } from '../../src/task-orchestrator/surfaces/pod-warm-switch.js';
|
|
44
44
|
import { getUserConfig } from '../../src/user-config.js';
|
|
45
45
|
import { beginGateShardAttempt, finalizeTerminalGateShards, gateShardsPath, recordGateShardsVerdict, writeGateShards, type GateJobTerminal, type GateShard } from '../../src/release-loop/gate-shards.js';
|
|
46
|
+
import { appendGateShardLedger, gateShardLedgerPath, type GateShardLedgerRow } from '../../src/release-loop/gate-shard-ledger.js';
|
|
47
|
+
import { markGateShardUnmeasured, nextGateShardAttempt, readGateShardManifest, readGateShardResult, saveGateShardResult } from './gate-shard-receipts.js';
|
|
48
|
+
import { planGateShardDispatch, type GateShardReceipt } from './gate-shard-dispatch.js';
|
|
46
49
|
import { defaultKubectl } from '../../src/task-orchestrator/surfaces/self-implement-pod.js';
|
|
47
50
|
import { executeTuiNodeCheck, type ExecutedTuiCheck } from './tui-node-checks.js';
|
|
48
51
|
import { runWithReleaseGateTraffic } from './gate-traffic-mode.js';
|
|
49
52
|
import { emitPodLifecycle, gatePodLifecycleEvent, gatePodOutcome } from '../../src/task-orchestrator/surfaces/pod-lifecycle-stages.js';
|
|
50
53
|
import { gateHostReservationRoot, reserveReleaseGateHosts } from '../../src/release-loop/gate-host-reservation.js';
|
|
51
54
|
import { normRoot } from '../../src/instance/resolve.js';
|
|
55
|
+
import { cleanRetryVerdict, junitFailedCaseIds, junitTestcaseCount, junitTestcaseFiles } from './gate-shard-load-flaky.js';
|
|
56
|
+
import { decideAndRecordGateAutoheal, gateAutohealCause, type GateAutohealDecision } from './gate-autoheal.js';
|
|
52
57
|
|
|
53
58
|
export interface WarmShadowObservation {
|
|
54
59
|
files: string[];
|
|
@@ -73,6 +78,10 @@ export function warmCommandMatchesRequest(command: WarmWorkerCommand, request: O
|
|
|
73
78
|
}
|
|
74
79
|
|
|
75
80
|
interface CommandResult { rc: number; output: string; passedIds?: string[]; stalledEnv?: Array<{ file: string; reason: 'no-output' }>; timedOut?: boolean;
|
|
81
|
+
/** Same-Pod per-file retry passed with complete log/JUnit evidence; these original ids are excluded from introduced. */
|
|
82
|
+
loadFlaky?: string[];
|
|
83
|
+
/** Files already given their one same-Pod retry; prevent the later timeout-only path from retrying them again. */
|
|
84
|
+
loadFlakyAttemptedFiles?: string[];
|
|
76
85
|
/** Identity and complete selected inventory of this Pod sweep; only the producer may issue it. */
|
|
77
86
|
podSweep?: { id: string; shards: number; files: string[] };
|
|
78
87
|
/** Complete, source-bound one-file Job result for shadow comparison; absent means unmeasured. */
|
|
@@ -218,6 +227,12 @@ export interface GateOptions {
|
|
|
218
227
|
recheckFileLimit?: number;
|
|
219
228
|
/** freshShards — GATE-SHARD-RESUME off: every shard starts a Pod even when this logDir already holds its result. */
|
|
220
229
|
pod?: { pool: string; shards?: number; shardTimeoutSeconds?: number; durationSource?: string; bunCache?: string; freshShards?: boolean;
|
|
230
|
+
/** Remeasure one named shard, or all incomplete shards with auto. Complete siblings keep their receipts. */
|
|
231
|
+
rerouteShard?: string | 'auto';
|
|
232
|
+
/** Pool context for the rerouted shard; the planner otherwise distributes missing shards over pool contexts. */
|
|
233
|
+
shardHost?: string;
|
|
234
|
+
/** Shard capability requirements are independent of host names; placement resolves them at run time. */
|
|
235
|
+
shardCapabilities?: Readonly<Record<string, { os?: 'darwin' | 'linux' }>>;
|
|
221
236
|
/** Opt-in gate shadow: provider fills the fenced lease identity. The Job remains authoritative. */
|
|
222
237
|
warmReplay?: (input: Omit<WarmWorkerCommand, 'leaseId' | 'generation' | 'workerId' | 'key'>) => Promise<{ command: WarmWorkerCommand | null; transport: WarmPodTransportResult }>;
|
|
223
238
|
warmQuarantine?: (command: WarmWorkerCommand, reason: string) => Promise<void>;
|
|
@@ -253,8 +268,8 @@ export interface GateOptions {
|
|
|
253
268
|
hostMirrorReaders?: GateHostMirrorReaders;
|
|
254
269
|
/** GATE-INSTALL-CACHE — concurrent `bun install`s per node (`release.loop.gatePodInstallSlots`); omitted = 4 · 0 = off. */
|
|
255
270
|
installSlots?: number;
|
|
256
|
-
/** GATE-MEM-ADMIT — seconds
|
|
257
|
-
*
|
|
271
|
+
/** GATE-MEM-ADMIT — seconds an active shard waits for pool admission. Root shards beyond the pool cap queue
|
|
272
|
+
* outside this timer; an exhausted wait stays unmeasured (`release.loop.gatePodAdmissionWaitSeconds`); 0 = off. */
|
|
258
273
|
admissionWaitSeconds?: number;
|
|
259
274
|
/** GATE-MEM-ADMIT — an injected admission (tests · embedders); config never sets it. Absent = the gate's own pool's. */
|
|
260
275
|
admission?: GateAdmission;
|
|
@@ -279,7 +294,8 @@ export interface GateOptions {
|
|
|
279
294
|
* of {@link gateShardAbortScript} on the gate's own pool; an injected pool without it runs without the mid-run watch. */
|
|
280
295
|
abortTests?: (job: { name: string; context?: string }, reason: string) => boolean | void;
|
|
281
296
|
/** GATE-LIVE-OBS — the shard state file (`gateShardsPath(version)`) this sweep keeps current; absent = not written. */
|
|
282
|
-
shardsFile?: { path: string; version: string
|
|
297
|
+
shardsFile?: { path: string; version: string; ledgerPath?: string; envFingerprint?: string;
|
|
298
|
+
receiptRoot?: string; base40?: string; expectedImageDigest?: string | null };
|
|
283
299
|
/** GATE-RISK-LANE — the ledgers `gateHistory` reads its watch list from; absent = no risk lane (the sweep plans as before). */
|
|
284
300
|
riskWatch?: { instanceRoots: readonly string[]; ledgerRoots: readonly string[] };
|
|
285
301
|
/** GATE-MULTI-HOST — is this pool member reachable (kubectl ns ⊕ ssh)? Injected by tests; absent = the real probe on
|
|
@@ -321,6 +337,9 @@ export interface GateResult {
|
|
|
321
337
|
baselineSource?: 'ledger' | 'instance' | 'cut-logs' | 'swept';
|
|
322
338
|
/** GATE-BASELINE-CACHE — isolated baseline files answered from the ledger (hits) or run now (misses). */
|
|
323
339
|
baselineCache?: { hits: number; misses: number };
|
|
340
|
+
/** A residual summary/name mismatch triggers an explicit baseline rerun before the gate stays fail-closed. */
|
|
341
|
+
attributionRecheck?: { summary: number; observed: number; identified: number; collapsed: number;
|
|
342
|
+
targetFiles: string[]; baselineResult: 'completed' | 'failed' | 'timeout' };
|
|
324
343
|
/** GATE-SPEED A4 — a swept baseline ran only the cut's failing files that exist at the previous release. */
|
|
325
344
|
baselineDeferred?: { cutFiles: number; swept: number };
|
|
326
345
|
durationMs: number;
|
|
@@ -341,14 +360,16 @@ export interface GateResult {
|
|
|
341
360
|
partialFailures?: SweepFailures;
|
|
342
361
|
/** GATE-PARTIAL — this verdict re-ran only these files and carried the prior gate's results for the rest. */
|
|
343
362
|
partial?: { priorCommit: string; files: number };
|
|
344
|
-
/**
|
|
345
|
-
* {@link GATE_TIMEOUT_RECHECK_MS} per-test limit: a warning (flaky under load), not an introduced regression. */
|
|
363
|
+
/** A failure that passed an evidence-checked isolated re-run: warning (flaky under load), not an introduced regression. */
|
|
346
364
|
loadFlaky?: string[];
|
|
347
365
|
/** File-level unhandled errors that reproduced or could not be measured in isolation. They keep the gate red. */
|
|
348
366
|
unmeasuredErrors?: Array<{ file: string; reason: 'reproduced' | 'timeout' | 'unreadable'; ids: string[] }>;
|
|
349
367
|
/** Shard-only unhandled errors whose file-level replay completed cleanly. */
|
|
350
368
|
interferenceErrors?: string[];
|
|
351
369
|
rehearsalTui?: ExecutedTuiCheck;
|
|
370
|
+
/** Recovery recommendation is persisted with the failed graph node so the resume coordinator can act on it. */
|
|
371
|
+
autoheal?: GateAutohealDecision;
|
|
372
|
+
autoheals?: GateAutohealDecision[];
|
|
352
373
|
}
|
|
353
374
|
|
|
354
375
|
const sha = /^[0-9a-f]{7,40}$/i;
|
|
@@ -523,6 +544,31 @@ const lastMatch = (output: string, pattern: RegExp): RegExpExecArray | null => {
|
|
|
523
544
|
return last;
|
|
524
545
|
};
|
|
525
546
|
const summaryCount = (output: string, label: string) => Number(lastMatch(output, new RegExp(`(?:^|\\n)\\s*(\\d+) ${label}s?\\s*(?=\\n|$)`))?.[1] ?? NaN);
|
|
547
|
+
/** `test:deterministic` runs several Bun commands. Its final clean command is not the verdict for earlier commands. */
|
|
548
|
+
function deterministicSubrunCounts(output: string): { fail: number; error: number } | undefined {
|
|
549
|
+
if (!/error: script "test:deterministic" exited with code 1/.test(output)) return undefined;
|
|
550
|
+
const lines = output.split(/\r?\n/).map((line) => line.trim());
|
|
551
|
+
const runs: Array<{ fail: number; error: number }> = [];
|
|
552
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
553
|
+
if (!/^Ran \d+ tests? across [1-9]\d* files?\./.test(lines[i]!)) continue;
|
|
554
|
+
let pass: number | undefined;
|
|
555
|
+
let fail: number | undefined;
|
|
556
|
+
let error = 0;
|
|
557
|
+
for (let j = i - 1; j >= Math.max(0, i - 8); j -= 1) {
|
|
558
|
+
const line = lines[j]!;
|
|
559
|
+
const count = /^(\d+) (pass|fail|skip|errors?|expect\(\) calls)$/.exec(line);
|
|
560
|
+
if (!line) continue;
|
|
561
|
+
if (!count) break;
|
|
562
|
+
if (count[2] === 'pass') { pass = Number(count[1]); break; }
|
|
563
|
+
if (count[2] === 'fail') fail = Number(count[1]);
|
|
564
|
+
if (count[2] === 'error' || count[2] === 'errors') error = Number(count[1]);
|
|
565
|
+
}
|
|
566
|
+
if (pass === undefined || fail === undefined) return undefined;
|
|
567
|
+
runs.push({ fail, error });
|
|
568
|
+
}
|
|
569
|
+
if (runs.length < 2) return undefined;
|
|
570
|
+
return runs.reduce((sum, run) => ({ fail: sum.fail + run.fail, error: sum.error + run.error }), { fail: 0, error: 0 });
|
|
571
|
+
}
|
|
526
572
|
/** bun reads a bare positional as a substring filter (`test` matches every *.test.ts); `./` makes it a path. */
|
|
527
573
|
const asPath = (path: string) => path.startsWith('./') || path.startsWith('/') ? path : `./${path}`;
|
|
528
574
|
/** A Pod path (`~/repo/src/…` · `../..~/repo/src/…`) as a repository-relative one. */
|
|
@@ -733,7 +779,8 @@ export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPa
|
|
|
733
779
|
// run_part body changes the shell hash once for this feature; keeping the
|
|
734
780
|
// multi-file invocation unchanged avoids another cache-key change.
|
|
735
781
|
const warmFile = part.length === 1 ? `WARM_REFERENCE_FILE=${quote(part[0]!)} ` : '';
|
|
736
|
-
|
|
782
|
+
const manifest = `printf '%s\\n' ${part.map(quote).join(' ')} > "$O/part-${index}.files"; `;
|
|
783
|
+
return `${manifest}${warmFile}run_part ${index} ${[...runner, ...(o.testTimeoutMs !== undefined ? ['--timeout', String(o.testTimeoutMs)] : spawnParts.includes(index) ? ['--timeout=120000'] : []), ...ignores, ...part.map(asPath)].map(quote).join(' ')}`;
|
|
737
784
|
});
|
|
738
785
|
// Only file-isolation batches use this path. A bounded group shares one clone/install, but every test process gets
|
|
739
786
|
// its own HOME, state and tmp directory; part logs/JUnit remain separate and the existing artifact reader judges them.
|
|
@@ -791,6 +838,24 @@ export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPa
|
|
|
791
838
|
...(spawnParts.length ? [' local test_start; test_start=$(date +%s)'] : []),
|
|
792
839
|
` { if [ -n "\${WARM_REFERENCE_FILE:-}" ]; then printf "[warm-worker] file=%s\\n" "$WARM_REFERENCE_FILE"; ${o.sourceBinding ? `printf "[warm-worker] cut=%s attempt=%s\\n" ${quote(o.sourceBinding.cutSha)} ${quote(o.sourceBinding.attemptId)}; ` : ''}fi; "\${run_env[@]}" timeout -k 10 "$left" "$@" --reporter=junit --reporter-outfile="$O/part-$i.junit.xml"; } > "$O/part-$i.log" 2>&1`,
|
|
793
840
|
' local prc=$?',
|
|
841
|
+
' if [ -s "$O/part-$i.junit.xml" ]; then',
|
|
842
|
+
' if bun run scripts/release-loop/gate-shard-load-flaky.ts "$O/part-$i.junit.xml" "$O/part-$i.retry-files" "$O/part-$i.files"; then',
|
|
843
|
+
' local retry_index=0 retry_left retry_rc retry_file',
|
|
844
|
+
' while IFS= read -r retry_file; do',
|
|
845
|
+
' [ -n "$retry_file" ] || continue',
|
|
846
|
+
' retry_left=$(( gate_end - $(date +%s) ))',
|
|
847
|
+
' [ "$retry_left" -gt 0 ] || break',
|
|
848
|
+
' [ "$retry_left" -le 120 ] || retry_left=120',
|
|
849
|
+
' stage "part-$i-retry-$retry_index-start"',
|
|
850
|
+
' : > "$O/part-$i.retry-$retry_index.started"',
|
|
851
|
+
' timeout -k 10 "$retry_left" "${run_env[@]}" bun run scripts/test-deterministic.ts "$retry_file" --timeout 120000 --reporter=junit --reporter-outfile="$O/part-$i.retry-$retry_index.junit.xml" > "$O/part-$i.retry-$retry_index.log" 2>&1',
|
|
852
|
+
' retry_rc=$?',
|
|
853
|
+
' echo "$retry_rc" > "$O/part-$i.retry-$retry_index.rc"',
|
|
854
|
+
' stage "part-$i-retry-$retry_index-end"',
|
|
855
|
+
' retry_index=$(( retry_index + 1 ))',
|
|
856
|
+
' done < "$O/part-$i.retry-files"',
|
|
857
|
+
' fi',
|
|
858
|
+
' fi',
|
|
794
859
|
' if [ -n "${WARM_REFERENCE_FILE:-}" ] && [ -f "$O/part-$i.junit.xml" ]; then',
|
|
795
860
|
' local junit_sha; junit_sha=$(sha256sum "$O/part-$i.junit.xml" 2>/dev/null | awk \'{print $1}\')',
|
|
796
861
|
' if [ -z "$junit_sha" ]; then junit_sha=$(shasum -a 256 "$O/part-$i.junit.xml" 2>/dev/null | awk \'{print $1}\'); fi',
|
|
@@ -1045,20 +1110,29 @@ export function gateShardAbortScript(reason: string, graceSeconds = GATE_SHARD_A
|
|
|
1045
1110
|
return [
|
|
1046
1111
|
`reason=${quote(reason.replace(/[^\w .:-]/g, '_'))}`,
|
|
1047
1112
|
`only=${quote(outbox ?? '')}`,
|
|
1048
|
-
|
|
1113
|
+
// Read after the final ')' because comm can itself contain spaces and ')'. starttime is field 20 in the suffix
|
|
1114
|
+
// (field 22 in proc stat); pairing it with PID prevents a recycled PID from validating an old group number.
|
|
1115
|
+
"proc_identity() { sed 's/^.*) //' \"/proc/$1/stat\" 2>/dev/null | awk '{print $3, $20}'; }",
|
|
1116
|
+
"pg_of() { proc_identity \"$1\" | awk '{print $1}'; }",
|
|
1049
1117
|
'root_pg=$(pg_of 1); my_pg=$(pg_of $$); found=0',
|
|
1050
1118
|
'for d in /proc/[0-9]*; do',
|
|
1051
1119
|
' pid=${d#/proc/}',
|
|
1052
1120
|
" out=$(tr '\\0' '\\n' < \"$d/cmdline\" 2>/dev/null | grep -m1 -E '^--reporter-outfile=.*/outbox/part-[0-9]+\\.junit\\.xml$') || continue",
|
|
1053
1121
|
' if [ -n "$only" ]; then case "$out" in "--reporter-outfile=$only/part-"*) ;; *) continue;; esac; fi',
|
|
1054
|
-
'
|
|
1055
|
-
'
|
|
1122
|
+
' identity=$(proc_identity "$pid")',
|
|
1123
|
+
' pg=${identity%% *}; started=${identity#* }',
|
|
1124
|
+
' case "$pg:$started" in *[!0-9:]*|:|*:|:*) continue;; esac',
|
|
1056
1125
|
' if [ "$pg" != "$pid" ] || [ "$pg" -le 1 ] || [ "$pg" = "$root_pg" ] || [ "$pg" = "$my_pg" ]; then continue; fi',
|
|
1126
|
+
// Re-read the process identity immediately before signaling. The same check guards delayed KILL so a PID/PGID
|
|
1127
|
+
// that disappeared and was reused during the grace interval is never signaled as if it were the original test.
|
|
1128
|
+
' [ "$(proc_identity "$pid")" = "$identity" ] || continue',
|
|
1057
1129
|
' out=${out#--reporter-outfile=}; dir=${out%/part-*}; part=${out##*/part-}; part=${part%.junit.xml}',
|
|
1058
|
-
'
|
|
1059
|
-
'
|
|
1060
|
-
'
|
|
1061
|
-
|
|
1130
|
+
' if [ "$(proc_identity "$pid")" = "$identity" ] && kill -TERM -- "-$pg" 2>/dev/null; then',
|
|
1131
|
+
' printf "%s\\n" "$reason" > "$dir/part-$part.abort"',
|
|
1132
|
+
' [ -f "$dir/shard.timeout" ] || echo "$part" > "$dir/shard.timeout"',
|
|
1133
|
+
' found=$((found + 1)); echo "ELANOUS_GATE_ABORT part=$part pgid=$pg"',
|
|
1134
|
+
` ( sleep ${Math.max(1, Math.floor(graceSeconds))}; [ "$(proc_identity "$pid")" = "$identity" ] && kill -KILL -- "-$pg" 2>/dev/null ) >/dev/null 2>&1 </dev/null &`,
|
|
1135
|
+
' fi',
|
|
1062
1136
|
'done',
|
|
1063
1137
|
'echo "ELANOUS_GATE_ABORT signalled=$found"',
|
|
1064
1138
|
'[ "$found" -gt 0 ]',
|
|
@@ -1092,6 +1166,10 @@ export interface ShardArtifacts {
|
|
|
1092
1166
|
finished: string[];
|
|
1093
1167
|
/** Raw logs of the parts that did not finish (diagnostics · last started file). */
|
|
1094
1168
|
unfinishedLog?: string;
|
|
1169
|
+
/** Original failure ids whose one same-Pod standalone retry completed cleanly. */
|
|
1170
|
+
loadFlaky?: string[];
|
|
1171
|
+
/** Files for which a same-Pod retry completed, even if it remained failing. */
|
|
1172
|
+
loadFlakyAttemptedFiles?: string[];
|
|
1095
1173
|
}
|
|
1096
1174
|
/** Reads a shard Pod's outbox: the part layout ({@link gateShardShell}) or the older single-run one (shard.log · shard.rc · junit.xml). */
|
|
1097
1175
|
export function readShardArtifacts(dir: string, parts: readonly (readonly string[])[]): ShardArtifacts {
|
|
@@ -1106,12 +1184,17 @@ export function readShardArtifacts(dir: string, parts: readonly (readonly string
|
|
|
1106
1184
|
}
|
|
1107
1185
|
const timedOut = read('shard.timeout') !== undefined;
|
|
1108
1186
|
if (installRc !== 0) return { output: install, ...(installRc !== undefined ? { rc: installRc } : {}), timedOut, finished: [] };
|
|
1109
|
-
const good: Array<{ log: string; junit?: string; rc: 0 | 1; files: readonly string[] }> = [];
|
|
1187
|
+
const good: Array<{ log: string; junit?: string; rc: 0 | 1; files: readonly string[]; loadFlaky?: string[]; loadFlakyAttemptedFiles?: string[] }> = [];
|
|
1110
1188
|
const bad: string[] = [];
|
|
1111
1189
|
const exitAnomaly: string[] = [];
|
|
1190
|
+
const attemptedFiles = new Set<string>();
|
|
1112
1191
|
let otherBad = false;
|
|
1113
1192
|
let badRc: number | undefined;
|
|
1114
1193
|
parts.forEach((files, index) => {
|
|
1194
|
+
const retryManifest = (read(`part-${index}.retry-files`) ?? '').split(/\r?\n/).filter(Boolean);
|
|
1195
|
+
retryManifest.forEach((file, retryIndex) => {
|
|
1196
|
+
if (files.includes(file) && read(`part-${index}.retry-${retryIndex}.started`) !== undefined) attemptedFiles.add(file);
|
|
1197
|
+
});
|
|
1115
1198
|
const log = read(`part-${index}.log`);
|
|
1116
1199
|
const rc = rcOf(read(`part-${index}.rc`));
|
|
1117
1200
|
const clean = log?.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '');
|
|
@@ -1122,9 +1205,32 @@ export function readShardArtifacts(dir: string, parts: readonly (readonly string
|
|
|
1122
1205
|
otherBad = true;
|
|
1123
1206
|
return;
|
|
1124
1207
|
}
|
|
1125
|
-
|
|
1208
|
+
const summary = clean === undefined ? undefined : lastMatch(clean, /Ran (\d+) tests? across (\d+) files?/);
|
|
1209
|
+
const summaryTests = Number(summary?.[1] ?? NaN);
|
|
1210
|
+
const summaryPass = clean === undefined ? NaN : summaryCount(clean, 'pass');
|
|
1211
|
+
const summaryFail = clean === undefined ? NaN : summaryCount(clean, 'fail');
|
|
1212
|
+
const summaryErrors = clean === undefined ? NaN : summaryCount(clean, 'error');
|
|
1213
|
+
const partJunit = read(`part-${index}.junit.xml`) ?? '';
|
|
1214
|
+
const testcaseFiles = junitTestcaseFiles(partJunit);
|
|
1215
|
+
const expectedFiles = [...files].sort();
|
|
1216
|
+
const timedOutComplete = rc === 124 && summary !== undefined && summary !== null && Number(summary[2]) === files.length
|
|
1217
|
+
&& summaryPass + summaryFail + (Number.isNaN(summaryErrors) ? 0 : summaryErrors) === summaryTests
|
|
1218
|
+
&& junitTestcaseCount(partJunit) === summaryTests
|
|
1219
|
+
&& testcaseFiles !== null && testcaseFiles.length === expectedFiles.length && expectedFiles.every((file, fileIndex) => testcaseFiles[fileIndex] === file);
|
|
1220
|
+
if (clean !== undefined && ((rc === 0 || rc === 1) || timedOutComplete) && summary) {
|
|
1126
1221
|
const junit = read(`part-${index}.junit.xml`);
|
|
1127
|
-
|
|
1222
|
+
const retryFiles = retryManifest;
|
|
1223
|
+
const loadFlaky = retryFiles.flatMap((file, retryIndex) => {
|
|
1224
|
+
if (!files.includes(file)) return [];
|
|
1225
|
+
const retryRc = rcOf(read(`part-${index}.retry-${retryIndex}.rc`));
|
|
1226
|
+
const retryLog = read(`part-${index}.retry-${retryIndex}.log`);
|
|
1227
|
+
const retryJunit = read(`part-${index}.retry-${retryIndex}.junit.xml`);
|
|
1228
|
+
const originalFailures = (junitFailedCaseIds(junit ?? '') ?? []).filter((id) => fileOf(id) === file);
|
|
1229
|
+
const verdict = cleanRetryVerdict(retryRc, retryLog, retryJunit, file, originalFailures);
|
|
1230
|
+
if (!verdict.clean) return [];
|
|
1231
|
+
return originalFailures;
|
|
1232
|
+
});
|
|
1233
|
+
good.push({ log: clean, rc: rc === 0 ? 0 : 1, files, ...(junit !== undefined ? { junit } : {}), ...(loadFlaky.length ? { loadFlaky } : {}) });
|
|
1128
1234
|
return;
|
|
1129
1235
|
}
|
|
1130
1236
|
if (log !== undefined) bad.push(`# part ${index} (${rc === undefined ? 'unfinished' : `rc=${rc}`})\n${log}`);
|
|
@@ -1150,6 +1256,9 @@ export function readShardArtifacts(dir: string, parts: readonly (readonly string
|
|
|
1150
1256
|
...(rc !== undefined ? { rc } : {}),
|
|
1151
1257
|
timedOut: !complete && timedOut && badRc === undefined,
|
|
1152
1258
|
finished: good.flatMap((part) => [...part.files]),
|
|
1259
|
+
...([ ...new Set(good.flatMap((part) => part.loadFlaky ?? [])) ].length
|
|
1260
|
+
? { loadFlaky: [...new Set(good.flatMap((part) => part.loadFlaky ?? []))].sort() } : {}),
|
|
1261
|
+
...(attemptedFiles.size ? { loadFlakyAttemptedFiles: [...attemptedFiles].sort() } : {}),
|
|
1153
1262
|
...(bad.length ? { unfinishedLog: bad.join('\n') } : {}),
|
|
1154
1263
|
...(exitAnomaly.length && !otherBad ? { exitCodeOnly: exitAnomaly } : {}),
|
|
1155
1264
|
};
|
|
@@ -1190,8 +1299,8 @@ const lastStartedTestFile = (output: string, paths: string[]): string | undefine
|
|
|
1190
1299
|
* split shape follows from depth and size alone, so the same plan reaches the same child names (TC 09:30 review ②).
|
|
1191
1300
|
* `runKey` hashes where and how the Pod ran the shard (pool · command · ignores · memory · deadline · Bun cache): a fix on the Pod side
|
|
1192
1301
|
* makes every old result a miss instead of re-reading an old failure as introduced (TC 09:48 review). */
|
|
1193
|
-
export function previousShardDescent(logDir: string, name: string, commit: string, files: readonly string[], runKey: string, parentAttempt?: string, splitChild?: string): { way: 'retry' | 'split'; attemptId: string } | undefined {
|
|
1194
|
-
const meta = (file: string): { commit?: unknown; files?: unknown; rc?: unknown; runKey?: unknown; attemptId?: unknown; parentAttempt?: unknown } | undefined => {
|
|
1302
|
+
export function previousShardDescent(logDir: string, name: string, commit: string, files: readonly string[], runKey: string, parentAttempt?: string, splitChild?: string): { way: 'retry' | 'split' | 'file-fanout'; attemptId: string } | undefined {
|
|
1303
|
+
const meta = (file: string): { commit?: unknown; files?: unknown; rc?: unknown; reason?: unknown; runKey?: unknown; attemptId?: unknown; parentAttempt?: unknown } | undefined => {
|
|
1195
1304
|
try { return JSON.parse(readFileSync(join(logDir, `${file}.json`), 'utf8')); } catch { return undefined; }
|
|
1196
1305
|
};
|
|
1197
1306
|
const own = meta(name);
|
|
@@ -1208,6 +1317,8 @@ export function previousShardDescent(logDir: string, name: string, commit: strin
|
|
|
1208
1317
|
&& (!whole || row.files.length === files.length);
|
|
1209
1318
|
};
|
|
1210
1319
|
if (child(`${name}-retry`, true)) return { way: 'retry', attemptId };
|
|
1320
|
+
// A stalled multi-file parent may have descended directly to one Pod per file. Keep that shape on resume.
|
|
1321
|
+
if (own.reason === 'stall' && files.length > 1 && child(`${name}-file-0`, false)) return { way: 'file-fanout', attemptId };
|
|
1211
1322
|
// Only the first child of the split the caller takes now (`-file-0` · `-c0` · `-0` · review r3): a record of another
|
|
1212
1323
|
// split shape is not followed.
|
|
1213
1324
|
return splitChild !== undefined && child(`${name}${splitChild}`, false) ? { way: 'split', attemptId } : undefined;
|
|
@@ -1239,10 +1350,11 @@ function failuresOf(run: CommandResult, label: string): SweepFailures {
|
|
|
1239
1350
|
const output = stripPodRepoRoot(run.output.replace(/\x1b\[[0-9;]*[A-Za-z]/g, ''));
|
|
1240
1351
|
const ran = lastMatch(output, /Ran (\d+) tests? across ([1-9]\d*) files?/);
|
|
1241
1352
|
const reportedErrors = summaryCount(output, 'error');
|
|
1242
|
-
|
|
1243
|
-
const
|
|
1244
|
-
const
|
|
1245
|
-
const
|
|
1353
|
+
// Validated Pod shards were checked separately before their logs were joined; only the host sweep needs subrun totals.
|
|
1354
|
+
const subruns = !run.validatedShards && run.rc === 1 ? deterministicSubrunCounts(output) : undefined;
|
|
1355
|
+
const errorCount = subruns?.error ?? (Number.isNaN(reportedErrors) ? 0 : reportedErrors);
|
|
1356
|
+
const incomplete = (reason: string) => new Error(`${label} incomplete (rc=${run.rc}; Ran=${ran ? ran[0] : 'missing'}; errors=${subruns || !Number.isNaN(reportedErrors) ? errorCount : 'missing'}; ${reason})\n${tail40(output)}`);
|
|
1357
|
+
const count = subruns?.fail ?? summaryCount(output, 'fail');
|
|
1246
1358
|
// Bun repeats failed test names under "N tests failed:". A validated Pod sweep joins
|
|
1247
1359
|
// separately checked shards, so a summary in one shard can precede real failures
|
|
1248
1360
|
// from later shards. Remove each repeated block, not everything after the last one.
|
|
@@ -1253,6 +1365,11 @@ function failuresOf(run: CommandResult, label: string): SweepFailures {
|
|
|
1253
1365
|
? output.replace(/^\d+ tests? failed:\r?\n(?:\(fail\)[^\r\n]*(?:\r?\n|$))*(?:[ \t]*\r?\n)*/gm, '')
|
|
1254
1366
|
: failureSummary ? output.slice(0, failureSummary.index) : output;
|
|
1255
1367
|
const occurrences = [...executionOutput.matchAll(/^[ \t]*\(fail\)[ \t]+.+?(?:[ \t]+\[[\d.]+(?:ms|s)\])?[ \t]*\r?$/gm)].length;
|
|
1368
|
+
const collapsed = collapseGateFailureAttempts(parseFailureAttempts(executionOutput));
|
|
1369
|
+
const failures = collapsed.failures;
|
|
1370
|
+
if (collapsed.collapsed > 0) debug.log('release-loop.gate', 'failure-attribution-deduplicated', {
|
|
1371
|
+
label, summary: count, observed: occurrences, identified: collapsed.failures.length, collapsed: collapsed.collapsed,
|
|
1372
|
+
});
|
|
1256
1373
|
// A test that leaves process.exitCode set leaks a non-zero exit into a run whose own summary is complete and clean
|
|
1257
1374
|
// (0.2.7 gate: agent-cli at v0.2.6 · 12 pass · 0 fail · rc 2). The summary is the verdict; read that exit as clean.
|
|
1258
1375
|
const leakedExit = !!ran && run.rc !== 0 && run.rc !== 1 && count === 0 && errorCount === 0 && occurrences === 0
|
|
@@ -1271,7 +1388,11 @@ function failuresOf(run: CommandResult, label: string): SweepFailures {
|
|
|
1271
1388
|
}
|
|
1272
1389
|
}
|
|
1273
1390
|
if (errors.length !== errorCount) throw incomplete(`error attribution identified=${errors.length}`);
|
|
1274
|
-
|
|
1391
|
+
const retryDuplicatesExplainCount = occurrences > count && collapsed.observed === occurrences && failures.length === count;
|
|
1392
|
+
if (Number.isFinite(count) && (failures.length !== count || (occurrences !== count && !retryDuplicatesExplainCount))) {
|
|
1393
|
+
throw new GateFailureAttributionMismatch(label, count, occurrences, failures.length, collapsed.collapsed, failures);
|
|
1394
|
+
}
|
|
1395
|
+
if (!Number.isFinite(count) || (count > 0 && failures.length === 0)
|
|
1275
1396
|
|| (rc === 1 && count === 0 && errorCount === 0) || (rc === 0 && count + errorCount > 0)) {
|
|
1276
1397
|
throw incomplete(`failure attribution incomplete: summary=${count}, identified=${failures.length}`);
|
|
1277
1398
|
}
|
|
@@ -1298,7 +1419,7 @@ export function readIsolatedRun(run: CommandResult, label: string, file: string)
|
|
|
1298
1419
|
if (!cleanNonzero) {
|
|
1299
1420
|
const read = failuresOf(run, label);
|
|
1300
1421
|
// Two (fail) lines folding into one id would undercount the file's failures: unreadable, not fewer.
|
|
1301
|
-
const failCount = summaryCount(output, 'fail');
|
|
1422
|
+
const failCount = deterministicSubrunCounts(output)?.fail ?? summaryCount(output, 'fail');
|
|
1302
1423
|
if (Number.isFinite(failCount) && read.failures.length !== failCount)
|
|
1303
1424
|
throw new Error(`${label} incomplete (rc=${run.rc}; failure attribution summary=${failCount}, identified=${read.failures.length})`);
|
|
1304
1425
|
return { kind: 'read', ...read };
|
|
@@ -1360,6 +1481,7 @@ class SettledRecheckFailures extends Error {
|
|
|
1360
1481
|
|
|
1361
1482
|
/** Isolated re-runs (host stall re-run · cut and baseline re-checks) are time-limited; sweeps and installs keep their full time. */
|
|
1362
1483
|
export const GATE_ISOLATED_TIMEOUT_MS = 300_000;
|
|
1484
|
+
export const GATE_ATTRIBUTION_RECHECK_FILE_LIMIT = 64;
|
|
1363
1485
|
/** Default final recheck window size; overflow continues in later windows. */
|
|
1364
1486
|
export const GATE_HOST_RECHECK_FILE_LIMIT = 32;
|
|
1365
1487
|
/** Two isolated test processes in each of four Pods = eight at most, with one checkout/install per eight files. */
|
|
@@ -1419,7 +1541,40 @@ export interface GateAdmission {
|
|
|
1419
1541
|
/** The pool-wide reading admission waits on (recommended · limitedBy · memorySlots); null before any reading. */
|
|
1420
1542
|
waitReason(): string | null;
|
|
1421
1543
|
}
|
|
1422
|
-
export const GATE_ADMISSION_WAIT_SECONDS_DEFAULT =
|
|
1544
|
+
export const GATE_ADMISSION_WAIT_SECONDS_DEFAULT = 1800;
|
|
1545
|
+
/** Queue pressure scales the default wait but never creates an unbounded admission timer. */
|
|
1546
|
+
export function gateAdmissionWaitSeconds(shards: number, slots: number): number {
|
|
1547
|
+
return Math.max(GATE_ADMISSION_WAIT_SECONDS_DEFAULT, Math.min(3600,
|
|
1548
|
+
Math.ceil(shards / Math.max(1, slots)) * 300));
|
|
1549
|
+
}
|
|
1550
|
+
export function createGateRootLimiter(slots: number): <T>(run: () => Promise<T>) => Promise<T> {
|
|
1551
|
+
if (!Number.isSafeInteger(slots) || slots < 1) throw new Error('invalid gate root slots');
|
|
1552
|
+
let active = 0;
|
|
1553
|
+
const waiters: Array<() => void> = [];
|
|
1554
|
+
return async <T>(run: () => Promise<T>): Promise<T> => {
|
|
1555
|
+
if (active >= slots) await new Promise<void>((resolve) => waiters.push(resolve));
|
|
1556
|
+
else active++;
|
|
1557
|
+
try { return await run(); }
|
|
1558
|
+
finally {
|
|
1559
|
+
const next = waiters.shift();
|
|
1560
|
+
if (next) next(); // transfer the occupied slot before another caller can enter
|
|
1561
|
+
else active--;
|
|
1562
|
+
}
|
|
1563
|
+
};
|
|
1564
|
+
}
|
|
1565
|
+
/** A direct gate has no preflight image pin. Its measured receipt still carries its own image digest. */
|
|
1566
|
+
export function gateReceiptReuseKey(receipt: GateShardReceipt | null, fingerprint: string, expectedImageDigest: string | null | undefined,
|
|
1567
|
+
hosts: ReadonlySet<string>): { envKey?: string; reason: string } {
|
|
1568
|
+
if (!receipt) return { reason: 'receipt-missing' };
|
|
1569
|
+
if (receipt.status !== 'complete') return { reason: 'receipt-unmeasured' };
|
|
1570
|
+
if (!hosts.has(receipt.host)) return { reason: 'receipt-host-outside-pool' };
|
|
1571
|
+
if (expectedImageDigest) return receipt.envKey === `${fingerprint}:${expectedImageDigest}`
|
|
1572
|
+
? { envKey: receipt.envKey, reason: 'image-pinned' } : { reason: 'receipt-image-or-env-drift' };
|
|
1573
|
+
const prefix = `${fingerprint}:`;
|
|
1574
|
+
if (!receipt.envKey.startsWith(prefix)) return { reason: 'receipt-env-drift' };
|
|
1575
|
+
return /^sha256:[a-f0-9]{64}$/i.test(receipt.envKey.slice(prefix.length))
|
|
1576
|
+
? { envKey: receipt.envKey, reason: 'measured-image' } : { reason: 'receipt-image-unmeasured' };
|
|
1577
|
+
}
|
|
1423
1578
|
export type GateAdmissionOutcome = 'granted' | 'admission-timeout' | 'admission-error' | 'off';
|
|
1424
1579
|
export interface GateAdmissionResult { outcome: GateAdmissionOutcome; waitedMs: number; reason?: string; release: () => void }
|
|
1425
1580
|
export const gateAdmissionWaitText = (reading: string | null) => `메모리 여유 대기(admission) · ${reading ?? '측정 전'}`.slice(0, 160);
|
|
@@ -1569,6 +1724,10 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1569
1724
|
const watchList = pod.riskWatch ? readGateRiskWatchList(pod.riskWatch) : [];
|
|
1570
1725
|
const riskPlan = planGateRiskLaneShards({ assignable, durations, shardCount, heavy, watchList });
|
|
1571
1726
|
const shards = riskPlan.shards;
|
|
1727
|
+
const receiptRoot = pod.shardsFile?.receiptRoot;
|
|
1728
|
+
const base40 = pod.shardsFile?.base40;
|
|
1729
|
+
const envFingerprint = pod.shardsFile?.envFingerprint;
|
|
1730
|
+
const expectedImageDigest = pod.shardsFile?.expectedImageDigest;
|
|
1572
1731
|
if (riskPlan.spawnFiles.length) debug.log('release-loop.gate', 'pod-spawn-lane', { files: riskPlan.spawnFiles, shards: riskPlan.spawnShardCount });
|
|
1573
1732
|
if (riskPlan.riskFiles.length) debug.log('release-loop.gate', 'pod-risk-lane-plan', { files: riskPlan.riskFiles, shard: 0,
|
|
1574
1733
|
plannedSeconds: riskPlan.riskPlannedSeconds, shards: shards.length, watch: watchList.length });
|
|
@@ -1590,6 +1749,56 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1590
1749
|
const ready = poolOverride ? undefined : checkPodPool(parsePodPool(pod.pool), kubectl);
|
|
1591
1750
|
if (ready && !ready.ok) throw new Error('pod command: 풀의 노드가 하나도 준비되지 않았다');
|
|
1592
1751
|
const poolScheduler = poolOverride ?? new PodPoolScheduler(ready!.ready);
|
|
1752
|
+
const receiptChecks = shards.map((item, shard) => {
|
|
1753
|
+
if (pod.freshShards) return { reason: 'fresh-shards' };
|
|
1754
|
+
if (!receiptRoot || !base40 || !envFingerprint) return { reason: 'receipt-context-missing' };
|
|
1755
|
+
const prior = readGateShardManifest(receiptRoot, `pod-${shard}`);
|
|
1756
|
+
const key = gateReceiptReuseKey(prior, envFingerprint, expectedImageDigest,
|
|
1757
|
+
new Set(poolScheduler.members.map((member) => member.context)));
|
|
1758
|
+
if (!key.envKey) return { reason: key.reason };
|
|
1759
|
+
const result = readGateShardResult(receiptRoot, { shard: { id: `pod-${shard}`, files: item.files },
|
|
1760
|
+
head40: commit, base40, envKey: key.envKey });
|
|
1761
|
+
return result ? { reason: key.reason, result } : { reason: 'receipt-identity-or-evidence-invalid' };
|
|
1762
|
+
});
|
|
1763
|
+
const receipts = receiptChecks.flatMap((check) => check.result ? [check.result.receipt] : []);
|
|
1764
|
+
const requiredOs = pod.shardCapabilities ?? {};
|
|
1765
|
+
const envKnownIds = loadEnvKnownFailures(tree);
|
|
1766
|
+
const envKnownFiles = new Set([...envKnownIds].map(fileOf));
|
|
1767
|
+
const macPassShardIds = shards.flatMap((shard, index) => {
|
|
1768
|
+
const prior = receipts.find((receipt) => receipt.id === `pod-${index}`);
|
|
1769
|
+
if (!prior) return [];
|
|
1770
|
+
const knownFailure = [...prior.failureIds, ...prior.errorIds].some((id) => envKnownIds.has(id));
|
|
1771
|
+
const knownUnmeasuredFile = prior.status === 'unmeasured' && shard.files.some((file) => envKnownFiles.has(file));
|
|
1772
|
+
return knownFailure || knownUnmeasuredFile ? [`pod-${index}`] : [];
|
|
1773
|
+
});
|
|
1774
|
+
for (const id of Object.keys(requiredOs)) if (!/^pod-\d+$/.test(id) || Number(id.slice(4)) >= shards.length)
|
|
1775
|
+
throw new Error(`unknown shard capability target: ${id}`);
|
|
1776
|
+
const needsPlatform = Object.values(requiredOs).some((value) => value.os)
|
|
1777
|
+
|| (pod.rerouteShard === 'auto' && macPassShardIds.length > 0);
|
|
1778
|
+
const hosts = poolScheduler.members.map((member) => {
|
|
1779
|
+
let platform: 'darwin' | 'linux' | 'other' = 'other';
|
|
1780
|
+
if (needsPlatform) {
|
|
1781
|
+
const system = member.sshHost ? remoteRun(member.sshHost, 'uname -s') :
|
|
1782
|
+
{ status: 0, stdout: process.platform, stderr: '' };
|
|
1783
|
+
if (system.status === 0) platform = /^(?:darwin)$/i.test(system.stdout.trim()) ? 'darwin'
|
|
1784
|
+
: /^(?:linux)$/i.test(system.stdout.trim()) ? 'linux' : 'other';
|
|
1785
|
+
}
|
|
1786
|
+
return { id: member.context, platform, available: true, reserved: false };
|
|
1787
|
+
});
|
|
1788
|
+
const hostPlan = planGateShardDispatch({ head40: commit, base40: base40 ?? commit,
|
|
1789
|
+
shards: shards.map((item, shard) => ({ id: `pod-${shard}`, files: item.files,
|
|
1790
|
+
...(requiredOs[`pod-${shard}`]?.os ? { requiredOs: requiredOs[`pod-${shard}`]!.os } : {}) })), receipts,
|
|
1791
|
+
hosts,
|
|
1792
|
+
macPassShardIds,
|
|
1793
|
+
...(pod.rerouteShard ? { rerouteShard: pod.rerouteShard } : {}), ...(pod.shardHost ? { shardHost: pod.shardHost } : {}) });
|
|
1794
|
+
// Leave ordinary shards to the live free-slot scheduler. A static plan can pick a busy member
|
|
1795
|
+
// while another is idle; pin only a capability or a deliberate cross-host replacement.
|
|
1796
|
+
const hostPins = new Map(hostPlan.flatMap((item) => item.kind === 'run'
|
|
1797
|
+
&& (requiredOs[item.shardId]?.os || (pod.rerouteShard === 'auto' && macPassShardIds.includes(item.shardId))
|
|
1798
|
+
|| (item.replacing && item.host !== item.replacing.host)
|
|
1799
|
+
|| (pod.shardHost && item.host === pod.shardHost)) ? [[item.shardId, item.host] as const] : []));
|
|
1800
|
+
const unavailable = hostPlan.find((item) => item.kind === 'unmeasured');
|
|
1801
|
+
if (unavailable) throw new Error(`gate shard ${unavailable.shardId} unmeasured: ${unavailable.reason}`);
|
|
1593
1802
|
const configCache = typeof pod.bunCache === 'string' ? pod.bunCache.trim() : '';
|
|
1594
1803
|
const envCache = process.env[POD_BUN_CACHE_HOST_PATH]?.trim();
|
|
1595
1804
|
const bunCache = configCache || envCache || undefined;
|
|
@@ -1600,11 +1809,13 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1600
1809
|
if (!Number.isSafeInteger(installSlots) || installSlots < 0) throw new Error('invalid pod sweep options');
|
|
1601
1810
|
debug.log('release-loop.gate', 'pod-install-slots', { slots: installSlots, source: pod.installSlots !== undefined ? 'config' : 'default' });
|
|
1602
1811
|
// GATE-MEM-ADMIT: every shard Job (retries and splits too) asks pool admission before it is placed.
|
|
1603
|
-
const
|
|
1812
|
+
const rootSlots = Math.max(1, poolScheduler.members.reduce((total, member) => total + member.capacity, 0));
|
|
1813
|
+
const admissionWaitSeconds = pod.admissionWaitSeconds ?? gateAdmissionWaitSeconds(shards.length, rootSlots);
|
|
1604
1814
|
if (!Number.isSafeInteger(admissionWaitSeconds) || admissionWaitSeconds < 0) throw new Error('invalid pod sweep options');
|
|
1605
1815
|
// An injected pool (tests) runs without admission unless `pod.admission` is passed; the gate's own pool asks it.
|
|
1606
1816
|
const gateAdmission = pod.admission ?? tools.admission ?? (poolOverride ? undefined : poolGateAdmission(poolScheduler));
|
|
1607
|
-
debug.log('release-loop.gate', 'pod-admission', { waitSeconds: admissionWaitSeconds, source: pod.admissionWaitSeconds !== undefined ? 'config' : 'default', enabled: !!gateAdmission && admissionWaitSeconds > 0
|
|
1817
|
+
debug.log('release-loop.gate', 'pod-admission', { waitSeconds: admissionWaitSeconds, source: pod.admissionWaitSeconds !== undefined ? 'config' : 'default', enabled: !!gateAdmission && admissionWaitSeconds > 0,
|
|
1818
|
+
rootSlots, shards: shards.length });
|
|
1608
1819
|
// GATE-POD-STALL-FAILFAST: a shard with no test output within the stall window is stopped and recorded unmeasured.
|
|
1609
1820
|
const stallSeconds = pod.stallSeconds ?? GATE_POD_STALL_SECONDS_DEFAULT;
|
|
1610
1821
|
const stallPollMs = pod.stallPollMs ?? GATE_POD_STALL_POLL_MS;
|
|
@@ -1695,7 +1906,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1695
1906
|
return { ...(podName ? { podName } : {}), ...(termination ? { termination } : {}), ...(logs?.status === 0 ? { logTail: logs.stdout } : {}),
|
|
1696
1907
|
...(errors.length ? { error: `수거 실패 · ${errors.join(' · ')}` } : {}) };
|
|
1697
1908
|
});
|
|
1698
|
-
type ShardRun = { output: string; rc: number; pass: number; fail: number; errors: number; ran: number; files: number; passedIds: string[] };
|
|
1909
|
+
type ShardRun = { output: string; rc: number; pass: number; fail: number; errors: number; ran: number; files: number; passedIds: string[]; loadFlaky?: string[]; loadFlakyAttemptedFiles?: string[] };
|
|
1699
1910
|
type ShardReason = NonNullable<GateResult['stalledShards']>[number]['reason'];
|
|
1700
1911
|
// 깊이 끝 OOM 조각은 «조각 전체»를 파일 하나씩 격리한다 — 상한 40 이면 나머지가 못 잰 채 남았다(09-30 ① 7번 조각 110 파일).
|
|
1701
1912
|
const isolationRemaining = shards.map((item) => item.files.length);
|
|
@@ -1722,29 +1933,88 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1722
1933
|
}, 0);
|
|
1723
1934
|
const knownTimes = assignable.map((file) => durations.get(file)).filter((value): value is number => value !== undefined && Number.isFinite(value) && value >= 0).sort((a, b) => a - b);
|
|
1724
1935
|
const unknownSeconds = knownTimes.length ? (knownTimes[Math.floor((knownTimes.length - 1) / 2)]! + knownTimes[Math.floor(knownTimes.length / 2)]!) / 2 : 1;
|
|
1725
|
-
// GATE-LIVE-OBS —
|
|
1726
|
-
//
|
|
1936
|
+
// GATE-LIVE-OBS — root rows stay stable; each recursive split/retry and tail gets its own row.
|
|
1937
|
+
// Otherwise a finished root Job can hide a still-running child (or tail) behind 80/80 done.
|
|
1727
1938
|
const board: GateShard[] = shards.map((item, index) => ({ id: `pod-${index}`, state: 'pending', plannedMin: Math.round(item.plannedSeconds / 6) / 10 }));
|
|
1939
|
+
const filesById = new Map(shards.map((item, index) => [`pod-${index}`, [...item.files]]));
|
|
1940
|
+
const detailsById = new Map<string, Partial<GateShardLedgerRow>>();
|
|
1941
|
+
const observeShard = (row: GateShard) => {
|
|
1942
|
+
if (!pod.shardsFile?.ledgerPath || !pod.shardsFile.envFingerprint) return;
|
|
1943
|
+
const detail = detailsById.get(row.id) ?? {};
|
|
1944
|
+
const logName = row.resultFile?.replace(/\.json$/, '') ?? null;
|
|
1945
|
+
const logPath = logName && logDir ? join(logDir, `${logName}.log`) : null;
|
|
1946
|
+
const junitPath = logName && logDir ? join(logDir, `${logName}.junit.xml`) : null;
|
|
1947
|
+
try {
|
|
1948
|
+
appendGateShardLedger(pod.shardsFile.ledgerPath, {
|
|
1949
|
+
v: 1, version: pod.shardsFile.version, phase: 'cut', shardId: row.id,
|
|
1950
|
+
files: filesById.get(row.id) ?? [], host: row.host ?? null, executor: 'pod',
|
|
1951
|
+
imageDigest: detail.imageDigest ?? null, commit, envFingerprint: pod.shardsFile.envFingerprint,
|
|
1952
|
+
startedAt: row.startedAt ?? null, endedAt: row.endedAt ?? null, observedAt: new Date().toISOString(),
|
|
1953
|
+
state: row.state, rc: row.rc ?? null, pass: detail.pass ?? null, fail: detail.fail ?? null,
|
|
1954
|
+
errors: detail.errors ?? null, ran: detail.ran ?? null,
|
|
1955
|
+
failureIds: detail.failureIds ?? null, errorIds: detail.errorIds ?? null,
|
|
1956
|
+
attribution: detail.attribution ?? 'unmeasured', logPath, junitPath,
|
|
1957
|
+
stopReason: detail.stopReason ?? row.waitReason ?? null, attemptId: row.attemptId ?? null,
|
|
1958
|
+
});
|
|
1959
|
+
} catch (error) { debug.log('release-loop.gate', 'shard-ledger-write-failed', { shard: row.id, error: String(error) }, { level: 'warn' }); }
|
|
1960
|
+
};
|
|
1728
1961
|
const flushBoard = () => {
|
|
1729
1962
|
if (!pod.shardsFile) return;
|
|
1730
1963
|
try { writeGateShards(pod.shardsFile.path, { v: 1, version: pod.shardsFile.version, updatedAt: new Date().toISOString(), shards: board }); }
|
|
1731
1964
|
catch (error) { debug.log('release-loop.gate', 'shards-write-failed', { path: pod.shardsFile.path, error: String(error) }); }
|
|
1732
1965
|
};
|
|
1733
|
-
const
|
|
1734
|
-
|
|
1966
|
+
const rowIndex = (shard: number, branch = '') => branch ? board.findIndex((row) => row.id === `pod-${shard}${branch}`) : shard;
|
|
1967
|
+
const markShard = (shard: number, patch: Partial<GateShard>, clear: Array<keyof GateShard> = [], branch = '') => {
|
|
1968
|
+
const index = rowIndex(shard, branch);
|
|
1969
|
+
const row = board[index];
|
|
1735
1970
|
if (!row) return;
|
|
1736
1971
|
const next: GateShard = { ...row, ...patch };
|
|
1737
1972
|
for (const key of clear) delete next[key];
|
|
1738
1973
|
if (JSON.stringify(next) === JSON.stringify(row)) return;
|
|
1739
|
-
board[
|
|
1974
|
+
board[index] = next;
|
|
1975
|
+
observeShard(next);
|
|
1740
1976
|
flushBoard();
|
|
1741
1977
|
};
|
|
1742
|
-
const resetShardAttempt = (shard: number) => {
|
|
1743
|
-
|
|
1978
|
+
const resetShardAttempt = (shard: number, branch = '', plannedMin?: number) => {
|
|
1979
|
+
if (branch && rowIndex(shard, branch) < 0) board.push({ id: `pod-${shard}${branch}`, state: 'pending', ...(plannedMin !== undefined ? { plannedMin } : {}) });
|
|
1980
|
+
const index = rowIndex(shard, branch);
|
|
1981
|
+
const row = board[index];
|
|
1744
1982
|
if (!row) return;
|
|
1745
|
-
board[
|
|
1983
|
+
board[index] = beginGateShardAttempt(row);
|
|
1984
|
+
delete board[index]!.rc;
|
|
1985
|
+
delete board[index]!.installSec;
|
|
1986
|
+
// The root is a logical aggregate while its descendants run, not the terminal state of its first Job.
|
|
1987
|
+
if (branch) {
|
|
1988
|
+
board[shard] = beginGateShardAttempt(board[shard]!);
|
|
1989
|
+
delete board[shard]!.rc;
|
|
1990
|
+
delete board[shard]!.installSec;
|
|
1991
|
+
observeShard(board[shard]!);
|
|
1992
|
+
}
|
|
1993
|
+
detailsById.delete(row.id);
|
|
1994
|
+
observeShard(board[index]!);
|
|
1746
1995
|
flushBoard();
|
|
1747
1996
|
};
|
|
1997
|
+
const recordShardResult = (id: string, done: { runs: ShardRun[]; stalled: NonNullable<GateResult['stalledShards']> }) => {
|
|
1998
|
+
const index = board.findIndex((row) => row.id === id);
|
|
1999
|
+
if (index < 0) return;
|
|
2000
|
+
const counts = done.runs.reduce((sum, run) => ({ pass: sum.pass + run.pass, fail: sum.fail + run.fail,
|
|
2001
|
+
errors: sum.errors + run.errors, ran: sum.ran + run.ran }), { pass: 0, fail: 0, errors: 0, ran: 0 });
|
|
2002
|
+
const failureIds: string[] = [];
|
|
2003
|
+
const errorIds: string[] = [];
|
|
2004
|
+
let attributed = done.stalled.length === 0 && done.runs.length > 0;
|
|
2005
|
+
for (const run of done.runs) {
|
|
2006
|
+
try {
|
|
2007
|
+
const parsed = failuresOf({ rc: run.rc, output: run.output, validatedShards: true }, `ledger ${id}`);
|
|
2008
|
+
failureIds.push(...parsed.failures); errorIds.push(...parsed.errors);
|
|
2009
|
+
} catch { attributed = false; }
|
|
2010
|
+
}
|
|
2011
|
+
const detail: Partial<GateShardLedgerRow> = { ...(detailsById.get(id) ?? {}), ...counts, failureIds: attributed ? [...new Set(failureIds)].sort() : null,
|
|
2012
|
+
errorIds: attributed ? [...new Set(errorIds)].sort() : null, attribution: attributed ? 'complete' : 'unmeasured',
|
|
2013
|
+
stopReason: done.stalled.map((item) => item.reason).join(', ') || null };
|
|
2014
|
+
detailsById.set(id, detail);
|
|
2015
|
+
observeShard(board[index]!);
|
|
2016
|
+
};
|
|
2017
|
+
for (const row of board) observeShard(row);
|
|
1748
2018
|
flushBoard();
|
|
1749
2019
|
// A result collector can lose its callback after the Kubernetes Job has already completed. Reconcile
|
|
1750
2020
|
// the public shard board independently so a finished Job cannot remain «running» indefinitely.
|
|
@@ -1758,6 +2028,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1758
2028
|
const index = board.findIndex((shard) => shard.id === item.shard);
|
|
1759
2029
|
if (index < 0) continue;
|
|
1760
2030
|
board[index] = reconciled.board.find((shard) => shard.id === item.shard)!;
|
|
2031
|
+
observeShard(board[index]!);
|
|
1761
2032
|
debug.log('release.gate', 'shard-stale-finalized', item);
|
|
1762
2033
|
}
|
|
1763
2034
|
flushBoard();
|
|
@@ -1768,22 +2039,25 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1768
2039
|
/** The gate's scheduler, seen through one shard: a slot wait and the Pod start become row states. */
|
|
1769
2040
|
// GATE-MEM-ADMIT: `placed` releases the shard's admission once a member is chosen — from then on the Job is
|
|
1770
2041
|
// Pending/Running in the cluster and the next admission reading counts it, so holding the lease would count it twice.
|
|
1771
|
-
const trackedPool = (shard: number,
|
|
2042
|
+
const trackedPool = (shard: number, branch: string, jobName: string, resultFile: string, attemptId: string, placed: (member: NonNullable<Awaited<ReturnType<PodPoolScheduler['tryAcquire']>>>) => void = () => {}, skipMembers: ReadonlySet<string> = new Set(), allowPin = false): PodPoolScheduler => ({
|
|
1772
2043
|
members: poolScheduler.members,
|
|
1773
2044
|
tryAcquire: async (...[skip, onEmpty, mode]: Parameters<PodPoolScheduler['tryAcquire']>) => {
|
|
1774
2045
|
// GATE-MULTI-HOST: a lost member is skipped while another member is left (all lost = the old behaviour).
|
|
1775
2046
|
const anyLeft = poolScheduler.members.some((m) => !downMembers.has(m.context));
|
|
1776
|
-
|
|
2047
|
+
// Only the first placement follows the plan; retries and unavailable hosts can move.
|
|
2048
|
+
const planned = allowPin && !skip?.size ? hostPins.get(`pod-${shard}`) : undefined;
|
|
2049
|
+
const pin = planned && !downMembers.has(planned) ? planned : undefined;
|
|
2050
|
+
const excluded = new Set([...(skip ?? []), ...skipMembers, ...(downMembers.size && anyLeft ? downMembers : []),
|
|
2051
|
+
...(pin ? poolScheduler.members.filter((m) => m.context !== pin).map((m) => m.context) : [])]);
|
|
1777
2052
|
const member = await poolScheduler.tryAcquire(excluded.size ? excluded : undefined, onEmpty, mode);
|
|
1778
2053
|
if (member) placed(member);
|
|
1779
|
-
if (member) markShard(shard, { state:
|
|
1780
|
-
else markShard(shard, { waitReason: 'Pod 자리 대기' });
|
|
2054
|
+
if (member) markShard(shard, { state: 'running', startedAt: board[rowIndex(shard, branch)]?.startedAt ?? new Date().toISOString(), host: member.context, job: jobName, resultFile, attemptId }, ['waitReason'], branch);
|
|
2055
|
+
else markShard(shard, { waitReason: 'Pod 자리 대기' }, [], branch);
|
|
1781
2056
|
return member;
|
|
1782
2057
|
},
|
|
1783
2058
|
release: (member: Parameters<PodPoolScheduler['release']>[0]) => poolScheduler.release(member),
|
|
1784
2059
|
}) as unknown as PodPoolScheduler;
|
|
1785
|
-
const
|
|
1786
|
-
if (branch || depth > 0 || parentAttempt !== undefined || hostRetries > 0) resetShardAttempt(shard);
|
|
2060
|
+
const runShardImpl = async (paths: string[], shard: number, depth = 0, branch = '', retriedNoOutput = false, fresh = pod.freshShards === true, parentAttempt?: string, hostRetries = 0, batchIsolation = false, skipMembers: ReadonlySet<string> = new Set(), singleShotTail = false): Promise<{ runs: ShardRun[]; stalled: NonNullable<GateResult['stalledShards']> }> => {
|
|
1787
2061
|
// 파일별 소요는 junit 으로 남긴다 — 콘솔 요약(판정 원천)은 그대로이고, 느린 시험 목록(K10 D4)·계층 분리(D2)의 자가 된다.
|
|
1788
2062
|
// GATE-TIMEOUT-HEAL: parts (each with its own junit) under a soft deadline inside this shard's own Job deadline.
|
|
1789
2063
|
const shardDeadline = shardDeadlineSeconds(batchIsolation ? Math.max(...paths.map((file) => estimated([file]))) * Math.ceil(paths.length / batchWidth) : estimated(paths), deadlineSeconds);
|
|
@@ -1805,6 +2079,8 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1805
2079
|
let batchArtifactsComplete = true;
|
|
1806
2080
|
let junit: string | undefined;
|
|
1807
2081
|
let rc: number | undefined;
|
|
2082
|
+
let shardLoadFlaky: string[] = [];
|
|
2083
|
+
let shardLoadFlakyAttemptedFiles: string[] = [];
|
|
1808
2084
|
let jobExitCode: number | undefined;
|
|
1809
2085
|
let batchArtifactDir: string | undefined;
|
|
1810
2086
|
let lastFile: string | undefined;
|
|
@@ -1847,13 +2123,13 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1847
2123
|
if (descent === 'retry') {
|
|
1848
2124
|
// «descend» is only the way down; «resumed» comes from each leaf whose log reads cleanly (review r4).
|
|
1849
2125
|
debug.log('release-loop.gate', 'pod-shard-descend', { shard, branch, files: paths.length, via: 'retry' });
|
|
1850
|
-
resetShardAttempt(shard);
|
|
2126
|
+
resetShardAttempt(shard, branch);
|
|
1851
2127
|
return runShard(paths, shard, depth, `${branch}-retry`, true, undefined, attemptId, hostRetries, batchIsolation, skipMembers);
|
|
1852
2128
|
}
|
|
1853
|
-
if (descent === 'split') {
|
|
1854
|
-
resetShardAttempt(shard);
|
|
2129
|
+
if (descent === 'split' || descent === 'file-fanout') {
|
|
2130
|
+
resetShardAttempt(shard, branch);
|
|
1855
2131
|
output = undefined;
|
|
1856
|
-
debug.log('release-loop.gate', 'pod-shard-descend', { shard, branch, files: paths.length, via:
|
|
2132
|
+
debug.log('release-loop.gate', 'pod-shard-descend', { shard, branch, files: paths.length, via: descent });
|
|
1857
2133
|
} else if (reused) {
|
|
1858
2134
|
output = reused.output;
|
|
1859
2135
|
junit = reused.junit;
|
|
@@ -1862,7 +2138,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1862
2138
|
lastFile = lastStartedTestFile(output, paths);
|
|
1863
2139
|
} else try {
|
|
1864
2140
|
stamp('admissionStart');
|
|
1865
|
-
admitted = await admitGateShard(gateAdmission, { waitMs: admissionWaitSeconds * 1000, onWait: (reason) => markShard(shard, { waitReason: reason }) });
|
|
2141
|
+
admitted = await admitGateShard(gateAdmission, { waitMs: admissionWaitSeconds * 1000, onWait: (reason) => markShard(shard, { waitReason: reason }, [], branch) });
|
|
1866
2142
|
stamp('admissionEnd');
|
|
1867
2143
|
if (admitted.outcome !== 'off') debug.log('release-loop.gate', 'shard-admission', { shard, branch, outcome: admitted.outcome, waitedMs: admitted.waitedMs, ...(admitted.reason ? { reason: admitted.reason } : {}) },
|
|
1868
2144
|
admitted.outcome === 'granted' ? undefined : { level: 'warn' });
|
|
@@ -1903,10 +2179,10 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1903
2179
|
graphNode: 'gate', version: pod.shardsFile?.version, commit, shard: `${shard + 1}-of-${shards.length}`,
|
|
1904
2180
|
reason: `release gate shard ${shard + 1}/${shards.length}`, pool: pod.pool, cpuReq: cpu.request,
|
|
1905
2181
|
memReq: memoryLimit ?? POD_ISOLATION_MEMORY_LIMIT, warmReused: false },
|
|
1906
|
-
pool: pod.pool, poolScheduler: trackedPool(shard,
|
|
2182
|
+
pool: pod.pool, poolScheduler: trackedPool(shard, branch, jobName, resultFile, attemptId, (member) => {
|
|
1907
2183
|
stamp('poolPlaced'); jobContext = member.context; jobContextForCapture = member.context; placedOn = member.context; placed();
|
|
1908
2184
|
if (branch === '-tail') tailPlacedHost = member.context;
|
|
1909
|
-
}, skipMembers), clone: true, source: { kind: 'commit', sha: commit }, deadlineSeconds: shardDeadline,
|
|
2185
|
+
}, skipMembers, depth === 0 && branch === '' && hostRetries === 0 && !retriedNoOutput && !batchIsolation && skipMembers.size === 0), clone: true, source: { kind: 'commit', sha: commit }, deadlineSeconds: shardDeadline,
|
|
1910
2186
|
...gateHostMirror(pod),
|
|
1911
2187
|
...(multiHost ? { statusUnreadableMs: memberUnreadableSeconds * 1000 } : {}),
|
|
1912
2188
|
...(bunCache ? { bunCache } : {}),
|
|
@@ -1935,6 +2211,31 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1935
2211
|
if (job.exitCode !== 0) captureFailure();
|
|
1936
2212
|
batchArtifactDir = job.artifactsDir;
|
|
1937
2213
|
const read = readShardArtifacts(job.artifactsDir, parts);
|
|
2214
|
+
if (logDir) {
|
|
2215
|
+
mkdirSync(logDir, { recursive: true });
|
|
2216
|
+
let retryArtifactIndex = 0;
|
|
2217
|
+
const retryEvidence = parts.flatMap((files, partIndex) => {
|
|
2218
|
+
const path = join(job.artifactsDir, `part-${partIndex}.retry-files`);
|
|
2219
|
+
if (!existsSync(path)) return [];
|
|
2220
|
+
const retryFiles = readFileSync(path, 'utf8').split(/\r?\n/).filter((file) => file && files.includes(file));
|
|
2221
|
+
return retryFiles.map((file, retryIndex) => {
|
|
2222
|
+
const prefix = `part-${partIndex}.retry-${retryIndex}`;
|
|
2223
|
+
const persisted: Record<string, unknown> = { file, started: existsSync(join(job.artifactsDir, `${prefix}.started`)), rc: existsSync(join(job.artifactsDir, `${prefix}.rc`))
|
|
2224
|
+
? readFileSync(join(job.artifactsDir, `${prefix}.rc`), 'utf8').trim() : null };
|
|
2225
|
+
for (const suffix of ['log', 'junit.xml'] as const) {
|
|
2226
|
+
const source = join(job.artifactsDir, `${prefix}.${suffix}`);
|
|
2227
|
+
if (!existsSync(source)) continue;
|
|
2228
|
+
const targetName = `pod-${shard}${branch}.retry-${retryArtifactIndex}.${suffix}`;
|
|
2229
|
+
copyFileSync(source, join(logDir, targetName)); chmodSync(join(logDir, targetName), 0o600);
|
|
2230
|
+
persisted[suffix === 'log' ? 'log' : 'junit'] = targetName;
|
|
2231
|
+
}
|
|
2232
|
+
retryArtifactIndex++;
|
|
2233
|
+
return persisted;
|
|
2234
|
+
});
|
|
2235
|
+
});
|
|
2236
|
+
if (retryEvidence.length) writeFileSync(join(logDir, `pod-${shard}${branch}.retries.json`), JSON.stringify(retryEvidence, null, 2) + '\n', { mode: 0o600 });
|
|
2237
|
+
else rmSync(join(logDir, `pod-${shard}${branch}.retries.json`), { force: true });
|
|
2238
|
+
}
|
|
1938
2239
|
batchArtifactsComplete = !batchIsolation || parts.every((_, index) =>
|
|
1939
2240
|
['rc', 'log', 'junit.xml'].every((suffix) => existsSync(join(job.artifactsDir, `part-${index}.${suffix}`))));
|
|
1940
2241
|
const stagesPath = join(job.artifactsDir, 'stages.tsv');
|
|
@@ -1949,6 +2250,8 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1949
2250
|
}
|
|
1950
2251
|
}
|
|
1951
2252
|
stamp('artifactsRead');
|
|
2253
|
+
shardLoadFlaky = read.loadFlaky ?? [];
|
|
2254
|
+
shardLoadFlakyAttemptedFiles = read.loadFlakyAttemptedFiles ?? [];
|
|
1952
2255
|
({ output, junit, rc, timedOut, finished, unfinishedLog, exitCodeOnly } = read);
|
|
1953
2256
|
// A stall counts only when the Job also left no verdict (a failed stop can let a late shard finish after all).
|
|
1954
2257
|
const stalledAt = watch?.stalled();
|
|
@@ -2001,6 +2304,8 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
2001
2304
|
if (unfinishedLog !== undefined) writeFileSync(unfinishedDestination, unfinishedLog, { mode: 0o600 });
|
|
2002
2305
|
else rmSync(unfinishedDestination, { force: true });
|
|
2003
2306
|
writeFileSync(join(logDir, `pod-${shard}${branch}.json`), JSON.stringify({ completed: true, jobName: jobNameForCapture, durationMs, rc: rc ?? null,
|
|
2307
|
+
...(shardLoadFlakyAttemptedFiles.length ? { samePodRetryFiles: shardLoadFlakyAttemptedFiles } : {}),
|
|
2308
|
+
...(shardLoadFlaky.length ? { loadFlaky: shardLoadFlaky } : {}),
|
|
2004
2309
|
...(stall ? { reason: 'stall', finishedFiles: 0, stallSeconds: stall.seconds, sinceCheckoutMs: stall.sinceCheckoutMs } : timedOut ? { reason: 'timeout', finishedFiles: finished.length } : {}), files: paths, plannedSeconds: estimated(paths), deadlineSeconds: shardDeadline, shardCount: shards.length, sweepId, commit, runKey, attemptId, ...(parentAttempt ? { parentAttempt } : {}), jobExitCode: jobExitCode ?? null, jobFailed, ...(placedOn ? { host: placedOn } : {}),
|
|
2005
2310
|
...(Object.keys(phaseTimes).length ? { phaseTimes } : {}), ...(podStages ? { podStages } : {}),
|
|
2006
2311
|
...(podLifecycle && Object.keys(podLifecycle).length ? { podLifecycle } : {}),
|
|
@@ -2029,6 +2334,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
2029
2334
|
}
|
|
2030
2335
|
}
|
|
2031
2336
|
const installSeconds = parseInstallSeconds(output ?? '');
|
|
2337
|
+
if (imageDigest) detailsById.set(`pod-${shard}${branch}`, { ...(detailsById.get(`pod-${shard}${branch}`) ?? {}), imageDigest });
|
|
2032
2338
|
// GATE-TIMEOUT-HEAL ③: a cut shard is «reason: timeout» (with the files that still finished), not a bare rc=null.
|
|
2033
2339
|
if (!reused && !descent) debug.log('release-loop.gate', 'pod-shard', { shard, files: paths, durationMs, rc: rc ?? null, attempt: depth + 1, installSeconds, gateDeps: parseGateDeps(output ?? ''), deadlineSeconds: shardDeadline, parts: parts.length, ...(placedOn ? { host: placedOn } : {}),
|
|
2034
2340
|
...(testSeconds !== undefined ? { testSeconds } : {}),
|
|
@@ -2037,7 +2343,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
2037
2343
|
// only for a Job that came back (a launch that failed before the Job returned has no Pod to describe).
|
|
2038
2344
|
if (!reused && !descent && podJobName !== undefined) emitPodLifecycle(gatePodLifecycleEvent({ runId: process.env.ELANOUS_RUN_ID ?? null, podJob: podJobName, host: placedOn ?? null,
|
|
2039
2345
|
shard: `pod-${shard}${branch}`, attemptId, phaseTimes, podStages, podLifecycle, outcome: gatePodOutcome({ jobFailed, stalled: !!stall, timedOut, rc }) }));
|
|
2040
|
-
if (!reused && !descent) markShard(shard, { ...(installSeconds !== null ? { installSec: installSeconds } : {}), ...(rc !== undefined ? { rc } : {}), ...(timedOut ? { state: 'timeout' as const } : {}) });
|
|
2346
|
+
if (!reused && !descent) markShard(shard, { ...(installSeconds !== null ? { installSec: installSeconds } : {}), ...(rc !== undefined ? { rc } : {}), ...(timedOut ? { state: 'timeout' as const } : {}) }, [], branch);
|
|
2041
2347
|
let clean = output?.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '');
|
|
2042
2348
|
const ran = clean && lastMatch(clean, /Ran (\d+) tests? across (\d+) files?/);
|
|
2043
2349
|
const passes = clean === undefined ? NaN : summaryCount(clean, 'pass');
|
|
@@ -2103,6 +2409,22 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
2103
2409
|
// 137 (killed · OOM) keeps the isolation path: re-running an OOM file only re-measures the same kill.
|
|
2104
2410
|
const diedBeforeTests = reason === 'job-failed' && output === undefined && !jobFailed && jobExitCode !== undefined && jobExitCode !== 0
|
|
2105
2411
|
&& jobExitCode !== 137 && (paths.length === 1 || jobExitCode === 5);
|
|
2412
|
+
// A watched stall in a multi-file Job has already spent its first deadline. Measure each file on its own
|
|
2413
|
+
// under the sweep-wide isolation permit pool instead of paying for another whole-bundle Job and split tree.
|
|
2414
|
+
// The same descent is replayed after a cut resume when the parent and first child records are attributed.
|
|
2415
|
+
if (reason === 'no-output' && paths.length > 1 && !singleShotTail && (stall || descent === 'file-fanout')) {
|
|
2416
|
+
const count = Math.min(paths.length, isolationRemaining[shard]!);
|
|
2417
|
+
isolationRemaining[shard]! -= count;
|
|
2418
|
+
debug.log('release-loop.gate', 'pod-stall-file-fanout', { shard, branch, files: paths.length, launched: count,
|
|
2419
|
+
resumed: descent === 'file-fanout', limit: isolationConcurrency });
|
|
2420
|
+
const children = await Promise.all(paths.slice(0, count).map((file, index) => isolatedFile(() =>
|
|
2421
|
+
runShard([file], shard, depth + 1, `${branch}-file-${index}`, false, undefined, attemptId))));
|
|
2422
|
+
return { runs: children.flatMap((child) => child.runs), stalled: [
|
|
2423
|
+
...children.flatMap((child) => child.stalled),
|
|
2424
|
+
...(paths.length > count ? [{ shard, files: paths.slice(count), reason: 'no-output' as const,
|
|
2425
|
+
...(placedOn ? { lastHost: placedOn } : {}) }] : []),
|
|
2426
|
+
] };
|
|
2427
|
+
}
|
|
2106
2428
|
if ((reason === 'no-output' || diedBeforeTests) && !retriedNoOutput) {
|
|
2107
2429
|
const retry = await runShard(paths, shard, depth, `${branch}-retry`, true, undefined, attemptId, hostRetries, batchIsolation, skipMembers);
|
|
2108
2430
|
debug.log('release-loop.gate', 'pod-shard-retry', { shard, files: paths, reason, outcome: retry.stalled[0]?.reason ?? 'ok' });
|
|
@@ -2231,11 +2553,33 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
2231
2553
|
if (reused) debug.log('release-loop.gate', 'pod-shard-resumed', { shard, branch, files: paths.length, rc, durationMs: reused.durationMs ?? null });
|
|
2232
2554
|
return { runs: [{ output: clean!, rc: rc!, pass: passes, fail: fails,
|
|
2233
2555
|
errors: Number.isNaN(errors) ? 0 : errors,
|
|
2234
|
-
ran: Number(ran![1]), files: Number(ran![2]), passedIds: junit ? junitPassedIds(junit) : []
|
|
2556
|
+
ran: Number(ran![1]), files: Number(ran![2]), passedIds: junit ? junitPassedIds(junit) : [],
|
|
2557
|
+
...(shardLoadFlaky.length ? { loadFlaky: shardLoadFlaky } : {}),
|
|
2558
|
+
...(shardLoadFlakyAttemptedFiles.length ? { loadFlakyAttemptedFiles: shardLoadFlakyAttemptedFiles } : {}) }], stalled: [] };
|
|
2559
|
+
};
|
|
2560
|
+
const runShard = async (...args: Parameters<typeof runShardImpl>): ReturnType<typeof runShardImpl> => {
|
|
2561
|
+
const [paths, shard, depth = 0, branch = '', , , parentAttempt, hostRetries = 0] = args;
|
|
2562
|
+
filesById.set(`pod-${shard}${branch}`, [...paths]);
|
|
2563
|
+
if (branch || depth > 0 || parentAttempt !== undefined || hostRetries > 0) {
|
|
2564
|
+
resetShardAttempt(shard, branch, Math.round(estimated(paths) / 6) / 10);
|
|
2565
|
+
}
|
|
2566
|
+
try {
|
|
2567
|
+
const done = await runShardImpl(...args);
|
|
2568
|
+
if (branch) markShard(shard, { state: !done.stalled.length ? 'done' : done.stalled.some((stall) => stall.reason === 'timeout') ? 'timeout' : 'failed', endedAt: new Date().toISOString() }, ['waitReason'], branch);
|
|
2569
|
+
if (branch) recordShardResult(`pod-${shard}${branch}`, done);
|
|
2570
|
+
return done;
|
|
2571
|
+
} catch (error) {
|
|
2572
|
+
if (branch) markShard(shard, { state: 'failed', endedAt: new Date().toISOString() }, ['waitReason'], branch);
|
|
2573
|
+
throw error;
|
|
2574
|
+
}
|
|
2235
2575
|
};
|
|
2236
2576
|
// 모든 조각이 끝난 뒤 판정한다 — 한 조각의 예외로 먼저 돌아가면 다른 Pod 가 도는 채로 정리가 시작된다(#22002 리뷰 R3).
|
|
2237
2577
|
// GATE-RISK-LANE — the risk shard's own wall time (planned vs measured in its log line); it never changes a verdict.
|
|
2238
2578
|
const shardDurations: number[] = [];
|
|
2579
|
+
// Keep root shards outside the timed Pod admission until a gate slot is available.
|
|
2580
|
+
// Otherwise a wide sweep starts every admission clock together and later waves expire before placement.
|
|
2581
|
+
const withRootSlot = createGateRootLimiter(rootSlots);
|
|
2582
|
+
debug.log('release-loop.gate', 'pod-root-concurrency', { limit: rootSlots, shards: shards.length });
|
|
2239
2583
|
let settled: Array<PromiseSettledResult<Awaited<ReturnType<typeof runShard>>>>;
|
|
2240
2584
|
if (shardsFile) {
|
|
2241
2585
|
reconcileTimer = setInterval(reconcileTick, 30_000);
|
|
@@ -2245,10 +2589,59 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
2245
2589
|
settled = await Promise.allSettled(shards.map(async (item, shard) => {
|
|
2246
2590
|
const shardStartedAt = clock();
|
|
2247
2591
|
try {
|
|
2248
|
-
const
|
|
2592
|
+
const shardId = `pod-${shard}`;
|
|
2593
|
+
const expected = hostPlan[shard]?.kind === 'reuse' ? receiptChecks[shard]?.result : null;
|
|
2594
|
+
if (expected) {
|
|
2595
|
+
const done = { runs: expected.runs, stalled: [] };
|
|
2596
|
+
shardDurations[shard] = 0;
|
|
2597
|
+
detailsById.set(shardId, { ...(detailsById.get(shardId) ?? {}), imageDigest: expected.receipt.envKey.slice(envFingerprint!.length + 1) });
|
|
2598
|
+
markShard(shard, { state: 'done', rc: done.runs.some((run) => run.fail || run.errors) ? 1 : 0,
|
|
2599
|
+
startedAt: expected.receipt.measuredAt, endedAt: new Date().toISOString(), host: expected.receipt.host }, ['waitReason']);
|
|
2600
|
+
recordShardResult(shardId, done);
|
|
2601
|
+
debug.log('release-loop.gate', 'shard-receipt-reused', { shard: shardId, host: expected.receipt.host, files: item.files.length });
|
|
2602
|
+
return done;
|
|
2603
|
+
}
|
|
2604
|
+
debug.log('release-loop.gate', 'shard-receipt-not-reused', { shard: shardId,
|
|
2605
|
+
reason: pod.rerouteShard === shardId ? 'reroute-requested'
|
|
2606
|
+
: receiptChecks[shard]?.result && pod.rerouteShard === 'auto' ? 'auto-reroute'
|
|
2607
|
+
: receiptChecks[shard]?.result ? 'dispatch-remeasure-required'
|
|
2608
|
+
: receiptChecks[shard]?.reason ?? 'receipt-unavailable',
|
|
2609
|
+
plan: hostPlan[shard]?.kind ?? 'none' });
|
|
2610
|
+
const receiptAttempt = receiptRoot ? nextGateShardAttempt(receiptRoot, shardId) : 1;
|
|
2611
|
+
if (receiptRoot && base40 && envFingerprint) {
|
|
2612
|
+
try {
|
|
2613
|
+
markGateShardUnmeasured(receiptRoot, { id: shardId, files: [...item.files], head40: commit, base40,
|
|
2614
|
+
host: 'pending', envKey: `${envFingerprint}:${expectedImageDigest ?? 'unmeasured'}`, status: 'unmeasured',
|
|
2615
|
+
failureIds: [], errorIds: [], evidencePath: 'pending', evidenceSha256: '0'.repeat(64),
|
|
2616
|
+
measuredAt: new Date().toISOString(), attempt: receiptAttempt });
|
|
2617
|
+
} catch (error) {
|
|
2618
|
+
debug.log('release-loop.gate', 'shard-receipt-invalidate-failed', { shard: shardId, error: String(error) }, { level: 'warn' });
|
|
2619
|
+
throw error;
|
|
2620
|
+
}
|
|
2621
|
+
}
|
|
2622
|
+
// A missing or deliberately replaced receipt must execute a new Job. The older log-only resume
|
|
2623
|
+
// has no base/environment/image binding and cannot authorize this cross-host measurement.
|
|
2624
|
+
// The sixth runShardImpl argument is `fresh`: a missing receipt must launch a new Job.
|
|
2625
|
+
const freshForReceipt = !!receiptRoot || pod.freshShards === true;
|
|
2626
|
+
const done = await withRootSlot(() => runShard(item.files, shard, 0, '', false, freshForReceipt));
|
|
2249
2627
|
shardDurations[shard] = clock() - shardStartedAt;
|
|
2250
2628
|
const state = !done.stalled.length ? 'done' as const : done.stalled.some((stall) => stall.reason === 'timeout') ? 'timeout' as const : 'failed' as const;
|
|
2251
|
-
markShard(shard, { state, endedAt: new Date().toISOString() }, ['waitReason']);
|
|
2629
|
+
markShard(shard, { state, rc: done.stalled.length || done.runs.some((run) => run.fail || run.errors) ? 1 : 0, endedAt: new Date().toISOString() }, ['waitReason']);
|
|
2630
|
+
recordShardResult(`pod-${shard}`, done);
|
|
2631
|
+
if (receiptRoot && base40 && envFingerprint && !done.stalled.length) {
|
|
2632
|
+
const imageDigest = detailsById.get(shardId)?.imageDigest;
|
|
2633
|
+
const attribution = detailsById.get(shardId)?.attribution;
|
|
2634
|
+
const row = board[shard];
|
|
2635
|
+
if (imageDigest && row?.host && attribution === 'complete') {
|
|
2636
|
+
try {
|
|
2637
|
+
saveGateShardResult(receiptRoot, { receipt: { id: shardId, files: [...item.files], head40: commit, base40,
|
|
2638
|
+
host: row.host, envKey: `${envFingerprint}:${imageDigest}`, status: 'complete',
|
|
2639
|
+
failureIds: detailsById.get(shardId)?.failureIds ?? [], errorIds: detailsById.get(shardId)?.errorIds ?? [],
|
|
2640
|
+
evidencePath: 'pending', evidenceSha256: '0'.repeat(64), measuredAt: new Date().toISOString(),
|
|
2641
|
+
attempt: receiptAttempt }, runs: done.runs });
|
|
2642
|
+
} catch (error) { debug.log('release-loop.gate', 'shard-receipt-write-failed', { shard: shardId, error: String(error) }, { level: 'warn' }); }
|
|
2643
|
+
}
|
|
2644
|
+
}
|
|
2252
2645
|
return done;
|
|
2253
2646
|
} catch (error) {
|
|
2254
2647
|
markShard(shard, { state: 'failed', endedAt: new Date().toISOString() }, ['waitReason']);
|
|
@@ -2274,6 +2667,8 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
2274
2667
|
// GATE-STALLED-TAIL-REMEASURE: after the sweep, give only a one-file no-output leaf one fresh attempt on a different
|
|
2275
2668
|
// pool member. Keep the original unmeasured evidence if no alternate is available or the remeasure is incomplete.
|
|
2276
2669
|
const remainingStalls: typeof stalledShards = [];
|
|
2670
|
+
const tailTouched = new Set<number>();
|
|
2671
|
+
const tailRc = new Map<number, number>();
|
|
2277
2672
|
for (const stalled of stalledShards) {
|
|
2278
2673
|
const alternatives = stalled.lastHost
|
|
2279
2674
|
? poolScheduler.members.filter((member) => member.context !== stalled.lastHost && !downMembers.has(member.context))
|
|
@@ -2284,6 +2679,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
2284
2679
|
}
|
|
2285
2680
|
debug.log('release-loop.gate', 'pod-stalled-tail-remeasure-start', { shard: stalled.shard, file: stalled.files[0], fromHost: stalled.lastHost,
|
|
2286
2681
|
availableHosts: alternatives.map((member) => member.context) });
|
|
2682
|
+
tailTouched.add(stalled.shard);
|
|
2287
2683
|
try {
|
|
2288
2684
|
tailPlacedHost = undefined;
|
|
2289
2685
|
const tail = await runShard(stalled.files, stalled.shard, 0, '-tail', true, true, undefined, 0, false, new Set([stalled.lastHost]), true);
|
|
@@ -2296,6 +2692,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
2296
2692
|
fromHost: stalled.lastHost, toHost: unresolved.lastHost ?? null, reason: unresolved.reason }, { level: 'warn' });
|
|
2297
2693
|
} else {
|
|
2298
2694
|
measured.push(...tail.runs);
|
|
2695
|
+
tailRc.set(stalled.shard, tail.runs.some((run) => run.fail || run.errors) ? 1 : 0);
|
|
2299
2696
|
debug.log('release-loop.gate', 'pod-stalled-tail-remeasure-complete', { shard: stalled.shard, file: stalled.files[0],
|
|
2300
2697
|
fromHost: stalled.lastHost, toHost: tailPlacedHost ?? null });
|
|
2301
2698
|
}
|
|
@@ -2306,6 +2703,10 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
2306
2703
|
}
|
|
2307
2704
|
}
|
|
2308
2705
|
stalledShards = remainingStalls;
|
|
2706
|
+
for (const shard of tailTouched) {
|
|
2707
|
+
const unresolved = stalledShards.some((item) => item.shard === shard);
|
|
2708
|
+
markShard(shard, { state: unresolved ? 'failed' : 'done', rc: unresolved ? 1 : tailRc.get(shard) ?? 0, endedAt: new Date().toISOString() }, ['waitReason']);
|
|
2709
|
+
}
|
|
2309
2710
|
const total = measured.reduce((sum, run) => ({ pass: sum.pass + run.pass, fail: sum.fail + run.fail,
|
|
2310
2711
|
errors: sum.errors + run.errors, ran: sum.ran + run.ran, files: sum.files + run.files }),
|
|
2311
2712
|
{ pass: 0, fail: 0, errors: 0, ran: 0, files: 0 });
|
|
@@ -2324,6 +2725,10 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
2324
2725
|
return { rc: total.fail + total.errors ? 1 : 0, output: outputs.join('\n') + `\n${total.pass} pass\n${total.fail} fail\n${total.errors} errors\nRan ${total.ran} tests across ${total.files} files.\n`, validatedShards: true,
|
|
2325
2726
|
podSweep: { id: sweepId, shards: shards.length, files: assignable },
|
|
2326
2727
|
passedIds: [...new Set(measured.flatMap((run) => run.passedIds))].sort(),
|
|
2728
|
+
...(measured.some((run) => run.loadFlaky?.length)
|
|
2729
|
+
? { loadFlaky: [...new Set(measured.flatMap((run) => run.loadFlaky ?? []))].sort() } : {}),
|
|
2730
|
+
...(measured.some((run) => run.loadFlakyAttemptedFiles?.length)
|
|
2731
|
+
? { loadFlakyAttemptedFiles: [...new Set(measured.flatMap((run) => run.loadFlakyAttemptedFiles ?? []))].sort() } : {}),
|
|
2327
2732
|
stalledEnv: stalledShards.flatMap((item) => item.files.map((file) => ({ file, reason: 'no-output' as const }))),
|
|
2328
2733
|
...(abortedShards ? { abortedShards } : {}),
|
|
2329
2734
|
...(riskLane ? { riskLane } : {}) };
|
|
@@ -2819,6 +3224,11 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
2819
3224
|
existsSync(join(root, 'release', opts.version, 'rehearsal', `${step}.json`)));
|
|
2820
3225
|
const currentImage = !explicitPartial && opts.graphRunId && opts.pod && rehearsalStampPresent
|
|
2821
3226
|
? imageProbe(opts.pod.pool) : null;
|
|
3227
|
+
const receiptImage = currentImage ?? (opts.graphRunId && opts.pod && process.env.ELANOUS_GRAPH_CONTEXT
|
|
3228
|
+
? imageProbe(opts.pod.pool) : null);
|
|
3229
|
+
const receiptBase = opts.baselineCommit ?? (opts.baselineVersion && roots.some((location) =>
|
|
3230
|
+
existsSync(join(location, 'release', opts.baselineVersion!, 'release.json')))
|
|
3231
|
+
? baselineReleaseCommit(opts.baselineVersion) : undefined);
|
|
2822
3232
|
const radius = !explicitPartial && opts.graphRunId && opts.pod && rehearsalStampPresent
|
|
2823
3233
|
? planHardRehearsalRadius({ root, repo, version: opts.version, forCommit: opts.commit,
|
|
2824
3234
|
graphPins: gateGraphPins(root, opts.graphRunId), envFingerprint: gateEnvFingerprint(process.env), imageDigest: currentImage })
|
|
@@ -2879,8 +3289,12 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
2879
3289
|
// the cut sweep puts those files first, the baseline sweep keeps its plan.
|
|
2880
3290
|
riskWatch: { instanceRoots: roots, ledgerRoots: roots },
|
|
2881
3291
|
// GATE-LIVE-OBS: the release ledger's shards.json (UX #24782) — never the run tree's universe; an explicit
|
|
2882
|
-
// --ledger-root
|
|
2883
|
-
shardsFile: { path:
|
|
3292
|
+
// --ledger-root or an explicit test instance keeps it in that ledger.
|
|
3293
|
+
shardsFile: { path: gateShardsPath(opts.version, ledger), version: opts.version,
|
|
3294
|
+
ledgerPath: gateShardLedgerPath(opts.version, ledger), envFingerprint: gateEnvFingerprint(process.env),
|
|
3295
|
+
receiptRoot: join(ledger, 'release', opts.version, 'gate-logs', 'cut', 'receipts'),
|
|
3296
|
+
base40: receiptBase,
|
|
3297
|
+
expectedImageDigest: receiptImage } } : undefined,
|
|
2884
3298
|
selected?.files);
|
|
2885
3299
|
let cutRun = await runCut(partial);
|
|
2886
3300
|
const radiusSelectionMatched = partial?.files && cutRun?.podSweep?.files
|
|
@@ -2945,7 +3359,45 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
2945
3359
|
debug.log('release-loop.gate', 'pod-shard-stalled-env', { files: stalled.map(({ file, reason }) => ({ file, reason })), local });
|
|
2946
3360
|
return measured;
|
|
2947
3361
|
};
|
|
2948
|
-
|
|
3362
|
+
let measuredCut: SweepFailures;
|
|
3363
|
+
try {
|
|
3364
|
+
measuredCut = cutRun ? await localStalled(cutRun, cutTree, opts.commit, 'cut') : { failures: [], errors: [] };
|
|
3365
|
+
} catch (error) {
|
|
3366
|
+
if (!(error instanceof GateFailureAttributionMismatch) || error.label !== 'cut sweep') throw error;
|
|
3367
|
+
const targetFiles = [...new Set(error.failures.map(fileOf))].sort();
|
|
3368
|
+
const attribution = { summary: error.summary, observed: error.observed, identified: error.identified,
|
|
3369
|
+
collapsed: error.collapsed, targetFiles,
|
|
3370
|
+
baselineResult: 'failed' as 'completed' | 'failed' | 'timeout' };
|
|
3371
|
+
result.attributionRecheck = attribution;
|
|
3372
|
+
debug.log('release-loop.gate', 'failure-attribution-baseline-recheck', { ...attribution, phase: 'started' });
|
|
3373
|
+
try {
|
|
3374
|
+
if (!targetFiles.length || targetFiles.length > GATE_ATTRIBUTION_RECHECK_FILE_LIMIT)
|
|
3375
|
+
throw new Error(`baseline recheck target count ${targetFiles.length} outside safe bound 1..${GATE_ATTRIBUTION_RECHECK_FILE_LIMIT}`);
|
|
3376
|
+
if (!work) throw new Error('gate workspace missing before baseline attribution recheck');
|
|
3377
|
+
const recheckTree = join(work, 'baseline-attribution-recheck');
|
|
3378
|
+
trees.push(recheckTree);
|
|
3379
|
+
const baseShaForRecheck = opts.baselineCommit ?? baselineReleaseCommit(opts.baselineVersion);
|
|
3380
|
+
await runner.add(recheckTree, baseShaForRecheck);
|
|
3381
|
+
for (const dir of [recheckTree, join(recheckTree, 'apps/pwa')])
|
|
3382
|
+
check(await runner.command('bun', ['install'], dir), `bun install ${dir}`);
|
|
3383
|
+
const rechecked = await runner.sweep(recheckTree,
|
|
3384
|
+
join(root, 'release', opts.version, 'gate-logs', 'baseline-attribution-recheck'),
|
|
3385
|
+
opts.pod ? { ...opts.pod, durationSource } : undefined,
|
|
3386
|
+
targetFiles);
|
|
3387
|
+
if (rechecked.timedOut || rechecked.abortedShards || rechecked.stalledEnv?.length) {
|
|
3388
|
+
attribution.baselineResult = 'timeout';
|
|
3389
|
+
throw new Error('baseline sweep timed out or stalled');
|
|
3390
|
+
}
|
|
3391
|
+
failuresOf(rechecked, 'baseline attribution recheck');
|
|
3392
|
+
attribution.baselineResult = 'completed';
|
|
3393
|
+
} catch (recheckError) {
|
|
3394
|
+
debug.log('release-loop.gate', 'failure-attribution-baseline-recheck', { ...attribution, phase: 'failed', error: String(recheckError) });
|
|
3395
|
+
throw new Error(`baseline recheck ${attribution.baselineResult}; cut attribution unresolved (summary=${attribution.summary}, identified=${attribution.identified})`);
|
|
3396
|
+
}
|
|
3397
|
+
debug.log('release-loop.gate', 'failure-attribution-baseline-recheck', { ...attribution, phase: 'completed',
|
|
3398
|
+
disposition: 'cut names remain unresolved; baseline success cannot identify unnamed cut failures; fail closed' });
|
|
3399
|
+
throw new Error(`baseline recheck completed; cut attribution still unresolved (summary=${attribution.summary}, identified=${attribution.identified})`);
|
|
3400
|
+
}
|
|
2949
3401
|
// A between-tests error proves only that the shared process failed. Re-run each affected file by itself using the
|
|
2950
3402
|
// same isolated path as ordinary new-failure rechecks, even when the error was present in the baseline ledger.
|
|
2951
3403
|
// Replace that file's shard observations with its isolated observations before the existing regression comparison.
|
|
@@ -2986,6 +3438,21 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
2986
3438
|
}
|
|
2987
3439
|
}
|
|
2988
3440
|
const cut = partial ? mergePartialFailures(partial, measuredCut) : measuredCut;
|
|
3441
|
+
const samePodFlaky = [...new Set(cutRun?.loadFlaky ?? [])].sort();
|
|
3442
|
+
if (samePodFlaky.length) {
|
|
3443
|
+
const flaky = new Set(samePodFlaky);
|
|
3444
|
+
cut.failures = cut.failures.filter((id) => !flaky.has(id));
|
|
3445
|
+
cut.errors = cut.errors.filter((id) => !flaky.has(id));
|
|
3446
|
+
result.loadFlaky = [...new Set([...(result.loadFlaky ?? []), ...samePodFlaky])].sort();
|
|
3447
|
+
debug.log('release-loop.gate', 'same-pod-load-flaky', { ids: samePodFlaky, files: [...new Set(samePodFlaky.map(fileOf))].sort() });
|
|
3448
|
+
try {
|
|
3449
|
+
const path = loadFlakyCensusPath(ledger);
|
|
3450
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
3451
|
+
const at = new Date().toISOString();
|
|
3452
|
+
appendFileSync(path, samePodFlaky.map((id) => JSON.stringify({ at, version: opts.version, commit: opts.commit, id,
|
|
3453
|
+
host: opts.pod ? `pod:${opts.pod.pool}` : opts.remote ?? 'local', retry: 'same-pod-standalone' }) + '\n').join(''), { mode: 0o600 });
|
|
3454
|
+
} catch (error) { debug.log('release-loop.gate', 'load-flaky-census-write-failed', { ids: samePodFlaky, error: String(error) }); }
|
|
3455
|
+
}
|
|
2989
3456
|
if (partial) result.partial = { priorCommit: partial.priorCommit, files: partial.files.length };
|
|
2990
3457
|
cutFailures = cut;
|
|
2991
3458
|
const baseSha = opts.baselineCommit ?? baselineReleaseCommit(opts.baselineVersion);
|
|
@@ -3129,7 +3596,8 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
3129
3596
|
// per-test limit, on the same runner (node-b when the gate is remote). Passing there = flaky under load (warning);
|
|
3130
3597
|
// an assertion failure, a failure again, or a run that cannot be read stays introduced.
|
|
3131
3598
|
const timedOut = cutCheck === undefined ? new Set<string>() : new Set(parseTimedOutFailures(stripPodRepoRoot(cutCheck.output)));
|
|
3132
|
-
const
|
|
3599
|
+
const samePodRetried = new Set(cutRun?.loadFlakyAttemptedFiles ?? []);
|
|
3600
|
+
const shaped = candidates.filter((id) => timedOut.has(id) && isolatedCut.failures.includes(id) && !samePodRetried.has(fileOf(id)));
|
|
3133
3601
|
if (shaped.length) {
|
|
3134
3602
|
const relaxed = opts.pod && runner.recheckPod
|
|
3135
3603
|
? await runner.recheckPod(file, opts.commit, { ...opts.pod, warmCutSha: opts.commit, warmBaselineSha: baseSha,
|
|
@@ -3348,11 +3816,13 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
3348
3816
|
...(result.interferenceErrors?.length ? { interferenceErrors: [...result.interferenceErrors].sort() } : {}),
|
|
3349
3817
|
};
|
|
3350
3818
|
const observed = { ...(result.selectionMs !== undefined ? { selectionMs: result.selectionMs } : {}),
|
|
3351
|
-
...(result.warmShadow?.length ? { warmShadow: result.warmShadow } : {})
|
|
3819
|
+
...(result.warmShadow?.length ? { warmShadow: result.warmShadow } : {}),
|
|
3820
|
+
...(result.attributionRecheck ? { attributionRecheck: result.attributionRecheck } : {}) };
|
|
3352
3821
|
// Receipt is advisory for a later radius gate. Missing identity makes reuse invalid, never this gate's verdict.
|
|
3353
3822
|
const baselineReceipt = gateBaselineReceipt({ root, version: opts.version, commit: opts.commit,
|
|
3354
3823
|
graphPins: gateGraphPins(root, opts.graphRunId) ?? (opts.rehearsalBaseline ? gateSourcePins(repo, opts.commit) : null),
|
|
3355
3824
|
envFingerprint: gateEnvFingerprint(process.env),
|
|
3825
|
+
...(result.attributionRecheck ? { attributionRecheck: result.attributionRecheck } : {}),
|
|
3356
3826
|
fullSweep: result.outcome === 'ok' && cutSweepUsable && !result.partial && !!cutFailures
|
|
3357
3827
|
&& !result.rcNonzeroNoFailures.length && !result.stalledShards?.length && !result.stalledEnv.length
|
|
3358
3828
|
&& !result.unmeasuredRecheckFiles?.length && !result.unmeasuredErrors?.length });
|
|
@@ -3418,7 +3888,7 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
3418
3888
|
// GATE-LIVE-OBS-RC-WORDING ⓒ — the shards board says «판정 전» until this lands; then «새 실패 N» only (UX · gate-shards.ts).
|
|
3419
3889
|
// Same ledger as the board's own writes (sweep's shardsFile). A write failure is logged and never changes the verdict.
|
|
3420
3890
|
if (result.outcome !== 'error' && opts.pod) {
|
|
3421
|
-
const path =
|
|
3891
|
+
const path = gateShardsPath(opts.version, ledger);
|
|
3422
3892
|
try {
|
|
3423
3893
|
const written = recordGateShardsVerdict(path, { introduced: result.introduced.length, preexisting: result.preexisting });
|
|
3424
3894
|
debug.log('release-loop.gate', 'shards-verdict', { version: opts.version, path, written, introduced: result.introduced.length, preexisting: result.preexisting });
|
|
@@ -3433,7 +3903,7 @@ export function graphGateResult(result: GateResult, graph: boolean) {
|
|
|
3433
3903
|
summary: (result.partial ? `부분 재검 ${result.partial.files}파일(앞 게이트 ${result.partial.priorCommit.slice(0, 9)} 결과 이음) · ` : '')
|
|
3434
3904
|
+ (result.outcome === 'ok' ? `새 회귀 ${result.introduced.length} · 기존 ${result.preexisting} · 고침 ${result.fixed ?? '못 셈(기준선 미루기)'}` : `게이트 ${result.outcome}: ${result.error ?? result.introduced.length + ' new regressions'}`) + ` · 환경 알려진 실패 ${result.knownEnv}`
|
|
3435
3905
|
+ (result.rcNonzeroNoFailures.length ? ` · ⚠ 종료코드만 비영 ${result.rcNonzeroNoFailures.length}` : '')
|
|
3436
|
-
+ (result.loadFlaky?.length ? ` · ⚠ 부하 흔들림 ${result.loadFlaky.length}(
|
|
3906
|
+
+ (result.loadFlaky?.length ? ` · ⚠ 부하 흔들림 ${result.loadFlaky.length}(독립 재실행 통과 · 막지 않음)` : '')
|
|
3437
3907
|
+ (result.stalledEnv.length ? ` · 환경 멈춤 ${result.stalledEnv.length} (Pod 밖 통과 ${result.stalledEnv.filter((item) => item.local === 'passed').length})` : '')
|
|
3438
3908
|
+ (result.abortedShards ? ` · 중단 조각 ${result.abortedShards}(중간 매달림 · 시험만 끊고 재확인)` : '') };
|
|
3439
3909
|
}
|
|
@@ -3483,6 +3953,12 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
3483
3953
|
if (fromGraph.gateExecutor !== undefined && !isGateExecutor(fromGraph.gateExecutor)) throw new Error(`gateExecutor must be pod or host: ${String(fromGraph.gateExecutor)}`);
|
|
3484
3954
|
// GATE-HOST-EXECUTOR: a host gate never reads the pod settings (a bad pod value must not block the fallback executor).
|
|
3485
3955
|
const hostExecutor = fromGraph.gateExecutor === 'host';
|
|
3956
|
+
const graphCapabilities = fromGraph.gateShardCapabilities;
|
|
3957
|
+
if (!hostExecutor && graphCapabilities !== undefined && (!graphCapabilities || typeof graphCapabilities !== 'object'
|
|
3958
|
+
|| Array.isArray(graphCapabilities) || Object.entries(graphCapabilities).some(([id, value]) =>
|
|
3959
|
+
!/^pod-\d+$/.test(id) || !value || typeof value !== 'object' || Array.isArray(value)
|
|
3960
|
+
|| !['darwin', 'linux'].includes((value as { os?: string }).os ?? ''))))
|
|
3961
|
+
throw new Error('gateShardCapabilities must map pod-N to {os: darwin|linux}');
|
|
3486
3962
|
const opts: GateOptions = {
|
|
3487
3963
|
rehearsalTui: fromGraph.rehearsalTui === true,
|
|
3488
3964
|
recheckFileLimit: fromGraph.gateRecheckFileLimit !== undefined ? Number(fromGraph.gateRecheckFileLimit) : undefined,
|
|
@@ -3504,11 +3980,16 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
3504
3980
|
...(fromGraph.gatePodStallSeconds !== undefined ? { stallSeconds: Number(fromGraph.gatePodStallSeconds) } : {}),
|
|
3505
3981
|
...(fromGraph.gatePodMidRunStallSeconds !== undefined ? { midRunStallSeconds: Number(fromGraph.gatePodMidRunStallSeconds) } : {}),
|
|
3506
3982
|
...(fromGraph.gatePodFreshShards === true ? { freshShards: true } : {}),
|
|
3983
|
+
...(typeof fromGraph.gateRerouteShard === 'string' ? { rerouteShard: fromGraph.gateRerouteShard } : {}),
|
|
3984
|
+
...(typeof fromGraph.gateShardHost === 'string' ? { shardHost: fromGraph.gateShardHost } : {}),
|
|
3985
|
+
...(graphCapabilities ? { shardCapabilities: graphCapabilities as Record<string, { os: 'darwin' | 'linux' }> } : {}),
|
|
3507
3986
|
...((cpu) => cpu ? { cpu } : {})(parsePodCpu(fromGraph.gatePodCpu, 'release.loop.gatePodCpu')),
|
|
3508
3987
|
} : undefined,
|
|
3509
3988
|
};
|
|
3510
3989
|
if (isGateExecutor(fromGraph.gateExecutor)) opts.executor = fromGraph.gateExecutor;
|
|
3511
3990
|
let freshShards = false;
|
|
3991
|
+
let rerouteShard: string | undefined;
|
|
3992
|
+
let shardHost: string | undefined;
|
|
3512
3993
|
for (let i = 0; i < args.length; i++) {
|
|
3513
3994
|
const arg = args[i];
|
|
3514
3995
|
if (arg === '--help') return 'help';
|
|
@@ -3516,6 +3997,18 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
3516
3997
|
if (arg === '--rehearsal-baseline') { opts.rehearsalBaseline = true; continue; }
|
|
3517
3998
|
// Applied after the loop and only to a Pod gate — on its own it must not turn a local gate into a Pod one (review r5).
|
|
3518
3999
|
if (arg === '--fresh-shards') { freshShards = true; continue; }
|
|
4000
|
+
if (arg === '--reroute-shard') {
|
|
4001
|
+
const value = args[++i];
|
|
4002
|
+
if (!value || !/^(?:auto|pod-\d+)$/.test(value)) throw new Error('--reroute-shard requires auto or pod-N');
|
|
4003
|
+
rerouteShard = value;
|
|
4004
|
+
continue;
|
|
4005
|
+
}
|
|
4006
|
+
if (arg === '--shard-host') {
|
|
4007
|
+
const value = args[++i];
|
|
4008
|
+
if (!value || value.startsWith('--')) throw new Error('--shard-host requires a pool context');
|
|
4009
|
+
shardHost = value;
|
|
4010
|
+
continue;
|
|
4011
|
+
}
|
|
3519
4012
|
if (arg === '--rehearsal-tui') { opts.rehearsalTui = true; continue; }
|
|
3520
4013
|
if (arg === '--recheck-file-limit') {
|
|
3521
4014
|
const value = args[++i];
|
|
@@ -3544,6 +4037,11 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
3544
4037
|
else if (key === 'remoteMirror') opts.remoteMirror = value;
|
|
3545
4038
|
}
|
|
3546
4039
|
if (freshShards && opts.pod) opts.pod = { ...opts.pod, freshShards: true };
|
|
4040
|
+
if ((rerouteShard || shardHost) && !opts.pod?.pool) throw new Error('shard reroute requires a Pod pool');
|
|
4041
|
+
if (shardHost && !rerouteShard) throw new Error('--shard-host requires --reroute-shard');
|
|
4042
|
+
if (opts.pod && rerouteShard) opts.pod = { ...opts.pod, rerouteShard, ...(shardHost ? { shardHost } : {}) };
|
|
4043
|
+
if (opts.pod?.rerouteShard && !/^(?:auto|pod-\d+)$/.test(opts.pod.rerouteShard)) throw new Error('gateRerouteShard must be auto or pod-N');
|
|
4044
|
+
if (opts.pod?.shardHost && !opts.pod.rerouteShard) throw new Error('gateShardHost requires gateRerouteShard');
|
|
3547
4045
|
// GATE-HOST-EXECUTOR: the host executor is the existing non-pod sweep on gateRemote — a configured pool must not pull it
|
|
3548
4046
|
// back into Pods, and without a host it would silently sweep on the runner's own machine.
|
|
3549
4047
|
if (opts.executor === 'host') {
|
|
@@ -3558,10 +4056,13 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
3558
4056
|
/** The node's run: judge, emit the graph result line, and return the process exit code (ok 0 · regression 1 · else 2).
|
|
3559
4057
|
* `runner` is a seam for tests; the process entry passes none. */
|
|
3560
4058
|
export async function runGateNode(argv: string[], env: NodeJS.ProcessEnv, runner?: GateRunner,
|
|
3561
|
-
judge: (opts: GateOptions, runner?: GateRunner) => Promise<GateResult> = judgeGate
|
|
4059
|
+
judge: (opts: GateOptions, runner?: GateRunner, imageProbe?: typeof preflightGateImageDigest) => Promise<GateResult> = judgeGate,
|
|
4060
|
+
imageProbe: typeof preflightGateImageDigest = preflightGateImageDigest): Promise<number> {
|
|
3562
4061
|
const start = Date.now();
|
|
3563
4062
|
let result: GateResult;
|
|
3564
4063
|
let version = '';
|
|
4064
|
+
let graphRunId: string | null = null;
|
|
4065
|
+
let autohealStoreRoot = '';
|
|
3565
4066
|
let placement: { executor: string | null; host: string | null } | undefined;
|
|
3566
4067
|
try {
|
|
3567
4068
|
const parsed = parseOptions(argv, env);
|
|
@@ -3576,12 +4077,14 @@ export async function runGateNode(argv: string[], env: NodeJS.ProcessEnv, runner
|
|
|
3576
4077
|
debug.log('release-loop.gate', 'executor', { version: opts.version, ...placement, requested: opts.executor ?? null });
|
|
3577
4078
|
}
|
|
3578
4079
|
if (opts === 'help') {
|
|
3579
|
-
console.log('Usage: bun scripts/release-loop/gate-node.ts --commit <sha> --version <v> [--baseline-version <prev>] [--baseline-commit <sha>] [--remote <ssh-host>] [--remote-mirror <path>] [--pod-pool <pool>] [--pod-shards <n>] [--pod-shard-timeout-seconds <n>] [--recheck-file-limit <n>] [--fresh-shards] [--executor <pod|host>] [--rehearsal-baseline] [--json]\nWithout flags, input.commit, input.version and input.previousVersion come from the JSON file at ELANOUS_GRAPH_CONTEXT.\n--executor host (input.gateExecutor) ignores every pod option and sweeps on --remote (input.gateRemote, required).\n--rehearsal-baseline pins source graph bytes only for a direct Pod gate on an isolated rc version.');
|
|
4080
|
+
console.log('Usage: bun scripts/release-loop/gate-node.ts --commit <sha> --version <v> [--baseline-version <prev>] [--baseline-commit <sha>] [--remote <ssh-host>] [--remote-mirror <path>] [--pod-pool <pool>] [--pod-shards <n>] [--pod-shard-timeout-seconds <n>] [--recheck-file-limit <n>] [--fresh-shards] [--reroute-shard <pod-N|auto>] [--shard-host <pool-context>] [--executor <pod|host>] [--rehearsal-baseline] [--json]\nWithout flags, input.commit, input.version and input.previousVersion come from the JSON file at ELANOUS_GRAPH_CONTEXT.\n--executor host (input.gateExecutor) ignores every pod option and sweeps on --remote (input.gateRemote, required).\n--rehearsal-baseline pins source graph bytes only for a direct Pod gate on an isolated rc version.');
|
|
3580
4081
|
return 0;
|
|
3581
4082
|
}
|
|
3582
4083
|
version = opts.version;
|
|
4084
|
+
autohealStoreRoot = opts.ledgerRoot ?? releaseLedgerRoot();
|
|
3583
4085
|
// The graph's existing gate-node execution path delegates verdict classification and persistence here.
|
|
3584
4086
|
const context = usesGraphContext(argv, env) ? readGraphContext(env) : undefined;
|
|
4087
|
+
graphRunId = context?.runId ?? null;
|
|
3585
4088
|
// Let the gate's existing SSH host validation report an invalid host before any reservation is published.
|
|
3586
4089
|
const hosts = opts.remote && !/^(?:[\w.-]+@)?[\w.-]+$/.test(opts.remote) ? [] : reservedGateHosts(opts);
|
|
3587
4090
|
const reservationRoot = context ? gateHostReservationRoot() : undefined;
|
|
@@ -3589,12 +4092,28 @@ export async function runGateNode(argv: string[], env: NodeJS.ProcessEnv, runner
|
|
|
3589
4092
|
throw new Error('release gate host reservation requires graph runId and pinned state root');
|
|
3590
4093
|
const releaseHosts = context && hosts.length
|
|
3591
4094
|
? reserveReleaseGateHosts(reservationRoot!, context.runId!, hosts) : () => undefined;
|
|
3592
|
-
try { result = await judge(context?.runId ? { ...opts, graphRunId: context.runId } : opts, runner); }
|
|
4095
|
+
try { result = await judge(context?.runId ? { ...opts, graphRunId: context.runId } : opts, runner, imageProbe); }
|
|
3593
4096
|
finally { releaseHosts(); }
|
|
3594
4097
|
} catch (error) {
|
|
3595
4098
|
result = { outcome: 'error', commit: '', introduced: [], preexisting: 0, fixed: 0, knownEnv: 0, knownEnvCleared: [], stalledEnv: [], rcNonzeroNoFailures: [], durationMs: Date.now() - start, error: String(error) };
|
|
3596
4099
|
placement ??= requestedGateExecutor(argv, env);
|
|
3597
4100
|
}
|
|
4101
|
+
if (versionPattern.test(version) && result.stalledShards?.length) {
|
|
4102
|
+
result.autoheals = result.stalledShards.map((shard, index) => {
|
|
4103
|
+
const shardKey = `pod-${shard.shard}`;
|
|
4104
|
+
const signature = createHash('sha256').update(JSON.stringify({ shard: shard.shard,
|
|
4105
|
+
files: [...shard.files].sort(), reason: shard.reason })).digest('hex');
|
|
4106
|
+
const cause = gateAutohealCause({ outcome: result.outcome, introduced: result.introduced, durationMs: result.durationMs,
|
|
4107
|
+
error: result.error, stalledShards: [shard], interferenceErrors: result.interferenceErrors, loadFlaky: result.loadFlaky }) ?? 'environment';
|
|
4108
|
+
const decision = decideAndRecordGateAutoheal({ cause, signature, version, runId: graphRunId ?? 'direct', shardKey,
|
|
4109
|
+
shardKeys: [shardKey], shardElapsedMs: result.durationMs,
|
|
4110
|
+
overallElapsedMs: index === 0 ? result.durationMs : 0,
|
|
4111
|
+
storeRoot: autohealStoreRoot || releaseLedgerRoot() });
|
|
4112
|
+
debug.log('release-loop.gate-autoheal', 'decision', { runId: graphRunId, version, ...decision });
|
|
4113
|
+
return decision;
|
|
4114
|
+
});
|
|
4115
|
+
result.autoheal = result.autoheals.find((decision) => decision.action === 'hitl') ?? result.autoheals[0];
|
|
4116
|
+
}
|
|
3598
4117
|
if (result.error) console.error(result.error);
|
|
3599
4118
|
debug.log('release-loop.gate', 'result', { version, outcome: result.outcome, ...(placement ?? {}) });
|
|
3600
4119
|
emitNodeResult({ ...graphGateResult(result, !!env.ELANOUS_GRAPH_CONTEXT), ...(placement ?? {}) });
|