elanous 0.2.28 → 0.2.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/pwa/out/404/index.html +1 -1
- package/apps/pwa/out/404.html +1 -1
- package/apps/pwa/out/_next/static/{y1ltt7oCZxVpC6zVPhZ2O → LSXfY0u9QtfDbejY0Ukf_}/_buildManifest.js +1 -1
- package/apps/pwa/out/_next/static/chunks/{ab4d57c2.e90bc2cb31949045.js → 0bb0ff36.b8d418980951e5ba.js} +7 -7
- package/apps/pwa/out/_next/static/chunks/1034-8b639cee57b8e4a8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/1085-6c2d35526f11f840.js +1 -0
- package/apps/pwa/out/_next/static/chunks/1220.1ff7d5cdc34b1ae5.js +27 -0
- package/apps/pwa/out/_next/static/chunks/1247.5884ec81b39fe2e7.js +287 -0
- package/apps/pwa/out/_next/static/chunks/1302.87681521a7a4661e.js +62 -0
- package/apps/pwa/out/_next/static/chunks/1324-59dae3002413623b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{5492-e05e26a033ba80d2.js → 1335-6731f392c35216eb.js} +3 -3
- package/apps/pwa/out/_next/static/chunks/1343-f40b5870ac1eda5a.js +1 -0
- package/apps/pwa/out/_next/static/chunks/146-b6cd56be11dda017.js +1 -0
- package/apps/pwa/out/_next/static/chunks/1654.32de4b324981a188.js +23 -0
- package/apps/pwa/out/_next/static/chunks/1834-50625904996a8fa8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/1849.1ce38efa6c527d9a.js +149 -0
- package/apps/pwa/out/_next/static/chunks/1913.7cc6fb6055b1ed3f.js +41 -0
- package/apps/pwa/out/_next/static/chunks/2001.6c7e0fd02190886c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2030.41fae98c7e907304.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2099-08b2f2dfba742c9b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2112-e4efc21a6505d5bc.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2134.ecee03a550ee7250.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{8578.dff22d988cc2b106.js → 2213.2ccbada92c33ea0f.js} +3 -3
- package/apps/pwa/out/_next/static/chunks/2236.66061832181cb745.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2422.b0bd272e7020d7e6.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2534.e85ae21ad0dac829.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{5081.a240b63fad28663a.js → 2871.0bca6137343b3f76.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/2898.fd781bc7cf01a7b7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2940-619bfd8ef337c611.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{2846.d6a433b8663819a5.js → 2985.39ccdecd36848b12.js} +4 -4
- package/apps/pwa/out/_next/static/chunks/3040.92ff9ffc9f971cb9.js +115 -0
- package/apps/pwa/out/_next/static/chunks/3144.6861096c7b4edef7.js +64 -0
- package/apps/pwa/out/_next/static/chunks/{f4dfbfc0.a69fb12f854882aa.js → 3267c0e6.d34854e7b3984a6e.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/3278.2e5d15b53a4091c8.js +2 -0
- package/apps/pwa/out/_next/static/chunks/3541-ce3a5424ed5ad7f1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/3583.a05e88cc8d594628.js +1 -0
- package/apps/pwa/out/_next/static/chunks/3771.e928edf2867d28c0.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{3143.257997a5f1ed79df.js → 3902.e1e71aaac850a045.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/4063-03284b5de59965f3.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{16e57f67.3ea19e893461e611.js → 409f1e57.d0649fc47691ebfe.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{1146.80d15ea7c6e6ee95.js → 4111.2bfedbb233ad206b.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{1516.83dd2c1b86b4e507.js → 4174.92ed04922709ac9e.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/4300-c7ec4eeabbb5f3f7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4483.646d5065199e1b55.js +36 -0
- package/apps/pwa/out/_next/static/chunks/4519.28199ff26b2fd80d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4581.ef2443578054544f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4608-8d894e9cbe4f08bf.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4639.ae48a01d1bb50a93.js +286 -0
- package/apps/pwa/out/_next/static/chunks/4739.08742ac668aada67.js +77 -0
- package/apps/pwa/out/_next/static/chunks/4745.67d6f99c245af3f7.js +226 -0
- package/apps/pwa/out/_next/static/chunks/4882-16cef71d6a1a32be.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{9938.b23e4d4dfa0114cd.js → 4999.c064a4000c2a9e94.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/5173-233188df2f42c626.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5260-69f35419bca263e7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5272.b92d1b72a15c25fe.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{1157.f08bf2b70915e176.js → 5304.04d24756f242a094.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/{7752.5cbdd0b433c54af9.js → 5307.60f2209e809bd627.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/5415-997980f828667736.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{9917-9e26abc5f7772735.js → 5458-16cd7a2b51b74c8c.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{6196.34d6b229079e9e28.js → 5525.86debf231e181813.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{4593.10f5b9bd702d4a6c.js → 5543.a438b8ab5a13ea47.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/5572-42c9084f27ca761e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5661-80470aac963d9f36.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5723.421338408b5c30c4.js +74 -0
- package/apps/pwa/out/_next/static/chunks/5809.64c3664267c50bce.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6000.64bc21ac8627f2f5.js +94 -0
- package/apps/pwa/out/_next/static/chunks/6013.39459b9c68eabf5f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/607-06466dff012b03ef.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{7483.ba01c6c209b81bba.js → 6092.a7f99e68b188ca66.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/60fb96c9.360b01d8d4927b7b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{9974.e77b4ff4884557b3.js → 6157.0c9af021de4fb31b.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/6192.a190b989b81e9743.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{3309.364573ceedbdfd5f.js → 6323.8672b4817a306928.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{94.4cfed21c0b781a06.js → 6433.fc573af22bbc9ab7.js} +4 -4
- package/apps/pwa/out/_next/static/chunks/6444.7d1fdcb9b09ae71c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{949.1e7d45ccb71580aa.js → 6517.4147878ac06d584d.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/6532.e34302010b2f0b68.js +32 -0
- package/apps/pwa/out/_next/static/chunks/6544cb48.a87fe20ddf137be2.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6546.74bc4ed82f19016d.js +104 -0
- package/apps/pwa/out/_next/static/chunks/6684-e7523ce7c2fad075.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6726.ebdc82feb161431e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6739.00fb6f892e5e392b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6745.934c4921bf15513a.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{2528.3868070ef1be69f9.js → 6767.ea607cf0c20a25af.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/6843.67bfd020da0926c5.js +93 -0
- package/apps/pwa/out/_next/static/chunks/6948.1062d4594652b5bf.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7138-eaf841fc1dc03463.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7174-cda29ee13d7790e7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7304-3c07583f4e405f4f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7371.80c05c653b08e907.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7391.ebae168025c91b6a.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{1743.563d3168e95825ef.js → 7502.ccd5b8065545edd0.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/7557.e325503a090a4310.js +8 -0
- package/apps/pwa/out/_next/static/chunks/7569-9ce9432c7f10989f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{5408.377c8c543635b81f.js → 7862.30a8d6482bb238eb.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/7886-55854585fadeded1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7947-d0759430f8042240.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{65ebff95.081daeff622d238e.js → 7a1887c4.f36b88fba31da8fd.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{2697.76ca2ac3efef6f3b.js → 8018.a32663f04a6fc5f9.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/8097-92f87bd108322f66.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8215-1661812b6e9ad71f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8260-19c921a51ca8ec3e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8440.3a5cf5c85aa76db8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8519-4599be2249161e68.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8547.b028b90c5c01ace6.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8587-cbe02a953a7f4990.js +1 -0
- package/apps/pwa/out/_next/static/chunks/877-2b977259c4b15655.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9002.e5658224e3dc1b6f.js +24 -0
- package/apps/pwa/out/_next/static/chunks/9019-0a4b40c50c7314fb.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9138.b6e80031afb1d973.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9188-d36e4fc869ddc166.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{4204.a409ab6250c0f6a8.js → 9221.7d22eaf353a2d4e7.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/9230.44f5039e796e4ab4.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{5785-ea8d2c633b6688e3.js → 9279-d28c5b7754d010ae.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/939.336acce73e98482c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9398-6aed28f0a96d0d6f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9407.e49124dedf0f5e72.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9610.c07cce4bfcc14ed7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9667-ce78c2ce91050ff9.js +1 -0
- package/apps/pwa/out/_next/static/chunks/997ce8a5-2ad501927003b41b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{9311.24c8f5ea793253dd.js → 9989.3694300e5f000b14.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/_not-found/page-339fabb3499da0e1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/approvals/page-bf3634a117322df7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/autopilot/page-3535a4a69f1fafaa.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/board/page-b9f51447f757fbf2.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/botlab/{page-fa46d4588e8442c2.js → page-2ebb8fd59b9461f9.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/bots/page-a2fc4e49a4fa4ba8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ceo/page-5a503541353e60a8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/chat/page-6025101384e361a5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/consult/page-f4b630cc8b5273cd.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/control/page-a645f00a06483e9a.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/decisions/page-61ac22cbab1ee4ec.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/design-check/page-e9bab7413b028b28.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/editor/page-c992eafe028b642e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/exec/page-e547e6ce55f1fb33.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/field/page-d27e1b9850fd54bf.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/healthz/route-339fabb3499da0e1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/inside/page-805ac2e137b7a121.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/intake/page-0fec29c7ba01894b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/layout-339a07410cd6f36b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/live/{page-ced0bcfc6542b3fc.js → page-349f2752d8cd37ad.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/live-run/page-d5c6dcf31c25cecc.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/loops/page-2fd93ed8c9f16580.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/market/page-9b564dbff0fefc35.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/missions/[id]/page-62f4475d3c38fb9b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/morning/page-814ba51385225776.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/not-found-397d479626e5a459.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/observatory/page-af1bba4e3251b728.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/checklist/page-3ad3be5e7d3fdce9.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/release/page-f219ccdbc0bf27df.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/seats/page-4da93ad35d4a8d28.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/outputs/page-456666766e5ce6fc.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/page-7404e377e8adb8c5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/reflection/page-14456240ed2a36ad.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/scheduler/page-3a658d5fa6d16a25.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/sessions/page-0157de6e85f8bb6b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/settings/devices/{page-cb249e81ed339318.js → page-0a07363b130182a1.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/settings/page-841d73d4ac44ed49.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/setup/done/page-6f43a3d1146e6377.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/setup/layout-339fabb3499da0e1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/setup/page-521ee23ac8c0abfa.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/share/page-a9992ee285efaa83.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/showroom/page-3d48e85fa89ec04f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/tasks/page-d2ced71132fac9b2.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/term/page-f08358bdf251adb9.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/today/page-233c71cd5f68f0d7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/trace/page-70151d78de2f375e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/vault/page-dcac33a3e2cb72c0.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workflows/chat-ui/page-149dc1caba411bb9.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workflows/page-0448145ccd5c45c1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workspace/page-9cb0c444903da65d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/worktrees/page-17e504c396e0e389.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{6def8a8e.c81f9ca41028eccf.js → b313b242.cb1265eea908db61.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{e87d89b9.2e71c0ae475d28a1.js → e5012ffe.f727e23c231f4342.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/ea695e3c-1f1d70d740cf3031.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{e117f801.e92bf10242653497.js → ebc7d829.264a0bcb6c750b50.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/framework-906af9bd029e166c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/main-a24847f9f684d469.js +1 -0
- package/apps/pwa/out/_next/static/chunks/main-app-7bdcd8aa7c8397d3.js +1 -0
- package/apps/pwa/out/_next/static/chunks/pages/_app-6ba46a5066fc226d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/pages/_error-a8dc6d07591f2a25.js +1 -0
- package/apps/pwa/out/_next/static/chunks/webpack-f13df28c15c55eee.js +1 -0
- package/apps/pwa/out/_next/static/css/{72becfd42807b50d.css → 3f5fc6f96e2cf7b2.css} +1 -1
- package/apps/pwa/out/approvals/index.html +1 -1
- package/apps/pwa/out/approvals/index.txt +23 -23
- package/apps/pwa/out/autopilot/index.html +1 -1
- package/apps/pwa/out/autopilot/index.txt +23 -23
- package/apps/pwa/out/board/index.html +1 -1
- package/apps/pwa/out/board/index.txt +23 -23
- package/apps/pwa/out/botlab/index.html +1 -1
- package/apps/pwa/out/botlab/index.txt +22 -22
- package/apps/pwa/out/bots/index.html +1 -1
- package/apps/pwa/out/bots/index.txt +22 -22
- package/apps/pwa/out/ceo/index.html +1 -1
- package/apps/pwa/out/ceo/index.txt +23 -23
- package/apps/pwa/out/chat/index.html +1 -1
- package/apps/pwa/out/chat/index.txt +22 -22
- package/apps/pwa/out/consult/index.html +1 -1
- package/apps/pwa/out/consult/index.txt +23 -23
- package/apps/pwa/out/control/index.html +1 -1
- package/apps/pwa/out/control/index.txt +23 -23
- package/apps/pwa/out/decisions/index.html +1 -1
- package/apps/pwa/out/decisions/index.txt +22 -22
- package/apps/pwa/out/design-check/index.html +1 -1
- package/apps/pwa/out/design-check/index.txt +23 -23
- package/apps/pwa/out/editor/index.html +1 -1
- package/apps/pwa/out/editor/index.txt +23 -23
- package/apps/pwa/out/exec/index.html +1 -1
- package/apps/pwa/out/exec/index.txt +23 -23
- package/apps/pwa/out/field/index.html +1 -1
- package/apps/pwa/out/field/index.txt +23 -23
- package/apps/pwa/out/healthz +1 -1
- package/apps/pwa/out/index.html +1 -1
- package/apps/pwa/out/index.txt +22 -22
- package/apps/pwa/out/inside/index.html +1 -1
- package/apps/pwa/out/inside/index.txt +22 -22
- package/apps/pwa/out/intake/index.html +1 -1
- package/apps/pwa/out/intake/index.txt +23 -23
- package/apps/pwa/out/live/index.html +1 -1
- package/apps/pwa/out/live/index.txt +23 -23
- package/apps/pwa/out/live-run/index.html +1 -1
- package/apps/pwa/out/live-run/index.txt +23 -23
- package/apps/pwa/out/loops/index.html +1 -1
- package/apps/pwa/out/loops/index.txt +22 -22
- package/apps/pwa/out/market/index.html +1 -1
- package/apps/pwa/out/market/index.txt +22 -22
- package/apps/pwa/out/missions/placeholder/index.html +1 -1
- package/apps/pwa/out/missions/placeholder/index.txt +22 -22
- package/apps/pwa/out/morning/index.html +1 -1
- package/apps/pwa/out/morning/index.txt +23 -23
- package/apps/pwa/out/observatory/index.html +1 -1
- package/apps/pwa/out/observatory/index.txt +23 -23
- package/apps/pwa/out/ops/checklist/index.html +1 -1
- package/apps/pwa/out/ops/checklist/index.txt +22 -22
- package/apps/pwa/out/ops/release/index.html +1 -1
- package/apps/pwa/out/ops/release/index.txt +22 -22
- package/apps/pwa/out/ops/seats/index.html +1 -1
- package/apps/pwa/out/ops/seats/index.txt +22 -22
- package/apps/pwa/out/outputs/index.html +1 -1
- package/apps/pwa/out/outputs/index.txt +22 -22
- package/apps/pwa/out/reflection/index.html +1 -1
- package/apps/pwa/out/reflection/index.txt +23 -23
- package/apps/pwa/out/scheduler/index.html +1 -1
- package/apps/pwa/out/scheduler/index.txt +23 -23
- package/apps/pwa/out/sessions/index.html +1 -1
- package/apps/pwa/out/sessions/index.txt +22 -22
- package/apps/pwa/out/settings/devices/index.html +1 -1
- package/apps/pwa/out/settings/devices/index.txt +23 -23
- package/apps/pwa/out/settings/index.html +1 -1
- package/apps/pwa/out/settings/index.txt +23 -23
- package/apps/pwa/out/setup/done/index.html +1 -1
- package/apps/pwa/out/setup/done/index.txt +23 -23
- package/apps/pwa/out/setup/index.html +1 -1
- package/apps/pwa/out/setup/index.txt +23 -23
- package/apps/pwa/out/share/index.html +1 -1
- package/apps/pwa/out/share/index.txt +23 -23
- package/apps/pwa/out/showroom/index.html +1 -1
- package/apps/pwa/out/showroom/index.txt +22 -22
- package/apps/pwa/out/tasks/index.html +1 -1
- package/apps/pwa/out/tasks/index.txt +23 -23
- package/apps/pwa/out/term/index.html +1 -1
- package/apps/pwa/out/term/index.txt +23 -23
- package/apps/pwa/out/today/index.html +1 -1
- package/apps/pwa/out/today/index.txt +23 -23
- package/apps/pwa/out/trace/index.html +1 -1
- package/apps/pwa/out/trace/index.txt +23 -23
- package/apps/pwa/out/vault/index.html +1 -1
- package/apps/pwa/out/vault/index.txt +23 -23
- package/apps/pwa/out/workflows/chat-ui/index.html +1 -1
- package/apps/pwa/out/workflows/chat-ui/index.txt +23 -23
- package/apps/pwa/out/workflows/index.html +1 -1
- package/apps/pwa/out/workflows/index.txt +23 -23
- package/apps/pwa/out/workspace/index.html +1 -1
- package/apps/pwa/out/workspace/index.txt +23 -23
- package/apps/pwa/out/worktrees/index.html +1 -1
- package/apps/pwa/out/worktrees/index.txt +23 -23
- package/bin/elanous.mjs +7 -3
- package/graphs/context/nightly-condense.yaml +1 -0
- package/graphs/intake/intake-daily.yaml +8 -1
- package/graphs/intake/recipes.yaml +5 -0
- package/graphs/lecture/lecture-note.yaml +2 -2
- package/graphs/ops/harness-gc.yaml +1 -0
- package/graphs/release/recipes.yaml +3 -0
- package/graphs/release/release-loop.yaml +15 -7
- package/graphs/units/recipes.yaml +3 -0
- package/graphs/units/release-gate-recheck.yaml +15 -0
- package/package.json +2 -1
- package/plugins/agent-team/agents/research.md +1 -0
- package/scripts/check-native-tool-wiring.ts +3 -2
- package/scripts/ci-android-unit-tests.ts +16 -0
- package/scripts/ci-typecheck-changed.ts +40 -11
- package/scripts/coord-channel-watch.sh +45 -19
- package/scripts/coord-role-pack.ts +222 -0
- package/scripts/cron-run.ts +96 -3
- package/scripts/detect-test-interference.ts +38 -5
- package/scripts/gate-pr-cache.ts +65 -0
- package/scripts/gh-surface/capture.ts +126 -0
- package/scripts/gh-surface/gh +40 -0
- package/scripts/import-graph-scc.ts +38 -0
- package/scripts/install.ps1 +19 -0
- package/scripts/intake-social.ts +181 -0
- package/scripts/lecture-note/node.ts +71 -4
- package/scripts/lib/android-toolchain.ts +32 -0
- package/scripts/lib/deterministic-env.ts +62 -0
- package/scripts/lib/gate-test-env.ts +19 -0
- package/scripts/lib/morning-report-options.ts +41 -0
- package/scripts/measure-goal-grounding-corpus.ts +2 -1
- package/scripts/pr-land-extra-remote-gates.ts +49 -0
- package/scripts/ratchets/test-interference-patterns.allowlist.txt +257 -0
- package/scripts/ratchets/test-interference-patterns.ts +367 -0
- package/scripts/release-loop/cut-branch.ts +4 -3
- package/scripts/release-loop/export-check-node.ts +0 -1
- package/scripts/release-loop/fixtures/warm-failure-assertion/sample.junit.xml +8 -0
- package/scripts/release-loop/fixtures/warm-job-029/sample.json +1 -0
- package/scripts/release-loop/fixtures/warm-job-029/sample.junit.xml +10 -0
- package/scripts/release-loop/gate-batch-safe.ts +23 -0
- package/scripts/release-loop/gate-closure.ts +12 -8
- package/scripts/release-loop/gate-diff.ts +1 -1
- package/scripts/release-loop/gate-history.ts +422 -0
- package/scripts/release-loop/gate-host-mirror.ts +66 -0
- package/scripts/release-loop/gate-node.ts +806 -94
- package/scripts/release-loop/gate-partial.ts +106 -9
- package/scripts/release-loop/gate-pr-cache.ts +66 -0
- package/scripts/release-loop/gate-prewarm-plan.ts +66 -0
- package/scripts/release-loop/gate-r2-verdict.ts +189 -0
- package/scripts/release-loop/gate-recheck-plan-node.ts +52 -0
- package/scripts/release-loop/gate-triage-node.ts +190 -0
- package/scripts/release-loop/gate-warm-artifacts.ts +253 -0
- package/scripts/release-loop/import-graph-scc-run.ts +54 -0
- package/scripts/release-loop/import-graph-scc.ts +215 -0
- package/scripts/release-loop/node-verdict.ts +2 -0
- package/scripts/release-loop/probe-pod-host-dirs-live.ts +113 -0
- package/scripts/release-loop/resume-release.ts +33 -4
- package/scripts/release-loop/resume.ts +13 -5
- package/scripts/release-loop/spawn-lane.ts +51 -0
- package/scripts/release-loop/tui-node-checks.ts +23 -0
- package/scripts/release-loop/tui-sim-node.ts +3 -2
- package/scripts/release-loop/unattended-release.ts +16 -0
- package/scripts/release-loop/version-node.ts +62 -4
- package/scripts/test-deterministic.ts +46 -73
- package/scripts/train-reach-grouping.ts +48 -0
- package/scripts/tsc-incremental-cache.ts +86 -0
- package/scripts/tsc-incremental-diagnostics.ts +76 -0
- package/scripts/tsc-main-safety-net.ts +2 -2
- package/scripts/usage-rollup.ts +11 -3
- package/src/acp/client.ts +4 -1
- package/src/acp/core-turn-bridge.ts +145 -24
- package/src/acp/elanous-extensions.ts +2 -0
- package/src/acp/server.ts +2 -0
- package/src/ad-pipeline/run.ts +4 -2
- package/src/agent/ref-grounding.ts +18 -0
- package/src/agent/runner.ts +169 -3
- package/src/agent/subagent-callable.ts +3 -2
- package/src/agent-mission/review-loop.ts +6 -3
- package/src/agent-substrate/acp-implementer.ts +14 -2
- package/src/agent-substrate/pr-reviewer.ts +41 -21
- package/src/agent-substrate/self-review-cli.ts +30 -4
- package/src/autopilot/pr-manager.ts +62 -7
- package/src/briefing/brief-items.ts +24 -22
- package/src/budget/pty-usage-reemit.ts +7 -0
- package/src/budget/run-usage-rollup.ts +9 -0
- package/src/budget/usage-daily-store.ts +98 -0
- package/src/capture/tui-self-report.ts +37 -4
- package/src/chat/chat-cli.ts +10 -2
- package/src/chat/index.ts +107 -20
- package/src/chat/input-control-key.ts +5 -3
- package/src/chat/pickers/modal-runtime.ts +1 -0
- package/src/chat/pickers/modals.ts +3 -1
- package/src/chat/tool-interrupt-behavior.ts +38 -0
- package/src/chat/tool-render/dispatch.ts +5 -1
- package/src/chat/turn-typeahead.ts +191 -29
- package/src/cli/__fixtures__/agent-help-before/agent.txt +4 -2
- package/src/cli/agent-cli.ts +41 -13
- package/src/cli/config-audit.ts +123 -0
- package/src/cli/dashboard-name-flag.ts +48 -0
- package/src/cli/decisions-cli.ts +80 -6
- package/src/cli/doctor-cli.ts +80 -0
- package/src/cli/log-event-names.ts +19 -0
- package/src/cli/mcp-login.ts +21 -5
- package/src/cli/msg-cli.ts +133 -2
- package/src/cli/pr-cli.ts +546 -112
- package/src/cli/pr-land-heavy-remote.ts +233 -0
- package/src/cli/pr-land-overlap-evidence.ts +167 -14
- package/src/cli/pr-land-pod-shadow-job.ts +83 -0
- package/src/cli/pr-land-pod-shadow.ts +16 -3
- package/src/cli/pr-land-virtual-rebase.ts +108 -0
- package/src/cli/pty-takeover-cli.ts +125 -16
- package/src/cli/release-cli.ts +122 -14
- package/src/cli/remote-resolve.ts +2 -1
- package/src/cli/seat-cli.ts +164 -19
- package/src/cli/self-update.ts +5 -2
- package/src/cli/tasks-cli.ts +105 -21
- package/src/cli/usage-cli.ts +146 -12
- package/src/cli/usage-daily-cli.ts +84 -0
- package/src/context-bus/context-first.ts +13 -0
- package/src/context-bus/context-now.ts +95 -7
- package/src/context-bus/ops-events.ts +75 -0
- package/src/control-plane/lease-holder.ts +12 -8
- package/src/control-plane/ledger.ts +163 -4
- package/src/control-plane/server.ts +91 -11
- package/src/core-turn/auth-rejected-notice.ts +41 -0
- package/src/core-turn/run-core-turn.ts +14 -2
- package/src/core-turn/run-goal-loop.ts +239 -4
- package/src/core-turn/seat-turn-cap.ts +79 -0
- package/src/core-turn/short-question.ts +78 -13
- package/src/core-turn/types.ts +10 -0
- package/src/dashboard/acp-active-model.ts +40 -0
- package/src/dashboard/acp-boot.ts +3 -0
- package/src/dashboard/auto-compact-runtime.ts +2 -0
- package/src/dashboard/boot-banner.ts +80 -0
- package/src/dashboard/dashboard-status-lines.ts +11 -0
- package/src/dashboard/detail-viewer-runtime.ts +15 -2
- package/src/dashboard/event-loop-stall-monitor.ts +4 -0
- package/src/dashboard/goal-loop-thinking.ts +16 -0
- package/src/dashboard/hitl-boot.ts +9 -4
- package/src/dashboard/index.ts +448 -998
- package/src/dashboard/input/chat-main-entry-runtime.ts +2 -0
- package/src/dashboard/input/chat-main-log-actions.ts +4 -0
- package/src/dashboard/input/chat-main-log-command.ts +12 -1
- package/src/dashboard/input/chat-main-plain-turn-runtime.ts +176 -4
- package/src/dashboard/input/chat-main-session.ts +14 -3
- package/src/dashboard/input/chat-main-slash-command.ts +7 -0
- package/src/dashboard/input/chat-main-turn.ts +14 -4
- package/src/dashboard/input/compact-prepass-recovery.ts +130 -0
- package/src/dashboard/input/dashboard-chord-actions.ts +10 -1
- package/src/dashboard/input/dashboard-chord-runtime.ts +2 -0
- package/src/dashboard/input/external-input-host.ts +71 -0
- package/src/dashboard/input/internal-turn.ts +261 -0
- package/src/dashboard/input/log-fold-binding.ts +18 -0
- package/src/dashboard/input/mouse-wiring.ts +17 -4
- package/src/dashboard/input/seat-autowake.ts +318 -0
- package/src/dashboard/input/turn-steer-bridge.ts +164 -0
- package/src/dashboard/log-click-runtime.ts +5 -3
- package/src/dashboard/loop-scheduler.ts +72 -0
- package/src/dashboard/running-build-status.ts +74 -0
- package/src/dashboard/seat-boot-preamble.ts +131 -0
- package/src/dashboard/seat-local-instructions.ts +305 -0
- package/src/dashboard/skill-runtime.ts +2 -0
- package/src/dashboard/slash-runtime/dashboard-handlers.ts +220 -1203
- package/src/dashboard/slash-runtime/help-from-registry.ts +2 -1
- package/src/dashboard/slash-runtime/index.ts +13 -0
- package/src/dashboard/slash-runtime/model-choices.ts +35 -0
- package/src/dashboard/term-slash-runtime.ts +15 -0
- package/src/dashboard/turn-busy-teardown.ts +50 -0
- package/src/dashboard/turn-preamble.ts +30 -2
- package/src/dashboard/turn-session-layers.ts +176 -0
- package/src/dashboard/turn-stream-formatter.ts +14 -6
- package/src/dashboard/turn-stream-presentation-applier.ts +8 -0
- package/src/decisions/decision-ledger.ts +45 -4
- package/src/discord-self-message.ts +4 -4
- package/src/domains/release-tool.ts +26 -10
- package/src/domains/surface-events.ts +4 -1
- package/src/domains/watch-default-scheduler.ts +8 -9
- package/src/field-note/transcribe.ts +17 -0
- package/src/fold-stack.ts +24 -1
- package/src/git-fs/dirty.ts +32 -4
- package/src/git-fs/index.ts +43 -1
- package/src/github/remote-call.ts +81 -0
- package/src/goals/registry.ts +46 -6
- package/src/goals/types.ts +7 -1
- package/src/graph-runner/runner.ts +2 -0
- package/src/grok/credential.ts +26 -0
- package/src/harness/harness-clean.ts +2 -1
- package/src/harness/harness-cli-command.ts +52 -5
- package/src/harness/harness-gc.ts +34 -6
- package/src/harness/harness-incidents.ts +57 -20
- package/src/harness/harness-pod-dispatch.ts +87 -10
- package/src/harness/harness-queue.ts +53 -4
- package/src/harness/harness-stop.ts +4 -3
- package/src/harness/harness-substrate-default.ts +78 -7
- package/src/harness/harness-worktree-auto.ts +2 -1
- package/src/harness/harness-worktrees.ts +3 -2
- package/src/hooks/dispatch.ts +2 -1
- package/src/index.ts +166 -26
- package/src/input/external-input-queue.ts +251 -0
- package/src/intake-plane/slash.ts +14 -3
- package/src/interactive-terminal-modal.ts +13 -0
- package/src/keybindings.ts +4 -1
- package/src/llm/route-decision.ts +25 -0
- package/src/llm/tool-loop-steered-rejection.ts +33 -0
- package/src/llm-stream-retry-display.ts +59 -0
- package/src/llm.ts +391 -74
- package/src/log-entry.ts +3 -3
- package/src/log-pane/search.ts +29 -4
- package/src/loops/agent-reply.ts +256 -0
- package/src/loops/budget.ts +37 -7
- package/src/loops/consensus.ts +348 -0
- package/src/loops/loop-cli.ts +3 -2
- package/src/loops/msg-address.ts +57 -0
- package/src/loops/msg-wake.ts +484 -0
- package/src/loops/observe.ts +19 -3
- package/src/loops/orchestrator/inbox.ts +60 -3
- package/src/loops/orchestrator/tick.ts +80 -6
- package/src/loops/orchestrator/traffic.ts +64 -1
- package/src/loops/registry.ts +72 -4
- package/src/market/signed-index.ts +14 -0
- package/src/mcp/mcp-oauth.ts +54 -5
- package/src/memory.ts +20 -5
- package/src/models/prompts.ts +28 -0
- package/src/msg/msg-sse-client.ts +223 -0
- package/src/msg/msg-store.ts +86 -2
- package/src/mss/logging/log-store.ts +225 -1
- package/src/native-tool-catalog.ts +125 -0
- package/src/nexus/DAEMON-THIN-inventory-2026-10-15.md +122 -0
- package/src/nexus/api/http-server.ts +59 -15
- package/src/nexus/api/loop-resources.ts +19 -3
- package/src/nexus/api/ops-api.ts +13 -0
- package/src/nexus/api/project-autopsy.ts +127 -0
- package/src/nexus/api/terminals.ts +3 -0
- package/src/nexus/api/workflow-approvals.ts +2 -0
- package/src/nexus/daemon-thin-inventory.md +37 -0
- package/src/nexus/index.ts +70 -0
- package/src/nexus/onboarding/first-boot-wizard.ts +56 -2
- package/src/nexus/state/state.ts +1 -0
- package/src/oauth/codex-account-store.ts +15 -3
- package/src/oauth/codex-billing-route.ts +31 -0
- package/src/oauth/store.ts +32 -7
- package/src/onboarding/first-run.ts +28 -3
- package/src/ops/overview-cache.ts +16 -4
- package/src/ops/overview-collect.ts +19 -2
- package/src/ops/resources.ts +82 -0
- package/src/panes/log-pane.ts +67 -8
- package/src/persona/presets.ts +44 -0
- package/src/plugins/maker/connector-maker.ts +41 -0
- package/src/plugins/maker/skill-maker.ts +42 -0
- package/src/plugins/maker/wizard-generate.ts +6 -4
- package/src/pr-tool/census.ts +288 -0
- package/src/preview/terminal.ts +14 -1
- package/src/prompt-cache/types.ts +2 -0
- package/src/prompt-enhance/enhance.ts +24 -8
- package/src/prompt-library/chat-conciseness.ts +13 -0
- package/src/prompt-library/universal-preamble.ts +283 -61
- package/src/pty-shell/pty-control-ipc.ts +4 -1
- package/src/pty-shell/pty-manifest.ts +77 -5
- package/src/pty-shell/registry.ts +19 -2
- package/src/pty-shell/tui-run-state.ts +253 -0
- package/src/release-loop/gate-host-reservation.ts +97 -0
- package/src/release-loop/landing-freeze.ts +5 -0
- package/src/release-loop/light-rc.ts +32 -4
- package/src/release-loop/placement.ts +92 -1
- package/src/release-loop/rehearsal-shard-plan.ts +149 -0
- package/src/release-loop/rehearsal.ts +73 -17
- package/src/render.ts +5 -1
- package/src/repl/index.ts +20 -2
- package/src/seat-loop/seat-knowhow.ts +282 -0
- package/src/seat-loop/seat-loop.ts +41 -2
- package/src/seat-loop/seat-report.ts +42 -2
- package/src/seat-loop/traffic-signal.ts +3 -0
- package/src/self-dev/ask-launch-flow.ts +24 -3
- package/src/self-dev/decompose-compose-graph.ts +28 -27
- package/src/self-dev/decompose-proposal.ts +53 -38
- package/src/self-dev/decompose.ts +17 -0
- package/src/self-dev/dev-pipeline.ts +31 -2
- package/src/self-dev/draft-residue.ts +24 -9
- package/src/self-dev/draft-sweep.ts +59 -4
- package/src/self-dev/external-call-zero.ts +77 -0
- package/src/self-dev/launch-preflight-text.ts +6 -0
- package/src/self-dev/launch-preflight.ts +25 -3
- package/src/self-dev/local-supervisor-judge.ts +185 -0
- package/src/self-dev/run-supervisor.ts +300 -24
- package/src/self-dev/run-types.ts +20 -0
- package/src/self-dev/shard-aggregate.ts +15 -2
- package/src/self-dev/shard-plan.ts +47 -0
- package/src/self-dev/shard-run.ts +135 -12
- package/src/self-dev/spawn-lane-files.ts +19 -0
- package/src/self-health/signatures.ts +132 -0
- package/src/self-implement/app-api-budget.ts +51 -11
- package/src/self-implement/config-gate.ts +2 -1
- package/src/self-implement/doctor-visit.ts +118 -0
- package/src/self-implement/expose-gate.ts +2 -1
- package/src/self-implement/gate-artifacts.ts +405 -0
- package/src/self-implement/gate-cli.ts +212 -18
- package/src/self-implement/gate-remote.ts +110 -25
- package/src/self-implement/gate-scenario.ts +85 -0
- package/src/self-implement/goal-author-decompose.ts +80 -16
- package/src/self-implement/goal-author.ts +270 -76
- package/src/self-implement/goal-authoring-grounding.ts +1 -1
- package/src/self-implement/goal-file-name.ts +65 -0
- package/src/self-implement/graph-authority.ts +38 -6
- package/src/self-implement/graph-overlay-condition.ts +92 -1
- package/src/self-implement/ground-goal.ts +2 -1
- package/src/self-implement/heal-triage.ts +79 -0
- package/src/self-implement/host-regate.ts +74 -2
- package/src/self-implement/orchestrator.ts +14 -123
- package/src/self-implement/pr-title.ts +123 -0
- package/src/self-implement/review-mustfix-class.ts +115 -0
- package/src/self-implement/run-checkpoint.ts +235 -0
- package/src/self-implement/run-ledger.ts +266 -17
- package/src/self-implement/run-stop.ts +4 -3
- package/src/self-implement/run-ttl.ts +28 -4
- package/src/self-implement/running-runs.ts +10 -6
- package/src/self-implement/seams.ts +2 -0
- package/src/self-implement/self-implement-runtime.ts +32 -0
- package/src/self-implement/shadow-stage.ts +7 -6
- package/src/self-implement/stop-autoheal.ts +2 -1
- package/src/self-implement/train-gate-remote.ts +37 -8
- package/src/self-implement/train-reach-grouping.ts +157 -0
- package/src/self-implement/train-scenario-layers.ts +133 -0
- package/src/self-implement/uncommitted-work.ts +2 -0
- package/src/session/pending-input.ts +81 -11
- package/src/session/session-start.ts +224 -0
- package/src/session-runtime/index.ts +7 -1
- package/src/session-runtime/parallel-dispatch.ts +22 -5
- package/src/skills/tools/dev-harness.ts +6 -4
- package/src/skills/tools/ops-ledger.ts +707 -0
- package/src/skills/tools/persistent-grounding.ts +47 -7
- package/src/skills/tools/shell.ts +117 -1
- package/src/status/bar.ts +32 -2
- package/src/status/status-provider-info.ts +58 -0
- package/src/steward/resync-tick.ts +114 -23
- package/src/steward/ta-repair-event-collect.ts +158 -0
- package/src/task-agent/card-evidence.ts +25 -1
- package/src/task-agent/green-sweep.ts +8 -3
- package/src/task-agent/judge.ts +76 -0
- package/src/task-agent/land-queue.ts +79 -0
- package/src/task-agent/live-moves.ts +30 -9
- package/src/task-agent/postland-card.ts +5 -0
- package/src/task-agent/prelaunch-already-done.ts +98 -0
- package/src/task-agent/real-launch-guard.ts +37 -0
- package/src/task-agent/rejudge-on-head.ts +93 -13
- package/src/task-agent/remote-rejudge-bind.ts +62 -13
- package/src/task-agent/repair-cycle.ts +58 -7
- package/src/task-agent/repair-worker.ts +7 -0
- package/src/task-agent/run-identity.ts +200 -0
- package/src/task-agent/shadow.ts +50 -3
- package/src/task-agent/task-hand.ts +158 -5
- package/src/task-agent/train-landing.ts +77 -0
- package/src/task-agent/work-order.ts +104 -0
- package/src/task-orchestrator/surfaces/kubectl-transient-retry.ts +51 -0
- package/src/task-orchestrator/surfaces/pod-command-job.ts +138 -22
- package/src/task-orchestrator/surfaces/pod-failure-reason.ts +88 -19
- package/src/task-orchestrator/surfaces/pod-host-acp-review.ts +2 -1
- package/src/task-orchestrator/surfaces/pod-image-source.ts +2 -1
- package/src/task-orchestrator/surfaces/pod-install-slots.ts +122 -15
- package/src/task-orchestrator/surfaces/pod-lease.ts +20 -6
- package/src/task-orchestrator/surfaces/pod-lifecycle-stages.ts +169 -0
- package/src/task-orchestrator/surfaces/pod-mirror-seed.ts +103 -0
- package/src/task-orchestrator/surfaces/pod-result-marker-wait.ts +33 -0
- package/src/task-orchestrator/surfaces/pod-reuse-cleanup.ts +63 -0
- package/src/task-orchestrator/surfaces/pod-source-receive.ts +19 -3
- package/src/task-orchestrator/surfaces/pod-warm-switch.ts +141 -0
- package/src/task-orchestrator/surfaces/pod-warm-ta-shadow.ts +85 -0
- package/src/task-orchestrator/surfaces/pod-warm-worker-protocol.ts +153 -0
- package/src/task-orchestrator/surfaces/pod-warm-worker-runtime.ts +208 -0
- package/src/task-orchestrator/surfaces/self-implement-pod.ts +237 -46
- package/src/telegram.ts +2 -4
- package/src/terminal/session-registry.ts +6 -0
- package/src/thinking-line.ts +12 -3
- package/src/tool-runtime/index.ts +4 -0
- package/src/tool-runtime/ops-ledger-runtimes.ts +84 -0
- package/src/tool-runtime/ref-consult-runtime.ts +17 -2
- package/src/tool-runtime/run-shell-runtime.ts +4 -1
- package/src/tool-runtime/self-cognition-runtimes.ts +5 -1
- package/src/tui/input/pickers.ts +732 -0
- package/src/tui/slash/mission.ts +492 -0
- package/src/tui/slash/term.ts +877 -0
- package/src/tui-client/dashboard-session.ts +9 -0
- package/src/tui.ts +53 -0
- package/src/user-config/gemini-server-tools.ts +9 -0
- package/src/user-config.ts +106 -21
- package/src/util/spawn-sync-output.ts +7 -0
- package/src/ux-sim/fake-bin.ts +154 -0
- package/src/ux-sim/fake-clock.ts +36 -0
- package/src/ux-sim/fake-llm.ts +29 -0
- package/src/ux-sim/headless-tui.ts +72 -0
- package/src/ux-sim/in-process-cli.ts +176 -0
- package/src/ux-sim/temp-git-remote.ts +82 -0
- package/src/version/packed-revision.json +1 -1
- package/src/video-pipeline/recipes/vlog.ts +25 -7
- package/apps/pwa/out/_next/static/chunks/1027.a0f04663d24c2779.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1038.676007ae505822e7.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1113.a37eadf7f740d955.js +0 -1
- package/apps/pwa/out/_next/static/chunks/112-fa7c526c20ce1405.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1144.5fa1fa1d48dbcd12.js +0 -36
- package/apps/pwa/out/_next/static/chunks/119b405e.8fdd0e44d1ff37aa.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1296.04560d98e73fbf51.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1314-f604ba013ec535b0.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1454-29d1d773d5996a49.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1464.f3209af0d8ae4ff2.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1729-d73dd25e564679d4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1843.5924c24295243261.js +0 -27
- package/apps/pwa/out/_next/static/chunks/1987.e28149724b92918b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2066-5ca18ffa5a4be68b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2240.d10a66695155fd93.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2391.e2cbc98d4abbb41a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2566-734cb83bbd405886.js +0 -1
- package/apps/pwa/out/_next/static/chunks/275.0e6bad414729bfa0.js +0 -32
- package/apps/pwa/out/_next/static/chunks/2803-33bdae771855a8d1.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3259-86f3be56e43eaf6a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3280-e177c33bfbdcfb0a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3294-2b6a6d45a46a6c1c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3522.758840fc5e7fe630.js +0 -287
- package/apps/pwa/out/_next/static/chunks/3525-a7c989c1a661e880.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3694.b1a66387756458c0.js +0 -1
- package/apps/pwa/out/_next/static/chunks/370.124f9e678a76fc01.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3797-9643105335e660fe.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4084.15ecaf4240bd9e77.js +0 -286
- package/apps/pwa/out/_next/static/chunks/4089-b3382b8843114eee.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4293.928a8d573c16aab7.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4402.f9238a3b18a7bd43.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4490-9b090b0f9427f902.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4548.13502d0ca3b4f862.js +0 -8
- package/apps/pwa/out/_next/static/chunks/4565-67e4c620597965c8.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4599-548bc96cda44a5a7.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4625.206574e33a9e7ae8.js +0 -104
- package/apps/pwa/out/_next/static/chunks/4862-392c9a71f0d8dfb4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5054.2906d1b02542edc5.js +0 -149
- package/apps/pwa/out/_next/static/chunks/5102-c822bbbee5a022a4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5194-8c93f2e7e4b6f717.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5305-bacd7eca0b9a5848.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5311.9071ee3f44fd7ff5.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5369.9f34353442daa421.js +0 -1
- package/apps/pwa/out/_next/static/chunks/589-49d730cb8138b397.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5904-264a6412ae3c9028.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5980.728531d5c566535c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/599-56f2c0a5cc4a7295.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6011.f525002e21bef0ac.js +0 -23
- package/apps/pwa/out/_next/static/chunks/6132.4381ab93683f1acf.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6149.2e698635e454d8f6.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6356-0afafc4230cf9767.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6406.64756073f7df35b8.js +0 -77
- package/apps/pwa/out/_next/static/chunks/6479-16e62aedafa0e310.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6599.b5246b46a2453329.js +0 -64
- package/apps/pwa/out/_next/static/chunks/6610.5c26eec011565acb.js +0 -1
- package/apps/pwa/out/_next/static/chunks/666.37d18474edc1c552.js +0 -2
- package/apps/pwa/out/_next/static/chunks/6904.a0b5b1ed4f2d2c20.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7247-1f4aba94aba50d9f.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7274-e71a7c5b7760cd02.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7290.1fcaeef3adb5d0d4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7477-2c64e42760f82f33.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7505-47002b76ff1aef6c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7600-11884ca4da68e5d9.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7607.40d8bb3689697ccb.js +0 -62
- package/apps/pwa/out/_next/static/chunks/7763.f51fcc19ca1414ec.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7785.846140c859b1adf6.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7807-a0fdac6393e0bf36.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7837-20a63b4b7e9342b1.js +0 -1
- package/apps/pwa/out/_next/static/chunks/800.7815414cf280545d.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8085-1e5fce59500a2cc2.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8100.415f2105faa63e97.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8290.701aa42ed13aa767.js +0 -226
- package/apps/pwa/out/_next/static/chunks/83a80514-b897b64df43ce108.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8444.d9c74214a3145a43.js +0 -93
- package/apps/pwa/out/_next/static/chunks/8569.98c915534928c7f9.js +0 -24
- package/apps/pwa/out/_next/static/chunks/8591.b1c7f876e5ec7cfb.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8715-f63756021d3ab989.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8879.776c2ec7732aa114.js +0 -115
- package/apps/pwa/out/_next/static/chunks/8881.ddca15b3c82c490c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8946.5da5c87f32a2e1a8.js +0 -41
- package/apps/pwa/out/_next/static/chunks/9117.080b9b75be90e14d.js +0 -74
- package/apps/pwa/out/_next/static/chunks/9167.f3ecb28f367d6e69.js +0 -94
- package/apps/pwa/out/_next/static/chunks/9224.66095370ee28c93b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9238-0682b013c83d6449.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9449.d3ed795da4befb1c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9638-d268329f2c6f4f8d.js +0 -1
- package/apps/pwa/out/_next/static/chunks/973cbf1b.3bbcf2b41548eca7.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9842-d6644eb53d3dc6bf.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9863.94ca3263e760696e.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/_not-found/page-d9babd8795f396a5.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/approvals/page-5ab1e47978577c5e.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/autopilot/page-24453f6a302816b4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/board/page-6c131d30bb1cf871.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/bots/page-dc0772b11861af51.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ceo/page-66ad46ed0279feb9.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/chat/page-5159dc8752c6cd29.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/consult/page-c5a2478861c08b87.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/control/page-f722a9dec084e08b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/decisions/page-256ffd365d32e100.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/design-check/page-e0de78ae7b99f01c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/editor/page-cfbdc44b708d9801.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/exec/page-4c741eb760f2f77c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/field/page-e9ba98f5f1440a21.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/healthz/route-d9babd8795f396a5.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/inside/page-dafc901bb1b72b5f.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/intake/page-2f6f303fa114ec24.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/layout-1634081c5272006c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/live-run/page-bb1f05cd51c38944.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/loops/page-646b50f63b6fe732.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/market/page-db069b48a59c8c00.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/missions/[id]/page-f736324afa2117c1.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/morning/page-6f131b51193a36f9.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/not-found-435f5e2faf02bc01.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/observatory/page-497e0eedf20e7890.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/checklist/page-a8d28fdf995e16ea.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/release/page-8966452228753c5e.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/seats/page-fd5d18656cb432b6.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/outputs/page-5663779cb1a37609.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/page-66a67eb19ebd976b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/reflection/page-3b492aa78cf3c4a6.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/scheduler/page-c71267e7681da982.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/sessions/page-a21d1e1797172692.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/settings/page-781cfc5114e097da.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/setup/done/page-478157214db6a023.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/setup/layout-d9babd8795f396a5.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/setup/page-86141f12cf090b7f.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/share/page-7256c73bb1c36751.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/showroom/page-5c738ca7adef3de5.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/tasks/page-0138c8091ec17224.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/term/page-cc2f24f16d85e735.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/today/page-979d318a534e92da.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/trace/page-182d394f7e6e096b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/vault/page-21050928a697ed7a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/workflows/chat-ui/page-44b9d807005a9030.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/workflows/page-601fb77340e63c9a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/workspace/page-75d3fb5d47af0edf.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/worktrees/page-11bf9e5725a72c22.js +0 -1
- package/apps/pwa/out/_next/static/chunks/dcf24383-2f82e35f2d827393.js +0 -1
- package/apps/pwa/out/_next/static/chunks/framework-bf135b997155e90e.js +0 -1
- package/apps/pwa/out/_next/static/chunks/main-5571537543bea91c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/main-app-25c51c7df7f3a442.js +0 -1
- package/apps/pwa/out/_next/static/chunks/pages/_app-e3b2c1f5d82abd65.js +0 -1
- package/apps/pwa/out/_next/static/chunks/pages/_error-083f2e23e9bc3641.js +0 -1
- package/apps/pwa/out/_next/static/chunks/webpack-e94d22b7a028217b.js +0 -1
- package/src/dashboard/bench-slash-runtime.ts +0 -40
- package/src/dashboard/input/search-modal-router.ts +0 -9
- package/src/dashboard/modals/pane-key-router.ts +0 -27
- package/src/dashboard/view-slash-runtime.ts +0 -48
- /package/apps/pwa/out/_next/static/{y1ltt7oCZxVpC6zVPhZ2O → LSXfY0u9QtfDbejY0Ukf_}/_ssgManifest.js +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import { spawn, spawnSync } from 'node:child_process';
|
|
3
3
|
import { appendFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
4
|
-
import { tmpdir } from 'node:os';
|
|
4
|
+
import { hostname, tmpdir } from 'node:os';
|
|
5
5
|
import { createHash, randomUUID } from 'node:crypto';
|
|
6
6
|
import { dirname, join, resolve } from 'node:path';
|
|
7
7
|
import { debug } from '../../src/debug/log.js';
|
|
@@ -11,25 +11,43 @@ import { effectiveInstanceRoot, releaseLedgerRoot } from '../../src/instance/res
|
|
|
11
11
|
import { getElanousConfigDirOverride } from '../../src/elanous-config-dir.js';
|
|
12
12
|
import { diffFailures, junitFailures, parseFailures, parseTimedOutFailures } from './gate-diff';
|
|
13
13
|
import { loadEnvKnownFailures, splitKnownEnv } from './env-known-failures';
|
|
14
|
+
import { loadGateBatchSafe } from './gate-batch-safe';
|
|
14
15
|
import { baselineFromCutLogs } from './baseline-from-logs';
|
|
15
|
-
import { mergePartialFailures, readPartialPlan, testFileOfId } from './gate-partial';
|
|
16
|
-
import { logPlanWalltime, planDurations,
|
|
16
|
+
import { mergePartialFailures, readPartialPlan, testFileOfId, verifyDocsOnlyCarry } from './gate-partial';
|
|
17
|
+
import { logPlanWalltime, planDurations, readFileMemory } from './shard-plan';
|
|
18
|
+
import { defaultGateHostMirrorReaders, resolveGateHostMirror, type GateHostMirror, type GateHostMirrorReaders } from './gate-host-mirror';
|
|
19
|
+
import { isGateSpawnLaneFile, planGateSpawnSeparatedShards } from './spawn-lane';
|
|
20
|
+
import { GATE_POD_ENV_BASELINE, gateTestBaseline } from '../lib/gate-test-env.js';
|
|
21
|
+
export { GATE_POD_ENV_BASELINE } from '../lib/gate-test-env.js';
|
|
17
22
|
import { readJunitFileSeconds } from './gate-method';
|
|
18
23
|
import { deriveCdpTestPatterns } from '../test-deterministic';
|
|
19
24
|
import { emitNodeResult, readGraphContext } from './node-verdict.js';
|
|
20
25
|
import { isGateExecutor, type GateExecutor } from './gate-executor.js';
|
|
21
|
-
import { runPodCommand, parsePodCpu, resolvePodCpu, type RunPodCommandOptions, type PodCommandResult, type PodCpu } from '../../src/task-orchestrator/surfaces/pod-command-job.js';
|
|
26
|
+
import { runPodCommand, parsePodCpu, podCpuMillis, resolvePodCpu, type RunPodCommandOptions, type PodCommandResult, type PodCpu } from '../../src/task-orchestrator/surfaces/pod-command-job.js';
|
|
22
27
|
import { POD_BUN_CACHE_HOST_PATH, parseInstallSeconds, podBunCacheVolume } from '../../src/task-orchestrator/surfaces/pod-bun-cache.js';
|
|
23
28
|
import { PodPoolScheduler, parsePodPool, checkPodPool, defaultRemoteRun, type PodPoolMember } from '../../src/task-orchestrator/surfaces/pod-pool.js';
|
|
24
29
|
import { POD_INSTALL_SLOTS_DEFAULT, POD_INSTALL_SLOTS_HOST_PATH, installSlotScript } from '../../src/task-orchestrator/surfaces/pod-install-slots.js';
|
|
25
30
|
import { gateDepsInstallScript, parseGateDeps } from '../../src/task-orchestrator/surfaces/pod-gate-deps.js';
|
|
31
|
+
import { writeWarmJobManifest } from './gate-warm-artifacts.js';
|
|
26
32
|
import { gateShardsPath, recordGateShardsVerdict, writeGateShards, type GateShard } from '../../src/release-loop/gate-shards.js';
|
|
27
33
|
import { defaultKubectl } from '../../src/task-orchestrator/surfaces/self-implement-pod.js';
|
|
34
|
+
import { executeTuiNodeCheck, type ExecutedTuiCheck } from './tui-node-checks.js';
|
|
35
|
+
import { emitPodLifecycle, gatePodLifecycleEvent, gatePodOutcome } from '../../src/task-orchestrator/surfaces/pod-lifecycle-stages.js';
|
|
36
|
+
import { gateHostReservationRoot, reserveReleaseGateHosts } from '../../src/release-loop/gate-host-reservation.js';
|
|
37
|
+
import { normRoot } from '../../src/instance/resolve.js';
|
|
28
38
|
|
|
29
|
-
interface CommandResult { rc: number; output: string; passedIds?: string[]; stalledEnv?: Array<{ file: string; reason: 'no-output' }>; timedOut?: boolean
|
|
39
|
+
interface CommandResult { rc: number; output: string; passedIds?: string[]; stalledEnv?: Array<{ file: string; reason: 'no-output' }>; timedOut?: boolean;
|
|
40
|
+
/** GATE-POD-STALL-FAILFAST ① — shard Pods whose tests the mid-run watch aborted in this sweep. */
|
|
41
|
+
abortedShards?: number;
|
|
42
|
+
/** Each contributing Pod leaf already passed failuresOf; repeated printed ids cannot add new failures. */
|
|
43
|
+
validatedShards?: boolean }
|
|
30
44
|
export interface GateRunner {
|
|
31
45
|
command(cmd: string, args: string[], cwd: string, limitMs?: number): Promise<CommandResult>;
|
|
32
46
|
localCommand(cmd: string, args: string[], cwd: string, limitMs?: number): Promise<CommandResult>;
|
|
47
|
+
/** Run one isolated test on the same Pod image and pinned commit as the sweep. */
|
|
48
|
+
recheckPod?(file: string, commit: string, pod: NonNullable<GateOptions['pod']>, timeoutMs: number, relaxed?: boolean): Promise<CommandResult>;
|
|
49
|
+
/** One checkout/install for a small independent file group; each file retains its own rc and log. */
|
|
50
|
+
recheckPodBatch?(files: readonly string[], commit: string, pod: NonNullable<GateOptions['pod']>, timeoutMs: number): Promise<Map<string, CommandResult>>;
|
|
33
51
|
/** `only` (GATE-PARTIAL) restricts the sweep to these test files; absent = every test file. */
|
|
34
52
|
sweep(tree: string, logDir?: string, pod?: GateOptions['pod'], only?: readonly string[]): Promise<CommandResult>;
|
|
35
53
|
add(tree: string, commit: string): Promise<void>;
|
|
@@ -52,13 +70,25 @@ export interface GateOptions {
|
|
|
52
70
|
instanceRoot?: string;
|
|
53
71
|
ledgerRoot?: string;
|
|
54
72
|
repo?: string;
|
|
73
|
+
/** Rehearsal's opt-in screen measurement; normal release gates keep their existing defaults. */
|
|
74
|
+
rehearsalTui?: boolean;
|
|
75
|
+
/** Files per final recheck window; every window is measured (default 32). */
|
|
76
|
+
recheckFileLimit?: number;
|
|
55
77
|
/** freshShards — GATE-SHARD-RESUME off: every shard starts a Pod even when this logDir already holds its result. */
|
|
56
78
|
pod?: { pool: string; shards?: number; shardTimeoutSeconds?: number; durationSource?: string; bunCache?: string; freshShards?: boolean;
|
|
79
|
+
/** Maximum concurrent file-isolation fallback Jobs across the entire sweep; omitted = 4. */
|
|
80
|
+
isolationConcurrency?: number;
|
|
57
81
|
/** GATE-SKIP-KNOWN-RETRY — failure ids the previous release already had (and env-known ones): a shard that fails only
|
|
58
82
|
* with these is not re-run in isolation; the verdict still classifies them against the baseline as before. */
|
|
59
83
|
knownFailures?: readonly string[];
|
|
60
84
|
/** GATE-SPEED A3① — shard Pod CPU request/limit (`release.loop.gatePodCpu`); omitted = request 1 / limit 4. */
|
|
61
85
|
cpu?: PodCpu;
|
|
86
|
+
/** GATE-HOSTMIRROR-EXPLICIT — host Git mirror for every gate Pod (wins over env/config); omitted = resolved once per
|
|
87
|
+
* runner by {@link resolveGateHostMirror} (env → config → operating config when isolated → none). */
|
|
88
|
+
hostMirror?: string;
|
|
89
|
+
/** GATE-HOSTMIRROR-EXPLICIT — injected readers (tests · embedders). Absent = the real ones; an injected pool without
|
|
90
|
+
* them reads env only — never a real config from a test. */
|
|
91
|
+
hostMirrorReaders?: GateHostMirrorReaders;
|
|
62
92
|
/** GATE-INSTALL-CACHE — concurrent `bun install`s per node (`release.loop.gatePodInstallSlots`); omitted = 4 · 0 = off. */
|
|
63
93
|
installSlots?: number;
|
|
64
94
|
/** GATE-MEM-ADMIT — seconds a shard waits for pool admission (memory headroom) before it launches anyway with
|
|
@@ -76,6 +106,13 @@ export interface GateOptions {
|
|
|
76
106
|
stopJob?: (job: { name: string; context?: string }) => boolean | void;
|
|
77
107
|
/** How often the stall watch reads the live log (ms); omitted = {@link GATE_POD_STALL_POLL_MS}. */
|
|
78
108
|
stallPollMs?: number;
|
|
109
|
+
/** GATE-POD-STALL-FAILFAST ① — seconds a shard may go without its part logs growing once test output began, before
|
|
110
|
+
* only its test process group is ended in the Pod (`gatePodMidRunStallSeconds`); omitted =
|
|
111
|
+
* {@link GATE_POD_MIDRUN_STALL_SECONDS_DEFAULT} · 0 = off. Needs the stall watch (live log). */
|
|
112
|
+
midRunStallSeconds?: number;
|
|
113
|
+
/** GATE-POD-STALL-FAILFAST ① — injected in-Pod test abort (tests · embedders); false = not taken. Absent = `kubectl exec`
|
|
114
|
+
* of {@link gateShardAbortScript} on the gate's own pool; an injected pool without it runs without the mid-run watch. */
|
|
115
|
+
abortTests?: (job: { name: string; context?: string }, reason: string) => boolean | void;
|
|
79
116
|
/** GATE-LIVE-OBS — the shard state file (`gateShardsPath(version)`) this sweep keeps current; absent = not written. */
|
|
80
117
|
shardsFile?: { path: string; version: string };
|
|
81
118
|
/** GATE-MULTI-HOST — is this pool member reachable (kubectl ns ⊕ ssh)? Injected by tests; absent = the real probe on
|
|
@@ -83,7 +120,9 @@ export interface GateOptions {
|
|
|
83
120
|
probeMember?: (member: PodPoolMember) => boolean;
|
|
84
121
|
/** GATE-MULTI-HOST — seconds a placed shard's Job status may stay unreadable before the shard counts as lost with its
|
|
85
122
|
* member; omitted = {@link GATE_MEMBER_UNREADABLE_SECONDS_DEFAULT}. Only a pool of two or more members uses it. */
|
|
86
|
-
memberUnreadableSeconds?: number
|
|
123
|
+
memberUnreadableSeconds?: number;
|
|
124
|
+
/** GATE-WARM-BOOT-SCAN — observation only: the sweep's test selection (list · filter · duration plan · shard plan) time. */
|
|
125
|
+
onSelectionTimed?: (timed: { ms: number; files: number }) => void };
|
|
87
126
|
}
|
|
88
127
|
export interface GateResult {
|
|
89
128
|
outcome: 'ok' | 'regression' | 'error';
|
|
@@ -96,12 +135,18 @@ export interface GateResult {
|
|
|
96
135
|
knownEnvCleared: string[];
|
|
97
136
|
stalledEnv: Array<{ file: string; reason: 'no-output' | 'isolated-timeout'; local: 'passed' | 'failed' | 'timeout' }>;
|
|
98
137
|
rcNonzeroNoFailures: Array<{ file: string; rc: number; pass: number; baseline: 'same' | 'clean' | 'missing' }>;
|
|
138
|
+
/** Files left unmeasured after isolated recheck; these cannot seed a reusable baseline. */
|
|
139
|
+
unmeasuredRecheckFiles?: string[];
|
|
140
|
+
/** GATE-POD-STALL-FAILFAST ① — cut sweep shard Pods ended by the mid-run abort (their files went the timeout way). */
|
|
141
|
+
abortedShards?: number;
|
|
99
142
|
baselineSource?: 'ledger' | 'instance' | 'cut-logs' | 'swept';
|
|
100
143
|
/** GATE-BASELINE-CACHE — isolated baseline files answered from the ledger (hits) or run now (misses). */
|
|
101
144
|
baselineCache?: { hits: number; misses: number };
|
|
102
145
|
/** GATE-SPEED A4 — a swept baseline ran only the cut's failing files that exist at the previous release. */
|
|
103
146
|
baselineDeferred?: { cutFiles: number; swept: number };
|
|
104
147
|
durationMs: number;
|
|
148
|
+
/** GATE-WARM-BOOT-SCAN — observation only: the cut sweep's test selection time (pod sweep · absent otherwise). */
|
|
149
|
+
selectionMs?: number;
|
|
105
150
|
error?: string;
|
|
106
151
|
stalledShards?: Array<{ shard: number; files: string[]; reason: 'incomplete' | 'no-output' | 'job-failed' | 'exit-code' | 'unattributed' | 'timeout'; lastFile?: string; summaryFailures?: number; namedFailures?: number; detail?: string;
|
|
107
152
|
/** GATE-POD-STALL-FAILFAST — the shard (and its one fresh retry) printed no test output within the stall window: unmeasured. */
|
|
@@ -112,6 +157,11 @@ export interface GateResult {
|
|
|
112
157
|
/** GATE-INTRO-RECHECK — new failures that were only the «timed out» shape and passed one isolated re-run with a
|
|
113
158
|
* {@link GATE_TIMEOUT_RECHECK_MS} per-test limit: a warning (flaky under load), not an introduced regression. */
|
|
114
159
|
loadFlaky?: string[];
|
|
160
|
+
/** File-level unhandled errors that reproduced or could not be measured in isolation. They keep the gate red. */
|
|
161
|
+
unmeasuredErrors?: Array<{ file: string; reason: 'reproduced' | 'timeout' | 'unreadable'; ids: string[] }>;
|
|
162
|
+
/** Shard-only unhandled errors whose file-level replay completed cleanly. */
|
|
163
|
+
interferenceErrors?: string[];
|
|
164
|
+
rehearsalTui?: ExecutedTuiCheck;
|
|
115
165
|
}
|
|
116
166
|
|
|
117
167
|
const sha = /^[0-9a-f]{7,40}$/i;
|
|
@@ -182,7 +232,6 @@ function unhandledErrorIds(output: string): string[] | undefined {
|
|
|
182
232
|
if (/^# Unhandled error between tests/.test(raw.trim())) {
|
|
183
233
|
if (!file) return undefined;
|
|
184
234
|
errors.push(`${file} > [error]`);
|
|
185
|
-
file = undefined;
|
|
186
235
|
}
|
|
187
236
|
}
|
|
188
237
|
return errors;
|
|
@@ -274,20 +323,72 @@ export function shardParts(paths: readonly string[], seconds: (file: string) =>
|
|
|
274
323
|
if (current.length) parts.push(current);
|
|
275
324
|
return parts;
|
|
276
325
|
}
|
|
326
|
+
/**
|
|
327
|
+
* GATE-ENV-BASELINE (W5 · 10-15) — the gate shard Pod's deterministic test env, exported before anything runs.
|
|
328
|
+
* 0.2.29 gate failures were the Pod's «environment», not the code: no LLM provider · `man` stubbed · CLI spawn timeouts.
|
|
329
|
+
* - `TZ=UTC` — the value `bun test` already uses when TZ is unset; pinned so a later image (or a shell profile) cannot move it.
|
|
330
|
+
* - `LANG`/`LC_ALL=C.UTF-8` — the image sets no locale (POSIX); C.UTF-8 ships with glibc, so no image rebuild.
|
|
331
|
+
* - `ELANOUS_TEST_FAKE_LLM=1` — the shared switch for the fake LLM provider. Its reader (UX device #27790) is not on main
|
|
332
|
+
* yet; until it lands the switch changes nothing. The reader yields to an explicit provider/endpoint, so a spawned CLI
|
|
333
|
+
* test that injects its own stub keeps it.
|
|
334
|
+
* Retries, splits and file isolation all re-enter `runShard`, which builds every Pod shell here — they carry it too.
|
|
335
|
+
* Provider keys are not set here on purpose: `test:deterministic` strips every credential-shaped key
|
|
336
|
+
* (`scripts/lib/deterministic-env.ts` `isCredentialKey`), so a test never reaches a real provider from the Pod.
|
|
337
|
+
* Binaries the image lacks (`man` pages — `git <sub> --help`) are not fixed here; see MANUAL-gate-regression-response-recipe §2a.
|
|
338
|
+
*/
|
|
339
|
+
export function gateEnvBaselineLine(env: Readonly<Record<string, string>> = gateTestBaseline(process.env)): string {
|
|
340
|
+
return `export ${Object.entries(env).map(([key, value]) => `${key}=${quote(value)}`).join(' ')}`;
|
|
341
|
+
}
|
|
277
342
|
/**
|
|
278
343
|
* The shard's Pod shell. Install once (behind the node's install slots), then run each part under the soft deadline,
|
|
279
344
|
* leaving `install.log`/`install.rc`, `part-<i>.log`/`.rc`/`.junit.xml`, and `shard.timeout` (the part index) when the
|
|
280
|
-
* deadline cut the shard.
|
|
345
|
+
* deadline cut the shard. Real child/PTY parts have an additional 120 s test limit; a cap hit also writes
|
|
346
|
+
* `shard.timeout` so the artifact reader does not mistake an unfinished child for a test verdict.
|
|
347
|
+
* A part without `.rc` did not finish. The shell itself always exits 0 (as before).
|
|
281
348
|
*/
|
|
282
|
-
export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPatterns: readonly string[]; softSeconds: number; cachePrefix: string; installSlots: number; depsDir?: string; beatSeconds?: number }): string {
|
|
283
|
-
|
|
349
|
+
export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPatterns: readonly string[]; softSeconds: number; cachePrefix: string; installSlots: number; depsDir?: string; beatSeconds?: number; parallelParts?: number; testTimeoutMs?: number; partLimitSeconds?: number; sourceBinding?: { cutSha: string; attemptId: string } }): string {
|
|
350
|
+
// The planner keeps spawn files in their own shards; reject a mixed part rather than silently lose its limit.
|
|
351
|
+
for (const part of o.parts) {
|
|
352
|
+
if (part.some(isGateSpawnLaneFile) && !part.every(isGateSpawnLaneFile)) throw new Error('gate spawn lane part mixed with regular tests');
|
|
353
|
+
}
|
|
354
|
+
const spawnParts = o.parts.flatMap((part, index) => part.length && part.every(isGateSpawnLaneFile) ? [index] : []);
|
|
355
|
+
const commands = o.parts.map((part, index) => {
|
|
284
356
|
const ignores = o.cdpPatterns.filter((pattern) => part.includes(pattern)).flatMap((pattern) => ['--path-ignore-patterns', pattern]);
|
|
285
|
-
|
|
286
|
-
|
|
357
|
+
// One-file runs keep named pass lines so their JUnit identities can be
|
|
358
|
+
// cross-checked. The package script's --dots removes those names.
|
|
359
|
+
const runner = part.length === 1 ? ['bun', 'run', 'scripts/test-deterministic.ts'] : ['bun', 'run', 'test:deterministic'];
|
|
360
|
+
// Bash scopes this assignment to run_part; the mixed-part shell test
|
|
361
|
+
// checks that a following multi-file part gets no source tag. The common
|
|
362
|
+
// run_part body changes the shell hash once for this feature; keeping the
|
|
363
|
+
// multi-file invocation unchanged avoids another cache-key change.
|
|
364
|
+
const warmFile = part.length === 1 ? `WARM_REFERENCE_FILE=${quote(part[0]!)} ` : '';
|
|
365
|
+
return `${warmFile}run_part ${index} ${[...runner, ...(o.testTimeoutMs !== undefined ? ['--timeout', String(o.testTimeoutMs)] : spawnParts.includes(index) ? ['--timeout=120000'] : []), ...ignores, ...part.map(asPath)].map(quote).join(' ')}`;
|
|
366
|
+
});
|
|
367
|
+
// Only file-isolation batches use this path. A bounded group shares one clone/install, but every test process gets
|
|
368
|
+
// its own HOME, state and tmp directory; part logs/JUnit remain separate and the existing artifact reader judges them.
|
|
369
|
+
const width = Math.max(1, Math.floor(o.parallelParts ?? 1));
|
|
370
|
+
const isolateParts = o.parallelParts !== undefined;
|
|
371
|
+
const run = width === 1 ? commands.join(' && ') : Array.from({ length: Math.ceil(commands.length / width) }, (_, group) => [
|
|
372
|
+
...commands.slice(group * width, (group + 1) * width).map((line) => `(${line}) & pids+=("$!")`),
|
|
373
|
+
'for pid in "${pids[@]}"; do wait "$pid" || true; done; pids=()',
|
|
374
|
+
].join('; ')).join('; ');
|
|
287
375
|
return [
|
|
288
376
|
'mkdir -p "$HOME/outbox"; O="$HOME/outbox"',
|
|
377
|
+
'stage_at() { printf "%s\\t%s\\n" "$1" "$2" >> "$O/stages.tsv"; }',
|
|
378
|
+
'stage() { stage_at "$(date +%s)" "$1"; }',
|
|
379
|
+
// This shell's first moment, captured before anything else runs so clone-end/shell-start keep their meaning.
|
|
380
|
+
'shell_t=$(date +%s)',
|
|
381
|
+
// Boot stages only: the container's age at shell entry. gate_end below keeps reading pod_age at its own spot.
|
|
382
|
+
`boot_age=$(awk -v t="$(cut -d' ' -f22 /proc/1/stat 2>/dev/null)" -v hz="$(getconf CLK_TCK 2>/dev/null)" '{ if (t > 0 && hz > 0) print int($1 - t / hz); else print 0 }' /proc/uptime 2>/dev/null)`,
|
|
383
|
+
'case "$boot_age" in ""|*[!0-9]*) boot_age=0;; esac',
|
|
384
|
+
// GATE-WARM-BOOT-STAGES: boot on the Pod's own clock — prep start = the child container's PID 1 (none when the age is
|
|
385
|
+
// unknown, never a guess), clone end = this shell's first line (the source checkout runs right before it).
|
|
386
|
+
'if [ "$boot_age" -gt 0 ]; then stage_at "$(( shell_t - boot_age ))" prep-start; fi',
|
|
387
|
+
'stage_at "$shell_t" clone-end',
|
|
388
|
+
'stage_at "$shell_t" shell-start',
|
|
289
389
|
...(o.beatSeconds ? gateBeatScript(o.beatSeconds, o.installSlots > 0) : []),
|
|
290
390
|
...(o.cachePrefix ? [o.cachePrefix] : []),
|
|
391
|
+
gateEnvBaselineLine(),
|
|
291
392
|
// Seconds since this container's PID 1 started (the clone ran before this shell) — 0 when /proc cannot tell.
|
|
292
393
|
`pod_age=$(awk -v t="$(cut -d' ' -f22 /proc/1/stat 2>/dev/null)" -v hz="$(getconf CLK_TCK 2>/dev/null)" '{ if (t > 0 && hz > 0) print int($1 - t / hz); else print 0 }' /proc/uptime 2>/dev/null)`,
|
|
293
394
|
'case "$pod_age" in ""|*[!0-9]*) pod_age=0;; esac',
|
|
@@ -295,19 +396,52 @@ export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPa
|
|
|
295
396
|
...(o.installSlots > 0 ? [installSlotScript({ slots: o.installSlots })] : []),
|
|
296
397
|
'run_part() {',
|
|
297
398
|
' local i=$1; shift',
|
|
399
|
+
...(spawnParts.length ? [' local spawn_part=0', ` case "$i" in ${spawnParts.join('|')}) spawn_part=1;; esac`] : []),
|
|
400
|
+
' stage "part-$i-start"',
|
|
401
|
+
...(isolateParts ? [
|
|
402
|
+
' local isolated; isolated=$(mktemp -d /tmp/gate-isolate.XXXXXX) || return 1',
|
|
403
|
+
' mkdir -p "$isolated/home" "$isolated/tmp" "$isolated/state" "$isolated/config" "$isolated/cache" "$isolated/data" "$isolated/xdg-state"',
|
|
404
|
+
' local source_repo="$PWD"',
|
|
405
|
+
' local -a run_env=(env HOME="$isolated/home" TMPDIR="$isolated/tmp" ELANOUS_STATE_DIR="$isolated/state" XDG_CONFIG_HOME="$isolated/config" XDG_CACHE_HOME="$isolated/cache" XDG_DATA_HOME="$isolated/data" XDG_STATE_HOME="$isolated/xdg-state")',
|
|
406
|
+
' stage "part-$i-setup-start"',
|
|
407
|
+
' local source_sha; source_sha=$(git rev-parse HEAD) || return 1',
|
|
408
|
+
' if ! "${run_env[@]}" git clone --shared --quiet --no-checkout "$source_repo" "$isolated/repo" > "$O/part-$i.setup.log" 2>&1 || ! "${run_env[@]}" git -C "$isolated/repo" checkout --detach --quiet "$source_sha" >> "$O/part-$i.setup.log" 2>&1; then cat "$O/part-$i.setup.log" > "$O/part-$i.log"; rm -rf "$isolated"; return 1; fi',
|
|
409
|
+
' ln -s "$source_repo/node_modules" "$isolated/repo/node_modules"',
|
|
410
|
+
' if [ -d "$source_repo/apps/pwa/node_modules" ]; then ln -s "$source_repo/apps/pwa/node_modules" "$isolated/repo/apps/pwa/node_modules"; fi',
|
|
411
|
+
' cd "$isolated/repo" || return 1',
|
|
412
|
+
' stage "part-$i-setup-end"',
|
|
413
|
+
] : [' local -a run_env=()']),
|
|
298
414
|
' local left=$(( gate_end - $(date +%s) ))',
|
|
299
415
|
' if [ "$left" -le 0 ]; then echo "$i" > "$O/shard.timeout"; return 1; fi',
|
|
300
|
-
'
|
|
416
|
+
...(spawnParts.length ? [' if [ "$spawn_part" -eq 1 ] && [ "$left" -gt 120 ]; then left=120; fi'] : []),
|
|
417
|
+
...(o.partLimitSeconds ? [` if [ "$left" -gt ${o.partLimitSeconds} ]; then left=${o.partLimitSeconds}; fi`] : []),
|
|
418
|
+
' stage "part-$i-test-start"',
|
|
419
|
+
...(spawnParts.length ? [' local test_start; test_start=$(date +%s)'] : []),
|
|
420
|
+
` { if [ -n "\${WARM_REFERENCE_FILE:-}" ]; then printf "[warm-worker] file=%s\\n" "$WARM_REFERENCE_FILE"; ${o.sourceBinding ? `printf "[warm-worker] cut=%s attempt=%s\\n" ${quote(o.sourceBinding.cutSha)} ${quote(o.sourceBinding.attemptId)}; ` : ''}fi; "\${run_env[@]}" timeout -k 10 "$left" "$@" --reporter=junit --reporter-outfile="$O/part-$i.junit.xml"; } > "$O/part-$i.log" 2>&1`,
|
|
301
421
|
' local prc=$?',
|
|
422
|
+
' if [ -n "${WARM_REFERENCE_FILE:-}" ] && [ -f "$O/part-$i.junit.xml" ]; then',
|
|
423
|
+
' local junit_sha; junit_sha=$(sha256sum "$O/part-$i.junit.xml" 2>/dev/null | awk \'{print $1}\')',
|
|
424
|
+
' if [ -z "$junit_sha" ]; then junit_sha=$(shasum -a 256 "$O/part-$i.junit.xml" 2>/dev/null | awk \'{print $1}\'); fi',
|
|
425
|
+
' if [[ "$junit_sha" =~ ^[a-f0-9]{64}$ ]]; then printf "[warm-worker] junit-sha256=%s\\n" "$junit_sha" >> "$O/part-$i.log"; fi',
|
|
426
|
+
' fi',
|
|
427
|
+
...(o.partLimitSeconds ? [' if [ "$prc" -eq 124 ] || [ "$prc" -eq 137 ]; then echo "$i" > "$O/part-$i.timeout"; fi'] : []),
|
|
428
|
+
...(spawnParts.length ? [' local test_end; test_end=$(date +%s)'] : []),
|
|
429
|
+
' stage "part-$i-test-end"',
|
|
430
|
+
...(isolateParts ? [' stage "part-$i-cleanup-start"', ' cd "$source_repo"', ' rm -rf "$isolated"', ' stage "part-$i-cleanup-end"'] : []),
|
|
431
|
+
' stage "part-$i-end"',
|
|
432
|
+
...(spawnParts.length ? [' if [ "$spawn_part" -eq 1 ] && { [ "$prc" -eq 124 ] || { [ "$test_end" -ge "$(( test_start + left ))" ] && { [ "$prc" -eq 137 ] || [ "$prc" -eq 143 ]; }; }; }; then echo "$i" > "$O/shard.timeout"; return 1; fi'] : []),
|
|
302
433
|
' if [ "$(date +%s)" -ge "$gate_end" ] && { [ "$prc" -eq 124 ] || [ "$prc" -eq 137 ] || [ "$prc" -eq 143 ]; }; then echo "$i" > "$O/shard.timeout"; return 1; fi',
|
|
303
434
|
' echo "$prc" > "$O/part-$i.rc"',
|
|
304
435
|
'}',
|
|
305
436
|
'if cd .. && cd repo; then',
|
|
437
|
+
' stage deps-start',
|
|
306
438
|
// GATE-IMAGE-DEPS: link the image's baked node_modules when the lockfiles match; only a miss installs (behind a slot).
|
|
307
439
|
...gateDepsInstallScript({ log: '"$O/install.log"', installSlots: o.installSlots, ...(o.depsDir ? { depsDir: o.depsDir } : {}) }).split('\n').map((line) => ` ${line}`),
|
|
308
440
|
' echo "$irc" > "$O/install.rc"',
|
|
309
|
-
|
|
441
|
+
' stage deps-end',
|
|
442
|
+
` if [ "$irc" -eq 0 ]; then stage test-start; ${width > 1 ? 'pids=(); ' : ''}${run || 'true'}; stage test-end; fi`,
|
|
310
443
|
'else echo 5 > "$O/install.rc"; fi',
|
|
444
|
+
'stage shell-end',
|
|
311
445
|
'true',
|
|
312
446
|
].join('\n');
|
|
313
447
|
}
|
|
@@ -317,6 +451,9 @@ export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPa
|
|
|
317
451
|
* shards ran to their 1697 s deadline with `finishedFiles: 0`, then retries and splits did the same for 2h+ and the node
|
|
318
452
|
* never finished. The beat lets the host see «no test output yet» while the Job runs:
|
|
319
453
|
* `ELANOUS_GATE_BEAT <seq> <slot|install|test> <0|1>` — the last field is 1 once any part log has a byte.
|
|
454
|
+
* GATE-POD-STALL-FAILFAST ① (mid-run): once there is test output the beat adds the part logs' total bytes —
|
|
455
|
+
* `ELANOUS_GATE_BEAT <seq> test 1 <bytes>` — so the host can see output stop after it started. The old 4-field line is
|
|
456
|
+
* still read (no bytes = no mid-run watch).
|
|
320
457
|
* It is observation only: the host hashes the shell without it into `runKey`.
|
|
321
458
|
*/
|
|
322
459
|
export function gateBeatScript(beatSeconds: number, installSlots: boolean): string[] {
|
|
@@ -324,13 +461,13 @@ export function gateBeatScript(beatSeconds: number, installSlots: boolean): stri
|
|
|
324
461
|
'gate_beat() {',
|
|
325
462
|
' local n=0 st t f',
|
|
326
463
|
' while :; do',
|
|
327
|
-
' n=$((n + 1)); t=0',
|
|
328
|
-
' for f in "$O"/part-*.log; do if [ -s "$f" ]; then t=1;
|
|
464
|
+
' n=$((n + 1)); t=0; b=0',
|
|
465
|
+
' for f in "$O"/part-*.log; do if [ -s "$f" ]; then t=1; b=$(( b + $(wc -c < "$f" 2>/dev/null || echo 0) )); fi; done',
|
|
329
466
|
' if [ -f "$O/install.rc" ]; then st=test',
|
|
330
467
|
// A deps miss with no slot line yet = waiting for an install slot (bounded by the slot script; not a stall).
|
|
331
468
|
...(installSlots ? [" elif [ -f \"$O/install.log\" ] && grep -q '^\\[gate\\] deps .* miss' \"$O/install.log\" 2>/dev/null && ! grep -q '^\\[gate\\] install-slot' \"$O/install.log\" 2>/dev/null; then st=slot"] : []),
|
|
332
469
|
' else st=install; fi',
|
|
333
|
-
' echo "ELANOUS_GATE_BEAT $n $st $t"',
|
|
470
|
+
' if [ "$t" = 1 ]; then echo "ELANOUS_GATE_BEAT $n $st $t $b"; else echo "ELANOUS_GATE_BEAT $n $st $t"; fi',
|
|
334
471
|
` sleep ${Math.max(1, Math.floor(beatSeconds))} >/dev/null 2>&1 </dev/null`,
|
|
335
472
|
' done',
|
|
336
473
|
'}',
|
|
@@ -343,30 +480,94 @@ export const GATE_POD_STALL_SECONDS_DEFAULT = 300;
|
|
|
343
480
|
export const GATE_POD_STALL_POLL_MS = 15_000;
|
|
344
481
|
/** At least ten beats per stall window, never slower than one per 30 s. */
|
|
345
482
|
export const gateStallBeatSeconds = (stallSeconds: number) => Math.min(30, Math.max(1, Math.floor(stallSeconds / 10)));
|
|
346
|
-
export interface ShardLiveLog { checkout: boolean; beat?: { seq: number; stage: 'slot' | 'install' | 'test'; testOutput: boolean
|
|
483
|
+
export interface ShardLiveLog { checkout: boolean; beat?: { seq: number; stage: 'slot' | 'install' | 'test'; testOutput: boolean;
|
|
484
|
+
/** Part logs' total bytes (mid-run progress) — absent on the old beat line. */
|
|
485
|
+
bytes?: number } }
|
|
347
486
|
/** What the Pod's live stdout says so far: the source checkout line and the latest beat. */
|
|
348
487
|
export function readShardLiveLog(text: string): ShardLiveLog {
|
|
349
488
|
let checkout = false;
|
|
350
489
|
let beat: ShardLiveLog['beat'];
|
|
351
490
|
for (const line of text.split(/\r?\n/)) {
|
|
352
491
|
if (/^ELANOUS_POD_SOURCE (?:commit|default|pr) /.test(line)) checkout = true;
|
|
353
|
-
const match = /^ELANOUS_GATE_BEAT (\d+) (slot|install|test) ([01])\s*$/.exec(line);
|
|
354
|
-
if (match) beat = { seq: Number(match[1]), stage: match[2] as 'slot' | 'install' | 'test', testOutput: match[3] === '1'
|
|
492
|
+
const match = /^ELANOUS_GATE_BEAT (\d+) (slot|install|test) ([01])(?: (\d+))?\s*$/.exec(line);
|
|
493
|
+
if (match) beat = { seq: Number(match[1]), stage: match[2] as 'slot' | 'install' | 'test', testOutput: match[3] === '1',
|
|
494
|
+
...(match[4] !== undefined ? { bytes: Number(match[4]) } : {}) };
|
|
355
495
|
}
|
|
356
496
|
return { checkout, ...(beat ? { beat } : {}) };
|
|
357
497
|
}
|
|
498
|
+
|
|
499
|
+
/** Kubernetes clock boundaries for scheduling and init. Child start → shell-start includes source checkout and setup. */
|
|
500
|
+
export function readGatePodLifecycle(text: string): Record<string, string> {
|
|
501
|
+
try {
|
|
502
|
+
const pod = (JSON.parse(text) as { items?: Array<{ metadata?: { creationTimestamp?: string }; status?: {
|
|
503
|
+
conditions?: Array<{ type?: string; lastTransitionTime?: string }>;
|
|
504
|
+
initContainerStatuses?: Array<{ name?: string; state?: { terminated?: { startedAt?: string; finishedAt?: string } } }>;
|
|
505
|
+
containerStatuses?: Array<{ name?: string; state?: { terminated?: { startedAt?: string; finishedAt?: string } } }>;
|
|
506
|
+
} }> }).items?.[0];
|
|
507
|
+
if (!pod) return {};
|
|
508
|
+
const out: Record<string, string> = {};
|
|
509
|
+
const put = (key: string, value?: string) => { if (value && Number.isFinite(Date.parse(value))) out[key] = value; };
|
|
510
|
+
put('podCreated', pod.metadata?.creationTimestamp);
|
|
511
|
+
put('podScheduled', pod.status?.conditions?.find((item) => item.type === 'PodScheduled')?.lastTransitionTime);
|
|
512
|
+
for (const item of pod.status?.initContainerStatuses ?? []) {
|
|
513
|
+
if (!item.name || !/^[a-z0-9-]+$/.test(item.name)) continue;
|
|
514
|
+
put(`init-${item.name}-start`, item.state?.terminated?.startedAt);
|
|
515
|
+
put(`init-${item.name}-end`, item.state?.terminated?.finishedAt);
|
|
516
|
+
}
|
|
517
|
+
const child = pod.status?.containerStatuses?.find((item) => item.name === 'child');
|
|
518
|
+
put('childStart', child?.state?.terminated?.startedAt);
|
|
519
|
+
put('childEnd', child?.state?.terminated?.finishedAt);
|
|
520
|
+
return out;
|
|
521
|
+
} catch { return {}; }
|
|
522
|
+
}
|
|
523
|
+
/** Seconds with at least one isolated test process running; setup gaps between parallel groups remain overhead. */
|
|
524
|
+
export function gateBatchTestWallSeconds(stages: Record<string, number> | undefined, partCount: number): number | undefined {
|
|
525
|
+
if (!stages || partCount < 1) return undefined;
|
|
526
|
+
const spans: Array<[number, number]> = [];
|
|
527
|
+
for (let index = 0; index < partCount; index++) {
|
|
528
|
+
const start = stages[`part-${index}-test-start`];
|
|
529
|
+
const end = stages[`part-${index}-test-end`];
|
|
530
|
+
if (!Number.isFinite(start) || !Number.isFinite(end) || end! < start!) return undefined;
|
|
531
|
+
spans.push([start!, end!]);
|
|
532
|
+
}
|
|
533
|
+
spans.sort((a, b) => a[0] - b[0]);
|
|
534
|
+
let covered = 0;
|
|
535
|
+
let [from, to] = spans[0]!;
|
|
536
|
+
for (const [start, end] of spans.slice(1)) {
|
|
537
|
+
if (start <= to) to = Math.max(to, end);
|
|
538
|
+
else { covered += to - from; from = start; to = end; }
|
|
539
|
+
}
|
|
540
|
+
return covered + to - from;
|
|
541
|
+
}
|
|
358
542
|
/**
|
|
359
543
|
* GATE-POD-STALL-FAILFAST — watches one running shard Job. Armed once the source checkout line is seen; disarmed for good
|
|
360
544
|
* the moment any test output exists (that shard then behaves exactly as before). Stalled = `stallSeconds` since the
|
|
361
545
|
* checkout with no test output, not counting time the beat reported an install-slot wait. A silent shell (no beat at all)
|
|
362
546
|
* counts like any other stage. Only successful reads decide; a read failure never stalls a shard. `graceMs` covers the
|
|
363
547
|
* beat's report lag: output that appeared just inside the window is reported by the next beat before the stop decides.
|
|
548
|
+
*
|
|
549
|
+
* GATE-POD-STALL-FAILFAST ① (mid-run): with `midRunStallSeconds` > 0 and an `abortTests`, the watch does not end at the
|
|
550
|
+
* first test output when the beat carries the part logs' bytes. It keeps the time the bytes last changed; no change for
|
|
551
|
+
* `midRunStallSeconds` (⊕ grace) = a mid-run hang: `onAbort`, then `abortTests` ends only the test process group in the
|
|
552
|
+
* Pod (never the Job — Job backoffLimit 0). One abort per watch; a beat without bytes (old shell) ends the watch as before.
|
|
364
553
|
*/
|
|
365
554
|
export function watchShardStall(o: { stallSeconds: number; pollMs: number; read: () => string | null; onStall: (sinceCheckoutMs: number) => void; now?: () => number; graceMs?: number;
|
|
366
555
|
/** Stops the Job; false = not taken, asked again every poll until it is or the watch ends (a stalled Job must not run to its deadline). */
|
|
367
|
-
stopJob?: () => boolean | void
|
|
556
|
+
stopJob?: () => boolean | void;
|
|
557
|
+
/** Mid-run hang window in seconds (0 / absent = off). */
|
|
558
|
+
midRunStallSeconds?: number;
|
|
559
|
+
onAbort?: (info: { sinceLastOutputMs: number }) => void;
|
|
560
|
+
/** Ends the Pod's test process group; false = not taken, asked again on the next polls (at most {@link GATE_SHARD_ABORT_TRIES}). */
|
|
561
|
+
abortTests?: () => boolean | void }): { stop(): void; stalled(): { sinceCheckoutMs: number } | undefined; aborted(): ShardAbort | undefined; state(): ShardStallWatchState } {
|
|
368
562
|
const now = o.now ?? Date.now;
|
|
563
|
+
const midRunMs = o.midRunStallSeconds && o.midRunStallSeconds > 0 && o.abortTests ? o.midRunStallSeconds * 1000 : 0;
|
|
564
|
+
/** Mid-run phase: the last bytes the beat reported and when they last changed. */
|
|
565
|
+
let midRun: { bytes: number; changedAt: number } | undefined;
|
|
566
|
+
let abort: ShardAbort | undefined;
|
|
567
|
+
let abortTries = 0;
|
|
369
568
|
let checkoutAt: number | undefined;
|
|
569
|
+
/** Observation only (GATE-WARM-BOOT-SCAN): when the first test output was seen, relative to the checkout. */
|
|
570
|
+
let firstTestAt: { checkout: number; at: number } | undefined;
|
|
370
571
|
let windowStart: number | undefined;
|
|
371
572
|
let lastSeq: number | undefined;
|
|
372
573
|
let hit: { sinceCheckoutMs: number } | undefined;
|
|
@@ -380,6 +581,15 @@ export function watchShardStall(o: { stallSeconds: number; pollMs: number; read:
|
|
|
380
581
|
try { taken = o.stopJob ? o.stopJob() : true; } catch { taken = false; }
|
|
381
582
|
if (taken === false && !ended) timer = setTimeout(askStop, o.pollMs);
|
|
382
583
|
};
|
|
584
|
+
const askAbort = () => {
|
|
585
|
+
timer = undefined;
|
|
586
|
+
if (ended) return;
|
|
587
|
+
abortTries++;
|
|
588
|
+
let taken: boolean | void = false;
|
|
589
|
+
try { taken = o.abortTests!(); } catch { taken = false; }
|
|
590
|
+
if (taken !== false && abort) abort.taken = true;
|
|
591
|
+
if (taken === false && !ended && abortTries < GATE_SHARD_ABORT_TRIES) timer = setTimeout(askAbort, o.pollMs);
|
|
592
|
+
};
|
|
383
593
|
const tick = () => {
|
|
384
594
|
timer = undefined;
|
|
385
595
|
if (done) return;
|
|
@@ -389,7 +599,31 @@ export function watchShardStall(o: { stallSeconds: number; pollMs: number; read:
|
|
|
389
599
|
if (text !== null) {
|
|
390
600
|
const t = now();
|
|
391
601
|
const live = readShardLiveLog(text);
|
|
392
|
-
if (
|
|
602
|
+
if (midRun) {
|
|
603
|
+
// A beat without bytes (cannot judge progress) ends the mid-run watch; a failed or empty read decides nothing.
|
|
604
|
+
if (live.beat?.testOutput && live.beat.bytes === undefined) { done = true; return; }
|
|
605
|
+
if (live.beat?.bytes !== undefined && live.beat.bytes !== midRun.bytes) midRun = { bytes: live.beat.bytes, changedAt: t };
|
|
606
|
+
else if (t - midRun.changedAt >= midRunMs + (o.graceMs ?? 0)) {
|
|
607
|
+
done = true;
|
|
608
|
+
abort = { reason: `mid-run-stall ${o.midRunStallSeconds}s`, at: new Date().toISOString(), by: 'auto', sinceLastOutputMs: t - midRun.changedAt, taken: false };
|
|
609
|
+
try { o.onAbort?.({ sinceLastOutputMs: abort.sinceLastOutputMs }); } catch { /* observation only */ }
|
|
610
|
+
askAbort();
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
613
|
+
timer = setTimeout(tick, o.pollMs);
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
if (live.beat?.testOutput) {
|
|
617
|
+
const seenCheckout = checkoutAt ?? (live.checkout ? t : undefined);
|
|
618
|
+
if (seenCheckout !== undefined) firstTestAt = { checkout: seenCheckout, at: t };
|
|
619
|
+
if (midRunMs && live.beat.bytes !== undefined) {
|
|
620
|
+
midRun = { bytes: live.beat.bytes, changedAt: t };
|
|
621
|
+
timer = setTimeout(tick, o.pollMs);
|
|
622
|
+
return;
|
|
623
|
+
}
|
|
624
|
+
done = true;
|
|
625
|
+
return;
|
|
626
|
+
}
|
|
393
627
|
if (live.checkout && checkoutAt === undefined) { checkoutAt = t; windowStart = t; }
|
|
394
628
|
if (live.beat && live.beat.seq !== lastSeq) {
|
|
395
629
|
lastSeq = live.beat.seq;
|
|
@@ -406,7 +640,69 @@ export function watchShardStall(o: { stallSeconds: number; pollMs: number; read:
|
|
|
406
640
|
timer = setTimeout(tick, o.pollMs);
|
|
407
641
|
};
|
|
408
642
|
timer = setTimeout(tick, o.pollMs);
|
|
409
|
-
return { stop: () => { done = true; ended = true; if (timer) clearTimeout(timer); }, stalled: () => hit
|
|
643
|
+
return { stop: () => { done = true; ended = true; if (timer) clearTimeout(timer); }, stalled: () => hit, aborted: () => abort,
|
|
644
|
+
state: () => ({ armed: checkoutAt !== undefined || firstTestAt !== undefined,
|
|
645
|
+
disarmedBy: hit ? 'stall-fired' : abort ? 'mid-run-stall' : firstTestAt ? 'test-output' : null,
|
|
646
|
+
firstTestAtSec: firstTestAt ? Math.round((firstTestAt.at - firstTestAt.checkout) / 100) / 10 : null,
|
|
647
|
+
...(abort ? { abort: { reason: abort.reason, at: abort.at, by: abort.by } } : {}) }) };
|
|
648
|
+
}
|
|
649
|
+
/** GATE-POD-STALL-FAILFAST ① — a mid-run abort: why (`mid-run-stall <N>s`), when, who (`auto`; a manual knob is slice 2). */
|
|
650
|
+
export type ShardAbort = { reason: string; at: string; by: 'auto'; sinceLastOutputMs: number;
|
|
651
|
+
/** The in-Pod abort was taken (false = every try failed — the Job then runs to its deadline as before). */
|
|
652
|
+
taken: boolean };
|
|
653
|
+
/** How many polls ask the in-Pod abort before giving up (the Job then runs to its deadline as before). */
|
|
654
|
+
export const GATE_SHARD_ABORT_TRIES = 3;
|
|
655
|
+
/** GATE-POD-STALL-FAILFAST ① — default mid-run hang window: well under a shard deadline (≈1500 s+), far over one test's limit. */
|
|
656
|
+
export const GATE_POD_MIDRUN_STALL_SECONDS_DEFAULT = 600;
|
|
657
|
+
/** Seconds between the in-Pod TERM and KILL of the test process group. */
|
|
658
|
+
export const GATE_SHARD_ABORT_GRACE_SECONDS = 10;
|
|
659
|
+
/** What one shard's stall watch saw — armed (checkout seen) · why it ended · first test output seconds after the checkout. */
|
|
660
|
+
export type ShardStallWatchState = { armed: boolean; disarmedBy: 'test-output' | 'stall-fired' | 'mid-run-stall' | null; firstTestAtSec: number | null;
|
|
661
|
+
abort?: Omit<ShardAbort, 'sinceLastOutputMs' | 'taken'> };
|
|
662
|
+
/**
|
|
663
|
+
* GATE-POD-STALL-FAILFAST ① — run inside the shard Pod (`kubectl exec … bash -c`): ends only the running test process
|
|
664
|
+
* groups — the group led by a process whose argv carries `--reporter-outfile=…/outbox/part-<i>.junit.xml` (GNU `timeout`
|
|
665
|
+
* leads its own group) — TERM now, KILL after the grace. It never signals PID 1's group or its own. Before the signal it
|
|
666
|
+
* leaves `part-<i>.abort` (the reason) and `shard.timeout`, so the outbox reads like a deadline cut of that part
|
|
667
|
+
* ({@link readShardArtifacts}: unfinished, timed out) while the shell finishes the remaining parts and its outbox.
|
|
668
|
+
* Exit 0 = at least one group signalled.
|
|
669
|
+
*/
|
|
670
|
+
export function gateShardAbortScript(reason: string, graceSeconds = GATE_SHARD_ABORT_GRACE_SECONDS,
|
|
671
|
+
/** Only this outbox's parts (tests: never another run's groups on the same machine); absent = any `…/outbox/` in the Pod. */
|
|
672
|
+
outbox?: string): string {
|
|
673
|
+
return [
|
|
674
|
+
`reason=${quote(reason.replace(/[^\w .:-]/g, '_'))}`,
|
|
675
|
+
`only=${quote(outbox ?? '')}`,
|
|
676
|
+
"pg_of() { sed 's/^.*) //' \"/proc/$1/stat\" 2>/dev/null | cut -d' ' -f3; }",
|
|
677
|
+
'root_pg=$(pg_of 1); my_pg=$(pg_of $$); found=0',
|
|
678
|
+
'for d in /proc/[0-9]*; do',
|
|
679
|
+
' pid=${d#/proc/}',
|
|
680
|
+
" out=$(tr '\\0' '\\n' < \"$d/cmdline\" 2>/dev/null | grep -m1 -E '^--reporter-outfile=.*/outbox/part-[0-9]+\\.junit\\.xml$') || continue",
|
|
681
|
+
' if [ -n "$only" ]; then case "$out" in "--reporter-outfile=$only/part-"*) ;; *) continue;; esac; fi',
|
|
682
|
+
' pg=$(pg_of "$pid")',
|
|
683
|
+
' case "$pg" in ""|*[!0-9]*) continue;; esac',
|
|
684
|
+
' if [ "$pg" != "$pid" ] || [ "$pg" -le 1 ] || [ "$pg" = "$root_pg" ] || [ "$pg" = "$my_pg" ]; then continue; fi',
|
|
685
|
+
' out=${out#--reporter-outfile=}; dir=${out%/part-*}; part=${out##*/part-}; part=${part%.junit.xml}',
|
|
686
|
+
' printf "%s\\n" "$reason" > "$dir/part-$part.abort"',
|
|
687
|
+
' [ -f "$dir/shard.timeout" ] || echo "$part" > "$dir/shard.timeout"',
|
|
688
|
+
' if kill -TERM -- "-$pg" 2>/dev/null; then found=$((found + 1)); echo "ELANOUS_GATE_ABORT part=$part pgid=$pg"; fi',
|
|
689
|
+
` ( sleep ${Math.max(1, Math.floor(graceSeconds))}; kill -KILL -- "-$pg" 2>/dev/null ) >/dev/null 2>&1 </dev/null &`,
|
|
690
|
+
'done',
|
|
691
|
+
'echo "ELANOUS_GATE_ABORT signalled=$found"',
|
|
692
|
+
'[ "$found" -gt 0 ]',
|
|
693
|
+
].join('\n');
|
|
694
|
+
}
|
|
695
|
+
/** The shard json's `stallWatch` record (observation only — never read by any judgment). */
|
|
696
|
+
export type ShardStallWatchRecord = Omit<ShardStallWatchState, 'disarmedBy'> & { disarmedBy: ShardStallWatchState['disarmedBy'] | 'off' | 'no-live-log'; stallSeconds: number };
|
|
697
|
+
export function shardStallWatchRecord(stallSeconds: number, hasLiveLog: boolean, state: ShardStallWatchState | undefined): ShardStallWatchRecord {
|
|
698
|
+
if (!(stallSeconds > 0)) return { armed: false, disarmedBy: 'off', firstTestAtSec: null, stallSeconds };
|
|
699
|
+
if (!hasLiveLog) return { armed: false, disarmedBy: 'no-live-log', firstTestAtSec: null, stallSeconds };
|
|
700
|
+
return { ...(state ?? { armed: false, disarmedBy: null, firstTestAtSec: null }), stallSeconds };
|
|
701
|
+
}
|
|
702
|
+
/** GATE-WARM-BOOT-SCAN — logs the selection time and hands it to the caller; an observer failure never stops the sweep. */
|
|
703
|
+
export function reportSelectionTimed(ms: number, files: number, observer?: (timed: { ms: number; files: number }) => void): void {
|
|
704
|
+
debug.log('release-loop.gate', 'selection-timed', { ms, files });
|
|
705
|
+
try { observer?.({ ms, files }); } catch { /* observation only */ }
|
|
410
706
|
}
|
|
411
707
|
/** Removes bun's own summary lines so one summary over several runs can follow (same rule as the sweep's merge). */
|
|
412
708
|
const stripSummary = (output: string) => output.replace(/(?:^|\n)\s*\d+ (?:pass|fail|errors?)\s*(?=\n|$)/g, '\n').replace(/Ran \d+ tests? across \d+ files?\.?/g, '');
|
|
@@ -447,6 +743,13 @@ export function readShardArtifacts(dir: string, parts: readonly (readonly string
|
|
|
447
743
|
const log = read(`part-${index}.log`);
|
|
448
744
|
const rc = rcOf(read(`part-${index}.rc`));
|
|
449
745
|
const clean = log?.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '');
|
|
746
|
+
// GATE-POD-STALL-FAILFAST ① — a part the mid-run abort ended reads like a deadline cut: unfinished, its exit (143/137) no verdict.
|
|
747
|
+
const abort = read(`part-${index}.abort`);
|
|
748
|
+
if (abort !== undefined) {
|
|
749
|
+
bad.push(`# part ${index} (abort: ${abort.trim()})\n${log ?? ''}`);
|
|
750
|
+
otherBad = true;
|
|
751
|
+
return;
|
|
752
|
+
}
|
|
450
753
|
if (clean !== undefined && (rc === 0 || rc === 1) && lastMatch(clean, /Ran (\d+) tests? across (\d+) files?/)) {
|
|
451
754
|
const junit = read(`part-${index}.junit.xml`);
|
|
452
755
|
good.push({ log: clean, rc, files, ...(junit !== undefined ? { junit } : {}) });
|
|
@@ -480,6 +783,9 @@ export function readShardArtifacts(dir: string, parts: readonly (readonly string
|
|
|
480
783
|
};
|
|
481
784
|
}
|
|
482
785
|
|
|
786
|
+
/** The line appended to pod-N.log when part output that never finished went to pod-N.unfinished.log (a reader of pod-N.log alone saw «no test output»). */
|
|
787
|
+
export const gateUnfinishedHint = (relativePath: string, bytes: number) => `[gate] unfinished: ${relativePath} · ${bytes}B (진행 중 part 출력은 그 파일)`;
|
|
788
|
+
|
|
483
789
|
// review-model-ab: Pod 에서 출력 없이 멈춤 2/2(10-04 0.2.11 · 로컬 단독 4/0 · 1.7s) — GATE-STALL 수리 전까지 Pod 밖.
|
|
484
790
|
export const POD_SWEEP_INTEGRATION_ONLY: readonly string[] = ['scripts/install.test.ts', 'scripts/review-model-ab.test.ts'];
|
|
485
791
|
/** Nightly-only whole-repository cases; gate runs the fixture cases in these same files. */
|
|
@@ -565,7 +871,16 @@ function failuresOf(run: CommandResult, label: string): SweepFailures {
|
|
|
565
871
|
const incomplete = (reason: string) => new Error(`${label} incomplete (rc=${run.rc}; Ran=${ran ? ran[0] : 'missing'}; errors=${Number.isNaN(reportedErrors) ? 'missing' : errorCount}; ${reason})\n${tail40(output)}`);
|
|
566
872
|
const count = summaryCount(output, 'fail');
|
|
567
873
|
const failures = parseFailures(output);
|
|
568
|
-
|
|
874
|
+
// Bun repeats failed test names under "N tests failed:". A validated Pod sweep joins
|
|
875
|
+
// separately checked shards, so a summary in one shard can precede real failures
|
|
876
|
+
// from later shards. Remove each repeated block, not everything after the last one.
|
|
877
|
+
// Unhandled-error summaries have a separate file-attribution path below and do not
|
|
878
|
+
// use Bun's repeated `(fail)` block format.
|
|
879
|
+
const failureSummary = [...output.matchAll(/^\d+ tests? failed:\s*$/gm)].at(-1);
|
|
880
|
+
const executionOutput = run.validatedShards
|
|
881
|
+
? output.replace(/^\d+ tests? failed:\r?\n(?:\(fail\)[^\r\n]*(?:\r?\n|$))*(?:[ \t]*\r?\n)*/gm, '')
|
|
882
|
+
: failureSummary ? output.slice(0, failureSummary.index) : output;
|
|
883
|
+
const occurrences = [...executionOutput.matchAll(/^[ \t]*\(fail\)[ \t]+.+?(?:[ \t]+\[[\d.]+(?:ms|s)\])?[ \t]*\r?$/gm)].length;
|
|
569
884
|
// A test that leaves process.exitCode set leaks a non-zero exit into a run whose own summary is complete and clean
|
|
570
885
|
// (0.2.7 gate: agent-cli at v0.2.6 · 12 pass · 0 fail · rc 2). The summary is the verdict; read that exit as clean.
|
|
571
886
|
const leakedExit = !!ran && run.rc !== 0 && run.rc !== 1 && count === 0 && errorCount === 0 && occurrences === 0
|
|
@@ -581,11 +896,10 @@ function failuresOf(run: CommandResult, label: string): SweepFailures {
|
|
|
581
896
|
if (/^# Unhandled error between tests/.test(raw.trim())) {
|
|
582
897
|
if (!file) throw incomplete('unattributed unhandled error');
|
|
583
898
|
errors.push(`${file} > [error]`);
|
|
584
|
-
file = undefined;
|
|
585
899
|
}
|
|
586
900
|
}
|
|
587
901
|
if (errors.length !== errorCount) throw incomplete(`error attribution identified=${errors.length}`);
|
|
588
|
-
if (!Number.isFinite(count) || count !== occurrences || (count > 0 && failures.length === 0)
|
|
902
|
+
if (!Number.isFinite(count) || (count !== occurrences && !(run.validatedShards && occurrences > count && failures.length === count)) || (count > 0 && failures.length === 0)
|
|
589
903
|
|| (rc === 1 && count === 0 && errorCount === 0) || (rc === 0 && count + errorCount > 0)) {
|
|
590
904
|
throw incomplete(`failure attribution incomplete: summary=${count}, identified=${failures.length}`);
|
|
591
905
|
}
|
|
@@ -634,17 +948,22 @@ export function readIsolatedRun(run: CommandResult, label: string, file: string)
|
|
|
634
948
|
}
|
|
635
949
|
|
|
636
950
|
class StalledPodShards extends Error {
|
|
637
|
-
constructor(readonly stalledShards: NonNullable<GateResult['stalledShards']>, readonly partialSummary: NonNullable<GateResult['partialSummary']
|
|
951
|
+
constructor(readonly stalledShards: NonNullable<GateResult['stalledShards']>, readonly partialSummary: NonNullable<GateResult['partialSummary']>, readonly abortedShards = 0) {
|
|
638
952
|
const unmeasured = stalledShards.filter((item) => item.stall);
|
|
639
953
|
const exitCode = stalledShards.filter((item) => item.reason === 'exit-code');
|
|
640
954
|
super(`pod sweep incomplete: stalled shards ${stalledShards.map(({ shard }) => shard).join(', ')}`
|
|
641
955
|
+ (unmeasured.length ? ` (unmeasured: no test output within ${unmeasured[0]!.stall!.seconds}s of checkout — shards ${[...new Set(unmeasured.map(({ shard }) => shard))].join(', ')})` : '')
|
|
642
|
-
+ (exitCode.length ? ` (exit-code anomaly: 0 fail but exit ≠ 0/1 — ${[...new Set(exitCode.flatMap(({ files }) => files))].join(', ')})` : '')
|
|
956
|
+
+ (exitCode.length ? ` (exit-code anomaly: 0 fail but exit ≠ 0/1 — ${[...new Set(exitCode.flatMap(({ files }) => files))].join(', ')})` : '')
|
|
957
|
+
+ (abortedShards ? ` (중단 조각 ${abortedShards}: mid-run-stall)` : ''));
|
|
643
958
|
}
|
|
644
959
|
}
|
|
645
960
|
|
|
646
961
|
/** Isolated re-runs (host stall re-run · cut and baseline re-checks) are time-limited; sweeps and installs keep their full time. */
|
|
647
962
|
export const GATE_ISOLATED_TIMEOUT_MS = 300_000;
|
|
963
|
+
/** Default final recheck window size; overflow continues in later windows. */
|
|
964
|
+
export const GATE_HOST_RECHECK_FILE_LIMIT = 32;
|
|
965
|
+
/** Two isolated test processes in each of four Pods = eight at most, with one checkout/install per eight files. */
|
|
966
|
+
export const GATE_RECHECK_POD_BATCH_FILES = 8;
|
|
648
967
|
/** GATE-INTRO-RECHECK (0.2.19) — the per-test limit of the one re-run given to a «timed out after 5000ms» new failure.
|
|
649
968
|
* 0.2.18: both first-gate introductions were 5 s limits missed at 6.04/6.06 s, and they alone cost an 84-minute re-gate. */
|
|
650
969
|
export const GATE_TIMEOUT_RECHECK_MS = 60_000;
|
|
@@ -750,6 +1069,15 @@ export async function admitGateShard(admission: GateAdmission | undefined, o: {
|
|
|
750
1069
|
|
|
751
1070
|
export function createGateRunner(repo: string, remote?: string, commandOverride?: GateRunner['command'], podCommand: (options: RunPodCommandOptions) => Promise<PodCommandResult> = runPodCommand, poolOverride?: PodPoolScheduler, podLogTail?: (job: PodCommandResult) => string): GateRunner {
|
|
752
1071
|
let remoteMirror: string | undefined;
|
|
1072
|
+
// GATE-HOSTMIRROR-EXPLICIT: resolved once per runner and handed to every podCommand (shard · retry · recheck).
|
|
1073
|
+
let gateMirror: GateHostMirror | undefined;
|
|
1074
|
+
const gateHostMirror = (pod: NonNullable<GateOptions['pod']>): { hostMirror?: string } => {
|
|
1075
|
+
if (!gateMirror) {
|
|
1076
|
+
gateMirror = resolveGateHostMirror(pod.hostMirror, pod.hostMirrorReaders ?? (poolOverride ? { env: process.env } : defaultGateHostMirrorReaders()));
|
|
1077
|
+
debug.log('release-loop.gate', 'host-mirror-resolved', { source: gateMirror.source, path: gateMirror.path });
|
|
1078
|
+
}
|
|
1079
|
+
return gateMirror.hostMirror ? { hostMirror: gateMirror.hostMirror } : {};
|
|
1080
|
+
};
|
|
753
1081
|
const localCommand: GateRunner['localCommand'] = async (cmd, args, cwd, limitMs) => {
|
|
754
1082
|
if (limitMs) return limitedLocalCommand(cmd, args, cwd, limitMs);
|
|
755
1083
|
const run = spawnSync(cmd, args, { cwd, encoding: 'utf8', maxBuffer: 128 * 1024 * 1024 });
|
|
@@ -771,14 +1099,17 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
771
1099
|
: localCommand);
|
|
772
1100
|
const podSweep = async (tree: string, logDir: string | undefined, pod: NonNullable<GateOptions['pod']>, only?: readonly string[]): Promise<CommandResult> => {
|
|
773
1101
|
const shardCount = pod.shards ?? 24;
|
|
1102
|
+
const isolationConcurrency = pod.isolationConcurrency ?? 4;
|
|
774
1103
|
// GATE-TIMEOUT-HEAL ①: the configured value is the floor; each shard's own deadline is shardDeadlineSeconds(planned).
|
|
775
1104
|
const deadlineSeconds = pod.shardTimeoutSeconds ?? POD_SHARD_TIMEOUT_FLOOR_SECONDS;
|
|
776
1105
|
if (!pod.pool || !Number.isSafeInteger(shardCount) || shardCount < 1
|
|
777
|
-
|| !Number.isSafeInteger(deadlineSeconds) || deadlineSeconds < 1
|
|
1106
|
+
|| !Number.isSafeInteger(deadlineSeconds) || deadlineSeconds < 1
|
|
1107
|
+
|| !Number.isSafeInteger(isolationConcurrency) || isolationConcurrency < 1) throw new Error('invalid pod sweep options');
|
|
778
1108
|
const head = await command('git', ['rev-parse', 'HEAD'], tree);
|
|
779
1109
|
check(head, 'cut tree HEAD');
|
|
780
1110
|
const commit = head.output.trim();
|
|
781
1111
|
if (!/^[0-9a-f]{40}$/.test(commit)) throw new Error('invalid cut tree HEAD');
|
|
1112
|
+
const selectionStart = Date.now();
|
|
782
1113
|
const listed = await command('git', ['ls-files', '*.test.*'], tree);
|
|
783
1114
|
check(listed, 'git ls-files tests');
|
|
784
1115
|
const allFiles = listed.output.split(/\r?\n/).filter((file) => /\.test\.(?:tsx?|jsx?|mts|cts)$/.test(file)).sort();
|
|
@@ -807,11 +1138,12 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
807
1138
|
// The TD1 measurements predate the fixture-only gate; do not isolate those five files at their old 2–13 GB peaks.
|
|
808
1139
|
const fixtureOnly = new Set([...GATE_NIGHTLY_AUDITS, 'test/guardian/dispatch-surface-contract.test.ts', 'test/user-config-mcp.test.ts']);
|
|
809
1140
|
const heavy = new Set(assignable.filter((file) => !fixtureOnly.has(file) && (memory.get(file) ?? 0) >= POD_HEAVY_FILE_MB));
|
|
810
|
-
const
|
|
1141
|
+
const { shards: lightShards, spawnFiles: spawnLane, spawnShardCount } = planGateSpawnSeparatedShards(assignable.filter((file) => !heavy.has(file)), durations, shardCount);
|
|
811
1142
|
const shards = [
|
|
812
|
-
...
|
|
1143
|
+
...lightShards,
|
|
813
1144
|
...[...heavy].map((file) => ({ files: [file], plannedSeconds: durations.get(file) ?? 0 })),
|
|
814
1145
|
];
|
|
1146
|
+
if (spawnLane.length) debug.log('release-loop.gate', 'pod-spawn-lane', { files: spawnLane, shards: spawnShardCount });
|
|
815
1147
|
logPlanWalltime(planWall, shards);
|
|
816
1148
|
if (heavy.size) debug.log('release-loop.gate', 'pod-heavy-alone', { files: [...heavy], thresholdMb: POD_HEAVY_FILE_MB, memoryLimit: POD_ISOLATION_MEMORY_LIMIT });
|
|
817
1149
|
const known = assignable.filter((file) => durations.has(file)).length;
|
|
@@ -820,9 +1152,13 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
820
1152
|
maxPlannedSeconds: Math.max(...shards.map((shard) => shard.plannedSeconds)),
|
|
821
1153
|
source: durations.size ? pod.durationSource : 'none',
|
|
822
1154
|
});
|
|
1155
|
+
reportSelectionTimed(Date.now() - selectionStart, assignable.length, pod.onSelectionTimed);
|
|
823
1156
|
// GATE-SPEED A3①: shard Pod CPU (`release.loop.gatePodCpu` · default request 1 / limit 4) — shards × request must fit the pool.
|
|
824
1157
|
const cpu = resolvePodCpu(pod.cpu);
|
|
1158
|
+
// Only evidence-backed opt-in files reach this batch; every other file retains its own Pod below.
|
|
1159
|
+
const batchWidth = Math.max(1, Math.min(4, Math.floor(podCpuMillis(cpu.limit)! / 1000)));
|
|
825
1160
|
debug.log('release-loop.gate', 'shard-resources', { shards: shards.length, cpuRequest: cpu.request, cpuLimit: cpu.limit, cpuSource: pod.cpu ? 'config' : 'default' });
|
|
1161
|
+
debug.log('release-loop.gate', 'isolation-batch-width', { batchWidth, cpuLimit: cpu.limit });
|
|
826
1162
|
const ready = poolOverride ? undefined : checkPodPool(parsePodPool(pod.pool), defaultKubectl);
|
|
827
1163
|
if (ready && !ready.ok) throw new Error('pod command: 풀의 노드가 하나도 준비되지 않았다');
|
|
828
1164
|
const poolScheduler = poolOverride ?? new PodPoolScheduler(ready!.ready);
|
|
@@ -861,6 +1197,19 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
861
1197
|
});
|
|
862
1198
|
const stallWatch = stallSeconds > 0 && !!liveLog;
|
|
863
1199
|
debug.log('release-loop.gate', 'pod-stall-watch', { stallSeconds, source: pod.stallSeconds !== undefined ? 'config' : 'default', enabled: stallWatch });
|
|
1200
|
+
// GATE-POD-STALL-FAILFAST ①: a hang after test output began ends only the Pod's test process group (never the Job).
|
|
1201
|
+
const midRunStallSeconds = pod.midRunStallSeconds ?? GATE_POD_MIDRUN_STALL_SECONDS_DEFAULT;
|
|
1202
|
+
if (!Number.isFinite(midRunStallSeconds) || midRunStallSeconds < 0) throw new Error('invalid pod sweep options');
|
|
1203
|
+
const abortTests = pod.abortTests ?? (poolOverride ? undefined : (job: { name: string; context?: string }, reason: string) => {
|
|
1204
|
+
if (!job.context) return false;
|
|
1205
|
+
const ran = defaultKubectl(['--context', job.context, '-n', 'elanous-test', 'exec', `job/${job.name}`, '-c', 'child', '--', 'bash', '-c', gateShardAbortScript(reason)]);
|
|
1206
|
+
if (ran.status !== 0) debug.log('release-loop.gate', 'shard-abort-failed', { job: job.name, rc: ran.status, out: `${ran.stdout}${ran.stderr}`.trim().slice(0, 300) }, { level: 'warn' });
|
|
1207
|
+
return ran.status === 0;
|
|
1208
|
+
});
|
|
1209
|
+
const midRunWatch = stallWatch && midRunStallSeconds > 0 && !!abortTests;
|
|
1210
|
+
debug.log('release-loop.gate', 'pod-midrun-stall-watch', { midRunStallSeconds, source: pod.midRunStallSeconds !== undefined ? 'config' : 'default', enabled: midRunWatch });
|
|
1211
|
+
/** Shard Pods the mid-run watch aborted in this sweep (the verdict summary's «중단 조각 n»). */
|
|
1212
|
+
let abortedShards = 0;
|
|
864
1213
|
// GATE-MULTI-HOST: members found unreachable during this gate — skipped for placement while any other member is left.
|
|
865
1214
|
const downMembers = new Set<string>();
|
|
866
1215
|
const multiHost = poolScheduler.members.length > 1;
|
|
@@ -894,7 +1243,23 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
894
1243
|
type ShardReason = NonNullable<GateResult['stalledShards']>[number]['reason'];
|
|
895
1244
|
// 깊이 끝 OOM 조각은 «조각 전체»를 파일 하나씩 격리한다 — 상한 40 이면 나머지가 못 잰 채 남았다(09-30 ① 7번 조각 110 파일).
|
|
896
1245
|
const isolationRemaining = shards.map((item) => item.files.length);
|
|
1246
|
+
// One sweep-wide permit pool: independent root shards must not each launch their own unbounded file fan-out.
|
|
1247
|
+
let activeIsolation = 0;
|
|
1248
|
+
const isolationWaiters: Array<() => void> = [];
|
|
1249
|
+
const isolatedFile = async <T>(run: () => Promise<T>): Promise<T> => {
|
|
1250
|
+
if (activeIsolation < isolationConcurrency) activeIsolation++;
|
|
1251
|
+
else await new Promise<void>((resolve) => isolationWaiters.push(resolve));
|
|
1252
|
+
try { return await run(); }
|
|
1253
|
+
finally {
|
|
1254
|
+
const next = isolationWaiters.shift();
|
|
1255
|
+
if (next) next(); // transfer this permit directly; queued callers cannot exceed the cap
|
|
1256
|
+
else activeIsolation--;
|
|
1257
|
+
}
|
|
1258
|
+
};
|
|
1259
|
+
debug.log('release-loop.gate', 'pod-isolation-concurrency', { limit: isolationConcurrency, source: pod.isolationConcurrency === undefined ? 'default' : 'config' });
|
|
897
1260
|
const knownFailures = new Set(pod.knownFailures ?? []);
|
|
1261
|
+
const batchSafe = loadGateBatchSafe(repo);
|
|
1262
|
+
debug.log('release-loop.gate', 'batch-safe-opt-in', { files: batchSafe.size });
|
|
898
1263
|
const estimated = (paths: string[]) => paths.reduce((sum, file) => {
|
|
899
1264
|
const value = durations.get(file);
|
|
900
1265
|
return sum + (value !== undefined && Number.isFinite(value) && value >= 0 ? value : unknownSeconds);
|
|
@@ -935,23 +1300,29 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
935
1300
|
},
|
|
936
1301
|
release: (member: Parameters<PodPoolScheduler['release']>[0]) => poolScheduler.release(member),
|
|
937
1302
|
}) as unknown as PodPoolScheduler;
|
|
938
|
-
const runShard = async (paths: string[], shard: number, depth = 0, branch = '', retriedNoOutput = false, fresh = pod.freshShards === true, parentAttempt?: string, hostRetries = 0): Promise<{ runs: ShardRun[]; stalled: NonNullable<GateResult['stalledShards']> }> => {
|
|
1303
|
+
const runShard = async (paths: string[], shard: number, depth = 0, branch = '', retriedNoOutput = false, fresh = pod.freshShards === true, parentAttempt?: string, hostRetries = 0, batchIsolation = false): Promise<{ runs: ShardRun[]; stalled: NonNullable<GateResult['stalledShards']> }> => {
|
|
939
1304
|
// 파일별 소요는 junit 으로 남긴다 — 콘솔 요약(판정 원천)은 그대로이고, 느린 시험 목록(K10 D4)·계층 분리(D2)의 자가 된다.
|
|
940
1305
|
// GATE-TIMEOUT-HEAL: parts (each with its own junit) under a soft deadline inside this shard's own Job deadline.
|
|
941
|
-
const shardDeadline = shardDeadlineSeconds(estimated(paths), deadlineSeconds);
|
|
942
|
-
const parts = shardParts(paths, (file) => estimated([file]));
|
|
943
|
-
const memoryLimit = paths.length === 1 && (depth >= 1 || heavy.has(paths[0]!)) ? POD_ISOLATION_MEMORY_LIMIT : undefined;
|
|
944
|
-
const shellOptions = { parts, cdpPatterns, softSeconds: softDeadlineSeconds(shardDeadline), cachePrefix, installSlots };
|
|
1306
|
+
const shardDeadline = shardDeadlineSeconds(batchIsolation ? Math.max(...paths.map((file) => estimated([file]))) * Math.ceil(paths.length / batchWidth) : estimated(paths), deadlineSeconds);
|
|
1307
|
+
const parts = batchIsolation ? paths.map((file) => [file]) : shardParts(paths, (file) => estimated([file]));
|
|
1308
|
+
const memoryLimit = batchIsolation || paths.length === 1 && (depth >= 1 || heavy.has(paths[0]!)) ? POD_ISOLATION_MEMORY_LIMIT : undefined;
|
|
1309
|
+
const shellOptions = { parts, cdpPatterns, softSeconds: softDeadlineSeconds(shardDeadline), cachePrefix, installSlots, ...(batchIsolation ? { parallelParts: batchWidth } : {}) };
|
|
945
1310
|
const keyShell = gateShardShell(shellOptions);
|
|
946
1311
|
// The stall beat only reports; runKey keeps hashing the shell without it, so earlier results stay reusable.
|
|
947
|
-
const podShell = stallWatch ? gateShardShell({ ...shellOptions, beatSeconds: gateStallBeatSeconds(stallSeconds) }) : keyShell;
|
|
948
1312
|
const runKey = createHash('sha256').update(JSON.stringify({ pool: pod.pool, podShell: keyShell, memoryLimit: memoryLimit ?? null, deadlineSeconds: shardDeadline, bunCache: bunCache ?? null })).digest('hex');
|
|
949
1313
|
let start = Date.now();
|
|
950
1314
|
let admitted: GateAdmissionResult | undefined;
|
|
951
1315
|
let output: string | undefined;
|
|
1316
|
+
const phaseTimes: Record<string, string> = {};
|
|
1317
|
+
const stamp = (phase: string) => { phaseTimes[phase] = new Date().toISOString(); };
|
|
1318
|
+
let podStages: Record<string, number> | undefined;
|
|
1319
|
+
let podLifecycle: Record<string, string> | undefined;
|
|
1320
|
+
let podJobName: string | undefined;
|
|
1321
|
+
let batchArtifactsComplete = true;
|
|
952
1322
|
let junit: string | undefined;
|
|
953
1323
|
let rc: number | undefined;
|
|
954
1324
|
let jobExitCode: number | undefined;
|
|
1325
|
+
let batchArtifactDir: string | undefined;
|
|
955
1326
|
let lastFile: string | undefined;
|
|
956
1327
|
let jobFailed = false;
|
|
957
1328
|
let jobError = '';
|
|
@@ -962,22 +1333,30 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
962
1333
|
let unfinishedLog: string | undefined;
|
|
963
1334
|
let exitCodeOnly: string[] | undefined;
|
|
964
1335
|
let stall: { seconds: number; sinceCheckoutMs: number } | undefined;
|
|
1336
|
+
let stallWatchState: ShardStallWatchState | undefined;
|
|
1337
|
+
let abort: ShardAbort | undefined;
|
|
965
1338
|
// The way down a previous attempt took wins over its own log: that log was not a verdict, or it would have no children.
|
|
966
1339
|
// The split this shard would take if it failed here — the same rule as the split code below (isolation at depth 2 ·
|
|
967
1340
|
// cap chunks for a big root shard · halves otherwise · none for one file).
|
|
968
1341
|
// Isolation follows only while this shard's isolation budget still covers every file (review r7) — the budget is the
|
|
969
1342
|
// shard's file count and isolations take disjoint files, so this holds unless the plan changed; if not, run the Pod.
|
|
970
|
-
const splitChild = depth === 2 && paths.length > 1 ? (isolationRemaining[shard]! >= paths.length
|
|
1343
|
+
const splitChild = depth === 2 && paths.length > 1 ? (isolationRemaining[shard]! >= paths.length
|
|
1344
|
+
? batchIsolation || !paths.every((file) => batchSafe.has(file)) ? '-file-0' : '-batch' : undefined)
|
|
971
1345
|
: depth < 2 && paths.length > 1 ? (depth === 0 && paths.length > POD_SHARD_FILE_CAP ? '-c0' : '-0') : undefined;
|
|
972
1346
|
const found = !fresh && logDir ? previousShardDescent(logDir, `pod-${shard}${branch}`, commit, paths, runKey, parentAttempt, splitChild) : undefined;
|
|
973
1347
|
const descent = found?.way;
|
|
974
1348
|
const reused = !fresh && !descent && logDir ? previousShardResult(logDir, `pod-${shard}${branch}`, commit, paths, runKey, parentAttempt) : undefined;
|
|
975
1349
|
// This shard's attempt id: the recorded one when its way down is followed, a new one when a Pod runs. Children carry it.
|
|
976
1350
|
const attemptId = found?.attemptId ?? randomUUID();
|
|
1351
|
+
// The Pod itself stamps its cut and attempt before the test starts. A
|
|
1352
|
+
// collector-created manifest cannot reassign another Job's bytes.
|
|
1353
|
+
const podShell = gateShardShell({ ...shellOptions,
|
|
1354
|
+
...(stallWatch ? { beatSeconds: gateStallBeatSeconds(stallSeconds) } : {}),
|
|
1355
|
+
sourceBinding: { cutSha: commit, attemptId } });
|
|
977
1356
|
if (descent === 'retry') {
|
|
978
1357
|
// «descend» is only the way down; «resumed» comes from each leaf whose log reads cleanly (review r4).
|
|
979
1358
|
debug.log('release-loop.gate', 'pod-shard-descend', { shard, branch, files: paths.length, via: 'retry' });
|
|
980
|
-
return runShard(paths, shard, depth, `${branch}-retry`, true, undefined, attemptId);
|
|
1359
|
+
return runShard(paths, shard, depth, `${branch}-retry`, true, undefined, attemptId, hostRetries, batchIsolation);
|
|
981
1360
|
}
|
|
982
1361
|
if (descent === 'split') {
|
|
983
1362
|
output = undefined;
|
|
@@ -989,7 +1368,9 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
989
1368
|
jobExitCode = 0;
|
|
990
1369
|
lastFile = lastStartedTestFile(output, paths);
|
|
991
1370
|
} else try {
|
|
1371
|
+
stamp('admissionStart');
|
|
992
1372
|
admitted = await admitGateShard(gateAdmission, { waitMs: admissionWaitSeconds * 1000, onWait: (reason) => markShard(shard, { waitReason: reason }) });
|
|
1373
|
+
stamp('admissionEnd');
|
|
993
1374
|
if (admitted.outcome !== 'off') debug.log('release-loop.gate', 'shard-admission', { shard, branch, outcome: admitted.outcome, waitedMs: admitted.waitedMs, ...(admitted.reason ? { reason: admitted.reason } : {}) },
|
|
994
1375
|
admitted.outcome === 'granted' ? undefined : { level: 'warn' });
|
|
995
1376
|
// The admission wait is not the shard's run time: deadline/timeout judgment and the measured duration start here.
|
|
@@ -1004,10 +1385,20 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1004
1385
|
stopJob: () => {
|
|
1005
1386
|
try { return stopJob(liveJob()); }
|
|
1006
1387
|
catch (error) { debug.log('release-loop.gate', 'pod-shard-stall-stop-failed', { job: jobName, error: String(error) }, { level: 'warn' }); return false; }
|
|
1007
|
-
}
|
|
1388
|
+
},
|
|
1389
|
+
...(midRunWatch ? {
|
|
1390
|
+
midRunStallSeconds,
|
|
1391
|
+
onAbort: ({ sinceLastOutputMs }: { sinceLastOutputMs: number }) => debug.log('release-loop.gate', 'shard-abort',
|
|
1392
|
+
{ shard, branch, reason: `mid-run-stall ${midRunStallSeconds}s`, by: 'auto', sinceLastOutputMs }, { level: 'warn' }),
|
|
1393
|
+
abortTests: () => {
|
|
1394
|
+
try { return abortTests!(liveJob(), `mid-run-stall ${midRunStallSeconds}s`); }
|
|
1395
|
+
catch (error) { debug.log('release-loop.gate', 'shard-abort-failed', { job: jobName, error: String(error) }, { level: 'warn' }); return false; }
|
|
1396
|
+
},
|
|
1397
|
+
} : {}) }) : undefined;
|
|
1008
1398
|
const placed = admitted.release;
|
|
1009
1399
|
const job = await podCommand({
|
|
1010
|
-
pool: pod.pool, poolScheduler: trackedPool(shard, depth > 0 || branch !== '', (member) => { jobContext = member.context; placedOn = member.context; placed(); }), clone: true, source: { kind: 'commit', sha: commit }, deadlineSeconds: shardDeadline,
|
|
1400
|
+
pool: pod.pool, poolScheduler: trackedPool(shard, depth > 0 || branch !== '', (member) => { stamp('poolPlaced'); jobContext = member.context; placedOn = member.context; placed(); }), clone: true, source: { kind: 'commit', sha: commit }, deadlineSeconds: shardDeadline,
|
|
1401
|
+
...gateHostMirror(pod),
|
|
1011
1402
|
...(multiHost ? { statusUnreadableMs: memberUnreadableSeconds * 1000 } : {}),
|
|
1012
1403
|
...(bunCache ? { bunCache } : {}),
|
|
1013
1404
|
...(installSlots > 0 ? { installSlots: POD_INSTALL_SLOTS_HOST_PATH } : {}),
|
|
@@ -1017,11 +1408,37 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1017
1408
|
name: jobName,
|
|
1018
1409
|
command: ['bash', '-lc', podShell],
|
|
1019
1410
|
}).finally(() => watch?.stop());
|
|
1411
|
+
stamp('jobReturned');
|
|
1412
|
+
podJobName = job.job;
|
|
1413
|
+
if (!poolOverride && placedOn) {
|
|
1414
|
+
try {
|
|
1415
|
+
const got = defaultKubectl(['--context', placedOn, '-n', 'elanous-test', 'get', 'pods', '-l', `job-name=${job.job}`, '-o', 'json']);
|
|
1416
|
+
if (got.status === 0) podLifecycle = readGatePodLifecycle(got.stdout);
|
|
1417
|
+
} catch { /* timings never change the verdict */ }
|
|
1418
|
+
}
|
|
1020
1419
|
jobExitCode = job.exitCode;
|
|
1420
|
+
batchArtifactDir = job.artifactsDir;
|
|
1021
1421
|
const read = readShardArtifacts(job.artifactsDir, parts);
|
|
1422
|
+
batchArtifactsComplete = !batchIsolation || parts.every((_, index) =>
|
|
1423
|
+
['rc', 'log', 'junit.xml'].every((suffix) => existsSync(join(job.artifactsDir, `part-${index}.${suffix}`))));
|
|
1424
|
+
const stagesPath = join(job.artifactsDir, 'stages.tsv');
|
|
1425
|
+
if (existsSync(stagesPath)) {
|
|
1426
|
+
try {
|
|
1427
|
+
podStages = Object.fromEntries(readFileSync(stagesPath, 'utf8').split(/\r?\n/).flatMap((line) => {
|
|
1428
|
+
const match = /^(\d+)\t([a-z0-9-]+)$/.exec(line);
|
|
1429
|
+
return match ? [[match[2]!, Number(match[1])]] : [];
|
|
1430
|
+
}));
|
|
1431
|
+
} catch (error) {
|
|
1432
|
+
debug.log('release-loop.gate', 'pod-stages-unreadable', { shard, branch, error: String(error) }, { level: 'warn' });
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
stamp('artifactsRead');
|
|
1022
1436
|
({ output, junit, rc, timedOut, finished, unfinishedLog, exitCodeOnly } = read);
|
|
1023
1437
|
// A stall counts only when the Job also left no verdict (a failed stop can let a late shard finish after all).
|
|
1024
1438
|
const stalledAt = watch?.stalled();
|
|
1439
|
+
stallWatchState = watch?.state();
|
|
1440
|
+
abort = watch?.aborted();
|
|
1441
|
+
if (abort?.taken) abortedShards++;
|
|
1025
1442
|
if (stalledAt && !finished.length && rc === undefined) stall = { seconds: stallSeconds, sinceCheckoutMs: stalledAt.sinceCheckoutMs };
|
|
1026
1443
|
// A Job the cluster cut at its own deadline leaves no outbox at all: past the deadline with nothing is a timeout too.
|
|
1027
1444
|
if (!timedOut && output === undefined && job.exitCode !== 0 && Date.now() - start >= shardDeadline * 1000) timedOut = true;
|
|
@@ -1039,15 +1456,26 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1039
1456
|
const durationMs = Date.now() - start;
|
|
1040
1457
|
const fileSeconds = junit === undefined ? undefined : Object.fromEntries(readJunitFileSeconds(junit));
|
|
1041
1458
|
const testSeconds = fileSeconds === undefined ? undefined : Object.values(fileSeconds).reduce((sum, seconds) => sum + seconds, 0);
|
|
1042
|
-
const
|
|
1459
|
+
const testWallSeconds = batchIsolation
|
|
1460
|
+
? gateBatchTestWallSeconds(podStages, parts.length)
|
|
1461
|
+
: podStages?.['test-start'] !== undefined && podStages?.['test-end'] !== undefined
|
|
1462
|
+
? Math.max(0, podStages['test-end'] - podStages['test-start']) : undefined;
|
|
1463
|
+
const podOverheadSeconds = testWallSeconds !== undefined || (!batchIsolation && testSeconds !== undefined)
|
|
1464
|
+
? Math.max(0, durationMs / 1000 - (testWallSeconds ?? testSeconds!)) : undefined;
|
|
1043
1465
|
// A reused shard keeps its files as they are — rewriting would replace the measured durationMs the next release plans by.
|
|
1044
1466
|
if (logDir && !reused && !descent) {
|
|
1045
1467
|
mkdirSync(logDir, { recursive: true });
|
|
1468
|
+
const warmPrefix = join(logDir, `pod-${shard}${branch}`);
|
|
1469
|
+
rmSync(`${warmPrefix}.warm-manifest.json`, { force: true });
|
|
1470
|
+
rmSync(`${warmPrefix}.rc`, { force: true });
|
|
1046
1471
|
// GATE-SHARD-RESUME: the json is what makes a log reusable, so it goes first and comes back last — a stop between
|
|
1047
1472
|
// the new log and the new json leaves no json, never the old Job's exit paired with the new log (review r8).
|
|
1048
1473
|
rmSync(join(logDir, `pod-${shard}${branch}.json`), { force: true });
|
|
1049
1474
|
const destination = join(logDir, `pod-${shard}${branch}.log`);
|
|
1050
|
-
|
|
1475
|
+
const unfinishedLogBytes = unfinishedLog !== undefined ? Buffer.byteLength(unfinishedLog, 'utf8') : undefined;
|
|
1476
|
+
// GATE-UNFINISHED-HINT: pod-N.log holds finished parts only — point a reader at the running parts' output (file only; the verdict reads `output`).
|
|
1477
|
+
if (output !== undefined) writeFileSync(destination, unfinishedLogBytes !== undefined
|
|
1478
|
+
? `${output}${output.endsWith('\n') || output === '' ? '' : '\n'}${gateUnfinishedHint(`pod-${shard}${branch}.unfinished.log`, unfinishedLogBytes)}\n` : output, { mode: 0o600 });
|
|
1051
1479
|
else rmSync(destination, { force: true });
|
|
1052
1480
|
const junitDestination = join(logDir, `pod-${shard}${branch}.junit.xml`);
|
|
1053
1481
|
if (junit !== undefined) writeFileSync(junitDestination, junit, { mode: 0o600 });
|
|
@@ -1057,14 +1485,29 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1057
1485
|
else rmSync(unfinishedDestination, { force: true });
|
|
1058
1486
|
writeFileSync(join(logDir, `pod-${shard}${branch}.json`), JSON.stringify({ durationMs, rc: rc ?? null,
|
|
1059
1487
|
...(stall ? { reason: 'stall', finishedFiles: 0, stallSeconds: stall.seconds, sinceCheckoutMs: stall.sinceCheckoutMs } : timedOut ? { reason: 'timeout', finishedFiles: finished.length } : {}), files: paths, plannedSeconds: estimated(paths), deadlineSeconds: shardDeadline, shardCount: shards.length, commit, runKey, attemptId, ...(parentAttempt ? { parentAttempt } : {}), jobExitCode: jobExitCode ?? null, jobFailed, ...(placedOn ? { host: placedOn } : {}),
|
|
1060
|
-
...(
|
|
1061
|
-
...(
|
|
1488
|
+
...(Object.keys(phaseTimes).length ? { phaseTimes } : {}), ...(podStages ? { podStages } : {}),
|
|
1489
|
+
...(podLifecycle && Object.keys(podLifecycle).length ? { podLifecycle } : {}),
|
|
1490
|
+
...(fileSeconds !== undefined ? { fileSeconds, testSeconds } : {}), ...(testWallSeconds !== undefined ? { testWallSeconds } : {}),
|
|
1491
|
+
...(podOverheadSeconds !== undefined ? { podOverheadSeconds } : {}),
|
|
1492
|
+
...(unfinishedLogBytes !== undefined ? { unfinishedLogBytes } : {}),
|
|
1493
|
+
...(admitted && admitted.outcome !== 'off' ? { admission: admitted.outcome, admissionWaitMs: admitted.waitedMs } : {}),
|
|
1494
|
+
...(abort ? { abort: { reason: `abort: ${abort.reason}`, at: abort.at, by: abort.by, sinceLastOutputMs: abort.sinceLastOutputMs, taken: abort.taken } } : {}),
|
|
1495
|
+
stallWatch: shardStallWatchRecord(stallSeconds, !!liveLog, stallWatchState) }) + '\n', { mode: 0o600 });
|
|
1496
|
+
if (paths.length === 1 && (rc === 0 || rc === 1) && !jobFailed && jobExitCode === 0
|
|
1497
|
+
&& output !== undefined && junit !== undefined) {
|
|
1498
|
+
writeFileSync(`${warmPrefix}.rc`, `${rc}\n`, { mode: 0o600 });
|
|
1499
|
+
writeWarmJobManifest(warmPrefix, paths[0]!, commit, attemptId);
|
|
1500
|
+
}
|
|
1062
1501
|
}
|
|
1063
1502
|
const installSeconds = parseInstallSeconds(output ?? '');
|
|
1064
1503
|
// GATE-TIMEOUT-HEAL ③: a cut shard is «reason: timeout» (with the files that still finished), not a bare rc=null.
|
|
1065
1504
|
if (!reused && !descent) debug.log('release-loop.gate', 'pod-shard', { shard, files: paths, durationMs, rc: rc ?? null, attempt: depth + 1, installSeconds, gateDeps: parseGateDeps(output ?? ''), deadlineSeconds: shardDeadline, parts: parts.length, ...(placedOn ? { host: placedOn } : {}),
|
|
1066
1505
|
...(testSeconds !== undefined ? { testSeconds } : {}),
|
|
1067
1506
|
...(stall ? { reason: 'stall', finishedFiles: 0, stallSeconds: stall.seconds } : timedOut ? { reason: 'timeout', finishedFiles: finished.length } : {}) });
|
|
1507
|
+
// POD-LIFECYCLE-STAGES: the same values the shard json carries, as one `pod.lifecycle stages` line (observation only) —
|
|
1508
|
+
// only for a Job that came back (a launch that failed before the Job returned has no Pod to describe).
|
|
1509
|
+
if (!reused && !descent && podJobName !== undefined) emitPodLifecycle(gatePodLifecycleEvent({ runId: process.env.ELANOUS_RUN_ID ?? null, podJob: podJobName, host: placedOn ?? null,
|
|
1510
|
+
shard: `pod-${shard}${branch}`, attemptId, phaseTimes, podStages, podLifecycle, outcome: gatePodOutcome({ jobFailed, stalled: !!stall, timedOut, rc }) }));
|
|
1068
1511
|
if (!reused && !descent) markShard(shard, { ...(installSeconds !== null ? { installSec: installSeconds } : {}), ...(rc !== undefined ? { rc } : {}), ...(timedOut ? { state: 'timeout' as const } : {}) });
|
|
1069
1512
|
let clean = output?.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '');
|
|
1070
1513
|
const ran = clean && lastMatch(clean, /Ran (\d+) tests? across (\d+) files?/);
|
|
@@ -1078,6 +1521,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1078
1521
|
// GATE-POD-STALL-FAILFAST: stopped with no test output — the existing «no output» reason (one fresh retry).
|
|
1079
1522
|
else if (stall) reason = 'no-output';
|
|
1080
1523
|
else if (timedOut) reason = 'timeout';
|
|
1524
|
+
else if (batchIsolation && !batchArtifactsComplete) reason = 'incomplete';
|
|
1081
1525
|
// GATE-POD-STALL-FAILFAST (OP): a complete summary with 0 fail · 0 error but an exit other than 0/1 is an exit-code
|
|
1082
1526
|
// anomaly (0.2.23: a test left process.exitCode set → «0 fail · rc 3» read as a stalled shard). Same split/isolation
|
|
1083
1527
|
// path as job-failed — only the label differs, so the verdict names the file instead of saying «stalled».
|
|
@@ -1116,14 +1560,14 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1116
1560
|
// A reused result that the current tool cannot read cleanly is not a verdict: run that shard for real.
|
|
1117
1561
|
if (reused && reason) {
|
|
1118
1562
|
debug.log('release-loop.gate', 'pod-shard-resume-miss', { shard, branch, reason });
|
|
1119
|
-
return runShard(paths, shard, depth, branch, retriedNoOutput, true, parentAttempt);
|
|
1563
|
+
return runShard(paths, shard, depth, branch, retriedNoOutput, true, parentAttempt, hostRetries, batchIsolation);
|
|
1120
1564
|
}
|
|
1121
1565
|
// GATE-MULTI-HOST: the Job failed because its member is gone (launch failed · status unreadable) — not the shard's
|
|
1122
1566
|
// fault, so no split: the same files run again on the members left. Bounded by the roster size.
|
|
1123
1567
|
if (jobFailed && placedOn && hostRetries < poolScheduler.members.length && memberLost(placedOn, shard, jobError)
|
|
1124
1568
|
&& poolScheduler.members.some((m) => !downMembers.has(m.context))) {
|
|
1125
1569
|
debug.log('release-loop.gate', 'pod-shard-host-retry', { shard, branch, files: paths.length, lostMember: placedOn, attempt: hostRetries + 1 }, { level: 'warn' });
|
|
1126
|
-
return runShard(paths, shard, depth, `${branch}-h${hostRetries}`, retriedNoOutput, undefined, attemptId, hostRetries + 1);
|
|
1570
|
+
return runShard(paths, shard, depth, `${branch}-h${hostRetries}`, retriedNoOutput, undefined, attemptId, hostRetries + 1, batchIsolation);
|
|
1127
1571
|
}
|
|
1128
1572
|
// GATE-TIMEOUT-HEAL ④: a Pod that died before any test ran (no outbox · jobExitCode≠0 — 0.2.19: isolated retries
|
|
1129
1573
|
// ending with exit 5, the source checkout) gets the same one fresh retry as «no output» instead of stalling the sweep.
|
|
@@ -1131,13 +1575,45 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1131
1575
|
const diedBeforeTests = reason === 'job-failed' && output === undefined && !jobFailed && jobExitCode !== undefined && jobExitCode !== 0
|
|
1132
1576
|
&& jobExitCode !== 137 && (paths.length === 1 || jobExitCode === 5);
|
|
1133
1577
|
if ((reason === 'no-output' || diedBeforeTests) && !retriedNoOutput) {
|
|
1134
|
-
const retry = await runShard(paths, shard, depth, `${branch}-retry`, true, undefined, attemptId);
|
|
1578
|
+
const retry = await runShard(paths, shard, depth, `${branch}-retry`, true, undefined, attemptId, hostRetries, batchIsolation);
|
|
1135
1579
|
debug.log('release-loop.gate', 'pod-shard-retry', { shard, files: paths, reason, outcome: retry.stalled[0]?.reason ?? 'ok' });
|
|
1136
1580
|
return retry;
|
|
1137
1581
|
}
|
|
1138
1582
|
// GATE-POD-STALL-FAILFAST: a stall after the fresh retry is unmeasured — no split, no isolation: smaller Pods on the
|
|
1139
1583
|
// same stalled node only re-measure the stall (10-10: retries and splits repeated it for 2h+).
|
|
1140
1584
|
if (stall && reason === 'no-output') return { runs: [], stalled: [{ shard, files: paths, reason, stall }] };
|
|
1585
|
+
// A shared Job may finish some file parts before another part loses its JUnit/log/rc. Keep every complete part's
|
|
1586
|
+
// original verdict, including a failure, and rerun only parts without the full artifact trio.
|
|
1587
|
+
if (batchIsolation && !batchArtifactsComplete && batchArtifactDir && reason) {
|
|
1588
|
+
const measured: ShardRun[] = [];
|
|
1589
|
+
const missing: string[] = [];
|
|
1590
|
+
parts.forEach((part, index) => {
|
|
1591
|
+
const file = part[0]!;
|
|
1592
|
+
try {
|
|
1593
|
+
const prefix = join(batchArtifactDir!, `part-${index}`);
|
|
1594
|
+
const partLog = readFileSync(`${prefix}.log`, 'utf8');
|
|
1595
|
+
const partJunit = readFileSync(`${prefix}.junit.xml`, 'utf8');
|
|
1596
|
+
const partRc = Number(readFileSync(`${prefix}.rc`, 'utf8').trim());
|
|
1597
|
+
const ranPart = lastMatch(partLog, /Ran (\d+) tests? across (\d+) files?/);
|
|
1598
|
+
const passPart = summaryCount(partLog, 'pass'), failPart = summaryCount(partLog, 'fail');
|
|
1599
|
+
const errorsPart = summaryCount(partLog, 'error');
|
|
1600
|
+
if ((partRc !== 0 && partRc !== 1) || !ranPart || Number(ranPart[2]) !== 1
|
|
1601
|
+
|| !Number.isFinite(passPart) || !Number.isFinite(failPart)) throw new Error('incomplete part');
|
|
1602
|
+
const attributed = failuresOf({ rc: partRc, output: partLog }, `pod-${shard} part-${index}`);
|
|
1603
|
+
if (attributed.failures.length !== failPart) throw new Error('unattributed part');
|
|
1604
|
+
measured.push({ output: partLog, rc: partRc, pass: passPart, fail: failPart,
|
|
1605
|
+
errors: Number.isFinite(errorsPart) ? errorsPart : 0, ran: Number(ranPart[1]), files: 1,
|
|
1606
|
+
passedIds: junitPassedIds(partJunit).map(podRelative) });
|
|
1607
|
+
} catch { missing.push(file); }
|
|
1608
|
+
});
|
|
1609
|
+
if (measured.length && missing.length) {
|
|
1610
|
+
debug.log('release-loop.gate', 'isolation-batch-partial', { shard, branch, measured: measured.length, missing });
|
|
1611
|
+
const children = await Promise.all(missing.map((file) => isolatedFile(() => runShard([file], shard, depth + 1,
|
|
1612
|
+
`${branch}-file-${paths.indexOf(file)}`, false, undefined, attemptId))));
|
|
1613
|
+
return { runs: [...measured, ...children.flatMap((child) => child.runs)],
|
|
1614
|
+
stalled: children.flatMap((child) => child.stalled) };
|
|
1615
|
+
}
|
|
1616
|
+
}
|
|
1141
1617
|
if (reason) {
|
|
1142
1618
|
// GATE-SKIP-KNOWN-RETRY — a shard whose summary is complete but whose failures could not be read back (unattributed)
|
|
1143
1619
|
// and whose every named failure is one the previous release already had is not re-run: re-running only re-measures
|
|
@@ -1162,7 +1638,8 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1162
1638
|
debug.log('release-loop.gate', 'retry-narrowed', { shard, before: paths.length, after: narrowed.length });
|
|
1163
1639
|
const kept = new Set(paths.filter((file) => !narrowed.includes(file)));
|
|
1164
1640
|
const passedIds = junitPassedIds(junit!).map(podRelative).filter((id) => { try { return kept.has(testFileOfId(id)); } catch { return false; } });
|
|
1165
|
-
const
|
|
1641
|
+
const directBatch = narrowed.length > 1 && narrowed.length <= POD_SHARD_FILE_CAP && narrowed.every((file) => batchSafe.has(file));
|
|
1642
|
+
const child = await runShard(narrowed, shard, directBatch ? 2 : depth, `${branch}-n`, false, undefined, attemptId, 0, directBatch);
|
|
1166
1643
|
return { runs: [{ output: '', rc: 0, pass: passedIds.length, fail: 0, errors: 0, ran: passedIds.length, files: kept.size, passedIds },
|
|
1167
1644
|
...child.runs], stalled: child.stalled };
|
|
1168
1645
|
}
|
|
@@ -1171,19 +1648,27 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1171
1648
|
if (depth === 2 && paths.length > 1) {
|
|
1172
1649
|
const count = Math.min(paths.length, isolationRemaining[shard]!);
|
|
1173
1650
|
isolationRemaining[shard]! -= count;
|
|
1174
|
-
const
|
|
1175
|
-
|
|
1651
|
+
const batchPaths = paths.slice(0, count);
|
|
1652
|
+
// The common path already ran every file as an isolated part in this same Job. Only an incomplete batch
|
|
1653
|
+
// needs a fresh per-file Pod, preserving the earlier fail-closed behavior when the shared Job died.
|
|
1654
|
+
const canBatch = batchPaths.length > 1 && batchPaths.every((file) => batchSafe.has(file));
|
|
1655
|
+
const batch = batchIsolation || !canBatch ? undefined : await runShard(batchPaths, shard, depth + 1, `${branch}-batch`, false, undefined, attemptId, 0, true);
|
|
1656
|
+
const retryPaths = !batch ? batchPaths : batchPaths.filter((file) => batch.stalled.some((item) => item.files.includes(file)));
|
|
1657
|
+
const children = !batch || batch.stalled.length ? await Promise.all(retryPaths.map((file) => isolatedFile(async () => {
|
|
1658
|
+
const child = await runShard([file], shard, depth + 1, `${branch}-file-${batchPaths.indexOf(file)}`, false, undefined, attemptId);
|
|
1176
1659
|
debug.log('release-loop.gate', 'pod-shard-isolate', { shard, file, outcome: child.stalled[0]?.reason ?? 'ok' });
|
|
1177
1660
|
return child;
|
|
1178
|
-
}));
|
|
1179
|
-
|
|
1661
|
+
}))) : [batch];
|
|
1662
|
+
debug.log('release-loop.gate', 'pod-shard-isolate-batch', { shard, files: batchPaths.length, fallback: !batch || batch.stalled.length > 0 });
|
|
1663
|
+
return { runs: [...(batch?.stalled.length ? batch.runs : []), ...children.flatMap((child) => child.runs)], stalled: [
|
|
1180
1664
|
...children.flatMap((child) => child.stalled),
|
|
1665
|
+
...(batch?.stalled.filter((item) => item.files.some((file) => !retryPaths.includes(file))) ?? []),
|
|
1181
1666
|
...(paths.length > count ? [{ shard, files: paths.slice(count), reason,
|
|
1182
1667
|
...(lastFile ? { lastFile } : {}),
|
|
1183
1668
|
}] : []),
|
|
1184
1669
|
] };
|
|
1185
1670
|
}
|
|
1186
|
-
if (depth
|
|
1671
|
+
if (depth >= 2 || paths.length === 1) return { runs: [], stalled: [{ shard, files: paths, reason,
|
|
1187
1672
|
...(lastFile ? { lastFile } : {}),
|
|
1188
1673
|
...(reason === 'unattributed' ? { summaryFailures: fails, namedFailures, ...(unattributedDetail ? { detail: unattributedDetail } : {}) } : {}),
|
|
1189
1674
|
}] };
|
|
@@ -1193,14 +1678,19 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1193
1678
|
if (depth === 0 && paths.length > POD_SHARD_FILE_CAP) {
|
|
1194
1679
|
const chunks = Array.from({ length: Math.ceil(paths.length / POD_SHARD_FILE_CAP) }, (_, i) => paths.slice(i * POD_SHARD_FILE_CAP, (i + 1) * POD_SHARD_FILE_CAP));
|
|
1195
1680
|
debug.log('release-loop.gate', 'pod-shard-cap-split', { shard, files: paths.length, chunks: chunks.length, cap: POD_SHARD_FILE_CAP, reason });
|
|
1196
|
-
|
|
1681
|
+
// Preserve the proven cap-chunk Job even with an empty opt-in list. Batch isolation is only for chunks whose
|
|
1682
|
+
// every file is explicitly safe; a regular cap chunk keeps the preexisting one-process Pod verdict.
|
|
1683
|
+
const children = await Promise.all(chunks.map((chunk, i) => runShard(chunk, shard, 2, `${branch}-c${i}`, false, undefined, attemptId, 0,
|
|
1684
|
+
chunk.length > 1 && chunk.every((file) => batchSafe.has(file)))));
|
|
1197
1685
|
return { runs: children.flatMap((child) => child.runs), stalled: children.flatMap((child) => child.stalled) };
|
|
1198
1686
|
}
|
|
1199
1687
|
debug.log('release-loop.gate', 'pod-shard-split', { shard, depth, files: paths, reason });
|
|
1200
1688
|
const middle = Math.ceil(paths.length / 2);
|
|
1201
1689
|
const children = await Promise.all([
|
|
1202
|
-
runShard(paths.slice(0, middle), shard, depth + 1, `${branch}-0`, false, undefined, attemptId
|
|
1203
|
-
|
|
1690
|
+
runShard(paths.slice(0, middle), shard, depth + 1, `${branch}-0`, false, undefined, attemptId, 0,
|
|
1691
|
+
depth + 1 === 2 && middle > 1 && paths.slice(0, middle).every((file) => batchSafe.has(file))),
|
|
1692
|
+
runShard(paths.slice(middle), shard, depth + 1, `${branch}-1`, false, undefined, attemptId, 0,
|
|
1693
|
+
depth + 1 === 2 && paths.length - middle > 1 && paths.slice(middle).every((file) => batchSafe.has(file))),
|
|
1204
1694
|
]);
|
|
1205
1695
|
return { runs: children.flatMap((child) => child.runs), stalled: children.flatMap((child) => child.stalled) };
|
|
1206
1696
|
}
|
|
@@ -1233,16 +1723,62 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1233
1723
|
if (unmeasured.length) debug.log('release-loop.gate', 'pod-sweep-unmeasured', { shards: [...new Set(unmeasured.map((item) => item.shard))], files: unmeasured.reduce((sum, item) => sum + item.files.length, 0),
|
|
1234
1724
|
stallSeconds, measuredFiles: total.files }, { level: 'warn' });
|
|
1235
1725
|
// An unmeasured (stalled) shard is never handed to the host re-run as an environment stall: the gate fails now.
|
|
1236
|
-
if (
|
|
1726
|
+
if (abortedShards) debug.log('release-loop.gate', 'pod-sweep-aborted', { shards: abortedShards, midRunStallSeconds }, { level: 'warn' });
|
|
1727
|
+
if (stalledShards.some((item) => item.reason !== 'no-output' || item.files.length !== 1 || item.stall)) throw new StalledPodShards(stalledShards, total, abortedShards);
|
|
1237
1728
|
if (total.ran === 0 && total.errors === 0 && !stalledShards.length) throw new Error('sweep incomplete: no tests ran');
|
|
1238
1729
|
const outputs = measured.map((run) => run.output.replace(/(?:^|\n)\s*\d+ (?:pass|fail|errors?)\s*(?=\n|$)/g, '\n').replace(/Ran \d+ tests? across \d+ files?\.?/g, ''));
|
|
1239
|
-
return { rc: total.fail + total.errors ? 1 : 0, output: outputs.join('\n') + `\n${total.pass} pass\n${total.fail} fail\n${total.errors} errors\nRan ${total.ran} tests across ${total.files} files.\n`,
|
|
1730
|
+
return { rc: total.fail + total.errors ? 1 : 0, output: outputs.join('\n') + `\n${total.pass} pass\n${total.fail} fail\n${total.errors} errors\nRan ${total.ran} tests across ${total.files} files.\n`, validatedShards: true,
|
|
1240
1731
|
passedIds: [...new Set(measured.flatMap((run) => run.passedIds))].sort(),
|
|
1241
|
-
stalledEnv: stalledShards.flatMap((item) => item.files.map((file) => ({ file, reason: 'no-output' as const })))
|
|
1732
|
+
stalledEnv: stalledShards.flatMap((item) => item.files.map((file) => ({ file, reason: 'no-output' as const }))),
|
|
1733
|
+
...(abortedShards ? { abortedShards } : {}) };
|
|
1734
|
+
};
|
|
1735
|
+
const runPodRecheckBatch = async (files: readonly string[], commit: string, pod: NonNullable<GateOptions['pod']>, timeoutMs: number, relaxed = false): Promise<Map<string, CommandResult>> => {
|
|
1736
|
+
if (!files.length || files.length > GATE_RECHECK_POD_BATCH_FILES || new Set(files).size !== files.length
|
|
1737
|
+
|| files.some((file) => !/^(?:[\w.-]+\/)*[\w.-]+\.test\.[cm]?[jt]sx?$/.test(file)
|
|
1738
|
+
|| file.split('/').some((part) => part === '.' || part === '..')))
|
|
1739
|
+
throw new Error('unsafe pod recheck files');
|
|
1740
|
+
if (!sha.test(commit)) throw new Error('invalid pod recheck commit');
|
|
1741
|
+
const installSlots = pod.installSlots ?? POD_INSTALL_SLOTS_DEFAULT;
|
|
1742
|
+
if (!Number.isSafeInteger(installSlots) || installSlots < 0) throw new Error('invalid pod recheck install slots');
|
|
1743
|
+
const bunCache = pod.bunCache?.trim() || process.env[POD_BUN_CACHE_HOST_PATH]?.trim();
|
|
1744
|
+
const partLimitSeconds = Math.ceil(timeoutMs / 1000);
|
|
1745
|
+
const deadlineSeconds = partLimitSeconds * Math.ceil(files.length / 2) + POD_SHARD_EMIT_RESERVE_SECONDS + 60;
|
|
1746
|
+
const shell = gateShardShell({ parts: files.map((file) => [file]), cdpPatterns: [], softSeconds: softDeadlineSeconds(deadlineSeconds),
|
|
1747
|
+
cachePrefix: bunCache ? podBunCacheVolume(bunCache).shellPrefix : '', installSlots, parallelParts: files.length > 1 ? 2 : 1,
|
|
1748
|
+
partLimitSeconds, ...(relaxed ? { testTimeoutMs: GATE_TIMEOUT_RECHECK_MS } : {}) });
|
|
1749
|
+
const job = await podCommand({ pool: pod.pool, ...(poolOverride ? { poolScheduler: poolOverride } : {}), clone: true,
|
|
1750
|
+
source: { kind: 'commit', sha: commit }, deadlineSeconds, ...(bunCache ? { bunCache } : {}), ...gateHostMirror(pod),
|
|
1751
|
+
...(installSlots > 0 ? { installSlots: POD_INSTALL_SLOTS_HOST_PATH } : {}), ...(pod.cpu ? { cpu: pod.cpu } : {}),
|
|
1752
|
+
memoryLimit: POD_ISOLATION_MEMORY_LIMIT, name: `gate-recheck-${randomUUID()}`, command: ['bash', '-lc', shell] });
|
|
1753
|
+
const artifact = (name: string) => {
|
|
1754
|
+
const path = join(job.artifactsDir, name);
|
|
1755
|
+
return existsSync(path) ? readFileSync(path, 'utf8') : undefined;
|
|
1756
|
+
};
|
|
1757
|
+
const installed = artifact('install.rc')?.trim() === '0';
|
|
1758
|
+
if (!installed || job.exitCode !== 0)
|
|
1759
|
+
throw new Error(`pod isolated recheck setup incomplete: ${commit.slice(0, 9)} job=${job.job} exit=${job.exitCode} install=${installed}`);
|
|
1760
|
+
const timedOut = artifact('shard.timeout') !== undefined;
|
|
1761
|
+
const results = new Map<string, CommandResult>();
|
|
1762
|
+
for (const [index, file] of files.entries()) {
|
|
1763
|
+
const rcText = artifact(`part-${index}.rc`)?.trim();
|
|
1764
|
+
const output = artifact(`part-${index}.log`);
|
|
1765
|
+
const rc = rcText && /^\d+$/.test(rcText) ? Number(rcText) : undefined;
|
|
1766
|
+
const partTimedOut = artifact(`part-${index}.timeout`) !== undefined;
|
|
1767
|
+
if (rc === undefined || output === undefined || partTimedOut) {
|
|
1768
|
+
if (!timedOut && !partTimedOut) throw new Error(`pod isolated recheck incomplete: ${file} @${commit.slice(0, 9)} job=${job.job} rc=${rc ?? 'missing'}`);
|
|
1769
|
+
results.set(file, { rc: 124, output: output ?? '', timedOut: true });
|
|
1770
|
+
} else results.set(file, { rc, output });
|
|
1771
|
+
debug.log('release-loop.gate', 'pod-isolated-recheck', { file, commit, job: job.job, rc: rc ?? 124, relaxed });
|
|
1772
|
+
}
|
|
1773
|
+
return results;
|
|
1242
1774
|
};
|
|
1243
1775
|
return {
|
|
1244
1776
|
command,
|
|
1245
1777
|
localCommand: commandOverride && !remote ? commandOverride : localCommand,
|
|
1778
|
+
async recheckPod(file, commit, pod, timeoutMs, relaxed = false) {
|
|
1779
|
+
return (await runPodRecheckBatch([file], commit, pod, timeoutMs, relaxed)).get(file)!;
|
|
1780
|
+
},
|
|
1781
|
+
recheckPodBatch: (files, commit, pod, timeoutMs) => runPodRecheckBatch(files, commit, pod, timeoutMs),
|
|
1246
1782
|
get remoteMirror() { return remoteMirror; },
|
|
1247
1783
|
set remoteMirror(path) { remoteMirror = path; },
|
|
1248
1784
|
async sweep(tree, logDir, pod, only) {
|
|
@@ -1354,7 +1890,7 @@ const fileOf = testFileOfId;
|
|
|
1354
1890
|
* GATE-BASELINE-CACHE (0.2.18) — the isolated `bun run test:deterministic <file>` result of one test file at one commit,
|
|
1355
1891
|
* kept in the machine ledger so a later gate does not run it again (0.2.17: the failing files were re-run on the previous
|
|
1356
1892
|
* release commit although that release had already measured them). Cache format harvested from draft #24500.
|
|
1357
|
-
* `host` is where the result was measured (`local
|
|
1893
|
+
* `host` is where the result was measured (`local`, the ssh host, or `pod:<pool>`) — another environment is a miss.
|
|
1358
1894
|
* Timeouts are never stored: an environment stall is measured again, not remembered.
|
|
1359
1895
|
*/
|
|
1360
1896
|
export interface BaselineFileResult { commit: string; file: string; host: string; failures: string[]; errors: string[]; missing: boolean }
|
|
@@ -1444,6 +1980,8 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1444
1980
|
|| (opts.baselineVersion && !versionPattern.test(opts.baselineVersion))
|
|
1445
1981
|
|| (opts.baselineCommit && !sha.test(opts.baselineCommit))) throw new Error('invalid commit or version');
|
|
1446
1982
|
if (!opts.baselineVersion) throw new Error('previous release version required (--baseline-version)');
|
|
1983
|
+
if (!Number.isSafeInteger(opts.recheckFileLimit ?? GATE_HOST_RECHECK_FILE_LIMIT) || (opts.recheckFileLimit ?? GATE_HOST_RECHECK_FILE_LIMIT) < 1)
|
|
1984
|
+
throw new Error('invalid gate recheck file limit');
|
|
1447
1985
|
if (opts.pod && (!opts.pod.pool || !Number.isSafeInteger(opts.pod.shards ?? 24) || (opts.pod.shards ?? 24) < 1
|
|
1448
1986
|
|| !Number.isSafeInteger(opts.pod.shardTimeoutSeconds ?? 1200) || (opts.pod.shardTimeoutSeconds ?? 1200) < 1)) throw new Error('invalid pod sweep options');
|
|
1449
1987
|
if (opts.remote && !/^(?:[\w.-]+@)?[\w.-]+$/.test(opts.remote)) throw new Error('invalid ssh host');
|
|
@@ -1487,6 +2025,23 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1487
2025
|
// Its logs go to cut-partial/ — the full cut/ logs stay whole (the next release reads them as a baseline source).
|
|
1488
2026
|
const partial = readPartialPlan(roots, opts.version, opts.commit);
|
|
1489
2027
|
if (partial) debug.log('release-loop.gate', 'partial-regate', { version: opts.version, priorCommit: partial.priorCommit, commit: opts.commit, files: partial.files.length });
|
|
2028
|
+
if (partial?.kind === 'carry') {
|
|
2029
|
+
// The plan alone is not evidence: the old cut and its verdict must still be complete and clean at this SHA.
|
|
2030
|
+
if (!verifyDocsOnlyCarry(repo, partial.priorCommit, opts.commit))
|
|
2031
|
+
throw new Error(`carry plan diff is not docs-only: ${partial.priorCommit}..${opts.commit}`);
|
|
2032
|
+
const trusted = roots.some((location) => {
|
|
2033
|
+
const dir = join(location, 'release', opts.version);
|
|
2034
|
+
const failuresPath = join(dir, 'gate-failures.json');
|
|
2035
|
+
const verdictPath = join(dir, 'gate-verdict.json');
|
|
2036
|
+
if (!existsSync(failuresPath) || !existsSync(verdictPath)) return false;
|
|
2037
|
+
const failures = JSON.parse(readFileSync(failuresPath, 'utf8')) as { commit?: unknown; failures?: unknown; errors?: unknown };
|
|
2038
|
+
const verdict = JSON.parse(readFileSync(verdictPath, 'utf8')) as { commit?: unknown; outcome?: unknown };
|
|
2039
|
+
return failures.commit === partial.priorCommit && Array.isArray(failures.failures) && failures.failures.length === 0
|
|
2040
|
+
&& (failures.errors === undefined || (Array.isArray(failures.errors) && failures.errors.length === 0))
|
|
2041
|
+
&& verdict.commit === partial.priorCommit && verdict.outcome === 'ok';
|
|
2042
|
+
});
|
|
2043
|
+
if (!trusted) throw new Error(`carry plan has no complete clean gate at ${partial.priorCommit}`);
|
|
2044
|
+
}
|
|
1490
2045
|
// GATE-SKIP-KNOWN-RETRY — the ids the verdict will not count as introduced (the recorded baseline ⊕ env-known), read
|
|
1491
2046
|
// before the sweep so a shard failing only with them skips its isolation retries. No recorded baseline = no skip.
|
|
1492
2047
|
let knownFailures: string[] = [];
|
|
@@ -1500,12 +2055,13 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1500
2055
|
} catch (error) { debug.log('release-loop.gate', 'known-failures-unavailable', { error: String(error) }); }
|
|
1501
2056
|
}
|
|
1502
2057
|
const durationSource = join(ledger, 'release', opts.baselineVersion, 'gate-logs', 'cut');
|
|
1503
|
-
const cutRun = await runner.sweep(cutTree, join(root, 'release', opts.version, 'gate-logs', partial ? 'cut-partial' : 'cut'),
|
|
1504
|
-
opts.pod ? { ...opts.pod, durationSource, ...(knownFailures.length ? { knownFailures } : {}),
|
|
2058
|
+
const cutRun = partial?.kind === 'carry' ? undefined : await runner.sweep(cutTree, join(root, 'release', opts.version, 'gate-logs', partial ? 'cut-partial' : 'cut'),
|
|
2059
|
+
opts.pod ? { ...opts.pod, durationSource, ...(knownFailures.length ? { knownFailures } : {}), onSelectionTimed: ({ ms }) => { result.selectionMs = ms; },
|
|
1505
2060
|
// GATE-LIVE-OBS: the release ledger's shards.json (UX #24782) — never the run tree's universe; an explicit
|
|
1506
2061
|
// --ledger-root (tests) keeps it in that ledger.
|
|
1507
2062
|
shardsFile: { path: opts.ledgerRoot ? gateShardsPath(opts.version, opts.ledgerRoot) : gateShardsPath(opts.version), version: opts.version } } : undefined,
|
|
1508
2063
|
partial?.files);
|
|
2064
|
+
if (cutRun?.abortedShards) result.abortedShards = cutRun.abortedShards;
|
|
1509
2065
|
const localStalled = async (run: CommandResult, tree: string, commit: string, label: string) => {
|
|
1510
2066
|
const stalled = run.stalledEnv ?? [];
|
|
1511
2067
|
const noCompletedShard = stalled.length && run.rc === 0
|
|
@@ -1545,7 +2101,46 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1545
2101
|
debug.log('release-loop.gate', 'pod-shard-stalled-env', { files: stalled.map(({ file, reason }) => ({ file, reason })), local });
|
|
1546
2102
|
return measured;
|
|
1547
2103
|
};
|
|
1548
|
-
const measuredCut = await localStalled(cutRun, cutTree, opts.commit, 'cut');
|
|
2104
|
+
const measuredCut = cutRun ? await localStalled(cutRun, cutTree, opts.commit, 'cut') : { failures: [], errors: [] };
|
|
2105
|
+
// A between-tests error proves only that the shared process failed. Re-run each affected file by itself using the
|
|
2106
|
+
// same isolated path as ordinary new-failure rechecks, even when the error was present in the baseline ledger.
|
|
2107
|
+
// Replace that file's shard observations with its isolated observations before the existing regression comparison.
|
|
2108
|
+
const prefetchedCut = new Map<string, CommandResult>();
|
|
2109
|
+
for (const file of [...new Set(measuredCut.errors.filter((id) => id.endsWith(' > [error]')).map(fileOf))].sort()) {
|
|
2110
|
+
let run: CommandResult | undefined;
|
|
2111
|
+
let reading: ReturnType<typeof readIsolatedRun> | undefined;
|
|
2112
|
+
let reason: 'reproduced' | 'timeout' | 'unreadable' | undefined;
|
|
2113
|
+
try {
|
|
2114
|
+
run = opts.pod && runner.recheckPod
|
|
2115
|
+
? await runner.recheckPod(file, opts.commit, opts.pod, GATE_ISOLATED_TIMEOUT_MS)
|
|
2116
|
+
: await runner.command('bun', ['run', 'test:deterministic', asPath(file)], cutTree, GATE_ISOLATED_TIMEOUT_MS);
|
|
2117
|
+
} catch (error) {
|
|
2118
|
+
reason = 'unreadable';
|
|
2119
|
+
debug.log('release-loop.gate', 'unhandled-isolated-run-failed', { file, error: String(error) });
|
|
2120
|
+
}
|
|
2121
|
+
if (run?.timedOut) reason = 'timeout';
|
|
2122
|
+
else if (run) {
|
|
2123
|
+
try {
|
|
2124
|
+
reading = readIsolatedRun(run, `cut unhandled isolated ${file}`, file);
|
|
2125
|
+
if (reading.kind === 'read' && reading.errors.length) reason = 'reproduced';
|
|
2126
|
+
} catch { reason = 'unreadable'; }
|
|
2127
|
+
}
|
|
2128
|
+
const originalIds = measuredCut.errors.filter((id) => fileOf(id) === file);
|
|
2129
|
+
measuredCut.failures = measuredCut.failures.filter((id) => fileOf(id) !== file);
|
|
2130
|
+
measuredCut.errors = measuredCut.errors.filter((id) => fileOf(id) !== file);
|
|
2131
|
+
if (reading?.kind === 'read') measuredCut.failures.push(...reading.failures);
|
|
2132
|
+
else if (reading?.kind === 'rc-nonzero-no-failures') measuredCut.errors.push(`${file} > [rc-nonzero]`);
|
|
2133
|
+
if (reason) {
|
|
2134
|
+
(result.unmeasuredErrors ??= []).push({ file, reason, ids: originalIds });
|
|
2135
|
+
debug.log('release-loop.gate', 'unhandled-isolated-unmeasured', { file, reason, ids: originalIds });
|
|
2136
|
+
} else {
|
|
2137
|
+
prefetchedCut.set(file, run!);
|
|
2138
|
+
if (reading?.kind === 'read' && !reading.failures.length) {
|
|
2139
|
+
(result.interferenceErrors ??= []).push(...originalIds);
|
|
2140
|
+
debug.log('release-loop.gate', 'unhandled-isolated-interference', { file, ids: originalIds });
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
2143
|
+
}
|
|
1549
2144
|
const cut = partial ? mergePartialFailures(partial, measuredCut) : measuredCut;
|
|
1550
2145
|
if (partial) result.partial = { priorCommit: partial.priorCommit, files: partial.files.length };
|
|
1551
2146
|
cutFailures = cut;
|
|
@@ -1598,20 +2193,20 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1598
2193
|
for (const id of [...baseline.failures, ...baseline.errors, ...cut.failures, ...cut.errors]) fileOf(id);
|
|
1599
2194
|
const known = opts.pod ? loadEnvKnownFailures(repo) : new Set<string>();
|
|
1600
2195
|
const cutIds = [...cut.failures, ...cut.errors];
|
|
1601
|
-
const cutHostFailed = new Set(cutRun
|
|
2196
|
+
const cutHostFailed = new Set(cutRun?.stalledEnv?.map(({ file }) => file) ?? []);
|
|
1602
2197
|
const baselineIds = [...baseline.failures, ...baseline.errors];
|
|
1603
2198
|
const countedCut = splitKnownEnv(cutIds, known);
|
|
1604
2199
|
const countedBaseline = splitKnownEnv(baselineIds, known);
|
|
1605
2200
|
knownEnvIds = countedCut.knownEnv;
|
|
1606
2201
|
result.knownEnv = knownEnvIds.length;
|
|
1607
|
-
result.knownEnvCleared = [...new Set(cutRun
|
|
2202
|
+
result.knownEnvCleared = [...new Set(cutRun?.passedIds ?? [])].filter((id) => known.has(id) && !cutIds.includes(id) && !cutRun?.stalledEnv?.some(({ file }) => fileOf(id) === file)).sort();
|
|
1608
2203
|
const diff = diffFailures(countedCut.counted, countedBaseline.counted);
|
|
1609
2204
|
result.fixed = result.baselineDeferred ? null : diff.fixed.length;
|
|
1610
2205
|
preexistingIds.push(...diff.common);
|
|
1611
2206
|
result.preexisting = preexistingIds.length;
|
|
1612
2207
|
// GATE-BASELINE-CACHE — only the files whose new failures reproduce in isolation reach the baseline, and each
|
|
1613
2208
|
// (commit, file, host) is run at most once across gates: this gate's cut results also seed the next gate's baseline.
|
|
1614
|
-
const cacheHost = opts.remote ?? 'local';
|
|
2209
|
+
const cacheHost = opts.pod && runner.recheckPod ? `pod:${opts.pod.pool}` : opts.remote ?? 'local';
|
|
1615
2210
|
const newFailureFiles = new Set(diff.newFailures.map(fileOf));
|
|
1616
2211
|
const baseCache = newFailureFiles.size ? readBaselineFileCache(ledger, baseSha) : new Map<string, BaselineFileResult>();
|
|
1617
2212
|
const cacheStats = { hits: 0, misses: 0 };
|
|
@@ -1625,13 +2220,27 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1625
2220
|
try { saveBaselineFileCache(ledger, row.commit, [row]); }
|
|
1626
2221
|
catch (error) { debug.log('release.gate', 'baseline-cache-write-failed', { commit: row.commit, file: row.file, error: String(error) }); }
|
|
1627
2222
|
};
|
|
1628
|
-
|
|
2223
|
+
const recheckFiles = [...newFailureFiles].sort();
|
|
2224
|
+
const recheckLimit = opts.recheckFileLimit ?? GATE_HOST_RECHECK_FILE_LIMIT;
|
|
2225
|
+
const recheckConcurrency = opts.pod?.isolationConcurrency ?? 4;
|
|
2226
|
+
if (opts.pod && (!Number.isSafeInteger(recheckConcurrency) || recheckConcurrency < 1)) throw new Error('invalid pod recheck concurrency');
|
|
2227
|
+
// Each window is bounded, but overflow is queued into the next one. A limit must never turn known failures into
|
|
2228
|
+
// unmeasured files. Pod checks use the same file-isolation fanout cap as the sweep; host fallback stays serial.
|
|
2229
|
+
debug.log('release-loop.gate', 'isolated-recheck-plan', { files: recheckFiles.length, window: recheckLimit,
|
|
2230
|
+
concurrency: opts.pod && runner.recheckPod ? recheckConcurrency : 1, executor: opts.pod && runner.recheckPod ? 'pod' : 'host',
|
|
2231
|
+
remote: opts.remote ?? null, podBatchFiles: opts.pod && runner.recheckPod && runner.recheckPodBatch ? GATE_RECHECK_POD_BATCH_FILES : 1 });
|
|
2232
|
+
const recheckStartedAt = Date.now();
|
|
2233
|
+
const recheckFile = async (file: string, index: number) => {
|
|
2234
|
+
debug.log('release-loop.gate', 'isolated-recheck-progress',
|
|
2235
|
+
{ index: index + 1, total: recheckFiles.length, file, elapsedSec: Math.round((Date.now() - recheckStartedAt) / 1000) });
|
|
1629
2236
|
// 0.2.15: the cut re-check of a new-failure file (InsidePage.test.tsx · headless Chrome) hung for good — limit it like the host re-run.
|
|
1630
|
-
const cutCheck = cutHostFailed.has(file) ? undefined :
|
|
2237
|
+
const cutCheck = cutHostFailed.has(file) ? undefined : prefetchedCut.get(file) ?? (opts.pod && runner.recheckPod
|
|
2238
|
+
? await runner.recheckPod(file, opts.commit, opts.pod, GATE_ISOLATED_TIMEOUT_MS)
|
|
2239
|
+
: await runner.command('bun', ['run', 'test:deterministic', asPath(file)], cutTree, GATE_ISOLATED_TIMEOUT_MS));
|
|
1631
2240
|
if (cutCheck?.timedOut) {
|
|
1632
2241
|
debug.log('release-loop.gate', 'isolated-timeout', { file, label: 'cut isolated', limitMs: GATE_ISOLATED_TIMEOUT_MS });
|
|
1633
2242
|
result.stalledEnv.push({ file, reason: 'isolated-timeout', local: 'timeout' });
|
|
1634
|
-
|
|
2243
|
+
return;
|
|
1635
2244
|
}
|
|
1636
2245
|
const cutReading = cutCheck === undefined ? undefined : readIsolatedRun(cutCheck, `cut isolated ${file}`, file);
|
|
1637
2246
|
const rcOnly = cutReading?.kind === 'rc-nonzero-no-failures' ? cutReading : undefined;
|
|
@@ -1648,7 +2257,9 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1648
2257
|
const timedOut = cutCheck === undefined ? new Set<string>() : new Set(parseTimedOutFailures(stripPodRepoRoot(cutCheck.output)));
|
|
1649
2258
|
const shaped = candidates.filter((id) => timedOut.has(id) && isolatedCut.failures.includes(id));
|
|
1650
2259
|
if (shaped.length) {
|
|
1651
|
-
const relaxed =
|
|
2260
|
+
const relaxed = opts.pod && runner.recheckPod
|
|
2261
|
+
? await runner.recheckPod(file, opts.commit, opts.pod, GATE_ISOLATED_TIMEOUT_MS, true)
|
|
2262
|
+
: await runner.command('bun', ['run', 'test:deterministic', asPath(file), '--timeout', String(GATE_TIMEOUT_RECHECK_MS)], cutTree, GATE_ISOLATED_TIMEOUT_MS);
|
|
1652
2263
|
let flaky: string[] = [];
|
|
1653
2264
|
let recheck: 'read' | 'timeout' | 'unreadable' = 'read';
|
|
1654
2265
|
if (relaxed.timedOut) recheck = 'timeout';
|
|
@@ -1671,7 +2282,7 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1671
2282
|
} catch (error) { debug.log('release-loop.gate', 'load-flaky-census-write-failed', { file, error: String(error) }); }
|
|
1672
2283
|
}
|
|
1673
2284
|
}
|
|
1674
|
-
if (candidates.length === 0)
|
|
2285
|
+
if (candidates.length === 0) return;
|
|
1675
2286
|
let oldFailures: Set<string>;
|
|
1676
2287
|
let baselineMissing = false;
|
|
1677
2288
|
const hit = baseCache.get(`${file}\0${cacheHost}`);
|
|
@@ -1681,15 +2292,17 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1681
2292
|
oldFailures = new Set([...hit.failures, ...hit.errors]);
|
|
1682
2293
|
} else {
|
|
1683
2294
|
cacheStats.misses++;
|
|
1684
|
-
const baselineTree = await getBaseTree();
|
|
1685
|
-
const previous =
|
|
2295
|
+
const baselineTree = opts.pod && runner.recheckPod ? undefined : await getBaseTree();
|
|
2296
|
+
const previous = opts.pod && runner.recheckPod
|
|
2297
|
+
? await runner.recheckPod(file, baseSha, opts.pod, GATE_ISOLATED_TIMEOUT_MS)
|
|
2298
|
+
: await runner.command('bun', ['run', 'test:deterministic', asPath(file)], baselineTree!, GATE_ISOLATED_TIMEOUT_MS);
|
|
1686
2299
|
if (previous.timedOut) {
|
|
1687
2300
|
debug.log('release-loop.gate', 'isolated-timeout', { file, label: 'baseline isolated', limitMs: GATE_ISOLATED_TIMEOUT_MS });
|
|
1688
2301
|
result.stalledEnv.push({ file, reason: 'isolated-timeout', local: 'timeout' });
|
|
1689
|
-
|
|
2302
|
+
return;
|
|
1690
2303
|
}
|
|
1691
2304
|
if (/No tests found|had no matches/i.test(previous.output) && previous.rc === 1) {
|
|
1692
|
-
const lookup = await runner.command('git', ['ls-tree', '--name-only', baseSha, '--', file], opts.remote ? baselineTree : repo);
|
|
2305
|
+
const lookup = await runner.command('git', ['ls-tree', '--name-only', baseSha, '--', file], opts.remote ? baselineTree ?? cutTree : repo);
|
|
1693
2306
|
if (lookup.rc !== 0 || lookup.output.trim()) throw new Error(`baseline isolated run incomplete: ${file}`);
|
|
1694
2307
|
oldFailures = new Set();
|
|
1695
2308
|
baselineMissing = true;
|
|
@@ -1712,16 +2325,60 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1712
2325
|
if (oldFailures.has(id)) preexistingIds.push(id);
|
|
1713
2326
|
else result.introduced.push(id);
|
|
1714
2327
|
}
|
|
2328
|
+
};
|
|
2329
|
+
const cap = opts.pod && runner.recheckPod ? recheckConcurrency : 1;
|
|
2330
|
+
const settleStarted = async <T>(jobs: Promise<T>[]): Promise<T[]> => {
|
|
2331
|
+
const settled = await Promise.allSettled(jobs);
|
|
2332
|
+
const rejected = settled.find((item): item is PromiseRejectedResult => item.status === 'rejected');
|
|
2333
|
+
if (rejected) throw rejected.reason;
|
|
2334
|
+
return settled.map((item) => (item as PromiseFulfilledResult<T>).value);
|
|
2335
|
+
};
|
|
2336
|
+
for (let start = 0; start < recheckFiles.length; start += recheckLimit) {
|
|
2337
|
+
const window = recheckFiles.slice(start, start + recheckLimit);
|
|
2338
|
+
debug.log('release-loop.gate', 'isolated-recheck-window', { start, files: window.length, total: recheckFiles.length });
|
|
2339
|
+
if (opts.pod && runner.recheckPod && runner.recheckPodBatch) {
|
|
2340
|
+
const pending = window.filter((file) => !cutHostFailed.has(file) && !prefetchedCut.has(file));
|
|
2341
|
+
const groups = Array.from({ length: Math.ceil(pending.length / GATE_RECHECK_POD_BATCH_FILES) }, (_, index) =>
|
|
2342
|
+
pending.slice(index * GATE_RECHECK_POD_BATCH_FILES, (index + 1) * GATE_RECHECK_POD_BATCH_FILES));
|
|
2343
|
+
for (let next = 0; next < groups.length; next += cap) {
|
|
2344
|
+
const batches = await settleStarted(groups.slice(next, next + cap).map((files) =>
|
|
2345
|
+
runner.recheckPodBatch!(files, opts.commit, opts.pod!, GATE_ISOLATED_TIMEOUT_MS)));
|
|
2346
|
+
for (const batch of batches) for (const [file, run] of batch) prefetchedCut.set(file, run);
|
|
2347
|
+
}
|
|
2348
|
+
}
|
|
2349
|
+
for (let next = 0; next < window.length; next += cap)
|
|
2350
|
+
await settleStarted(window.slice(next, next + cap).map((file, slot) => recheckFile(file, start + next + slot)));
|
|
2351
|
+
for (const file of window) prefetchedCut.delete(file);
|
|
1715
2352
|
}
|
|
2353
|
+
result.introduced.sort();
|
|
2354
|
+
preexistingIds.sort();
|
|
2355
|
+
result.loadFlaky?.sort();
|
|
1716
2356
|
result.preexisting = preexistingIds.length;
|
|
1717
2357
|
result.baselineCache = cacheStats;
|
|
1718
2358
|
debug.log('release.gate', 'baseline-cache', { hits: cacheStats.hits, misses: cacheStats.misses, commit: baseSha });
|
|
1719
|
-
|
|
2359
|
+
const timedOutFiles = [...new Set(result.stalledEnv.filter((item) => item.reason === 'isolated-timeout').map((item) => item.file))].sort();
|
|
2360
|
+
const unmeasuredFiles = [...new Set([...timedOutFiles, ...(result.unmeasuredErrors ?? []).map((item) => item.file)])].sort();
|
|
2361
|
+
if (unmeasuredFiles.length) result.unmeasuredRecheckFiles = unmeasuredFiles;
|
|
2362
|
+
if (timedOutFiles.length || result.unmeasuredErrors?.length) {
|
|
2363
|
+
result.outcome = 'error';
|
|
2364
|
+
result.error = [
|
|
2365
|
+
timedOutFiles.length ? `isolated recheck timed out: ${timedOutFiles.join(', ')}` : undefined,
|
|
2366
|
+
result.unmeasuredErrors?.length ? `unhandled errors unmeasured after isolated recheck: ${result.unmeasuredErrors.map((item) => `${item.file} (${item.reason})`).join(', ')}` : undefined,
|
|
2367
|
+
].filter(Boolean).join('; ');
|
|
2368
|
+
} else result.outcome = result.introduced.length ? 'regression' : 'ok';
|
|
2369
|
+
if (opts.rehearsalTui) {
|
|
2370
|
+
result.rehearsalTui = await executeTuiNodeCheck(opts.commit, repo, runner.localCommand);
|
|
2371
|
+
if (result.rehearsalTui.outcome !== 'ok') {
|
|
2372
|
+
result.outcome = 'error';
|
|
2373
|
+
result.error = result.rehearsalTui.detail;
|
|
2374
|
+
}
|
|
2375
|
+
}
|
|
1720
2376
|
} catch (error) {
|
|
1721
2377
|
result.error = error instanceof Error ? error.message : String(error);
|
|
1722
2378
|
if (error instanceof StalledPodShards) {
|
|
1723
2379
|
result.stalledShards = error.stalledShards;
|
|
1724
2380
|
result.partialSummary = error.partialSummary;
|
|
2381
|
+
if (error.abortedShards) result.abortedShards = error.abortedShards;
|
|
1725
2382
|
}
|
|
1726
2383
|
} finally {
|
|
1727
2384
|
if (baseSnapshot && work) {
|
|
@@ -1741,7 +2398,7 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1741
2398
|
if (opts.remote) runner.remoteMirror = undefined;
|
|
1742
2399
|
result.durationMs = Date.now() - start;
|
|
1743
2400
|
}
|
|
1744
|
-
if (cutFailures && !result.stalledShards?.length && !result.stalledEnv.length) {
|
|
2401
|
+
if (cutFailures && !result.stalledShards?.length && !result.stalledEnv.length && !result.unmeasuredRecheckFiles?.length) {
|
|
1745
2402
|
try {
|
|
1746
2403
|
for (const location of roots) {
|
|
1747
2404
|
const path = failureFile(location, opts.version);
|
|
@@ -1758,31 +2415,54 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1758
2415
|
result.error = `baseline persistence: ${String(error)}`;
|
|
1759
2416
|
}
|
|
1760
2417
|
}
|
|
1761
|
-
|
|
2418
|
+
{
|
|
1762
2419
|
const ids = {
|
|
1763
2420
|
introduced: [...result.introduced].sort(),
|
|
1764
2421
|
knownEnv: [...knownEnvIds].sort(),
|
|
1765
2422
|
preexisting: [...preexistingIds].sort(),
|
|
1766
2423
|
loadFlaky: [...(result.loadFlaky ?? [])].sort(),
|
|
2424
|
+
...(result.unmeasuredRecheckFiles?.length ? { unmeasuredRecheckFiles: [...result.unmeasuredRecheckFiles].sort() } : {}),
|
|
2425
|
+
...(result.unmeasuredErrors?.length ? { unmeasuredErrors: result.unmeasuredErrors } : {}),
|
|
2426
|
+
...(result.interferenceErrors?.length ? { interferenceErrors: [...result.interferenceErrors].sort() } : {}),
|
|
1767
2427
|
};
|
|
2428
|
+
const observed = result.selectionMs !== undefined ? { selectionMs: result.selectionMs } : {};
|
|
1768
2429
|
try {
|
|
1769
|
-
const
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
2430
|
+
for (const location of roots) {
|
|
2431
|
+
const path = verdictFile(location, opts.version);
|
|
2432
|
+
if (existsSync(path)) readFileSync(path, 'utf8');
|
|
2433
|
+
}
|
|
2434
|
+
for (const location of roots) {
|
|
2435
|
+
const path = verdictFile(location, opts.version);
|
|
2436
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
2437
|
+
const temp = join(dirname(path), `.gate-verdict-${process.pid}-${randomUUID()}.tmp`);
|
|
2438
|
+
try {
|
|
2439
|
+
writeFileSync(temp, JSON.stringify({ version: opts.version, commit: opts.commit, outcome: result.outcome, ...ids, ...observed }, null, 2) + '\n', { mode: 0o600 });
|
|
2440
|
+
chmodSync(temp, 0o600);
|
|
2441
|
+
renameSync(temp, path);
|
|
2442
|
+
} finally { if (existsSync(temp)) rmSync(temp); }
|
|
2443
|
+
}
|
|
1777
2444
|
} catch (error) {
|
|
1778
2445
|
result.outcome = 'error';
|
|
1779
2446
|
result.error = `verdict persistence: ${String(error)}`;
|
|
2447
|
+
const path = verdictFile(ledger, opts.version);
|
|
2448
|
+
try {
|
|
2449
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
2450
|
+
const temp = join(dirname(path), `.gate-verdict-${process.pid}-${randomUUID()}.tmp`);
|
|
2451
|
+
try {
|
|
2452
|
+
writeFileSync(temp, JSON.stringify({ version: opts.version, commit: opts.commit, outcome: result.outcome, ...ids, ...observed }, null, 2) + '\n', { mode: 0o600 });
|
|
2453
|
+
chmodSync(temp, 0o600);
|
|
2454
|
+
renameSync(temp, path);
|
|
2455
|
+
} finally { if (existsSync(temp)) rmSync(temp); }
|
|
2456
|
+
} catch (persistenceError) {
|
|
2457
|
+
result.error += `; error verdict persistence: ${String(persistenceError)}`;
|
|
2458
|
+
}
|
|
1780
2459
|
}
|
|
1781
|
-
|
|
1782
|
-
version: opts.version, commit: opts.commit,
|
|
2460
|
+
debug.log('release-loop.gate', 'judged', {
|
|
2461
|
+
version: opts.version, commit: opts.commit, outcome: result.outcome,
|
|
1783
2462
|
introduced: result.introduced.length, preexisting: result.preexisting,
|
|
1784
2463
|
knownEnv: result.knownEnv, loadFlaky: ids.loadFlaky.length,
|
|
1785
2464
|
introducedIds: ids.introduced, knownEnvIds: ids.knownEnv, preexistingIds: ids.preexisting, loadFlakyIds: ids.loadFlaky,
|
|
2465
|
+
...(result.outcome === 'error' ? { error: result.error } : {}),
|
|
1786
2466
|
});
|
|
1787
2467
|
}
|
|
1788
2468
|
// GATE-LIVE-OBS-RC-WORDING ⓒ — the shards board says «판정 전» until this lands; then «새 실패 N» only (UX · gate-shards.ts).
|
|
@@ -1804,7 +2484,8 @@ export function graphGateResult(result: GateResult, graph: boolean) {
|
|
|
1804
2484
|
+ (result.outcome === 'ok' ? `새 회귀 ${result.introduced.length} · 기존 ${result.preexisting} · 고침 ${result.fixed ?? '못 셈(기준선 미루기)'}` : `게이트 ${result.outcome}: ${result.error ?? result.introduced.length + ' new regressions'}`) + ` · 환경 알려진 실패 ${result.knownEnv}`
|
|
1805
2485
|
+ (result.rcNonzeroNoFailures.length ? ` · ⚠ 종료코드만 비영 ${result.rcNonzeroNoFailures.length}` : '')
|
|
1806
2486
|
+ (result.loadFlaky?.length ? ` · ⚠ 부하 흔들림 ${result.loadFlaky.length}(시간 초과 꼴 · ${GATE_TIMEOUT_RECHECK_MS / 1000}s 재실행 통과 · 막지 않음)` : '')
|
|
1807
|
-
+ (result.stalledEnv.length ? ` · 환경 멈춤 ${result.stalledEnv.length} (Pod 밖 통과 ${result.stalledEnv.filter((item) => item.local === 'passed').length})` : '')
|
|
2487
|
+
+ (result.stalledEnv.length ? ` · 환경 멈춤 ${result.stalledEnv.length} (Pod 밖 통과 ${result.stalledEnv.filter((item) => item.local === 'passed').length})` : '')
|
|
2488
|
+
+ (result.abortedShards ? ` · 중단 조각 ${result.abortedShards}(중간 매달림 · 시험만 끊고 재확인)` : '') };
|
|
1808
2489
|
}
|
|
1809
2490
|
|
|
1810
2491
|
/** GATE-HOST-EXECUTOR — which executor produced (or will produce) this verdict, for the node output and observation. */
|
|
@@ -1813,12 +2494,18 @@ export function gateExecutorOf(opts: Pick<GateOptions, 'pod' | 'remote'>): { exe
|
|
|
1813
2494
|
return opts.pod ? { executor: 'pod', host: null } : { executor: 'host', host: opts.remote ?? 'local' };
|
|
1814
2495
|
}
|
|
1815
2496
|
|
|
2497
|
+
/** A Pod without an SSH host runs on this machine; it does not reserve every remote gate host. */
|
|
2498
|
+
export function reservedGateHosts(opts: Pick<GateOptions, 'pod' | 'remote'>): string[] {
|
|
2499
|
+
return opts.pod ? [...new Set(parsePodPool(opts.pod.pool).map(member => member.sshHost ?? hostname()))]
|
|
2500
|
+
: [opts.remote?.split('@').at(-1) ?? hostname()];
|
|
2501
|
+
}
|
|
2502
|
+
|
|
1816
2503
|
/** What was asked for, read best-effort when the options themselves were refused — so a failed gate still says which executor. */
|
|
1817
2504
|
export function requestedGateExecutor(args: string[], env: NodeJS.ProcessEnv): { executor: string | null; host: string | null } {
|
|
1818
2505
|
let executor: unknown;
|
|
1819
2506
|
let host: unknown;
|
|
1820
2507
|
try {
|
|
1821
|
-
if (
|
|
2508
|
+
if (usesGraphContext(args, env)) {
|
|
1822
2509
|
const input = readGraphContext(env).input as Record<string, unknown>;
|
|
1823
2510
|
executor = input.gateExecutor ?? (typeof input.gatePodPool === 'string' ? 'pod' : undefined);
|
|
1824
2511
|
host = input.gateRemote;
|
|
@@ -1832,8 +2519,12 @@ export function requestedGateExecutor(args: string[], env: NodeJS.ProcessEnv): {
|
|
|
1832
2519
|
return { executor: typeof executor === 'string' ? executor : null, host: executor !== 'pod' && typeof host === 'string' ? host : null };
|
|
1833
2520
|
}
|
|
1834
2521
|
|
|
2522
|
+
function usesGraphContext(args: string[], env: NodeJS.ProcessEnv): boolean {
|
|
2523
|
+
return !!env.ELANOUS_GRAPH_CONTEXT && args.length > 0 && args.every(arg => arg === '--json');
|
|
2524
|
+
}
|
|
2525
|
+
|
|
1835
2526
|
export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOptions | 'help' {
|
|
1836
|
-
const context =
|
|
2527
|
+
const context = usesGraphContext(args, env) ? readGraphContext(env) : undefined;
|
|
1837
2528
|
const fromGraph: Record<string, unknown> = context?.input ?? {};
|
|
1838
2529
|
const graphCommit = context?.outputs['version-release']?.commit;
|
|
1839
2530
|
if (typeof graphCommit === 'string') fromGraph.commit = graphCommit;
|
|
@@ -1843,6 +2534,8 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
1843
2534
|
// GATE-HOST-EXECUTOR: a host gate never reads the pod settings (a bad pod value must not block the fallback executor).
|
|
1844
2535
|
const hostExecutor = fromGraph.gateExecutor === 'host';
|
|
1845
2536
|
const opts: GateOptions = {
|
|
2537
|
+
rehearsalTui: fromGraph.rehearsalTui === true,
|
|
2538
|
+
recheckFileLimit: fromGraph.gateRecheckFileLimit !== undefined ? Number(fromGraph.gateRecheckFileLimit) : undefined,
|
|
1846
2539
|
commit: typeof fromGraph.commit === 'string' ? fromGraph.commit : '',
|
|
1847
2540
|
version: typeof fromGraph.version === 'string' ? fromGraph.version : '',
|
|
1848
2541
|
baselineVersion: typeof fromGraph.previousVersion === 'string' ? fromGraph.previousVersion : undefined,
|
|
@@ -1857,7 +2550,9 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
1857
2550
|
...(typeof fromGraph.gatePodBunCache === 'string' && fromGraph.gatePodBunCache.trim() ? { bunCache: fromGraph.gatePodBunCache } : {}),
|
|
1858
2551
|
...(fromGraph.gatePodInstallSlots !== undefined ? { installSlots: Number(fromGraph.gatePodInstallSlots) } : {}),
|
|
1859
2552
|
...(fromGraph.gatePodAdmissionWaitSeconds !== undefined ? { admissionWaitSeconds: Number(fromGraph.gatePodAdmissionWaitSeconds) } : {}),
|
|
2553
|
+
...(fromGraph.gatePodIsolationConcurrency !== undefined ? { isolationConcurrency: Number(fromGraph.gatePodIsolationConcurrency) } : {}),
|
|
1860
2554
|
...(fromGraph.gatePodStallSeconds !== undefined ? { stallSeconds: Number(fromGraph.gatePodStallSeconds) } : {}),
|
|
2555
|
+
...(fromGraph.gatePodMidRunStallSeconds !== undefined ? { midRunStallSeconds: Number(fromGraph.gatePodMidRunStallSeconds) } : {}),
|
|
1861
2556
|
...(fromGraph.gatePodFreshShards === true ? { freshShards: true } : {}),
|
|
1862
2557
|
...((cpu) => cpu ? { cpu } : {})(parsePodCpu(fromGraph.gatePodCpu, 'release.loop.gatePodCpu')),
|
|
1863
2558
|
} : undefined,
|
|
@@ -1870,6 +2565,13 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
1870
2565
|
if (arg === '--json') continue;
|
|
1871
2566
|
// Applied after the loop and only to a Pod gate — on its own it must not turn a local gate into a Pod one (review r5).
|
|
1872
2567
|
if (arg === '--fresh-shards') { freshShards = true; continue; }
|
|
2568
|
+
if (arg === '--rehearsal-tui') { opts.rehearsalTui = true; continue; }
|
|
2569
|
+
if (arg === '--recheck-file-limit') {
|
|
2570
|
+
const value = args[++i];
|
|
2571
|
+
if (!value || !/^\d+$/.test(value) || Number(value) < 1 || !Number.isSafeInteger(Number(value))) throw new Error('--recheck-file-limit requires a positive integer');
|
|
2572
|
+
opts.recheckFileLimit = Number(value);
|
|
2573
|
+
continue;
|
|
2574
|
+
}
|
|
1873
2575
|
if (arg === '--executor') {
|
|
1874
2576
|
const value = args[++i];
|
|
1875
2577
|
if (!isGateExecutor(value)) throw new Error(`--executor must be pod or host: ${value ?? ''}`);
|
|
@@ -1916,11 +2618,21 @@ export async function runGateNode(argv: string[], env: NodeJS.ProcessEnv, runner
|
|
|
1916
2618
|
debug.log('release-loop.gate', 'executor', { version: opts.version, ...placement, requested: opts.executor ?? null });
|
|
1917
2619
|
}
|
|
1918
2620
|
if (opts === 'help') {
|
|
1919
|
-
console.log('Usage: bun scripts/release-loop/gate-node.ts --commit <sha> --version <v> [--baseline-version <prev>] [--baseline-commit <sha>] [--remote <ssh-host>] [--remote-mirror <path>] [--pod-pool <pool>] [--pod-shards <n>] [--pod-shard-timeout-seconds <n>] [--fresh-shards] [--executor <pod|host>] [--json]\nWithout flags, input.commit, input.version and input.previousVersion come from the JSON file at ELANOUS_GRAPH_CONTEXT.\n--executor host (input.gateExecutor) ignores every pod option and sweeps on --remote (input.gateRemote, required).');
|
|
2621
|
+
console.log('Usage: bun scripts/release-loop/gate-node.ts --commit <sha> --version <v> [--baseline-version <prev>] [--baseline-commit <sha>] [--remote <ssh-host>] [--remote-mirror <path>] [--pod-pool <pool>] [--pod-shards <n>] [--pod-shard-timeout-seconds <n>] [--recheck-file-limit <n>] [--fresh-shards] [--executor <pod|host>] [--json]\nWithout flags, input.commit, input.version and input.previousVersion come from the JSON file at ELANOUS_GRAPH_CONTEXT.\n--executor host (input.gateExecutor) ignores every pod option and sweeps on --remote (input.gateRemote, required).');
|
|
1920
2622
|
return 0;
|
|
1921
2623
|
}
|
|
1922
2624
|
version = opts.version;
|
|
1923
|
-
|
|
2625
|
+
// The graph's existing gate-node execution path delegates verdict classification and persistence here.
|
|
2626
|
+
const context = usesGraphContext(argv, env) ? readGraphContext(env) : undefined;
|
|
2627
|
+
// Let the gate's existing SSH host validation report an invalid host before any reservation is published.
|
|
2628
|
+
const hosts = opts.remote && !/^(?:[\w.-]+@)?[\w.-]+$/.test(opts.remote) ? [] : reservedGateHosts(opts);
|
|
2629
|
+
const reservationRoot = context ? gateHostReservationRoot() : undefined;
|
|
2630
|
+
if (context && (!context.runId || !env.ELANOUS_STATE_DIR || normRoot(env.ELANOUS_STATE_DIR) !== reservationRoot))
|
|
2631
|
+
throw new Error('release gate host reservation requires graph runId and pinned state root');
|
|
2632
|
+
const releaseHosts = context && hosts.length
|
|
2633
|
+
? reserveReleaseGateHosts(reservationRoot!, context.runId!, hosts) : () => undefined;
|
|
2634
|
+
try { result = await (runner ? judgeGate(opts, runner) : judgeGate(opts)); }
|
|
2635
|
+
finally { releaseHosts(); }
|
|
1924
2636
|
} catch (error) {
|
|
1925
2637
|
result = { outcome: 'error', commit: '', introduced: [], preexisting: 0, fixed: 0, knownEnv: 0, knownEnvCleared: [], stalledEnv: [], rcNonzeroNoFailures: [], durationMs: Date.now() - start, error: String(error) };
|
|
1926
2638
|
placement ??= requestedGateExecutor(argv, env);
|