elanous 0.2.29 → 0.2.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/pwa/out/404/index.html +1 -1
- package/apps/pwa/out/404.html +1 -1
- package/apps/pwa/out/_next/static/{cIpv5nU9AycwYWZI9H589 → LSXfY0u9QtfDbejY0Ukf_}/_buildManifest.js +1 -1
- package/apps/pwa/out/_next/static/chunks/{058a98ba.7eb28d5e6dc03565.js → 0bb0ff36.b8d418980951e5ba.js} +7 -7
- package/apps/pwa/out/_next/static/chunks/1034-8b639cee57b8e4a8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/1085-6c2d35526f11f840.js +1 -0
- package/apps/pwa/out/_next/static/chunks/1220.1ff7d5cdc34b1ae5.js +27 -0
- package/apps/pwa/out/_next/static/chunks/1247.5884ec81b39fe2e7.js +287 -0
- package/apps/pwa/out/_next/static/chunks/1302.87681521a7a4661e.js +62 -0
- package/apps/pwa/out/_next/static/chunks/1324-59dae3002413623b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{6013-2f769370d8b4c2c6.js → 1335-6731f392c35216eb.js} +3 -3
- package/apps/pwa/out/_next/static/chunks/1343-f40b5870ac1eda5a.js +1 -0
- package/apps/pwa/out/_next/static/chunks/146-b6cd56be11dda017.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{8332.6f0f49df772e163b.js → 1654.32de4b324981a188.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/1834-50625904996a8fa8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/1849.1ce38efa6c527d9a.js +149 -0
- package/apps/pwa/out/_next/static/chunks/{8795.32573a185af18c7b.js → 1913.7cc6fb6055b1ed3f.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/{419.736c0d6f426249a7.js → 2001.6c7e0fd02190886c.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/2030.41fae98c7e907304.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2099-08b2f2dfba742c9b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2112-e4efc21a6505d5bc.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2134.ecee03a550ee7250.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{6767.1fcfc9b8be83975f.js → 2213.2ccbada92c33ea0f.js} +16 -16
- package/apps/pwa/out/_next/static/chunks/2236.66061832181cb745.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2422.b0bd272e7020d7e6.js +1 -0
- package/apps/pwa/out/_next/static/chunks/2534.e85ae21ad0dac829.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{4759.1f76713aefb87db5.js → 2871.0bca6137343b3f76.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{6776.06ff1e3b464cc607.js → 2898.fd781bc7cf01a7b7.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{1782-59db1b5aa06df38a.js → 2940-619bfd8ef337c611.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{8471.8d2a8c74f09c36ff.js → 2985.39ccdecd36848b12.js} +4 -4
- package/apps/pwa/out/_next/static/chunks/3040.92ff9ffc9f971cb9.js +115 -0
- package/apps/pwa/out/_next/static/chunks/3144.6861096c7b4edef7.js +64 -0
- package/apps/pwa/out/_next/static/chunks/{bc324b7c.02a09dd93f6a399b.js → 3267c0e6.d34854e7b3984a6e.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/3278.2e5d15b53a4091c8.js +2 -0
- package/apps/pwa/out/_next/static/chunks/{7725-4f83d5541fbdb81b.js → 3541-ce3a5424ed5ad7f1.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/3583.a05e88cc8d594628.js +1 -0
- package/apps/pwa/out/_next/static/chunks/3771.e928edf2867d28c0.js +1 -0
- package/apps/pwa/out/_next/static/chunks/3902.e1e71aaac850a045.js +34 -0
- package/apps/pwa/out/_next/static/chunks/4063-03284b5de59965f3.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{46bd8bea.2f4b4099172fcdb7.js → 409f1e57.d0649fc47691ebfe.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/4111.2bfedbb233ad206b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{1142.65d002789ab3af7d.js → 4174.92ed04922709ac9e.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/4300-c7ec4eeabbb5f3f7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4483.646d5065199e1b55.js +36 -0
- package/apps/pwa/out/_next/static/chunks/4519.28199ff26b2fd80d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4581.ef2443578054544f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4608-8d894e9cbe4f08bf.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4639.ae48a01d1bb50a93.js +286 -0
- package/apps/pwa/out/_next/static/chunks/4739.08742ac668aada67.js +77 -0
- package/apps/pwa/out/_next/static/chunks/4745.67d6f99c245af3f7.js +226 -0
- package/apps/pwa/out/_next/static/chunks/4882-16cef71d6a1a32be.js +1 -0
- package/apps/pwa/out/_next/static/chunks/4999.c064a4000c2a9e94.js +79 -0
- package/apps/pwa/out/_next/static/chunks/5173-233188df2f42c626.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5260-69f35419bca263e7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5272.b92d1b72a15c25fe.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5304.04d24756f242a094.js +148 -0
- package/apps/pwa/out/_next/static/chunks/5307.60f2209e809bd627.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5415-997980f828667736.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{3396-97b97928c49190c9.js → 5458-16cd7a2b51b74c8c.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{3403.7199205e65124486.js → 5525.86debf231e181813.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{9143.ff14585db1d275ff.js → 5543.a438b8ab5a13ea47.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/5572-42c9084f27ca761e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5661-80470aac963d9f36.js +1 -0
- package/apps/pwa/out/_next/static/chunks/5723.421338408b5c30c4.js +74 -0
- package/apps/pwa/out/_next/static/chunks/5809.64c3664267c50bce.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6000.64bc21ac8627f2f5.js +94 -0
- package/apps/pwa/out/_next/static/chunks/6013.39459b9c68eabf5f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/607-06466dff012b03ef.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{8398.73a19b39715eda04.js → 6092.a7f99e68b188ca66.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/60fb96c9.360b01d8d4927b7b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{5591.a35e7a217734a8b9.js → 6157.0c9af021de4fb31b.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/6192.a190b989b81e9743.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6323.8672b4817a306928.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{7375.050ea0d72968e49f.js → 6433.fc573af22bbc9ab7.js} +17 -17
- package/apps/pwa/out/_next/static/chunks/6444.7d1fdcb9b09ae71c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6517.4147878ac06d584d.js +90 -0
- package/apps/pwa/out/_next/static/chunks/6532.e34302010b2f0b68.js +32 -0
- package/apps/pwa/out/_next/static/chunks/6544cb48.a87fe20ddf137be2.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6546.74bc4ed82f19016d.js +104 -0
- package/apps/pwa/out/_next/static/chunks/6684-e7523ce7c2fad075.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6726.ebdc82feb161431e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6739.00fb6f892e5e392b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6745.934c4921bf15513a.js +1 -0
- package/apps/pwa/out/_next/static/chunks/6767.ea607cf0c20a25af.js +24 -0
- package/apps/pwa/out/_next/static/chunks/6843.67bfd020da0926c5.js +93 -0
- package/apps/pwa/out/_next/static/chunks/6948.1062d4594652b5bf.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7138-eaf841fc1dc03463.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7174-cda29ee13d7790e7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7304-3c07583f4e405f4f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7371.80c05c653b08e907.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7391.ebae168025c91b6a.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{1852.7f7b53991fcb99ef.js → 7502.ccd5b8065545edd0.js} +2 -2
- package/apps/pwa/out/_next/static/chunks/7557.e325503a090a4310.js +8 -0
- package/apps/pwa/out/_next/static/chunks/7569-9ce9432c7f10989f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{6534.1ab00f370db7b105.js → 7862.30a8d6482bb238eb.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/7886-55854585fadeded1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/7947-d0759430f8042240.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{285adb1a.5654e605dddf86d2.js → 7a1887c4.f36b88fba31da8fd.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/8018.a32663f04a6fc5f9.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8097-92f87bd108322f66.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8215-1661812b6e9ad71f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8260-19c921a51ca8ec3e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8440.3a5cf5c85aa76db8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8519-4599be2249161e68.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8547.b028b90c5c01ace6.js +1 -0
- package/apps/pwa/out/_next/static/chunks/8587-cbe02a953a7f4990.js +1 -0
- package/apps/pwa/out/_next/static/chunks/877-2b977259c4b15655.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9002.e5658224e3dc1b6f.js +24 -0
- package/apps/pwa/out/_next/static/chunks/9019-0a4b40c50c7314fb.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{1516.dfaceff3fd2d0e0d.js → 9138.b6e80031afb1d973.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/9188-d36e4fc869ddc166.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{9907.49be0fcadaf214b5.js → 9221.7d22eaf353a2d4e7.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/9230.44f5039e796e4ab4.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{8767-2f6a634f5abeaa01.js → 9279-d28c5b7754d010ae.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/939.336acce73e98482c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{3160-95c96f4d2a21dd26.js → 9398-6aed28f0a96d0d6f.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/9407.e49124dedf0f5e72.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9610.c07cce4bfcc14ed7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/9667-ce78c2ce91050ff9.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{328bb3ab-6628a1114fe3fb3b.js → 997ce8a5-2ad501927003b41b.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/{6853.cc5174a9dee59593.js → 9989.3694300e5f000b14.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/_not-found/page-339fabb3499da0e1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/approvals/page-bf3634a117322df7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/autopilot/page-3535a4a69f1fafaa.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/board/page-b9f51447f757fbf2.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/botlab/{page-a5635eba0d73b84e.js → page-2ebb8fd59b9461f9.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/bots/page-a2fc4e49a4fa4ba8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ceo/page-5a503541353e60a8.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/chat/page-6025101384e361a5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/consult/page-f4b630cc8b5273cd.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/control/page-a645f00a06483e9a.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/decisions/page-61ac22cbab1ee4ec.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/design-check/page-e9bab7413b028b28.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/editor/page-c992eafe028b642e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/exec/page-e547e6ce55f1fb33.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/field/page-d27e1b9850fd54bf.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/healthz/route-339fabb3499da0e1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/inside/page-805ac2e137b7a121.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/intake/page-0fec29c7ba01894b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/layout-339a07410cd6f36b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/live/page-349f2752d8cd37ad.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/live-run/page-d5c6dcf31c25cecc.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/loops/page-2fd93ed8c9f16580.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/market/page-9b564dbff0fefc35.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/missions/[id]/page-62f4475d3c38fb9b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/morning/page-814ba51385225776.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/not-found-397d479626e5a459.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/observatory/page-af1bba4e3251b728.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/checklist/page-3ad3be5e7d3fdce9.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/release/page-f219ccdbc0bf27df.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/ops/seats/page-4da93ad35d4a8d28.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/outputs/page-456666766e5ce6fc.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/page-7404e377e8adb8c5.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/reflection/page-14456240ed2a36ad.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/scheduler/page-3a658d5fa6d16a25.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/sessions/page-0157de6e85f8bb6b.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/settings/devices/page-0a07363b130182a1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/settings/page-841d73d4ac44ed49.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/setup/done/page-6f43a3d1146e6377.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/setup/layout-339fabb3499da0e1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/setup/{page-29900ed0011b2c83.js → page-521ee23ac8c0abfa.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/share/page-a9992ee285efaa83.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/showroom/page-3d48e85fa89ec04f.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/tasks/page-d2ced71132fac9b2.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/term/page-f08358bdf251adb9.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/today/page-233c71cd5f68f0d7.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/trace/page-70151d78de2f375e.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/vault/page-dcac33a3e2cb72c0.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workflows/chat-ui/{page-9f92737b8766009a.js → page-149dc1caba411bb9.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/app/workflows/page-0448145ccd5c45c1.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/workspace/page-9cb0c444903da65d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/app/worktrees/page-17e504c396e0e389.js +1 -0
- package/apps/pwa/out/_next/static/chunks/{425cfc4d.fb91158e3971c39f.js → b313b242.cb1265eea908db61.js} +1 -1
- package/apps/pwa/out/_next/static/chunks/e5012ffe.f727e23c231f4342.js +122 -0
- package/apps/pwa/out/_next/static/chunks/ea695e3c-1f1d70d740cf3031.js +1 -0
- package/apps/pwa/out/_next/static/chunks/ebc7d829.264a0bcb6c750b50.js +1 -0
- package/apps/pwa/out/_next/static/chunks/framework-906af9bd029e166c.js +1 -0
- package/apps/pwa/out/_next/static/chunks/main-a24847f9f684d469.js +1 -0
- package/apps/pwa/out/_next/static/chunks/main-app-7bdcd8aa7c8397d3.js +1 -0
- package/apps/pwa/out/_next/static/chunks/pages/_app-6ba46a5066fc226d.js +1 -0
- package/apps/pwa/out/_next/static/chunks/pages/_error-a8dc6d07591f2a25.js +1 -0
- package/apps/pwa/out/_next/static/chunks/webpack-f13df28c15c55eee.js +1 -0
- package/apps/pwa/out/_next/static/css/{72becfd42807b50d.css → 3f5fc6f96e2cf7b2.css} +1 -1
- package/apps/pwa/out/approvals/index.html +1 -1
- package/apps/pwa/out/approvals/index.txt +23 -23
- package/apps/pwa/out/autopilot/index.html +1 -1
- package/apps/pwa/out/autopilot/index.txt +23 -23
- package/apps/pwa/out/board/index.html +1 -1
- package/apps/pwa/out/board/index.txt +23 -23
- package/apps/pwa/out/botlab/index.html +1 -1
- package/apps/pwa/out/botlab/index.txt +22 -22
- package/apps/pwa/out/bots/index.html +1 -1
- package/apps/pwa/out/bots/index.txt +22 -22
- package/apps/pwa/out/ceo/index.html +1 -1
- package/apps/pwa/out/ceo/index.txt +23 -23
- package/apps/pwa/out/chat/index.html +1 -1
- package/apps/pwa/out/chat/index.txt +22 -22
- package/apps/pwa/out/consult/index.html +1 -1
- package/apps/pwa/out/consult/index.txt +23 -23
- package/apps/pwa/out/control/index.html +1 -1
- package/apps/pwa/out/control/index.txt +23 -23
- package/apps/pwa/out/decisions/index.html +1 -1
- package/apps/pwa/out/decisions/index.txt +22 -22
- package/apps/pwa/out/design-check/index.html +1 -1
- package/apps/pwa/out/design-check/index.txt +23 -23
- package/apps/pwa/out/editor/index.html +1 -1
- package/apps/pwa/out/editor/index.txt +23 -23
- package/apps/pwa/out/exec/index.html +1 -1
- package/apps/pwa/out/exec/index.txt +23 -23
- package/apps/pwa/out/field/index.html +1 -1
- package/apps/pwa/out/field/index.txt +23 -23
- package/apps/pwa/out/healthz +1 -1
- package/apps/pwa/out/index.html +1 -1
- package/apps/pwa/out/index.txt +22 -22
- package/apps/pwa/out/inside/index.html +1 -1
- package/apps/pwa/out/inside/index.txt +22 -22
- package/apps/pwa/out/intake/index.html +1 -1
- package/apps/pwa/out/intake/index.txt +23 -23
- package/apps/pwa/out/live/index.html +1 -1
- package/apps/pwa/out/live/index.txt +23 -23
- package/apps/pwa/out/live-run/index.html +1 -1
- package/apps/pwa/out/live-run/index.txt +23 -23
- package/apps/pwa/out/loops/index.html +1 -1
- package/apps/pwa/out/loops/index.txt +22 -22
- package/apps/pwa/out/market/index.html +1 -1
- package/apps/pwa/out/market/index.txt +22 -22
- package/apps/pwa/out/missions/placeholder/index.html +1 -1
- package/apps/pwa/out/missions/placeholder/index.txt +22 -22
- package/apps/pwa/out/morning/index.html +1 -1
- package/apps/pwa/out/morning/index.txt +23 -23
- package/apps/pwa/out/observatory/index.html +1 -1
- package/apps/pwa/out/observatory/index.txt +23 -23
- package/apps/pwa/out/ops/checklist/index.html +1 -1
- package/apps/pwa/out/ops/checklist/index.txt +22 -22
- package/apps/pwa/out/ops/release/index.html +1 -1
- package/apps/pwa/out/ops/release/index.txt +22 -22
- package/apps/pwa/out/ops/seats/index.html +1 -1
- package/apps/pwa/out/ops/seats/index.txt +22 -22
- package/apps/pwa/out/outputs/index.html +1 -1
- package/apps/pwa/out/outputs/index.txt +22 -22
- package/apps/pwa/out/reflection/index.html +1 -1
- package/apps/pwa/out/reflection/index.txt +23 -23
- package/apps/pwa/out/scheduler/index.html +1 -1
- package/apps/pwa/out/scheduler/index.txt +23 -23
- package/apps/pwa/out/sessions/index.html +1 -1
- package/apps/pwa/out/sessions/index.txt +22 -22
- package/apps/pwa/out/settings/devices/index.html +1 -1
- package/apps/pwa/out/settings/devices/index.txt +23 -23
- package/apps/pwa/out/settings/index.html +1 -1
- package/apps/pwa/out/settings/index.txt +23 -23
- package/apps/pwa/out/setup/done/index.html +1 -1
- package/apps/pwa/out/setup/done/index.txt +23 -23
- package/apps/pwa/out/setup/index.html +1 -1
- package/apps/pwa/out/setup/index.txt +23 -23
- package/apps/pwa/out/share/index.html +1 -1
- package/apps/pwa/out/share/index.txt +23 -23
- package/apps/pwa/out/showroom/index.html +1 -1
- package/apps/pwa/out/showroom/index.txt +22 -22
- package/apps/pwa/out/tasks/index.html +1 -1
- package/apps/pwa/out/tasks/index.txt +23 -23
- package/apps/pwa/out/term/index.html +1 -1
- package/apps/pwa/out/term/index.txt +23 -23
- package/apps/pwa/out/today/index.html +1 -1
- package/apps/pwa/out/today/index.txt +23 -23
- package/apps/pwa/out/trace/index.html +1 -1
- package/apps/pwa/out/trace/index.txt +23 -23
- package/apps/pwa/out/vault/index.html +1 -1
- package/apps/pwa/out/vault/index.txt +23 -23
- package/apps/pwa/out/workflows/chat-ui/index.html +1 -1
- package/apps/pwa/out/workflows/chat-ui/index.txt +23 -23
- package/apps/pwa/out/workflows/index.html +1 -1
- package/apps/pwa/out/workflows/index.txt +23 -23
- package/apps/pwa/out/workspace/index.html +1 -1
- package/apps/pwa/out/workspace/index.txt +23 -23
- package/apps/pwa/out/worktrees/index.html +1 -1
- package/apps/pwa/out/worktrees/index.txt +23 -23
- package/bin/elanous.mjs +7 -3
- package/graphs/context/nightly-condense.yaml +1 -0
- package/graphs/intake/intake-daily.yaml +8 -1
- package/graphs/intake/recipes.yaml +5 -0
- package/graphs/lecture/lecture-note.yaml +2 -2
- package/graphs/ops/harness-gc.yaml +1 -0
- package/graphs/release/recipes.yaml +3 -0
- package/graphs/release/release-loop.yaml +9 -5
- package/graphs/units/recipes.yaml +3 -0
- package/graphs/units/release-gate-recheck.yaml +15 -0
- package/package.json +2 -1
- package/scripts/check-native-tool-wiring.ts +3 -2
- package/scripts/ci-android-unit-tests.ts +16 -0
- package/scripts/ci-typecheck-changed.ts +37 -9
- package/scripts/coord-channel-watch.sh +22 -16
- package/scripts/coord-role-pack.ts +81 -2
- package/scripts/cron-run.ts +96 -3
- package/scripts/detect-test-interference.ts +38 -5
- package/scripts/gate-pr-cache.ts +65 -0
- package/scripts/gh-surface/capture.ts +126 -0
- package/scripts/gh-surface/gh +40 -0
- package/scripts/import-graph-scc.ts +38 -0
- package/scripts/install.ps1 +19 -0
- package/scripts/intake-social.ts +181 -0
- package/scripts/lecture-note/node.ts +71 -4
- package/scripts/lib/android-toolchain.ts +32 -0
- package/scripts/lib/deterministic-env.ts +62 -0
- package/scripts/lib/gate-test-env.ts +19 -0
- package/scripts/measure-goal-grounding-corpus.ts +2 -1
- package/scripts/ratchets/test-interference-patterns.allowlist.txt +257 -0
- package/scripts/ratchets/test-interference-patterns.ts +367 -0
- package/scripts/release-loop/cut-branch.ts +4 -3
- package/scripts/release-loop/export-check-node.ts +0 -1
- package/scripts/release-loop/fixtures/warm-failure-assertion/sample.junit.xml +8 -0
- package/scripts/release-loop/fixtures/warm-job-029/sample.json +1 -0
- package/scripts/release-loop/fixtures/warm-job-029/sample.junit.xml +10 -0
- package/scripts/release-loop/gate-closure.ts +12 -8
- package/scripts/release-loop/gate-diff.ts +1 -1
- package/scripts/release-loop/gate-host-mirror.ts +66 -0
- package/scripts/release-loop/gate-node.ts +523 -75
- package/scripts/release-loop/gate-partial.ts +106 -9
- package/scripts/release-loop/gate-pr-cache.ts +66 -0
- package/scripts/release-loop/gate-prewarm-plan.ts +66 -0
- package/scripts/release-loop/gate-r2-verdict.ts +189 -0
- package/scripts/release-loop/gate-recheck-plan-node.ts +52 -0
- package/scripts/release-loop/gate-warm-artifacts.ts +253 -0
- package/scripts/release-loop/import-graph-scc-run.ts +54 -0
- package/scripts/release-loop/import-graph-scc.ts +215 -0
- package/scripts/release-loop/node-verdict.ts +2 -0
- package/scripts/release-loop/probe-pod-host-dirs-live.ts +113 -0
- package/scripts/release-loop/resume-release.ts +33 -4
- package/scripts/release-loop/resume.ts +13 -5
- package/scripts/release-loop/spawn-lane.ts +2 -19
- package/scripts/release-loop/tui-sim-node.ts +3 -2
- package/scripts/release-loop/unattended-release.ts +16 -0
- package/scripts/test-deterministic.ts +5 -61
- package/scripts/train-reach-grouping.ts +48 -0
- package/scripts/tsc-incremental-cache.ts +86 -0
- package/scripts/tsc-incremental-diagnostics.ts +76 -0
- package/scripts/tsc-main-safety-net.ts +2 -2
- package/src/acp/client.ts +4 -1
- package/src/acp/core-turn-bridge.ts +50 -5
- package/src/ad-pipeline/run.ts +4 -2
- package/src/agent/runner.ts +169 -3
- package/src/agent/subagent-callable.ts +3 -2
- package/src/agent-mission/review-loop.ts +6 -3
- package/src/agent-substrate/acp-implementer.ts +14 -2
- package/src/agent-substrate/pr-reviewer.ts +12 -0
- package/src/agent-substrate/self-review-cli.ts +30 -4
- package/src/autopilot/pr-manager.ts +62 -7
- package/src/briefing/brief-items.ts +24 -22
- package/src/capture/tui-self-report.ts +15 -0
- package/src/chat/chat-cli.ts +10 -2
- package/src/chat/index.ts +77 -11
- package/src/chat/input-control-key.ts +5 -3
- package/src/chat/tool-interrupt-behavior.ts +38 -0
- package/src/chat/tool-render/dispatch.ts +5 -1
- package/src/chat/turn-typeahead.ts +75 -20
- package/src/cli/__fixtures__/agent-help-before/agent.txt +4 -2
- package/src/cli/agent-cli.ts +41 -13
- package/src/cli/config-audit.ts +123 -0
- package/src/cli/decisions-cli.ts +80 -6
- package/src/cli/log-event-names.ts +7 -0
- package/src/cli/mcp-login.ts +21 -5
- package/src/cli/msg-cli.ts +88 -0
- package/src/cli/pr-cli.ts +214 -57
- package/src/cli/pr-land-heavy-remote.ts +2 -2
- package/src/cli/pr-land-overlap-evidence.ts +63 -9
- package/src/cli/pr-land-pod-shadow-job.ts +83 -0
- package/src/cli/pr-land-pod-shadow.ts +16 -3
- package/src/cli/pr-land-virtual-rebase.ts +108 -0
- package/src/cli/release-cli.ts +80 -14
- package/src/cli/seat-cli.ts +138 -3
- package/src/cli/self-update.ts +5 -2
- package/src/cli/tasks-cli.ts +97 -19
- package/src/context-bus/context-first.ts +13 -0
- package/src/context-bus/context-now.ts +13 -2
- package/src/context-bus/ops-events.ts +75 -0
- package/src/control-plane/lease-holder.ts +12 -8
- package/src/control-plane/ledger.ts +163 -4
- package/src/control-plane/server.ts +91 -11
- package/src/core-turn/run-core-turn.ts +1 -0
- package/src/core-turn/run-goal-loop.ts +222 -3
- package/src/core-turn/seat-turn-cap.ts +79 -0
- package/src/core-turn/short-question.ts +38 -6
- package/src/core-turn/types.ts +3 -0
- package/src/dashboard/acp-boot.ts +3 -0
- package/src/dashboard/index.ts +224 -615
- package/src/dashboard/input/chat-main-entry-runtime.ts +1 -1
- package/src/dashboard/input/chat-main-plain-turn-runtime.ts +4 -1
- package/src/dashboard/input/chat-main-session.ts +8 -1
- package/src/dashboard/input/chat-main-turn.ts +12 -3
- package/src/dashboard/input/compact-prepass-recovery.ts +33 -0
- package/src/dashboard/input/external-input-host.ts +26 -0
- package/src/dashboard/input/internal-turn.ts +261 -0
- package/src/dashboard/input/seat-autowake.ts +318 -0
- package/src/dashboard/input/turn-steer-bridge.ts +70 -0
- package/src/dashboard/loop-scheduler.ts +72 -0
- package/src/dashboard/seat-boot-preamble.ts +18 -0
- package/src/dashboard/skill-runtime.ts +2 -0
- package/src/dashboard/slash-runtime/dashboard-handlers.ts +126 -1181
- package/src/dashboard/slash-runtime/help-from-registry.ts +2 -1
- package/src/dashboard/slash-runtime/index.ts +13 -0
- package/src/dashboard/turn-busy-teardown.ts +50 -0
- package/src/decisions/decision-ledger.ts +45 -4
- package/src/discord-self-message.ts +4 -4
- package/src/domains/release-tool.ts +18 -1
- package/src/field-note/transcribe.ts +17 -0
- package/src/github/remote-call.ts +81 -0
- package/src/goals/registry.ts +46 -6
- package/src/goals/types.ts +7 -1
- package/src/graph-runner/runner.ts +1 -0
- package/src/grok/credential.ts +26 -0
- package/src/harness/harness-clean.ts +2 -1
- package/src/harness/harness-cli-command.ts +52 -5
- package/src/harness/harness-gc.ts +2 -1
- package/src/harness/harness-incidents.ts +57 -20
- package/src/harness/harness-pod-dispatch.ts +87 -10
- package/src/harness/harness-queue.ts +53 -4
- package/src/harness/harness-stop.ts +4 -3
- package/src/harness/harness-substrate-default.ts +78 -7
- package/src/harness/harness-worktree-auto.ts +2 -1
- package/src/harness/harness-worktrees.ts +3 -2
- package/src/hooks/dispatch.ts +2 -1
- package/src/index.ts +152 -25
- package/src/input/external-input-queue.ts +33 -1
- package/src/intake-plane/slash.ts +14 -3
- package/src/keybindings.ts +4 -1
- package/src/llm/tool-loop-steered-rejection.ts +33 -0
- package/src/llm.ts +138 -19
- package/src/loops/consensus.ts +348 -0
- package/src/loops/msg-wake.ts +206 -4
- package/src/loops/observe.ts +19 -3
- package/src/loops/orchestrator/tick.ts +64 -10
- package/src/loops/orchestrator/traffic.ts +64 -1
- package/src/loops/registry.ts +30 -0
- package/src/market/signed-index.ts +14 -0
- package/src/mcp/mcp-oauth.ts +54 -5
- package/src/models/prompts.ts +28 -0
- package/src/msg/msg-sse-client.ts +223 -0
- package/src/msg/msg-store.ts +86 -2
- package/src/mss/logging/log-store.ts +225 -1
- package/src/nexus/DAEMON-THIN-inventory-2026-10-15.md +122 -0
- package/src/nexus/api/http-server.ts +54 -15
- package/src/nexus/api/loop-resources.ts +19 -3
- package/src/nexus/api/ops-api.ts +11 -0
- package/src/nexus/daemon-thin-inventory.md +37 -0
- package/src/nexus/index.ts +70 -0
- package/src/nexus/state/state.ts +1 -0
- package/src/oauth/store.ts +32 -7
- package/src/onboarding/first-run.ts +28 -3
- package/src/ops/overview-cache.ts +16 -4
- package/src/ops/resources.ts +82 -0
- package/src/plugins/maker/connector-maker.ts +41 -0
- package/src/plugins/maker/skill-maker.ts +42 -0
- package/src/pr-tool/census.ts +288 -0
- package/src/prompt-library/chat-conciseness.ts +13 -0
- package/src/prompt-library/universal-preamble.ts +10 -5
- package/src/release-loop/gate-host-reservation.ts +97 -0
- package/src/release-loop/landing-freeze.ts +5 -0
- package/src/release-loop/light-rc.ts +5 -2
- package/src/release-loop/placement.ts +92 -1
- package/src/release-loop/rehearsal-shard-plan.ts +149 -0
- package/src/release-loop/rehearsal.ts +73 -17
- package/src/repl/index.ts +20 -2
- package/src/seat-loop/seat-knowhow.ts +282 -0
- package/src/seat-loop/seat-loop.ts +41 -2
- package/src/seat-loop/traffic-signal.ts +3 -0
- package/src/self-dev/ask-launch-flow.ts +24 -3
- package/src/self-dev/decompose-compose-graph.ts +28 -27
- package/src/self-dev/decompose-proposal.ts +53 -38
- package/src/self-dev/decompose.ts +17 -0
- package/src/self-dev/dev-pipeline.ts +31 -2
- package/src/self-dev/draft-residue.ts +24 -9
- package/src/self-dev/draft-sweep.ts +59 -4
- package/src/self-dev/external-call-zero.ts +77 -0
- package/src/self-dev/launch-preflight-text.ts +6 -0
- package/src/self-dev/launch-preflight.ts +25 -3
- package/src/self-dev/local-supervisor-judge.ts +185 -0
- package/src/self-dev/run-supervisor.ts +229 -19
- package/src/self-dev/run-types.ts +20 -0
- package/src/self-dev/shard-aggregate.ts +13 -2
- package/src/self-dev/shard-plan.ts +47 -0
- package/src/self-dev/shard-run.ts +112 -10
- package/src/self-dev/spawn-lane-files.ts +19 -0
- package/src/self-implement/app-api-budget.ts +51 -11
- package/src/self-implement/config-gate.ts +2 -1
- package/src/self-implement/expose-gate.ts +2 -1
- package/src/self-implement/gate-artifacts.ts +405 -0
- package/src/self-implement/gate-cli.ts +201 -16
- package/src/self-implement/gate-remote.ts +110 -25
- package/src/self-implement/gate-scenario.ts +85 -0
- package/src/self-implement/goal-author.ts +160 -8
- package/src/self-implement/graph-authority.ts +38 -6
- package/src/self-implement/graph-overlay-condition.ts +92 -1
- package/src/self-implement/heal-triage.ts +79 -0
- package/src/self-implement/host-regate.ts +61 -3
- package/src/self-implement/orchestrator.ts +14 -123
- package/src/self-implement/pr-title.ts +123 -0
- package/src/self-implement/run-checkpoint.ts +235 -0
- package/src/self-implement/run-ledger.ts +155 -10
- package/src/self-implement/run-stop.ts +4 -3
- package/src/self-implement/run-ttl.ts +28 -4
- package/src/self-implement/seams.ts +2 -0
- package/src/self-implement/shadow-stage.ts +7 -6
- package/src/self-implement/stop-autoheal.ts +2 -1
- package/src/self-implement/train-gate-remote.ts +37 -8
- package/src/self-implement/train-reach-grouping.ts +157 -0
- package/src/self-implement/train-scenario-layers.ts +133 -0
- package/src/self-implement/uncommitted-work.ts +2 -0
- package/src/session/pending-input.ts +19 -0
- package/src/session/session-start.ts +224 -0
- package/src/session-runtime/parallel-dispatch.ts +22 -5
- package/src/skills/tools/dev-harness.ts +6 -4
- package/src/skills/tools/shell.ts +1 -1
- package/src/steward/resync-tick.ts +114 -23
- package/src/steward/ta-repair-event-collect.ts +158 -0
- package/src/task-agent/card-evidence.ts +25 -1
- package/src/task-agent/green-sweep.ts +8 -3
- package/src/task-agent/judge.ts +76 -0
- package/src/task-agent/land-queue.ts +79 -0
- package/src/task-agent/live-moves.ts +30 -9
- package/src/task-agent/postland-card.ts +5 -0
- package/src/task-agent/prelaunch-already-done.ts +98 -0
- package/src/task-agent/real-launch-guard.ts +37 -0
- package/src/task-agent/rejudge-on-head.ts +89 -12
- package/src/task-agent/remote-rejudge-bind.ts +28 -8
- package/src/task-agent/repair-cycle.ts +45 -7
- package/src/task-agent/repair-worker.ts +7 -0
- package/src/task-agent/run-identity.ts +200 -0
- package/src/task-agent/shadow.ts +50 -3
- package/src/task-agent/task-hand.ts +158 -5
- package/src/task-agent/train-landing.ts +77 -0
- package/src/task-agent/work-order.ts +104 -0
- package/src/task-orchestrator/surfaces/pod-command-job.ts +88 -28
- package/src/task-orchestrator/surfaces/pod-failure-reason.ts +88 -19
- package/src/task-orchestrator/surfaces/pod-host-acp-review.ts +2 -1
- package/src/task-orchestrator/surfaces/pod-image-source.ts +2 -1
- package/src/task-orchestrator/surfaces/pod-install-slots.ts +121 -9
- package/src/task-orchestrator/surfaces/pod-lease.ts +20 -6
- package/src/task-orchestrator/surfaces/pod-lifecycle-stages.ts +169 -0
- package/src/task-orchestrator/surfaces/pod-mirror-seed.ts +27 -0
- package/src/task-orchestrator/surfaces/pod-result-marker-wait.ts +33 -0
- package/src/task-orchestrator/surfaces/pod-reuse-cleanup.ts +63 -0
- package/src/task-orchestrator/surfaces/pod-source-receive.ts +19 -3
- package/src/task-orchestrator/surfaces/pod-warm-switch.ts +141 -0
- package/src/task-orchestrator/surfaces/pod-warm-ta-shadow.ts +85 -0
- package/src/task-orchestrator/surfaces/pod-warm-worker-protocol.ts +153 -0
- package/src/task-orchestrator/surfaces/pod-warm-worker-runtime.ts +208 -0
- package/src/task-orchestrator/surfaces/self-implement-pod.ts +132 -15
- package/src/telegram.ts +2 -4
- package/src/tui/input/pickers.ts +732 -0
- package/src/tui/slash/mission.ts +492 -0
- package/src/tui/slash/term.ts +877 -0
- package/src/tui-client/dashboard-session.ts +9 -0
- package/src/tui.ts +48 -0
- package/src/user-config/gemini-server-tools.ts +9 -0
- package/src/user-config.ts +53 -16
- package/src/util/spawn-sync-output.ts +7 -0
- package/src/ux-sim/fake-bin.ts +154 -0
- package/src/ux-sim/fake-clock.ts +36 -0
- package/src/ux-sim/headless-tui.ts +72 -0
- package/src/ux-sim/in-process-cli.ts +2 -1
- package/src/ux-sim/temp-git-remote.ts +82 -0
- package/src/version/packed-revision.json +1 -1
- package/src/video-pipeline/recipes/vlog.ts +25 -7
- package/apps/pwa/out/_next/static/chunks/1015-9cba22a5a4554797.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1084-4e66af7376a18db7.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1106.3a6410b7b0c70ad4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1127.32fc642f70423506.js +0 -8
- package/apps/pwa/out/_next/static/chunks/1189-d9b8e1995a831fae.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1224.6da98c7ad0eb7dd8.js +0 -34
- package/apps/pwa/out/_next/static/chunks/1370.4e3e2558f111fd5d.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1514-b7da377fdc4eb17a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1577.3c24c32584e10d70.js +0 -286
- package/apps/pwa/out/_next/static/chunks/161.130a569855c4d68c.js +0 -24
- package/apps/pwa/out/_next/static/chunks/1864.16da9556e78a9236.js +0 -1
- package/apps/pwa/out/_next/static/chunks/1b6f6b2a.aff090122be3726e.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2341.e8a9135e5b910dd5.js +0 -36
- package/apps/pwa/out/_next/static/chunks/257.10afdbe3520f2fc2.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2658.87550e692dc493f6.js +0 -64
- package/apps/pwa/out/_next/static/chunks/2673.85e2f02efae7a85f.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2766-bee623b37472ed63.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2794.6d836872b0bf43e2.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2806.eb57ced1a528ed39.js +0 -1
- package/apps/pwa/out/_next/static/chunks/283.7a121c4910e6dad7.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2870.1dc29aa5fc01f09f.js +0 -115
- package/apps/pwa/out/_next/static/chunks/293.fd38e77ce5d48103.js +0 -1
- package/apps/pwa/out/_next/static/chunks/2993-5952a1cd3fd5d441.js +0 -1
- package/apps/pwa/out/_next/static/chunks/32-a523427c5bd5ed00.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3280.af38002a6424a905.js +0 -90
- package/apps/pwa/out/_next/static/chunks/3292-5ddffdb2f517b44f.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3523.d3231daaf7aba993.js +0 -1
- package/apps/pwa/out/_next/static/chunks/3763.577de3181e1e6ea0.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4138-cc56b0938ef72783.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4255-8350c43d307a8875.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4352.a4954693499c2714.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4421-f2dcbc6fd80ec013.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4513-51b94aeee37c9905.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4928-c3b72f99aa588699.js +0 -1
- package/apps/pwa/out/_next/static/chunks/4986-f9fc0283fb325219.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5014.671a7b9f69c2b7b1.js +0 -94
- package/apps/pwa/out/_next/static/chunks/5137.d398e24710b3feb6.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5298.0bea74885cabe754.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5315.74c53a39739b11c4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5317-8b1df8d02105cd07.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5358-33051f41a2b239af.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5388.a2ec02501f714ad5.js +0 -62
- package/apps/pwa/out/_next/static/chunks/5395.e6ac948a2d15058d.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5409-f282bf533c151849.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5466.d0f4aba9d52acfa8.js +0 -27
- package/apps/pwa/out/_next/static/chunks/5564.3d28b353470fe305.js +0 -104
- package/apps/pwa/out/_next/static/chunks/569.b0cbfb4dee4a167a.js +0 -93
- package/apps/pwa/out/_next/static/chunks/5835-72465b73a8bb8d74.js +0 -1
- package/apps/pwa/out/_next/static/chunks/5941.045d6093139e1f7a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6228.47cabb83564dfe54.js +0 -24
- package/apps/pwa/out/_next/static/chunks/6457-8d1993002a87d0fe.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6513.ef54cfae52351982.js +0 -2
- package/apps/pwa/out/_next/static/chunks/6582.13d245bf87df69f1.js +0 -148
- package/apps/pwa/out/_next/static/chunks/6655.c33f6acc2c61bb89.js +0 -149
- package/apps/pwa/out/_next/static/chunks/6731-9d7195af289c8d4c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6733.0be05b1db9279fc5.js +0 -79
- package/apps/pwa/out/_next/static/chunks/6742-90937146253a8b9e.js +0 -1
- package/apps/pwa/out/_next/static/chunks/687-816858f77a396a4e.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6888.72c7b8cfb02a7e61.js +0 -1
- package/apps/pwa/out/_next/static/chunks/6993.c17c828ac3ef3253.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7076.9f27ef666b0cefdd.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7113.1403c5d451804e8c.js +0 -287
- package/apps/pwa/out/_next/static/chunks/7169.9dd603c726c2cd7c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7247-e3ec21f761f86710.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7261-a47b72f96f5199df.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7335-5cda81ac90310077.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7356-c2e4af090546e2d4.js +0 -1
- package/apps/pwa/out/_next/static/chunks/738-59518742078fda71.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7385.a7375f10db73243a.js +0 -77
- package/apps/pwa/out/_next/static/chunks/7445.cf1cf2fe109bcd96.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7505.a2057432ab6ef99a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/7515.31f45fac1f9c70cb.js +0 -74
- package/apps/pwa/out/_next/static/chunks/7658-d4ce8c72a9548a91.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8107-0b1ae46bc5bcb788.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8132.bfea2f016ce8d992.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8672.e299832bc7d18ee2.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8735.f1ef798a5f260f6f.js +0 -226
- package/apps/pwa/out/_next/static/chunks/8774-88f67f7d8d53d299.js +0 -1
- package/apps/pwa/out/_next/static/chunks/8782.252e5a405d86e4ed.js +0 -32
- package/apps/pwa/out/_next/static/chunks/909-9ed7aa0dba804316.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9100.22b0301256827007.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9140-dde77575570bc71e.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9292-a7ad9a01c7044b93.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9517.00627dc51b02821c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9725.dcbb585652d8e9c5.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9761.a3ebf5c389c24ddc.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9972.834d2b2b7d48f1ed.js +0 -1
- package/apps/pwa/out/_next/static/chunks/9995-1a90486f5ae01546.js +0 -1
- package/apps/pwa/out/_next/static/chunks/a435ad95.577add051b4facb2.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/_not-found/page-83372d8b5863ebf7.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/approvals/page-d69256d7613c140a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/autopilot/page-352c5e9ba3f9b858.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/board/page-69f1d725524201a8.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/bots/page-ad9181d84285e4ea.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ceo/page-86feb08080261a2f.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/chat/page-ffa15987e4756d03.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/consult/page-11a3ab44ac327cc1.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/control/page-6309b11c4264d24d.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/decisions/page-8800472079612e87.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/design-check/page-d387bf3f2301882a.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/editor/page-048f88d8c8658da6.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/exec/page-6c14f9739b7f6880.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/field/page-2834f44f64e98d74.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/healthz/route-83372d8b5863ebf7.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/inside/page-a13df6f88e824887.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/intake/page-8a4397248754cf2b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/layout-daf0bd329360a336.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/live/page-f7ef1c28764cb476.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/live-run/page-61e74f28b9353dac.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/loops/page-8aaa6dedc4c10dfb.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/market/page-294602ba649be829.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/missions/[id]/page-5fa5f41ed3fce323.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/morning/page-33bf667a5fb297bf.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/not-found-5ae3cbd6e5dba515.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/observatory/page-eb867d16ad6fdede.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/checklist/page-1ab789fa68682f7e.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/release/page-6fd41b5b6ff13a71.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/ops/seats/page-fb3d5f7642c99fba.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/outputs/page-70f705b2b5ed42ca.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/page-c9c93b358a7667b5.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/reflection/page-1563c556b81c96ed.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/scheduler/page-ce89c20c45b3a673.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/sessions/page-423ae63d786f4ee9.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/settings/devices/page-591a5378a4c4447c.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/settings/page-a250496b19a5824b.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/setup/done/page-c36a9277febc5911.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/setup/layout-83372d8b5863ebf7.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/share/page-a5de57edc6d7f942.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/showroom/page-16bdd4c9f40498df.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/tasks/page-75e946e6e71cc6d8.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/term/page-9eb56166774eb256.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/today/page-1c5f9ef35dbb22c5.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/trace/page-657ff9985e508498.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/vault/page-22dc72499da68fb2.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/workflows/page-094ad334e40f8169.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/workspace/page-8bcd379852ff6287.js +0 -1
- package/apps/pwa/out/_next/static/chunks/app/worktrees/page-6b04413f280cd344.js +0 -1
- package/apps/pwa/out/_next/static/chunks/de70ab42-d28d47437208d458.js +0 -1
- package/apps/pwa/out/_next/static/chunks/f800c5e3.46dbd7f2af8c3fce.js +0 -1
- package/apps/pwa/out/_next/static/chunks/f8bc19ad.47579f6d684b0760.js +0 -122
- package/apps/pwa/out/_next/static/chunks/framework-ca9ac4609fb9fed8.js +0 -1
- package/apps/pwa/out/_next/static/chunks/main-app-0dd08d32d15a030f.js +0 -1
- package/apps/pwa/out/_next/static/chunks/main-f0399d33955c4e01.js +0 -1
- package/apps/pwa/out/_next/static/chunks/pages/_app-d42caca6f8ac2211.js +0 -1
- package/apps/pwa/out/_next/static/chunks/pages/_error-42e713c13aa41ca8.js +0 -1
- package/apps/pwa/out/_next/static/chunks/webpack-8fbf8f61a377aa16.js +0 -1
- /package/apps/pwa/out/_next/static/{cIpv5nU9AycwYWZI9H589 → LSXfY0u9QtfDbejY0Ukf_}/_ssgManifest.js +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import { spawn, spawnSync } from 'node:child_process';
|
|
3
3
|
import { appendFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
4
|
-
import { tmpdir } from 'node:os';
|
|
4
|
+
import { hostname, tmpdir } from 'node:os';
|
|
5
5
|
import { createHash, randomUUID } from 'node:crypto';
|
|
6
6
|
import { dirname, join, resolve } from 'node:path';
|
|
7
7
|
import { debug } from '../../src/debug/log.js';
|
|
@@ -13,9 +13,12 @@ import { diffFailures, junitFailures, parseFailures, parseTimedOutFailures } fro
|
|
|
13
13
|
import { loadEnvKnownFailures, splitKnownEnv } from './env-known-failures';
|
|
14
14
|
import { loadGateBatchSafe } from './gate-batch-safe';
|
|
15
15
|
import { baselineFromCutLogs } from './baseline-from-logs';
|
|
16
|
-
import { mergePartialFailures, readPartialPlan, testFileOfId } from './gate-partial';
|
|
16
|
+
import { mergePartialFailures, readPartialPlan, testFileOfId, verifyDocsOnlyCarry } from './gate-partial';
|
|
17
17
|
import { logPlanWalltime, planDurations, readFileMemory } from './shard-plan';
|
|
18
|
+
import { defaultGateHostMirrorReaders, resolveGateHostMirror, type GateHostMirror, type GateHostMirrorReaders } from './gate-host-mirror';
|
|
18
19
|
import { isGateSpawnLaneFile, planGateSpawnSeparatedShards } from './spawn-lane';
|
|
20
|
+
import { GATE_POD_ENV_BASELINE, gateTestBaseline } from '../lib/gate-test-env.js';
|
|
21
|
+
export { GATE_POD_ENV_BASELINE } from '../lib/gate-test-env.js';
|
|
19
22
|
import { readJunitFileSeconds } from './gate-method';
|
|
20
23
|
import { deriveCdpTestPatterns } from '../test-deterministic';
|
|
21
24
|
import { emitNodeResult, readGraphContext } from './node-verdict.js';
|
|
@@ -25,14 +28,26 @@ import { POD_BUN_CACHE_HOST_PATH, parseInstallSeconds, podBunCacheVolume } from
|
|
|
25
28
|
import { PodPoolScheduler, parsePodPool, checkPodPool, defaultRemoteRun, type PodPoolMember } from '../../src/task-orchestrator/surfaces/pod-pool.js';
|
|
26
29
|
import { POD_INSTALL_SLOTS_DEFAULT, POD_INSTALL_SLOTS_HOST_PATH, installSlotScript } from '../../src/task-orchestrator/surfaces/pod-install-slots.js';
|
|
27
30
|
import { gateDepsInstallScript, parseGateDeps } from '../../src/task-orchestrator/surfaces/pod-gate-deps.js';
|
|
31
|
+
import { writeWarmJobManifest } from './gate-warm-artifacts.js';
|
|
28
32
|
import { gateShardsPath, recordGateShardsVerdict, writeGateShards, type GateShard } from '../../src/release-loop/gate-shards.js';
|
|
29
33
|
import { defaultKubectl } from '../../src/task-orchestrator/surfaces/self-implement-pod.js';
|
|
30
34
|
import { executeTuiNodeCheck, type ExecutedTuiCheck } from './tui-node-checks.js';
|
|
35
|
+
import { emitPodLifecycle, gatePodLifecycleEvent, gatePodOutcome } from '../../src/task-orchestrator/surfaces/pod-lifecycle-stages.js';
|
|
36
|
+
import { gateHostReservationRoot, reserveReleaseGateHosts } from '../../src/release-loop/gate-host-reservation.js';
|
|
37
|
+
import { normRoot } from '../../src/instance/resolve.js';
|
|
31
38
|
|
|
32
|
-
interface CommandResult { rc: number; output: string; passedIds?: string[]; stalledEnv?: Array<{ file: string; reason: 'no-output' }>; timedOut?: boolean
|
|
39
|
+
interface CommandResult { rc: number; output: string; passedIds?: string[]; stalledEnv?: Array<{ file: string; reason: 'no-output' }>; timedOut?: boolean;
|
|
40
|
+
/** GATE-POD-STALL-FAILFAST ① — shard Pods whose tests the mid-run watch aborted in this sweep. */
|
|
41
|
+
abortedShards?: number;
|
|
42
|
+
/** Each contributing Pod leaf already passed failuresOf; repeated printed ids cannot add new failures. */
|
|
43
|
+
validatedShards?: boolean }
|
|
33
44
|
export interface GateRunner {
|
|
34
45
|
command(cmd: string, args: string[], cwd: string, limitMs?: number): Promise<CommandResult>;
|
|
35
46
|
localCommand(cmd: string, args: string[], cwd: string, limitMs?: number): Promise<CommandResult>;
|
|
47
|
+
/** Run one isolated test on the same Pod image and pinned commit as the sweep. */
|
|
48
|
+
recheckPod?(file: string, commit: string, pod: NonNullable<GateOptions['pod']>, timeoutMs: number, relaxed?: boolean): Promise<CommandResult>;
|
|
49
|
+
/** One checkout/install for a small independent file group; each file retains its own rc and log. */
|
|
50
|
+
recheckPodBatch?(files: readonly string[], commit: string, pod: NonNullable<GateOptions['pod']>, timeoutMs: number): Promise<Map<string, CommandResult>>;
|
|
36
51
|
/** `only` (GATE-PARTIAL) restricts the sweep to these test files; absent = every test file. */
|
|
37
52
|
sweep(tree: string, logDir?: string, pod?: GateOptions['pod'], only?: readonly string[]): Promise<CommandResult>;
|
|
38
53
|
add(tree: string, commit: string): Promise<void>;
|
|
@@ -57,13 +72,23 @@ export interface GateOptions {
|
|
|
57
72
|
repo?: string;
|
|
58
73
|
/** Rehearsal's opt-in screen measurement; normal release gates keep their existing defaults. */
|
|
59
74
|
rehearsalTui?: boolean;
|
|
75
|
+
/** Files per final recheck window; every window is measured (default 32). */
|
|
76
|
+
recheckFileLimit?: number;
|
|
60
77
|
/** freshShards — GATE-SHARD-RESUME off: every shard starts a Pod even when this logDir already holds its result. */
|
|
61
78
|
pod?: { pool: string; shards?: number; shardTimeoutSeconds?: number; durationSource?: string; bunCache?: string; freshShards?: boolean;
|
|
79
|
+
/** Maximum concurrent file-isolation fallback Jobs across the entire sweep; omitted = 4. */
|
|
80
|
+
isolationConcurrency?: number;
|
|
62
81
|
/** GATE-SKIP-KNOWN-RETRY — failure ids the previous release already had (and env-known ones): a shard that fails only
|
|
63
82
|
* with these is not re-run in isolation; the verdict still classifies them against the baseline as before. */
|
|
64
83
|
knownFailures?: readonly string[];
|
|
65
84
|
/** GATE-SPEED A3① — shard Pod CPU request/limit (`release.loop.gatePodCpu`); omitted = request 1 / limit 4. */
|
|
66
85
|
cpu?: PodCpu;
|
|
86
|
+
/** GATE-HOSTMIRROR-EXPLICIT — host Git mirror for every gate Pod (wins over env/config); omitted = resolved once per
|
|
87
|
+
* runner by {@link resolveGateHostMirror} (env → config → operating config when isolated → none). */
|
|
88
|
+
hostMirror?: string;
|
|
89
|
+
/** GATE-HOSTMIRROR-EXPLICIT — injected readers (tests · embedders). Absent = the real ones; an injected pool without
|
|
90
|
+
* them reads env only — never a real config from a test. */
|
|
91
|
+
hostMirrorReaders?: GateHostMirrorReaders;
|
|
67
92
|
/** GATE-INSTALL-CACHE — concurrent `bun install`s per node (`release.loop.gatePodInstallSlots`); omitted = 4 · 0 = off. */
|
|
68
93
|
installSlots?: number;
|
|
69
94
|
/** GATE-MEM-ADMIT — seconds a shard waits for pool admission (memory headroom) before it launches anyway with
|
|
@@ -81,6 +106,13 @@ export interface GateOptions {
|
|
|
81
106
|
stopJob?: (job: { name: string; context?: string }) => boolean | void;
|
|
82
107
|
/** How often the stall watch reads the live log (ms); omitted = {@link GATE_POD_STALL_POLL_MS}. */
|
|
83
108
|
stallPollMs?: number;
|
|
109
|
+
/** GATE-POD-STALL-FAILFAST ① — seconds a shard may go without its part logs growing once test output began, before
|
|
110
|
+
* only its test process group is ended in the Pod (`gatePodMidRunStallSeconds`); omitted =
|
|
111
|
+
* {@link GATE_POD_MIDRUN_STALL_SECONDS_DEFAULT} · 0 = off. Needs the stall watch (live log). */
|
|
112
|
+
midRunStallSeconds?: number;
|
|
113
|
+
/** GATE-POD-STALL-FAILFAST ① — injected in-Pod test abort (tests · embedders); false = not taken. Absent = `kubectl exec`
|
|
114
|
+
* of {@link gateShardAbortScript} on the gate's own pool; an injected pool without it runs without the mid-run watch. */
|
|
115
|
+
abortTests?: (job: { name: string; context?: string }, reason: string) => boolean | void;
|
|
84
116
|
/** GATE-LIVE-OBS — the shard state file (`gateShardsPath(version)`) this sweep keeps current; absent = not written. */
|
|
85
117
|
shardsFile?: { path: string; version: string };
|
|
86
118
|
/** GATE-MULTI-HOST — is this pool member reachable (kubectl ns ⊕ ssh)? Injected by tests; absent = the real probe on
|
|
@@ -88,7 +120,9 @@ export interface GateOptions {
|
|
|
88
120
|
probeMember?: (member: PodPoolMember) => boolean;
|
|
89
121
|
/** GATE-MULTI-HOST — seconds a placed shard's Job status may stay unreadable before the shard counts as lost with its
|
|
90
122
|
* member; omitted = {@link GATE_MEMBER_UNREADABLE_SECONDS_DEFAULT}. Only a pool of two or more members uses it. */
|
|
91
|
-
memberUnreadableSeconds?: number
|
|
123
|
+
memberUnreadableSeconds?: number;
|
|
124
|
+
/** GATE-WARM-BOOT-SCAN — observation only: the sweep's test selection (list · filter · duration plan · shard plan) time. */
|
|
125
|
+
onSelectionTimed?: (timed: { ms: number; files: number }) => void };
|
|
92
126
|
}
|
|
93
127
|
export interface GateResult {
|
|
94
128
|
outcome: 'ok' | 'regression' | 'error';
|
|
@@ -101,14 +135,18 @@ export interface GateResult {
|
|
|
101
135
|
knownEnvCleared: string[];
|
|
102
136
|
stalledEnv: Array<{ file: string; reason: 'no-output' | 'isolated-timeout'; local: 'passed' | 'failed' | 'timeout' }>;
|
|
103
137
|
rcNonzeroNoFailures: Array<{ file: string; rc: number; pass: number; baseline: 'same' | 'clean' | 'missing' }>;
|
|
104
|
-
/**
|
|
138
|
+
/** Files left unmeasured after isolated recheck; these cannot seed a reusable baseline. */
|
|
105
139
|
unmeasuredRecheckFiles?: string[];
|
|
140
|
+
/** GATE-POD-STALL-FAILFAST ① — cut sweep shard Pods ended by the mid-run abort (their files went the timeout way). */
|
|
141
|
+
abortedShards?: number;
|
|
106
142
|
baselineSource?: 'ledger' | 'instance' | 'cut-logs' | 'swept';
|
|
107
143
|
/** GATE-BASELINE-CACHE — isolated baseline files answered from the ledger (hits) or run now (misses). */
|
|
108
144
|
baselineCache?: { hits: number; misses: number };
|
|
109
145
|
/** GATE-SPEED A4 — a swept baseline ran only the cut's failing files that exist at the previous release. */
|
|
110
146
|
baselineDeferred?: { cutFiles: number; swept: number };
|
|
111
147
|
durationMs: number;
|
|
148
|
+
/** GATE-WARM-BOOT-SCAN — observation only: the cut sweep's test selection time (pod sweep · absent otherwise). */
|
|
149
|
+
selectionMs?: number;
|
|
112
150
|
error?: string;
|
|
113
151
|
stalledShards?: Array<{ shard: number; files: string[]; reason: 'incomplete' | 'no-output' | 'job-failed' | 'exit-code' | 'unattributed' | 'timeout'; lastFile?: string; summaryFailures?: number; namedFailures?: number; detail?: string;
|
|
114
152
|
/** GATE-POD-STALL-FAILFAST — the shard (and its one fresh retry) printed no test output within the stall window: unmeasured. */
|
|
@@ -119,6 +157,10 @@ export interface GateResult {
|
|
|
119
157
|
/** GATE-INTRO-RECHECK — new failures that were only the «timed out» shape and passed one isolated re-run with a
|
|
120
158
|
* {@link GATE_TIMEOUT_RECHECK_MS} per-test limit: a warning (flaky under load), not an introduced regression. */
|
|
121
159
|
loadFlaky?: string[];
|
|
160
|
+
/** File-level unhandled errors that reproduced or could not be measured in isolation. They keep the gate red. */
|
|
161
|
+
unmeasuredErrors?: Array<{ file: string; reason: 'reproduced' | 'timeout' | 'unreadable'; ids: string[] }>;
|
|
162
|
+
/** Shard-only unhandled errors whose file-level replay completed cleanly. */
|
|
163
|
+
interferenceErrors?: string[];
|
|
122
164
|
rehearsalTui?: ExecutedTuiCheck;
|
|
123
165
|
}
|
|
124
166
|
|
|
@@ -190,7 +232,6 @@ function unhandledErrorIds(output: string): string[] | undefined {
|
|
|
190
232
|
if (/^# Unhandled error between tests/.test(raw.trim())) {
|
|
191
233
|
if (!file) return undefined;
|
|
192
234
|
errors.push(`${file} > [error]`);
|
|
193
|
-
file = undefined;
|
|
194
235
|
}
|
|
195
236
|
}
|
|
196
237
|
return errors;
|
|
@@ -295,13 +336,7 @@ export function shardParts(paths: readonly string[], seconds: (file: string) =>
|
|
|
295
336
|
* (`scripts/lib/deterministic-env.ts` `isCredentialKey`), so a test never reaches a real provider from the Pod.
|
|
296
337
|
* Binaries the image lacks (`man` pages — `git <sub> --help`) are not fixed here; see MANUAL-gate-regression-response-recipe §2a.
|
|
297
338
|
*/
|
|
298
|
-
export
|
|
299
|
-
TZ: 'UTC',
|
|
300
|
-
LANG: 'C.UTF-8',
|
|
301
|
-
LC_ALL: 'C.UTF-8',
|
|
302
|
-
ELANOUS_TEST_FAKE_LLM: '1',
|
|
303
|
-
});
|
|
304
|
-
export function gateEnvBaselineLine(env: Readonly<Record<string, string>> = GATE_POD_ENV_BASELINE): string {
|
|
339
|
+
export function gateEnvBaselineLine(env: Readonly<Record<string, string>> = gateTestBaseline(process.env)): string {
|
|
305
340
|
return `export ${Object.entries(env).map(([key, value]) => `${key}=${quote(value)}`).join(' ')}`;
|
|
306
341
|
}
|
|
307
342
|
/**
|
|
@@ -311,7 +346,7 @@ export function gateEnvBaselineLine(env: Readonly<Record<string, string>> = GATE
|
|
|
311
346
|
* `shard.timeout` so the artifact reader does not mistake an unfinished child for a test verdict.
|
|
312
347
|
* A part without `.rc` did not finish. The shell itself always exits 0 (as before).
|
|
313
348
|
*/
|
|
314
|
-
export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPatterns: readonly string[]; softSeconds: number; cachePrefix: string; installSlots: number; depsDir?: string; beatSeconds?: number; parallelParts?: number }): string {
|
|
349
|
+
export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPatterns: readonly string[]; softSeconds: number; cachePrefix: string; installSlots: number; depsDir?: string; beatSeconds?: number; parallelParts?: number; testTimeoutMs?: number; partLimitSeconds?: number; sourceBinding?: { cutSha: string; attemptId: string } }): string {
|
|
315
350
|
// The planner keeps spawn files in their own shards; reject a mixed part rather than silently lose its limit.
|
|
316
351
|
for (const part of o.parts) {
|
|
317
352
|
if (part.some(isGateSpawnLaneFile) && !part.every(isGateSpawnLaneFile)) throw new Error('gate spawn lane part mixed with regular tests');
|
|
@@ -319,7 +354,15 @@ export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPa
|
|
|
319
354
|
const spawnParts = o.parts.flatMap((part, index) => part.length && part.every(isGateSpawnLaneFile) ? [index] : []);
|
|
320
355
|
const commands = o.parts.map((part, index) => {
|
|
321
356
|
const ignores = o.cdpPatterns.filter((pattern) => part.includes(pattern)).flatMap((pattern) => ['--path-ignore-patterns', pattern]);
|
|
322
|
-
|
|
357
|
+
// One-file runs keep named pass lines so their JUnit identities can be
|
|
358
|
+
// cross-checked. The package script's --dots removes those names.
|
|
359
|
+
const runner = part.length === 1 ? ['bun', 'run', 'scripts/test-deterministic.ts'] : ['bun', 'run', 'test:deterministic'];
|
|
360
|
+
// Bash scopes this assignment to run_part; the mixed-part shell test
|
|
361
|
+
// checks that a following multi-file part gets no source tag. The common
|
|
362
|
+
// run_part body changes the shell hash once for this feature; keeping the
|
|
363
|
+
// multi-file invocation unchanged avoids another cache-key change.
|
|
364
|
+
const warmFile = part.length === 1 ? `WARM_REFERENCE_FILE=${quote(part[0]!)} ` : '';
|
|
365
|
+
return `${warmFile}run_part ${index} ${[...runner, ...(o.testTimeoutMs !== undefined ? ['--timeout', String(o.testTimeoutMs)] : spawnParts.includes(index) ? ['--timeout=120000'] : []), ...ignores, ...part.map(asPath)].map(quote).join(' ')}`;
|
|
323
366
|
});
|
|
324
367
|
// Only file-isolation batches use this path. A bounded group shares one clone/install, but every test process gets
|
|
325
368
|
// its own HOME, state and tmp directory; part logs/JUnit remain separate and the existing artifact reader judges them.
|
|
@@ -371,10 +414,17 @@ export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPa
|
|
|
371
414
|
' local left=$(( gate_end - $(date +%s) ))',
|
|
372
415
|
' if [ "$left" -le 0 ]; then echo "$i" > "$O/shard.timeout"; return 1; fi',
|
|
373
416
|
...(spawnParts.length ? [' if [ "$spawn_part" -eq 1 ] && [ "$left" -gt 120 ]; then left=120; fi'] : []),
|
|
417
|
+
...(o.partLimitSeconds ? [` if [ "$left" -gt ${o.partLimitSeconds} ]; then left=${o.partLimitSeconds}; fi`] : []),
|
|
374
418
|
' stage "part-$i-test-start"',
|
|
375
419
|
...(spawnParts.length ? [' local test_start; test_start=$(date +%s)'] : []),
|
|
376
|
-
|
|
420
|
+
` { if [ -n "\${WARM_REFERENCE_FILE:-}" ]; then printf "[warm-worker] file=%s\\n" "$WARM_REFERENCE_FILE"; ${o.sourceBinding ? `printf "[warm-worker] cut=%s attempt=%s\\n" ${quote(o.sourceBinding.cutSha)} ${quote(o.sourceBinding.attemptId)}; ` : ''}fi; "\${run_env[@]}" timeout -k 10 "$left" "$@" --reporter=junit --reporter-outfile="$O/part-$i.junit.xml"; } > "$O/part-$i.log" 2>&1`,
|
|
377
421
|
' local prc=$?',
|
|
422
|
+
' if [ -n "${WARM_REFERENCE_FILE:-}" ] && [ -f "$O/part-$i.junit.xml" ]; then',
|
|
423
|
+
' local junit_sha; junit_sha=$(sha256sum "$O/part-$i.junit.xml" 2>/dev/null | awk \'{print $1}\')',
|
|
424
|
+
' if [ -z "$junit_sha" ]; then junit_sha=$(shasum -a 256 "$O/part-$i.junit.xml" 2>/dev/null | awk \'{print $1}\'); fi',
|
|
425
|
+
' if [[ "$junit_sha" =~ ^[a-f0-9]{64}$ ]]; then printf "[warm-worker] junit-sha256=%s\\n" "$junit_sha" >> "$O/part-$i.log"; fi',
|
|
426
|
+
' fi',
|
|
427
|
+
...(o.partLimitSeconds ? [' if [ "$prc" -eq 124 ] || [ "$prc" -eq 137 ]; then echo "$i" > "$O/part-$i.timeout"; fi'] : []),
|
|
378
428
|
...(spawnParts.length ? [' local test_end; test_end=$(date +%s)'] : []),
|
|
379
429
|
' stage "part-$i-test-end"',
|
|
380
430
|
...(isolateParts ? [' stage "part-$i-cleanup-start"', ' cd "$source_repo"', ' rm -rf "$isolated"', ' stage "part-$i-cleanup-end"'] : []),
|
|
@@ -401,6 +451,9 @@ export function gateShardShell(o: { parts: readonly (readonly string[])[]; cdpPa
|
|
|
401
451
|
* shards ran to their 1697 s deadline with `finishedFiles: 0`, then retries and splits did the same for 2h+ and the node
|
|
402
452
|
* never finished. The beat lets the host see «no test output yet» while the Job runs:
|
|
403
453
|
* `ELANOUS_GATE_BEAT <seq> <slot|install|test> <0|1>` — the last field is 1 once any part log has a byte.
|
|
454
|
+
* GATE-POD-STALL-FAILFAST ① (mid-run): once there is test output the beat adds the part logs' total bytes —
|
|
455
|
+
* `ELANOUS_GATE_BEAT <seq> test 1 <bytes>` — so the host can see output stop after it started. The old 4-field line is
|
|
456
|
+
* still read (no bytes = no mid-run watch).
|
|
404
457
|
* It is observation only: the host hashes the shell without it into `runKey`.
|
|
405
458
|
*/
|
|
406
459
|
export function gateBeatScript(beatSeconds: number, installSlots: boolean): string[] {
|
|
@@ -408,13 +461,13 @@ export function gateBeatScript(beatSeconds: number, installSlots: boolean): stri
|
|
|
408
461
|
'gate_beat() {',
|
|
409
462
|
' local n=0 st t f',
|
|
410
463
|
' while :; do',
|
|
411
|
-
' n=$((n + 1)); t=0',
|
|
412
|
-
' for f in "$O"/part-*.log; do if [ -s "$f" ]; then t=1;
|
|
464
|
+
' n=$((n + 1)); t=0; b=0',
|
|
465
|
+
' for f in "$O"/part-*.log; do if [ -s "$f" ]; then t=1; b=$(( b + $(wc -c < "$f" 2>/dev/null || echo 0) )); fi; done',
|
|
413
466
|
' if [ -f "$O/install.rc" ]; then st=test',
|
|
414
467
|
// A deps miss with no slot line yet = waiting for an install slot (bounded by the slot script; not a stall).
|
|
415
468
|
...(installSlots ? [" elif [ -f \"$O/install.log\" ] && grep -q '^\\[gate\\] deps .* miss' \"$O/install.log\" 2>/dev/null && ! grep -q '^\\[gate\\] install-slot' \"$O/install.log\" 2>/dev/null; then st=slot"] : []),
|
|
416
469
|
' else st=install; fi',
|
|
417
|
-
' echo "ELANOUS_GATE_BEAT $n $st $t"',
|
|
470
|
+
' if [ "$t" = 1 ]; then echo "ELANOUS_GATE_BEAT $n $st $t $b"; else echo "ELANOUS_GATE_BEAT $n $st $t"; fi',
|
|
418
471
|
` sleep ${Math.max(1, Math.floor(beatSeconds))} >/dev/null 2>&1 </dev/null`,
|
|
419
472
|
' done',
|
|
420
473
|
'}',
|
|
@@ -427,15 +480,18 @@ export const GATE_POD_STALL_SECONDS_DEFAULT = 300;
|
|
|
427
480
|
export const GATE_POD_STALL_POLL_MS = 15_000;
|
|
428
481
|
/** At least ten beats per stall window, never slower than one per 30 s. */
|
|
429
482
|
export const gateStallBeatSeconds = (stallSeconds: number) => Math.min(30, Math.max(1, Math.floor(stallSeconds / 10)));
|
|
430
|
-
export interface ShardLiveLog { checkout: boolean; beat?: { seq: number; stage: 'slot' | 'install' | 'test'; testOutput: boolean
|
|
483
|
+
export interface ShardLiveLog { checkout: boolean; beat?: { seq: number; stage: 'slot' | 'install' | 'test'; testOutput: boolean;
|
|
484
|
+
/** Part logs' total bytes (mid-run progress) — absent on the old beat line. */
|
|
485
|
+
bytes?: number } }
|
|
431
486
|
/** What the Pod's live stdout says so far: the source checkout line and the latest beat. */
|
|
432
487
|
export function readShardLiveLog(text: string): ShardLiveLog {
|
|
433
488
|
let checkout = false;
|
|
434
489
|
let beat: ShardLiveLog['beat'];
|
|
435
490
|
for (const line of text.split(/\r?\n/)) {
|
|
436
491
|
if (/^ELANOUS_POD_SOURCE (?:commit|default|pr) /.test(line)) checkout = true;
|
|
437
|
-
const match = /^ELANOUS_GATE_BEAT (\d+) (slot|install|test) ([01])\s*$/.exec(line);
|
|
438
|
-
if (match) beat = { seq: Number(match[1]), stage: match[2] as 'slot' | 'install' | 'test', testOutput: match[3] === '1'
|
|
492
|
+
const match = /^ELANOUS_GATE_BEAT (\d+) (slot|install|test) ([01])(?: (\d+))?\s*$/.exec(line);
|
|
493
|
+
if (match) beat = { seq: Number(match[1]), stage: match[2] as 'slot' | 'install' | 'test', testOutput: match[3] === '1',
|
|
494
|
+
...(match[4] !== undefined ? { bytes: Number(match[4]) } : {}) };
|
|
439
495
|
}
|
|
440
496
|
return { checkout, ...(beat ? { beat } : {}) };
|
|
441
497
|
}
|
|
@@ -489,12 +545,29 @@ export function gateBatchTestWallSeconds(stages: Record<string, number> | undefi
|
|
|
489
545
|
* checkout with no test output, not counting time the beat reported an install-slot wait. A silent shell (no beat at all)
|
|
490
546
|
* counts like any other stage. Only successful reads decide; a read failure never stalls a shard. `graceMs` covers the
|
|
491
547
|
* beat's report lag: output that appeared just inside the window is reported by the next beat before the stop decides.
|
|
548
|
+
*
|
|
549
|
+
* GATE-POD-STALL-FAILFAST ① (mid-run): with `midRunStallSeconds` > 0 and an `abortTests`, the watch does not end at the
|
|
550
|
+
* first test output when the beat carries the part logs' bytes. It keeps the time the bytes last changed; no change for
|
|
551
|
+
* `midRunStallSeconds` (⊕ grace) = a mid-run hang: `onAbort`, then `abortTests` ends only the test process group in the
|
|
552
|
+
* Pod (never the Job — Job backoffLimit 0). One abort per watch; a beat without bytes (old shell) ends the watch as before.
|
|
492
553
|
*/
|
|
493
554
|
export function watchShardStall(o: { stallSeconds: number; pollMs: number; read: () => string | null; onStall: (sinceCheckoutMs: number) => void; now?: () => number; graceMs?: number;
|
|
494
555
|
/** Stops the Job; false = not taken, asked again every poll until it is or the watch ends (a stalled Job must not run to its deadline). */
|
|
495
|
-
stopJob?: () => boolean | void
|
|
556
|
+
stopJob?: () => boolean | void;
|
|
557
|
+
/** Mid-run hang window in seconds (0 / absent = off). */
|
|
558
|
+
midRunStallSeconds?: number;
|
|
559
|
+
onAbort?: (info: { sinceLastOutputMs: number }) => void;
|
|
560
|
+
/** Ends the Pod's test process group; false = not taken, asked again on the next polls (at most {@link GATE_SHARD_ABORT_TRIES}). */
|
|
561
|
+
abortTests?: () => boolean | void }): { stop(): void; stalled(): { sinceCheckoutMs: number } | undefined; aborted(): ShardAbort | undefined; state(): ShardStallWatchState } {
|
|
496
562
|
const now = o.now ?? Date.now;
|
|
563
|
+
const midRunMs = o.midRunStallSeconds && o.midRunStallSeconds > 0 && o.abortTests ? o.midRunStallSeconds * 1000 : 0;
|
|
564
|
+
/** Mid-run phase: the last bytes the beat reported and when they last changed. */
|
|
565
|
+
let midRun: { bytes: number; changedAt: number } | undefined;
|
|
566
|
+
let abort: ShardAbort | undefined;
|
|
567
|
+
let abortTries = 0;
|
|
497
568
|
let checkoutAt: number | undefined;
|
|
569
|
+
/** Observation only (GATE-WARM-BOOT-SCAN): when the first test output was seen, relative to the checkout. */
|
|
570
|
+
let firstTestAt: { checkout: number; at: number } | undefined;
|
|
498
571
|
let windowStart: number | undefined;
|
|
499
572
|
let lastSeq: number | undefined;
|
|
500
573
|
let hit: { sinceCheckoutMs: number } | undefined;
|
|
@@ -508,6 +581,15 @@ export function watchShardStall(o: { stallSeconds: number; pollMs: number; read:
|
|
|
508
581
|
try { taken = o.stopJob ? o.stopJob() : true; } catch { taken = false; }
|
|
509
582
|
if (taken === false && !ended) timer = setTimeout(askStop, o.pollMs);
|
|
510
583
|
};
|
|
584
|
+
const askAbort = () => {
|
|
585
|
+
timer = undefined;
|
|
586
|
+
if (ended) return;
|
|
587
|
+
abortTries++;
|
|
588
|
+
let taken: boolean | void = false;
|
|
589
|
+
try { taken = o.abortTests!(); } catch { taken = false; }
|
|
590
|
+
if (taken !== false && abort) abort.taken = true;
|
|
591
|
+
if (taken === false && !ended && abortTries < GATE_SHARD_ABORT_TRIES) timer = setTimeout(askAbort, o.pollMs);
|
|
592
|
+
};
|
|
511
593
|
const tick = () => {
|
|
512
594
|
timer = undefined;
|
|
513
595
|
if (done) return;
|
|
@@ -517,7 +599,31 @@ export function watchShardStall(o: { stallSeconds: number; pollMs: number; read:
|
|
|
517
599
|
if (text !== null) {
|
|
518
600
|
const t = now();
|
|
519
601
|
const live = readShardLiveLog(text);
|
|
520
|
-
if (
|
|
602
|
+
if (midRun) {
|
|
603
|
+
// A beat without bytes (cannot judge progress) ends the mid-run watch; a failed or empty read decides nothing.
|
|
604
|
+
if (live.beat?.testOutput && live.beat.bytes === undefined) { done = true; return; }
|
|
605
|
+
if (live.beat?.bytes !== undefined && live.beat.bytes !== midRun.bytes) midRun = { bytes: live.beat.bytes, changedAt: t };
|
|
606
|
+
else if (t - midRun.changedAt >= midRunMs + (o.graceMs ?? 0)) {
|
|
607
|
+
done = true;
|
|
608
|
+
abort = { reason: `mid-run-stall ${o.midRunStallSeconds}s`, at: new Date().toISOString(), by: 'auto', sinceLastOutputMs: t - midRun.changedAt, taken: false };
|
|
609
|
+
try { o.onAbort?.({ sinceLastOutputMs: abort.sinceLastOutputMs }); } catch { /* observation only */ }
|
|
610
|
+
askAbort();
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
613
|
+
timer = setTimeout(tick, o.pollMs);
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
if (live.beat?.testOutput) {
|
|
617
|
+
const seenCheckout = checkoutAt ?? (live.checkout ? t : undefined);
|
|
618
|
+
if (seenCheckout !== undefined) firstTestAt = { checkout: seenCheckout, at: t };
|
|
619
|
+
if (midRunMs && live.beat.bytes !== undefined) {
|
|
620
|
+
midRun = { bytes: live.beat.bytes, changedAt: t };
|
|
621
|
+
timer = setTimeout(tick, o.pollMs);
|
|
622
|
+
return;
|
|
623
|
+
}
|
|
624
|
+
done = true;
|
|
625
|
+
return;
|
|
626
|
+
}
|
|
521
627
|
if (live.checkout && checkoutAt === undefined) { checkoutAt = t; windowStart = t; }
|
|
522
628
|
if (live.beat && live.beat.seq !== lastSeq) {
|
|
523
629
|
lastSeq = live.beat.seq;
|
|
@@ -534,7 +640,69 @@ export function watchShardStall(o: { stallSeconds: number; pollMs: number; read:
|
|
|
534
640
|
timer = setTimeout(tick, o.pollMs);
|
|
535
641
|
};
|
|
536
642
|
timer = setTimeout(tick, o.pollMs);
|
|
537
|
-
return { stop: () => { done = true; ended = true; if (timer) clearTimeout(timer); }, stalled: () => hit
|
|
643
|
+
return { stop: () => { done = true; ended = true; if (timer) clearTimeout(timer); }, stalled: () => hit, aborted: () => abort,
|
|
644
|
+
state: () => ({ armed: checkoutAt !== undefined || firstTestAt !== undefined,
|
|
645
|
+
disarmedBy: hit ? 'stall-fired' : abort ? 'mid-run-stall' : firstTestAt ? 'test-output' : null,
|
|
646
|
+
firstTestAtSec: firstTestAt ? Math.round((firstTestAt.at - firstTestAt.checkout) / 100) / 10 : null,
|
|
647
|
+
...(abort ? { abort: { reason: abort.reason, at: abort.at, by: abort.by } } : {}) }) };
|
|
648
|
+
}
|
|
649
|
+
/** GATE-POD-STALL-FAILFAST ① — a mid-run abort: why (`mid-run-stall <N>s`), when, who (`auto`; a manual knob is slice 2). */
|
|
650
|
+
export type ShardAbort = { reason: string; at: string; by: 'auto'; sinceLastOutputMs: number;
|
|
651
|
+
/** The in-Pod abort was taken (false = every try failed — the Job then runs to its deadline as before). */
|
|
652
|
+
taken: boolean };
|
|
653
|
+
/** How many polls ask the in-Pod abort before giving up (the Job then runs to its deadline as before). */
|
|
654
|
+
export const GATE_SHARD_ABORT_TRIES = 3;
|
|
655
|
+
/** GATE-POD-STALL-FAILFAST ① — default mid-run hang window: well under a shard deadline (≈1500 s+), far over one test's limit. */
|
|
656
|
+
export const GATE_POD_MIDRUN_STALL_SECONDS_DEFAULT = 600;
|
|
657
|
+
/** Seconds between the in-Pod TERM and KILL of the test process group. */
|
|
658
|
+
export const GATE_SHARD_ABORT_GRACE_SECONDS = 10;
|
|
659
|
+
/** What one shard's stall watch saw — armed (checkout seen) · why it ended · first test output seconds after the checkout. */
|
|
660
|
+
export type ShardStallWatchState = { armed: boolean; disarmedBy: 'test-output' | 'stall-fired' | 'mid-run-stall' | null; firstTestAtSec: number | null;
|
|
661
|
+
abort?: Omit<ShardAbort, 'sinceLastOutputMs' | 'taken'> };
|
|
662
|
+
/**
|
|
663
|
+
* GATE-POD-STALL-FAILFAST ① — run inside the shard Pod (`kubectl exec … bash -c`): ends only the running test process
|
|
664
|
+
* groups — the group led by a process whose argv carries `--reporter-outfile=…/outbox/part-<i>.junit.xml` (GNU `timeout`
|
|
665
|
+
* leads its own group) — TERM now, KILL after the grace. It never signals PID 1's group or its own. Before the signal it
|
|
666
|
+
* leaves `part-<i>.abort` (the reason) and `shard.timeout`, so the outbox reads like a deadline cut of that part
|
|
667
|
+
* ({@link readShardArtifacts}: unfinished, timed out) while the shell finishes the remaining parts and its outbox.
|
|
668
|
+
* Exit 0 = at least one group signalled.
|
|
669
|
+
*/
|
|
670
|
+
export function gateShardAbortScript(reason: string, graceSeconds = GATE_SHARD_ABORT_GRACE_SECONDS,
|
|
671
|
+
/** Only this outbox's parts (tests: never another run's groups on the same machine); absent = any `…/outbox/` in the Pod. */
|
|
672
|
+
outbox?: string): string {
|
|
673
|
+
return [
|
|
674
|
+
`reason=${quote(reason.replace(/[^\w .:-]/g, '_'))}`,
|
|
675
|
+
`only=${quote(outbox ?? '')}`,
|
|
676
|
+
"pg_of() { sed 's/^.*) //' \"/proc/$1/stat\" 2>/dev/null | cut -d' ' -f3; }",
|
|
677
|
+
'root_pg=$(pg_of 1); my_pg=$(pg_of $$); found=0',
|
|
678
|
+
'for d in /proc/[0-9]*; do',
|
|
679
|
+
' pid=${d#/proc/}',
|
|
680
|
+
" out=$(tr '\\0' '\\n' < \"$d/cmdline\" 2>/dev/null | grep -m1 -E '^--reporter-outfile=.*/outbox/part-[0-9]+\\.junit\\.xml$') || continue",
|
|
681
|
+
' if [ -n "$only" ]; then case "$out" in "--reporter-outfile=$only/part-"*) ;; *) continue;; esac; fi',
|
|
682
|
+
' pg=$(pg_of "$pid")',
|
|
683
|
+
' case "$pg" in ""|*[!0-9]*) continue;; esac',
|
|
684
|
+
' if [ "$pg" != "$pid" ] || [ "$pg" -le 1 ] || [ "$pg" = "$root_pg" ] || [ "$pg" = "$my_pg" ]; then continue; fi',
|
|
685
|
+
' out=${out#--reporter-outfile=}; dir=${out%/part-*}; part=${out##*/part-}; part=${part%.junit.xml}',
|
|
686
|
+
' printf "%s\\n" "$reason" > "$dir/part-$part.abort"',
|
|
687
|
+
' [ -f "$dir/shard.timeout" ] || echo "$part" > "$dir/shard.timeout"',
|
|
688
|
+
' if kill -TERM -- "-$pg" 2>/dev/null; then found=$((found + 1)); echo "ELANOUS_GATE_ABORT part=$part pgid=$pg"; fi',
|
|
689
|
+
` ( sleep ${Math.max(1, Math.floor(graceSeconds))}; kill -KILL -- "-$pg" 2>/dev/null ) >/dev/null 2>&1 </dev/null &`,
|
|
690
|
+
'done',
|
|
691
|
+
'echo "ELANOUS_GATE_ABORT signalled=$found"',
|
|
692
|
+
'[ "$found" -gt 0 ]',
|
|
693
|
+
].join('\n');
|
|
694
|
+
}
|
|
695
|
+
/** The shard json's `stallWatch` record (observation only — never read by any judgment). */
|
|
696
|
+
export type ShardStallWatchRecord = Omit<ShardStallWatchState, 'disarmedBy'> & { disarmedBy: ShardStallWatchState['disarmedBy'] | 'off' | 'no-live-log'; stallSeconds: number };
|
|
697
|
+
export function shardStallWatchRecord(stallSeconds: number, hasLiveLog: boolean, state: ShardStallWatchState | undefined): ShardStallWatchRecord {
|
|
698
|
+
if (!(stallSeconds > 0)) return { armed: false, disarmedBy: 'off', firstTestAtSec: null, stallSeconds };
|
|
699
|
+
if (!hasLiveLog) return { armed: false, disarmedBy: 'no-live-log', firstTestAtSec: null, stallSeconds };
|
|
700
|
+
return { ...(state ?? { armed: false, disarmedBy: null, firstTestAtSec: null }), stallSeconds };
|
|
701
|
+
}
|
|
702
|
+
/** GATE-WARM-BOOT-SCAN — logs the selection time and hands it to the caller; an observer failure never stops the sweep. */
|
|
703
|
+
export function reportSelectionTimed(ms: number, files: number, observer?: (timed: { ms: number; files: number }) => void): void {
|
|
704
|
+
debug.log('release-loop.gate', 'selection-timed', { ms, files });
|
|
705
|
+
try { observer?.({ ms, files }); } catch { /* observation only */ }
|
|
538
706
|
}
|
|
539
707
|
/** Removes bun's own summary lines so one summary over several runs can follow (same rule as the sweep's merge). */
|
|
540
708
|
const stripSummary = (output: string) => output.replace(/(?:^|\n)\s*\d+ (?:pass|fail|errors?)\s*(?=\n|$)/g, '\n').replace(/Ran \d+ tests? across \d+ files?\.?/g, '');
|
|
@@ -575,6 +743,13 @@ export function readShardArtifacts(dir: string, parts: readonly (readonly string
|
|
|
575
743
|
const log = read(`part-${index}.log`);
|
|
576
744
|
const rc = rcOf(read(`part-${index}.rc`));
|
|
577
745
|
const clean = log?.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '');
|
|
746
|
+
// GATE-POD-STALL-FAILFAST ① — a part the mid-run abort ended reads like a deadline cut: unfinished, its exit (143/137) no verdict.
|
|
747
|
+
const abort = read(`part-${index}.abort`);
|
|
748
|
+
if (abort !== undefined) {
|
|
749
|
+
bad.push(`# part ${index} (abort: ${abort.trim()})\n${log ?? ''}`);
|
|
750
|
+
otherBad = true;
|
|
751
|
+
return;
|
|
752
|
+
}
|
|
578
753
|
if (clean !== undefined && (rc === 0 || rc === 1) && lastMatch(clean, /Ran (\d+) tests? across (\d+) files?/)) {
|
|
579
754
|
const junit = read(`part-${index}.junit.xml`);
|
|
580
755
|
good.push({ log: clean, rc, files, ...(junit !== undefined ? { junit } : {}) });
|
|
@@ -608,6 +783,9 @@ export function readShardArtifacts(dir: string, parts: readonly (readonly string
|
|
|
608
783
|
};
|
|
609
784
|
}
|
|
610
785
|
|
|
786
|
+
/** The line appended to pod-N.log when part output that never finished went to pod-N.unfinished.log (a reader of pod-N.log alone saw «no test output»). */
|
|
787
|
+
export const gateUnfinishedHint = (relativePath: string, bytes: number) => `[gate] unfinished: ${relativePath} · ${bytes}B (진행 중 part 출력은 그 파일)`;
|
|
788
|
+
|
|
611
789
|
// review-model-ab: Pod 에서 출력 없이 멈춤 2/2(10-04 0.2.11 · 로컬 단독 4/0 · 1.7s) — GATE-STALL 수리 전까지 Pod 밖.
|
|
612
790
|
export const POD_SWEEP_INTEGRATION_ONLY: readonly string[] = ['scripts/install.test.ts', 'scripts/review-model-ab.test.ts'];
|
|
613
791
|
/** Nightly-only whole-repository cases; gate runs the fixture cases in these same files. */
|
|
@@ -693,7 +871,16 @@ function failuresOf(run: CommandResult, label: string): SweepFailures {
|
|
|
693
871
|
const incomplete = (reason: string) => new Error(`${label} incomplete (rc=${run.rc}; Ran=${ran ? ran[0] : 'missing'}; errors=${Number.isNaN(reportedErrors) ? 'missing' : errorCount}; ${reason})\n${tail40(output)}`);
|
|
694
872
|
const count = summaryCount(output, 'fail');
|
|
695
873
|
const failures = parseFailures(output);
|
|
696
|
-
|
|
874
|
+
// Bun repeats failed test names under "N tests failed:". A validated Pod sweep joins
|
|
875
|
+
// separately checked shards, so a summary in one shard can precede real failures
|
|
876
|
+
// from later shards. Remove each repeated block, not everything after the last one.
|
|
877
|
+
// Unhandled-error summaries have a separate file-attribution path below and do not
|
|
878
|
+
// use Bun's repeated `(fail)` block format.
|
|
879
|
+
const failureSummary = [...output.matchAll(/^\d+ tests? failed:\s*$/gm)].at(-1);
|
|
880
|
+
const executionOutput = run.validatedShards
|
|
881
|
+
? output.replace(/^\d+ tests? failed:\r?\n(?:\(fail\)[^\r\n]*(?:\r?\n|$))*(?:[ \t]*\r?\n)*/gm, '')
|
|
882
|
+
: failureSummary ? output.slice(0, failureSummary.index) : output;
|
|
883
|
+
const occurrences = [...executionOutput.matchAll(/^[ \t]*\(fail\)[ \t]+.+?(?:[ \t]+\[[\d.]+(?:ms|s)\])?[ \t]*\r?$/gm)].length;
|
|
697
884
|
// A test that leaves process.exitCode set leaks a non-zero exit into a run whose own summary is complete and clean
|
|
698
885
|
// (0.2.7 gate: agent-cli at v0.2.6 · 12 pass · 0 fail · rc 2). The summary is the verdict; read that exit as clean.
|
|
699
886
|
const leakedExit = !!ran && run.rc !== 0 && run.rc !== 1 && count === 0 && errorCount === 0 && occurrences === 0
|
|
@@ -709,11 +896,10 @@ function failuresOf(run: CommandResult, label: string): SweepFailures {
|
|
|
709
896
|
if (/^# Unhandled error between tests/.test(raw.trim())) {
|
|
710
897
|
if (!file) throw incomplete('unattributed unhandled error');
|
|
711
898
|
errors.push(`${file} > [error]`);
|
|
712
|
-
file = undefined;
|
|
713
899
|
}
|
|
714
900
|
}
|
|
715
901
|
if (errors.length !== errorCount) throw incomplete(`error attribution identified=${errors.length}`);
|
|
716
|
-
if (!Number.isFinite(count) || count !== occurrences || (count > 0 && failures.length === 0)
|
|
902
|
+
if (!Number.isFinite(count) || (count !== occurrences && !(run.validatedShards && occurrences > count && failures.length === count)) || (count > 0 && failures.length === 0)
|
|
717
903
|
|| (rc === 1 && count === 0 && errorCount === 0) || (rc === 0 && count + errorCount > 0)) {
|
|
718
904
|
throw incomplete(`failure attribution incomplete: summary=${count}, identified=${failures.length}`);
|
|
719
905
|
}
|
|
@@ -762,19 +948,22 @@ export function readIsolatedRun(run: CommandResult, label: string, file: string)
|
|
|
762
948
|
}
|
|
763
949
|
|
|
764
950
|
class StalledPodShards extends Error {
|
|
765
|
-
constructor(readonly stalledShards: NonNullable<GateResult['stalledShards']>, readonly partialSummary: NonNullable<GateResult['partialSummary']
|
|
951
|
+
constructor(readonly stalledShards: NonNullable<GateResult['stalledShards']>, readonly partialSummary: NonNullable<GateResult['partialSummary']>, readonly abortedShards = 0) {
|
|
766
952
|
const unmeasured = stalledShards.filter((item) => item.stall);
|
|
767
953
|
const exitCode = stalledShards.filter((item) => item.reason === 'exit-code');
|
|
768
954
|
super(`pod sweep incomplete: stalled shards ${stalledShards.map(({ shard }) => shard).join(', ')}`
|
|
769
955
|
+ (unmeasured.length ? ` (unmeasured: no test output within ${unmeasured[0]!.stall!.seconds}s of checkout — shards ${[...new Set(unmeasured.map(({ shard }) => shard))].join(', ')})` : '')
|
|
770
|
-
+ (exitCode.length ? ` (exit-code anomaly: 0 fail but exit ≠ 0/1 — ${[...new Set(exitCode.flatMap(({ files }) => files))].join(', ')})` : '')
|
|
956
|
+
+ (exitCode.length ? ` (exit-code anomaly: 0 fail but exit ≠ 0/1 — ${[...new Set(exitCode.flatMap(({ files }) => files))].join(', ')})` : '')
|
|
957
|
+
+ (abortedShards ? ` (중단 조각 ${abortedShards}: mid-run-stall)` : ''));
|
|
771
958
|
}
|
|
772
959
|
}
|
|
773
960
|
|
|
774
961
|
/** Isolated re-runs (host stall re-run · cut and baseline re-checks) are time-limited; sweeps and installs keep their full time. */
|
|
775
962
|
export const GATE_ISOLATED_TIMEOUT_MS = 300_000;
|
|
776
|
-
/**
|
|
963
|
+
/** Default final recheck window size; overflow continues in later windows. */
|
|
777
964
|
export const GATE_HOST_RECHECK_FILE_LIMIT = 32;
|
|
965
|
+
/** Two isolated test processes in each of four Pods = eight at most, with one checkout/install per eight files. */
|
|
966
|
+
export const GATE_RECHECK_POD_BATCH_FILES = 8;
|
|
778
967
|
/** GATE-INTRO-RECHECK (0.2.19) — the per-test limit of the one re-run given to a «timed out after 5000ms» new failure.
|
|
779
968
|
* 0.2.18: both first-gate introductions were 5 s limits missed at 6.04/6.06 s, and they alone cost an 84-minute re-gate. */
|
|
780
969
|
export const GATE_TIMEOUT_RECHECK_MS = 60_000;
|
|
@@ -880,6 +1069,15 @@ export async function admitGateShard(admission: GateAdmission | undefined, o: {
|
|
|
880
1069
|
|
|
881
1070
|
export function createGateRunner(repo: string, remote?: string, commandOverride?: GateRunner['command'], podCommand: (options: RunPodCommandOptions) => Promise<PodCommandResult> = runPodCommand, poolOverride?: PodPoolScheduler, podLogTail?: (job: PodCommandResult) => string): GateRunner {
|
|
882
1071
|
let remoteMirror: string | undefined;
|
|
1072
|
+
// GATE-HOSTMIRROR-EXPLICIT: resolved once per runner and handed to every podCommand (shard · retry · recheck).
|
|
1073
|
+
let gateMirror: GateHostMirror | undefined;
|
|
1074
|
+
const gateHostMirror = (pod: NonNullable<GateOptions['pod']>): { hostMirror?: string } => {
|
|
1075
|
+
if (!gateMirror) {
|
|
1076
|
+
gateMirror = resolveGateHostMirror(pod.hostMirror, pod.hostMirrorReaders ?? (poolOverride ? { env: process.env } : defaultGateHostMirrorReaders()));
|
|
1077
|
+
debug.log('release-loop.gate', 'host-mirror-resolved', { source: gateMirror.source, path: gateMirror.path });
|
|
1078
|
+
}
|
|
1079
|
+
return gateMirror.hostMirror ? { hostMirror: gateMirror.hostMirror } : {};
|
|
1080
|
+
};
|
|
883
1081
|
const localCommand: GateRunner['localCommand'] = async (cmd, args, cwd, limitMs) => {
|
|
884
1082
|
if (limitMs) return limitedLocalCommand(cmd, args, cwd, limitMs);
|
|
885
1083
|
const run = spawnSync(cmd, args, { cwd, encoding: 'utf8', maxBuffer: 128 * 1024 * 1024 });
|
|
@@ -901,14 +1099,17 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
901
1099
|
: localCommand);
|
|
902
1100
|
const podSweep = async (tree: string, logDir: string | undefined, pod: NonNullable<GateOptions['pod']>, only?: readonly string[]): Promise<CommandResult> => {
|
|
903
1101
|
const shardCount = pod.shards ?? 24;
|
|
1102
|
+
const isolationConcurrency = pod.isolationConcurrency ?? 4;
|
|
904
1103
|
// GATE-TIMEOUT-HEAL ①: the configured value is the floor; each shard's own deadline is shardDeadlineSeconds(planned).
|
|
905
1104
|
const deadlineSeconds = pod.shardTimeoutSeconds ?? POD_SHARD_TIMEOUT_FLOOR_SECONDS;
|
|
906
1105
|
if (!pod.pool || !Number.isSafeInteger(shardCount) || shardCount < 1
|
|
907
|
-
|| !Number.isSafeInteger(deadlineSeconds) || deadlineSeconds < 1
|
|
1106
|
+
|| !Number.isSafeInteger(deadlineSeconds) || deadlineSeconds < 1
|
|
1107
|
+
|| !Number.isSafeInteger(isolationConcurrency) || isolationConcurrency < 1) throw new Error('invalid pod sweep options');
|
|
908
1108
|
const head = await command('git', ['rev-parse', 'HEAD'], tree);
|
|
909
1109
|
check(head, 'cut tree HEAD');
|
|
910
1110
|
const commit = head.output.trim();
|
|
911
1111
|
if (!/^[0-9a-f]{40}$/.test(commit)) throw new Error('invalid cut tree HEAD');
|
|
1112
|
+
const selectionStart = Date.now();
|
|
912
1113
|
const listed = await command('git', ['ls-files', '*.test.*'], tree);
|
|
913
1114
|
check(listed, 'git ls-files tests');
|
|
914
1115
|
const allFiles = listed.output.split(/\r?\n/).filter((file) => /\.test\.(?:tsx?|jsx?|mts|cts)$/.test(file)).sort();
|
|
@@ -951,6 +1152,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
951
1152
|
maxPlannedSeconds: Math.max(...shards.map((shard) => shard.plannedSeconds)),
|
|
952
1153
|
source: durations.size ? pod.durationSource : 'none',
|
|
953
1154
|
});
|
|
1155
|
+
reportSelectionTimed(Date.now() - selectionStart, assignable.length, pod.onSelectionTimed);
|
|
954
1156
|
// GATE-SPEED A3①: shard Pod CPU (`release.loop.gatePodCpu` · default request 1 / limit 4) — shards × request must fit the pool.
|
|
955
1157
|
const cpu = resolvePodCpu(pod.cpu);
|
|
956
1158
|
// Only evidence-backed opt-in files reach this batch; every other file retains its own Pod below.
|
|
@@ -995,6 +1197,19 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
995
1197
|
});
|
|
996
1198
|
const stallWatch = stallSeconds > 0 && !!liveLog;
|
|
997
1199
|
debug.log('release-loop.gate', 'pod-stall-watch', { stallSeconds, source: pod.stallSeconds !== undefined ? 'config' : 'default', enabled: stallWatch });
|
|
1200
|
+
// GATE-POD-STALL-FAILFAST ①: a hang after test output began ends only the Pod's test process group (never the Job).
|
|
1201
|
+
const midRunStallSeconds = pod.midRunStallSeconds ?? GATE_POD_MIDRUN_STALL_SECONDS_DEFAULT;
|
|
1202
|
+
if (!Number.isFinite(midRunStallSeconds) || midRunStallSeconds < 0) throw new Error('invalid pod sweep options');
|
|
1203
|
+
const abortTests = pod.abortTests ?? (poolOverride ? undefined : (job: { name: string; context?: string }, reason: string) => {
|
|
1204
|
+
if (!job.context) return false;
|
|
1205
|
+
const ran = defaultKubectl(['--context', job.context, '-n', 'elanous-test', 'exec', `job/${job.name}`, '-c', 'child', '--', 'bash', '-c', gateShardAbortScript(reason)]);
|
|
1206
|
+
if (ran.status !== 0) debug.log('release-loop.gate', 'shard-abort-failed', { job: job.name, rc: ran.status, out: `${ran.stdout}${ran.stderr}`.trim().slice(0, 300) }, { level: 'warn' });
|
|
1207
|
+
return ran.status === 0;
|
|
1208
|
+
});
|
|
1209
|
+
const midRunWatch = stallWatch && midRunStallSeconds > 0 && !!abortTests;
|
|
1210
|
+
debug.log('release-loop.gate', 'pod-midrun-stall-watch', { midRunStallSeconds, source: pod.midRunStallSeconds !== undefined ? 'config' : 'default', enabled: midRunWatch });
|
|
1211
|
+
/** Shard Pods the mid-run watch aborted in this sweep (the verdict summary's «중단 조각 n»). */
|
|
1212
|
+
let abortedShards = 0;
|
|
998
1213
|
// GATE-MULTI-HOST: members found unreachable during this gate — skipped for placement while any other member is left.
|
|
999
1214
|
const downMembers = new Set<string>();
|
|
1000
1215
|
const multiHost = poolScheduler.members.length > 1;
|
|
@@ -1028,6 +1243,20 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1028
1243
|
type ShardReason = NonNullable<GateResult['stalledShards']>[number]['reason'];
|
|
1029
1244
|
// 깊이 끝 OOM 조각은 «조각 전체»를 파일 하나씩 격리한다 — 상한 40 이면 나머지가 못 잰 채 남았다(09-30 ① 7번 조각 110 파일).
|
|
1030
1245
|
const isolationRemaining = shards.map((item) => item.files.length);
|
|
1246
|
+
// One sweep-wide permit pool: independent root shards must not each launch their own unbounded file fan-out.
|
|
1247
|
+
let activeIsolation = 0;
|
|
1248
|
+
const isolationWaiters: Array<() => void> = [];
|
|
1249
|
+
const isolatedFile = async <T>(run: () => Promise<T>): Promise<T> => {
|
|
1250
|
+
if (activeIsolation < isolationConcurrency) activeIsolation++;
|
|
1251
|
+
else await new Promise<void>((resolve) => isolationWaiters.push(resolve));
|
|
1252
|
+
try { return await run(); }
|
|
1253
|
+
finally {
|
|
1254
|
+
const next = isolationWaiters.shift();
|
|
1255
|
+
if (next) next(); // transfer this permit directly; queued callers cannot exceed the cap
|
|
1256
|
+
else activeIsolation--;
|
|
1257
|
+
}
|
|
1258
|
+
};
|
|
1259
|
+
debug.log('release-loop.gate', 'pod-isolation-concurrency', { limit: isolationConcurrency, source: pod.isolationConcurrency === undefined ? 'default' : 'config' });
|
|
1031
1260
|
const knownFailures = new Set(pod.knownFailures ?? []);
|
|
1032
1261
|
const batchSafe = loadGateBatchSafe(repo);
|
|
1033
1262
|
debug.log('release-loop.gate', 'batch-safe-opt-in', { files: batchSafe.size });
|
|
@@ -1080,7 +1309,6 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1080
1309
|
const shellOptions = { parts, cdpPatterns, softSeconds: softDeadlineSeconds(shardDeadline), cachePrefix, installSlots, ...(batchIsolation ? { parallelParts: batchWidth } : {}) };
|
|
1081
1310
|
const keyShell = gateShardShell(shellOptions);
|
|
1082
1311
|
// The stall beat only reports; runKey keeps hashing the shell without it, so earlier results stay reusable.
|
|
1083
|
-
const podShell = stallWatch ? gateShardShell({ ...shellOptions, beatSeconds: gateStallBeatSeconds(stallSeconds) }) : keyShell;
|
|
1084
1312
|
const runKey = createHash('sha256').update(JSON.stringify({ pool: pod.pool, podShell: keyShell, memoryLimit: memoryLimit ?? null, deadlineSeconds: shardDeadline, bunCache: bunCache ?? null })).digest('hex');
|
|
1085
1313
|
let start = Date.now();
|
|
1086
1314
|
let admitted: GateAdmissionResult | undefined;
|
|
@@ -1089,6 +1317,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1089
1317
|
const stamp = (phase: string) => { phaseTimes[phase] = new Date().toISOString(); };
|
|
1090
1318
|
let podStages: Record<string, number> | undefined;
|
|
1091
1319
|
let podLifecycle: Record<string, string> | undefined;
|
|
1320
|
+
let podJobName: string | undefined;
|
|
1092
1321
|
let batchArtifactsComplete = true;
|
|
1093
1322
|
let junit: string | undefined;
|
|
1094
1323
|
let rc: number | undefined;
|
|
@@ -1104,6 +1333,8 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1104
1333
|
let unfinishedLog: string | undefined;
|
|
1105
1334
|
let exitCodeOnly: string[] | undefined;
|
|
1106
1335
|
let stall: { seconds: number; sinceCheckoutMs: number } | undefined;
|
|
1336
|
+
let stallWatchState: ShardStallWatchState | undefined;
|
|
1337
|
+
let abort: ShardAbort | undefined;
|
|
1107
1338
|
// The way down a previous attempt took wins over its own log: that log was not a verdict, or it would have no children.
|
|
1108
1339
|
// The split this shard would take if it failed here — the same rule as the split code below (isolation at depth 2 ·
|
|
1109
1340
|
// cap chunks for a big root shard · halves otherwise · none for one file).
|
|
@@ -1117,6 +1348,11 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1117
1348
|
const reused = !fresh && !descent && logDir ? previousShardResult(logDir, `pod-${shard}${branch}`, commit, paths, runKey, parentAttempt) : undefined;
|
|
1118
1349
|
// This shard's attempt id: the recorded one when its way down is followed, a new one when a Pod runs. Children carry it.
|
|
1119
1350
|
const attemptId = found?.attemptId ?? randomUUID();
|
|
1351
|
+
// The Pod itself stamps its cut and attempt before the test starts. A
|
|
1352
|
+
// collector-created manifest cannot reassign another Job's bytes.
|
|
1353
|
+
const podShell = gateShardShell({ ...shellOptions,
|
|
1354
|
+
...(stallWatch ? { beatSeconds: gateStallBeatSeconds(stallSeconds) } : {}),
|
|
1355
|
+
sourceBinding: { cutSha: commit, attemptId } });
|
|
1120
1356
|
if (descent === 'retry') {
|
|
1121
1357
|
// «descend» is only the way down; «resumed» comes from each leaf whose log reads cleanly (review r4).
|
|
1122
1358
|
debug.log('release-loop.gate', 'pod-shard-descend', { shard, branch, files: paths.length, via: 'retry' });
|
|
@@ -1149,10 +1385,20 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1149
1385
|
stopJob: () => {
|
|
1150
1386
|
try { return stopJob(liveJob()); }
|
|
1151
1387
|
catch (error) { debug.log('release-loop.gate', 'pod-shard-stall-stop-failed', { job: jobName, error: String(error) }, { level: 'warn' }); return false; }
|
|
1152
|
-
}
|
|
1388
|
+
},
|
|
1389
|
+
...(midRunWatch ? {
|
|
1390
|
+
midRunStallSeconds,
|
|
1391
|
+
onAbort: ({ sinceLastOutputMs }: { sinceLastOutputMs: number }) => debug.log('release-loop.gate', 'shard-abort',
|
|
1392
|
+
{ shard, branch, reason: `mid-run-stall ${midRunStallSeconds}s`, by: 'auto', sinceLastOutputMs }, { level: 'warn' }),
|
|
1393
|
+
abortTests: () => {
|
|
1394
|
+
try { return abortTests!(liveJob(), `mid-run-stall ${midRunStallSeconds}s`); }
|
|
1395
|
+
catch (error) { debug.log('release-loop.gate', 'shard-abort-failed', { job: jobName, error: String(error) }, { level: 'warn' }); return false; }
|
|
1396
|
+
},
|
|
1397
|
+
} : {}) }) : undefined;
|
|
1153
1398
|
const placed = admitted.release;
|
|
1154
1399
|
const job = await podCommand({
|
|
1155
1400
|
pool: pod.pool, poolScheduler: trackedPool(shard, depth > 0 || branch !== '', (member) => { stamp('poolPlaced'); jobContext = member.context; placedOn = member.context; placed(); }), clone: true, source: { kind: 'commit', sha: commit }, deadlineSeconds: shardDeadline,
|
|
1401
|
+
...gateHostMirror(pod),
|
|
1156
1402
|
...(multiHost ? { statusUnreadableMs: memberUnreadableSeconds * 1000 } : {}),
|
|
1157
1403
|
...(bunCache ? { bunCache } : {}),
|
|
1158
1404
|
...(installSlots > 0 ? { installSlots: POD_INSTALL_SLOTS_HOST_PATH } : {}),
|
|
@@ -1163,6 +1409,7 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1163
1409
|
command: ['bash', '-lc', podShell],
|
|
1164
1410
|
}).finally(() => watch?.stop());
|
|
1165
1411
|
stamp('jobReturned');
|
|
1412
|
+
podJobName = job.job;
|
|
1166
1413
|
if (!poolOverride && placedOn) {
|
|
1167
1414
|
try {
|
|
1168
1415
|
const got = defaultKubectl(['--context', placedOn, '-n', 'elanous-test', 'get', 'pods', '-l', `job-name=${job.job}`, '-o', 'json']);
|
|
@@ -1189,6 +1436,9 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1189
1436
|
({ output, junit, rc, timedOut, finished, unfinishedLog, exitCodeOnly } = read);
|
|
1190
1437
|
// A stall counts only when the Job also left no verdict (a failed stop can let a late shard finish after all).
|
|
1191
1438
|
const stalledAt = watch?.stalled();
|
|
1439
|
+
stallWatchState = watch?.state();
|
|
1440
|
+
abort = watch?.aborted();
|
|
1441
|
+
if (abort?.taken) abortedShards++;
|
|
1192
1442
|
if (stalledAt && !finished.length && rc === undefined) stall = { seconds: stallSeconds, sinceCheckoutMs: stalledAt.sinceCheckoutMs };
|
|
1193
1443
|
// A Job the cluster cut at its own deadline leaves no outbox at all: past the deadline with nothing is a timeout too.
|
|
1194
1444
|
if (!timedOut && output === undefined && job.exitCode !== 0 && Date.now() - start >= shardDeadline * 1000) timedOut = true;
|
|
@@ -1215,11 +1465,17 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1215
1465
|
// A reused shard keeps its files as they are — rewriting would replace the measured durationMs the next release plans by.
|
|
1216
1466
|
if (logDir && !reused && !descent) {
|
|
1217
1467
|
mkdirSync(logDir, { recursive: true });
|
|
1468
|
+
const warmPrefix = join(logDir, `pod-${shard}${branch}`);
|
|
1469
|
+
rmSync(`${warmPrefix}.warm-manifest.json`, { force: true });
|
|
1470
|
+
rmSync(`${warmPrefix}.rc`, { force: true });
|
|
1218
1471
|
// GATE-SHARD-RESUME: the json is what makes a log reusable, so it goes first and comes back last — a stop between
|
|
1219
1472
|
// the new log and the new json leaves no json, never the old Job's exit paired with the new log (review r8).
|
|
1220
1473
|
rmSync(join(logDir, `pod-${shard}${branch}.json`), { force: true });
|
|
1221
1474
|
const destination = join(logDir, `pod-${shard}${branch}.log`);
|
|
1222
|
-
|
|
1475
|
+
const unfinishedLogBytes = unfinishedLog !== undefined ? Buffer.byteLength(unfinishedLog, 'utf8') : undefined;
|
|
1476
|
+
// GATE-UNFINISHED-HINT: pod-N.log holds finished parts only — point a reader at the running parts' output (file only; the verdict reads `output`).
|
|
1477
|
+
if (output !== undefined) writeFileSync(destination, unfinishedLogBytes !== undefined
|
|
1478
|
+
? `${output}${output.endsWith('\n') || output === '' ? '' : '\n'}${gateUnfinishedHint(`pod-${shard}${branch}.unfinished.log`, unfinishedLogBytes)}\n` : output, { mode: 0o600 });
|
|
1223
1479
|
else rmSync(destination, { force: true });
|
|
1224
1480
|
const junitDestination = join(logDir, `pod-${shard}${branch}.junit.xml`);
|
|
1225
1481
|
if (junit !== undefined) writeFileSync(junitDestination, junit, { mode: 0o600 });
|
|
@@ -1233,13 +1489,25 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1233
1489
|
...(podLifecycle && Object.keys(podLifecycle).length ? { podLifecycle } : {}),
|
|
1234
1490
|
...(fileSeconds !== undefined ? { fileSeconds, testSeconds } : {}), ...(testWallSeconds !== undefined ? { testWallSeconds } : {}),
|
|
1235
1491
|
...(podOverheadSeconds !== undefined ? { podOverheadSeconds } : {}),
|
|
1236
|
-
...(
|
|
1492
|
+
...(unfinishedLogBytes !== undefined ? { unfinishedLogBytes } : {}),
|
|
1493
|
+
...(admitted && admitted.outcome !== 'off' ? { admission: admitted.outcome, admissionWaitMs: admitted.waitedMs } : {}),
|
|
1494
|
+
...(abort ? { abort: { reason: `abort: ${abort.reason}`, at: abort.at, by: abort.by, sinceLastOutputMs: abort.sinceLastOutputMs, taken: abort.taken } } : {}),
|
|
1495
|
+
stallWatch: shardStallWatchRecord(stallSeconds, !!liveLog, stallWatchState) }) + '\n', { mode: 0o600 });
|
|
1496
|
+
if (paths.length === 1 && (rc === 0 || rc === 1) && !jobFailed && jobExitCode === 0
|
|
1497
|
+
&& output !== undefined && junit !== undefined) {
|
|
1498
|
+
writeFileSync(`${warmPrefix}.rc`, `${rc}\n`, { mode: 0o600 });
|
|
1499
|
+
writeWarmJobManifest(warmPrefix, paths[0]!, commit, attemptId);
|
|
1500
|
+
}
|
|
1237
1501
|
}
|
|
1238
1502
|
const installSeconds = parseInstallSeconds(output ?? '');
|
|
1239
1503
|
// GATE-TIMEOUT-HEAL ③: a cut shard is «reason: timeout» (with the files that still finished), not a bare rc=null.
|
|
1240
1504
|
if (!reused && !descent) debug.log('release-loop.gate', 'pod-shard', { shard, files: paths, durationMs, rc: rc ?? null, attempt: depth + 1, installSeconds, gateDeps: parseGateDeps(output ?? ''), deadlineSeconds: shardDeadline, parts: parts.length, ...(placedOn ? { host: placedOn } : {}),
|
|
1241
1505
|
...(testSeconds !== undefined ? { testSeconds } : {}),
|
|
1242
1506
|
...(stall ? { reason: 'stall', finishedFiles: 0, stallSeconds: stall.seconds } : timedOut ? { reason: 'timeout', finishedFiles: finished.length } : {}) });
|
|
1507
|
+
// POD-LIFECYCLE-STAGES: the same values the shard json carries, as one `pod.lifecycle stages` line (observation only) —
|
|
1508
|
+
// only for a Job that came back (a launch that failed before the Job returned has no Pod to describe).
|
|
1509
|
+
if (!reused && !descent && podJobName !== undefined) emitPodLifecycle(gatePodLifecycleEvent({ runId: process.env.ELANOUS_RUN_ID ?? null, podJob: podJobName, host: placedOn ?? null,
|
|
1510
|
+
shard: `pod-${shard}${branch}`, attemptId, phaseTimes, podStages, podLifecycle, outcome: gatePodOutcome({ jobFailed, stalled: !!stall, timedOut, rc }) }));
|
|
1243
1511
|
if (!reused && !descent) markShard(shard, { ...(installSeconds !== null ? { installSec: installSeconds } : {}), ...(rc !== undefined ? { rc } : {}), ...(timedOut ? { state: 'timeout' as const } : {}) });
|
|
1244
1512
|
let clean = output?.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '');
|
|
1245
1513
|
const ran = clean && lastMatch(clean, /Ran (\d+) tests? across (\d+) files?/);
|
|
@@ -1340,8 +1608,8 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1340
1608
|
});
|
|
1341
1609
|
if (measured.length && missing.length) {
|
|
1342
1610
|
debug.log('release-loop.gate', 'isolation-batch-partial', { shard, branch, measured: measured.length, missing });
|
|
1343
|
-
const children = await Promise.all(missing.map((file) => runShard([file], shard, depth + 1,
|
|
1344
|
-
`${branch}-file-${paths.indexOf(file)}`, false, undefined, attemptId)));
|
|
1611
|
+
const children = await Promise.all(missing.map((file) => isolatedFile(() => runShard([file], shard, depth + 1,
|
|
1612
|
+
`${branch}-file-${paths.indexOf(file)}`, false, undefined, attemptId))));
|
|
1345
1613
|
return { runs: [...measured, ...children.flatMap((child) => child.runs)],
|
|
1346
1614
|
stalled: children.flatMap((child) => child.stalled) };
|
|
1347
1615
|
}
|
|
@@ -1386,11 +1654,11 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1386
1654
|
const canBatch = batchPaths.length > 1 && batchPaths.every((file) => batchSafe.has(file));
|
|
1387
1655
|
const batch = batchIsolation || !canBatch ? undefined : await runShard(batchPaths, shard, depth + 1, `${branch}-batch`, false, undefined, attemptId, 0, true);
|
|
1388
1656
|
const retryPaths = !batch ? batchPaths : batchPaths.filter((file) => batch.stalled.some((item) => item.files.includes(file)));
|
|
1389
|
-
const children = !batch || batch.stalled.length ? await Promise.all(retryPaths.map(async (
|
|
1657
|
+
const children = !batch || batch.stalled.length ? await Promise.all(retryPaths.map((file) => isolatedFile(async () => {
|
|
1390
1658
|
const child = await runShard([file], shard, depth + 1, `${branch}-file-${batchPaths.indexOf(file)}`, false, undefined, attemptId);
|
|
1391
1659
|
debug.log('release-loop.gate', 'pod-shard-isolate', { shard, file, outcome: child.stalled[0]?.reason ?? 'ok' });
|
|
1392
1660
|
return child;
|
|
1393
|
-
})) : [batch];
|
|
1661
|
+
}))) : [batch];
|
|
1394
1662
|
debug.log('release-loop.gate', 'pod-shard-isolate-batch', { shard, files: batchPaths.length, fallback: !batch || batch.stalled.length > 0 });
|
|
1395
1663
|
return { runs: [...(batch?.stalled.length ? batch.runs : []), ...children.flatMap((child) => child.runs)], stalled: [
|
|
1396
1664
|
...children.flatMap((child) => child.stalled),
|
|
@@ -1455,16 +1723,62 @@ export function createGateRunner(repo: string, remote?: string, commandOverride?
|
|
|
1455
1723
|
if (unmeasured.length) debug.log('release-loop.gate', 'pod-sweep-unmeasured', { shards: [...new Set(unmeasured.map((item) => item.shard))], files: unmeasured.reduce((sum, item) => sum + item.files.length, 0),
|
|
1456
1724
|
stallSeconds, measuredFiles: total.files }, { level: 'warn' });
|
|
1457
1725
|
// An unmeasured (stalled) shard is never handed to the host re-run as an environment stall: the gate fails now.
|
|
1458
|
-
if (
|
|
1726
|
+
if (abortedShards) debug.log('release-loop.gate', 'pod-sweep-aborted', { shards: abortedShards, midRunStallSeconds }, { level: 'warn' });
|
|
1727
|
+
if (stalledShards.some((item) => item.reason !== 'no-output' || item.files.length !== 1 || item.stall)) throw new StalledPodShards(stalledShards, total, abortedShards);
|
|
1459
1728
|
if (total.ran === 0 && total.errors === 0 && !stalledShards.length) throw new Error('sweep incomplete: no tests ran');
|
|
1460
1729
|
const outputs = measured.map((run) => run.output.replace(/(?:^|\n)\s*\d+ (?:pass|fail|errors?)\s*(?=\n|$)/g, '\n').replace(/Ran \d+ tests? across \d+ files?\.?/g, ''));
|
|
1461
|
-
return { rc: total.fail + total.errors ? 1 : 0, output: outputs.join('\n') + `\n${total.pass} pass\n${total.fail} fail\n${total.errors} errors\nRan ${total.ran} tests across ${total.files} files.\n`,
|
|
1730
|
+
return { rc: total.fail + total.errors ? 1 : 0, output: outputs.join('\n') + `\n${total.pass} pass\n${total.fail} fail\n${total.errors} errors\nRan ${total.ran} tests across ${total.files} files.\n`, validatedShards: true,
|
|
1462
1731
|
passedIds: [...new Set(measured.flatMap((run) => run.passedIds))].sort(),
|
|
1463
|
-
stalledEnv: stalledShards.flatMap((item) => item.files.map((file) => ({ file, reason: 'no-output' as const })))
|
|
1732
|
+
stalledEnv: stalledShards.flatMap((item) => item.files.map((file) => ({ file, reason: 'no-output' as const }))),
|
|
1733
|
+
...(abortedShards ? { abortedShards } : {}) };
|
|
1734
|
+
};
|
|
1735
|
+
const runPodRecheckBatch = async (files: readonly string[], commit: string, pod: NonNullable<GateOptions['pod']>, timeoutMs: number, relaxed = false): Promise<Map<string, CommandResult>> => {
|
|
1736
|
+
if (!files.length || files.length > GATE_RECHECK_POD_BATCH_FILES || new Set(files).size !== files.length
|
|
1737
|
+
|| files.some((file) => !/^(?:[\w.-]+\/)*[\w.-]+\.test\.[cm]?[jt]sx?$/.test(file)
|
|
1738
|
+
|| file.split('/').some((part) => part === '.' || part === '..')))
|
|
1739
|
+
throw new Error('unsafe pod recheck files');
|
|
1740
|
+
if (!sha.test(commit)) throw new Error('invalid pod recheck commit');
|
|
1741
|
+
const installSlots = pod.installSlots ?? POD_INSTALL_SLOTS_DEFAULT;
|
|
1742
|
+
if (!Number.isSafeInteger(installSlots) || installSlots < 0) throw new Error('invalid pod recheck install slots');
|
|
1743
|
+
const bunCache = pod.bunCache?.trim() || process.env[POD_BUN_CACHE_HOST_PATH]?.trim();
|
|
1744
|
+
const partLimitSeconds = Math.ceil(timeoutMs / 1000);
|
|
1745
|
+
const deadlineSeconds = partLimitSeconds * Math.ceil(files.length / 2) + POD_SHARD_EMIT_RESERVE_SECONDS + 60;
|
|
1746
|
+
const shell = gateShardShell({ parts: files.map((file) => [file]), cdpPatterns: [], softSeconds: softDeadlineSeconds(deadlineSeconds),
|
|
1747
|
+
cachePrefix: bunCache ? podBunCacheVolume(bunCache).shellPrefix : '', installSlots, parallelParts: files.length > 1 ? 2 : 1,
|
|
1748
|
+
partLimitSeconds, ...(relaxed ? { testTimeoutMs: GATE_TIMEOUT_RECHECK_MS } : {}) });
|
|
1749
|
+
const job = await podCommand({ pool: pod.pool, ...(poolOverride ? { poolScheduler: poolOverride } : {}), clone: true,
|
|
1750
|
+
source: { kind: 'commit', sha: commit }, deadlineSeconds, ...(bunCache ? { bunCache } : {}), ...gateHostMirror(pod),
|
|
1751
|
+
...(installSlots > 0 ? { installSlots: POD_INSTALL_SLOTS_HOST_PATH } : {}), ...(pod.cpu ? { cpu: pod.cpu } : {}),
|
|
1752
|
+
memoryLimit: POD_ISOLATION_MEMORY_LIMIT, name: `gate-recheck-${randomUUID()}`, command: ['bash', '-lc', shell] });
|
|
1753
|
+
const artifact = (name: string) => {
|
|
1754
|
+
const path = join(job.artifactsDir, name);
|
|
1755
|
+
return existsSync(path) ? readFileSync(path, 'utf8') : undefined;
|
|
1756
|
+
};
|
|
1757
|
+
const installed = artifact('install.rc')?.trim() === '0';
|
|
1758
|
+
if (!installed || job.exitCode !== 0)
|
|
1759
|
+
throw new Error(`pod isolated recheck setup incomplete: ${commit.slice(0, 9)} job=${job.job} exit=${job.exitCode} install=${installed}`);
|
|
1760
|
+
const timedOut = artifact('shard.timeout') !== undefined;
|
|
1761
|
+
const results = new Map<string, CommandResult>();
|
|
1762
|
+
for (const [index, file] of files.entries()) {
|
|
1763
|
+
const rcText = artifact(`part-${index}.rc`)?.trim();
|
|
1764
|
+
const output = artifact(`part-${index}.log`);
|
|
1765
|
+
const rc = rcText && /^\d+$/.test(rcText) ? Number(rcText) : undefined;
|
|
1766
|
+
const partTimedOut = artifact(`part-${index}.timeout`) !== undefined;
|
|
1767
|
+
if (rc === undefined || output === undefined || partTimedOut) {
|
|
1768
|
+
if (!timedOut && !partTimedOut) throw new Error(`pod isolated recheck incomplete: ${file} @${commit.slice(0, 9)} job=${job.job} rc=${rc ?? 'missing'}`);
|
|
1769
|
+
results.set(file, { rc: 124, output: output ?? '', timedOut: true });
|
|
1770
|
+
} else results.set(file, { rc, output });
|
|
1771
|
+
debug.log('release-loop.gate', 'pod-isolated-recheck', { file, commit, job: job.job, rc: rc ?? 124, relaxed });
|
|
1772
|
+
}
|
|
1773
|
+
return results;
|
|
1464
1774
|
};
|
|
1465
1775
|
return {
|
|
1466
1776
|
command,
|
|
1467
1777
|
localCommand: commandOverride && !remote ? commandOverride : localCommand,
|
|
1778
|
+
async recheckPod(file, commit, pod, timeoutMs, relaxed = false) {
|
|
1779
|
+
return (await runPodRecheckBatch([file], commit, pod, timeoutMs, relaxed)).get(file)!;
|
|
1780
|
+
},
|
|
1781
|
+
recheckPodBatch: (files, commit, pod, timeoutMs) => runPodRecheckBatch(files, commit, pod, timeoutMs),
|
|
1468
1782
|
get remoteMirror() { return remoteMirror; },
|
|
1469
1783
|
set remoteMirror(path) { remoteMirror = path; },
|
|
1470
1784
|
async sweep(tree, logDir, pod, only) {
|
|
@@ -1576,7 +1890,7 @@ const fileOf = testFileOfId;
|
|
|
1576
1890
|
* GATE-BASELINE-CACHE (0.2.18) — the isolated `bun run test:deterministic <file>` result of one test file at one commit,
|
|
1577
1891
|
* kept in the machine ledger so a later gate does not run it again (0.2.17: the failing files were re-run on the previous
|
|
1578
1892
|
* release commit although that release had already measured them). Cache format harvested from draft #24500.
|
|
1579
|
-
* `host` is where the result was measured (`local
|
|
1893
|
+
* `host` is where the result was measured (`local`, the ssh host, or `pod:<pool>`) — another environment is a miss.
|
|
1580
1894
|
* Timeouts are never stored: an environment stall is measured again, not remembered.
|
|
1581
1895
|
*/
|
|
1582
1896
|
export interface BaselineFileResult { commit: string; file: string; host: string; failures: string[]; errors: string[]; missing: boolean }
|
|
@@ -1666,6 +1980,8 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1666
1980
|
|| (opts.baselineVersion && !versionPattern.test(opts.baselineVersion))
|
|
1667
1981
|
|| (opts.baselineCommit && !sha.test(opts.baselineCommit))) throw new Error('invalid commit or version');
|
|
1668
1982
|
if (!opts.baselineVersion) throw new Error('previous release version required (--baseline-version)');
|
|
1983
|
+
if (!Number.isSafeInteger(opts.recheckFileLimit ?? GATE_HOST_RECHECK_FILE_LIMIT) || (opts.recheckFileLimit ?? GATE_HOST_RECHECK_FILE_LIMIT) < 1)
|
|
1984
|
+
throw new Error('invalid gate recheck file limit');
|
|
1669
1985
|
if (opts.pod && (!opts.pod.pool || !Number.isSafeInteger(opts.pod.shards ?? 24) || (opts.pod.shards ?? 24) < 1
|
|
1670
1986
|
|| !Number.isSafeInteger(opts.pod.shardTimeoutSeconds ?? 1200) || (opts.pod.shardTimeoutSeconds ?? 1200) < 1)) throw new Error('invalid pod sweep options');
|
|
1671
1987
|
if (opts.remote && !/^(?:[\w.-]+@)?[\w.-]+$/.test(opts.remote)) throw new Error('invalid ssh host');
|
|
@@ -1709,6 +2025,23 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1709
2025
|
// Its logs go to cut-partial/ — the full cut/ logs stay whole (the next release reads them as a baseline source).
|
|
1710
2026
|
const partial = readPartialPlan(roots, opts.version, opts.commit);
|
|
1711
2027
|
if (partial) debug.log('release-loop.gate', 'partial-regate', { version: opts.version, priorCommit: partial.priorCommit, commit: opts.commit, files: partial.files.length });
|
|
2028
|
+
if (partial?.kind === 'carry') {
|
|
2029
|
+
// The plan alone is not evidence: the old cut and its verdict must still be complete and clean at this SHA.
|
|
2030
|
+
if (!verifyDocsOnlyCarry(repo, partial.priorCommit, opts.commit))
|
|
2031
|
+
throw new Error(`carry plan diff is not docs-only: ${partial.priorCommit}..${opts.commit}`);
|
|
2032
|
+
const trusted = roots.some((location) => {
|
|
2033
|
+
const dir = join(location, 'release', opts.version);
|
|
2034
|
+
const failuresPath = join(dir, 'gate-failures.json');
|
|
2035
|
+
const verdictPath = join(dir, 'gate-verdict.json');
|
|
2036
|
+
if (!existsSync(failuresPath) || !existsSync(verdictPath)) return false;
|
|
2037
|
+
const failures = JSON.parse(readFileSync(failuresPath, 'utf8')) as { commit?: unknown; failures?: unknown; errors?: unknown };
|
|
2038
|
+
const verdict = JSON.parse(readFileSync(verdictPath, 'utf8')) as { commit?: unknown; outcome?: unknown };
|
|
2039
|
+
return failures.commit === partial.priorCommit && Array.isArray(failures.failures) && failures.failures.length === 0
|
|
2040
|
+
&& (failures.errors === undefined || (Array.isArray(failures.errors) && failures.errors.length === 0))
|
|
2041
|
+
&& verdict.commit === partial.priorCommit && verdict.outcome === 'ok';
|
|
2042
|
+
});
|
|
2043
|
+
if (!trusted) throw new Error(`carry plan has no complete clean gate at ${partial.priorCommit}`);
|
|
2044
|
+
}
|
|
1712
2045
|
// GATE-SKIP-KNOWN-RETRY — the ids the verdict will not count as introduced (the recorded baseline ⊕ env-known), read
|
|
1713
2046
|
// before the sweep so a shard failing only with them skips its isolation retries. No recorded baseline = no skip.
|
|
1714
2047
|
let knownFailures: string[] = [];
|
|
@@ -1722,12 +2055,13 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1722
2055
|
} catch (error) { debug.log('release-loop.gate', 'known-failures-unavailable', { error: String(error) }); }
|
|
1723
2056
|
}
|
|
1724
2057
|
const durationSource = join(ledger, 'release', opts.baselineVersion, 'gate-logs', 'cut');
|
|
1725
|
-
const cutRun = await runner.sweep(cutTree, join(root, 'release', opts.version, 'gate-logs', partial ? 'cut-partial' : 'cut'),
|
|
1726
|
-
opts.pod ? { ...opts.pod, durationSource, ...(knownFailures.length ? { knownFailures } : {}),
|
|
2058
|
+
const cutRun = partial?.kind === 'carry' ? undefined : await runner.sweep(cutTree, join(root, 'release', opts.version, 'gate-logs', partial ? 'cut-partial' : 'cut'),
|
|
2059
|
+
opts.pod ? { ...opts.pod, durationSource, ...(knownFailures.length ? { knownFailures } : {}), onSelectionTimed: ({ ms }) => { result.selectionMs = ms; },
|
|
1727
2060
|
// GATE-LIVE-OBS: the release ledger's shards.json (UX #24782) — never the run tree's universe; an explicit
|
|
1728
2061
|
// --ledger-root (tests) keeps it in that ledger.
|
|
1729
2062
|
shardsFile: { path: opts.ledgerRoot ? gateShardsPath(opts.version, opts.ledgerRoot) : gateShardsPath(opts.version), version: opts.version } } : undefined,
|
|
1730
2063
|
partial?.files);
|
|
2064
|
+
if (cutRun?.abortedShards) result.abortedShards = cutRun.abortedShards;
|
|
1731
2065
|
const localStalled = async (run: CommandResult, tree: string, commit: string, label: string) => {
|
|
1732
2066
|
const stalled = run.stalledEnv ?? [];
|
|
1733
2067
|
const noCompletedShard = stalled.length && run.rc === 0
|
|
@@ -1767,7 +2101,46 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1767
2101
|
debug.log('release-loop.gate', 'pod-shard-stalled-env', { files: stalled.map(({ file, reason }) => ({ file, reason })), local });
|
|
1768
2102
|
return measured;
|
|
1769
2103
|
};
|
|
1770
|
-
const measuredCut = await localStalled(cutRun, cutTree, opts.commit, 'cut');
|
|
2104
|
+
const measuredCut = cutRun ? await localStalled(cutRun, cutTree, opts.commit, 'cut') : { failures: [], errors: [] };
|
|
2105
|
+
// A between-tests error proves only that the shared process failed. Re-run each affected file by itself using the
|
|
2106
|
+
// same isolated path as ordinary new-failure rechecks, even when the error was present in the baseline ledger.
|
|
2107
|
+
// Replace that file's shard observations with its isolated observations before the existing regression comparison.
|
|
2108
|
+
const prefetchedCut = new Map<string, CommandResult>();
|
|
2109
|
+
for (const file of [...new Set(measuredCut.errors.filter((id) => id.endsWith(' > [error]')).map(fileOf))].sort()) {
|
|
2110
|
+
let run: CommandResult | undefined;
|
|
2111
|
+
let reading: ReturnType<typeof readIsolatedRun> | undefined;
|
|
2112
|
+
let reason: 'reproduced' | 'timeout' | 'unreadable' | undefined;
|
|
2113
|
+
try {
|
|
2114
|
+
run = opts.pod && runner.recheckPod
|
|
2115
|
+
? await runner.recheckPod(file, opts.commit, opts.pod, GATE_ISOLATED_TIMEOUT_MS)
|
|
2116
|
+
: await runner.command('bun', ['run', 'test:deterministic', asPath(file)], cutTree, GATE_ISOLATED_TIMEOUT_MS);
|
|
2117
|
+
} catch (error) {
|
|
2118
|
+
reason = 'unreadable';
|
|
2119
|
+
debug.log('release-loop.gate', 'unhandled-isolated-run-failed', { file, error: String(error) });
|
|
2120
|
+
}
|
|
2121
|
+
if (run?.timedOut) reason = 'timeout';
|
|
2122
|
+
else if (run) {
|
|
2123
|
+
try {
|
|
2124
|
+
reading = readIsolatedRun(run, `cut unhandled isolated ${file}`, file);
|
|
2125
|
+
if (reading.kind === 'read' && reading.errors.length) reason = 'reproduced';
|
|
2126
|
+
} catch { reason = 'unreadable'; }
|
|
2127
|
+
}
|
|
2128
|
+
const originalIds = measuredCut.errors.filter((id) => fileOf(id) === file);
|
|
2129
|
+
measuredCut.failures = measuredCut.failures.filter((id) => fileOf(id) !== file);
|
|
2130
|
+
measuredCut.errors = measuredCut.errors.filter((id) => fileOf(id) !== file);
|
|
2131
|
+
if (reading?.kind === 'read') measuredCut.failures.push(...reading.failures);
|
|
2132
|
+
else if (reading?.kind === 'rc-nonzero-no-failures') measuredCut.errors.push(`${file} > [rc-nonzero]`);
|
|
2133
|
+
if (reason) {
|
|
2134
|
+
(result.unmeasuredErrors ??= []).push({ file, reason, ids: originalIds });
|
|
2135
|
+
debug.log('release-loop.gate', 'unhandled-isolated-unmeasured', { file, reason, ids: originalIds });
|
|
2136
|
+
} else {
|
|
2137
|
+
prefetchedCut.set(file, run!);
|
|
2138
|
+
if (reading?.kind === 'read' && !reading.failures.length) {
|
|
2139
|
+
(result.interferenceErrors ??= []).push(...originalIds);
|
|
2140
|
+
debug.log('release-loop.gate', 'unhandled-isolated-interference', { file, ids: originalIds });
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
2143
|
+
}
|
|
1771
2144
|
const cut = partial ? mergePartialFailures(partial, measuredCut) : measuredCut;
|
|
1772
2145
|
if (partial) result.partial = { priorCommit: partial.priorCommit, files: partial.files.length };
|
|
1773
2146
|
cutFailures = cut;
|
|
@@ -1820,20 +2193,20 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1820
2193
|
for (const id of [...baseline.failures, ...baseline.errors, ...cut.failures, ...cut.errors]) fileOf(id);
|
|
1821
2194
|
const known = opts.pod ? loadEnvKnownFailures(repo) : new Set<string>();
|
|
1822
2195
|
const cutIds = [...cut.failures, ...cut.errors];
|
|
1823
|
-
const cutHostFailed = new Set(cutRun
|
|
2196
|
+
const cutHostFailed = new Set(cutRun?.stalledEnv?.map(({ file }) => file) ?? []);
|
|
1824
2197
|
const baselineIds = [...baseline.failures, ...baseline.errors];
|
|
1825
2198
|
const countedCut = splitKnownEnv(cutIds, known);
|
|
1826
2199
|
const countedBaseline = splitKnownEnv(baselineIds, known);
|
|
1827
2200
|
knownEnvIds = countedCut.knownEnv;
|
|
1828
2201
|
result.knownEnv = knownEnvIds.length;
|
|
1829
|
-
result.knownEnvCleared = [...new Set(cutRun
|
|
2202
|
+
result.knownEnvCleared = [...new Set(cutRun?.passedIds ?? [])].filter((id) => known.has(id) && !cutIds.includes(id) && !cutRun?.stalledEnv?.some(({ file }) => fileOf(id) === file)).sort();
|
|
1830
2203
|
const diff = diffFailures(countedCut.counted, countedBaseline.counted);
|
|
1831
2204
|
result.fixed = result.baselineDeferred ? null : diff.fixed.length;
|
|
1832
2205
|
preexistingIds.push(...diff.common);
|
|
1833
2206
|
result.preexisting = preexistingIds.length;
|
|
1834
2207
|
// GATE-BASELINE-CACHE — only the files whose new failures reproduce in isolation reach the baseline, and each
|
|
1835
2208
|
// (commit, file, host) is run at most once across gates: this gate's cut results also seed the next gate's baseline.
|
|
1836
|
-
const cacheHost = opts.remote ?? 'local';
|
|
2209
|
+
const cacheHost = opts.pod && runner.recheckPod ? `pod:${opts.pod.pool}` : opts.remote ?? 'local';
|
|
1837
2210
|
const newFailureFiles = new Set(diff.newFailures.map(fileOf));
|
|
1838
2211
|
const baseCache = newFailureFiles.size ? readBaselineFileCache(ledger, baseSha) : new Map<string, BaselineFileResult>();
|
|
1839
2212
|
const cacheStats = { hits: 0, misses: 0 };
|
|
@@ -1848,20 +2221,26 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1848
2221
|
catch (error) { debug.log('release.gate', 'baseline-cache-write-failed', { commit: row.commit, file: row.file, error: String(error) }); }
|
|
1849
2222
|
};
|
|
1850
2223
|
const recheckFiles = [...newFailureFiles].sort();
|
|
1851
|
-
const
|
|
1852
|
-
|
|
1853
|
-
|
|
2224
|
+
const recheckLimit = opts.recheckFileLimit ?? GATE_HOST_RECHECK_FILE_LIMIT;
|
|
2225
|
+
const recheckConcurrency = opts.pod?.isolationConcurrency ?? 4;
|
|
2226
|
+
if (opts.pod && (!Number.isSafeInteger(recheckConcurrency) || recheckConcurrency < 1)) throw new Error('invalid pod recheck concurrency');
|
|
2227
|
+
// Each window is bounded, but overflow is queued into the next one. A limit must never turn known failures into
|
|
2228
|
+
// unmeasured files. Pod checks use the same file-isolation fanout cap as the sweep; host fallback stays serial.
|
|
2229
|
+
debug.log('release-loop.gate', 'isolated-recheck-plan', { files: recheckFiles.length, window: recheckLimit,
|
|
2230
|
+
concurrency: opts.pod && runner.recheckPod ? recheckConcurrency : 1, executor: opts.pod && runner.recheckPod ? 'pod' : 'host',
|
|
2231
|
+
remote: opts.remote ?? null, podBatchFiles: opts.pod && runner.recheckPod && runner.recheckPodBatch ? GATE_RECHECK_POD_BATCH_FILES : 1 });
|
|
1854
2232
|
const recheckStartedAt = Date.now();
|
|
1855
|
-
const
|
|
1856
|
-
for (const [index, file] of selectedRechecks.entries()) {
|
|
2233
|
+
const recheckFile = async (file: string, index: number) => {
|
|
1857
2234
|
debug.log('release-loop.gate', 'isolated-recheck-progress',
|
|
1858
|
-
{ index: index + 1, total:
|
|
2235
|
+
{ index: index + 1, total: recheckFiles.length, file, elapsedSec: Math.round((Date.now() - recheckStartedAt) / 1000) });
|
|
1859
2236
|
// 0.2.15: the cut re-check of a new-failure file (InsidePage.test.tsx · headless Chrome) hung for good — limit it like the host re-run.
|
|
1860
|
-
const cutCheck = cutHostFailed.has(file) ? undefined :
|
|
2237
|
+
const cutCheck = cutHostFailed.has(file) ? undefined : prefetchedCut.get(file) ?? (opts.pod && runner.recheckPod
|
|
2238
|
+
? await runner.recheckPod(file, opts.commit, opts.pod, GATE_ISOLATED_TIMEOUT_MS)
|
|
2239
|
+
: await runner.command('bun', ['run', 'test:deterministic', asPath(file)], cutTree, GATE_ISOLATED_TIMEOUT_MS));
|
|
1861
2240
|
if (cutCheck?.timedOut) {
|
|
1862
2241
|
debug.log('release-loop.gate', 'isolated-timeout', { file, label: 'cut isolated', limitMs: GATE_ISOLATED_TIMEOUT_MS });
|
|
1863
2242
|
result.stalledEnv.push({ file, reason: 'isolated-timeout', local: 'timeout' });
|
|
1864
|
-
|
|
2243
|
+
return;
|
|
1865
2244
|
}
|
|
1866
2245
|
const cutReading = cutCheck === undefined ? undefined : readIsolatedRun(cutCheck, `cut isolated ${file}`, file);
|
|
1867
2246
|
const rcOnly = cutReading?.kind === 'rc-nonzero-no-failures' ? cutReading : undefined;
|
|
@@ -1878,7 +2257,9 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1878
2257
|
const timedOut = cutCheck === undefined ? new Set<string>() : new Set(parseTimedOutFailures(stripPodRepoRoot(cutCheck.output)));
|
|
1879
2258
|
const shaped = candidates.filter((id) => timedOut.has(id) && isolatedCut.failures.includes(id));
|
|
1880
2259
|
if (shaped.length) {
|
|
1881
|
-
const relaxed =
|
|
2260
|
+
const relaxed = opts.pod && runner.recheckPod
|
|
2261
|
+
? await runner.recheckPod(file, opts.commit, opts.pod, GATE_ISOLATED_TIMEOUT_MS, true)
|
|
2262
|
+
: await runner.command('bun', ['run', 'test:deterministic', asPath(file), '--timeout', String(GATE_TIMEOUT_RECHECK_MS)], cutTree, GATE_ISOLATED_TIMEOUT_MS);
|
|
1882
2263
|
let flaky: string[] = [];
|
|
1883
2264
|
let recheck: 'read' | 'timeout' | 'unreadable' = 'read';
|
|
1884
2265
|
if (relaxed.timedOut) recheck = 'timeout';
|
|
@@ -1901,7 +2282,7 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1901
2282
|
} catch (error) { debug.log('release-loop.gate', 'load-flaky-census-write-failed', { file, error: String(error) }); }
|
|
1902
2283
|
}
|
|
1903
2284
|
}
|
|
1904
|
-
if (candidates.length === 0)
|
|
2285
|
+
if (candidates.length === 0) return;
|
|
1905
2286
|
let oldFailures: Set<string>;
|
|
1906
2287
|
let baselineMissing = false;
|
|
1907
2288
|
const hit = baseCache.get(`${file}\0${cacheHost}`);
|
|
@@ -1911,15 +2292,17 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1911
2292
|
oldFailures = new Set([...hit.failures, ...hit.errors]);
|
|
1912
2293
|
} else {
|
|
1913
2294
|
cacheStats.misses++;
|
|
1914
|
-
const baselineTree = await getBaseTree();
|
|
1915
|
-
const previous =
|
|
2295
|
+
const baselineTree = opts.pod && runner.recheckPod ? undefined : await getBaseTree();
|
|
2296
|
+
const previous = opts.pod && runner.recheckPod
|
|
2297
|
+
? await runner.recheckPod(file, baseSha, opts.pod, GATE_ISOLATED_TIMEOUT_MS)
|
|
2298
|
+
: await runner.command('bun', ['run', 'test:deterministic', asPath(file)], baselineTree!, GATE_ISOLATED_TIMEOUT_MS);
|
|
1916
2299
|
if (previous.timedOut) {
|
|
1917
2300
|
debug.log('release-loop.gate', 'isolated-timeout', { file, label: 'baseline isolated', limitMs: GATE_ISOLATED_TIMEOUT_MS });
|
|
1918
2301
|
result.stalledEnv.push({ file, reason: 'isolated-timeout', local: 'timeout' });
|
|
1919
|
-
|
|
2302
|
+
return;
|
|
1920
2303
|
}
|
|
1921
2304
|
if (/No tests found|had no matches/i.test(previous.output) && previous.rc === 1) {
|
|
1922
|
-
const lookup = await runner.command('git', ['ls-tree', '--name-only', baseSha, '--', file], opts.remote ? baselineTree : repo);
|
|
2305
|
+
const lookup = await runner.command('git', ['ls-tree', '--name-only', baseSha, '--', file], opts.remote ? baselineTree ?? cutTree : repo);
|
|
1923
2306
|
if (lookup.rc !== 0 || lookup.output.trim()) throw new Error(`baseline isolated run incomplete: ${file}`);
|
|
1924
2307
|
oldFailures = new Set();
|
|
1925
2308
|
baselineMissing = true;
|
|
@@ -1942,14 +2325,46 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1942
2325
|
if (oldFailures.has(id)) preexistingIds.push(id);
|
|
1943
2326
|
else result.introduced.push(id);
|
|
1944
2327
|
}
|
|
2328
|
+
};
|
|
2329
|
+
const cap = opts.pod && runner.recheckPod ? recheckConcurrency : 1;
|
|
2330
|
+
const settleStarted = async <T>(jobs: Promise<T>[]): Promise<T[]> => {
|
|
2331
|
+
const settled = await Promise.allSettled(jobs);
|
|
2332
|
+
const rejected = settled.find((item): item is PromiseRejectedResult => item.status === 'rejected');
|
|
2333
|
+
if (rejected) throw rejected.reason;
|
|
2334
|
+
return settled.map((item) => (item as PromiseFulfilledResult<T>).value);
|
|
2335
|
+
};
|
|
2336
|
+
for (let start = 0; start < recheckFiles.length; start += recheckLimit) {
|
|
2337
|
+
const window = recheckFiles.slice(start, start + recheckLimit);
|
|
2338
|
+
debug.log('release-loop.gate', 'isolated-recheck-window', { start, files: window.length, total: recheckFiles.length });
|
|
2339
|
+
if (opts.pod && runner.recheckPod && runner.recheckPodBatch) {
|
|
2340
|
+
const pending = window.filter((file) => !cutHostFailed.has(file) && !prefetchedCut.has(file));
|
|
2341
|
+
const groups = Array.from({ length: Math.ceil(pending.length / GATE_RECHECK_POD_BATCH_FILES) }, (_, index) =>
|
|
2342
|
+
pending.slice(index * GATE_RECHECK_POD_BATCH_FILES, (index + 1) * GATE_RECHECK_POD_BATCH_FILES));
|
|
2343
|
+
for (let next = 0; next < groups.length; next += cap) {
|
|
2344
|
+
const batches = await settleStarted(groups.slice(next, next + cap).map((files) =>
|
|
2345
|
+
runner.recheckPodBatch!(files, opts.commit, opts.pod!, GATE_ISOLATED_TIMEOUT_MS)));
|
|
2346
|
+
for (const batch of batches) for (const [file, run] of batch) prefetchedCut.set(file, run);
|
|
2347
|
+
}
|
|
2348
|
+
}
|
|
2349
|
+
for (let next = 0; next < window.length; next += cap)
|
|
2350
|
+
await settleStarted(window.slice(next, next + cap).map((file, slot) => recheckFile(file, start + next + slot)));
|
|
2351
|
+
for (const file of window) prefetchedCut.delete(file);
|
|
1945
2352
|
}
|
|
2353
|
+
result.introduced.sort();
|
|
2354
|
+
preexistingIds.sort();
|
|
2355
|
+
result.loadFlaky?.sort();
|
|
1946
2356
|
result.preexisting = preexistingIds.length;
|
|
1947
2357
|
result.baselineCache = cacheStats;
|
|
1948
2358
|
debug.log('release.gate', 'baseline-cache', { hits: cacheStats.hits, misses: cacheStats.misses, commit: baseSha });
|
|
1949
|
-
|
|
1950
|
-
|
|
2359
|
+
const timedOutFiles = [...new Set(result.stalledEnv.filter((item) => item.reason === 'isolated-timeout').map((item) => item.file))].sort();
|
|
2360
|
+
const unmeasuredFiles = [...new Set([...timedOutFiles, ...(result.unmeasuredErrors ?? []).map((item) => item.file)])].sort();
|
|
2361
|
+
if (unmeasuredFiles.length) result.unmeasuredRecheckFiles = unmeasuredFiles;
|
|
2362
|
+
if (timedOutFiles.length || result.unmeasuredErrors?.length) {
|
|
1951
2363
|
result.outcome = 'error';
|
|
1952
|
-
result.error =
|
|
2364
|
+
result.error = [
|
|
2365
|
+
timedOutFiles.length ? `isolated recheck timed out: ${timedOutFiles.join(', ')}` : undefined,
|
|
2366
|
+
result.unmeasuredErrors?.length ? `unhandled errors unmeasured after isolated recheck: ${result.unmeasuredErrors.map((item) => `${item.file} (${item.reason})`).join(', ')}` : undefined,
|
|
2367
|
+
].filter(Boolean).join('; ');
|
|
1953
2368
|
} else result.outcome = result.introduced.length ? 'regression' : 'ok';
|
|
1954
2369
|
if (opts.rehearsalTui) {
|
|
1955
2370
|
result.rehearsalTui = await executeTuiNodeCheck(opts.commit, repo, runner.localCommand);
|
|
@@ -1963,6 +2378,7 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1963
2378
|
if (error instanceof StalledPodShards) {
|
|
1964
2379
|
result.stalledShards = error.stalledShards;
|
|
1965
2380
|
result.partialSummary = error.partialSummary;
|
|
2381
|
+
if (error.abortedShards) result.abortedShards = error.abortedShards;
|
|
1966
2382
|
}
|
|
1967
2383
|
} finally {
|
|
1968
2384
|
if (baseSnapshot && work) {
|
|
@@ -1982,7 +2398,7 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
1982
2398
|
if (opts.remote) runner.remoteMirror = undefined;
|
|
1983
2399
|
result.durationMs = Date.now() - start;
|
|
1984
2400
|
}
|
|
1985
|
-
if (cutFailures && !result.stalledShards?.length && !result.stalledEnv.length) {
|
|
2401
|
+
if (cutFailures && !result.stalledShards?.length && !result.stalledEnv.length && !result.unmeasuredRecheckFiles?.length) {
|
|
1986
2402
|
try {
|
|
1987
2403
|
for (const location of roots) {
|
|
1988
2404
|
const path = failureFile(location, opts.version);
|
|
@@ -2006,7 +2422,10 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
2006
2422
|
preexisting: [...preexistingIds].sort(),
|
|
2007
2423
|
loadFlaky: [...(result.loadFlaky ?? [])].sort(),
|
|
2008
2424
|
...(result.unmeasuredRecheckFiles?.length ? { unmeasuredRecheckFiles: [...result.unmeasuredRecheckFiles].sort() } : {}),
|
|
2425
|
+
...(result.unmeasuredErrors?.length ? { unmeasuredErrors: result.unmeasuredErrors } : {}),
|
|
2426
|
+
...(result.interferenceErrors?.length ? { interferenceErrors: [...result.interferenceErrors].sort() } : {}),
|
|
2009
2427
|
};
|
|
2428
|
+
const observed = result.selectionMs !== undefined ? { selectionMs: result.selectionMs } : {};
|
|
2010
2429
|
try {
|
|
2011
2430
|
for (const location of roots) {
|
|
2012
2431
|
const path = verdictFile(location, opts.version);
|
|
@@ -2017,7 +2436,7 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
2017
2436
|
mkdirSync(dirname(path), { recursive: true });
|
|
2018
2437
|
const temp = join(dirname(path), `.gate-verdict-${process.pid}-${randomUUID()}.tmp`);
|
|
2019
2438
|
try {
|
|
2020
|
-
writeFileSync(temp, JSON.stringify({ version: opts.version, commit: opts.commit, outcome: result.outcome, ...ids }, null, 2) + '\n', { mode: 0o600 });
|
|
2439
|
+
writeFileSync(temp, JSON.stringify({ version: opts.version, commit: opts.commit, outcome: result.outcome, ...ids, ...observed }, null, 2) + '\n', { mode: 0o600 });
|
|
2021
2440
|
chmodSync(temp, 0o600);
|
|
2022
2441
|
renameSync(temp, path);
|
|
2023
2442
|
} finally { if (existsSync(temp)) rmSync(temp); }
|
|
@@ -2030,7 +2449,7 @@ export async function judgeGate(opts: GateOptions, runner: GateRunner = createGa
|
|
|
2030
2449
|
mkdirSync(dirname(path), { recursive: true });
|
|
2031
2450
|
const temp = join(dirname(path), `.gate-verdict-${process.pid}-${randomUUID()}.tmp`);
|
|
2032
2451
|
try {
|
|
2033
|
-
writeFileSync(temp, JSON.stringify({ version: opts.version, commit: opts.commit, outcome: result.outcome, ...ids }, null, 2) + '\n', { mode: 0o600 });
|
|
2452
|
+
writeFileSync(temp, JSON.stringify({ version: opts.version, commit: opts.commit, outcome: result.outcome, ...ids, ...observed }, null, 2) + '\n', { mode: 0o600 });
|
|
2034
2453
|
chmodSync(temp, 0o600);
|
|
2035
2454
|
renameSync(temp, path);
|
|
2036
2455
|
} finally { if (existsSync(temp)) rmSync(temp); }
|
|
@@ -2065,7 +2484,8 @@ export function graphGateResult(result: GateResult, graph: boolean) {
|
|
|
2065
2484
|
+ (result.outcome === 'ok' ? `새 회귀 ${result.introduced.length} · 기존 ${result.preexisting} · 고침 ${result.fixed ?? '못 셈(기준선 미루기)'}` : `게이트 ${result.outcome}: ${result.error ?? result.introduced.length + ' new regressions'}`) + ` · 환경 알려진 실패 ${result.knownEnv}`
|
|
2066
2485
|
+ (result.rcNonzeroNoFailures.length ? ` · ⚠ 종료코드만 비영 ${result.rcNonzeroNoFailures.length}` : '')
|
|
2067
2486
|
+ (result.loadFlaky?.length ? ` · ⚠ 부하 흔들림 ${result.loadFlaky.length}(시간 초과 꼴 · ${GATE_TIMEOUT_RECHECK_MS / 1000}s 재실행 통과 · 막지 않음)` : '')
|
|
2068
|
-
+ (result.stalledEnv.length ? ` · 환경 멈춤 ${result.stalledEnv.length} (Pod 밖 통과 ${result.stalledEnv.filter((item) => item.local === 'passed').length})` : '')
|
|
2487
|
+
+ (result.stalledEnv.length ? ` · 환경 멈춤 ${result.stalledEnv.length} (Pod 밖 통과 ${result.stalledEnv.filter((item) => item.local === 'passed').length})` : '')
|
|
2488
|
+
+ (result.abortedShards ? ` · 중단 조각 ${result.abortedShards}(중간 매달림 · 시험만 끊고 재확인)` : '') };
|
|
2069
2489
|
}
|
|
2070
2490
|
|
|
2071
2491
|
/** GATE-HOST-EXECUTOR — which executor produced (or will produce) this verdict, for the node output and observation. */
|
|
@@ -2074,12 +2494,18 @@ export function gateExecutorOf(opts: Pick<GateOptions, 'pod' | 'remote'>): { exe
|
|
|
2074
2494
|
return opts.pod ? { executor: 'pod', host: null } : { executor: 'host', host: opts.remote ?? 'local' };
|
|
2075
2495
|
}
|
|
2076
2496
|
|
|
2497
|
+
/** A Pod without an SSH host runs on this machine; it does not reserve every remote gate host. */
|
|
2498
|
+
export function reservedGateHosts(opts: Pick<GateOptions, 'pod' | 'remote'>): string[] {
|
|
2499
|
+
return opts.pod ? [...new Set(parsePodPool(opts.pod.pool).map(member => member.sshHost ?? hostname()))]
|
|
2500
|
+
: [opts.remote?.split('@').at(-1) ?? hostname()];
|
|
2501
|
+
}
|
|
2502
|
+
|
|
2077
2503
|
/** What was asked for, read best-effort when the options themselves were refused — so a failed gate still says which executor. */
|
|
2078
2504
|
export function requestedGateExecutor(args: string[], env: NodeJS.ProcessEnv): { executor: string | null; host: string | null } {
|
|
2079
2505
|
let executor: unknown;
|
|
2080
2506
|
let host: unknown;
|
|
2081
2507
|
try {
|
|
2082
|
-
if (
|
|
2508
|
+
if (usesGraphContext(args, env)) {
|
|
2083
2509
|
const input = readGraphContext(env).input as Record<string, unknown>;
|
|
2084
2510
|
executor = input.gateExecutor ?? (typeof input.gatePodPool === 'string' ? 'pod' : undefined);
|
|
2085
2511
|
host = input.gateRemote;
|
|
@@ -2093,8 +2519,12 @@ export function requestedGateExecutor(args: string[], env: NodeJS.ProcessEnv): {
|
|
|
2093
2519
|
return { executor: typeof executor === 'string' ? executor : null, host: executor !== 'pod' && typeof host === 'string' ? host : null };
|
|
2094
2520
|
}
|
|
2095
2521
|
|
|
2522
|
+
function usesGraphContext(args: string[], env: NodeJS.ProcessEnv): boolean {
|
|
2523
|
+
return !!env.ELANOUS_GRAPH_CONTEXT && args.length > 0 && args.every(arg => arg === '--json');
|
|
2524
|
+
}
|
|
2525
|
+
|
|
2096
2526
|
export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOptions | 'help' {
|
|
2097
|
-
const context =
|
|
2527
|
+
const context = usesGraphContext(args, env) ? readGraphContext(env) : undefined;
|
|
2098
2528
|
const fromGraph: Record<string, unknown> = context?.input ?? {};
|
|
2099
2529
|
const graphCommit = context?.outputs['version-release']?.commit;
|
|
2100
2530
|
if (typeof graphCommit === 'string') fromGraph.commit = graphCommit;
|
|
@@ -2105,6 +2535,7 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
2105
2535
|
const hostExecutor = fromGraph.gateExecutor === 'host';
|
|
2106
2536
|
const opts: GateOptions = {
|
|
2107
2537
|
rehearsalTui: fromGraph.rehearsalTui === true,
|
|
2538
|
+
recheckFileLimit: fromGraph.gateRecheckFileLimit !== undefined ? Number(fromGraph.gateRecheckFileLimit) : undefined,
|
|
2108
2539
|
commit: typeof fromGraph.commit === 'string' ? fromGraph.commit : '',
|
|
2109
2540
|
version: typeof fromGraph.version === 'string' ? fromGraph.version : '',
|
|
2110
2541
|
baselineVersion: typeof fromGraph.previousVersion === 'string' ? fromGraph.previousVersion : undefined,
|
|
@@ -2119,7 +2550,9 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
2119
2550
|
...(typeof fromGraph.gatePodBunCache === 'string' && fromGraph.gatePodBunCache.trim() ? { bunCache: fromGraph.gatePodBunCache } : {}),
|
|
2120
2551
|
...(fromGraph.gatePodInstallSlots !== undefined ? { installSlots: Number(fromGraph.gatePodInstallSlots) } : {}),
|
|
2121
2552
|
...(fromGraph.gatePodAdmissionWaitSeconds !== undefined ? { admissionWaitSeconds: Number(fromGraph.gatePodAdmissionWaitSeconds) } : {}),
|
|
2553
|
+
...(fromGraph.gatePodIsolationConcurrency !== undefined ? { isolationConcurrency: Number(fromGraph.gatePodIsolationConcurrency) } : {}),
|
|
2122
2554
|
...(fromGraph.gatePodStallSeconds !== undefined ? { stallSeconds: Number(fromGraph.gatePodStallSeconds) } : {}),
|
|
2555
|
+
...(fromGraph.gatePodMidRunStallSeconds !== undefined ? { midRunStallSeconds: Number(fromGraph.gatePodMidRunStallSeconds) } : {}),
|
|
2123
2556
|
...(fromGraph.gatePodFreshShards === true ? { freshShards: true } : {}),
|
|
2124
2557
|
...((cpu) => cpu ? { cpu } : {})(parsePodCpu(fromGraph.gatePodCpu, 'release.loop.gatePodCpu')),
|
|
2125
2558
|
} : undefined,
|
|
@@ -2133,6 +2566,12 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv): GateOption
|
|
|
2133
2566
|
// Applied after the loop and only to a Pod gate — on its own it must not turn a local gate into a Pod one (review r5).
|
|
2134
2567
|
if (arg === '--fresh-shards') { freshShards = true; continue; }
|
|
2135
2568
|
if (arg === '--rehearsal-tui') { opts.rehearsalTui = true; continue; }
|
|
2569
|
+
if (arg === '--recheck-file-limit') {
|
|
2570
|
+
const value = args[++i];
|
|
2571
|
+
if (!value || !/^\d+$/.test(value) || Number(value) < 1 || !Number.isSafeInteger(Number(value))) throw new Error('--recheck-file-limit requires a positive integer');
|
|
2572
|
+
opts.recheckFileLimit = Number(value);
|
|
2573
|
+
continue;
|
|
2574
|
+
}
|
|
2136
2575
|
if (arg === '--executor') {
|
|
2137
2576
|
const value = args[++i];
|
|
2138
2577
|
if (!isGateExecutor(value)) throw new Error(`--executor must be pod or host: ${value ?? ''}`);
|
|
@@ -2179,12 +2618,21 @@ export async function runGateNode(argv: string[], env: NodeJS.ProcessEnv, runner
|
|
|
2179
2618
|
debug.log('release-loop.gate', 'executor', { version: opts.version, ...placement, requested: opts.executor ?? null });
|
|
2180
2619
|
}
|
|
2181
2620
|
if (opts === 'help') {
|
|
2182
|
-
console.log('Usage: bun scripts/release-loop/gate-node.ts --commit <sha> --version <v> [--baseline-version <prev>] [--baseline-commit <sha>] [--remote <ssh-host>] [--remote-mirror <path>] [--pod-pool <pool>] [--pod-shards <n>] [--pod-shard-timeout-seconds <n>] [--fresh-shards] [--executor <pod|host>] [--json]\nWithout flags, input.commit, input.version and input.previousVersion come from the JSON file at ELANOUS_GRAPH_CONTEXT.\n--executor host (input.gateExecutor) ignores every pod option and sweeps on --remote (input.gateRemote, required).');
|
|
2621
|
+
console.log('Usage: bun scripts/release-loop/gate-node.ts --commit <sha> --version <v> [--baseline-version <prev>] [--baseline-commit <sha>] [--remote <ssh-host>] [--remote-mirror <path>] [--pod-pool <pool>] [--pod-shards <n>] [--pod-shard-timeout-seconds <n>] [--recheck-file-limit <n>] [--fresh-shards] [--executor <pod|host>] [--json]\nWithout flags, input.commit, input.version and input.previousVersion come from the JSON file at ELANOUS_GRAPH_CONTEXT.\n--executor host (input.gateExecutor) ignores every pod option and sweeps on --remote (input.gateRemote, required).');
|
|
2183
2622
|
return 0;
|
|
2184
2623
|
}
|
|
2185
2624
|
version = opts.version;
|
|
2186
2625
|
// The graph's existing gate-node execution path delegates verdict classification and persistence here.
|
|
2187
|
-
|
|
2626
|
+
const context = usesGraphContext(argv, env) ? readGraphContext(env) : undefined;
|
|
2627
|
+
// Let the gate's existing SSH host validation report an invalid host before any reservation is published.
|
|
2628
|
+
const hosts = opts.remote && !/^(?:[\w.-]+@)?[\w.-]+$/.test(opts.remote) ? [] : reservedGateHosts(opts);
|
|
2629
|
+
const reservationRoot = context ? gateHostReservationRoot() : undefined;
|
|
2630
|
+
if (context && (!context.runId || !env.ELANOUS_STATE_DIR || normRoot(env.ELANOUS_STATE_DIR) !== reservationRoot))
|
|
2631
|
+
throw new Error('release gate host reservation requires graph runId and pinned state root');
|
|
2632
|
+
const releaseHosts = context && hosts.length
|
|
2633
|
+
? reserveReleaseGateHosts(reservationRoot!, context.runId!, hosts) : () => undefined;
|
|
2634
|
+
try { result = await (runner ? judgeGate(opts, runner) : judgeGate(opts)); }
|
|
2635
|
+
finally { releaseHosts(); }
|
|
2188
2636
|
} catch (error) {
|
|
2189
2637
|
result = { outcome: 'error', commit: '', introduced: [], preexisting: 0, fixed: 0, knownEnv: 0, knownEnvCleared: [], stalledEnv: [], rcNonzeroNoFailures: [], durationMs: Date.now() - start, error: String(error) };
|
|
2190
2638
|
placement ??= requestedGateExecutor(argv, env);
|