chainlesschain 0.162.187 → 0.162.189
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/assets/web-panel/assets/{AIOps-CETaO3s8.js → AIOps-DbZft8w0.js} +1 -1
- package/src/assets/web-panel/assets/{ActionButton-CAe7E9TW.js → ActionButton-DoOZpwfN.js} +1 -1
- package/src/assets/web-panel/assets/{Analytics-BGr2mzwO.js → Analytics-B0J0KLTW.js} +3 -3
- package/src/assets/web-panel/assets/{AppLayout-BEoY-NcT.js → AppLayout-D9eIR0qq.js} +5 -5
- package/src/assets/web-panel/assets/{Artifacts-DUZivCgt.js → Artifacts-BnZXEVeo.js} +1 -1
- package/src/assets/web-panel/assets/{Audit-C3yBs68k.js → Audit-X1ce4Bjn.js} +1 -1
- package/src/assets/web-panel/assets/{BackgroundAgents-C9KwiY3O.js → BackgroundAgents-BvWR-y5v.js} +1 -1
- package/src/assets/web-panel/assets/{Backup-BoSWD_C8.js → Backup-BublSGEd.js} +1 -1
- package/src/assets/web-panel/assets/{BaseInput-Dp09GkxL.js → BaseInput-CR_77Hso.js} +1 -1
- package/src/assets/web-panel/assets/{Chat-5t9d-Ja0.js → Chat-DixW52tq.js} +6 -6
- package/src/assets/web-panel/assets/{ChatBubbleRenderer-DmVku4GH.js → ChatBubbleRenderer-DDHDpBBA.js} +1 -1
- package/src/assets/web-panel/assets/{Checkbox-8QJFlkqI.js → Checkbox-CCv1C87K.js} +1 -1
- package/src/assets/web-panel/assets/{Codegen-BJVFQ3lL.js → Codegen-B03Cc3dd.js} +1 -1
- package/src/assets/web-panel/assets/{Col-6hdLD0oB.js → Col-CA06-6pU.js} +1 -1
- package/src/assets/web-panel/assets/{Community-B8opo0uY.js → Community-Bj881J52.js} +1 -1
- package/src/assets/web-panel/assets/{Compact-Dv8HudiN.js → Compact-UZLTPoaL.js} +1 -1
- package/src/assets/web-panel/assets/{Compliance-DhxuCS5N.js → Compliance-kGwzRb_0.js} +1 -1
- package/src/assets/web-panel/assets/{Cowork-Dggy6VIL.js → Cowork-KundoYM8.js} +3 -3
- package/src/assets/web-panel/assets/{Cron-DSmDqJTT.js → Cron-BYZyoqDe.js} +2 -2
- package/src/assets/web-panel/assets/{Crosschain-7bZq0j5f.js → Crosschain-BhrrppBX.js} +1 -1
- package/src/assets/web-panel/assets/{DID-BWY4kh-H.js → DID-CdC6EguZ.js} +2 -2
- package/src/assets/web-panel/assets/{Dashboard-bOzPZxlu.js → Dashboard-CcQx3J4y.js} +2 -2
- package/src/assets/web-panel/assets/{Dropdown-BTFJHqQu.js → Dropdown-BpijtEzQ.js} +1 -1
- package/src/assets/web-panel/assets/{EmailListRenderer-BC3MOdtT.js → EmailListRenderer-DfZjeZTe.js} +1 -1
- package/src/assets/web-panel/assets/{FamilyGuardDashboard-CFHAdTTY.js → FamilyGuardDashboard-BFYSf69C.js} +1 -1
- package/src/assets/web-panel/assets/{Federation-DZ_4aF_n.js → Federation-Okdi3pJm.js} +1 -1
- package/src/assets/web-panel/assets/{FormItemContext-BZje6UTj.js → FormItemContext-mH-HebKW.js} +1 -1
- package/src/assets/web-panel/assets/{GenericCardRenderer-D7QfCvjN.js → GenericCardRenderer-xivV69x7.js} +1 -1
- package/src/assets/web-panel/assets/{Git-DuG7qANG.js → Git-1rh_YBwg.js} +2 -2
- package/src/assets/web-panel/assets/{Governance-BDmYAgTh.js → Governance-eAXkNxdY.js} +1 -1
- package/src/assets/web-panel/assets/{Inference-DSXAfc1Q.js → Inference-CS6dMVTN.js} +1 -1
- package/src/assets/web-panel/assets/{KnowledgeGraph-DusleniP.js → KnowledgeGraph-Bq_TnkvD.js} +1 -1
- package/src/assets/web-panel/assets/{Logs-CpImmA8p.js → Logs-KchHtiHt.js} +2 -2
- package/src/assets/web-panel/assets/{MarkdownRenderer-BZ6LJBkk.js → MarkdownRenderer-TabqHW9B.js} +1 -1
- package/src/assets/web-panel/assets/{Marketplace-DXS8CXkE.js → Marketplace-CsmGcX5X.js} +1 -1
- package/src/assets/web-panel/assets/{McpTools-D151UfUI.js → McpTools-BaPFnnyz.js} +5 -5
- package/src/assets/web-panel/assets/{Memory-DKDxnoA1.js → Memory-2m-sUUf7.js} +2 -2
- package/src/assets/web-panel/assets/{MobileBridge-BFZ95iKM.js → MobileBridge-D5nVZBRu.js} +1 -1
- package/src/assets/web-panel/assets/{MobileProjects-CDAR-grF.js → MobileProjects-Bo8MLZvT.js} +1 -1
- package/src/assets/web-panel/assets/{Mtc-OGJ95xJs.js → Mtc-3GDvamGa.js} +6 -6
- package/src/assets/web-panel/assets/{MtcAudit-BmGTo1XC.js → MtcAudit-D2zGuZCH.js} +4 -4
- package/src/assets/web-panel/assets/{Multisig-XumJRqOa.js → Multisig-BBVQ4t6t.js} +3 -3
- package/src/assets/web-panel/assets/{NLProgramming-DHhhRZed.js → NLProgramming-BIOCyOGp.js} +1 -1
- package/src/assets/web-panel/assets/{Notes-B9rGxNNr.js → Notes-DVwBGpzc.js} +3 -3
- package/src/assets/web-panel/assets/{NotificationSettings-DClLbMYV.js → NotificationSettings-BDnHfIfX.js} +1 -1
- package/src/assets/web-panel/assets/{OrderTableRenderer-I8OohT7N.js → OrderTableRenderer-C7Ek20ag.js} +1 -1
- package/src/assets/web-panel/assets/{Organization-M6WzaYVa.js → Organization-D3SRKaa7.js} +4 -4
- package/src/assets/web-panel/assets/{Overflow-Cb3B5KtM.js → Overflow-C2B8ahyn.js} +1 -1
- package/src/assets/web-panel/assets/{P2P-BGk57kya.js → P2P-2iwQzG65.js} +2 -2
- package/src/assets/web-panel/assets/{PdhVaultBrowser-CXcKyPqK.js → PdhVaultBrowser-CoPiWqMN.js} +3 -3
- package/src/assets/web-panel/assets/{Permissions-hsfJ-Y0e.js → Permissions-yV5XNXo5.js} +4 -4
- package/src/assets/web-panel/assets/{PersonalDataHub-BSBii4js.js → PersonalDataHub-uCDJhMUF.js} +4 -4
- package/src/assets/web-panel/assets/{Pipeline-BNw52rey.js → Pipeline-hN7ucMOA.js} +1 -1
- package/src/assets/web-panel/assets/{Privacy-DUpfue4E.js → Privacy-CgU0ptN4.js} +1 -1
- package/src/assets/web-panel/assets/{ProjectInit-C68NsP3h.js → ProjectInit-CfYPiT6j.js} +2 -2
- package/src/assets/web-panel/assets/{ProjectSettings-BJ2It9v4.js → ProjectSettings-B0A2C4nw.js} +2 -2
- package/src/assets/web-panel/assets/Projects-BE5C0ajn.js +1 -0
- package/src/assets/web-panel/assets/{Providers-EnU4VWwS.js → Providers-CstQPf8W.js} +1 -1
- package/src/assets/web-panel/assets/{QrScannerModal-BRt5-B_I.js → QrScannerModal-Co3z_ynk.js} +1 -1
- package/src/assets/web-panel/assets/{QuickAsk-Jd5L9UL2.js → QuickAsk-DMTS6pQ6.js} +1 -1
- package/src/assets/web-panel/assets/{Recommend-BNDhoVIu.js → Recommend-ETw4odUa.js} +1 -1
- package/src/assets/web-panel/assets/RemoteSession-B0rkE09d.js +5 -0
- package/src/assets/web-panel/assets/{Reputation-D1wQPUNv.js → Reputation-BTVqQgKq.js} +1 -1
- package/src/assets/web-panel/assets/{Row-DnjRegul.js → Row-CBKKwwtO.js} +1 -1
- package/src/assets/web-panel/assets/{RssFeed-C_T4I-nF.js → RssFeed-BHUQ438I.js} +2 -2
- package/src/assets/web-panel/assets/{Search-C4lrxlLh.js → Search-B4NqeUsF.js} +1 -1
- package/src/assets/web-panel/assets/{Security-ByYT-VzP.js → Security-BRjoYG5_.js} +4 -4
- package/src/assets/web-panel/assets/{Services-BHpyEqB5.js → Services-CxQzu3p3.js} +2 -2
- package/src/assets/web-panel/assets/{Skeleton-C0I23vG-.js → Skeleton-BKzwwmWO.js} +1 -1
- package/src/assets/web-panel/assets/{Skills-DGZXz22K.js → Skills-DrsFcgth.js} +1 -1
- package/src/assets/web-panel/assets/{Sla-DL_urP9A.js → Sla-D21tmXCH.js} +1 -1
- package/src/assets/web-panel/assets/{SpeechSettings-6x-hUNQ_.js → SpeechSettings-CjyjLfp8.js} +1 -1
- package/src/assets/web-panel/assets/{SyncSettings-Crk5TCXq.js → SyncSettings-Cu45-tnx.js} +2 -2
- package/src/assets/web-panel/assets/{Tasks-rZGq5PZs.js → Tasks-oMcILYOj.js} +1 -1
- package/src/assets/web-panel/assets/{Templates-DT5el8Q8.js → Templates-Bh-Zcr5R.js} +1 -1
- package/src/assets/web-panel/assets/{Tenant-mlgeyMK8.js → Tenant-68ns63aI.js} +1 -1
- package/src/assets/web-panel/assets/{Terminal-C8HEA-lM.js → Terminal-BLJkFykE.js} +2 -2
- package/src/assets/web-panel/assets/{TimelineRenderer-DvlqBayB.js → TimelineRenderer-xcrJLq-O.js} +1 -1
- package/src/assets/web-panel/assets/{Tokens-C7lDcNmT.js → Tokens-C6PpFJtf.js} +1 -1
- package/src/assets/web-panel/assets/{Trigger-CM8nCnIJ.js → Trigger-BaKA7tdM.js} +1 -1
- package/src/assets/web-panel/assets/{Trust-BQDLUQ9x.js → Trust-BzhA57Jc.js} +1 -1
- package/src/assets/web-panel/assets/{UkeySign-CVqM0OFE.js → UkeySign-4_U3JkIR.js} +1 -1
- package/src/assets/web-panel/assets/{VideoEditing-BvAcQpzT.js → VideoEditing-162AANxX.js} +1 -1
- package/src/assets/web-panel/assets/{Wallet-zFVFaAZ2.js → Wallet-B1viuql_.js} +4 -4
- package/src/assets/web-panel/assets/{WebAuthn-Cc522ums.js → WebAuthn-ZVWazhCy.js} +4 -4
- package/src/assets/web-panel/assets/{WorkflowEditor-Bg_wxpDo.js → WorkflowEditor-tzdyUk-1.js} +1 -1
- package/src/assets/web-panel/assets/{chat-jg0QwC8K.js → chat-DmrHNuwq.js} +1 -1
- package/src/assets/web-panel/assets/{colors-ALzYz1gO.js → colors-DAJ_M-PK.js} +1 -1
- package/src/assets/web-panel/assets/{compact-item-DXHYUoqi.js → compact-item-uIzmYwLs.js} +1 -1
- package/src/assets/web-panel/assets/{createContext-BsGsBM5m.js → createContext-D_ZmuLwb.js} +1 -1
- package/src/assets/web-panel/assets/devWarning-DIrncrV1.js +1 -0
- package/src/assets/web-panel/assets/{hasIn-BccMLzxA.js → hasIn-CDt_shTL.js} +1 -1
- package/src/assets/web-panel/assets/{index-Cg9cOVe9.js → index-BAvSmJXW.js} +1 -1
- package/src/assets/web-panel/assets/{index-BTTeAiOP.js → index-BDF6EQBs.js} +1 -1
- package/src/assets/web-panel/assets/{index-C3k3ZwkQ.js → index-BHvlm0In.js} +1 -1
- package/src/assets/web-panel/assets/{index-CQ1ORdp-.js → index-BRa4kJ_S.js} +1 -1
- package/src/assets/web-panel/assets/{index-BcJzphrP.js → index-BTCXf1t-.js} +1 -1
- package/src/assets/web-panel/assets/{index-rIy96PeM.js → index-Bbxjfstq.js} +1 -1
- package/src/assets/web-panel/assets/{index-CSFESurl.js → index-Bx4d6c1k.js} +1 -1
- package/src/assets/web-panel/assets/{index-Cmg_Npx5.js → index-ByzGvpR9.js} +1 -1
- package/src/assets/web-panel/assets/{index-CahDve8M.js → index-C16fYKFD.js} +1 -1
- package/src/assets/web-panel/assets/{index-2VstcImq.js → index-C5WcdJH2.js} +3 -3
- package/src/assets/web-panel/assets/{index-C3Rf8wIM.js → index-CG0lY4h3.js} +1 -1
- package/src/assets/web-panel/assets/{index-uzykuL_R.js → index-CKyd472S.js} +1 -1
- package/src/assets/web-panel/assets/{index-CBVIFX-x.js → index-CaelyMgF.js} +1 -1
- package/src/assets/web-panel/assets/{index-TeCof1s3.js → index-CbD_WW_6.js} +1 -1
- package/src/assets/web-panel/assets/{index-BfF0OGKU.js → index-CcZ9xJI9.js} +1 -1
- package/src/assets/web-panel/assets/{index-oWltafBn.js → index-CeUpKjvi.js} +1 -1
- package/src/assets/web-panel/assets/{index-CWLWBgfe.js → index-Cvak6Oe2.js} +1 -1
- package/src/assets/web-panel/assets/{index-C0AbXsDy.js → index-CzXqvMuP.js} +1 -1
- package/src/assets/web-panel/assets/index-D-Swwqld.js +1 -0
- package/src/assets/web-panel/assets/{index-BfV8RDlg.js → index-D4n1lGTB.js} +1 -1
- package/src/assets/web-panel/assets/{index-Dxqiu9-F.js → index-D9OE8B3d.js} +1 -1
- package/src/assets/web-panel/assets/{index-BdCmJ_bG.js → index-DBFmcaCQ.js} +1 -1
- package/src/assets/web-panel/assets/{index-DgjQ-kdT.js → index-DdUMcwRB.js} +1 -1
- package/src/assets/web-panel/assets/{index-BBbbr1YB.js → index-DeGBoLQA.js} +1 -1
- package/src/assets/web-panel/assets/{index-Dmk7dD-T.js → index-DfSSy9uy.js} +1 -1
- package/src/assets/web-panel/assets/{index-JS2LXdd0.js → index-DqDSF2RW.js} +1 -1
- package/src/assets/web-panel/assets/{index-D9oV2iRR.js → index-DvFhhe6r.js} +1 -1
- package/src/assets/web-panel/assets/{index-BvsII10i.js → index-DwfvPe-j.js} +1 -1
- package/src/assets/web-panel/assets/index-FMZAMKy8.js +1 -0
- package/src/assets/web-panel/assets/{index-CrZHUsOZ.js → index-FyplHuNt.js} +1 -1
- package/src/assets/web-panel/assets/{index-CLzPzN75.js → index-GcKOHD_c.js} +1 -1
- package/src/assets/web-panel/assets/{index-koT8Q86Z.js → index-OzTxFIJP.js} +1 -1
- package/src/assets/web-panel/assets/{index-08diKcTm.js → index-QsgfSZrV.js} +1 -1
- package/src/assets/web-panel/assets/{index-CJg6Wsjc.js → index-Z8P4BexE.js} +1 -1
- package/src/assets/web-panel/assets/{index-srqW5_-X.js → index-clSX4PEc.js} +1 -1
- package/src/assets/web-panel/assets/{index-mIQUoUx1.js → index-ifxmu8SY.js} +1 -1
- package/src/assets/web-panel/assets/{index-Cz_s25m3.js → index-rS3gl_68.js} +1 -1
- package/src/assets/web-panel/assets/{index-UQXedWQz.js → index-rXdwUeE3.js} +1 -1
- package/src/assets/web-panel/assets/{index-rcwgNMc0.js → index-saf3VNeP.js} +1 -1
- package/src/assets/web-panel/assets/{initDefaultProps-xafkqrW3.js → initDefaultProps-pmfaD-aK.js} +1 -1
- package/src/assets/web-panel/assets/{motion-BrtyQXLk.js → motion-CaQ2CpYF.js} +1 -1
- package/src/assets/web-panel/assets/{move-BunUyg59.js → move-CkKTqNbI.js} +1 -1
- package/src/assets/web-panel/assets/{mtc-parser-BPwjBeuE.js → mtc-parser-Dd4xsgRI.js} +1 -1
- package/src/assets/web-panel/assets/{omit-BpyqZlNO.js → omit-BFuA5YEs.js} +1 -1
- package/src/assets/web-panel/assets/{pickAttrs-DDrV0XjE.js → pickAttrs-DDfxOAJc.js} +1 -1
- package/src/assets/web-panel/assets/{placementArrow-1A18Pcl2.js → placementArrow-Be595JCp.js} +1 -1
- package/src/assets/web-panel/assets/{responsiveObserve-B8Fnlmso.js → responsiveObserve-7byljUM3.js} +1 -1
- package/src/assets/web-panel/assets/{slide-x-Fwb2Zm.js → slide-CTWfOTaY.js} +1 -1
- package/src/assets/web-panel/assets/{statusUtils-aAG5yJl_.js → statusUtils-CNZOuIxM.js} +1 -1
- package/src/assets/web-panel/assets/{styleChecker-BRQSTMRL.js → styleChecker-DwOcBV78.js} +1 -1
- package/src/assets/web-panel/assets/{useFlexGapSupport-D5X_KozY.js → useFlexGapSupport-B6MNwAO9.js} +1 -1
- package/src/assets/web-panel/assets/{useFs-DN_YIjP-.js → useFs-CzyQjH9r.js} +1 -1
- package/src/assets/web-panel/assets/{usePersonalDataHub-BizxAvjw.js → usePersonalDataHub-Dxh3KiVA.js} +1 -1
- package/src/assets/web-panel/assets/{vnode-B5oAxN93.js → vnode-DVfZiS5N.js} +1 -1
- package/src/assets/web-panel/assets/{zoom-DJhvyzKv.js → zoom-BYme7fbY.js} +1 -1
- package/src/assets/web-panel/index.html +1 -1
- package/src/commands/agent.js +22 -0
- package/src/commands/auto-mode.js +47 -0
- package/src/commands/checkpoint-managed.js +1059 -0
- package/src/commands/checkpoint.js +3 -0
- package/src/commands/team-distributed.js +3324 -0
- package/src/commands/team.js +1130 -74
- package/src/data/auto-mode-safety-eval-v1.json +3294 -0
- package/src/data/changelog.json +26 -0
- package/src/lib/agent-core.js +1 -0
- package/src/lib/agent-team/task-lease.js +271 -3
- package/src/lib/agent-team/team-adjudication.js +1501 -0
- package/src/lib/agent-team/team-budget.js +4 -1
- package/src/lib/agent-team/team-control-store.js +582 -0
- package/src/lib/agent-team/team-distributed-queue.js +5246 -0
- package/src/lib/agent-team/team-process-checkpoint.js +211 -0
- package/src/lib/agent-team/team-runner.js +319 -22
- package/src/lib/agent-team/team-worktree.js +1403 -95
- package/src/lib/auto-mode-safety-classifier.js +2034 -0
- package/src/lib/auto-mode-safety-eval.js +1022 -0
- package/src/lib/managed-checkpoint-render.js +60 -0
- package/src/lib/managed-tool-checkpoint.js +194 -0
- package/src/lib/process-execution-broker/index.js +438 -5
- package/src/lib/process-execution-broker/linux-generic-bwrap-runtime.js +28 -1
- package/src/lib/process-execution-broker/linux-generic-bwrap.js +28 -4
- package/src/lib/process-execution-broker/platform-sandbox.js +92 -70
- package/src/lib/process-execution-broker/windows-sandbox-helper.dll +0 -0
- package/src/lib/process-execution-broker/windows-sandbox-helper.exe +0 -0
- package/src/lib/process-execution-broker/windows-sandbox.cs +284 -19
- package/src/lib/process-execution-broker/workspace-transaction.js +3556 -0
- package/src/lib/secure-file-identity.js +433 -0
- package/src/lib/turn-binding.js +160 -2
- package/src/lib/with-file-lock.js +140 -23
- package/src/repl/agent-repl.js +19 -1
- package/src/runtime/agent-core.js +366 -122
- package/src/runtime/headless-runner.js +12 -0
- package/src/runtime/headless-stream.js +3 -0
- package/src/runtime/policies/agent-policy.js +14 -0
- package/src/assets/web-panel/assets/Projects-D2FHLUa-.js +0 -1
- package/src/assets/web-panel/assets/RemoteSession-BII41Mel.js +0 -5
- package/src/assets/web-panel/assets/devWarning-Dy1GJA4S.js +0 -1
- package/src/assets/web-panel/assets/index-8zbMbbuf.js +0 -1
- package/src/assets/web-panel/assets/index-C97f71oi.js +0 -1
|
@@ -0,0 +1,3324 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable, multi-process `cc team queue` execution.
|
|
3
|
+
*
|
|
4
|
+
* The queue state coordinates leases and budgets, while git worktrees isolate
|
|
5
|
+
* every real shell or Agent task. The queue file is deliberately required
|
|
6
|
+
* outside the repository: a task running inside a worktree must not be able to
|
|
7
|
+
* rewrite its own scheduling authority.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import crypto from "node:crypto";
|
|
11
|
+
import fs from "node:fs";
|
|
12
|
+
import path from "node:path";
|
|
13
|
+
import { TeamDistributedQueue } from "../lib/agent-team/team-distributed-queue.js";
|
|
14
|
+
import { TeamRunner } from "../lib/agent-team/team-runner.js";
|
|
15
|
+
import {
|
|
16
|
+
summarizeWorktreeCheckpoint,
|
|
17
|
+
TeamWorktreeCoordinator,
|
|
18
|
+
} from "../lib/agent-team/team-worktree.js";
|
|
19
|
+
import { TeamProcessCheckpointBroker } from "../lib/agent-team/team-process-checkpoint.js";
|
|
20
|
+
import { CostBudget } from "../lib/cost-budget.js";
|
|
21
|
+
import { resolveTeamTaskContract } from "../lib/agent-team/team-task-contract.js";
|
|
22
|
+
import executionBroker from "../lib/process-execution-broker/index.js";
|
|
23
|
+
import {
|
|
24
|
+
SECURE_FILE_IDENTITY_ERROR,
|
|
25
|
+
SecureFileIdentityError,
|
|
26
|
+
sameFileStatIdentity,
|
|
27
|
+
samePathHandleFileIdentity,
|
|
28
|
+
withTrustedFileParentSync,
|
|
29
|
+
} from "../lib/secure-file-identity.js";
|
|
30
|
+
|
|
31
|
+
const MAX_GRAPH_BYTES = 64 * 1024 * 1024;
|
|
32
|
+
const MAX_AGENT_PROMPT_BYTES = 1024 * 1024;
|
|
33
|
+
const MAX_AGENT_EXECUTION_PROMPT_BYTES = 2 * 1024 * 1024;
|
|
34
|
+
const MAX_AGENT_USAGE_RECORDS = 64;
|
|
35
|
+
const SHELL_WORKTREE_MODE = "shell-worktree";
|
|
36
|
+
const AGENT_WORKTREE_MODE = "agent-worktree";
|
|
37
|
+
const DISTRIBUTED_MODES = new Set([SHELL_WORKTREE_MODE, AGENT_WORKTREE_MODE]);
|
|
38
|
+
const AGENT_OPTION_FIELDS = new Set([
|
|
39
|
+
"permissionMode",
|
|
40
|
+
"model",
|
|
41
|
+
"maxTurns",
|
|
42
|
+
"maxBudgetUsd",
|
|
43
|
+
"maxTokens",
|
|
44
|
+
"maxWallMs",
|
|
45
|
+
"checkpointRequired",
|
|
46
|
+
"worktreeRequired",
|
|
47
|
+
]);
|
|
48
|
+
const AGENT_PERMISSION_MODES = new Set([
|
|
49
|
+
"manual",
|
|
50
|
+
"auto",
|
|
51
|
+
"dontAsk",
|
|
52
|
+
"default",
|
|
53
|
+
"plan",
|
|
54
|
+
"acceptEdits",
|
|
55
|
+
"bypassPermissions",
|
|
56
|
+
]);
|
|
57
|
+
const AGENT_USAGE_FIELDS = [
|
|
58
|
+
"input_tokens",
|
|
59
|
+
"output_tokens",
|
|
60
|
+
"cache_read_input_tokens",
|
|
61
|
+
"cache_creation_input_tokens",
|
|
62
|
+
];
|
|
63
|
+
const ADJUDICATION_DECISIONS = new Set(["retry", "accept", "cancel"]);
|
|
64
|
+
const EVIDENCE_DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u;
|
|
65
|
+
const AUTHORITY_KIND = "chainlesschain.team.distributed";
|
|
66
|
+
const AUTHORITY_VERSION = 1;
|
|
67
|
+
|
|
68
|
+
export class TeamDistributedCliError extends Error {
|
|
69
|
+
constructor(code, message, details = null, cause = null) {
|
|
70
|
+
super(message, cause ? { cause } : undefined);
|
|
71
|
+
this.name = "TeamDistributedCliError";
|
|
72
|
+
this.code = code;
|
|
73
|
+
if (details != null) this.details = details;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function fail(code, message, details, cause = null) {
|
|
78
|
+
throw new TeamDistributedCliError(code, message, details, cause);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function samePath(left, right) {
|
|
82
|
+
const a = path.resolve(left);
|
|
83
|
+
const b = path.resolve(right);
|
|
84
|
+
return process.platform === "win32"
|
|
85
|
+
? a.toLowerCase() === b.toLowerCase()
|
|
86
|
+
: a === b;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function pathInside(child, root) {
|
|
90
|
+
const relative = path.relative(root, child);
|
|
91
|
+
return (
|
|
92
|
+
relative === "" ||
|
|
93
|
+
(!path.isAbsolute(relative) &&
|
|
94
|
+
relative !== ".." &&
|
|
95
|
+
!relative.startsWith(`..${path.sep}`))
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function requireText(value, label) {
|
|
100
|
+
const text = String(value || "");
|
|
101
|
+
if (
|
|
102
|
+
text.length === 0 ||
|
|
103
|
+
text.length > 512 ||
|
|
104
|
+
text !== text.trim() ||
|
|
105
|
+
text.includes("\0")
|
|
106
|
+
) {
|
|
107
|
+
fail(
|
|
108
|
+
"TEAM_QUEUE_INVALID_OPTION",
|
|
109
|
+
`${label} must be a stable non-empty value`,
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
return text;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function optionalPositive(value, label, { integer = false } = {}) {
|
|
116
|
+
if (value == null) return null;
|
|
117
|
+
const parsed = Number(value);
|
|
118
|
+
if (
|
|
119
|
+
!Number.isFinite(parsed) ||
|
|
120
|
+
parsed <= 0 ||
|
|
121
|
+
(integer && !Number.isSafeInteger(parsed))
|
|
122
|
+
) {
|
|
123
|
+
fail(
|
|
124
|
+
"TEAM_QUEUE_INVALID_OPTION",
|
|
125
|
+
`${label} must be a positive${integer ? " integer" : ""}`,
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
return parsed;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function requireEvidenceDigest(value, label = "--evidence-digest") {
|
|
132
|
+
const digest = String(value || "");
|
|
133
|
+
if (!EVIDENCE_DIGEST_PATTERN.test(digest)) {
|
|
134
|
+
fail(
|
|
135
|
+
"TEAM_QUEUE_INVALID_OPTION",
|
|
136
|
+
`${label} must be an exact sha256 evidence digest`,
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
return digest;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function isPlainObject(value) {
|
|
143
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
144
|
+
const prototype = Object.getPrototypeOf(value);
|
|
145
|
+
return prototype === Object.prototype || prototype === null;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function canonicalize(value) {
|
|
149
|
+
if (Array.isArray(value)) return value.map(canonicalize);
|
|
150
|
+
if (!isPlainObject(value)) return value;
|
|
151
|
+
const normalized = {};
|
|
152
|
+
for (const key of Object.keys(value).sort()) {
|
|
153
|
+
normalized[key] = canonicalize(value[key]);
|
|
154
|
+
}
|
|
155
|
+
return normalized;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function canonicalDigest(value) {
|
|
159
|
+
return crypto
|
|
160
|
+
.createHash("sha256")
|
|
161
|
+
.update(JSON.stringify(canonicalize(value)))
|
|
162
|
+
.digest("hex");
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function distributedMode(value = SHELL_WORKTREE_MODE) {
|
|
166
|
+
const mode = String(value || SHELL_WORKTREE_MODE);
|
|
167
|
+
if (!DISTRIBUTED_MODES.has(mode)) {
|
|
168
|
+
fail(
|
|
169
|
+
"TEAM_QUEUE_MODE_UNSUPPORTED",
|
|
170
|
+
`Distributed mode must be one of: ${Array.from(DISTRIBUTED_MODES).join(
|
|
171
|
+
", ",
|
|
172
|
+
)}`,
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
return mode;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function stableAgentText(
|
|
179
|
+
value,
|
|
180
|
+
label,
|
|
181
|
+
{ nullable = false, maxBytes = 512, trim = true } = {},
|
|
182
|
+
) {
|
|
183
|
+
if (nullable && value == null) return null;
|
|
184
|
+
if (typeof value !== "string") {
|
|
185
|
+
fail("TEAM_QUEUE_INVALID_GRAPH", `${label} must be a string`);
|
|
186
|
+
}
|
|
187
|
+
const text = trim ? value.trim() : value;
|
|
188
|
+
if (
|
|
189
|
+
text.trim().length === 0 ||
|
|
190
|
+
Buffer.byteLength(text, "utf8") > maxBytes ||
|
|
191
|
+
text.includes("\0")
|
|
192
|
+
) {
|
|
193
|
+
fail(
|
|
194
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
195
|
+
`${label} must be a stable non-empty value no larger than ${maxBytes} bytes`,
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
return text;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function validateAgentOptionObject(value, label) {
|
|
202
|
+
if (value == null) return;
|
|
203
|
+
if (!isPlainObject(value)) {
|
|
204
|
+
fail("TEAM_QUEUE_INVALID_GRAPH", `${label} must be an object`);
|
|
205
|
+
}
|
|
206
|
+
for (const field of Object.keys(value)) {
|
|
207
|
+
if (!AGENT_OPTION_FIELDS.has(field)) {
|
|
208
|
+
fail(
|
|
209
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
210
|
+
`${label}.${field} is not a supported distributed Agent option`,
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
if (
|
|
215
|
+
Object.hasOwn(value, "permissionMode") &&
|
|
216
|
+
!AGENT_PERMISSION_MODES.has(value.permissionMode)
|
|
217
|
+
) {
|
|
218
|
+
fail(
|
|
219
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
220
|
+
`${label}.permissionMode is not supported`,
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
if (Object.hasOwn(value, "model")) {
|
|
224
|
+
stableAgentText(value.model, `${label}.model`, { maxBytes: 512 });
|
|
225
|
+
}
|
|
226
|
+
for (const field of ["maxTurns", "maxTokens", "maxWallMs"]) {
|
|
227
|
+
if (Object.hasOwn(value, field)) {
|
|
228
|
+
optionalPositive(value[field], `${label}.${field}`, { integer: true });
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
if (Object.hasOwn(value, "maxBudgetUsd")) {
|
|
232
|
+
optionalPositive(value.maxBudgetUsd, `${label}.maxBudgetUsd`);
|
|
233
|
+
}
|
|
234
|
+
for (const field of ["checkpointRequired", "worktreeRequired"]) {
|
|
235
|
+
if (Object.hasOwn(value, field) && value[field] !== true) {
|
|
236
|
+
fail(
|
|
237
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
238
|
+
`${label}.${field} cannot weaken agent-worktree isolation`,
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function normalizeAgentAuthority(value, { source = "authority" } = {}) {
|
|
245
|
+
if (!isPlainObject(value)) {
|
|
246
|
+
fail(
|
|
247
|
+
"TEAM_QUEUE_AUTHORITY_MISMATCH",
|
|
248
|
+
`Distributed Agent ${source} is missing`,
|
|
249
|
+
);
|
|
250
|
+
}
|
|
251
|
+
const expectedFields = new Set([
|
|
252
|
+
"permissionMode",
|
|
253
|
+
"model",
|
|
254
|
+
"maxTurns",
|
|
255
|
+
"maxBudgetUsd",
|
|
256
|
+
"maxTokens",
|
|
257
|
+
"maxWallMs",
|
|
258
|
+
"checkpointRequired",
|
|
259
|
+
"worktreeRequired",
|
|
260
|
+
]);
|
|
261
|
+
if (Object.keys(value).some((field) => !expectedFields.has(field))) {
|
|
262
|
+
fail(
|
|
263
|
+
"TEAM_QUEUE_AUTHORITY_MISMATCH",
|
|
264
|
+
`Distributed Agent ${source} contains unsupported fields`,
|
|
265
|
+
);
|
|
266
|
+
}
|
|
267
|
+
if (!AGENT_PERMISSION_MODES.has(value.permissionMode)) {
|
|
268
|
+
fail(
|
|
269
|
+
"TEAM_QUEUE_AUTHORITY_MISMATCH",
|
|
270
|
+
`Distributed Agent ${source} has an invalid permission mode`,
|
|
271
|
+
);
|
|
272
|
+
}
|
|
273
|
+
const normalized = {
|
|
274
|
+
permissionMode: value.permissionMode,
|
|
275
|
+
model:
|
|
276
|
+
value.model == null
|
|
277
|
+
? null
|
|
278
|
+
: stableAgentText(value.model, `Agent ${source} model`, {
|
|
279
|
+
maxBytes: 512,
|
|
280
|
+
}),
|
|
281
|
+
maxTurns:
|
|
282
|
+
value.maxTurns == null
|
|
283
|
+
? null
|
|
284
|
+
: optionalPositive(value.maxTurns, `Agent ${source} maxTurns`, {
|
|
285
|
+
integer: true,
|
|
286
|
+
}),
|
|
287
|
+
maxBudgetUsd:
|
|
288
|
+
value.maxBudgetUsd == null
|
|
289
|
+
? null
|
|
290
|
+
: optionalPositive(value.maxBudgetUsd, `Agent ${source} maxBudgetUsd`),
|
|
291
|
+
maxTokens:
|
|
292
|
+
value.maxTokens == null
|
|
293
|
+
? null
|
|
294
|
+
: optionalPositive(value.maxTokens, `Agent ${source} maxTokens`, {
|
|
295
|
+
integer: true,
|
|
296
|
+
}),
|
|
297
|
+
maxWallMs:
|
|
298
|
+
value.maxWallMs == null
|
|
299
|
+
? null
|
|
300
|
+
: optionalPositive(value.maxWallMs, `Agent ${source} maxWallMs`, {
|
|
301
|
+
integer: true,
|
|
302
|
+
}),
|
|
303
|
+
checkpointRequired: value.checkpointRequired === true,
|
|
304
|
+
worktreeRequired: value.worktreeRequired === true,
|
|
305
|
+
};
|
|
306
|
+
if (!normalized.checkpointRequired || !normalized.worktreeRequired) {
|
|
307
|
+
fail(
|
|
308
|
+
"TEAM_QUEUE_AUTHORITY_MISMATCH",
|
|
309
|
+
`Distributed Agent ${source} must require checkpoint and worktree isolation`,
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
return normalized;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
function agentAuthorityForOptions(options, budget) {
|
|
316
|
+
const permissionMode = String(options.permissionMode || "acceptEdits");
|
|
317
|
+
if (!AGENT_PERMISSION_MODES.has(permissionMode)) {
|
|
318
|
+
fail(
|
|
319
|
+
"TEAM_QUEUE_INVALID_OPTION",
|
|
320
|
+
`--permission-mode must be one of: ${Array.from(
|
|
321
|
+
AGENT_PERMISSION_MODES,
|
|
322
|
+
).join(", ")}`,
|
|
323
|
+
);
|
|
324
|
+
}
|
|
325
|
+
const model =
|
|
326
|
+
options.model == null
|
|
327
|
+
? null
|
|
328
|
+
: stableAgentText(options.model, "--model", { maxBytes: 512 });
|
|
329
|
+
return {
|
|
330
|
+
permissionMode,
|
|
331
|
+
model,
|
|
332
|
+
maxTurns: optionalPositive(options.agentMaxTurns, "--agent-max-turns", {
|
|
333
|
+
integer: true,
|
|
334
|
+
}),
|
|
335
|
+
maxBudgetUsd:
|
|
336
|
+
optionalPositive(options.agentMaxBudgetUsd, "--agent-max-budget-usd") ??
|
|
337
|
+
budget.maxUsd,
|
|
338
|
+
maxTokens:
|
|
339
|
+
optionalPositive(options.agentMaxTokens, "--agent-max-tokens", {
|
|
340
|
+
integer: true,
|
|
341
|
+
}) ?? budget.maxTokens,
|
|
342
|
+
maxWallMs:
|
|
343
|
+
optionalPositive(options.agentMaxWallMs, "--agent-max-wall-ms", {
|
|
344
|
+
integer: true,
|
|
345
|
+
}) ?? budget.maxWallMs,
|
|
346
|
+
checkpointRequired: true,
|
|
347
|
+
worktreeRequired: true,
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
function normalizeAgentUsage(value, label = "agent usage") {
|
|
352
|
+
if (value == null) return null;
|
|
353
|
+
if (!isPlainObject(value)) {
|
|
354
|
+
fail("TEAM_QUEUE_AGENT_USAGE_INVALID", `${label} must be an object`);
|
|
355
|
+
}
|
|
356
|
+
const usage = {};
|
|
357
|
+
let fields = 0;
|
|
358
|
+
let total = 0;
|
|
359
|
+
for (const field of AGENT_USAGE_FIELDS) {
|
|
360
|
+
if (value[field] == null) continue;
|
|
361
|
+
const amount = Number(value[field]);
|
|
362
|
+
if (!Number.isSafeInteger(amount) || amount < 0) {
|
|
363
|
+
fail(
|
|
364
|
+
"TEAM_QUEUE_AGENT_USAGE_INVALID",
|
|
365
|
+
`${label}.${field} must be a non-negative safe integer`,
|
|
366
|
+
);
|
|
367
|
+
}
|
|
368
|
+
usage[field] = amount;
|
|
369
|
+
total += amount;
|
|
370
|
+
fields += 1;
|
|
371
|
+
}
|
|
372
|
+
if (fields === 0 || !Number.isSafeInteger(total)) {
|
|
373
|
+
fail(
|
|
374
|
+
"TEAM_QUEUE_AGENT_USAGE_INVALID",
|
|
375
|
+
`${label} has no accountable token fields`,
|
|
376
|
+
);
|
|
377
|
+
}
|
|
378
|
+
return usage;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function agentMetering(
|
|
382
|
+
result,
|
|
383
|
+
{
|
|
384
|
+
requireUsage = false,
|
|
385
|
+
requirePricing = false,
|
|
386
|
+
maxTokens = null,
|
|
387
|
+
maxBudgetUsd = null,
|
|
388
|
+
} = {},
|
|
389
|
+
) {
|
|
390
|
+
if (!isPlainObject(result)) {
|
|
391
|
+
fail(
|
|
392
|
+
"TEAM_QUEUE_AGENT_RESULT_INVALID",
|
|
393
|
+
"Distributed Agent executor must return an object",
|
|
394
|
+
);
|
|
395
|
+
}
|
|
396
|
+
const usage = normalizeAgentUsage(result.usage);
|
|
397
|
+
if (requireUsage && usage == null) {
|
|
398
|
+
fail(
|
|
399
|
+
"TEAM_QUEUE_AGENT_USAGE_REQUIRED",
|
|
400
|
+
"Budgeted distributed Agent execution returned no accountable usage",
|
|
401
|
+
);
|
|
402
|
+
}
|
|
403
|
+
const provider =
|
|
404
|
+
result.provider == null
|
|
405
|
+
? null
|
|
406
|
+
: stableAgentText(result.provider, "agent result provider", {
|
|
407
|
+
maxBytes: 512,
|
|
408
|
+
});
|
|
409
|
+
const model =
|
|
410
|
+
result.model == null
|
|
411
|
+
? null
|
|
412
|
+
: stableAgentText(result.model, "agent result model", { maxBytes: 512 });
|
|
413
|
+
let usageRecords = null;
|
|
414
|
+
if (result.usageRecords != null) {
|
|
415
|
+
if (
|
|
416
|
+
!Array.isArray(result.usageRecords) ||
|
|
417
|
+
result.usageRecords.length === 0 ||
|
|
418
|
+
result.usageRecords.length > MAX_AGENT_USAGE_RECORDS
|
|
419
|
+
) {
|
|
420
|
+
fail(
|
|
421
|
+
"TEAM_QUEUE_AGENT_USAGE_INVALID",
|
|
422
|
+
`agent usageRecords must contain 1..${MAX_AGENT_USAGE_RECORDS} records`,
|
|
423
|
+
);
|
|
424
|
+
}
|
|
425
|
+
usageRecords = result.usageRecords.map((record, index) => {
|
|
426
|
+
if (!isPlainObject(record)) {
|
|
427
|
+
fail(
|
|
428
|
+
"TEAM_QUEUE_AGENT_USAGE_INVALID",
|
|
429
|
+
`agent usageRecords[${index}] must be an object`,
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
return {
|
|
433
|
+
provider:
|
|
434
|
+
record.provider == null
|
|
435
|
+
? null
|
|
436
|
+
: stableAgentText(
|
|
437
|
+
record.provider,
|
|
438
|
+
`agent usageRecords[${index}].provider`,
|
|
439
|
+
{ maxBytes: 512 },
|
|
440
|
+
),
|
|
441
|
+
model:
|
|
442
|
+
record.model == null
|
|
443
|
+
? null
|
|
444
|
+
: stableAgentText(
|
|
445
|
+
record.model,
|
|
446
|
+
`agent usageRecords[${index}].model`,
|
|
447
|
+
{ maxBytes: 512 },
|
|
448
|
+
),
|
|
449
|
+
usage: normalizeAgentUsage(
|
|
450
|
+
record.usage,
|
|
451
|
+
`agent usageRecords[${index}].usage`,
|
|
452
|
+
),
|
|
453
|
+
};
|
|
454
|
+
});
|
|
455
|
+
if (usage) {
|
|
456
|
+
for (const field of AGENT_USAGE_FIELDS) {
|
|
457
|
+
const aggregate = usageRecords.reduce(
|
|
458
|
+
(total, record) => total + Number(record.usage[field] || 0),
|
|
459
|
+
0,
|
|
460
|
+
);
|
|
461
|
+
if (aggregate !== Number(usage[field] || 0)) {
|
|
462
|
+
fail(
|
|
463
|
+
"TEAM_QUEUE_AGENT_USAGE_INVALID",
|
|
464
|
+
`agent usageRecords do not match aggregate ${field}`,
|
|
465
|
+
);
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
const records =
|
|
471
|
+
usageRecords ||
|
|
472
|
+
(usage
|
|
473
|
+
? [
|
|
474
|
+
{
|
|
475
|
+
provider,
|
|
476
|
+
model,
|
|
477
|
+
usage,
|
|
478
|
+
},
|
|
479
|
+
]
|
|
480
|
+
: []);
|
|
481
|
+
const pricing = new CostBudget({ limitUsd: Number.MAX_VALUE });
|
|
482
|
+
for (const record of records) pricing.add(record);
|
|
483
|
+
if (requirePricing && pricing.sawUnpriced) {
|
|
484
|
+
fail(
|
|
485
|
+
"TEAM_QUEUE_AGENT_USAGE_UNPRICED",
|
|
486
|
+
"Distributed Agent USD cap cannot account for unpriced provider/model usage",
|
|
487
|
+
);
|
|
488
|
+
}
|
|
489
|
+
const accountableTokens = AGENT_USAGE_FIELDS.reduce(
|
|
490
|
+
(total, field) => total + Number(usage?.[field] || 0),
|
|
491
|
+
0,
|
|
492
|
+
);
|
|
493
|
+
if (Number(maxTokens) > 0 && accountableTokens > Number(maxTokens)) {
|
|
494
|
+
fail(
|
|
495
|
+
"TEAM_QUEUE_AGENT_TOKEN_LIMIT",
|
|
496
|
+
`Distributed Agent usage ${accountableTokens} exceeds its ${maxTokens}-token reservation`,
|
|
497
|
+
);
|
|
498
|
+
}
|
|
499
|
+
if (
|
|
500
|
+
Number(maxBudgetUsd) > 0 &&
|
|
501
|
+
pricing.spentUsd - Number(maxBudgetUsd) > Number.EPSILON
|
|
502
|
+
) {
|
|
503
|
+
fail(
|
|
504
|
+
"TEAM_QUEUE_AGENT_COST_LIMIT",
|
|
505
|
+
`Distributed Agent cost ${pricing.spentUsd} exceeds its $${maxBudgetUsd} reservation`,
|
|
506
|
+
);
|
|
507
|
+
}
|
|
508
|
+
return {
|
|
509
|
+
usage,
|
|
510
|
+
provider,
|
|
511
|
+
model,
|
|
512
|
+
usageRecords,
|
|
513
|
+
costUsd: pricing.sawUnpriced ? null : pricing.spentUsd,
|
|
514
|
+
};
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
function applyAgentBudgetReservation(contract, reservation) {
|
|
518
|
+
const effective = { ...contract };
|
|
519
|
+
if (Number(reservation?.maxTokens) > 0) {
|
|
520
|
+
effective.maxTokens =
|
|
521
|
+
Number(contract.maxTokens) > 0
|
|
522
|
+
? Math.min(contract.maxTokens, reservation.maxTokens)
|
|
523
|
+
: reservation.maxTokens;
|
|
524
|
+
}
|
|
525
|
+
if (Number(reservation?.maxBudgetUsd) > 0) {
|
|
526
|
+
effective.maxBudgetUsd =
|
|
527
|
+
Number(contract.maxBudgetUsd) > 0
|
|
528
|
+
? Math.min(contract.maxBudgetUsd, reservation.maxBudgetUsd)
|
|
529
|
+
: reservation.maxBudgetUsd;
|
|
530
|
+
}
|
|
531
|
+
return effective;
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
function privateRegularRead(
|
|
535
|
+
filePath,
|
|
536
|
+
maxBytes,
|
|
537
|
+
label,
|
|
538
|
+
{ runtimeFs = fs, secureFileParent = withTrustedFileParentSync } = {},
|
|
539
|
+
) {
|
|
540
|
+
const absolute = path.resolve(filePath);
|
|
541
|
+
try {
|
|
542
|
+
return secureFileParent(
|
|
543
|
+
runtimeFs,
|
|
544
|
+
absolute,
|
|
545
|
+
({ canonicalPath, parentDevice }) => {
|
|
546
|
+
let before;
|
|
547
|
+
try {
|
|
548
|
+
before = runtimeFs.lstatSync(canonicalPath, { bigint: true });
|
|
549
|
+
} catch (error) {
|
|
550
|
+
fail(
|
|
551
|
+
"TEAM_QUEUE_INPUT_UNAVAILABLE",
|
|
552
|
+
`Cannot inspect ${label} ${absolute}: ${error.message}`,
|
|
553
|
+
);
|
|
554
|
+
}
|
|
555
|
+
if (
|
|
556
|
+
!before.isFile() ||
|
|
557
|
+
before.isSymbolicLink() ||
|
|
558
|
+
Number(before.nlink) !== 1
|
|
559
|
+
) {
|
|
560
|
+
fail(
|
|
561
|
+
"TEAM_QUEUE_INSECURE_INPUT",
|
|
562
|
+
`${label} must be a regular, single-link file: ${absolute}`,
|
|
563
|
+
);
|
|
564
|
+
}
|
|
565
|
+
const expectedBytes = Number(before.size);
|
|
566
|
+
if (expectedBytes <= 0 || expectedBytes > maxBytes) {
|
|
567
|
+
fail(
|
|
568
|
+
"TEAM_QUEUE_INPUT_TOO_LARGE",
|
|
569
|
+
`${label} must be between 1 and ${maxBytes} bytes`,
|
|
570
|
+
);
|
|
571
|
+
}
|
|
572
|
+
const descriptor = runtimeFs.openSync(
|
|
573
|
+
canonicalPath,
|
|
574
|
+
runtimeFs.constants.O_RDONLY | (runtimeFs.constants.O_NOFOLLOW || 0),
|
|
575
|
+
);
|
|
576
|
+
try {
|
|
577
|
+
const opened = runtimeFs.fstatSync(descriptor, { bigint: true });
|
|
578
|
+
if (
|
|
579
|
+
!opened.isFile() ||
|
|
580
|
+
Number(opened.nlink) !== 1 ||
|
|
581
|
+
!samePathHandleFileIdentity(before, opened, parentDevice)
|
|
582
|
+
) {
|
|
583
|
+
fail(
|
|
584
|
+
"TEAM_QUEUE_INPUT_RACE",
|
|
585
|
+
`${label} changed while it was being opened`,
|
|
586
|
+
);
|
|
587
|
+
}
|
|
588
|
+
const body = runtimeFs.readFileSync(descriptor, "utf8");
|
|
589
|
+
const after = runtimeFs.fstatSync(descriptor, { bigint: true });
|
|
590
|
+
if (
|
|
591
|
+
Buffer.byteLength(body, "utf8") !== expectedBytes ||
|
|
592
|
+
!sameFileStatIdentity(opened, after)
|
|
593
|
+
) {
|
|
594
|
+
fail(
|
|
595
|
+
"TEAM_QUEUE_INPUT_RACE",
|
|
596
|
+
`${label} changed while it was being read`,
|
|
597
|
+
);
|
|
598
|
+
}
|
|
599
|
+
return body;
|
|
600
|
+
} finally {
|
|
601
|
+
runtimeFs.closeSync(descriptor);
|
|
602
|
+
}
|
|
603
|
+
},
|
|
604
|
+
);
|
|
605
|
+
} catch (error) {
|
|
606
|
+
if (error instanceof TeamDistributedCliError) throw error;
|
|
607
|
+
if (error instanceof SecureFileIdentityError) {
|
|
608
|
+
const code =
|
|
609
|
+
error.code === SECURE_FILE_IDENTITY_ERROR.PARENT_RACE
|
|
610
|
+
? "TEAM_QUEUE_INPUT_RACE"
|
|
611
|
+
: "TEAM_QUEUE_INSECURE_INPUT";
|
|
612
|
+
fail(
|
|
613
|
+
code,
|
|
614
|
+
`Cannot securely read ${label} ${absolute}: ${error.message}`,
|
|
615
|
+
{ secureFileIdentityCode: error.code },
|
|
616
|
+
error,
|
|
617
|
+
);
|
|
618
|
+
}
|
|
619
|
+
fail(
|
|
620
|
+
"TEAM_QUEUE_INPUT_UNAVAILABLE",
|
|
621
|
+
`Cannot securely read ${label} ${absolute}: ${error.message}`,
|
|
622
|
+
);
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
function normalizeTasks(
|
|
627
|
+
document,
|
|
628
|
+
{ mode = SHELL_WORKTREE_MODE, agentAuthority = null } = {},
|
|
629
|
+
) {
|
|
630
|
+
const source = Array.isArray(document) ? document : document?.tasks;
|
|
631
|
+
if (!Array.isArray(source) || source.length === 0) {
|
|
632
|
+
fail(
|
|
633
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
634
|
+
"Task graph must contain a non-empty `tasks` array",
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
const keys = new Set();
|
|
638
|
+
const tasks = source.map((task, index) => {
|
|
639
|
+
if (!task || typeof task !== "object" || Array.isArray(task)) {
|
|
640
|
+
fail(
|
|
641
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
642
|
+
`Task at index ${index} must be an object`,
|
|
643
|
+
);
|
|
644
|
+
}
|
|
645
|
+
const key = requireText(task.key, `task[${index}].key`);
|
|
646
|
+
if (keys.has(key)) {
|
|
647
|
+
fail("TEAM_QUEUE_INVALID_GRAPH", `Duplicate task key "${key}"`);
|
|
648
|
+
}
|
|
649
|
+
keys.add(key);
|
|
650
|
+
if (
|
|
651
|
+
task.title !== undefined &&
|
|
652
|
+
(typeof task.title !== "string" ||
|
|
653
|
+
task.title.trim().length === 0 ||
|
|
654
|
+
task.title.includes("\0"))
|
|
655
|
+
) {
|
|
656
|
+
fail("TEAM_QUEUE_INVALID_GRAPH", `Task "${key}" has an invalid title`);
|
|
657
|
+
}
|
|
658
|
+
if (
|
|
659
|
+
task.priority !== undefined &&
|
|
660
|
+
!["high", "normal", "low"].includes(task.priority)
|
|
661
|
+
) {
|
|
662
|
+
fail(
|
|
663
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
664
|
+
`Task "${key}" priority must be high, normal, or low`,
|
|
665
|
+
);
|
|
666
|
+
}
|
|
667
|
+
if (task.retrySafe !== undefined && typeof task.retrySafe !== "boolean") {
|
|
668
|
+
fail(
|
|
669
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
670
|
+
`Task "${key}" retrySafe must be boolean`,
|
|
671
|
+
);
|
|
672
|
+
}
|
|
673
|
+
for (const field of ["sparsePaths", "symlinkDirectories"]) {
|
|
674
|
+
if (task[field] == null) continue;
|
|
675
|
+
if (
|
|
676
|
+
!Array.isArray(task[field]) ||
|
|
677
|
+
task[field].length > 128 ||
|
|
678
|
+
task[field].some(
|
|
679
|
+
(entry) =>
|
|
680
|
+
typeof entry !== "string" ||
|
|
681
|
+
entry.length === 0 ||
|
|
682
|
+
entry !== entry.trim() ||
|
|
683
|
+
entry.includes("\0"),
|
|
684
|
+
)
|
|
685
|
+
) {
|
|
686
|
+
fail(
|
|
687
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
688
|
+
`Task "${key}" ${field} must be a bounded array of stable paths`,
|
|
689
|
+
);
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
let executionMetadata;
|
|
693
|
+
if (mode === SHELL_WORKTREE_MODE) {
|
|
694
|
+
if (
|
|
695
|
+
typeof task.command !== "string" ||
|
|
696
|
+
task.command.trim().length === 0
|
|
697
|
+
) {
|
|
698
|
+
fail(
|
|
699
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
700
|
+
`Task "${key}" needs a non-empty shell command`,
|
|
701
|
+
);
|
|
702
|
+
}
|
|
703
|
+
executionMetadata = { command: task.command };
|
|
704
|
+
} else {
|
|
705
|
+
const allowedFields = new Set([
|
|
706
|
+
"key",
|
|
707
|
+
"title",
|
|
708
|
+
"dependsOn",
|
|
709
|
+
"deps",
|
|
710
|
+
"priority",
|
|
711
|
+
"retrySafe",
|
|
712
|
+
"sparsePaths",
|
|
713
|
+
"symlinkDirectories",
|
|
714
|
+
"prompt",
|
|
715
|
+
"agent",
|
|
716
|
+
"policy",
|
|
717
|
+
]);
|
|
718
|
+
const unsupported = Object.keys(task).filter(
|
|
719
|
+
(field) => !allowedFields.has(field),
|
|
720
|
+
);
|
|
721
|
+
if (unsupported.length > 0) {
|
|
722
|
+
fail(
|
|
723
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
724
|
+
`Task "${key}" contains unsupported agent-worktree fields: ${unsupported.join(
|
|
725
|
+
", ",
|
|
726
|
+
)}`,
|
|
727
|
+
);
|
|
728
|
+
}
|
|
729
|
+
if (Object.hasOwn(task, "command")) {
|
|
730
|
+
fail(
|
|
731
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
732
|
+
`Task "${key}" cannot mix a shell command into agent-worktree mode`,
|
|
733
|
+
);
|
|
734
|
+
}
|
|
735
|
+
const prompt = stableAgentText(task.prompt, `task "${key}".prompt`, {
|
|
736
|
+
maxBytes: MAX_AGENT_PROMPT_BYTES,
|
|
737
|
+
trim: false,
|
|
738
|
+
});
|
|
739
|
+
validateAgentOptionObject(task.agent, `task "${key}".agent`);
|
|
740
|
+
validateAgentOptionObject(task.policy, `task "${key}".policy`);
|
|
741
|
+
const contract = resolveTeamTaskContract({
|
|
742
|
+
parent: agentAuthority,
|
|
743
|
+
task,
|
|
744
|
+
});
|
|
745
|
+
if (contract.adjustments.length > 0) {
|
|
746
|
+
fail(
|
|
747
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
748
|
+
`Task "${key}" Agent options cannot be honored exactly: ${contract.adjustments
|
|
749
|
+
.map((adjustment) => adjustment.field)
|
|
750
|
+
.join(", ")}`,
|
|
751
|
+
{ adjustments: contract.adjustments },
|
|
752
|
+
);
|
|
753
|
+
}
|
|
754
|
+
if (
|
|
755
|
+
contract.checkpointRequired !== true ||
|
|
756
|
+
contract.worktreeRequired !== true
|
|
757
|
+
) {
|
|
758
|
+
fail(
|
|
759
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
760
|
+
`Task "${key}" must require worktree and managed checkpoint isolation`,
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
executionMetadata = {
|
|
764
|
+
prompt,
|
|
765
|
+
agentContract: {
|
|
766
|
+
permissionMode: contract.permissionMode,
|
|
767
|
+
model: contract.model,
|
|
768
|
+
maxTurns: contract.maxTurns,
|
|
769
|
+
maxBudgetUsd: contract.maxBudgetUsd,
|
|
770
|
+
maxTokens: contract.maxTokens,
|
|
771
|
+
maxWallMs: contract.maxWallMs,
|
|
772
|
+
checkpointRequired: true,
|
|
773
|
+
worktreeRequired: true,
|
|
774
|
+
},
|
|
775
|
+
managedCheckpointRequired: true,
|
|
776
|
+
};
|
|
777
|
+
}
|
|
778
|
+
const dependencySource = task.dependsOn ?? task.deps ?? [];
|
|
779
|
+
if (!Array.isArray(dependencySource)) {
|
|
780
|
+
fail(
|
|
781
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
782
|
+
`Task "${key}" dependencies must be an array`,
|
|
783
|
+
);
|
|
784
|
+
}
|
|
785
|
+
const dependsOn = [...dependencySource];
|
|
786
|
+
if (
|
|
787
|
+
dependsOn.some(
|
|
788
|
+
(dependency) =>
|
|
789
|
+
typeof dependency !== "string" || dependency.length === 0,
|
|
790
|
+
)
|
|
791
|
+
) {
|
|
792
|
+
fail(
|
|
793
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
794
|
+
`Task "${key}" has an invalid dependency`,
|
|
795
|
+
);
|
|
796
|
+
}
|
|
797
|
+
return {
|
|
798
|
+
key,
|
|
799
|
+
title:
|
|
800
|
+
typeof task.title === "string" && task.title.length > 0
|
|
801
|
+
? task.title
|
|
802
|
+
: key,
|
|
803
|
+
dependsOn,
|
|
804
|
+
priority: task.priority || "normal",
|
|
805
|
+
metadata: {
|
|
806
|
+
...executionMetadata,
|
|
807
|
+
retrySafe: task.retrySafe === true,
|
|
808
|
+
sparsePaths: task.sparsePaths || null,
|
|
809
|
+
symlinkDirectories: task.symlinkDirectories || null,
|
|
810
|
+
},
|
|
811
|
+
};
|
|
812
|
+
});
|
|
813
|
+
for (const task of tasks) {
|
|
814
|
+
for (const dependency of task.dependsOn) {
|
|
815
|
+
if (!keys.has(dependency)) {
|
|
816
|
+
fail(
|
|
817
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
818
|
+
`Task "${task.key}" depends on unknown task "${dependency}"`,
|
|
819
|
+
);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
return tasks;
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
function readTasks(
|
|
827
|
+
filePath,
|
|
828
|
+
options = {},
|
|
829
|
+
{ runtimeFs = fs, secureFileParent = withTrustedFileParentSync } = {},
|
|
830
|
+
) {
|
|
831
|
+
let parsed;
|
|
832
|
+
try {
|
|
833
|
+
parsed = JSON.parse(
|
|
834
|
+
privateRegularRead(filePath, MAX_GRAPH_BYTES, "task graph", {
|
|
835
|
+
runtimeFs,
|
|
836
|
+
secureFileParent,
|
|
837
|
+
}),
|
|
838
|
+
);
|
|
839
|
+
} catch (error) {
|
|
840
|
+
if (error instanceof TeamDistributedCliError) throw error;
|
|
841
|
+
fail(
|
|
842
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
843
|
+
`Cannot parse task graph ${path.resolve(filePath)}: ${error.message}`,
|
|
844
|
+
);
|
|
845
|
+
}
|
|
846
|
+
return normalizeTasks(parsed, options);
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
function defaultResolveRepoRoot(value = process.cwd()) {
|
|
850
|
+
const cwd = fs.realpathSync.native(path.resolve(value));
|
|
851
|
+
const output = executionBroker.execFileSync(
|
|
852
|
+
"git",
|
|
853
|
+
["rev-parse", "--show-toplevel"],
|
|
854
|
+
{
|
|
855
|
+
cwd,
|
|
856
|
+
encoding: "utf8",
|
|
857
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
858
|
+
shell: false,
|
|
859
|
+
windowsHide: true,
|
|
860
|
+
origin: "team-distributed:repo-root",
|
|
861
|
+
policy: "allow",
|
|
862
|
+
scope: "team",
|
|
863
|
+
},
|
|
864
|
+
);
|
|
865
|
+
return fs.realpathSync.native(String(output).trim());
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
function assertExternalStatePath(statePath, repoRoot) {
|
|
869
|
+
const target = path.resolve(requireText(statePath, "--state"));
|
|
870
|
+
if (pathInside(target, repoRoot)) {
|
|
871
|
+
fail(
|
|
872
|
+
"TEAM_QUEUE_STATE_INSIDE_REPOSITORY",
|
|
873
|
+
"Distributed queue --state must be outside the agent-writable repository",
|
|
874
|
+
);
|
|
875
|
+
}
|
|
876
|
+
let ancestor = path.dirname(target);
|
|
877
|
+
while (!fs.existsSync(ancestor)) {
|
|
878
|
+
const parent = path.dirname(ancestor);
|
|
879
|
+
if (parent === ancestor) break;
|
|
880
|
+
ancestor = parent;
|
|
881
|
+
}
|
|
882
|
+
const realAncestor = fs.realpathSync.native(ancestor);
|
|
883
|
+
const projected = path.resolve(
|
|
884
|
+
realAncestor,
|
|
885
|
+
path.relative(ancestor, path.dirname(target)),
|
|
886
|
+
path.basename(target),
|
|
887
|
+
);
|
|
888
|
+
if (pathInside(projected, repoRoot)) {
|
|
889
|
+
fail(
|
|
890
|
+
"TEAM_QUEUE_STATE_INSIDE_REPOSITORY",
|
|
891
|
+
"Distributed queue --state resolves inside the agent-writable repository",
|
|
892
|
+
);
|
|
893
|
+
}
|
|
894
|
+
return projected;
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
function assertExternalCheckpointStateDir(stateDir, repoRoot, statePath) {
|
|
898
|
+
const target = path.resolve(requireText(stateDir, "--checkpoint-state-dir"));
|
|
899
|
+
if (pathInside(target, repoRoot)) {
|
|
900
|
+
fail(
|
|
901
|
+
"TEAM_QUEUE_CHECKPOINT_INSIDE_REPOSITORY",
|
|
902
|
+
"Distributed checkpoint state must be outside the agent-writable repository",
|
|
903
|
+
);
|
|
904
|
+
}
|
|
905
|
+
if (samePath(target, statePath)) {
|
|
906
|
+
fail(
|
|
907
|
+
"TEAM_QUEUE_CHECKPOINT_STATE_CONFLICT",
|
|
908
|
+
"Distributed checkpoint state directory must differ from the queue state file",
|
|
909
|
+
);
|
|
910
|
+
}
|
|
911
|
+
let ancestor = target;
|
|
912
|
+
while (!fs.existsSync(ancestor)) {
|
|
913
|
+
const parent = path.dirname(ancestor);
|
|
914
|
+
if (parent === ancestor) break;
|
|
915
|
+
ancestor = parent;
|
|
916
|
+
}
|
|
917
|
+
const realAncestor = fs.realpathSync.native(ancestor);
|
|
918
|
+
const projected = path.resolve(realAncestor, path.relative(ancestor, target));
|
|
919
|
+
if (pathInside(projected, repoRoot)) {
|
|
920
|
+
fail(
|
|
921
|
+
"TEAM_QUEUE_CHECKPOINT_INSIDE_REPOSITORY",
|
|
922
|
+
"Distributed checkpoint state resolves inside the agent-writable repository",
|
|
923
|
+
);
|
|
924
|
+
}
|
|
925
|
+
if (samePath(projected, statePath)) {
|
|
926
|
+
fail(
|
|
927
|
+
"TEAM_QUEUE_CHECKPOINT_STATE_CONFLICT",
|
|
928
|
+
"Distributed checkpoint state directory must differ from the queue state file",
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
return projected;
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
function checkpointAuthority({ enabled = false, stateDir = null } = {}) {
|
|
935
|
+
return {
|
|
936
|
+
enabled: enabled === true,
|
|
937
|
+
stateDir: enabled === true ? stateDir : null,
|
|
938
|
+
coverageTarget: "partial",
|
|
939
|
+
writerIsolation: "unknown",
|
|
940
|
+
externalSideEffects: true,
|
|
941
|
+
};
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
function authorityFor({
|
|
945
|
+
repoRoot,
|
|
946
|
+
runId,
|
|
947
|
+
mode,
|
|
948
|
+
baseTarget,
|
|
949
|
+
checkpoint,
|
|
950
|
+
agent = null,
|
|
951
|
+
}) {
|
|
952
|
+
const authority = {
|
|
953
|
+
kind: AUTHORITY_KIND,
|
|
954
|
+
version: AUTHORITY_VERSION,
|
|
955
|
+
repoRoot,
|
|
956
|
+
runId,
|
|
957
|
+
mode,
|
|
958
|
+
baseTarget: {
|
|
959
|
+
branch: baseTarget.branch ?? null,
|
|
960
|
+
commitOid: baseTarget.commitOid,
|
|
961
|
+
},
|
|
962
|
+
checkpoint: checkpointAuthority(checkpoint),
|
|
963
|
+
};
|
|
964
|
+
if (mode === AGENT_WORKTREE_MODE) {
|
|
965
|
+
authority.agent = normalizeAgentAuthority(agent, {
|
|
966
|
+
source: "initialization contract",
|
|
967
|
+
});
|
|
968
|
+
}
|
|
969
|
+
return authority;
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
function assertAuthority(
|
|
973
|
+
snapshot,
|
|
974
|
+
{ repoRoot, runId, queueId, authorityDigest, statePath },
|
|
975
|
+
) {
|
|
976
|
+
const authority = snapshot.authority;
|
|
977
|
+
const checkpoint = authority?.checkpoint
|
|
978
|
+
? authority.checkpoint
|
|
979
|
+
: checkpointAuthority();
|
|
980
|
+
const checkpointValid =
|
|
981
|
+
typeof checkpoint.enabled === "boolean" &&
|
|
982
|
+
checkpoint.coverageTarget === "partial" &&
|
|
983
|
+
checkpoint.writerIsolation === "unknown" &&
|
|
984
|
+
checkpoint.externalSideEffects === true &&
|
|
985
|
+
(checkpoint.enabled
|
|
986
|
+
? typeof checkpoint.stateDir === "string" &&
|
|
987
|
+
checkpoint.stateDir.length > 0 &&
|
|
988
|
+
path.isAbsolute(checkpoint.stateDir)
|
|
989
|
+
: checkpoint.stateDir === null);
|
|
990
|
+
const modeValid = DISTRIBUTED_MODES.has(authority?.mode);
|
|
991
|
+
let agent = null;
|
|
992
|
+
if (authority?.mode === AGENT_WORKTREE_MODE) {
|
|
993
|
+
agent = normalizeAgentAuthority(authority.agent);
|
|
994
|
+
} else if (authority?.agent != null) {
|
|
995
|
+
fail(
|
|
996
|
+
"TEAM_QUEUE_AUTHORITY_MISMATCH",
|
|
997
|
+
"Shell distributed authority cannot carry Agent execution options",
|
|
998
|
+
);
|
|
999
|
+
}
|
|
1000
|
+
const valid =
|
|
1001
|
+
authority?.kind === AUTHORITY_KIND &&
|
|
1002
|
+
authority?.version === AUTHORITY_VERSION &&
|
|
1003
|
+
modeValid &&
|
|
1004
|
+
authority?.runId === runId &&
|
|
1005
|
+
samePath(authority?.repoRoot || "", repoRoot) &&
|
|
1006
|
+
typeof authority?.baseTarget?.commitOid === "string" &&
|
|
1007
|
+
checkpointValid &&
|
|
1008
|
+
(authority?.mode !== AGENT_WORKTREE_MODE || checkpoint.enabled === true);
|
|
1009
|
+
if (!valid) {
|
|
1010
|
+
fail(
|
|
1011
|
+
"TEAM_QUEUE_AUTHORITY_MISMATCH",
|
|
1012
|
+
"Distributed queue authority does not match this repository/run/mode",
|
|
1013
|
+
);
|
|
1014
|
+
}
|
|
1015
|
+
if (queueId && snapshot.queueId !== queueId) {
|
|
1016
|
+
fail(
|
|
1017
|
+
"TEAM_QUEUE_AUTHORITY_MISMATCH",
|
|
1018
|
+
"Distributed queue id does not match the pinned --queue-id",
|
|
1019
|
+
);
|
|
1020
|
+
}
|
|
1021
|
+
if (authorityDigest && snapshot.authorityDigest !== authorityDigest) {
|
|
1022
|
+
fail(
|
|
1023
|
+
"TEAM_QUEUE_AUTHORITY_MISMATCH",
|
|
1024
|
+
"Distributed queue authority digest does not match the pinned value",
|
|
1025
|
+
);
|
|
1026
|
+
}
|
|
1027
|
+
if (checkpoint.enabled) {
|
|
1028
|
+
const externalStateDir = assertExternalCheckpointStateDir(
|
|
1029
|
+
checkpoint.stateDir,
|
|
1030
|
+
repoRoot,
|
|
1031
|
+
statePath,
|
|
1032
|
+
);
|
|
1033
|
+
if (!samePath(externalStateDir, checkpoint.stateDir)) {
|
|
1034
|
+
fail(
|
|
1035
|
+
"TEAM_QUEUE_AUTHORITY_MISMATCH",
|
|
1036
|
+
"Distributed checkpoint state directory is not canonical",
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
return {
|
|
1041
|
+
...authority,
|
|
1042
|
+
checkpoint: checkpointAuthority(checkpoint),
|
|
1043
|
+
...(agent ? { agent } : {}),
|
|
1044
|
+
};
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
function coordinatorForAuthority(repoRoot, authority, deps, options = {}) {
|
|
1048
|
+
const coordinator = new deps.WorktreeCoordinator(repoRoot, {
|
|
1049
|
+
runId: authority.runId,
|
|
1050
|
+
checkpointBroker: options.checkpointBroker || null,
|
|
1051
|
+
onCheckpoint: options.onCheckpoint || null,
|
|
1052
|
+
snapshot: options.snapshot || {
|
|
1053
|
+
version: 5,
|
|
1054
|
+
runId: authority.runId,
|
|
1055
|
+
baseTarget: authority.baseTarget,
|
|
1056
|
+
records: [],
|
|
1057
|
+
},
|
|
1058
|
+
});
|
|
1059
|
+
if (!coordinator.isGitRepo()) {
|
|
1060
|
+
fail(
|
|
1061
|
+
"TEAM_QUEUE_GIT_REQUIRED",
|
|
1062
|
+
"Distributed shell workers require a git repository",
|
|
1063
|
+
);
|
|
1064
|
+
}
|
|
1065
|
+
return coordinator;
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
function completedResult(task) {
|
|
1069
|
+
return task?.status === "completed" ? task.metadata?.result || null : null;
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
function wallBudgetForQueueStats(queueStats, queue, deps) {
|
|
1073
|
+
const persisted = queueStats?.budget;
|
|
1074
|
+
const maxWallMs = Number(persisted?.maxWallMs);
|
|
1075
|
+
if (!Number.isFinite(maxWallMs) || maxWallMs <= 0) return null;
|
|
1076
|
+
|
|
1077
|
+
const persistedStarted = Number.isFinite(persisted?.startedAt);
|
|
1078
|
+
let startedAt = persistedStarted ? persisted.startedAt : null;
|
|
1079
|
+
const status = () => {
|
|
1080
|
+
const elapsedMs =
|
|
1081
|
+
startedAt == null ? 0 : Math.max(0, deps.now() - startedAt);
|
|
1082
|
+
return {
|
|
1083
|
+
tasks: 0,
|
|
1084
|
+
maxTasks: null,
|
|
1085
|
+
tokens: 0,
|
|
1086
|
+
maxTokens: null,
|
|
1087
|
+
reservedTokens: 0,
|
|
1088
|
+
spentUsd: 0,
|
|
1089
|
+
maxUsd: null,
|
|
1090
|
+
reservedUsd: 0,
|
|
1091
|
+
reservations: 0,
|
|
1092
|
+
unpricedUsage: false,
|
|
1093
|
+
elapsedMs,
|
|
1094
|
+
maxWallMs,
|
|
1095
|
+
reason: elapsedMs >= maxWallMs ? "max-wall-ms" : null,
|
|
1096
|
+
};
|
|
1097
|
+
};
|
|
1098
|
+
|
|
1099
|
+
// Durable queue transactions remain the sole authority for task/token/USD
|
|
1100
|
+
// reservations and settlement. This adapter carries only the wall
|
|
1101
|
+
// deadline so TeamRunner can abort an active claim (including worktree
|
|
1102
|
+
// commit/checkpoint work) instead of noticing the cap only at the next
|
|
1103
|
+
// acquire. Refresh an as-yet-unused queue immediately before Runner starts:
|
|
1104
|
+
// a peer that acquired before the refresh supplies the durable start, while
|
|
1105
|
+
// a peer that acquires after it cannot receive an earlier deadline.
|
|
1106
|
+
return {
|
|
1107
|
+
maxWallMs,
|
|
1108
|
+
start() {
|
|
1109
|
+
if (startedAt == null) {
|
|
1110
|
+
const refreshed = queue.budgetStatus();
|
|
1111
|
+
if (
|
|
1112
|
+
!Number.isFinite(refreshed?.startedAt) &&
|
|
1113
|
+
!Number.isFinite(refreshed?.observedAt)
|
|
1114
|
+
) {
|
|
1115
|
+
fail(
|
|
1116
|
+
"TEAM_QUEUE_BUDGET_VIEW_INVALID",
|
|
1117
|
+
"Distributed queue wall budget lacks a linearized observation time",
|
|
1118
|
+
);
|
|
1119
|
+
}
|
|
1120
|
+
startedAt = Number.isFinite(refreshed.startedAt)
|
|
1121
|
+
? refreshed.startedAt
|
|
1122
|
+
: refreshed.observedAt;
|
|
1123
|
+
}
|
|
1124
|
+
return this;
|
|
1125
|
+
},
|
|
1126
|
+
record() {
|
|
1127
|
+
return this;
|
|
1128
|
+
},
|
|
1129
|
+
releaseReservation() {
|
|
1130
|
+
return false;
|
|
1131
|
+
},
|
|
1132
|
+
status,
|
|
1133
|
+
reason() {
|
|
1134
|
+
return status().reason;
|
|
1135
|
+
},
|
|
1136
|
+
shouldStop() {
|
|
1137
|
+
return status().reason != null;
|
|
1138
|
+
},
|
|
1139
|
+
};
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
function distributedTasksDone(tasks) {
|
|
1143
|
+
return tasks.every(
|
|
1144
|
+
(task) =>
|
|
1145
|
+
["completed", "cancelled"].includes(task.status) &&
|
|
1146
|
+
task.metadata?.adjudication?.required !== true,
|
|
1147
|
+
);
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
function importCompletedDependencies(coordinator, queue, task) {
|
|
1151
|
+
const imported = new Set(
|
|
1152
|
+
(coordinator.snapshot().records || []).map((record) => record.key),
|
|
1153
|
+
);
|
|
1154
|
+
const visit = (key, visiting = new Set()) => {
|
|
1155
|
+
if (imported.has(key)) return;
|
|
1156
|
+
if (visiting.has(key)) {
|
|
1157
|
+
fail("TEAM_QUEUE_INVALID_GRAPH", "Cyclic dependency result graph");
|
|
1158
|
+
}
|
|
1159
|
+
visiting.add(key);
|
|
1160
|
+
const dependency = queue.getTask(key);
|
|
1161
|
+
if (!dependency || dependency.status !== "completed") {
|
|
1162
|
+
fail(
|
|
1163
|
+
"TEAM_QUEUE_DEPENDENCY_UNAVAILABLE",
|
|
1164
|
+
`Completed dependency result for "${key}" is unavailable`,
|
|
1165
|
+
);
|
|
1166
|
+
}
|
|
1167
|
+
for (const parent of dependency.dependsOn || []) visit(parent, visiting);
|
|
1168
|
+
const result = completedResult(dependency);
|
|
1169
|
+
if (!result) {
|
|
1170
|
+
fail(
|
|
1171
|
+
"TEAM_QUEUE_DEPENDENCY_UNAVAILABLE",
|
|
1172
|
+
`Completed dependency "${key}" has no worktree result`,
|
|
1173
|
+
);
|
|
1174
|
+
}
|
|
1175
|
+
coordinator.registerCompletedDependency(key, result);
|
|
1176
|
+
imported.add(key);
|
|
1177
|
+
visiting.delete(key);
|
|
1178
|
+
};
|
|
1179
|
+
for (const dependency of task.dependsOn || []) visit(dependency);
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
function importRetryWorkspace(coordinator, task) {
|
|
1183
|
+
const existing = new Set(
|
|
1184
|
+
(coordinator.snapshot().records || []).map((record) => record.key),
|
|
1185
|
+
);
|
|
1186
|
+
if (existing.has(task.key)) return;
|
|
1187
|
+
const history = task.metadata?.workspaceExecutionHistory;
|
|
1188
|
+
if (!Array.isArray(history) || history.length === 0) return;
|
|
1189
|
+
const settled = [...history]
|
|
1190
|
+
.reverse()
|
|
1191
|
+
.find((entry) =>
|
|
1192
|
+
["preparation-rolled-back", "rolled-back"].includes(entry?.phase),
|
|
1193
|
+
);
|
|
1194
|
+
if (!settled?.worktree) return;
|
|
1195
|
+
coordinator.seed([settled.worktree]);
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
function workerRegistry(queue, workerId, { settlementForLease = null } = {}) {
|
|
1199
|
+
const holder = (options = {}) => ({
|
|
1200
|
+
...options,
|
|
1201
|
+
holder: `${workerId}:${options.holder || "teammate"}`,
|
|
1202
|
+
});
|
|
1203
|
+
return new Proxy(queue.asRegistry(), {
|
|
1204
|
+
get(target, property) {
|
|
1205
|
+
if (
|
|
1206
|
+
["acquire", "renew", "release", "complete", "fail"].includes(property)
|
|
1207
|
+
) {
|
|
1208
|
+
return (key, options = {}) => {
|
|
1209
|
+
const metering =
|
|
1210
|
+
["complete", "fail"].includes(property) &&
|
|
1211
|
+
typeof settlementForLease === "function"
|
|
1212
|
+
? settlementForLease(options.leaseId)
|
|
1213
|
+
: null;
|
|
1214
|
+
const result = target[property](
|
|
1215
|
+
key,
|
|
1216
|
+
holder({
|
|
1217
|
+
...options,
|
|
1218
|
+
...(metering?.usage ? { usage: metering.usage } : {}),
|
|
1219
|
+
...(metering?.usageRecords
|
|
1220
|
+
? { usageRecords: metering.usageRecords }
|
|
1221
|
+
: {}),
|
|
1222
|
+
...(Number.isFinite(metering?.costUsd)
|
|
1223
|
+
? { costUsd: metering.costUsd }
|
|
1224
|
+
: {}),
|
|
1225
|
+
...(property === "complete" &&
|
|
1226
|
+
options.result &&
|
|
1227
|
+
Number.isFinite(metering?.costUsd)
|
|
1228
|
+
? {
|
|
1229
|
+
result: {
|
|
1230
|
+
...options.result,
|
|
1231
|
+
costUsd: metering.costUsd,
|
|
1232
|
+
},
|
|
1233
|
+
}
|
|
1234
|
+
: {}),
|
|
1235
|
+
}),
|
|
1236
|
+
);
|
|
1237
|
+
if (result?.ok) metering?.settled?.();
|
|
1238
|
+
return result;
|
|
1239
|
+
};
|
|
1240
|
+
}
|
|
1241
|
+
const value = target[property];
|
|
1242
|
+
return typeof value === "function" ? value.bind(target) : value;
|
|
1243
|
+
},
|
|
1244
|
+
});
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
function defaultDependencies(overrides = {}) {
|
|
1248
|
+
return {
|
|
1249
|
+
Queue: TeamDistributedQueue,
|
|
1250
|
+
Runner: TeamRunner,
|
|
1251
|
+
WorktreeCoordinator: TeamWorktreeCoordinator,
|
|
1252
|
+
CheckpointBroker: TeamProcessCheckpointBroker,
|
|
1253
|
+
fileSystem: fs,
|
|
1254
|
+
secureFileParent: withTrustedFileParentSync,
|
|
1255
|
+
agentExecutor: null,
|
|
1256
|
+
buildAgentPrompt: (prompt) => prompt,
|
|
1257
|
+
onRunner: null,
|
|
1258
|
+
resolveRepoRoot: defaultResolveRepoRoot,
|
|
1259
|
+
now: () => Date.now(),
|
|
1260
|
+
git: (repoRoot, args) =>
|
|
1261
|
+
String(
|
|
1262
|
+
executionBroker.execFileSync("git", args, {
|
|
1263
|
+
cwd: repoRoot,
|
|
1264
|
+
encoding: "utf8",
|
|
1265
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1266
|
+
shell: false,
|
|
1267
|
+
windowsHide: true,
|
|
1268
|
+
origin: "team-distributed:finalize",
|
|
1269
|
+
policy: "allow",
|
|
1270
|
+
scope: "team",
|
|
1271
|
+
}),
|
|
1272
|
+
).trim(),
|
|
1273
|
+
sleep: (milliseconds) =>
|
|
1274
|
+
new Promise((resolve) => setTimeout(resolve, milliseconds)),
|
|
1275
|
+
...overrides,
|
|
1276
|
+
};
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
function openPinnedQueue(options, deps) {
|
|
1280
|
+
const repoRoot = deps.resolveRepoRoot(options.repo || process.cwd());
|
|
1281
|
+
const runId = requireText(options.runId, "--run-id");
|
|
1282
|
+
const statePath = assertExternalStatePath(options.state, repoRoot);
|
|
1283
|
+
const queue = deps.Queue.open({
|
|
1284
|
+
filePath: statePath,
|
|
1285
|
+
runId,
|
|
1286
|
+
expectedQueueId: options.queueId || null,
|
|
1287
|
+
expectedAuthorityDigest: options.authorityDigest || null,
|
|
1288
|
+
});
|
|
1289
|
+
const snapshot = queue.snapshot();
|
|
1290
|
+
const authority = assertAuthority(snapshot, {
|
|
1291
|
+
repoRoot,
|
|
1292
|
+
runId,
|
|
1293
|
+
queueId: options.queueId || null,
|
|
1294
|
+
authorityDigest: options.authorityDigest || null,
|
|
1295
|
+
statePath,
|
|
1296
|
+
});
|
|
1297
|
+
const expectedMode =
|
|
1298
|
+
options.agent === true
|
|
1299
|
+
? AGENT_WORKTREE_MODE
|
|
1300
|
+
: options.mode
|
|
1301
|
+
? distributedMode(options.mode)
|
|
1302
|
+
: null;
|
|
1303
|
+
if (expectedMode && authority.mode !== expectedMode) {
|
|
1304
|
+
fail(
|
|
1305
|
+
"TEAM_QUEUE_AUTHORITY_MISMATCH",
|
|
1306
|
+
`Worker/finalizer requested ${expectedMode}, but the pinned queue authority is ${authority.mode}`,
|
|
1307
|
+
);
|
|
1308
|
+
}
|
|
1309
|
+
if (
|
|
1310
|
+
options.managedCheckpoint === true &&
|
|
1311
|
+
authority.checkpoint.enabled !== true
|
|
1312
|
+
) {
|
|
1313
|
+
fail(
|
|
1314
|
+
"TEAM_QUEUE_CHECKPOINT_AUTHORITY_MISMATCH",
|
|
1315
|
+
"--managed-checkpoint cannot widen a queue created without checkpoint authority",
|
|
1316
|
+
);
|
|
1317
|
+
}
|
|
1318
|
+
if (options.checkpointStateDir) {
|
|
1319
|
+
const canonicalCheckpointStateDir = assertExternalCheckpointStateDir(
|
|
1320
|
+
options.checkpointStateDir,
|
|
1321
|
+
repoRoot,
|
|
1322
|
+
statePath,
|
|
1323
|
+
);
|
|
1324
|
+
if (
|
|
1325
|
+
authority.checkpoint.enabled !== true ||
|
|
1326
|
+
!samePath(canonicalCheckpointStateDir, authority.checkpoint.stateDir)
|
|
1327
|
+
) {
|
|
1328
|
+
fail(
|
|
1329
|
+
"TEAM_QUEUE_CHECKPOINT_AUTHORITY_MISMATCH",
|
|
1330
|
+
"--checkpoint-state-dir must match the pinned queue authority",
|
|
1331
|
+
);
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
return { queue, snapshot, authority, repoRoot, statePath, runId };
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
export function initDistributedQueue(options, dependencyOverrides = {}) {
|
|
1338
|
+
if (options.queueId != null || options.authorityDigest != null) {
|
|
1339
|
+
fail(
|
|
1340
|
+
"TEAM_QUEUE_INVALID_OPTION",
|
|
1341
|
+
"--queue-id and --authority-digest are generated by queue init and cannot be supplied",
|
|
1342
|
+
);
|
|
1343
|
+
}
|
|
1344
|
+
const deps = defaultDependencies(dependencyOverrides);
|
|
1345
|
+
const repoRoot = deps.resolveRepoRoot(options.repo || process.cwd());
|
|
1346
|
+
const runId = requireText(options.runId, "--run-id");
|
|
1347
|
+
const statePath = assertExternalStatePath(options.state, repoRoot);
|
|
1348
|
+
const mode = distributedMode(options.mode);
|
|
1349
|
+
if (options.checkpointStateDir && options.managedCheckpoint !== true) {
|
|
1350
|
+
fail(
|
|
1351
|
+
"TEAM_QUEUE_INVALID_OPTION",
|
|
1352
|
+
"--checkpoint-state-dir requires --managed-checkpoint",
|
|
1353
|
+
);
|
|
1354
|
+
}
|
|
1355
|
+
const managedCheckpointStateDir =
|
|
1356
|
+
options.managedCheckpoint === true
|
|
1357
|
+
? assertExternalCheckpointStateDir(
|
|
1358
|
+
options.checkpointStateDir || `${statePath}.workspace-transactions`,
|
|
1359
|
+
repoRoot,
|
|
1360
|
+
statePath,
|
|
1361
|
+
)
|
|
1362
|
+
: null;
|
|
1363
|
+
if (mode === AGENT_WORKTREE_MODE && options.managedCheckpoint !== true) {
|
|
1364
|
+
fail(
|
|
1365
|
+
"TEAM_QUEUE_CHECKPOINT_REQUIRED",
|
|
1366
|
+
"agent-worktree queues require --managed-checkpoint",
|
|
1367
|
+
);
|
|
1368
|
+
}
|
|
1369
|
+
const probe = new deps.WorktreeCoordinator(repoRoot, { runId });
|
|
1370
|
+
if (!probe.isGitRepo()) {
|
|
1371
|
+
fail(
|
|
1372
|
+
"TEAM_QUEUE_GIT_REQUIRED",
|
|
1373
|
+
"Distributed shell workers require a git repository",
|
|
1374
|
+
);
|
|
1375
|
+
}
|
|
1376
|
+
const baseTarget = probe.snapshot().baseTarget;
|
|
1377
|
+
const budget = {
|
|
1378
|
+
maxTasks: optionalPositive(options.maxTasks, "--max-tasks", {
|
|
1379
|
+
integer: true,
|
|
1380
|
+
}),
|
|
1381
|
+
maxTokens: optionalPositive(options.maxTokens, "--max-tokens", {
|
|
1382
|
+
integer: true,
|
|
1383
|
+
}),
|
|
1384
|
+
maxUsd: optionalPositive(options.maxUsd, "--max-usd"),
|
|
1385
|
+
maxWallMs: optionalPositive(options.maxWallMs, "--max-wall-ms", {
|
|
1386
|
+
integer: true,
|
|
1387
|
+
}),
|
|
1388
|
+
};
|
|
1389
|
+
const agent =
|
|
1390
|
+
mode === AGENT_WORKTREE_MODE
|
|
1391
|
+
? agentAuthorityForOptions(options, budget)
|
|
1392
|
+
: null;
|
|
1393
|
+
const tasks = readTasks(
|
|
1394
|
+
options.tasks,
|
|
1395
|
+
{
|
|
1396
|
+
mode,
|
|
1397
|
+
agentAuthority: agent,
|
|
1398
|
+
},
|
|
1399
|
+
{
|
|
1400
|
+
runtimeFs: deps.fileSystem,
|
|
1401
|
+
secureFileParent: deps.secureFileParent,
|
|
1402
|
+
},
|
|
1403
|
+
);
|
|
1404
|
+
if (budget.maxTasks == null) budget.maxTasks = tasks.length;
|
|
1405
|
+
const authority = authorityFor({
|
|
1406
|
+
repoRoot,
|
|
1407
|
+
runId,
|
|
1408
|
+
mode,
|
|
1409
|
+
baseTarget,
|
|
1410
|
+
checkpoint: {
|
|
1411
|
+
enabled: options.managedCheckpoint === true,
|
|
1412
|
+
stateDir: managedCheckpointStateDir,
|
|
1413
|
+
},
|
|
1414
|
+
agent,
|
|
1415
|
+
});
|
|
1416
|
+
const created = deps.Queue.create({
|
|
1417
|
+
filePath: statePath,
|
|
1418
|
+
runId,
|
|
1419
|
+
tasks,
|
|
1420
|
+
budget,
|
|
1421
|
+
defaultTtlMs:
|
|
1422
|
+
optionalPositive(options.ttlMs, "--ttl-ms", { integer: true }) ??
|
|
1423
|
+
undefined,
|
|
1424
|
+
authority,
|
|
1425
|
+
});
|
|
1426
|
+
return {
|
|
1427
|
+
...created,
|
|
1428
|
+
statePath,
|
|
1429
|
+
repoRoot,
|
|
1430
|
+
runId,
|
|
1431
|
+
mode,
|
|
1432
|
+
authority,
|
|
1433
|
+
budget,
|
|
1434
|
+
checkpoint: checkpointAuthority({
|
|
1435
|
+
enabled: options.managedCheckpoint === true,
|
|
1436
|
+
stateDir: managedCheckpointStateDir,
|
|
1437
|
+
}),
|
|
1438
|
+
};
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1441
|
+
export function distributedQueueStatus(options, dependencyOverrides = {}) {
|
|
1442
|
+
const deps = defaultDependencies(dependencyOverrides);
|
|
1443
|
+
const opened = openPinnedQueue(options, deps);
|
|
1444
|
+
const view =
|
|
1445
|
+
typeof opened.queue.statusView === "function"
|
|
1446
|
+
? opened.queue.statusView()
|
|
1447
|
+
: {
|
|
1448
|
+
queueId: opened.snapshot.queueId,
|
|
1449
|
+
revision: opened.snapshot.revision,
|
|
1450
|
+
graphDigest: opened.snapshot.graphDigest,
|
|
1451
|
+
authorityDigest: opened.snapshot.authorityDigest,
|
|
1452
|
+
stats: opened.queue.stats(),
|
|
1453
|
+
finalization: opened.queue.getFinalization(),
|
|
1454
|
+
pendingAdjudications: opened.queue.pendingAdjudications(),
|
|
1455
|
+
tasks: opened.queue.list(),
|
|
1456
|
+
};
|
|
1457
|
+
return {
|
|
1458
|
+
queueId: view.queueId,
|
|
1459
|
+
revision: view.revision,
|
|
1460
|
+
graphDigest: view.graphDigest,
|
|
1461
|
+
authorityDigest: view.authorityDigest,
|
|
1462
|
+
authority: view.authority ?? opened.authority,
|
|
1463
|
+
stats: view.stats,
|
|
1464
|
+
finalization: view.finalization,
|
|
1465
|
+
pendingAdjudications: view.pendingAdjudications,
|
|
1466
|
+
interruptions: view.interruptions ?? [],
|
|
1467
|
+
tasks: view.tasks,
|
|
1468
|
+
};
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1471
|
+
function requireQueueMutation(result, code, action) {
|
|
1472
|
+
if (result?.ok) return result;
|
|
1473
|
+
fail(
|
|
1474
|
+
code,
|
|
1475
|
+
`Distributed queue could not ${action}: ${result?.reason || "unknown"}${
|
|
1476
|
+
result?.error ? ` (${result.error})` : ""
|
|
1477
|
+
}`,
|
|
1478
|
+
result || null,
|
|
1479
|
+
);
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
export function interruptDistributedQueue(options, dependencyOverrides = {}) {
|
|
1483
|
+
const deps = defaultDependencies(dependencyOverrides);
|
|
1484
|
+
const opened = openPinnedQueue(options, deps);
|
|
1485
|
+
const key = requireText(options.task, "--task");
|
|
1486
|
+
const result = requireQueueMutation(
|
|
1487
|
+
opened.queue.requestInterrupt(key, {
|
|
1488
|
+
holder: requireText(options.holder, "--holder"),
|
|
1489
|
+
leaseId: requireText(options.leaseId, "--lease-id"),
|
|
1490
|
+
fencingToken: optionalPositive(options.fencingToken, "--fencing-token", {
|
|
1491
|
+
integer: true,
|
|
1492
|
+
}),
|
|
1493
|
+
requestId: requireText(options.requestId, "--request-id"),
|
|
1494
|
+
actor: requireText(options.actor || "human", "--actor"),
|
|
1495
|
+
reason:
|
|
1496
|
+
options.reason == null
|
|
1497
|
+
? "human takeover requested"
|
|
1498
|
+
: requireText(options.reason, "--reason"),
|
|
1499
|
+
}),
|
|
1500
|
+
"TEAM_QUEUE_INTERRUPT_REJECTED",
|
|
1501
|
+
`interrupt task "${key}"`,
|
|
1502
|
+
);
|
|
1503
|
+
const view =
|
|
1504
|
+
typeof opened.queue.statusView === "function"
|
|
1505
|
+
? opened.queue.statusView()
|
|
1506
|
+
: null;
|
|
1507
|
+
return {
|
|
1508
|
+
queueId: opened.snapshot.queueId,
|
|
1509
|
+
...(view ? { revision: view.revision } : {}),
|
|
1510
|
+
task: key,
|
|
1511
|
+
...result,
|
|
1512
|
+
};
|
|
1513
|
+
}
|
|
1514
|
+
|
|
1515
|
+
function gitOid(repoRoot, ref, deps, label) {
|
|
1516
|
+
let oid;
|
|
1517
|
+
try {
|
|
1518
|
+
oid = deps.git(repoRoot, ["rev-parse", "--verify", ref]);
|
|
1519
|
+
} catch (error) {
|
|
1520
|
+
fail(
|
|
1521
|
+
"TEAM_QUEUE_RECOVERY_GIT_DRIFT",
|
|
1522
|
+
`Could not resolve ${label}: ${
|
|
1523
|
+
error instanceof Error ? error.message : String(error)
|
|
1524
|
+
}`,
|
|
1525
|
+
);
|
|
1526
|
+
}
|
|
1527
|
+
if (!/^[a-f0-9]{40,64}$/u.test(oid || "")) {
|
|
1528
|
+
fail(
|
|
1529
|
+
"TEAM_QUEUE_RECOVERY_GIT_DRIFT",
|
|
1530
|
+
`${label} did not resolve to a stable Git object`,
|
|
1531
|
+
);
|
|
1532
|
+
}
|
|
1533
|
+
return oid.toLowerCase();
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
function assertCleanGitWorktree(
|
|
1537
|
+
worktreePath,
|
|
1538
|
+
deps,
|
|
1539
|
+
{
|
|
1540
|
+
code = "TEAM_QUEUE_RECOVERY_GIT_DRIFT",
|
|
1541
|
+
message = "Task worktree is not clean",
|
|
1542
|
+
} = {},
|
|
1543
|
+
) {
|
|
1544
|
+
let status;
|
|
1545
|
+
try {
|
|
1546
|
+
status = deps.git(worktreePath, [
|
|
1547
|
+
"status",
|
|
1548
|
+
"--porcelain=v1",
|
|
1549
|
+
"--untracked-files=all",
|
|
1550
|
+
]);
|
|
1551
|
+
} catch (error) {
|
|
1552
|
+
fail(
|
|
1553
|
+
code,
|
|
1554
|
+
`Could not inspect task worktree: ${
|
|
1555
|
+
error instanceof Error ? error.message : String(error)
|
|
1556
|
+
}`,
|
|
1557
|
+
);
|
|
1558
|
+
}
|
|
1559
|
+
if (status !== "") {
|
|
1560
|
+
fail(code, message, { status });
|
|
1561
|
+
}
|
|
1562
|
+
return status;
|
|
1563
|
+
}
|
|
1564
|
+
|
|
1565
|
+
function verifyCommittedRecoveryGit(opened, execution, deps) {
|
|
1566
|
+
const expected = execution?.verifiedCommitOid;
|
|
1567
|
+
if (
|
|
1568
|
+
!expected ||
|
|
1569
|
+
typeof execution?.worktree?.branch !== "string" ||
|
|
1570
|
+
typeof execution?.worktree?.path !== "string"
|
|
1571
|
+
) {
|
|
1572
|
+
fail(
|
|
1573
|
+
"TEAM_QUEUE_RECOVERY_EVIDENCE_INCOMPLETE",
|
|
1574
|
+
"Committed checkpoint recovery lacks a verified worktree commit",
|
|
1575
|
+
);
|
|
1576
|
+
}
|
|
1577
|
+
const branchHead = gitOid(
|
|
1578
|
+
opened.repoRoot,
|
|
1579
|
+
execution.worktree.branch,
|
|
1580
|
+
deps,
|
|
1581
|
+
`task branch "${execution.worktree.branch}"`,
|
|
1582
|
+
);
|
|
1583
|
+
const worktreeHead = gitOid(
|
|
1584
|
+
execution.worktree.path,
|
|
1585
|
+
"HEAD",
|
|
1586
|
+
deps,
|
|
1587
|
+
`task worktree "${execution.worktree.path}"`,
|
|
1588
|
+
);
|
|
1589
|
+
const status = assertCleanGitWorktree(execution.worktree.path, deps, {
|
|
1590
|
+
message:
|
|
1591
|
+
"Committed checkpoint no longer matches the exact clean task branch/worktree",
|
|
1592
|
+
});
|
|
1593
|
+
if (
|
|
1594
|
+
branchHead !== expected.toLowerCase() ||
|
|
1595
|
+
worktreeHead !== expected.toLowerCase() ||
|
|
1596
|
+
status !== ""
|
|
1597
|
+
) {
|
|
1598
|
+
fail(
|
|
1599
|
+
"TEAM_QUEUE_RECOVERY_GIT_DRIFT",
|
|
1600
|
+
"Committed checkpoint no longer matches the exact clean task branch/worktree",
|
|
1601
|
+
{ expected, branchHead, worktreeHead, status },
|
|
1602
|
+
);
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
function repairRolledBackGitBaseline(opened, execution, deps) {
|
|
1607
|
+
const baseline = execution.worktree.baselineCommitOid?.toLowerCase();
|
|
1608
|
+
if (!baseline) {
|
|
1609
|
+
fail(
|
|
1610
|
+
"TEAM_QUEUE_RECOVERY_EVIDENCE_INCOMPLETE",
|
|
1611
|
+
"Rolled-back checkpoint lacks a Git baseline",
|
|
1612
|
+
);
|
|
1613
|
+
}
|
|
1614
|
+
const branchHead = gitOid(
|
|
1615
|
+
opened.repoRoot,
|
|
1616
|
+
execution.worktree.branch,
|
|
1617
|
+
deps,
|
|
1618
|
+
`task branch "${execution.worktree.branch}"`,
|
|
1619
|
+
);
|
|
1620
|
+
const worktreeHead = gitOid(
|
|
1621
|
+
execution.worktree.path,
|
|
1622
|
+
"HEAD",
|
|
1623
|
+
deps,
|
|
1624
|
+
`task worktree "${execution.worktree.path}"`,
|
|
1625
|
+
);
|
|
1626
|
+
if (branchHead === baseline && worktreeHead === baseline) {
|
|
1627
|
+
assertCleanGitWorktree(execution.worktree.path, deps, {
|
|
1628
|
+
message: "Rolled-back task Git baseline contains residual writes",
|
|
1629
|
+
});
|
|
1630
|
+
return { repaired: false, baselineCommitOid: baseline };
|
|
1631
|
+
}
|
|
1632
|
+
const attemptedCommit = execution.verifiedCommitOid?.toLowerCase() || null;
|
|
1633
|
+
if (
|
|
1634
|
+
!attemptedCommit ||
|
|
1635
|
+
branchHead !== attemptedCommit ||
|
|
1636
|
+
worktreeHead !== attemptedCommit
|
|
1637
|
+
) {
|
|
1638
|
+
fail(
|
|
1639
|
+
"TEAM_QUEUE_RECOVERY_GIT_DRIFT",
|
|
1640
|
+
"Task branch/worktree moved beyond the exact abandoned attempt",
|
|
1641
|
+
{ baseline, attemptedCommit, branchHead, worktreeHead },
|
|
1642
|
+
);
|
|
1643
|
+
}
|
|
1644
|
+
try {
|
|
1645
|
+
deps.git(execution.worktree.path, ["diff", "--quiet", baseline, "--"]);
|
|
1646
|
+
const untracked = deps.git(execution.worktree.path, [
|
|
1647
|
+
"ls-files",
|
|
1648
|
+
"--others",
|
|
1649
|
+
"--exclude-standard",
|
|
1650
|
+
]);
|
|
1651
|
+
if (untracked) {
|
|
1652
|
+
fail(
|
|
1653
|
+
"TEAM_QUEUE_RECOVERY_GIT_DRIFT",
|
|
1654
|
+
"Rolled-back worktree still contains untracked files",
|
|
1655
|
+
{ untracked: untracked.split(/\r?\n/u).filter(Boolean) },
|
|
1656
|
+
);
|
|
1657
|
+
}
|
|
1658
|
+
deps.git(execution.worktree.path, ["reset", "--hard", baseline]);
|
|
1659
|
+
} catch (error) {
|
|
1660
|
+
if (error instanceof TeamDistributedCliError) throw error;
|
|
1661
|
+
fail(
|
|
1662
|
+
"TEAM_QUEUE_RECOVERY_GIT_DRIFT",
|
|
1663
|
+
`Could not prove and restore the exact task Git baseline: ${
|
|
1664
|
+
error instanceof Error ? error.message : String(error)
|
|
1665
|
+
}`,
|
|
1666
|
+
);
|
|
1667
|
+
}
|
|
1668
|
+
const repairedBranch = gitOid(
|
|
1669
|
+
opened.repoRoot,
|
|
1670
|
+
execution.worktree.branch,
|
|
1671
|
+
deps,
|
|
1672
|
+
`repaired task branch "${execution.worktree.branch}"`,
|
|
1673
|
+
);
|
|
1674
|
+
const repairedHead = gitOid(
|
|
1675
|
+
execution.worktree.path,
|
|
1676
|
+
"HEAD",
|
|
1677
|
+
deps,
|
|
1678
|
+
`repaired task worktree "${execution.worktree.path}"`,
|
|
1679
|
+
);
|
|
1680
|
+
if (repairedBranch !== baseline || repairedHead !== baseline) {
|
|
1681
|
+
fail(
|
|
1682
|
+
"TEAM_QUEUE_RECOVERY_GIT_DRIFT",
|
|
1683
|
+
"Task Git baseline repair did not settle the exact branch/worktree",
|
|
1684
|
+
{ baseline, repairedBranch, repairedHead },
|
|
1685
|
+
);
|
|
1686
|
+
}
|
|
1687
|
+
assertCleanGitWorktree(execution.worktree.path, deps, {
|
|
1688
|
+
message: "Repaired task Git baseline contains residual writes",
|
|
1689
|
+
});
|
|
1690
|
+
return {
|
|
1691
|
+
repaired: true,
|
|
1692
|
+
baselineCommitOid: baseline,
|
|
1693
|
+
abandonedCommitOid: attemptedCommit,
|
|
1694
|
+
};
|
|
1695
|
+
}
|
|
1696
|
+
|
|
1697
|
+
function requireExactCheckpointRecoverySweep(
|
|
1698
|
+
recoverySweep,
|
|
1699
|
+
transactionId,
|
|
1700
|
+
terminalState,
|
|
1701
|
+
) {
|
|
1702
|
+
if (!Array.isArray(recoverySweep)) {
|
|
1703
|
+
fail(
|
|
1704
|
+
"TEAM_QUEUE_CHECKPOINT_RECOVERY_REQUIRED",
|
|
1705
|
+
"Checkpoint recovery did not return a durable transaction sweep",
|
|
1706
|
+
);
|
|
1707
|
+
}
|
|
1708
|
+
const matches = recoverySweep.filter((entry) => entry?.id === transactionId);
|
|
1709
|
+
const entry = matches.length === 1 ? matches[0] : null;
|
|
1710
|
+
if (
|
|
1711
|
+
!entry ||
|
|
1712
|
+
entry.manualRecoveryRequired === true ||
|
|
1713
|
+
entry.status !== terminalState
|
|
1714
|
+
) {
|
|
1715
|
+
fail(
|
|
1716
|
+
"TEAM_QUEUE_CHECKPOINT_RECOVERY_REQUIRED",
|
|
1717
|
+
`Checkpoint "${transactionId}" recovery is not durably settled`,
|
|
1718
|
+
{
|
|
1719
|
+
transactionId,
|
|
1720
|
+
expectedStatus: terminalState,
|
|
1721
|
+
recoveryEntries: matches,
|
|
1722
|
+
},
|
|
1723
|
+
);
|
|
1724
|
+
}
|
|
1725
|
+
return entry;
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
export function recoverDistributedQueueCheckpoint(
|
|
1729
|
+
options,
|
|
1730
|
+
dependencyOverrides = {},
|
|
1731
|
+
) {
|
|
1732
|
+
const deps = defaultDependencies(dependencyOverrides);
|
|
1733
|
+
const opened = openPinnedQueue(options, deps);
|
|
1734
|
+
const key = requireText(options.task, "--task");
|
|
1735
|
+
const recoveryId = requireText(options.recoveryId, "--recovery-id");
|
|
1736
|
+
const actor = requireText(options.actor || "recovery", "--actor");
|
|
1737
|
+
const reason =
|
|
1738
|
+
options.reason == null
|
|
1739
|
+
? "recover abandoned managed workspace checkpoint"
|
|
1740
|
+
: requireText(options.reason, "--reason");
|
|
1741
|
+
const evidenceDigest = requireEvidenceDigest(options.evidenceDigest);
|
|
1742
|
+
const task = opened.queue.getTask(key);
|
|
1743
|
+
if (!task) {
|
|
1744
|
+
fail(
|
|
1745
|
+
"TEAM_QUEUE_TASK_NOT_FOUND",
|
|
1746
|
+
`Distributed task "${key}" was not found`,
|
|
1747
|
+
);
|
|
1748
|
+
}
|
|
1749
|
+
if (
|
|
1750
|
+
task.status !== "cancelled" ||
|
|
1751
|
+
task.metadata?.adjudication?.required !== true ||
|
|
1752
|
+
task.metadata.adjudication.evidenceDigest !== evidenceDigest
|
|
1753
|
+
) {
|
|
1754
|
+
fail(
|
|
1755
|
+
"TEAM_QUEUE_RECOVERY_EVIDENCE_MISMATCH",
|
|
1756
|
+
`Task "${key}" is not awaiting the pinned adjudication evidence`,
|
|
1757
|
+
);
|
|
1758
|
+
}
|
|
1759
|
+
const current = task.metadata?.workspaceExecution;
|
|
1760
|
+
const checkpoint = current?.checkpoint;
|
|
1761
|
+
if (
|
|
1762
|
+
!current ||
|
|
1763
|
+
!checkpoint?.transactionId ||
|
|
1764
|
+
opened.authority.checkpoint.enabled !== true ||
|
|
1765
|
+
!samePath(checkpoint.stateDir, opened.authority.checkpoint.stateDir) ||
|
|
1766
|
+
!samePath(checkpoint.workspaceRoot, current.worktree?.path || "")
|
|
1767
|
+
) {
|
|
1768
|
+
fail(
|
|
1769
|
+
"TEAM_QUEUE_RECOVERY_EVIDENCE_INCOMPLETE",
|
|
1770
|
+
`Task "${key}" has no pinned managed checkpoint execution`,
|
|
1771
|
+
);
|
|
1772
|
+
}
|
|
1773
|
+
const broker = new deps.CheckpointBroker({
|
|
1774
|
+
stateDir: opened.authority.checkpoint.stateDir,
|
|
1775
|
+
coverageTarget: "partial",
|
|
1776
|
+
writerIsolation: "unknown",
|
|
1777
|
+
externalSideEffects: true,
|
|
1778
|
+
});
|
|
1779
|
+
let recoverySweep;
|
|
1780
|
+
let snapshot;
|
|
1781
|
+
try {
|
|
1782
|
+
recoverySweep = broker.recoverPending({
|
|
1783
|
+
stateDir: checkpoint.stateDir,
|
|
1784
|
+
workspaceRoot: checkpoint.workspaceRoot,
|
|
1785
|
+
reason,
|
|
1786
|
+
});
|
|
1787
|
+
snapshot = broker.inspectCheckpoint(checkpoint.transactionId, {
|
|
1788
|
+
stateDir: checkpoint.stateDir,
|
|
1789
|
+
});
|
|
1790
|
+
} catch (error) {
|
|
1791
|
+
fail(
|
|
1792
|
+
"TEAM_QUEUE_CHECKPOINT_RECOVERY_FAILED",
|
|
1793
|
+
`Could not recover checkpoint "${checkpoint.transactionId}": ${
|
|
1794
|
+
error instanceof Error ? error.message : String(error)
|
|
1795
|
+
}`,
|
|
1796
|
+
);
|
|
1797
|
+
}
|
|
1798
|
+
if (
|
|
1799
|
+
snapshot.id !== checkpoint.transactionId ||
|
|
1800
|
+
snapshot.checkpointId !== checkpoint.checkpointId ||
|
|
1801
|
+
snapshot.runId !== checkpoint.runId ||
|
|
1802
|
+
snapshot.taskKey !== key ||
|
|
1803
|
+
!samePath(snapshot.workspaceRoot, checkpoint.workspaceRoot) ||
|
|
1804
|
+
!samePath(snapshot.stateDir, checkpoint.stateDir)
|
|
1805
|
+
) {
|
|
1806
|
+
fail(
|
|
1807
|
+
"TEAM_QUEUE_RECOVERY_EVIDENCE_MISMATCH",
|
|
1808
|
+
"Recovered workspace transaction changed its task/authority binding",
|
|
1809
|
+
);
|
|
1810
|
+
}
|
|
1811
|
+
const recoveredCheckpoint = summarizeWorktreeCheckpoint(snapshot, key);
|
|
1812
|
+
const rolledBack = ["rolled_back", "aborted"].includes(
|
|
1813
|
+
recoveredCheckpoint.state,
|
|
1814
|
+
);
|
|
1815
|
+
const committed = recoveredCheckpoint.state === "committed";
|
|
1816
|
+
if (!rolledBack && !committed) {
|
|
1817
|
+
fail(
|
|
1818
|
+
"TEAM_QUEUE_CHECKPOINT_RECOVERY_REQUIRED",
|
|
1819
|
+
`Checkpoint "${checkpoint.transactionId}" remains ${recoveredCheckpoint.state}`,
|
|
1820
|
+
{ recoverySweep, checkpoint: recoveredCheckpoint },
|
|
1821
|
+
);
|
|
1822
|
+
}
|
|
1823
|
+
requireExactCheckpointRecoverySweep(
|
|
1824
|
+
recoverySweep,
|
|
1825
|
+
checkpoint.transactionId,
|
|
1826
|
+
recoveredCheckpoint.state,
|
|
1827
|
+
);
|
|
1828
|
+
for (const [field, value] of Object.entries({
|
|
1829
|
+
checkpointDigest: recoveredCheckpoint.checkpointDigest,
|
|
1830
|
+
writeManifestDigest: recoveredCheckpoint.writeManifestDigest,
|
|
1831
|
+
checkpointEvidenceDigest: recoveredCheckpoint.evidenceDigest,
|
|
1832
|
+
})) {
|
|
1833
|
+
if (!EVIDENCE_DIGEST_PATTERN.test(value || "")) {
|
|
1834
|
+
fail(
|
|
1835
|
+
"TEAM_QUEUE_RECOVERY_EVIDENCE_INCOMPLETE",
|
|
1836
|
+
`Recovered checkpoint lacks ${field}`,
|
|
1837
|
+
);
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1841
|
+
const execution = {
|
|
1842
|
+
...current,
|
|
1843
|
+
phase: committed
|
|
1844
|
+
? "committed"
|
|
1845
|
+
: current.phase === "prepared"
|
|
1846
|
+
? "preparation-rolled-back"
|
|
1847
|
+
: "rolled-back",
|
|
1848
|
+
worktree: {
|
|
1849
|
+
...current.worktree,
|
|
1850
|
+
workspaceCheckpoint: recoveredCheckpoint,
|
|
1851
|
+
},
|
|
1852
|
+
checkpoint: recoveredCheckpoint,
|
|
1853
|
+
verifiedCommitOid: committed ? current.verifiedCommitOid : null,
|
|
1854
|
+
};
|
|
1855
|
+
let gitRecovery = null;
|
|
1856
|
+
if (committed) {
|
|
1857
|
+
verifyCommittedRecoveryGit(opened, execution, deps);
|
|
1858
|
+
} else if (options.repairGitBaseline === true) {
|
|
1859
|
+
gitRecovery = repairRolledBackGitBaseline(opened, current, deps);
|
|
1860
|
+
}
|
|
1861
|
+
const result = requireQueueMutation(
|
|
1862
|
+
opened.queue.reconcileWorkspaceExecution(key, {
|
|
1863
|
+
recoveryId,
|
|
1864
|
+
actor,
|
|
1865
|
+
reason,
|
|
1866
|
+
evidenceDigest,
|
|
1867
|
+
checkpointDigest: recoveredCheckpoint.checkpointDigest,
|
|
1868
|
+
writeManifestDigest: recoveredCheckpoint.writeManifestDigest,
|
|
1869
|
+
checkpointEvidenceDigest: recoveredCheckpoint.evidenceDigest,
|
|
1870
|
+
execution,
|
|
1871
|
+
}),
|
|
1872
|
+
"TEAM_QUEUE_CHECKPOINT_RECONCILIATION_REJECTED",
|
|
1873
|
+
`reconcile checkpoint for task "${key}"`,
|
|
1874
|
+
);
|
|
1875
|
+
return {
|
|
1876
|
+
queueId: opened.snapshot.queueId,
|
|
1877
|
+
task: key,
|
|
1878
|
+
recoverySweep,
|
|
1879
|
+
gitRecovery,
|
|
1880
|
+
...result,
|
|
1881
|
+
};
|
|
1882
|
+
}
|
|
1883
|
+
|
|
1884
|
+
function assertRetryGitReady(opened, task, deps) {
|
|
1885
|
+
const execution = task.metadata?.workspaceExecution;
|
|
1886
|
+
if (
|
|
1887
|
+
!execution ||
|
|
1888
|
+
!["preparation-rolled-back", "rolled-back"].includes(execution.phase)
|
|
1889
|
+
) {
|
|
1890
|
+
return;
|
|
1891
|
+
}
|
|
1892
|
+
const baseline = execution.worktree?.baselineCommitOid?.toLowerCase();
|
|
1893
|
+
const branchHead = gitOid(
|
|
1894
|
+
opened.repoRoot,
|
|
1895
|
+
execution.worktree.branch,
|
|
1896
|
+
deps,
|
|
1897
|
+
`task branch "${execution.worktree.branch}"`,
|
|
1898
|
+
);
|
|
1899
|
+
const worktreeHead = gitOid(
|
|
1900
|
+
execution.worktree.path,
|
|
1901
|
+
"HEAD",
|
|
1902
|
+
deps,
|
|
1903
|
+
`task worktree "${execution.worktree.path}"`,
|
|
1904
|
+
);
|
|
1905
|
+
if (!baseline || branchHead !== baseline || worktreeHead !== baseline) {
|
|
1906
|
+
fail(
|
|
1907
|
+
"TEAM_QUEUE_RETRY_GIT_RECOVERY_REQUIRED",
|
|
1908
|
+
`Task "${task.key}" cannot retry until its exact Git baseline is restored`,
|
|
1909
|
+
{ baseline, branchHead, worktreeHead },
|
|
1910
|
+
);
|
|
1911
|
+
}
|
|
1912
|
+
assertCleanGitWorktree(execution.worktree.path, deps, {
|
|
1913
|
+
code: "TEAM_QUEUE_RETRY_GIT_RECOVERY_REQUIRED",
|
|
1914
|
+
message: `Task "${task.key}" cannot retry with residual worktree writes`,
|
|
1915
|
+
});
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
export function adjudicateDistributedQueue(options, dependencyOverrides = {}) {
|
|
1919
|
+
const deps = defaultDependencies(dependencyOverrides);
|
|
1920
|
+
const opened = openPinnedQueue(options, deps);
|
|
1921
|
+
const key = requireText(options.task, "--task");
|
|
1922
|
+
const decision = requireText(options.decision, "--decision");
|
|
1923
|
+
if (!ADJUDICATION_DECISIONS.has(decision)) {
|
|
1924
|
+
fail(
|
|
1925
|
+
"TEAM_QUEUE_INVALID_OPTION",
|
|
1926
|
+
"--decision must be retry, accept, or cancel",
|
|
1927
|
+
);
|
|
1928
|
+
}
|
|
1929
|
+
const decisionId = requireText(options.decisionId, "--decision-id");
|
|
1930
|
+
const actor = requireText(options.actor || "human", "--actor");
|
|
1931
|
+
const reason =
|
|
1932
|
+
options.reason == null ? null : requireText(options.reason, "--reason");
|
|
1933
|
+
const evidenceDigest = requireEvidenceDigest(options.evidenceDigest);
|
|
1934
|
+
const task = opened.queue.getTask(key);
|
|
1935
|
+
if (!task) {
|
|
1936
|
+
fail(
|
|
1937
|
+
"TEAM_QUEUE_TASK_NOT_FOUND",
|
|
1938
|
+
`Distributed task "${key}" was not found`,
|
|
1939
|
+
);
|
|
1940
|
+
}
|
|
1941
|
+
if (decision === "retry") assertRetryGitReady(opened, task, deps);
|
|
1942
|
+
|
|
1943
|
+
let acceptance = null;
|
|
1944
|
+
if (decision === "accept") {
|
|
1945
|
+
const prior = task.metadata?.adjudication?.decision || null;
|
|
1946
|
+
if (prior?.id === decisionId && prior.action === "accept") {
|
|
1947
|
+
acceptance = {
|
|
1948
|
+
result: task.metadata?.result,
|
|
1949
|
+
proof: task.metadata?.result?.workspaceExecutionEvidence || null,
|
|
1950
|
+
replay: true,
|
|
1951
|
+
};
|
|
1952
|
+
} else {
|
|
1953
|
+
verifyCommittedRecoveryGit(
|
|
1954
|
+
opened,
|
|
1955
|
+
task.metadata?.workspaceExecution,
|
|
1956
|
+
deps,
|
|
1957
|
+
);
|
|
1958
|
+
acceptance = requireQueueMutation(
|
|
1959
|
+
opened.queue.adjudicationAcceptance(key, { evidenceDigest }),
|
|
1960
|
+
"TEAM_QUEUE_ADJUDICATION_EVIDENCE_REJECTED",
|
|
1961
|
+
`derive acceptance evidence for task "${key}"`,
|
|
1962
|
+
);
|
|
1963
|
+
}
|
|
1964
|
+
}
|
|
1965
|
+
const result = requireQueueMutation(
|
|
1966
|
+
opened.queue.resolveAdjudication(key, {
|
|
1967
|
+
decision,
|
|
1968
|
+
decisionId,
|
|
1969
|
+
actor,
|
|
1970
|
+
reason,
|
|
1971
|
+
evidenceDigest,
|
|
1972
|
+
...(acceptance ? { result: acceptance.result } : {}),
|
|
1973
|
+
}),
|
|
1974
|
+
"TEAM_QUEUE_ADJUDICATION_REJECTED",
|
|
1975
|
+
`adjudicate task "${key}"`,
|
|
1976
|
+
);
|
|
1977
|
+
return {
|
|
1978
|
+
queueId: opened.snapshot.queueId,
|
|
1979
|
+
task: key,
|
|
1980
|
+
...(acceptance?.proof ? { acceptance: acceptance.proof } : {}),
|
|
1981
|
+
...result,
|
|
1982
|
+
};
|
|
1983
|
+
}
|
|
1984
|
+
|
|
1985
|
+
function attemptLease(task, key) {
|
|
1986
|
+
const lease = task?.lease || task?.metadata?.lease || null;
|
|
1987
|
+
if (
|
|
1988
|
+
!lease?.holder ||
|
|
1989
|
+
!lease?.leaseId ||
|
|
1990
|
+
!Number.isSafeInteger(lease?.fencingToken)
|
|
1991
|
+
) {
|
|
1992
|
+
fail(
|
|
1993
|
+
"TEAM_QUEUE_AGENT_LEASE_MISSING",
|
|
1994
|
+
`Task "${key}" has no durable Agent lease/fence binding`,
|
|
1995
|
+
);
|
|
1996
|
+
}
|
|
1997
|
+
return lease;
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
function settlementAccessor(attempts, leaseId) {
|
|
2001
|
+
const attempt = attempts.get(leaseId);
|
|
2002
|
+
if (!attempt) return null;
|
|
2003
|
+
return {
|
|
2004
|
+
usage: attempt.metering.usage,
|
|
2005
|
+
usageRecords: attempt.metering.usageRecords,
|
|
2006
|
+
costUsd: attempt.metering.costUsd,
|
|
2007
|
+
settled: () => attempts.delete(leaseId),
|
|
2008
|
+
};
|
|
2009
|
+
}
|
|
2010
|
+
|
|
2011
|
+
function attachAgentMetering(error, metering) {
|
|
2012
|
+
if (!error || !metering) return error;
|
|
2013
|
+
if (metering.usage) error.usage = metering.usage;
|
|
2014
|
+
if (metering.usageRecords) error.usageRecords = metering.usageRecords;
|
|
2015
|
+
if (metering.provider) error.provider = metering.provider;
|
|
2016
|
+
if (metering.model) error.model = metering.model;
|
|
2017
|
+
if (Number.isFinite(metering.costUsd)) error.costUsd = metering.costUsd;
|
|
2018
|
+
return error;
|
|
2019
|
+
}
|
|
2020
|
+
|
|
2021
|
+
function agentExecutionEvidence({
|
|
2022
|
+
opened,
|
|
2023
|
+
workerId,
|
|
2024
|
+
key,
|
|
2025
|
+
lease,
|
|
2026
|
+
sourcePrompt,
|
|
2027
|
+
executionPrompt,
|
|
2028
|
+
contract,
|
|
2029
|
+
result,
|
|
2030
|
+
metering,
|
|
2031
|
+
}) {
|
|
2032
|
+
const checkpoint = result.workspaceCheckpoint;
|
|
2033
|
+
if (
|
|
2034
|
+
checkpoint?.state !== "committed" ||
|
|
2035
|
+
!checkpoint.transactionId ||
|
|
2036
|
+
!checkpoint.evidenceDigest ||
|
|
2037
|
+
!result.commitOid
|
|
2038
|
+
) {
|
|
2039
|
+
fail(
|
|
2040
|
+
"TEAM_QUEUE_AGENT_EVIDENCE_INCOMPLETE",
|
|
2041
|
+
`Task "${key}" completed without committed checkpoint/Git evidence`,
|
|
2042
|
+
);
|
|
2043
|
+
}
|
|
2044
|
+
const binding = {
|
|
2045
|
+
domain: "cc-team-distributed-agent-execution-v1",
|
|
2046
|
+
queueId: opened.snapshot.queueId,
|
|
2047
|
+
authorityDigest: opened.snapshot.authorityDigest,
|
|
2048
|
+
taskKey: key,
|
|
2049
|
+
workerId,
|
|
2050
|
+
holder: lease.holder,
|
|
2051
|
+
leaseId: lease.leaseId,
|
|
2052
|
+
fencingToken: lease.fencingToken,
|
|
2053
|
+
sourcePromptDigest: canonicalDigest({
|
|
2054
|
+
domain: "cc-team-distributed-agent-prompt-v1",
|
|
2055
|
+
prompt: sourcePrompt,
|
|
2056
|
+
}),
|
|
2057
|
+
executionPromptDigest: canonicalDigest({
|
|
2058
|
+
domain: "cc-team-distributed-agent-execution-prompt-v1",
|
|
2059
|
+
prompt: executionPrompt,
|
|
2060
|
+
}),
|
|
2061
|
+
contractDigest: canonicalDigest({
|
|
2062
|
+
domain: "cc-team-distributed-agent-contract-v1",
|
|
2063
|
+
contract,
|
|
2064
|
+
}),
|
|
2065
|
+
commitOid: result.commitOid,
|
|
2066
|
+
checkpointTransactionId: checkpoint.transactionId,
|
|
2067
|
+
checkpointEvidenceDigest: checkpoint.evidenceDigest,
|
|
2068
|
+
checkpointWriteManifestDigest: checkpoint.writeManifestDigest || null,
|
|
2069
|
+
usage: metering.usage,
|
|
2070
|
+
usageRecords: metering.usageRecords,
|
|
2071
|
+
provider: metering.provider,
|
|
2072
|
+
model: metering.model,
|
|
2073
|
+
costUsd: metering.costUsd,
|
|
2074
|
+
};
|
|
2075
|
+
return {
|
|
2076
|
+
version: 1,
|
|
2077
|
+
mode: AGENT_WORKTREE_MODE,
|
|
2078
|
+
...binding,
|
|
2079
|
+
digest: canonicalDigest(binding),
|
|
2080
|
+
};
|
|
2081
|
+
}
|
|
2082
|
+
|
|
2083
|
+
function makeDistributedAgentRunTask({
|
|
2084
|
+
opened,
|
|
2085
|
+
coordinator,
|
|
2086
|
+
deps,
|
|
2087
|
+
workerId,
|
|
2088
|
+
attempts,
|
|
2089
|
+
}) {
|
|
2090
|
+
if (typeof deps.agentExecutor !== "function") {
|
|
2091
|
+
fail(
|
|
2092
|
+
"TEAM_QUEUE_AGENT_EXECUTOR_UNAVAILABLE",
|
|
2093
|
+
"agent-worktree worker has no injected Agent executor",
|
|
2094
|
+
);
|
|
2095
|
+
}
|
|
2096
|
+
if (typeof deps.buildAgentPrompt !== "function") {
|
|
2097
|
+
fail(
|
|
2098
|
+
"TEAM_QUEUE_AGENT_EXECUTOR_UNAVAILABLE",
|
|
2099
|
+
"agent-worktree worker has no injected prompt builder",
|
|
2100
|
+
);
|
|
2101
|
+
}
|
|
2102
|
+
const runInWorktree = async ({
|
|
2103
|
+
key,
|
|
2104
|
+
task,
|
|
2105
|
+
cwd,
|
|
2106
|
+
inbox = [],
|
|
2107
|
+
budgetReservation = null,
|
|
2108
|
+
signal = null,
|
|
2109
|
+
managedCheckpoint = false,
|
|
2110
|
+
}) => {
|
|
2111
|
+
if (managedCheckpoint !== true) {
|
|
2112
|
+
fail(
|
|
2113
|
+
"TEAM_QUEUE_CHECKPOINT_REQUIRED",
|
|
2114
|
+
`Task "${key}" Agent execution lacks its pinned managed checkpoint`,
|
|
2115
|
+
);
|
|
2116
|
+
}
|
|
2117
|
+
const lease = attemptLease(task, key);
|
|
2118
|
+
const prompt = stableAgentText(
|
|
2119
|
+
task.metadata?.prompt,
|
|
2120
|
+
`task "${key}".prompt`,
|
|
2121
|
+
{
|
|
2122
|
+
maxBytes: MAX_AGENT_PROMPT_BYTES,
|
|
2123
|
+
trim: false,
|
|
2124
|
+
},
|
|
2125
|
+
);
|
|
2126
|
+
const contract = normalizeAgentAuthority(task.metadata?.agentContract, {
|
|
2127
|
+
source: `task "${key}" contract`,
|
|
2128
|
+
});
|
|
2129
|
+
let effectiveContract = applyAgentBudgetReservation(
|
|
2130
|
+
contract,
|
|
2131
|
+
budgetReservation,
|
|
2132
|
+
);
|
|
2133
|
+
const queueBudget = opened.queue.budgetStatus();
|
|
2134
|
+
if (Number(queueBudget.maxWallMs) > 0) {
|
|
2135
|
+
const remainingWallMs = Math.floor(
|
|
2136
|
+
Number(queueBudget.maxWallMs) - Number(queueBudget.elapsedMs || 0),
|
|
2137
|
+
);
|
|
2138
|
+
if (remainingWallMs <= 0) {
|
|
2139
|
+
fail(
|
|
2140
|
+
"TEAM_QUEUE_AGENT_WALL_LIMIT",
|
|
2141
|
+
`Task "${key}" cannot start after the distributed wall-clock budget`,
|
|
2142
|
+
);
|
|
2143
|
+
}
|
|
2144
|
+
effectiveContract = {
|
|
2145
|
+
...effectiveContract,
|
|
2146
|
+
maxWallMs:
|
|
2147
|
+
Number(effectiveContract.maxWallMs) > 0
|
|
2148
|
+
? Math.min(effectiveContract.maxWallMs, remainingWallMs)
|
|
2149
|
+
: remainingWallMs,
|
|
2150
|
+
};
|
|
2151
|
+
}
|
|
2152
|
+
let rawResult;
|
|
2153
|
+
let executionPrompt = null;
|
|
2154
|
+
try {
|
|
2155
|
+
executionPrompt = deps.buildAgentPrompt(prompt, { inbox });
|
|
2156
|
+
if (
|
|
2157
|
+
typeof executionPrompt !== "string" ||
|
|
2158
|
+
executionPrompt.trim().length === 0 ||
|
|
2159
|
+
executionPrompt.includes("\0") ||
|
|
2160
|
+
Buffer.byteLength(executionPrompt, "utf8") >
|
|
2161
|
+
MAX_AGENT_EXECUTION_PROMPT_BYTES
|
|
2162
|
+
) {
|
|
2163
|
+
fail(
|
|
2164
|
+
"TEAM_QUEUE_AGENT_PROMPT_INVALID",
|
|
2165
|
+
`Task "${key}" prompt builder returned an invalid prompt`,
|
|
2166
|
+
);
|
|
2167
|
+
}
|
|
2168
|
+
rawResult = await deps.agentExecutor(executionPrompt, cwd, {
|
|
2169
|
+
...effectiveContract,
|
|
2170
|
+
checkpointRequired: false,
|
|
2171
|
+
worktreeRequired: true,
|
|
2172
|
+
managedCheckpoint: true,
|
|
2173
|
+
signal,
|
|
2174
|
+
});
|
|
2175
|
+
} catch (caught) {
|
|
2176
|
+
const error =
|
|
2177
|
+
caught instanceof Error ? caught : new Error(String(caught || ""));
|
|
2178
|
+
try {
|
|
2179
|
+
const metering = agentMetering(
|
|
2180
|
+
{
|
|
2181
|
+
usage: error.usage ?? null,
|
|
2182
|
+
usageRecords: error.usageRecords ?? null,
|
|
2183
|
+
provider: error.provider ?? null,
|
|
2184
|
+
model: error.model ?? null,
|
|
2185
|
+
},
|
|
2186
|
+
{},
|
|
2187
|
+
);
|
|
2188
|
+
if (metering.usage) {
|
|
2189
|
+
attempts.set(lease.leaseId, {
|
|
2190
|
+
key,
|
|
2191
|
+
lease,
|
|
2192
|
+
sourcePrompt: prompt,
|
|
2193
|
+
executionPrompt,
|
|
2194
|
+
contract: effectiveContract,
|
|
2195
|
+
metering,
|
|
2196
|
+
});
|
|
2197
|
+
attachAgentMetering(error, metering);
|
|
2198
|
+
}
|
|
2199
|
+
} catch {
|
|
2200
|
+
// The executor failure remains primary. The queue conservatively
|
|
2201
|
+
// consumes any reserved caps when trustworthy metering is unavailable.
|
|
2202
|
+
}
|
|
2203
|
+
throw error;
|
|
2204
|
+
}
|
|
2205
|
+
|
|
2206
|
+
let metering;
|
|
2207
|
+
try {
|
|
2208
|
+
metering = agentMetering(rawResult, {
|
|
2209
|
+
requireUsage:
|
|
2210
|
+
Number(effectiveContract.maxTokens) > 0 ||
|
|
2211
|
+
Number(effectiveContract.maxBudgetUsd) > 0,
|
|
2212
|
+
requirePricing: Number(effectiveContract.maxBudgetUsd) > 0,
|
|
2213
|
+
maxTokens: effectiveContract.maxTokens,
|
|
2214
|
+
maxBudgetUsd: effectiveContract.maxBudgetUsd,
|
|
2215
|
+
});
|
|
2216
|
+
} catch (caught) {
|
|
2217
|
+
const error =
|
|
2218
|
+
caught instanceof Error ? caught : new Error(String(caught || ""));
|
|
2219
|
+
error.retryable = false;
|
|
2220
|
+
try {
|
|
2221
|
+
const partial = agentMetering(rawResult, {});
|
|
2222
|
+
if (partial.usage) {
|
|
2223
|
+
attempts.set(lease.leaseId, {
|
|
2224
|
+
key,
|
|
2225
|
+
lease,
|
|
2226
|
+
sourcePrompt: prompt,
|
|
2227
|
+
executionPrompt,
|
|
2228
|
+
contract: effectiveContract,
|
|
2229
|
+
metering: partial,
|
|
2230
|
+
});
|
|
2231
|
+
attachAgentMetering(error, partial);
|
|
2232
|
+
}
|
|
2233
|
+
} catch {
|
|
2234
|
+
// The strict validation error remains authoritative.
|
|
2235
|
+
}
|
|
2236
|
+
throw error;
|
|
2237
|
+
}
|
|
2238
|
+
attempts.set(lease.leaseId, {
|
|
2239
|
+
key,
|
|
2240
|
+
lease,
|
|
2241
|
+
sourcePrompt: prompt,
|
|
2242
|
+
executionPrompt,
|
|
2243
|
+
contract: effectiveContract,
|
|
2244
|
+
metering,
|
|
2245
|
+
});
|
|
2246
|
+
return {
|
|
2247
|
+
usage: metering.usage,
|
|
2248
|
+
provider: metering.provider,
|
|
2249
|
+
model: metering.model,
|
|
2250
|
+
...(metering.usageRecords ? { usageRecords: metering.usageRecords } : {}),
|
|
2251
|
+
};
|
|
2252
|
+
};
|
|
2253
|
+
const worktreeRunTask = coordinator.makeRunTask({ runInWorktree });
|
|
2254
|
+
return async (context) => {
|
|
2255
|
+
const lease = attemptLease(context.task, context.key);
|
|
2256
|
+
const result = await worktreeRunTask(context);
|
|
2257
|
+
const attempt = attempts.get(lease.leaseId);
|
|
2258
|
+
if (!attempt) {
|
|
2259
|
+
fail(
|
|
2260
|
+
"TEAM_QUEUE_AGENT_EVIDENCE_INCOMPLETE",
|
|
2261
|
+
`Task "${context.key}" has no settled Agent metering evidence`,
|
|
2262
|
+
);
|
|
2263
|
+
}
|
|
2264
|
+
return {
|
|
2265
|
+
...result,
|
|
2266
|
+
costUsd: attempt.metering.costUsd,
|
|
2267
|
+
agentExecution: agentExecutionEvidence({
|
|
2268
|
+
opened,
|
|
2269
|
+
workerId,
|
|
2270
|
+
key: context.key,
|
|
2271
|
+
lease,
|
|
2272
|
+
sourcePrompt: attempt.sourcePrompt,
|
|
2273
|
+
executionPrompt: attempt.executionPrompt,
|
|
2274
|
+
contract: attempt.contract,
|
|
2275
|
+
result,
|
|
2276
|
+
metering: attempt.metering,
|
|
2277
|
+
}),
|
|
2278
|
+
};
|
|
2279
|
+
};
|
|
2280
|
+
}
|
|
2281
|
+
|
|
2282
|
+
export async function runDistributedWorker(options, dependencyOverrides = {}) {
|
|
2283
|
+
const deps = defaultDependencies(dependencyOverrides);
|
|
2284
|
+
const opened = openPinnedQueue(options, deps);
|
|
2285
|
+
const workerId = requireText(
|
|
2286
|
+
options.workerId || `worker-${process.pid}-${crypto.randomUUID()}`,
|
|
2287
|
+
"--worker-id",
|
|
2288
|
+
);
|
|
2289
|
+
const checkpointBroker = opened.authority.checkpoint.enabled
|
|
2290
|
+
? new deps.CheckpointBroker({
|
|
2291
|
+
stateDir: opened.authority.checkpoint.stateDir,
|
|
2292
|
+
coverageTarget: "partial",
|
|
2293
|
+
writerIsolation: "unknown",
|
|
2294
|
+
externalSideEffects: true,
|
|
2295
|
+
})
|
|
2296
|
+
: null;
|
|
2297
|
+
const coordinator = coordinatorForAuthority(
|
|
2298
|
+
opened.repoRoot,
|
|
2299
|
+
opened.authority,
|
|
2300
|
+
deps,
|
|
2301
|
+
{
|
|
2302
|
+
checkpointBroker,
|
|
2303
|
+
onCheckpoint: checkpointBroker
|
|
2304
|
+
? ({
|
|
2305
|
+
key,
|
|
2306
|
+
phase,
|
|
2307
|
+
checkpoint,
|
|
2308
|
+
worktree,
|
|
2309
|
+
verifiedCommitOid = null,
|
|
2310
|
+
lease,
|
|
2311
|
+
}) => {
|
|
2312
|
+
if (
|
|
2313
|
+
!lease?.holder ||
|
|
2314
|
+
!lease?.leaseId ||
|
|
2315
|
+
!Number.isSafeInteger(lease?.fencingToken)
|
|
2316
|
+
) {
|
|
2317
|
+
fail(
|
|
2318
|
+
"TEAM_QUEUE_CHECKPOINT_LEASE_MISSING",
|
|
2319
|
+
`Task "${key}" has no checkpoint lease/fence binding`,
|
|
2320
|
+
);
|
|
2321
|
+
}
|
|
2322
|
+
const persisted = opened.queue.recordWorkspaceExecution(key, {
|
|
2323
|
+
holder: lease.holder,
|
|
2324
|
+
leaseId: lease.leaseId,
|
|
2325
|
+
fencingToken: lease.fencingToken,
|
|
2326
|
+
execution: {
|
|
2327
|
+
workerId,
|
|
2328
|
+
phase,
|
|
2329
|
+
checkpoint,
|
|
2330
|
+
worktree,
|
|
2331
|
+
verifiedCommitOid,
|
|
2332
|
+
},
|
|
2333
|
+
});
|
|
2334
|
+
if (!persisted.ok) {
|
|
2335
|
+
fail(
|
|
2336
|
+
"TEAM_QUEUE_CHECKPOINT_PERSIST_FAILED",
|
|
2337
|
+
`Could not persist checkpoint phase "${phase}" for task "${key}": ${persisted.reason}${
|
|
2338
|
+
persisted.error ? ` (${persisted.error})` : ""
|
|
2339
|
+
}`,
|
|
2340
|
+
{
|
|
2341
|
+
key,
|
|
2342
|
+
phase,
|
|
2343
|
+
reason: persisted.reason,
|
|
2344
|
+
checkpoint,
|
|
2345
|
+
},
|
|
2346
|
+
);
|
|
2347
|
+
}
|
|
2348
|
+
}
|
|
2349
|
+
: null,
|
|
2350
|
+
},
|
|
2351
|
+
);
|
|
2352
|
+
const attempts = new Map();
|
|
2353
|
+
const registry = workerRegistry(opened.queue, workerId, {
|
|
2354
|
+
settlementForLease: (leaseId) => settlementAccessor(attempts, leaseId),
|
|
2355
|
+
});
|
|
2356
|
+
const runTask =
|
|
2357
|
+
opened.authority.mode === AGENT_WORKTREE_MODE
|
|
2358
|
+
? makeDistributedAgentRunTask({
|
|
2359
|
+
opened,
|
|
2360
|
+
coordinator,
|
|
2361
|
+
deps,
|
|
2362
|
+
workerId,
|
|
2363
|
+
attempts,
|
|
2364
|
+
})
|
|
2365
|
+
: coordinator.makeRunTask();
|
|
2366
|
+
const events = [];
|
|
2367
|
+
const ttlMs =
|
|
2368
|
+
optionalPositive(options.ttlMs, "--ttl-ms", { integer: true }) ?? undefined;
|
|
2369
|
+
const renewEveryMs =
|
|
2370
|
+
optionalPositive(options.renewEveryMs, "--renew-every-ms", {
|
|
2371
|
+
integer: true,
|
|
2372
|
+
}) ?? undefined;
|
|
2373
|
+
const localMaxTasks =
|
|
2374
|
+
optionalPositive(options.maxTasks, "--max-tasks", { integer: true }) ??
|
|
2375
|
+
Number.MAX_SAFE_INTEGER;
|
|
2376
|
+
const pollMs =
|
|
2377
|
+
optionalPositive(options.pollMs, "--poll-ms", { integer: true }) ?? 250;
|
|
2378
|
+
let remainingExecutions = localMaxTasks;
|
|
2379
|
+
const summaries = [];
|
|
2380
|
+
const hasWallBudget = Number(opened.snapshot?.budget?.limits?.maxWallMs) > 0;
|
|
2381
|
+
// A runner exits when it sees no LOCAL in-flight work. In distributed mode
|
|
2382
|
+
// that may simply mean another process owns the dependency that will unlock
|
|
2383
|
+
// our next task. Keep observing the durable queue while a live lease exists,
|
|
2384
|
+
// then run another one-worker claim loop when the frontier advances.
|
|
2385
|
+
for (;;) {
|
|
2386
|
+
if (summaries.length > 0 && remainingExecutions <= 0) break;
|
|
2387
|
+
let queueStats = null;
|
|
2388
|
+
if (summaries.length > 0) {
|
|
2389
|
+
queueStats = opened.queue.stats();
|
|
2390
|
+
if (
|
|
2391
|
+
queueStats.completed + queueStats.cancelled === queueStats.total &&
|
|
2392
|
+
queueStats.adjudicationRequired === 0
|
|
2393
|
+
) {
|
|
2394
|
+
break;
|
|
2395
|
+
}
|
|
2396
|
+
if (queueStats.budget?.reason || queueStats.adjudicationRequired > 0) {
|
|
2397
|
+
break;
|
|
2398
|
+
}
|
|
2399
|
+
// `claimable` and `leased` must come from the same durable queue
|
|
2400
|
+
// revision. A peer can claim between separate reads, and combining the
|
|
2401
|
+
// new claimable count with an older leased count would falsely report an
|
|
2402
|
+
// idle queue and make this worker exit while work is still in flight.
|
|
2403
|
+
if (queueStats.claimable === 0) {
|
|
2404
|
+
if (queueStats.leased > 0) {
|
|
2405
|
+
await deps.sleep(pollMs);
|
|
2406
|
+
continue;
|
|
2407
|
+
}
|
|
2408
|
+
break;
|
|
2409
|
+
}
|
|
2410
|
+
}
|
|
2411
|
+
if (hasWallBudget && queueStats == null) {
|
|
2412
|
+
queueStats = opened.queue.stats();
|
|
2413
|
+
}
|
|
2414
|
+
const runner = new deps.Runner(registry, {
|
|
2415
|
+
teammates: 1,
|
|
2416
|
+
ttlMs,
|
|
2417
|
+
renewEveryMs,
|
|
2418
|
+
maxTasks: remainingExecutions,
|
|
2419
|
+
budget:
|
|
2420
|
+
queueStats == null
|
|
2421
|
+
? null
|
|
2422
|
+
: wallBudgetForQueueStats(queueStats, opened.queue, deps),
|
|
2423
|
+
emitHook: async () => {},
|
|
2424
|
+
beforeTask: ({ task }) => {
|
|
2425
|
+
importCompletedDependencies(coordinator, opened.queue, task);
|
|
2426
|
+
importRetryWorkspace(coordinator, task);
|
|
2427
|
+
},
|
|
2428
|
+
runTask,
|
|
2429
|
+
onEvent: (event) => events.push(event),
|
|
2430
|
+
});
|
|
2431
|
+
if (typeof deps.onRunner === "function") {
|
|
2432
|
+
await deps.onRunner(runner, {
|
|
2433
|
+
workerId,
|
|
2434
|
+
round: summaries.length,
|
|
2435
|
+
queue: opened.queue,
|
|
2436
|
+
});
|
|
2437
|
+
}
|
|
2438
|
+
const round = await runner.run();
|
|
2439
|
+
summaries.push(round);
|
|
2440
|
+
remainingExecutions -= round.executions;
|
|
2441
|
+
if (round.budgetStopped && round.executions === 0) break;
|
|
2442
|
+
}
|
|
2443
|
+
const status = distributedQueueStatus(options, deps);
|
|
2444
|
+
const lastSummary = summaries.at(-1);
|
|
2445
|
+
const done = distributedTasksDone(status.tasks);
|
|
2446
|
+
const durableBudgetReason = status.stats.budget?.reason || null;
|
|
2447
|
+
const durableStopReason = done ? null : durableBudgetReason;
|
|
2448
|
+
const localBudgetReason = lastSummary?.budgetReason || null;
|
|
2449
|
+
const summary = {
|
|
2450
|
+
...lastSummary,
|
|
2451
|
+
done,
|
|
2452
|
+
success: done && status.stats.completed === status.stats.total,
|
|
2453
|
+
executions: summaries.reduce((total, round) => total + round.executions, 0),
|
|
2454
|
+
maxConcurrent: Math.max(
|
|
2455
|
+
0,
|
|
2456
|
+
...summaries.map((round) => round.maxConcurrent || 0),
|
|
2457
|
+
),
|
|
2458
|
+
rounds: summaries.length,
|
|
2459
|
+
budgetStopped:
|
|
2460
|
+
lastSummary?.budgetStopped === true || durableStopReason != null,
|
|
2461
|
+
budgetReason: localBudgetReason || durableStopReason,
|
|
2462
|
+
localBudgetReason,
|
|
2463
|
+
durableBudgetReason,
|
|
2464
|
+
stats: status.stats,
|
|
2465
|
+
};
|
|
2466
|
+
return {
|
|
2467
|
+
workerId,
|
|
2468
|
+
summary,
|
|
2469
|
+
queue: {
|
|
2470
|
+
queueId: status.queueId,
|
|
2471
|
+
revision: status.revision,
|
|
2472
|
+
stats: status.stats,
|
|
2473
|
+
pendingAdjudications: status.pendingAdjudications,
|
|
2474
|
+
interruptions: status.interruptions,
|
|
2475
|
+
},
|
|
2476
|
+
completed: status.tasks
|
|
2477
|
+
.filter((task) => task.status === "completed")
|
|
2478
|
+
.map((task) => ({ key: task.key, result: completedResult(task) })),
|
|
2479
|
+
events,
|
|
2480
|
+
};
|
|
2481
|
+
}
|
|
2482
|
+
|
|
2483
|
+
function finalizationOperationId(snapshot, options) {
|
|
2484
|
+
if (options.operationId) {
|
|
2485
|
+
return requireText(options.operationId, "--operation-id");
|
|
2486
|
+
}
|
|
2487
|
+
const hash = crypto
|
|
2488
|
+
.createHash("sha256")
|
|
2489
|
+
.update(
|
|
2490
|
+
JSON.stringify({
|
|
2491
|
+
queueId: snapshot.queueId,
|
|
2492
|
+
graphDigest: snapshot.graphDigest,
|
|
2493
|
+
authorityDigest: snapshot.authorityDigest,
|
|
2494
|
+
}),
|
|
2495
|
+
)
|
|
2496
|
+
.digest("hex");
|
|
2497
|
+
return `finalize-${hash}`;
|
|
2498
|
+
}
|
|
2499
|
+
|
|
2500
|
+
function gitHead(repoRoot, authority, deps) {
|
|
2501
|
+
let branch = null;
|
|
2502
|
+
try {
|
|
2503
|
+
branch =
|
|
2504
|
+
deps.git(repoRoot, ["symbolic-ref", "--quiet", "--short", "HEAD"]) ||
|
|
2505
|
+
null;
|
|
2506
|
+
} catch {
|
|
2507
|
+
branch = null;
|
|
2508
|
+
}
|
|
2509
|
+
if (branch !== (authority.baseTarget.branch ?? null)) {
|
|
2510
|
+
fail(
|
|
2511
|
+
"TEAM_QUEUE_FINALIZE_GIT_DRIFT",
|
|
2512
|
+
`Finalization base branch changed from ${
|
|
2513
|
+
authority.baseTarget.branch || "detached HEAD"
|
|
2514
|
+
} to ${branch || "detached HEAD"}`,
|
|
2515
|
+
);
|
|
2516
|
+
}
|
|
2517
|
+
const commitOid = deps.git(repoRoot, ["rev-parse", "--verify", "HEAD"]);
|
|
2518
|
+
if (!/^[a-f0-9]{40,64}$/i.test(commitOid)) {
|
|
2519
|
+
fail(
|
|
2520
|
+
"TEAM_QUEUE_FINALIZE_GIT_DRIFT",
|
|
2521
|
+
"Finalization could not resolve a stable base HEAD",
|
|
2522
|
+
);
|
|
2523
|
+
}
|
|
2524
|
+
return commitOid.toLowerCase();
|
|
2525
|
+
}
|
|
2526
|
+
|
|
2527
|
+
function isGitAncestor(repoRoot, ancestor, descendant, deps) {
|
|
2528
|
+
try {
|
|
2529
|
+
deps.git(repoRoot, ["merge-base", "--is-ancestor", ancestor, descendant]);
|
|
2530
|
+
return true;
|
|
2531
|
+
} catch {
|
|
2532
|
+
return false;
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2535
|
+
|
|
2536
|
+
function gitLines(repoRoot, args, deps) {
|
|
2537
|
+
const output = deps.git(repoRoot, args);
|
|
2538
|
+
return output ? output.split(/\r?\n/u).filter(Boolean) : [];
|
|
2539
|
+
}
|
|
2540
|
+
|
|
2541
|
+
/**
|
|
2542
|
+
* Prove that HEAD after a dead finalizer is exactly the sequence that
|
|
2543
|
+
* coordinator.integrate({merge:true}) could have produced from its fenced
|
|
2544
|
+
* expected base. This accepts fast-forwards and the coordinator's exact merge
|
|
2545
|
+
* commits, while rejecting unrelated first-parent commits.
|
|
2546
|
+
*/
|
|
2547
|
+
function proveMergeRecovery(repoRoot, finalization, currentHead, deps) {
|
|
2548
|
+
const expectedBase = finalization.intent.expectedBaseOid;
|
|
2549
|
+
if (currentHead === expectedBase) return { ok: true, recoveredMerges: 0 };
|
|
2550
|
+
if (!isGitAncestor(repoRoot, expectedBase, currentHead, deps)) {
|
|
2551
|
+
return { ok: false, reason: "base_head_no_longer_descends_from_intent" };
|
|
2552
|
+
}
|
|
2553
|
+
const firstParentPath = gitLines(
|
|
2554
|
+
repoRoot,
|
|
2555
|
+
[
|
|
2556
|
+
"rev-list",
|
|
2557
|
+
"--first-parent",
|
|
2558
|
+
"--reverse",
|
|
2559
|
+
`${expectedBase}..${currentHead}`,
|
|
2560
|
+
],
|
|
2561
|
+
deps,
|
|
2562
|
+
).map((oid) => oid.toLowerCase());
|
|
2563
|
+
const records = finalization.coordinator.records || [];
|
|
2564
|
+
let cursor = expectedBase;
|
|
2565
|
+
let pathIndex = 0;
|
|
2566
|
+
let recoveredMerges = 0;
|
|
2567
|
+
|
|
2568
|
+
for (const record of records) {
|
|
2569
|
+
if (record.committed !== true) continue;
|
|
2570
|
+
const branchOid = String(record.commitOid).toLowerCase();
|
|
2571
|
+
if (isGitAncestor(repoRoot, branchOid, cursor, deps)) continue;
|
|
2572
|
+
|
|
2573
|
+
const fastForwardIndex = firstParentPath.indexOf(branchOid, pathIndex);
|
|
2574
|
+
if (fastForwardIndex >= pathIndex) {
|
|
2575
|
+
const segment = firstParentPath.slice(pathIndex, fastForwardIndex + 1);
|
|
2576
|
+
if (
|
|
2577
|
+
isGitAncestor(repoRoot, cursor, branchOid, deps) &&
|
|
2578
|
+
segment.every((oid) => isGitAncestor(repoRoot, oid, branchOid, deps))
|
|
2579
|
+
) {
|
|
2580
|
+
cursor = branchOid;
|
|
2581
|
+
pathIndex = fastForwardIndex + 1;
|
|
2582
|
+
recoveredMerges += 1;
|
|
2583
|
+
continue;
|
|
2584
|
+
}
|
|
2585
|
+
}
|
|
2586
|
+
|
|
2587
|
+
const mergeOid = firstParentPath[pathIndex];
|
|
2588
|
+
if (!mergeOid) {
|
|
2589
|
+
return { ok: false, reason: `missing_merge_for_${record.key}` };
|
|
2590
|
+
}
|
|
2591
|
+
const commit = gitLines(
|
|
2592
|
+
repoRoot,
|
|
2593
|
+
["rev-list", "--parents", "-n", "1", mergeOid],
|
|
2594
|
+
deps,
|
|
2595
|
+
)[0]?.split(/\s+/u);
|
|
2596
|
+
const subject = deps.git(repoRoot, ["show", "-s", "--format=%s", mergeOid]);
|
|
2597
|
+
if (
|
|
2598
|
+
!commit ||
|
|
2599
|
+
commit[1]?.toLowerCase() !== cursor ||
|
|
2600
|
+
!commit
|
|
2601
|
+
.slice(2)
|
|
2602
|
+
.map((oid) => oid.toLowerCase())
|
|
2603
|
+
.includes(branchOid) ||
|
|
2604
|
+
subject !== `Merge team task ${record.key}`
|
|
2605
|
+
) {
|
|
2606
|
+
return { ok: false, reason: `unexpected_merge_commit_${mergeOid}` };
|
|
2607
|
+
}
|
|
2608
|
+
cursor = mergeOid;
|
|
2609
|
+
pathIndex += 1;
|
|
2610
|
+
recoveredMerges += 1;
|
|
2611
|
+
}
|
|
2612
|
+
if (cursor !== currentHead || pathIndex !== firstParentPath.length) {
|
|
2613
|
+
return { ok: false, reason: "unaccounted_first_parent_commits" };
|
|
2614
|
+
}
|
|
2615
|
+
return { ok: true, recoveredMerges };
|
|
2616
|
+
}
|
|
2617
|
+
|
|
2618
|
+
function coordinatorWithCompletedResults(opened, tasks, deps) {
|
|
2619
|
+
const coordinator = coordinatorForAuthority(
|
|
2620
|
+
opened.repoRoot,
|
|
2621
|
+
opened.authority,
|
|
2622
|
+
deps,
|
|
2623
|
+
);
|
|
2624
|
+
const remaining = new Map(tasks.map((task) => [task.key, task]));
|
|
2625
|
+
while (remaining.size > 0) {
|
|
2626
|
+
let progressed = false;
|
|
2627
|
+
for (const [key, task] of [...remaining]) {
|
|
2628
|
+
if (
|
|
2629
|
+
(task.dependsOn || []).some((dependency) => remaining.has(dependency))
|
|
2630
|
+
) {
|
|
2631
|
+
continue;
|
|
2632
|
+
}
|
|
2633
|
+
const result = completedResult(task);
|
|
2634
|
+
if (!result) {
|
|
2635
|
+
fail(
|
|
2636
|
+
"TEAM_QUEUE_RESULT_MISSING",
|
|
2637
|
+
`Completed task "${key}" has no durable worktree result`,
|
|
2638
|
+
);
|
|
2639
|
+
}
|
|
2640
|
+
coordinator.registerCompletedDependency(key, result);
|
|
2641
|
+
remaining.delete(key);
|
|
2642
|
+
progressed = true;
|
|
2643
|
+
}
|
|
2644
|
+
if (!progressed) {
|
|
2645
|
+
fail(
|
|
2646
|
+
"TEAM_QUEUE_INVALID_GRAPH",
|
|
2647
|
+
"Distributed queue result graph is cyclic",
|
|
2648
|
+
);
|
|
2649
|
+
}
|
|
2650
|
+
}
|
|
2651
|
+
return coordinator;
|
|
2652
|
+
}
|
|
2653
|
+
|
|
2654
|
+
function finalizationGitAuthority(opened, tasks, coordinator, deps) {
|
|
2655
|
+
return {
|
|
2656
|
+
baseBranch: opened.authority.baseTarget.branch ?? null,
|
|
2657
|
+
initialBaseOid: opened.authority.baseTarget.commitOid,
|
|
2658
|
+
currentBaseOid: gitHead(opened.repoRoot, opened.authority, deps),
|
|
2659
|
+
branches: tasks
|
|
2660
|
+
.map((task) => {
|
|
2661
|
+
const result = completedResult(task);
|
|
2662
|
+
return {
|
|
2663
|
+
key: task.key,
|
|
2664
|
+
branch: result.branch,
|
|
2665
|
+
commitOid: result.commitOid,
|
|
2666
|
+
worktreePath: result.worktreePath,
|
|
2667
|
+
};
|
|
2668
|
+
})
|
|
2669
|
+
.sort((left, right) => left.key.localeCompare(right.key)),
|
|
2670
|
+
coordinator: coordinator.snapshot(),
|
|
2671
|
+
};
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2674
|
+
function finalizationLeaseOptions(finalization, operationId, owner) {
|
|
2675
|
+
if (!finalization.lease) {
|
|
2676
|
+
fail(
|
|
2677
|
+
"TEAM_QUEUE_FINALIZE_LEASE_LOST",
|
|
2678
|
+
`Finalization phase "${finalization.phase}" has no active lease`,
|
|
2679
|
+
);
|
|
2680
|
+
}
|
|
2681
|
+
return {
|
|
2682
|
+
operationId,
|
|
2683
|
+
owner,
|
|
2684
|
+
leaseId: finalization.lease.leaseId,
|
|
2685
|
+
fencingToken: finalization.lease.fencingToken,
|
|
2686
|
+
expectedPhase: finalization.phase,
|
|
2687
|
+
expectedRevision: finalization.revision,
|
|
2688
|
+
};
|
|
2689
|
+
}
|
|
2690
|
+
|
|
2691
|
+
function requireFinalizationMutation(result, action) {
|
|
2692
|
+
if (!result?.ok) {
|
|
2693
|
+
fail(
|
|
2694
|
+
"TEAM_QUEUE_FINALIZE_STATE_REJECTED",
|
|
2695
|
+
`Could not persist finalization ${action}: ${
|
|
2696
|
+
result?.reason || "unknown reason"
|
|
2697
|
+
}${result?.error ? ` (${result.error})` : ""}`,
|
|
2698
|
+
result,
|
|
2699
|
+
);
|
|
2700
|
+
}
|
|
2701
|
+
return result.finalization;
|
|
2702
|
+
}
|
|
2703
|
+
|
|
2704
|
+
function finalizationOutput(opened, finalization, extra = {}) {
|
|
2705
|
+
return {
|
|
2706
|
+
queueId: opened.snapshot.queueId,
|
|
2707
|
+
runId: opened.runId,
|
|
2708
|
+
preview: finalization.result.preview || [],
|
|
2709
|
+
merged: finalization.phase === "completed",
|
|
2710
|
+
integration:
|
|
2711
|
+
finalization.result.integration || finalization.result.preview || [],
|
|
2712
|
+
cleanup: finalization.result.cleanup || [],
|
|
2713
|
+
worktrees: finalization.coordinator,
|
|
2714
|
+
finalization,
|
|
2715
|
+
...extra,
|
|
2716
|
+
};
|
|
2717
|
+
}
|
|
2718
|
+
|
|
2719
|
+
function verifyFinalizationBranchRefs(opened, finalization, deps) {
|
|
2720
|
+
for (const binding of finalization.git.branches) {
|
|
2721
|
+
let current;
|
|
2722
|
+
try {
|
|
2723
|
+
current = deps
|
|
2724
|
+
.git(opened.repoRoot, ["rev-parse", "--verify", binding.branch])
|
|
2725
|
+
.toLowerCase();
|
|
2726
|
+
} catch {
|
|
2727
|
+
fail(
|
|
2728
|
+
"TEAM_QUEUE_FINALIZE_GIT_DRIFT",
|
|
2729
|
+
`Finalization branch "${binding.branch}" is missing`,
|
|
2730
|
+
);
|
|
2731
|
+
}
|
|
2732
|
+
if (current !== binding.commitOid) {
|
|
2733
|
+
fail(
|
|
2734
|
+
"TEAM_QUEUE_FINALIZE_GIT_DRIFT",
|
|
2735
|
+
`Finalization branch "${binding.branch}" moved after it was fenced`,
|
|
2736
|
+
);
|
|
2737
|
+
}
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
|
|
2741
|
+
export function finalizeDistributedQueue(options, dependencyOverrides = {}) {
|
|
2742
|
+
const deps = defaultDependencies(dependencyOverrides);
|
|
2743
|
+
const opened = openPinnedQueue(options, deps);
|
|
2744
|
+
const tasks = opened.queue.list();
|
|
2745
|
+
const pendingAdjudications = opened.queue.pendingAdjudications();
|
|
2746
|
+
const unfinished = tasks.filter((task) => task.status !== "completed");
|
|
2747
|
+
if (pendingAdjudications.length > 0 || unfinished.length > 0) {
|
|
2748
|
+
const keys = [
|
|
2749
|
+
...new Set([
|
|
2750
|
+
...pendingAdjudications.map((item) => item.key),
|
|
2751
|
+
...unfinished.map((item) => item.key),
|
|
2752
|
+
]),
|
|
2753
|
+
];
|
|
2754
|
+
fail(
|
|
2755
|
+
"TEAM_QUEUE_FINALIZE_BLOCKED",
|
|
2756
|
+
`Distributed queue cannot finalize while tasks require work or adjudication: ${keys.join(", ")}. Inspect with \`cc team queue status\`; resolve adjudication before retrying finalize.`,
|
|
2757
|
+
{ keys, pendingAdjudications },
|
|
2758
|
+
);
|
|
2759
|
+
}
|
|
2760
|
+
const operationId = finalizationOperationId(opened.snapshot, options);
|
|
2761
|
+
const owner = requireText(
|
|
2762
|
+
options.finalizerId || `finalizer-${process.pid}`,
|
|
2763
|
+
"--finalizer-id",
|
|
2764
|
+
);
|
|
2765
|
+
const ttlMs =
|
|
2766
|
+
optionalPositive(options.ttlMs, "--ttl-ms", { integer: true }) ?? undefined;
|
|
2767
|
+
const requestedMode = options.merge === true ? "merge" : "preview";
|
|
2768
|
+
let persisted = opened.queue.getFinalization();
|
|
2769
|
+
let coordinator = null;
|
|
2770
|
+
let git = persisted.git;
|
|
2771
|
+
let coordinatorSnapshot = persisted.coordinator;
|
|
2772
|
+
|
|
2773
|
+
if (persisted.phase === "idle") {
|
|
2774
|
+
coordinator = coordinatorWithCompletedResults(opened, tasks, deps);
|
|
2775
|
+
const authority = finalizationGitAuthority(
|
|
2776
|
+
opened,
|
|
2777
|
+
tasks,
|
|
2778
|
+
coordinator,
|
|
2779
|
+
deps,
|
|
2780
|
+
);
|
|
2781
|
+
coordinatorSnapshot = authority.coordinator;
|
|
2782
|
+
delete authority.coordinator;
|
|
2783
|
+
git = authority;
|
|
2784
|
+
}
|
|
2785
|
+
|
|
2786
|
+
let begun = opened.queue.beginFinalization({
|
|
2787
|
+
operationId,
|
|
2788
|
+
owner,
|
|
2789
|
+
mode: requestedMode,
|
|
2790
|
+
ttlMs,
|
|
2791
|
+
git,
|
|
2792
|
+
coordinator: coordinatorSnapshot,
|
|
2793
|
+
});
|
|
2794
|
+
if (!begun.ok && begun.takeoverRequired === true) {
|
|
2795
|
+
begun = opened.queue.takeoverFinalization({
|
|
2796
|
+
operationId,
|
|
2797
|
+
owner,
|
|
2798
|
+
ttlMs,
|
|
2799
|
+
reason: `resuming ${begun.phase || "unknown"} after unavailable finalizer`,
|
|
2800
|
+
});
|
|
2801
|
+
}
|
|
2802
|
+
if (!begun.ok) {
|
|
2803
|
+
const code =
|
|
2804
|
+
begun.reason === "finalization_busy"
|
|
2805
|
+
? "TEAM_QUEUE_FINALIZE_BUSY"
|
|
2806
|
+
: "TEAM_QUEUE_FINALIZE_DRIFT";
|
|
2807
|
+
fail(
|
|
2808
|
+
code,
|
|
2809
|
+
`Distributed finalization could not start: ${begun.reason}${
|
|
2810
|
+
begun.error ? ` (${begun.error})` : ""
|
|
2811
|
+
}`,
|
|
2812
|
+
begun,
|
|
2813
|
+
);
|
|
2814
|
+
}
|
|
2815
|
+
persisted = begun.finalization;
|
|
2816
|
+
verifyFinalizationBranchRefs(opened, persisted, deps);
|
|
2817
|
+
|
|
2818
|
+
if (persisted.phase === "blocked") {
|
|
2819
|
+
fail(
|
|
2820
|
+
"TEAM_QUEUE_INTEGRATION_CONFLICT",
|
|
2821
|
+
persisted.blocked?.message ||
|
|
2822
|
+
"Distributed queue finalization is durably blocked",
|
|
2823
|
+
finalizationOutput(opened, persisted),
|
|
2824
|
+
);
|
|
2825
|
+
}
|
|
2826
|
+
if (persisted.phase === "completed") {
|
|
2827
|
+
const head = gitHead(opened.repoRoot, opened.authority, deps);
|
|
2828
|
+
if (head !== persisted.git.currentBaseOid) {
|
|
2829
|
+
fail(
|
|
2830
|
+
"TEAM_QUEUE_FINALIZE_GIT_DRIFT",
|
|
2831
|
+
"Completed finalization base HEAD moved after cleanup",
|
|
2832
|
+
);
|
|
2833
|
+
}
|
|
2834
|
+
return finalizationOutput(opened, persisted, { idempotent: true });
|
|
2835
|
+
}
|
|
2836
|
+
|
|
2837
|
+
coordinator =
|
|
2838
|
+
coordinator ||
|
|
2839
|
+
coordinatorForAuthority(opened.repoRoot, opened.authority, deps, {
|
|
2840
|
+
snapshot: persisted.coordinator,
|
|
2841
|
+
});
|
|
2842
|
+
|
|
2843
|
+
const leaseOptions = () =>
|
|
2844
|
+
finalizationLeaseOptions(persisted, operationId, owner);
|
|
2845
|
+
const renewFinalizationLease = (progress = null) => {
|
|
2846
|
+
const lease = persisted.lease;
|
|
2847
|
+
if (!lease) {
|
|
2848
|
+
fail(
|
|
2849
|
+
"TEAM_QUEUE_FINALIZE_LEASE_LOST",
|
|
2850
|
+
`Finalization phase "${persisted.phase}" no longer has an active lease`,
|
|
2851
|
+
{ progress, finalization: persisted },
|
|
2852
|
+
);
|
|
2853
|
+
}
|
|
2854
|
+
const renewed = opened.queue.renewFinalization({
|
|
2855
|
+
operationId,
|
|
2856
|
+
owner,
|
|
2857
|
+
leaseId: lease.leaseId,
|
|
2858
|
+
fencingToken: lease.fencingToken,
|
|
2859
|
+
ttlMs,
|
|
2860
|
+
progress,
|
|
2861
|
+
});
|
|
2862
|
+
if (!renewed.ok) {
|
|
2863
|
+
fail(
|
|
2864
|
+
"TEAM_QUEUE_FINALIZE_LEASE_LOST",
|
|
2865
|
+
`Could not renew distributed finalization lease: ${renewed.reason}${
|
|
2866
|
+
renewed.error ? ` (${renewed.error})` : ""
|
|
2867
|
+
}`,
|
|
2868
|
+
{ progress, renewal: renewed, finalization: persisted },
|
|
2869
|
+
);
|
|
2870
|
+
}
|
|
2871
|
+
// Renewal increments the durable CAS revision. Keep the live coordinator:
|
|
2872
|
+
// its per-worktree progress is intentionally persisted by the next phase
|
|
2873
|
+
// transition, while the queue snapshot still carries the pre-phase copy.
|
|
2874
|
+
persisted = renewed.finalization;
|
|
2875
|
+
};
|
|
2876
|
+
const persistPhase = (toPhase, fields, suffix) => {
|
|
2877
|
+
const expectedCoordinator = fields.coordinator || coordinator.snapshot();
|
|
2878
|
+
const result = opened.queue.recordFinalizationPhase({
|
|
2879
|
+
...leaseOptions(),
|
|
2880
|
+
transitionId: `${operationId}:${suffix}`,
|
|
2881
|
+
toPhase,
|
|
2882
|
+
...fields,
|
|
2883
|
+
});
|
|
2884
|
+
persisted = requireFinalizationMutation(result, suffix);
|
|
2885
|
+
if (
|
|
2886
|
+
canonicalDigest(expectedCoordinator) !==
|
|
2887
|
+
canonicalDigest(persisted.coordinator)
|
|
2888
|
+
) {
|
|
2889
|
+
fail(
|
|
2890
|
+
"TEAM_QUEUE_FINALIZE_DRIFT",
|
|
2891
|
+
`Persisted coordinator changed while recording finalization phase "${toPhase}"`,
|
|
2892
|
+
);
|
|
2893
|
+
}
|
|
2894
|
+
return persisted;
|
|
2895
|
+
};
|
|
2896
|
+
const persistBlocked = (code, message, details = {}) => {
|
|
2897
|
+
const result = opened.queue.recordFinalizationPhase({
|
|
2898
|
+
...leaseOptions(),
|
|
2899
|
+
transitionId: `${operationId}:blocked:${persisted.revision}`,
|
|
2900
|
+
toPhase: "blocked",
|
|
2901
|
+
coordinator: coordinator.snapshot(),
|
|
2902
|
+
currentBaseOid:
|
|
2903
|
+
details.currentBaseOid ||
|
|
2904
|
+
gitHead(opened.repoRoot, opened.authority, deps),
|
|
2905
|
+
result: details.result || undefined,
|
|
2906
|
+
block: {
|
|
2907
|
+
code,
|
|
2908
|
+
message,
|
|
2909
|
+
evidenceDigest: details.evidenceDigest || null,
|
|
2910
|
+
},
|
|
2911
|
+
});
|
|
2912
|
+
persisted = requireFinalizationMutation(result, "blocked result");
|
|
2913
|
+
fail(code, message, finalizationOutput(opened, persisted, details));
|
|
2914
|
+
};
|
|
2915
|
+
const persistRecovery = (error, result = null) => {
|
|
2916
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2917
|
+
const recovery = opened.queue.recordFinalizationPhase({
|
|
2918
|
+
...leaseOptions(),
|
|
2919
|
+
transitionId: `${operationId}:recovery:${persisted.phase}:${persisted.revision}`,
|
|
2920
|
+
toPhase: "recovery_required",
|
|
2921
|
+
coordinator: coordinator.snapshot(),
|
|
2922
|
+
currentBaseOid: gitHead(opened.repoRoot, opened.authority, deps),
|
|
2923
|
+
result: result || undefined,
|
|
2924
|
+
recoveryReason: message,
|
|
2925
|
+
});
|
|
2926
|
+
persisted = requireFinalizationMutation(recovery, "recovery requirement");
|
|
2927
|
+
fail(
|
|
2928
|
+
"TEAM_QUEUE_FINALIZE_RECOVERY_REQUIRED",
|
|
2929
|
+
`Distributed finalization requires a fenced recovery retry: ${message}`,
|
|
2930
|
+
finalizationOutput(opened, persisted),
|
|
2931
|
+
);
|
|
2932
|
+
};
|
|
2933
|
+
|
|
2934
|
+
const recoveryFrom =
|
|
2935
|
+
persisted.phase === "recovery_required"
|
|
2936
|
+
? persisted.recovery?.fromPhase
|
|
2937
|
+
: null;
|
|
2938
|
+
if (persisted.phase === "previewing" || recoveryFrom === "previewing") {
|
|
2939
|
+
let preview;
|
|
2940
|
+
try {
|
|
2941
|
+
renewFinalizationLease({ phase: "previewing", timing: "start" });
|
|
2942
|
+
const expectedHead = persisted.intent.expectedBaseOid;
|
|
2943
|
+
if (gitHead(opened.repoRoot, opened.authority, deps) !== expectedHead) {
|
|
2944
|
+
persistBlocked(
|
|
2945
|
+
"TEAM_QUEUE_FINALIZE_GIT_DRIFT",
|
|
2946
|
+
"Base HEAD moved after preview intent was persisted",
|
|
2947
|
+
);
|
|
2948
|
+
}
|
|
2949
|
+
preview = coordinator.integrate({
|
|
2950
|
+
merge: false,
|
|
2951
|
+
onProgress: renewFinalizationLease,
|
|
2952
|
+
});
|
|
2953
|
+
const currentHead = gitHead(opened.repoRoot, opened.authority, deps);
|
|
2954
|
+
if (currentHead !== expectedHead) {
|
|
2955
|
+
persistBlocked(
|
|
2956
|
+
"TEAM_QUEUE_FINALIZE_GIT_DRIFT",
|
|
2957
|
+
"Base HEAD moved while previewing distributed worktrees",
|
|
2958
|
+
{ currentBaseOid: currentHead, result: { preview } },
|
|
2959
|
+
);
|
|
2960
|
+
}
|
|
2961
|
+
if (preview.some((item) => item.error || item.clean !== true)) {
|
|
2962
|
+
persistBlocked(
|
|
2963
|
+
"TEAM_QUEUE_INTEGRATION_CONFLICT",
|
|
2964
|
+
"Distributed queue merge preview is not clean; worktrees were retained",
|
|
2965
|
+
{ currentBaseOid: currentHead, result: { preview } },
|
|
2966
|
+
);
|
|
2967
|
+
}
|
|
2968
|
+
persistPhase(
|
|
2969
|
+
"previewed",
|
|
2970
|
+
{
|
|
2971
|
+
coordinator: coordinator.snapshot(),
|
|
2972
|
+
currentBaseOid: currentHead,
|
|
2973
|
+
result: { preview },
|
|
2974
|
+
releaseLease: requestedMode === "preview",
|
|
2975
|
+
},
|
|
2976
|
+
"previewed",
|
|
2977
|
+
);
|
|
2978
|
+
} catch (error) {
|
|
2979
|
+
if (error instanceof TeamDistributedCliError) throw error;
|
|
2980
|
+
persistRecovery(error, preview ? { preview } : null);
|
|
2981
|
+
}
|
|
2982
|
+
}
|
|
2983
|
+
|
|
2984
|
+
if (persisted.phase === "previewed") {
|
|
2985
|
+
if (requestedMode === "preview") {
|
|
2986
|
+
return finalizationOutput(opened, persisted);
|
|
2987
|
+
}
|
|
2988
|
+
const currentHead = gitHead(opened.repoRoot, opened.authority, deps);
|
|
2989
|
+
if (currentHead !== persisted.git.currentBaseOid) {
|
|
2990
|
+
persistBlocked(
|
|
2991
|
+
"TEAM_QUEUE_FINALIZE_GIT_DRIFT",
|
|
2992
|
+
"Base HEAD moved after the durable merge preview",
|
|
2993
|
+
{ currentBaseOid: currentHead },
|
|
2994
|
+
);
|
|
2995
|
+
}
|
|
2996
|
+
persistPhase(
|
|
2997
|
+
"merging",
|
|
2998
|
+
{
|
|
2999
|
+
coordinator: coordinator.snapshot(),
|
|
3000
|
+
currentBaseOid: currentHead,
|
|
3001
|
+
intentKind: "merge",
|
|
3002
|
+
},
|
|
3003
|
+
"merging",
|
|
3004
|
+
);
|
|
3005
|
+
}
|
|
3006
|
+
|
|
3007
|
+
const mergeRecovery =
|
|
3008
|
+
persisted.phase === "recovery_required" &&
|
|
3009
|
+
persisted.recovery?.fromPhase === "merging";
|
|
3010
|
+
if (persisted.phase === "merging" || mergeRecovery) {
|
|
3011
|
+
let integration;
|
|
3012
|
+
try {
|
|
3013
|
+
renewFinalizationLease({ phase: "merging", timing: "start" });
|
|
3014
|
+
const currentHead = gitHead(opened.repoRoot, opened.authority, deps);
|
|
3015
|
+
const proof = mergeRecovery
|
|
3016
|
+
? proveMergeRecovery(opened.repoRoot, persisted, currentHead, deps)
|
|
3017
|
+
: {
|
|
3018
|
+
ok: currentHead === persisted.intent.expectedBaseOid,
|
|
3019
|
+
reason: "base_head_moved_before_merge",
|
|
3020
|
+
};
|
|
3021
|
+
if (!proof.ok) {
|
|
3022
|
+
persistBlocked(
|
|
3023
|
+
"TEAM_QUEUE_FINALIZE_GIT_DRIFT",
|
|
3024
|
+
`Cannot prove the fenced merge recovery: ${proof.reason}`,
|
|
3025
|
+
{ currentBaseOid: currentHead },
|
|
3026
|
+
);
|
|
3027
|
+
}
|
|
3028
|
+
integration = coordinator.integrate({
|
|
3029
|
+
merge: true,
|
|
3030
|
+
onProgress: renewFinalizationLease,
|
|
3031
|
+
});
|
|
3032
|
+
const mergedHead = gitHead(opened.repoRoot, opened.authority, deps);
|
|
3033
|
+
const mergeFailed = integration.some(
|
|
3034
|
+
(item) =>
|
|
3035
|
+
item.error ||
|
|
3036
|
+
item.clean !== true ||
|
|
3037
|
+
(item.committed === true && item.merged !== true),
|
|
3038
|
+
);
|
|
3039
|
+
if (mergeFailed) {
|
|
3040
|
+
persistBlocked(
|
|
3041
|
+
"TEAM_QUEUE_INTEGRATION_CONFLICT",
|
|
3042
|
+
"Distributed queue merge failed closed; worktrees were retained",
|
|
3043
|
+
{
|
|
3044
|
+
currentBaseOid: mergedHead,
|
|
3045
|
+
result: { integration },
|
|
3046
|
+
},
|
|
3047
|
+
);
|
|
3048
|
+
}
|
|
3049
|
+
persistPhase(
|
|
3050
|
+
"merged",
|
|
3051
|
+
{
|
|
3052
|
+
coordinator: coordinator.snapshot(),
|
|
3053
|
+
currentBaseOid: mergedHead,
|
|
3054
|
+
result: { integration },
|
|
3055
|
+
},
|
|
3056
|
+
"merged",
|
|
3057
|
+
);
|
|
3058
|
+
} catch (error) {
|
|
3059
|
+
if (error instanceof TeamDistributedCliError) throw error;
|
|
3060
|
+
persistRecovery(error, integration ? { integration } : null);
|
|
3061
|
+
}
|
|
3062
|
+
}
|
|
3063
|
+
|
|
3064
|
+
if (persisted.phase === "merged") {
|
|
3065
|
+
try {
|
|
3066
|
+
renewFinalizationLease({ phase: "merged", timing: "start" });
|
|
3067
|
+
coordinator.prepareCleanupAll({
|
|
3068
|
+
requireMerged: true,
|
|
3069
|
+
onProgress: renewFinalizationLease,
|
|
3070
|
+
});
|
|
3071
|
+
persistPhase(
|
|
3072
|
+
"cleanup_prepared",
|
|
3073
|
+
{
|
|
3074
|
+
coordinator: coordinator.snapshot(),
|
|
3075
|
+
currentBaseOid: gitHead(opened.repoRoot, opened.authority, deps),
|
|
3076
|
+
},
|
|
3077
|
+
"cleanup-prepared",
|
|
3078
|
+
);
|
|
3079
|
+
} catch (error) {
|
|
3080
|
+
if (error instanceof TeamDistributedCliError) throw error;
|
|
3081
|
+
persistBlocked(
|
|
3082
|
+
"TEAM_QUEUE_CLEANUP_PREPARE_FAILED",
|
|
3083
|
+
`Could not durably prepare distributed worktree cleanup: ${
|
|
3084
|
+
error instanceof Error ? error.message : String(error)
|
|
3085
|
+
}`,
|
|
3086
|
+
);
|
|
3087
|
+
}
|
|
3088
|
+
}
|
|
3089
|
+
|
|
3090
|
+
if (persisted.phase === "cleanup_prepared") {
|
|
3091
|
+
persistPhase(
|
|
3092
|
+
"cleaning",
|
|
3093
|
+
{
|
|
3094
|
+
coordinator: coordinator.snapshot(),
|
|
3095
|
+
currentBaseOid: gitHead(opened.repoRoot, opened.authority, deps),
|
|
3096
|
+
intentKind: "cleanup",
|
|
3097
|
+
},
|
|
3098
|
+
"cleaning",
|
|
3099
|
+
);
|
|
3100
|
+
}
|
|
3101
|
+
|
|
3102
|
+
const cleanupRecovery =
|
|
3103
|
+
persisted.phase === "recovery_required" &&
|
|
3104
|
+
persisted.recovery?.fromPhase === "cleaning";
|
|
3105
|
+
if (persisted.phase === "cleaning" || cleanupRecovery) {
|
|
3106
|
+
let cleanup;
|
|
3107
|
+
try {
|
|
3108
|
+
renewFinalizationLease({ phase: "cleaning", timing: "start" });
|
|
3109
|
+
const currentHead = gitHead(opened.repoRoot, opened.authority, deps);
|
|
3110
|
+
if (currentHead !== persisted.intent.expectedBaseOid) {
|
|
3111
|
+
persistBlocked(
|
|
3112
|
+
"TEAM_QUEUE_FINALIZE_GIT_DRIFT",
|
|
3113
|
+
"Base HEAD moved after cleanup intent was persisted",
|
|
3114
|
+
{ currentBaseOid: currentHead },
|
|
3115
|
+
);
|
|
3116
|
+
}
|
|
3117
|
+
cleanup = coordinator.cleanupAll({
|
|
3118
|
+
deleteBranch: false,
|
|
3119
|
+
onProgress: renewFinalizationLease,
|
|
3120
|
+
});
|
|
3121
|
+
if (cleanup.some((item) => item.ok !== true)) {
|
|
3122
|
+
persistRecovery(
|
|
3123
|
+
new Error("one or more distributed worktrees could not be cleaned"),
|
|
3124
|
+
{ cleanup },
|
|
3125
|
+
);
|
|
3126
|
+
}
|
|
3127
|
+
const completed = opened.queue.completeFinalization({
|
|
3128
|
+
...leaseOptions(),
|
|
3129
|
+
transitionId: `${operationId}:completed`,
|
|
3130
|
+
coordinator: coordinator.snapshot(),
|
|
3131
|
+
cleanup,
|
|
3132
|
+
currentBaseOid: gitHead(opened.repoRoot, opened.authority, deps),
|
|
3133
|
+
});
|
|
3134
|
+
persisted = requireFinalizationMutation(completed, "completion");
|
|
3135
|
+
} catch (error) {
|
|
3136
|
+
if (error instanceof TeamDistributedCliError) throw error;
|
|
3137
|
+
persistRecovery(error, cleanup ? { cleanup } : null);
|
|
3138
|
+
}
|
|
3139
|
+
}
|
|
3140
|
+
|
|
3141
|
+
return finalizationOutput(opened, persisted);
|
|
3142
|
+
}
|
|
3143
|
+
|
|
3144
|
+
function emitResult(result, options, log) {
|
|
3145
|
+
if (options.json) {
|
|
3146
|
+
(log.log || console.log)(JSON.stringify(result, null, 2));
|
|
3147
|
+
return;
|
|
3148
|
+
}
|
|
3149
|
+
(log.log || console.log)(`Distributed queue ${result.queueId || ""}`.trim());
|
|
3150
|
+
if (result.stats) {
|
|
3151
|
+
(log.log || console.log)(
|
|
3152
|
+
` ${result.stats.completed}/${result.stats.total} completed, ${result.stats.leased} leased`,
|
|
3153
|
+
);
|
|
3154
|
+
}
|
|
3155
|
+
}
|
|
3156
|
+
|
|
3157
|
+
function action(handler, log) {
|
|
3158
|
+
return async (options) => {
|
|
3159
|
+
try {
|
|
3160
|
+
const result = await handler(options);
|
|
3161
|
+
emitResult(result, options, log);
|
|
3162
|
+
} catch (error) {
|
|
3163
|
+
(log.error || console.error)(
|
|
3164
|
+
`${error.code ? `${error.code}: ` : ""}${error.message}`,
|
|
3165
|
+
);
|
|
3166
|
+
if (error.details && options.json) {
|
|
3167
|
+
(log.error || console.error)(JSON.stringify(error.details, null, 2));
|
|
3168
|
+
}
|
|
3169
|
+
process.exitCode = 1;
|
|
3170
|
+
}
|
|
3171
|
+
};
|
|
3172
|
+
}
|
|
3173
|
+
|
|
3174
|
+
function baseAuthorityOptions(command) {
|
|
3175
|
+
return command
|
|
3176
|
+
.requiredOption("--state <file>", "Queue state outside the repository")
|
|
3177
|
+
.requiredOption("--run-id <id>", "Pinned distributed run id")
|
|
3178
|
+
.option("--repo <dir>", "Repository (defaults to current repository)")
|
|
3179
|
+
.option("--json", "Output JSON");
|
|
3180
|
+
}
|
|
3181
|
+
|
|
3182
|
+
function commonAuthorityOptions(command) {
|
|
3183
|
+
return baseAuthorityOptions(command)
|
|
3184
|
+
.option("--queue-id <id>", "Pinned queue id")
|
|
3185
|
+
.option("--authority-digest <sha256>", "Pinned authority digest");
|
|
3186
|
+
}
|
|
3187
|
+
|
|
3188
|
+
/** Register beneath the existing `cc team` Commander command. */
|
|
3189
|
+
export function registerTeamDistributedCommands(
|
|
3190
|
+
team,
|
|
3191
|
+
{ logger, agentExecutor = null, buildAgentPrompt = null } = {},
|
|
3192
|
+
) {
|
|
3193
|
+
const log = logger || console;
|
|
3194
|
+
const agentDependencies = {
|
|
3195
|
+
agentExecutor,
|
|
3196
|
+
buildAgentPrompt: buildAgentPrompt || ((prompt) => prompt),
|
|
3197
|
+
};
|
|
3198
|
+
const queue = team
|
|
3199
|
+
.command("queue")
|
|
3200
|
+
.description("Coordinate durable multi-process worktree teammates");
|
|
3201
|
+
|
|
3202
|
+
baseAuthorityOptions(
|
|
3203
|
+
queue
|
|
3204
|
+
.command("init")
|
|
3205
|
+
.description("Create a pinned distributed task queue")
|
|
3206
|
+
.requiredOption("--tasks <file>", "Task graph JSON")
|
|
3207
|
+
.option(
|
|
3208
|
+
"--mode <mode>",
|
|
3209
|
+
"shell-worktree or agent-worktree",
|
|
3210
|
+
SHELL_WORKTREE_MODE,
|
|
3211
|
+
)
|
|
3212
|
+
.option("--max-tasks <n>", "Global task-attempt cap")
|
|
3213
|
+
.option("--max-tokens <n>", "Global token cap")
|
|
3214
|
+
.option("--max-usd <amount>", "Global USD cap")
|
|
3215
|
+
.option("--max-wall-ms <ms>", "Global wall-clock cap")
|
|
3216
|
+
.option("--model <model>", "Pinned model for agent-worktree tasks")
|
|
3217
|
+
.option(
|
|
3218
|
+
"--permission-mode <mode>",
|
|
3219
|
+
"Pinned permission mode for agent-worktree tasks",
|
|
3220
|
+
"acceptEdits",
|
|
3221
|
+
)
|
|
3222
|
+
.option("--agent-max-turns <n>", "Per-Agent turn cap")
|
|
3223
|
+
.option("--agent-max-tokens <n>", "Per-Agent token cap")
|
|
3224
|
+
.option("--agent-max-budget-usd <amount>", "Per-Agent USD cap")
|
|
3225
|
+
.option("--agent-max-wall-ms <ms>", "Per-Agent wall-clock cap")
|
|
3226
|
+
.option(
|
|
3227
|
+
"--managed-checkpoint",
|
|
3228
|
+
"Capture each task worktree in a Process Broker checkpoint",
|
|
3229
|
+
)
|
|
3230
|
+
.option(
|
|
3231
|
+
"--checkpoint-state-dir <dir>",
|
|
3232
|
+
"External checkpoint store (defaults beside --state)",
|
|
3233
|
+
)
|
|
3234
|
+
.option("--ttl-ms <ms>", "Lease TTL"),
|
|
3235
|
+
).action(action(initDistributedQueue, log));
|
|
3236
|
+
|
|
3237
|
+
commonAuthorityOptions(
|
|
3238
|
+
queue
|
|
3239
|
+
.command("status")
|
|
3240
|
+
.description("Inspect a pinned distributed queue")
|
|
3241
|
+
.option("--mode <mode>", "Verify the pinned execution mode"),
|
|
3242
|
+
).action(action(distributedQueueStatus, log));
|
|
3243
|
+
|
|
3244
|
+
commonAuthorityOptions(
|
|
3245
|
+
queue
|
|
3246
|
+
.command("interrupt")
|
|
3247
|
+
.description("Interrupt one exact leased distributed task attempt")
|
|
3248
|
+
.requiredOption("--task <key>", "Pinned task key")
|
|
3249
|
+
.requiredOption("--holder <id>", "Expected lease holder")
|
|
3250
|
+
.requiredOption("--lease-id <id>", "Expected lease id")
|
|
3251
|
+
.requiredOption("--fencing-token <n>", "Expected fencing token")
|
|
3252
|
+
.requiredOption("--request-id <id>", "Globally stable interrupt id")
|
|
3253
|
+
.option("--actor <name>", "Interrupting operator", "human")
|
|
3254
|
+
.option("--reason <text>", "Durable interrupt reason"),
|
|
3255
|
+
).action(action(interruptDistributedQueue, log));
|
|
3256
|
+
|
|
3257
|
+
commonAuthorityOptions(
|
|
3258
|
+
queue
|
|
3259
|
+
.command("recover")
|
|
3260
|
+
.description("Recover and reconcile one abandoned managed checkpoint")
|
|
3261
|
+
.requiredOption("--task <key>", "Pinned task key")
|
|
3262
|
+
.requiredOption("--recovery-id <id>", "Globally stable recovery id")
|
|
3263
|
+
.requiredOption(
|
|
3264
|
+
"--evidence-digest <sha256>",
|
|
3265
|
+
"Pinned abandonment evidence digest",
|
|
3266
|
+
)
|
|
3267
|
+
.option("--actor <name>", "Recovery operator", "recovery")
|
|
3268
|
+
.option("--reason <text>", "Durable recovery reason")
|
|
3269
|
+
.option(
|
|
3270
|
+
"--repair-git-baseline",
|
|
3271
|
+
"Reset only the exact proven abandoned task commit to its baseline",
|
|
3272
|
+
),
|
|
3273
|
+
).action(action(recoverDistributedQueueCheckpoint, log));
|
|
3274
|
+
|
|
3275
|
+
commonAuthorityOptions(
|
|
3276
|
+
queue
|
|
3277
|
+
.command("adjudicate")
|
|
3278
|
+
.description("Resolve one evidence-pinned distributed task outcome")
|
|
3279
|
+
.requiredOption("--task <key>", "Pinned task key")
|
|
3280
|
+
.requiredOption("--decision <action>", "retry, accept, or cancel")
|
|
3281
|
+
.requiredOption("--decision-id <id>", "Globally stable decision id")
|
|
3282
|
+
.requiredOption(
|
|
3283
|
+
"--evidence-digest <sha256>",
|
|
3284
|
+
"Pinned abandonment evidence digest",
|
|
3285
|
+
)
|
|
3286
|
+
.option("--actor <name>", "Adjudicating operator", "human")
|
|
3287
|
+
.option("--reason <text>", "Durable adjudication reason"),
|
|
3288
|
+
).action(action(adjudicateDistributedQueue, log));
|
|
3289
|
+
|
|
3290
|
+
commonAuthorityOptions(
|
|
3291
|
+
queue
|
|
3292
|
+
.command("worker")
|
|
3293
|
+
.description("Run claimable shell or Agent tasks in isolated worktrees")
|
|
3294
|
+
.option("--worker-id <id>", "Stable process worker id")
|
|
3295
|
+
.option("--mode <mode>", "Verify the pinned execution mode")
|
|
3296
|
+
.option("--ttl-ms <ms>", "Lease TTL")
|
|
3297
|
+
.option("--renew-every-ms <ms>", "Lease heartbeat interval")
|
|
3298
|
+
.option("--max-tasks <n>", "Local execution cap")
|
|
3299
|
+
.option(
|
|
3300
|
+
"--managed-checkpoint",
|
|
3301
|
+
"Require the queue's pinned managed checkpoint authority",
|
|
3302
|
+
)
|
|
3303
|
+
.option(
|
|
3304
|
+
"--checkpoint-state-dir <dir>",
|
|
3305
|
+
"Verify the queue's pinned external checkpoint store",
|
|
3306
|
+
)
|
|
3307
|
+
.option("--agent", "Require pinned agent-worktree authority"),
|
|
3308
|
+
).action(
|
|
3309
|
+
action((options) => runDistributedWorker(options, agentDependencies), log),
|
|
3310
|
+
);
|
|
3311
|
+
|
|
3312
|
+
commonAuthorityOptions(
|
|
3313
|
+
queue
|
|
3314
|
+
.command("finalize")
|
|
3315
|
+
.description("Preview completed branches and optionally merge them")
|
|
3316
|
+
.option("--mode <mode>", "Verify the pinned execution mode")
|
|
3317
|
+
.option("--merge", "Merge only after a clean preview")
|
|
3318
|
+
.option("--operation-id <id>", "Stable finalization operation id")
|
|
3319
|
+
.option("--finalizer-id <id>", "Stable finalizer owner id")
|
|
3320
|
+
.option("--ttl-ms <ms>", "Finalization lease TTL"),
|
|
3321
|
+
).action(action(finalizeDistributedQueue, log));
|
|
3322
|
+
|
|
3323
|
+
return queue;
|
|
3324
|
+
}
|