audit-tools 0.33.4 → 0.33.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit/cli/args.d.ts +1 -1
- package/dist/audit/cli/args.js +1 -1
- package/dist/audit/cli/hybridDispatch.d.ts +9 -2
- package/dist/audit/cli/hybridDispatch.d.ts.map +1 -1
- package/dist/audit/cli/hybridDispatch.js +10 -6
- package/dist/audit/cli/hybridDispatch.js.map +1 -1
- package/dist/audit/cli/nextStepCommand.js +8 -8
- package/dist/audit/cli/nextStepCommand.js.map +1 -1
- package/dist/audit/cli/nextStepHelpers.d.ts.map +1 -1
- package/dist/audit/cli/nextStepHelpers.js +45 -43
- package/dist/audit/cli/nextStepHelpers.js.map +1 -1
- package/dist/audit/cli/reviewRun.d.ts +1 -0
- package/dist/audit/cli/reviewRun.d.ts.map +1 -1
- package/dist/audit/cli/reviewRun.js +5 -1
- package/dist/audit/cli/reviewRun.js.map +1 -1
- package/dist/audit/cli/rollingAuditDispatch.d.ts +2 -32
- package/dist/audit/cli/rollingAuditDispatch.d.ts.map +1 -1
- package/dist/audit/cli/rollingAuditDispatch.js +9 -39
- package/dist/audit/cli/rollingAuditDispatch.js.map +1 -1
- package/dist/remediate/steps/dispatch/marshal.d.ts.map +1 -1
- package/dist/remediate/steps/dispatch/marshal.js +8 -1
- package/dist/remediate/steps/dispatch/marshal.js.map +1 -1
- package/dist/remediate/steps/dispatch/waveScheduling.d.ts +9 -2
- package/dist/remediate/steps/dispatch/waveScheduling.d.ts.map +1 -1
- package/dist/remediate/steps/dispatch/waveScheduling.js +29 -19
- package/dist/remediate/steps/dispatch/waveScheduling.js.map +1 -1
- package/dist/remediate/steps/nextStep.d.ts +79 -65
- package/dist/remediate/steps/nextStep.d.ts.map +1 -1
- package/dist/remediate/steps/nextStep.js +342 -376
- package/dist/remediate/steps/nextStep.js.map +1 -1
- package/dist/remediate/steps/sessionConfigLoad.js +1 -1
- package/dist/remediate/steps/sessionConfigLoad.js.map +1 -1
- package/dist/shared/dispatch/admissionLoop.d.ts +31 -9
- package/dist/shared/dispatch/admissionLoop.d.ts.map +1 -1
- package/dist/shared/dispatch/admissionLoop.js +87 -3
- package/dist/shared/dispatch/admissionLoop.js.map +1 -1
- package/dist/shared/dispatch/dispatchQuotaContract.d.ts +7 -7
- package/dist/shared/dispatch/rollingDispatch.d.ts +18 -1
- package/dist/shared/dispatch/rollingDispatch.d.ts.map +1 -1
- package/dist/shared/dispatch/rollingDispatch.js +35 -9
- package/dist/shared/dispatch/rollingDispatch.js.map +1 -1
- package/dist/shared/index.d.ts +2 -2
- package/dist/shared/index.d.ts.map +1 -1
- package/dist/shared/index.js +3 -3
- package/dist/shared/index.js.map +1 -1
- package/dist/shared/providers/auditorSources.d.ts +39 -44
- package/dist/shared/providers/auditorSources.d.ts.map +1 -1
- package/dist/shared/providers/auditorSources.js +90 -63
- package/dist/shared/providers/auditorSources.js.map +1 -1
- package/dist/shared/providers/claudeWorkerProvider.d.ts +12 -11
- package/dist/shared/providers/claudeWorkerProvider.d.ts.map +1 -1
- package/dist/shared/providers/claudeWorkerProvider.js +23 -16
- package/dist/shared/providers/claudeWorkerProvider.js.map +1 -1
- package/dist/shared/providers/providerConfirmation.d.ts +2 -2
- package/dist/shared/providers/providerConfirmation.js +2 -2
- package/dist/shared/providers/providerConfirmation.js.map +1 -1
- package/dist/shared/providers/providerPathGuard.d.ts +33 -4
- package/dist/shared/providers/providerPathGuard.d.ts.map +1 -1
- package/dist/shared/providers/providerPathGuard.js +33 -4
- package/dist/shared/providers/providerPathGuard.js.map +1 -1
- package/dist/shared/providers/proxyCatalog.d.ts +19 -16
- package/dist/shared/providers/proxyCatalog.d.ts.map +1 -1
- package/dist/shared/providers/proxyCatalog.js +272 -181
- package/dist/shared/providers/proxyCatalog.js.map +1 -1
- package/dist/shared/providers/sharedProviderConfirmation.d.ts +1 -1
- package/dist/shared/providers/sharedProviderConfirmation.js +2 -2
- package/dist/shared/providers/sharedProviderConfirmation.js.map +1 -1
- package/dist/shared/quota/apiPool.d.ts +76 -44
- package/dist/shared/quota/apiPool.d.ts.map +1 -1
- package/dist/shared/quota/apiPool.js +142 -75
- package/dist/shared/quota/apiPool.js.map +1 -1
- package/dist/shared/quota/capacity.d.ts +1 -1
- package/dist/shared/types/providerConfirmation.d.ts +2 -2
- package/dist/shared/types/sessionConfig.d.ts +9 -8
- package/dist/shared/types/sessionConfig.d.ts.map +1 -1
- package/dist/shared/types/sessionConfig.js +1 -1
- package/dist/shared/types/sessionConfig.js.map +1 -1
- package/dist/shared/validation/sessionConfig.js +1 -1
- package/dist/shared/validation/sessionConfig.js.map +1 -1
- package/package.json +1 -1
|
@@ -3,14 +3,14 @@ import { existsSync, statSync } from "node:fs";
|
|
|
3
3
|
import { mkdir, readFile, rename } from "node:fs/promises";
|
|
4
4
|
import { dirname, join, resolve } from "node:path";
|
|
5
5
|
import { StateStore } from "../state/store.js";
|
|
6
|
-
import { readConfirmedDispatchPolicy, resolveDispatchExclusion, readOptionalJsonFile, readValidatedRepoSessionIntent, stagedAndUntracked, writeJsonFile, writeTextFile, buildAuditDeliverablePair, formatValidationIssues, isRecord, withFsRetry, RunLogger, DISPATCH_PROMPT_HANDOFF_NOTE, renderHostScratchNote, hostScratchDir, renderQuotaCoverageNudge, renderTokenBudgetView, coerceJsonObjectArg, driveRolling, resolveLedgerBudgets, setQuotaStateDir, detectHostDispatchWall, admissionBlockedOnBudget, reconcileAdmissionLeasesFromQuotaFile, buildQuotaPausedTerminal, interpretFreeFormIntent, advance, decideFrictionTriage, buildFrictionTriageBlock, planHybridDispatch, readSettledPools, addSettledPool, isPoolSettlingOutcome, isInProcessWorkerProvider,
|
|
6
|
+
import { readConfirmedDispatchPolicy, resolveDispatchExclusion, readOptionalJsonFile, readValidatedRepoSessionIntent, stagedAndUntracked, writeJsonFile, writeTextFile, buildAuditDeliverablePair, formatValidationIssues, isRecord, withFsRetry, RunLogger, DISPATCH_PROMPT_HANDOFF_NOTE, renderHostScratchNote, hostScratchDir, renderQuotaCoverageNudge, renderTokenBudgetView, coerceJsonObjectArg, driveRolling, resolveLedgerBudgets, setQuotaStateDir, detectHostDispatchWall, admissionBlockedOnBudget, reconcileAdmissionLeasesFromQuotaFile, buildQuotaPausedTerminal, interpretFreeFormIntent, advance, decideFrictionTriage, buildFrictionTriageBlock, planHybridDispatch, readSettledPools, addSettledPool, isPoolSettlingOutcome, isInProcessWorkerProvider, sourceByPoolId, classifyProvider, selectDispatchDriver, renderDispatchDriverInstruction, HostSessionQuotaSource, buildProviderModelKey, captureStepBoundaryFriction, captureCostDriftFriction, captureCreditExhaustionFriction, captureQuotaUnclassifiedFriction, captureModelUnavailableFriction, capturePacketTooLargeFriction, LENSES, SEVERITIES, resolveHostProviderName, resolveHostDispatchProviderName, resolveHostDispatchCapability as sharedResolveHostDispatchCapability, resolveAutonomousMode, resolveRollingEngineFlag, DEFAULT_CONTEXT_TOKENS } from "audit-tools/shared";
|
|
7
7
|
import { readRemediationAccessMemory, computeBlockContinuityScores } from "../state/accessMemory.js";
|
|
8
8
|
import { applyPlanPipeline, buildCoverageLedger } from "../phases/plan.js";
|
|
9
9
|
import { groundExtractedFindings } from "../phases/grounding.js";
|
|
10
10
|
import { runTriagePhase } from "../phases/triage.js";
|
|
11
11
|
import { runClosePhase } from "../phases/close.js";
|
|
12
12
|
import { validateRemediationPlan } from "../validation/remediationState.js";
|
|
13
|
-
import { mergeImplementResults, prepareImplementDispatch, readExtractedPlanIfPresent, buildConfirmedPools, executeNodeInWorktree, blockScopesFromPlan,
|
|
13
|
+
import { mergeImplementResults, prepareImplementDispatch, readExtractedPlanIfPresent, buildConfirmedPools, executeNodeInWorktree, blockScopesFromPlan, targetedCommandsForBlock, } from "./dispatch.js";
|
|
14
14
|
import { makeProviderNodeDispatcher } from "./providerNodeDispatch.js";
|
|
15
15
|
import { prepareHostRollingDispatch, nodeClaimRegistry, nodeSettledPoolsPath } from "./rollingSession.js";
|
|
16
16
|
import { ClaimRegistry } from "../../shared/quota/claimRegistry.js";
|
|
@@ -509,12 +509,16 @@ export async function driveRollingDispatch(levels, options) {
|
|
|
509
509
|
...(continuity > 0 ? { continuity } : {}),
|
|
510
510
|
};
|
|
511
511
|
},
|
|
512
|
-
toPacket: (b) =>
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
512
|
+
toPacket: (b) => {
|
|
513
|
+
const requiredTier = options.tierForBlock?.(b);
|
|
514
|
+
return {
|
|
515
|
+
id: b.block_id,
|
|
516
|
+
payload: { block_id: b.block_id },
|
|
517
|
+
estimatedTokens: estimateTokens(b),
|
|
518
|
+
complexity: 0.5,
|
|
519
|
+
...(requiredTier ? { requiredTier } : {}),
|
|
520
|
+
};
|
|
521
|
+
},
|
|
518
522
|
dispatchPacket: async (packet, slot) => options.dispatchNode(blockById.get(packet.payload.block_id), slot),
|
|
519
523
|
...(options.root !== undefined ? { root: options.root } : {}),
|
|
520
524
|
...(options.onCostDrift ? { onCostDrift: options.onCostDrift } : {}),
|
|
@@ -538,18 +542,9 @@ export async function driveRollingDispatch(levels, options) {
|
|
|
538
542
|
levels: run.levels.map((l) => ({ blockIds: l.nodeIds, results: l.results })),
|
|
539
543
|
rebuilds: run.rebuilds,
|
|
540
544
|
...(run.terminal ? { terminal: run.terminal } : {}),
|
|
545
|
+
exhaustedPoolIds: run.exhaustedPoolIds,
|
|
541
546
|
};
|
|
542
547
|
}
|
|
543
|
-
/**
|
|
544
|
-
* Whether session config names an EXPLICIT backend the orchestrator self-drives as
|
|
545
|
-
* the per-node implement worker — the shared `isHeadlessPrimaryProvider` predicate
|
|
546
|
-
* (H3), remediate policy: command-shaped primaries allowed (implement nodes carry
|
|
547
|
-
* per-node worker commands). The conversation host (claude-code) and IDE-bound
|
|
548
|
-
* providers stay excluded; "auto" stays on the conversation host-subagent default.
|
|
549
|
-
*/
|
|
550
|
-
function resolvesToInProcessDispatchProvider(sessionConfig) {
|
|
551
|
-
return isHeadlessPrimaryProvider(sessionConfig?.provider, { commandWorkers: true });
|
|
552
|
-
}
|
|
553
548
|
/**
|
|
554
549
|
* Whether a confirmed pool is one the orchestrator launches IN-PROCESS this cycle
|
|
555
550
|
* (vs. the conversation host's subagent pool) — the shared
|
|
@@ -588,21 +583,23 @@ export async function driveRollingImplementDispatch(options) {
|
|
|
588
583
|
const { root, artifactsDir, runId } = options;
|
|
589
584
|
// Prepare the dispatch plan (eligible verified-complete frontier) with the SAME
|
|
590
585
|
// quota-derived sizing the wave path uses. This writes per-node prompts +
|
|
591
|
-
// dispatch-plan.json + dispatch-quota.json.
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
586
|
+
// dispatch-plan.json + dispatch-quota.json. A partition-scoped hybrid caller
|
|
587
|
+
// supplies its already-prepared plan instead (planOverride — see its docblock).
|
|
588
|
+
const plan = options.planOverride ??
|
|
589
|
+
(await prepareImplementDispatch({ root, artifactsDir }, runId, undefined, {
|
|
590
|
+
hostMaxConcurrent: options.waveOptions?.hostMaxConcurrent,
|
|
591
|
+
sessionConfig: options.sessionConfig,
|
|
592
|
+
hostContextTokens: options.waveOptions?.hostContextTokens,
|
|
593
|
+
hostOutputTokens: options.waveOptions?.hostOutputTokens,
|
|
594
|
+
hostModels: options.waveOptions?.hostModels,
|
|
595
|
+
hostModelId: options.waveOptions?.hostModelId,
|
|
596
|
+
// Each node runs in its own worktree, so its prompt is rooted there.
|
|
597
|
+
worktreeRootedPrompts: true,
|
|
598
|
+
// The in-process rolling engine admits + leases per-packet itself, so the
|
|
599
|
+
// dispatch-quota grant here must NOT lease (a host grant lease would
|
|
600
|
+
// double-count the same work against the shared account budget).
|
|
601
|
+
grantLeases: false,
|
|
602
|
+
}));
|
|
606
603
|
if (plan.items.length === 0) {
|
|
607
604
|
return null;
|
|
608
605
|
}
|
|
@@ -625,6 +622,12 @@ export async function driveRollingImplementDispatch(options) {
|
|
|
625
622
|
const referencedFilesByBlock = new Map(plan.items
|
|
626
623
|
.filter((i) => typeof i.block_id === "string")
|
|
627
624
|
.map((i) => [i.block_id, i.access?.read_paths ?? []]));
|
|
625
|
+
// Per-block capability floor from the plan's model hints (F4) — the same tier
|
|
626
|
+
// the contract's admission packets carry, so the engine's packet→pool
|
|
627
|
+
// selection enforces what the contract displays.
|
|
628
|
+
const tierByBlock = new Map(plan.items
|
|
629
|
+
.filter((i) => typeof i.block_id === "string")
|
|
630
|
+
.flatMap((i) => (i.model_hint ? [[i.block_id, i.model_hint.tier]] : [])));
|
|
628
631
|
// The RETAINED host-session source: threaded through pool sizing AND the
|
|
629
632
|
// dispatcher's escalation hooks so the bounded re-limit chain (recordLimit →
|
|
630
633
|
// escalate → strand → quota_escalation friction) is fed end-to-end. Its
|
|
@@ -650,20 +653,22 @@ export async function driveRollingImplementDispatch(options) {
|
|
|
650
653
|
},
|
|
651
654
|
});
|
|
652
655
|
// Confirmed pools: quota-derived concurrency, never the raw host flag (INV-QD-11).
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
656
|
+
// A partition-scoped caller supplies its coordinator's already-built set instead.
|
|
657
|
+
const confirmedPools = options.poolsOverride ??
|
|
658
|
+
(await buildConfirmedPools({
|
|
659
|
+
sessionConfig: options.sessionConfig,
|
|
660
|
+
hostMaxConcurrent: options.waveOptions?.hostMaxConcurrent,
|
|
661
|
+
hostContextTokens: options.waveOptions?.hostContextTokens,
|
|
662
|
+
hostOutputTokens: options.waveOptions?.hostOutputTokens,
|
|
663
|
+
hostModels: options.waveOptions?.hostModels,
|
|
664
|
+
hostModelId: options.waveOptions?.hostModelId,
|
|
665
|
+
hostSession,
|
|
666
|
+
// The operator's Gate-0 exclusions, applied as a set-difference over freshly
|
|
667
|
+
// gathered reach. Read here because this layer owns `root`; self-spawn-blocked is
|
|
668
|
+
// recomputed against THIS process's env rather than inherited from the auditor
|
|
669
|
+
// that wrote the confirmation.
|
|
670
|
+
excludedBackends: resolveDispatchExclusion(await readConfirmedDispatchPolicy(options.root)),
|
|
671
|
+
}));
|
|
667
672
|
// The live per-node worker: the configured provider, launched with the node's
|
|
668
673
|
// worktree-rooted prompt and cwd = its worktree. Tests inject `options.dispatchNode`
|
|
669
674
|
// to exercise the engine without spawning a real worker. A node on a source-backed
|
|
@@ -683,11 +688,22 @@ export async function driveRollingImplementDispatch(options) {
|
|
|
683
688
|
if (!state)
|
|
684
689
|
return null;
|
|
685
690
|
const plannedBlockIds = new Set(resultPathByBlock.keys());
|
|
691
|
+
// Partition scope (H2): when the caller assigned this driver a coordinator
|
|
692
|
+
// partition, only those planned blocks are driven — everything else stays
|
|
693
|
+
// pending for the caller's other pools (the host share included).
|
|
694
|
+
const partitionIds = options.blocksOverride !== undefined ? new Set(options.blocksOverride) : null;
|
|
686
695
|
const allLevels = rollingDependencyLevels(state);
|
|
687
696
|
// Keep only the blocks that were actually planned this dispatch (eligible now).
|
|
688
697
|
const levels = allLevels
|
|
689
|
-
.map((level) => level.filter((b) => plannedBlockIds.has(b.block_id)))
|
|
698
|
+
.map((level) => level.filter((b) => plannedBlockIds.has(b.block_id) && (partitionIds === null || partitionIds.has(b.block_id))))
|
|
690
699
|
.filter((level) => level.length > 0);
|
|
700
|
+
// Empty partition → an empty-but-SHAPED result, never null: `null` means "no
|
|
701
|
+
// eligible work, run the merge" to existing callers, and a partition caller
|
|
702
|
+
// following that recipe against the just-written full-frontier plan would
|
|
703
|
+
// terminal-block every undriven block (review h2c2 F1).
|
|
704
|
+
if (partitionIds !== null && levels.length === 0) {
|
|
705
|
+
return { nodes: [], rebuilds: 0, state_status: state.status, exhausted_pool_ids: [] };
|
|
706
|
+
}
|
|
691
707
|
const nodeOutcomes = [];
|
|
692
708
|
// The SAME file-backed claim registry the host-subagent driver claims through
|
|
693
709
|
// (`nodeClaimRegistry`, keyed only to run + artifacts dir). Claiming a node here
|
|
@@ -697,7 +713,9 @@ export async function driveRollingImplementDispatch(options) {
|
|
|
697
713
|
// life of this run: a `rate_limited` re-queue re-enters for the same block, so a
|
|
698
714
|
// node already claimed by THIS driver reuses its token rather than self-colliding.
|
|
699
715
|
const registry = nodeClaimRegistry(artifactsDir, runId);
|
|
700
|
-
|
|
716
|
+
// Seeded with any coordinator-held claims the caller hands off (hybrid partition
|
|
717
|
+
// drive) — an adopted node skips self-claiming and releases with the adopted token.
|
|
718
|
+
const claimTokens = new Map(options.claimOwnerTokens ?? []);
|
|
701
719
|
// Per-node worktree dispatch + verify-before-accept, wrapped so the rolling
|
|
702
720
|
// engine's dispatchNode callback always RESOLVES (never rejects).
|
|
703
721
|
const dispatchNodeWithWorktree = async (block, slot) => {
|
|
@@ -710,7 +728,7 @@ export async function driveRollingImplementDispatch(options) {
|
|
|
710
728
|
if (!claimTokens.has(block.block_id)) {
|
|
711
729
|
const claim = await registry.claim(block.block_id, "in-process");
|
|
712
730
|
if (!claim.acquired) {
|
|
713
|
-
nodeOutcomes.push({ block_id: block.block_id, outcome: "success", verify_passed: false, merged: false });
|
|
731
|
+
nodeOutcomes.push({ block_id: block.block_id, outcome: "success", verify_passed: false, merged: false, pool_id: slot.poolId });
|
|
714
732
|
return {
|
|
715
733
|
packet: { id: block.block_id, payload: { block_id: block.block_id }, estimatedTokens: 0, complexity: 0.5 },
|
|
716
734
|
outcome: "success",
|
|
@@ -743,6 +761,9 @@ export async function driveRollingImplementDispatch(options) {
|
|
|
743
761
|
outcome: accept.outcome,
|
|
744
762
|
verify_passed: accept.verifyPassed,
|
|
745
763
|
merged: accept.merged,
|
|
764
|
+
// Real per-node pool attribution (H2 plan D2) — the slot the engine bound,
|
|
765
|
+
// so a partition caller can settle the RIGHT pool from a node's outcome.
|
|
766
|
+
pool_id: slot.poolId,
|
|
746
767
|
});
|
|
747
768
|
// Release the claim ONLY on a terminal accept. A `rate_limited`,
|
|
748
769
|
// `credit_exhausted`, or `quota_unclassified` worker re-queues (still owned
|
|
@@ -775,6 +796,7 @@ export async function driveRollingImplementDispatch(options) {
|
|
|
775
796
|
hostSession,
|
|
776
797
|
continuityScores,
|
|
777
798
|
scopeForBlock: (block) => writePathsByBlock.get(block.block_id) ?? block.touched_files ?? [],
|
|
799
|
+
tierForBlock: (block) => tierByBlock.get(block.block_id),
|
|
778
800
|
// Reactive cost verification: a declared-free source pool observed charging has
|
|
779
801
|
// been demoted by the engine; surface it as reviewable friction so the operator
|
|
780
802
|
// reconciles the stale `cost_per_mtok:0` (single step-boundary chokepoint).
|
|
@@ -808,6 +830,22 @@ export async function driveRollingImplementDispatch(options) {
|
|
|
808
830
|
capturePacketTooLargeFriction(artifactsDir, runId, info, "remediate-code");
|
|
809
831
|
},
|
|
810
832
|
});
|
|
833
|
+
// Partition-scoped drive (H2): run-level lifecycle belongs to the CALLER — the
|
|
834
|
+
// engine terminal is surfaced on the result but never persisted (a backend-only
|
|
835
|
+
// wall must not pause the whole run while the host share proceeds), and the
|
|
836
|
+
// deterministic merge is the caller's, run once over ALL partitions.
|
|
837
|
+
if (partitionIds !== null) {
|
|
838
|
+
// Freshly persisted status; falls back to the pre-drive snapshot only if
|
|
839
|
+
// state.json vanished mid-drive (review h2c2 F4 — documented, not silent).
|
|
840
|
+
const current = await new StateStore(artifactsDir).loadState();
|
|
841
|
+
return {
|
|
842
|
+
nodes: nodeOutcomes,
|
|
843
|
+
rebuilds: Math.max(0, levels.length - 1),
|
|
844
|
+
state_status: (current ?? state).status,
|
|
845
|
+
exhausted_pool_ids: driven.exhaustedPoolIds,
|
|
846
|
+
...(driven.terminal ? { terminal: driven.terminal } : {}),
|
|
847
|
+
};
|
|
848
|
+
}
|
|
811
849
|
// Piece D — persist the rolling engine's partial-completion terminal onto state
|
|
812
850
|
// BEFORE the merge, so the merge can SKIP-block the quota_paused stranded nodes
|
|
813
851
|
// (their worker rate-limited → no result file, but they must stay PENDING for a
|
|
@@ -829,6 +867,8 @@ export async function driveRollingImplementDispatch(options) {
|
|
|
829
867
|
nodes: nodeOutcomes,
|
|
830
868
|
rebuilds: Math.max(0, levels.length - 1),
|
|
831
869
|
state_status: merged.status,
|
|
870
|
+
exhausted_pool_ids: driven.exhaustedPoolIds,
|
|
871
|
+
...(driven.terminal ? { terminal: driven.terminal } : {}),
|
|
832
872
|
};
|
|
833
873
|
}
|
|
834
874
|
/**
|
|
@@ -838,85 +878,6 @@ export async function driveRollingImplementDispatch(options) {
|
|
|
838
878
|
* uses (`driveRollingDispatch`'s `() => 2000`), keeping the two paths consistent.
|
|
839
879
|
*/
|
|
840
880
|
const HYBRID_NODE_TOKEN_ESTIMATE = 2000;
|
|
841
|
-
/**
|
|
842
|
-
* Run the A-8 coordinator's IN-PROCESS partition this cycle. Each node was already
|
|
843
|
-
* claimed by the coordinator and assigned to a backend pool (NIM / codex / …), so it
|
|
844
|
-
* is launched on THAT pool's provider — binding the slot's providerName to the
|
|
845
|
-
* per-node assignment is what routes a node to its assigned backend — cwd-confined to
|
|
846
|
-
* its worktree, through the shared `executeNodeInWorktree` lifecycle (commit → verify
|
|
847
|
-
* → write-scope → merge); then the coordinator's claim is released.
|
|
848
|
-
*
|
|
849
|
-
* Nodes run concurrently: the coordinator's proactive split already bounded the
|
|
850
|
-
* partition to the backend pools' capacity, so the partition size IS the safe
|
|
851
|
-
* concurrency. The host partition runs in parallel via the host-subagent driver;
|
|
852
|
-
* both write accept-outcome sidecars the run-level `mergeImplementResults` reconciles.
|
|
853
|
-
*
|
|
854
|
-
* Run-once (no in-pass re-queue): a node whose worker rate-limits or errors is not
|
|
855
|
-
* merged (its worktree drops) and is routed to triage by the deterministic merge —
|
|
856
|
-
* bounded, never a livelock. (Cross-cycle settled-pool re-balancing is a follow-up.)
|
|
857
|
-
*/
|
|
858
|
-
export async function executeInProcessPartition(params) {
|
|
859
|
-
const { root, artifactsDir, runId, sessionConfig, partition, plan, coordinator, registry } = params;
|
|
860
|
-
if (partition.length === 0)
|
|
861
|
-
return { nodes: [] };
|
|
862
|
-
const allBlockScopes = blockScopesFromPlan(plan);
|
|
863
|
-
const withBlockId = plan.items.filter((i) => typeof i.block_id === "string");
|
|
864
|
-
const promptPathByBlock = new Map(withBlockId.map((i) => [i.block_id, i.prompt_path]));
|
|
865
|
-
const referencedFilesByBlock = new Map(withBlockId.map((i) => [i.block_id, i.access?.read_paths ?? []]));
|
|
866
|
-
const resultPathByBlock = new Map(withBlockId.map((i) => [i.block_id, i.result_path]));
|
|
867
|
-
const state = await new StateStore(artifactsDir).loadState();
|
|
868
|
-
const blockById = new Map((state?.plan?.blocks ?? []).map((b) => [b.block_id, b]));
|
|
869
|
-
const dispatchNode = params.dispatchNode ??
|
|
870
|
-
makeProviderNodeDispatcher({
|
|
871
|
-
root,
|
|
872
|
-
artifactsDir,
|
|
873
|
-
runId,
|
|
874
|
-
sessionConfig,
|
|
875
|
-
promptPathByBlock,
|
|
876
|
-
referencedFilesByBlock,
|
|
877
|
-
sourceByPoolId: params.sourceByPoolId,
|
|
878
|
-
});
|
|
879
|
-
const nodes = await Promise.all(partition.map(async (a) => {
|
|
880
|
-
// `a` IS the coordinator's NodeAssignment — release it directly on terminal.
|
|
881
|
-
const resultPath = resultPathByBlock.get(a.nodeId);
|
|
882
|
-
if (!resultPath) {
|
|
883
|
-
// No prepared prompt/result for this node — release + mark error (the merge
|
|
884
|
-
// routes it to triage); never silently drop a claimed node.
|
|
885
|
-
await coordinator.release(a);
|
|
886
|
-
return { block_id: a.nodeId, outcome: "error", verify_passed: false, merged: false };
|
|
887
|
-
}
|
|
888
|
-
const block = blockById.get(a.nodeId) ?? { block_id: a.nodeId };
|
|
889
|
-
const slot = {
|
|
890
|
-
providerName: a.providerName,
|
|
891
|
-
hostModel: a.hostModel,
|
|
892
|
-
poolId: a.poolId,
|
|
893
|
-
};
|
|
894
|
-
const { accept } = await executeNodeInWorktree({
|
|
895
|
-
block,
|
|
896
|
-
slot,
|
|
897
|
-
root,
|
|
898
|
-
artifactsDir,
|
|
899
|
-
runId,
|
|
900
|
-
resultPath,
|
|
901
|
-
seedPaths: declaredPathsFromPlan(plan, a.nodeId),
|
|
902
|
-
allBlockScopes,
|
|
903
|
-
additionalVerifyCommands: state ? targetedCommandsForBlock(state, a.nodeId) : [],
|
|
904
|
-
dispatchNode,
|
|
905
|
-
// Merge-time ownership gate (OD3 layer 2): the SAME lease the coordinator
|
|
906
|
-
// claimed `a` under. Omitted when the caller didn't supply a registry.
|
|
907
|
-
...(registry ? { ownership: { registry, nodeId: a.nodeId, ownerToken: a.ownerToken } } : {}),
|
|
908
|
-
});
|
|
909
|
-
// Run-once → terminal; free the coordinator claim (token-checked).
|
|
910
|
-
await coordinator.release(a);
|
|
911
|
-
return {
|
|
912
|
-
block_id: a.nodeId,
|
|
913
|
-
outcome: accept.outcome,
|
|
914
|
-
verify_passed: accept.verifyPassed,
|
|
915
|
-
merged: accept.merged,
|
|
916
|
-
};
|
|
917
|
-
}));
|
|
918
|
-
return { nodes };
|
|
919
|
-
}
|
|
920
881
|
// ---------------------------------------------------------------------------
|
|
921
882
|
// Tool-owned final completion gate (INV-RS-10) + coarse re-block (INV-RS-09)
|
|
922
883
|
// ---------------------------------------------------------------------------
|
|
@@ -1225,80 +1186,23 @@ async function buildImplementDispatchStep(ctx) {
|
|
|
1225
1186
|
hostModels: resolvedHostModels,
|
|
1226
1187
|
hostModelId: resolvedHostModelId,
|
|
1227
1188
|
};
|
|
1228
|
-
//
|
|
1229
|
-
//
|
|
1230
|
-
//
|
|
1231
|
-
//
|
|
1232
|
-
//
|
|
1233
|
-
//
|
|
1234
|
-
//
|
|
1235
|
-
//
|
|
1236
|
-
//
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
// primary backend provider (one account ⇒ host self-drives as a single pool),
|
|
1245
|
-
// else host + demoted-source pools double-book that one meter.
|
|
1246
|
-
const demoteBackendToSource = shouldDemotePrimaryInProcess({
|
|
1247
|
-
sessionConfig: sessionConfigImpl,
|
|
1248
|
-
hostCanDispatch: canDispatchImpl,
|
|
1249
|
-
});
|
|
1250
|
-
if (rollingEngineEnabled &&
|
|
1251
|
-
!demoteBackendToSource &&
|
|
1252
|
-
resolvesToInProcessDispatchProvider(sessionConfigImpl)) {
|
|
1253
|
-
const driven = await driveRollingImplementDispatch({
|
|
1254
|
-
root,
|
|
1255
|
-
artifactsDir,
|
|
1256
|
-
runId,
|
|
1257
|
-
sessionConfig: sessionConfigImpl ?? null,
|
|
1258
|
-
// Per-node verify (targeted_commands) owns each node's build/test; an
|
|
1259
|
-
// inter-level "shared surface" rebuild is a monorepo-self-remediation concern
|
|
1260
|
-
// the host-driven paths handle, not a generic target-repo step → no-op here.
|
|
1261
|
-
rebuildSharedBetweenLevels: async () => { },
|
|
1262
|
-
waveOptions: {
|
|
1263
|
-
hostMaxConcurrent: resolvedHostMaxConcurrent,
|
|
1264
|
-
hostContextTokens: resolvedHostContextTokens,
|
|
1265
|
-
hostOutputTokens: resolvedHostOutputTokens,
|
|
1266
|
-
hostModels: resolvedHostModels,
|
|
1267
|
-
hostModelId: resolvedHostModelId,
|
|
1268
|
-
},
|
|
1189
|
+
// H2+H4 collapse: ONE fan-out over the eligible pool set. `buildConfirmedPools`
|
|
1190
|
+
// folds the configured primary in-process backend in as a source pool
|
|
1191
|
+
// UNCONDITIONALLY (no demote flag; command-shaped primaries included under
|
|
1192
|
+
// remediate's policy — plan D3) and includes the conversation host as a member
|
|
1193
|
+
// pool iff it can dispatch subagents. Headless is the degenerate "no host pool
|
|
1194
|
+
// in the set" case: the engine drives the whole frontier itself. The same-agent
|
|
1195
|
+
// case (conversation host IS the primary backend) is the shared cross-class
|
|
1196
|
+
// dedup's D1 collision rule — the engine/source pool survives, so one account is
|
|
1197
|
+
// never double-booked across a host pool and a folded source.
|
|
1198
|
+
if (rollingEngineEnabled) {
|
|
1199
|
+
// D5: the host-session quota key follows the DRIVER identity — an in-process
|
|
1200
|
+
// worker primary keys to the conversation host; an explicit IDE/host provider
|
|
1201
|
+
// passes through verbatim (never re-keyed to the literal claude-code, the
|
|
1202
|
+
// founding-bug misattribution class [[capability-is-per-auditor-not-per-audit]]).
|
|
1203
|
+
const hybridProviderName = resolveHostDispatchProviderName(sessionConfigImpl, {
|
|
1204
|
+
commandWorkers: true,
|
|
1269
1205
|
});
|
|
1270
|
-
// null = no eligible pending work this pass; the engine merges internally once
|
|
1271
|
-
// it has run, so only the empty-frontier case needs a merge here. Either way the
|
|
1272
|
-
// implement frontier is resolved — transition on the freshly-merged state so the
|
|
1273
|
-
// engine re-scans (triage / closing) without recursion.
|
|
1274
|
-
if (driven === null) {
|
|
1275
|
-
const merged = await mergeImplementResults({ root, artifactsDir }, runId);
|
|
1276
|
-
return { kind: "transition", state: merged };
|
|
1277
|
-
}
|
|
1278
|
-
return { kind: "transition", state: await store.loadState() };
|
|
1279
|
-
}
|
|
1280
|
-
if (rollingEngineEnabled && canDispatchImpl) {
|
|
1281
|
-
// A-8 hybrid spill: when an in-process backend pool is ALSO confirmed (a
|
|
1282
|
-
// configured NIM/openai-compatible endpoint alongside the conversation host),
|
|
1283
|
-
// split the eligible frontier across BOTH pool classes via the shared
|
|
1284
|
-
// HybridSpillCoordinator (single claimant, proactive capacity split) — the
|
|
1285
|
-
// orchestrator runs the in-process partition THIS cycle while the host spawns
|
|
1286
|
-
// subagents for its partition. Pure host-subagent dispatch falls out when no
|
|
1287
|
-
// backend pool is confirmed (the coordinator has nothing to split against).
|
|
1288
|
-
// Retained host-session source for the hybrid pool-sizing pre-wall throttle
|
|
1289
|
-
// (this branch previously sized pools with no account-wall awareness at all,
|
|
1290
|
-
// unlike the primary path above). This branch has its own already-working,
|
|
1291
|
-
// bounded rate-limited/settle mechanism below (DC-4) rather than routing
|
|
1292
|
-
// through HostSessionQuotaSource.recordLimit/isEscalated, so onEscalation is
|
|
1293
|
-
// unused here — the source only feeds buildConfirmedPools' sizing.
|
|
1294
|
-
// Defect-1: the host-session quota key must follow the CONVERSATION HOST, not a
|
|
1295
|
-
// demoted backend — key it to the auto-detected conversation host (B1: codex
|
|
1296
|
-
// when inside a Codex session, else claude-code; --host-provider overrides)
|
|
1297
|
-
// when the configured primary is a demotable backend that this attended run is
|
|
1298
|
-
// fanning out onto as a source.
|
|
1299
|
-
const hybridProviderName = demoteBackendToSource
|
|
1300
|
-
? resolveConversationHostProvider({ sessionConfig: sessionConfigImpl })
|
|
1301
|
-
: resolveHostProviderName(sessionConfigImpl);
|
|
1302
1206
|
const hybridHostSessionModelKey = buildProviderModelKey(hybridProviderName, sessionConfigImpl
|
|
1303
1207
|
?.block_quota?.host_model ??
|
|
1304
1208
|
resolvedHostModelId ??
|
|
@@ -1314,187 +1218,248 @@ async function buildImplementDispatchStep(ctx) {
|
|
|
1314
1218
|
hostModels: resolvedHostModels,
|
|
1315
1219
|
hostModelId: resolvedHostModelId,
|
|
1316
1220
|
hostSession: hybridHostSession,
|
|
1317
|
-
//
|
|
1318
|
-
|
|
1319
|
-
// source-pool set so the split fans across host + backend + NIM.
|
|
1320
|
-
demotePrimaryInProcess: demoteBackendToSource,
|
|
1221
|
+
// Attendance IS pool-set membership: headless ⇒ no host pool in the set.
|
|
1222
|
+
hostCanDispatch: canDispatchImpl,
|
|
1321
1223
|
// The operator's Gate-0 exclusions, applied as a set-difference over freshly
|
|
1322
1224
|
// gathered reach; self-spawn-blocked recomputed against THIS process's env.
|
|
1323
1225
|
excludedBackends: resolveDispatchExclusion(await readConfirmedDispatchPolicy(root)),
|
|
1324
1226
|
});
|
|
1325
1227
|
const backendPools = confirmedPools.filter(isInProcessPool);
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
//
|
|
1329
|
-
//
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1228
|
+
if (!canDispatchImpl) {
|
|
1229
|
+
// Headless: no host pool in the eligible set — when any dispatchable
|
|
1230
|
+
// backend pool is confirmed, the engine drives the FULL rolling implement
|
|
1231
|
+
// dispatch itself (no blocksOverride: the driver's full path owns terminal
|
|
1232
|
+
// persistence + the final merge), cwd-confined to each node's worktree,
|
|
1233
|
+
// sharing the same `acceptNodeWorktree` core as the host-subagent driver.
|
|
1234
|
+
// No pool at all ⇒ fall through to the sequential host step below.
|
|
1235
|
+
if (backendPools.length > 0) {
|
|
1236
|
+
const driven = await driveRollingImplementDispatch({
|
|
1237
|
+
root,
|
|
1238
|
+
artifactsDir,
|
|
1239
|
+
runId,
|
|
1240
|
+
sessionConfig: sessionConfigImpl ?? null,
|
|
1241
|
+
// Per-node verify (targeted_commands) owns each node's build/test; an
|
|
1242
|
+
// inter-level "shared surface" rebuild is a monorepo-self-remediation concern
|
|
1243
|
+
// the host-driven paths handle, not a generic target-repo step → no-op here.
|
|
1244
|
+
rebuildSharedBetweenLevels: async () => { },
|
|
1245
|
+
waveOptions: {
|
|
1246
|
+
hostMaxConcurrent: resolvedHostMaxConcurrent,
|
|
1247
|
+
hostContextTokens: resolvedHostContextTokens,
|
|
1248
|
+
hostOutputTokens: resolvedHostOutputTokens,
|
|
1249
|
+
hostModels: resolvedHostModels,
|
|
1250
|
+
hostModelId: resolvedHostModelId,
|
|
1251
|
+
},
|
|
1252
|
+
// The eligible pool set built above (source pools only — headless),
|
|
1253
|
+
// so the drive routes across every confirmed backend pool.
|
|
1254
|
+
poolsOverride: backendPools,
|
|
1255
|
+
});
|
|
1256
|
+
// null = no eligible pending work this pass; the engine merges internally once
|
|
1257
|
+
// it has run, so only the empty-frontier case needs a merge here. Either way the
|
|
1258
|
+
// implement frontier is resolved — transition on the freshly-merged state so the
|
|
1259
|
+
// engine re-scans (triage / closing) without recursion.
|
|
1260
|
+
if (driven === null) {
|
|
1261
|
+
const merged = await mergeImplementResults({ root, artifactsDir }, runId);
|
|
1262
|
+
return { kind: "transition", state: merged };
|
|
1263
|
+
}
|
|
1264
|
+
return { kind: "transition", state: await store.loadState() };
|
|
1340
1265
|
}
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
//
|
|
1344
|
-
//
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
//
|
|
1348
|
-
//
|
|
1349
|
-
//
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
registry
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
.
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1266
|
+
}
|
|
1267
|
+
if (canDispatchImpl) {
|
|
1268
|
+
// A-8 hybrid spill: when an in-process backend pool is ALSO confirmed (a
|
|
1269
|
+
// configured NIM/openai-compatible endpoint alongside the conversation host),
|
|
1270
|
+
// split the eligible frontier across BOTH pool classes via the shared
|
|
1271
|
+
// HybridSpillCoordinator (single claimant, proactive capacity split) — the
|
|
1272
|
+
// orchestrator runs the in-process partition THIS cycle while the host spawns
|
|
1273
|
+
// subagents for its partition. Pure host-subagent dispatch falls out when no
|
|
1274
|
+
// backend pool is confirmed (the coordinator has nothing to split against).
|
|
1275
|
+
// The hybrid host-session source above fed `buildConfirmedPools`' pool-sizing
|
|
1276
|
+
// pre-wall throttle; this branch has its own bounded rate-limited/settle
|
|
1277
|
+
// mechanism below (DC-4) rather than routing through
|
|
1278
|
+
// HostSessionQuotaSource.recordLimit/isEscalated.
|
|
1279
|
+
let rolling;
|
|
1280
|
+
if (backendPools.length > 0) {
|
|
1281
|
+
// Prepare the frontier ONCE (worktree-rooted prompts) so the coordinator
|
|
1282
|
+
// split, the partition drive (via planOverride), and the host driver all
|
|
1283
|
+
// read the same plan — the partition drive deliberately does NOT re-prepare
|
|
1284
|
+
// (see DriveRollingImplementDispatchOptions.planOverride for the lease
|
|
1285
|
+
// overlap this shape carries and why a re-prepare would clobber the
|
|
1286
|
+
// host-share dispatch-quota).
|
|
1287
|
+
const plan = await prepareImplementDispatch({ root, artifactsDir }, runId, undefined, {
|
|
1288
|
+
...waveOptsImpl,
|
|
1289
|
+
worktreeRootedPrompts: true,
|
|
1290
|
+
});
|
|
1291
|
+
const frontier = plan.items
|
|
1292
|
+
.filter((i) => typeof i.block_id === "string")
|
|
1293
|
+
.map((i) => ({ id: i.block_id, estimatedTokens: HYBRID_NODE_TOKEN_ESTIMATE }));
|
|
1294
|
+
if (frontier.length === 0) {
|
|
1295
|
+
const merged = await mergeImplementResults({ root, artifactsDir }, runId);
|
|
1296
|
+
return { kind: "transition", state: merged };
|
|
1297
|
+
}
|
|
1298
|
+
// One coordinator over the shared claim registry splits + claims each node to
|
|
1299
|
+
// exactly one pool. DC-4: the settled set is cross-cycle (persisted) — a backend
|
|
1300
|
+
// pool that exhausted on a prior cycle is excluded here, so its work falls to the
|
|
1301
|
+
// host-subagent pool instead of re-looping on a dead backend.
|
|
1302
|
+
const settledPath = nodeSettledPoolsPath(artifactsDir, runId);
|
|
1303
|
+
const settled = await readSettledPools(settledPath);
|
|
1304
|
+
// Hoisted so the coordinator's claim registry and the ownership gate's
|
|
1305
|
+
// heartbeat probe are the SAME file-backed instance — never independently
|
|
1306
|
+
// re-derived by path (the coordinator's own `claimRegistry` field is private).
|
|
1307
|
+
const hybridClaimRegistry = nodeClaimRegistry(artifactsDir, runId);
|
|
1308
|
+
const partition = await planHybridDispatch({
|
|
1309
|
+
frontier,
|
|
1310
|
+
pools: confirmedPools,
|
|
1311
|
+
sessionConfig: sessionConfigImpl ?? {},
|
|
1312
|
+
claimRegistry: hybridClaimRegistry,
|
|
1313
|
+
readSettled: () => settled,
|
|
1314
|
+
onSettle: async (id) => {
|
|
1315
|
+
settled.add(id);
|
|
1316
|
+
await addSettledPool(settledPath, id);
|
|
1317
|
+
},
|
|
1318
|
+
isInProcess: isInProcessPool,
|
|
1319
|
+
});
|
|
1320
|
+
// Drive the in-process partition through the ROLLING ENGINE (H2 plan D2 —
|
|
1321
|
+
// `executeInProcessPartition`'s direct Promise.all executor is deleted; one
|
|
1322
|
+
// core, one driver): each node runs on a coordinator-claimed backend pool,
|
|
1323
|
+
// launched FROM that pool's source config, with the engine's full hook set
|
|
1324
|
+
// live (413 → packet_too_large re-queue + friction, verbatim quota harvest,
|
|
1325
|
+
// cost-drift / credit-exhaustion / model-unavailable capture) instead of the
|
|
1326
|
+
// hand-replicated friction blocks this replaces. The coordinator's claims
|
|
1327
|
+
// are ADOPTED (claimOwnerTokens) — same registry, no self-collision — and
|
|
1328
|
+
// run-level lifecycle stays HERE: the drive is partition-scoped
|
|
1329
|
+
// (blocksOverride), so a backend-only wall never persists a run terminal and
|
|
1330
|
+
// the final merge remains this caller's.
|
|
1331
|
+
if (partition.inProcess.length > 0) {
|
|
1332
|
+
// DC-4 (review h2c3 F1): the engine's per-packet selection binds freely
|
|
1333
|
+
// across `poolsOverride`, so a pool settled on a PRIOR cycle must be
|
|
1334
|
+
// filtered out here — the coordinator's claim walk already excluded it,
|
|
1335
|
+
// and re-offering it would re-die on the same dead pool every cycle.
|
|
1336
|
+
const liveBackendPools = backendPools.filter((p) => !settled.has(p.id));
|
|
1337
|
+
const driven = await driveRollingImplementDispatch({
|
|
1338
|
+
root,
|
|
1339
|
+
artifactsDir,
|
|
1340
|
+
runId,
|
|
1341
|
+
sessionConfig: sessionConfigImpl ?? null,
|
|
1342
|
+
rebuildSharedBetweenLevels: async () => { },
|
|
1343
|
+
waveOptions: {
|
|
1344
|
+
hostMaxConcurrent: resolvedHostMaxConcurrent,
|
|
1345
|
+
hostContextTokens: resolvedHostContextTokens,
|
|
1346
|
+
hostOutputTokens: resolvedHostOutputTokens,
|
|
1347
|
+
hostModels: resolvedHostModels,
|
|
1348
|
+
hostModelId: resolvedHostModelId,
|
|
1349
|
+
},
|
|
1350
|
+
blocksOverride: partition.inProcess.map((a) => a.nodeId),
|
|
1351
|
+
poolsOverride: liveBackendPools,
|
|
1352
|
+
planOverride: plan,
|
|
1353
|
+
claimOwnerTokens: new Map(partition.inProcess.map((a) => [a.nodeId, a.ownerToken])),
|
|
1354
|
+
});
|
|
1355
|
+
// Free any coordinator claim still held (a non-terminal outcome keeps its
|
|
1356
|
+
// claim through the drive); terminal accepts already released in-driver,
|
|
1357
|
+
// so this is a token-checked no-op for them.
|
|
1358
|
+
for (const a of partition.inProcess) {
|
|
1359
|
+
await partition.coordinator.release(a);
|
|
1360
|
+
}
|
|
1361
|
+
// DC-4 cross-cycle settle (D2 iii — PRESERVED across the engine
|
|
1362
|
+
// migration): a backend pool whose node rate-limited, credit-exhausted,
|
|
1363
|
+
// went model-unavailable, or died quota-unclassified → settle it
|
|
1364
|
+
// (cross-cycle) so the next cycle routes its share to the host pool. This
|
|
1365
|
+
// partition spans multiple cycles, each with a fresh in-process dispatcher
|
|
1366
|
+
// (fresh in-memory exhaustedPoolIds AND pausedPoolResetAt), so the
|
|
1367
|
+
// engine's reversible pause evaporates at the cycle boundary — the BROAD
|
|
1368
|
+
// `isPoolSettlingOutcome` predicate (incl. reset-bearing 429 +
|
|
1369
|
+
// quota_unclassified; see settledPools.ts for the divergence rationale)
|
|
1370
|
+
// therefore applies over the drive's real per-node pool attribution, plus
|
|
1371
|
+
// the engine's own terminal-exhaustion set. `packet_too_large` is
|
|
1372
|
+
// deliberately NOT a settle trigger (step D): a 413 is a per-(node,pool)
|
|
1373
|
+
// sizing fact, surfaced via the engine's onPacketTooLarge friction hook.
|
|
1374
|
+
// `driven` is null only when the plan/state vanished mid-cycle (nothing was
|
|
1375
|
+
// dispatched) — no outcomes ⇒ nothing to settle.
|
|
1376
|
+
const settlingPoolIds = new Set((driven?.nodes ?? [])
|
|
1377
|
+
.filter((n) => isPoolSettlingOutcome(n.outcome))
|
|
1378
|
+
.map((n) => n.pool_id));
|
|
1379
|
+
for (const poolId of driven?.exhausted_pool_ids ?? []) {
|
|
1380
|
+
settlingPoolIds.add(poolId);
|
|
1381
|
+
}
|
|
1382
|
+
const liveBackendPoolIds = new Set(liveBackendPools.map((p) => p.id));
|
|
1383
|
+
for (const poolId of settlingPoolIds) {
|
|
1384
|
+
if (liveBackendPoolIds.has(poolId)) {
|
|
1385
|
+
await partition.coordinator.settlePool(poolId);
|
|
1386
|
+
// The settle FACT itself is reviewable friction (review h2c3 F2):
|
|
1387
|
+
// the engine hooks capture the per-death evidence (verbatim 429
|
|
1388
|
+
// text, credit exhaustion, 404), but reset-bearing rate limits have
|
|
1389
|
+
// no engine hook, and no hook records "this pool is now settled for
|
|
1390
|
+
// the run". One record per (run, pool) via the dedupe chokepoint.
|
|
1391
|
+
void captureStepBoundaryFriction(artifactsDir, runId, {
|
|
1392
|
+
eventType: "quota_escalation",
|
|
1393
|
+
discriminator: `pool-settled:${poolId}`,
|
|
1394
|
+
note: "Hybrid in-process backend pool settled for this run (rate-limited / credit-exhausted / model-unavailable / quota-unclassified); its remaining share routes to the host on later cycles.",
|
|
1395
|
+
severity: "high",
|
|
1396
|
+
category: "trap",
|
|
1397
|
+
area: "dispatch/quota",
|
|
1398
|
+
}, "remediate-code");
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
}
|
|
1402
|
+
// The backend carried the whole batch (or every host node was contested by a
|
|
1403
|
+
// peer driver) → nothing for the host this cycle; merge what landed + transition.
|
|
1404
|
+
if (partition.host.length === 0) {
|
|
1405
|
+
const merged = await mergeImplementResults({ root, artifactsDir }, runId);
|
|
1406
|
+
return { kind: "transition", state: merged };
|
|
1407
|
+
}
|
|
1408
|
+
// Hand the host partition (pre-claimed) to the host-subagent driver.
|
|
1409
|
+
rolling = await prepareHostRollingDispatch({ root, artifactsDir }, runId, waveOptsImpl, {
|
|
1410
|
+
plan,
|
|
1411
|
+
partition: partition.host.map((a) => ({ block_id: a.nodeId, ownerToken: a.ownerToken })),
|
|
1412
|
+
});
|
|
1405
1413
|
}
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
// {eventType, runId, discriminator}, so a chronically-exhausted backend pool
|
|
1409
|
-
// re-hitting the same block_id across cycles collapses to one record.
|
|
1410
|
-
for (const blockId of settledOutcomeBlocks) {
|
|
1411
|
-
void captureStepBoundaryFriction(artifactsDir, runId, {
|
|
1412
|
-
eventType: "quota_escalation",
|
|
1413
|
-
discriminator: blockId,
|
|
1414
|
-
note: "A-8 hybrid in-process node rate-limited / credit-exhausted / model-unavailable / quota-unclassified; its backend pool was settled for this run.",
|
|
1415
|
-
severity: "high",
|
|
1416
|
-
category: "trap",
|
|
1417
|
-
area: "dispatch/quota",
|
|
1418
|
-
}, "remediate-code");
|
|
1414
|
+
else {
|
|
1415
|
+
rolling = await prepareHostRollingDispatch({ root, artifactsDir }, runId, waveOptsImpl);
|
|
1419
1416
|
}
|
|
1420
|
-
//
|
|
1421
|
-
//
|
|
1422
|
-
//
|
|
1423
|
-
//
|
|
1424
|
-
//
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
area: "dispatch/quota",
|
|
1435
|
-
}, "remediate-code");
|
|
1417
|
+
// Increment B residual (a): the hybrid host-subagent driver hit the cooldown wall
|
|
1418
|
+
// (admission over-granted the throttled set). Reconcile the reserved leases (the
|
|
1419
|
+
// pause skips the merge that would) and set the resumable `quota_paused` terminal
|
|
1420
|
+
// so the `partial_terminal` obligation emits it this same advance; the ungranted
|
|
1421
|
+
// nodes stay PENDING and re-dispatch on resume. The in-process partition (above)
|
|
1422
|
+
// already ran on its own cooldown-safe pool, so its work still lands.
|
|
1423
|
+
if (rolling.wall) {
|
|
1424
|
+
await reconcileAdmissionLeasesFromQuotaFile(rolling.quotaPath);
|
|
1425
|
+
const paused = await store.loadState();
|
|
1426
|
+
if (paused) {
|
|
1427
|
+
paused.partial_completion_terminal = buildQuotaPausedTerminal(rolling.wall.strandedBlockIds, rolling.wall.detected.earliestResetAt);
|
|
1428
|
+
await store.saveState(paused);
|
|
1429
|
+
}
|
|
1430
|
+
return { kind: "transition", state: paused };
|
|
1436
1431
|
}
|
|
1437
|
-
//
|
|
1438
|
-
//
|
|
1439
|
-
if (
|
|
1440
|
-
|
|
1441
|
-
return { kind: "transition", state:
|
|
1432
|
+
// Everything eligible may already be done/skipped — fold straight to merge
|
|
1433
|
+
// rather than emitting a dispatch step with zero nodes.
|
|
1434
|
+
if (rolling.session.frontier.length === 0) {
|
|
1435
|
+
await mergeImplementResults({ root, artifactsDir }, runId);
|
|
1436
|
+
return { kind: "transition", state: await store.loadState() };
|
|
1442
1437
|
}
|
|
1443
|
-
//
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1438
|
+
// S-BROKER-WIRING: pick the dispatch DRIVER (delegate the rolling loop to a
|
|
1439
|
+
// dedicated dispatcher subagent vs. drive it from the top host) off the
|
|
1440
|
+
// single classification + the live frontier/slot count — not host prose.
|
|
1441
|
+
const hostProvider = resolveHostProviderName(sessionConfigImpl);
|
|
1442
|
+
const driverSelection = selectDispatchDriver({
|
|
1443
|
+
classification: classifyProvider(hostProvider),
|
|
1444
|
+
eligibleItemCount: rolling.session.frontier.length,
|
|
1445
|
+
// The granted set's size IS the instantaneous admission width — there is no
|
|
1446
|
+
// separate concurrency number. The whole granted set runs at once, so the
|
|
1447
|
+
// driver-selection "slots" is the granted-set size.
|
|
1448
|
+
slots: rolling.session.frontier.length,
|
|
1447
1449
|
});
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
if (paused) {
|
|
1462
|
-
paused.partial_completion_terminal = buildQuotaPausedTerminal(rolling.wall.strandedBlockIds, rolling.wall.detected.earliestResetAt);
|
|
1463
|
-
await store.saveState(paused);
|
|
1464
|
-
}
|
|
1465
|
-
return { kind: "transition", state: paused };
|
|
1466
|
-
}
|
|
1467
|
-
// Everything eligible may already be done/skipped — fold straight to merge
|
|
1468
|
-
// rather than emitting a dispatch step with zero nodes.
|
|
1469
|
-
if (rolling.session.frontier.length === 0) {
|
|
1470
|
-
await mergeImplementResults({ root, artifactsDir }, runId);
|
|
1471
|
-
return { kind: "transition", state: await store.loadState() };
|
|
1472
|
-
}
|
|
1473
|
-
// S-BROKER-WIRING: pick the dispatch DRIVER (delegate the rolling loop to a
|
|
1474
|
-
// dedicated dispatcher subagent vs. drive it from the top host) off the
|
|
1475
|
-
// single classification + the live frontier/slot count — not host prose.
|
|
1476
|
-
const hostProvider = resolveHostProviderName(sessionConfigImpl);
|
|
1477
|
-
const driverSelection = selectDispatchDriver({
|
|
1478
|
-
classification: classifyProvider(hostProvider),
|
|
1479
|
-
eligibleItemCount: rolling.session.frontier.length,
|
|
1480
|
-
// The granted set's size IS the instantaneous admission width — there is no
|
|
1481
|
-
// separate concurrency number. The whole granted set runs at once, so the
|
|
1482
|
-
// driver-selection "slots" is the granted-set size.
|
|
1483
|
-
slots: rolling.session.frontier.length,
|
|
1484
|
-
});
|
|
1485
|
-
const rollMerge = loaderCommand(`merge-implement-results --run-id ${runId}`);
|
|
1486
|
-
const rollNext = loaderCommand("next-step");
|
|
1487
|
-
const acceptCmd = loaderCommand(`accept-node --id <BLOCK_ID> --run-id ${runId}`);
|
|
1488
|
-
const nodeLines = rolling.initial
|
|
1489
|
-
.map((n) => `- \`${n.block_id}\` — prompt: \`${n.prompt_path}\` — worktree (subagent cwd): \`${n.worktree_root}\``)
|
|
1490
|
-
.join("\n");
|
|
1491
|
-
return { kind: "emit", step: await writeCurrentStep({
|
|
1492
|
-
stepKind: "dispatch_implement_rolling",
|
|
1493
|
-
status: "ready",
|
|
1494
|
-
runId,
|
|
1495
|
-
repoRoot: root,
|
|
1496
|
-
artifactsDir,
|
|
1497
|
-
prompt: `
|
|
1450
|
+
const rollMerge = loaderCommand(`merge-implement-results --run-id ${runId}`);
|
|
1451
|
+
const rollNext = loaderCommand("next-step");
|
|
1452
|
+
const acceptCmd = loaderCommand(`accept-node --id <BLOCK_ID> --run-id ${runId}`);
|
|
1453
|
+
const nodeLines = rolling.initial
|
|
1454
|
+
.map((n) => `- \`${n.block_id}\` — prompt: \`${n.prompt_path}\` — worktree (subagent cwd): \`${n.worktree_root}\``)
|
|
1455
|
+
.join("\n");
|
|
1456
|
+
return { kind: "emit", step: await writeCurrentStep({
|
|
1457
|
+
stepKind: "dispatch_implement_rolling",
|
|
1458
|
+
status: "ready",
|
|
1459
|
+
runId,
|
|
1460
|
+
repoRoot: root,
|
|
1461
|
+
artifactsDir,
|
|
1462
|
+
prompt: `
|
|
1498
1463
|
# Dispatch Implementation Work (host-subagent rolling, worktree-isolated)
|
|
1499
1464
|
|
|
1500
1465
|
Each granted node runs in its OWN git worktree (hard isolation between nodes). The
|
|
@@ -1539,10 +1504,11 @@ ${DISPATCH_PROMPT_HANDOFF_NOTE}
|
|
|
1539
1504
|
|
|
1540
1505
|
${renderHostScratchNote(hostScratchDir(artifactsDir, runId))}
|
|
1541
1506
|
`,
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1507
|
+
allowedCommands: [acceptCmd, rollMerge, rollNext],
|
|
1508
|
+
stopCondition: "Stop after every node has been accepted (accept-node returns done), results merged, and next-step has been run.",
|
|
1509
|
+
artifactPaths: { dispatch_plan: rolling.planPath, dispatch_quota: rolling.quotaPath },
|
|
1510
|
+
}) };
|
|
1511
|
+
}
|
|
1546
1512
|
}
|
|
1547
1513
|
// Rolling per-node dispatch: prepare EVERY currently-eligible node (deps all
|
|
1548
1514
|
// verified-complete), never a single artificially-serialized block. There is
|