audit-tools 0.34.29 → 0.34.31

package/dist/remediate/steps/nextStep.js

@@ -3,7 +3,7 @@ import { existsSync, statSync } from "node:fs";
 import { mkdir, readFile, rename } from "node:fs/promises";
 import { dirname, join, resolve } from "node:path";
 import { StateStore } from "../state/store.js";
-import { readConfirmedDispatchPolicy, readConfirmedCapabilityRanks, resolveDispatchExclusion, readOptionalJsonFile, readValidatedRepoSessionIntent, stagedAndUntracked, writeJsonFile, writeTextFile, buildAuditDeliverablePair, formatValidationIssues, isRecord, withFsRetry, RunLogger, DISPATCH_PROMPT_HANDOFF_NOTE, renderHostScratchNote, hostScratchDir, renderQuotaCoverageNudge, renderTokenBudgetView, coerceJsonObjectArg, driveRolling, resolveLedgerBudgets, setQuotaStateDir, detectHostDispatchWall, admissionBlockedOnBudget, classifyEmptyGrantCause, reconcileAdmissionLeasesFromQuotaFile, buildQuotaPausedTerminal, interpretFreeFormIntent, advance, decideFrictionTriage, buildFrictionTriageBlock, planHybridDispatch, readSettledPools, addSettledPool, isPoolSettlingOutcome, isInProcessWorkerProvider, sourceByPoolId, classifyProvider, selectDispatchDriver, renderDispatchDriverInstruction, HostSessionQuotaSource, quotaPoolKey, captureStepBoundaryFriction, captureZeroCapacityFriction, captureCostDriftFriction, captureCreditExhaustionFriction, captureQuotaUnclassifiedFriction, captureModelUnavailableFriction, capturePacketTooLargeFriction, createDispatchDecisionLog, LENSES, SEVERITIES, resolveHostProviderName, resolveHostDispatchProviderName, resolveHostDispatchCapability as sharedResolveHostDispatchCapability, resolveAutonomousMode, resolveRollingEngineFlag, DEFAULT_CONTEXT_TOKENS } from "audit-tools/shared";
+import { discardOnSchemaVersionMismatch, readConfirmedDispatchPolicy, readConfirmedCapabilityRanks, resolveDispatchExclusion, readOptionalJsonFile, readValidatedRepoSessionIntent, stagedAndUntracked, writeJsonFile, writeTextFile, buildAuditDeliverablePair, formatValidationIssues, isRecord, withFsRetry, RunLogger, DISPATCH_PROMPT_HANDOFF_NOTE, renderHostScratchNote, hostScratchDir, renderQuotaCoverageNudge, renderTokenBudgetView, coerceJsonObjectArg, driveRolling, resolveLedgerBudgets, setQuotaStateDir, detectHostDispatchWall, admissionBlockedOnBudget, classifyEmptyGrantCause, reconcileAdmissionLeasesFromQuotaFile, buildQuotaPausedTerminal, interpretFreeFormIntent, advance, decideFrictionTriage, buildFrictionTriageBlock, planHybridDispatch, readSettledPools, addSettledPool, isPoolSettlingOutcome, isInProcessWorkerProvider, sourceByPoolId, classifyProvider, selectDispatchDriver, renderDispatchDriverInstruction, HostSessionQuotaSource, quotaPoolKey, captureStepBoundaryFriction, captureZeroCapacityFriction, captureCostDriftFriction, captureCreditExhaustionFriction, captureQuotaUnclassifiedFriction, captureModelUnavailableFriction, capturePacketTooLargeFriction, createDispatchDecisionLog, LENSES, SEVERITIES, resolveHostProviderName, resolveHostDispatchProviderName, resolveHostDispatchCapability as sharedResolveHostDispatchCapability, resolveAutonomousMode, resolveRollingEngineFlag, DEFAULT_CONTEXT_TOKENS } from "audit-tools/shared";
 import { readRemediationAccessMemory, computeBlockContinuityScores } from "../state/accessMemory.js";
 import { applyPlanPipeline, buildCoverageLedger } from "../phases/plan.js";
 import { groundExtractedFindings } from "../phases/grounding.js";
@@ -18,7 +18,7 @@ import { claimWithBackoff, withClaimHeartbeat } from "../../shared/quota/claimLe
 import { AUDIT_FINDINGS_FILENAME, AUDIT_REPORT_FILENAME, auditArtifactsDir, auditFindingsPath, auditReportPath, auditToolsDir, nodeClaimsPath, promotedAuditFindingsPath, promotedAuditReportPath, remediationArtifactsDir, } from "../../shared/io/auditToolsPaths.js";
 import { resolveRepoRoot } from "../../shared/io/repoRoot.js";
 import { writeCurrentStep } from "./stepWriter.js";
-import { dependencyVerifiedComplete } from "./stepUtils.js";
+import { dependencyAwaitingClarification, dependencyVerifiedComplete, } from "./stepUtils.js";
 import { isTerminalStatus, isVerifiedCompleteStatus, } from "../state/itemStatus.js";
 import { deduplicateCrossLensFindings, fixupBlocksAfterDedup, } from "../dedup/crossLensDedup.js";
 import { checkAffectedFileIntegrity } from "../utils/fileIntegrity.js";
@@ -26,7 +26,7 @@ import { resolveIntakeStep } from "./intakeResolver.js";
 import { runToolOwnedFinalGate, applyCoarseReblock, readFinalGateSidecar, writeFinalGateSidecar, } from "./finalGate.js";
 import { buildNextContractPipelineStep, shouldEnterContractPipeline, writePathASeedFromFindings, buildLeanExtractedPlan, } from "./contractPipeline.js";
 import { contractArtifactExists, contractPipelineDir, } from "../contractPipeline/artifactStore.js";
-import { buildReviewRequest, applyReviewResolution, isResolutionForRequest, } from "../review/reviewGate.js";
+import { buildReviewRequest, applyReviewResolution, isResolutionForRequest, REVIEW_REQUEST_SCHEMA_VERSION, } from "../review/reviewGate.js";
 import { buildAutonomousReviewDecision } from "../review/autonomousGate.js";
 import { runFindingFilterPass } from "../findingFilter.js";
 import { intakePaths, isIntakeReady, readIntakeArtifacts, resolveManifestSources, } from "../intake.js";
@@ -320,19 +320,35 @@ function implementableBlocks(state) {
         }));
 }
 /**
- * Pending nodes that are NOT eligible because at least one dependency did not
- * reach a verified-complete disposition (a prerequisite was skipped, blocked, or
- * is still pending). Once no eligible block remains, these are dead-ended: the
- * rolling scheduler marks them `blocked` (their upstream surface never landed)
- * rather than looping forever. Used by `handlePlanning` to make that transition
- * deterministic.
+ * Pending nodes that are genuinely DEAD-ENDED: at least one dependency did not
+ * reach a verified-complete disposition and never will (a prerequisite was
+ * skipped or blocked, or the edges are cyclic). Once no eligible block remains,
+ * these are marked `blocked` — their upstream surface never landed — rather than
+ * looping forever.
+ *
+ * A node held only by an UNANSWERED WORKER QUESTION is deliberately excluded
+ * (`dependencyAwaitingClarification`): since the clarification round is deferred
+ * to the end of the implement phase, such a node reaches this sweep while its
+ * answer is still outstanding, and blocking it would report "upstream failed"
+ * for what is really "awaiting an answer". It stays `pending` and is re-decided
+ * once the answer lands.
  */
 function blockedByUnsatisfiedDependency(state) {
     if (!state.plan || !state.items)
         return [];
     return state.plan.blocks.filter((block) => !dependencyVerifiedComplete(block, state) &&
+        !dependencyAwaitingClarification(block, state) &&
         block.items.some((findingId) => state.items?.[findingId]?.status === "pending"));
 }
+/**
+ * Whether any item is paused on a worker question that has not been answered yet.
+ * Drives the deferred clarification round (the `deferred_clarification`
+ * obligation): the question waits until the implement frontier drains, then is
+ * asked in one batched window.
+ */
+function hasUnansweredClarification(state) {
+    return Object.values(state.items ?? {}).some((it) => it.status === "needs_clarification");
+}
 // ---------------------------------------------------------------------------
 // Rolling per-node scheduler (CP-BLOCK-N-rolling-scheduler)
 // ---------------------------------------------------------------------------
@@ -495,8 +511,8 @@ export async function driveRollingDispatch(levels, options) {
     if (options.quotaStateDir) {
         setQuotaStateDir(options.quotaStateDir);
     }
-    const estimateTokens = options.estimateTokens ?? (() => 2000);
-    const scopeForBlock = options.scopeForBlock ?? ((b) => b.touched_files ?? []);
+    const estimateTokens = options.estimateTokens;
+    const scopeForBlock = options.scopeForBlock ?? ((b) => b.touched_files);
     const allBlocks = levels.flat();
     const blockById = new Map(allBlocks.map((b) => [b.block_id, b]));
     const hostSession = options.hostSession;
@@ -787,7 +803,7 @@ export async function driveRollingImplementDispatch(options) {
             artifactsDir,
             runId,
             resultPath,
-            seedPaths: block.touched_files ?? [],
+            seedPaths: block.touched_files,
             allBlockScopes,
             additionalVerifyCommands: targetedCommandsForBlock(state, block.block_id),
             dispatchNode,
@@ -819,12 +835,15 @@ export async function driveRollingImplementDispatch(options) {
     // the merge-time write-scope gate reads — for file-ownership-disjoint admission
     // (INV-SOO). A block with no plan scope falls back to its touched_files.
     const writePathsByBlock = new Map(allBlockScopes.map((s) => [s.block_id, s.write_paths]));
+    const tokensByBlock = new Map(plan.items
+        .filter((i) => typeof i.block_id === "string")
+        .map((i) => [i.block_id, i.estimated_input_tokens]));
     // Continuity bias (context-efficiency track, increment 2d): load the harvested
     // access-memory and reduce it to a per-block mass keyed on each block's declared
     // source surface (`touched_files` — the clean surface the harvest attributes to,
     // excluding the synthetic result-path in the dispatch write-scope). Absent on the
     // first pass (no merge yet) ⇒ empty map ⇒ pure block_id sub-wave ordering.
-    const continuityScores = computeBlockContinuityScores(await readRemediationAccessMemory(artifactsDir), levels.flat(), (block) => block.touched_files ?? []);
+    const continuityScores = computeBlockContinuityScores(await readRemediationAccessMemory(artifactsDir), levels.flat(), (block) => block.touched_files);
     const driven = await driveRollingDispatch(levels, {
         confirmedPools,
         sessionConfig: options.sessionConfig ?? {},
@@ -834,7 +853,19 @@ export async function driveRollingImplementDispatch(options) {
         root,
         hostSession,
         continuityScores,
-        scopeForBlock: (block) => writePathsByBlock.get(block.block_id) ?? block.touched_files ?? [],
+        scopeForBlock: (block) => writePathsByBlock.get(block.block_id) ?? block.touched_files,
+        // The node's real size, read off the plan item that already carries it. No
+        // fallback: a block in the drive that is absent from the plan is a producer
+        // desync, and defaulting it would silently restore the flat estimate that
+        // made every node look identical to the fit gates.
+        estimateTokens: (block) => {
+            const estimate = tokensByBlock.get(block.block_id);
+            if (estimate === undefined) {
+                throw new Error(`Block ${block.block_id} is being dispatched but carries no dispatch-plan item, ` +
+                    `so its token estimate is unknown. The drive and the plan have desynced.`);
+            }
+            return estimate;
+        },
         tierForBlock: (block) => tierByBlock.get(block.block_id),
         // Reactive cost verification: a declared-free source pool observed charging has
         // been demoted by the engine; surface it as reviewable friction so the operator
@@ -914,19 +945,6 @@ export async function driveRollingImplementDispatch(options) {
         ...(driven.terminal ? { terminal: driven.terminal } : {}),
     };
 }
-/**
- * Flat per-node input-token estimate for the A-8 hybrid frontier split. The
- * coordinator splits by per-pool SLOTS (concurrency) — host vs. backend — so a
- * uniform estimate is sufficient here; it matches the default the in-process engine
- * uses (`driveRollingDispatch`'s `() => 2000`), keeping the two paths consistent.
- *
- * ⚠ **Deliberately still flat.** Replacing it with the real per-node estimate was
- * built and REVERTED (2026-07-24): it is accurate, and that is precisely the
- * problem — an honest estimate makes "this node fits no pool" REACHABLE, and both
- * consumers mishandle that case terminally rather than resumably. See the backlog
- * entry; the fit gates cannot consume a real estimate until that routing is fixed.
- */
-const HYBRID_NODE_TOKEN_ESTIMATE = 2000;
 // ---------------------------------------------------------------------------
 // Tool-owned final completion gate (INV-RS-10) + coarse re-block (INV-RS-09)
 // ---------------------------------------------------------------------------
@@ -1338,7 +1356,7 @@ async function buildImplementDispatchStep(ctx) {
                 });
                 const frontier = plan.items
                     .filter((i) => typeof i.block_id === "string")
-                    .map((i) => ({ id: i.block_id, estimatedTokens: HYBRID_NODE_TOKEN_ESTIMATE }));
+                    .map((i) => ({ id: i.block_id, estimatedTokens: i.estimated_input_tokens }));
                 if (frontier.length === 0) {
                     const merged = await mergeImplementResults({ root, artifactsDir }, runId);
                     return { kind: "transition", state: merged };
@@ -1939,7 +1957,11 @@ async function handlePendingExtractedPlan(root, artifactsDir, existing, extracte
                 run_start_dirty: [...stagedAndUntracked(root)].sort(),
             };
         }
-        const reviewDecision = await readOptionalJsonFile(reviewDecisionPath(artifactsDir));
+        // Discarded on mismatch, so the gate re-asks rather than replaying operator
+        // decisions under semantics they were not made under. Re-asking costs a repeat
+        // answer; applying them blind could act on a keep/decline that no longer means
+        // what it meant when it was recorded.
+        const reviewDecision = discardOnSchemaVersionMismatch(await readOptionalJsonFile(reviewDecisionPath(artifactsDir)), REVIEW_DECISION_SCHEMA_VERSION);
         // Coverage ledger. Path A (structured_audit): the single filter pass ran at
         // intake over the ORIGINAL findings and persisted its dispositions — build
         // coverage over those originals so every audit finding gets exactly one
@@ -2113,8 +2135,8 @@ async function runReviewApprovalGate(root, artifactsDir, survivors, autonomous =
             };
         }
         // Consume the resolution into a durable, reasoned decision record.
-        const request = (await readOptionalJsonFile(requestPath)) ??
-            buildReviewRequest(survivors, randomRunId("path-a-review"));
+        // Regenerable: a stale-schema request is treated as absent and rebuilt below.
+        const request = discardOnSchemaVersionMismatch(await readOptionalJsonFile(requestPath), REVIEW_REQUEST_SCHEMA_VERSION) ?? buildReviewRequest(survivors, randomRunId("path-a-review"));
         const resolution = await readOptionalJsonFile(resolutionPath);
         if (!isResolutionForRequest(request, resolution)) {
             // Stale cross-run resolution (plan_id mismatch): archive it and RE-HALT
@@ -2147,7 +2169,9 @@ async function runReviewApprovalGate(root, artifactsDir, survivors, autonomous =
     // decision approves a SUBSET with declined EMPTY (leftovers live but not
     // approved), so keying the replay on the declined set alone would silently
     // re-approve every leftover on the next call.
-    const decision = await readOptionalJsonFile(decisionPath);
+    // Discarded on mismatch — see the note at the sibling read: a stale-schema
+    // decision record re-asks rather than replaying stale operator intent.
+    const decision = discardOnSchemaVersionMismatch(await readOptionalJsonFile(decisionPath), REVIEW_DECISION_SCHEMA_VERSION);
     const declined = decision?.declined ?? [];
     const declinedIds = new Set(declined.map((d) => d.finding_id));
     const approvedIds = decision ? new Set(decision.approved_ids ?? []) : undefined;
@@ -2712,7 +2736,17 @@ async function applyPlanClarificationResolution(artifactsDir, state, store) {
         await withFsRetry(() => rename(resolutionPath, `${resolutionPath}.consumed-${Date.now()}`));
     }
     const remainingPending = state.plan.findings.some((f) => state.items?.[f.id]?.status === "pending");
-    state.status = remainingPending ? "implementing" : "closing";
+    // Undecided-remainder guard, mirroring triage's still-blocked guard: a
+    // resolution covering only SOME of the paused items must not fall through to
+    // closing, which would force-close the undecided ones as `abandoned` and drop
+    // their questions unanswered. Load-bearing now that the round runs at the
+    // DRAINED end of the implement phase, where `remainingPending` is normally
+    // false — the fall-through it guards is the common case, not a corner.
+    state.status = remainingPending
+        ? "implementing"
+        : hasUnansweredClarification(state)
+            ? "waiting_for_clarification"
+            : "closing";
     state.clarifications = [];
     state.closing_plan ??= { action: "none" };
     await store.saveState(state);
@@ -2789,8 +2823,8 @@ async function runPlanningReviewGate(root, artifactsDir, state, store) {
         return handleWaitingForReviewApproval(root, artifactsDir, request);
     }
     // Resolution present: consume it into a durable, reasoned decision record.
-    const request = (await readOptionalJsonFile(requestPath)) ??
-        buildReviewRequest(findings, reviewPlanId);
+    // Regenerable: a stale-schema request is treated as absent and rebuilt below.
+    const request = discardOnSchemaVersionMismatch(await readOptionalJsonFile(requestPath), REVIEW_REQUEST_SCHEMA_VERSION) ?? buildReviewRequest(findings, reviewPlanId);
     const resolution = await readOptionalJsonFile(resolutionPath);
     if (!isResolutionForRequest(request, resolution)) {
         // Stale cross-run resolution: archive it and re-halt with the live request.
@@ -3751,6 +3785,7 @@ const MAIN_PRIORITY = [
     "waiting_for_triage",
     "planning_documentable",
     "partial_terminal",
+    "deferred_clarification",
     "implementing",
     "triage",
     "planning_zero",
@@ -3875,6 +3910,35 @@ function buildMainObligations(ctx) {
                 return handleAllTerminalTransition(root, artifactsDir, s, store, options, runLogger);
             },
         },
+        {
+            // Deferred clarification round. A worker question no longer freezes the run
+            // at merge time; it waits HERE — at the END of the implement phase, once the
+            // eligible dispatch frontier has drained — so every sibling's remaining work
+            // lands first and the questions are asked in one batched window (the goals
+            // doc's bounded-window promise).
+            //
+            // Ordered ABOVE `implementing` only because the derive already requires an
+            // empty frontier: while any node is dispatchable this obligation is
+            // satisfied and `implementing` dispatches it. Ordered above `triage` /
+            // `all_terminal` / `closing` so an unanswered question can never be swept
+            // past into close (triage with no blocked items routes straight to closing,
+            // which would force-close the paused item as `abandoned` and lose the
+            // question). It is NOT the only mid-phase halt — `partial_terminal` still
+            // owns the quota pause, and keeps its higher slot.
+            id: "deferred_clarification",
+            derive: (state) => state != null &&
+                (state.status === "implementing" || state.status === "triage") &&
+                hasUnansweredClarification(state) &&
+                implementableBlocks(state).length === 0
+                ? "missing"
+                : "satisfied",
+            execute: async (state) => {
+                const s = requireState(state);
+                s.status = "waiting_for_clarification";
+                await store.saveState(s);
+                return { kind: "transition", state: s };
+            },
+        },
         {
             id: "implementing",
             derive: (state) => state?.status === "implementing" ? "missing" : "satisfied",