@tea-agent/loop-agent 0.35.2 → 0.35.4-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -0
- package/CHANGELOG.md +43 -1
- package/README.md +1 -1
- package/bin/loop-agent.js +37 -1
- package/dist/build-stamp.json +6 -0
- package/dist/cli/program.js +2 -2
- package/dist/executors/dag-pi-executor.js +44 -0
- package/dist/shared/package-metadata.js +42 -0
- package/dist/worker/console/chat/assistant-content.js +11 -0
- package/dist/worker/console/chat/pi-runtime.js +6 -2
- package/dist/worker/console/chat/turn-process.js +17 -9
- package/dist/worker/console/chat/workspace-landing.js +1 -1
- package/dist/worker/console/static/assets/index-DuVLjCIT.js +57 -0
- package/dist/worker/console/static/index.html +1 -1
- package/dist/worker/console/static-src/app/useRecoveryConsole.js +5 -0
- package/dist/worker/console/static-src/operator-chat/chat-sse-events.js +15 -3
- package/dist/worker/console/static-src/operator-chat/refs.js +3 -0
- package/dist/worker/console/static-src/operator-chat/useChatSessions.js +3 -0
- package/dist/worker/console/static-src/operator-chat/useChatThread.js +1 -0
- package/dist/worker/loop-agent/loop-agent-client.js +17 -3
- package/dist/worker/observability/read-model.js +20 -0
- package/dist/worker/observe/spec-evidence.js +3 -8
- package/dist/worker/observe/static/views/dag-inspector.js +6 -71
- package/dist/worker/preflight.js +2 -1
- package/dist/workflows/dag/backend-test-scenario-param.js +33 -23
- package/dist/workflows/dag/contract-output-registry.js +14 -0
- package/dist/workflows/dag/contract-validator-registrations.js +8 -0
- package/dist/workflows/dag/dynamic-runtime/shared.js +9 -1
- package/dist/workflows/dag/failure-routing.js +9 -4
- package/dist/workflows/dag/frontend-implementation-contract.js +233 -39
- package/dist/workflows/dag/frontend-prewrite-gate.js +364 -61
- package/dist/workflows/dag/frontend-recovery-plan.js +73 -0
- package/dist/workflows/dag/frontend-recovery-root-manifest.js +123 -0
- package/dist/workflows/dag/frontend-recovery-run.js +539 -0
- package/dist/workflows/dag/frontend-repair.js +219 -18
- package/dist/workflows/dag/frontend-verification-trace.js +47 -32
- package/dist/workflows/dag/frontend-writer-recovery.js +106 -0
- package/dist/workflows/dag/frontend-writer-rollback.js +821 -0
- package/dist/workflows/dag/init-hybrid.js +49 -24
- package/dist/workflows/dag/lifecycle.js +4 -0
- package/dist/workflows/dag/node-execution.js +89 -0
- package/dist/workflows/dag/recovery-recommendation.js +58 -0
- package/dist/workflows/dag/report.js +6 -0
- package/dist/workflows/dag/runner.js +245 -11
- package/dist/workflows/dag/scheduler.js +257 -3
- package/dist/workflows/dag/types.js +130 -2
- package/docs/templates/frontend-task-constraints.md +13 -7
- package/package.json +4 -3
- package/dist/worker/console/static/assets/index-gVHrlqI9.js +0 -56
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readdir, readFile } from "node:fs/promises";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
2
3
|
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
3
4
|
import { hostname } from "node:os";
|
|
4
5
|
import path from "node:path";
|
|
@@ -6,7 +7,7 @@ import { isHardBudgetBreached, resolveEffectiveMaxConcurrent, } from "../../appl
|
|
|
6
7
|
import { readCandidateRecord } from "../../infrastructure/evaluation/candidate-store.js";
|
|
7
8
|
import { CANONICAL_TASK_ID_PATTERN, formatLocalCompactDate, } from "../../task/runtime.js";
|
|
8
9
|
import { assertFrozenBudget, initRunBudgetLedger, preflightBudgetOrBreach, recordFinishedNodeBudget, writeBudgetLedgerArtifacts, } from "./budget-enforcement.js";
|
|
9
|
-
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
|
|
10
|
+
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readDagRunSpec, readDagRunState, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
|
|
10
11
|
import { moveToCompletedRunDir, moveToPausedRunDir, prepareRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
|
|
11
12
|
import { evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
|
|
12
13
|
import { createDagNodeExecutor } from "./executor-registry.js";
|
|
@@ -21,8 +22,9 @@ import { createSkillSnapshot, prepareSkillSnapshotForContinuation, writeSkillSna
|
|
|
21
22
|
import { captureWorkspaceCheckpoint, WORKSPACE_CHECKPOINT_START_REL, WORKSPACE_CHECKPOINT_TERMINAL_REL, writeWorkspaceCheckpoint, } from "./workspace-checkpoint.js";
|
|
22
23
|
import { buildNodePrompt, buildNodePromptWithResolvedSkillInstructions, executeDagNode, } from "./node-execution.js";
|
|
23
24
|
import { runConvergencePassController, } from "./convergence/controller.js";
|
|
24
|
-
import { executeDagRanksOnce, isConditionSkippedReason } from "./scheduler.js";
|
|
25
|
+
import { executeDagRanksOnce, FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, FRONTEND_WRITER_NODE_IDS, isConditionSkippedReason, readFrontendPrewriteResult, } from "./scheduler.js";
|
|
25
26
|
import { topoSortToRanks } from "./topo.js";
|
|
27
|
+
import { isFrontendWriterTransientPartialWrite } from "./frontend-writer-recovery.js";
|
|
26
28
|
import { parseDagSpec, resolveModelForTask, } from "./types.js";
|
|
27
29
|
import { executeDynamicCondition } from "./dynamic-runtime/condition.js";
|
|
28
30
|
import { executeDynamicLoopUntil } from "./dynamic-runtime/loop-until.js";
|
|
@@ -448,6 +450,39 @@ async function executeDagCheckpoint(input) {
|
|
|
448
450
|
void persistState().catch(() => { });
|
|
449
451
|
}, runnerLivenessPolicy.heartbeatIntervalMs);
|
|
450
452
|
heartbeatTimer.unref();
|
|
453
|
+
// Graceful terminal persistence: an outer SIGTERM/SIGINT (operator hard
|
|
454
|
+
// timeout, supervision layer, or shell wall-clock) must not leave the run
|
|
455
|
+
// orphaned in RUNNING with a dead heartbeat. Persist a terminal failed
|
|
456
|
+
// state with an explicit terminalReason before exiting so downstream
|
|
457
|
+
// liveness/doctor tooling sees a diagnosable terminal instead of an orphan.
|
|
458
|
+
let terminalSignal;
|
|
459
|
+
const shutdownOnSignal = (signal) => {
|
|
460
|
+
if (terminalSignal)
|
|
461
|
+
return;
|
|
462
|
+
terminalSignal = signal;
|
|
463
|
+
clearInterval(heartbeatTimer);
|
|
464
|
+
if (state.runner)
|
|
465
|
+
state.runner.heartbeatAt = new Date().toISOString();
|
|
466
|
+
state.status = "failed";
|
|
467
|
+
state.finishedAt = new Date().toISOString();
|
|
468
|
+
state.terminalReason = `runner terminated by ${signal} before run completion`;
|
|
469
|
+
console.warn(`[run-dag] received ${signal}; persisting terminal state and exiting`);
|
|
470
|
+
const exit = () => process.exit(signal === "SIGINT" ? 130 : 143);
|
|
471
|
+
// Best-effort atomic persist with a hard exit deadline so a stuck write
|
|
472
|
+
// queue cannot keep the process alive past the supervisor's kill window.
|
|
473
|
+
const hardExit = setTimeout(exit, 5000);
|
|
474
|
+
hardExit.unref();
|
|
475
|
+
void persistState()
|
|
476
|
+
.catch(() => { })
|
|
477
|
+
.finally(() => {
|
|
478
|
+
clearTimeout(hardExit);
|
|
479
|
+
exit();
|
|
480
|
+
});
|
|
481
|
+
};
|
|
482
|
+
const onSigTerm = () => shutdownOnSignal("SIGTERM");
|
|
483
|
+
const onSigInt = () => shutdownOnSignal("SIGINT");
|
|
484
|
+
process.once("SIGTERM", onSigTerm);
|
|
485
|
+
process.once("SIGINT", onSigInt);
|
|
451
486
|
try {
|
|
452
487
|
const tasksById = new Map(spec.tasks.map((task) => [task.id, task]));
|
|
453
488
|
const baseExecuteNode = input.executeNode ??
|
|
@@ -504,6 +539,7 @@ async function executeDagCheckpoint(input) {
|
|
|
504
539
|
tasksById,
|
|
505
540
|
maxConcurrent,
|
|
506
541
|
persistState,
|
|
542
|
+
runDir,
|
|
507
543
|
abortSignal: input.abortSignal,
|
|
508
544
|
createExecuteNodeForRank: (rankWriterNodeIds) => buildRankAwareExecuteNode({
|
|
509
545
|
baseExecuteNode,
|
|
@@ -595,17 +631,54 @@ async function executeDagCheckpoint(input) {
|
|
|
595
631
|
runDir = await moveToPausedRunDir(runDir, pausedRunDir);
|
|
596
632
|
}
|
|
597
633
|
else {
|
|
598
|
-
finalizeTerminalRunStatus(state, spec.tasks.length);
|
|
634
|
+
const { recoveryPending } = await finalizeTerminalRunStatus(state, spec.tasks.length, runDir, cwd);
|
|
599
635
|
await persistState();
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
636
|
+
if (recoveryPending) {
|
|
637
|
+
// Recovery coordinator (phase 3c AC-2/AC-3): materialize the reserved
|
|
638
|
+
// child and run it synchronously. Dynamic import avoids a static
|
|
639
|
+
// runner ↔ frontend-recovery-run module cycle. stageRecoveryChild
|
|
640
|
+
// locates the parent via the canonical active/ layout, so recovery
|
|
641
|
+
// only fires when this run is still the canonical active run dir
|
|
642
|
+
// (direct runDagContinuation callers using an arbitrary runDir keep
|
|
643
|
+
// the previous scheduler-layer skip semantics).
|
|
644
|
+
const canonicalActiveDir = getDagRunDir(cwd, "active", state.runId);
|
|
645
|
+
if (path.resolve(runDir) === path.resolve(canonicalActiveDir)) {
|
|
646
|
+
const { stageRecoveryChild } = await import("./frontend-recovery-run.js");
|
|
647
|
+
const staged = await stageRecoveryChild({
|
|
648
|
+
cwd,
|
|
649
|
+
parentRunId: state.runId,
|
|
650
|
+
});
|
|
651
|
+
// stageRecoveryChild CAS-advanced the parent's on-disk lineage
|
|
652
|
+
// (child-staging → child-running + childRunId). Refresh the in-memory
|
|
653
|
+
// copy so the terminal persistState below does not regress the parent
|
|
654
|
+
// back to stale child-staging.
|
|
655
|
+
const parentOnDisk = await readDagRunState(runDir);
|
|
656
|
+
state.frontendRecoveryState = parentOnDisk.frontendRecoveryState;
|
|
657
|
+
const childState = await readDagRunState(staged.childRunDir);
|
|
658
|
+
const childSpec = await readDagRunSpec(staged.childRunDir);
|
|
659
|
+
await runDagContinuation({
|
|
660
|
+
cwd,
|
|
661
|
+
spec: childSpec,
|
|
662
|
+
state: childState,
|
|
663
|
+
runDir: staged.childRunDir,
|
|
664
|
+
maxConcurrent,
|
|
665
|
+
executeNode: input.executeNode,
|
|
666
|
+
observer: input.observer,
|
|
667
|
+
abortSignal: input.abortSignal,
|
|
668
|
+
});
|
|
669
|
+
}
|
|
604
670
|
}
|
|
605
|
-
|
|
606
|
-
|
|
671
|
+
await notifyRunObserver(input.observer, "onRunFinish", state);
|
|
672
|
+
if (!recoveryPending) {
|
|
673
|
+
try {
|
|
674
|
+
const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
|
|
675
|
+
await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
|
|
676
|
+
}
|
|
677
|
+
catch (error) {
|
|
678
|
+
console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
|
|
679
|
+
}
|
|
680
|
+
runDir = await moveToCompletedRunDir(runDir, completedRunDir);
|
|
607
681
|
}
|
|
608
|
-
runDir = await moveToCompletedRunDir(runDir, completedRunDir);
|
|
609
682
|
}
|
|
610
683
|
await relocateRunArtifactPaths({
|
|
611
684
|
runDir,
|
|
@@ -628,6 +701,8 @@ async function executeDagCheckpoint(input) {
|
|
|
628
701
|
}
|
|
629
702
|
finally {
|
|
630
703
|
clearInterval(heartbeatTimer);
|
|
704
|
+
process.removeListener("SIGTERM", onSigTerm);
|
|
705
|
+
process.removeListener("SIGINT", onSigInt);
|
|
631
706
|
}
|
|
632
707
|
}
|
|
633
708
|
async function notifyRunObserver(observer, event, state) {
|
|
@@ -695,7 +770,153 @@ function isSuccessfulConvergenceTerminal(state) {
|
|
|
695
770
|
return true;
|
|
696
771
|
return SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS.has(reason);
|
|
697
772
|
}
|
|
698
|
-
function
|
|
773
|
+
function buildFrontendRecoveryIntent(state, failureSource) {
|
|
774
|
+
const existing = state.frontendRecoveryState;
|
|
775
|
+
const requestId = existing?.requestId ?? randomUUID();
|
|
776
|
+
return {
|
|
777
|
+
schemaVersion: 1,
|
|
778
|
+
phase: "child-staging",
|
|
779
|
+
requestId,
|
|
780
|
+
recoveryRootRunId: existing?.recoveryRootRunId ?? state.runId,
|
|
781
|
+
parentRunId: state.runId,
|
|
782
|
+
...(existing?.childRunId ? { childRunId: existing.childRunId } : {}),
|
|
783
|
+
attemptId: existing?.attemptId ?? `recovery-${requestId}`,
|
|
784
|
+
attemptIndex: existing?.attemptIndex ?? 0,
|
|
785
|
+
continuationCount: existing?.continuationCount ?? 0,
|
|
786
|
+
...(failureSource ? { failureSource } : {}),
|
|
787
|
+
revision: (existing?.revision ?? 0) + 1,
|
|
788
|
+
};
|
|
789
|
+
}
|
|
790
|
+
function buildCandidateContractInvalidResult(state, result, artifactHash) {
|
|
791
|
+
const recovery = state.frontendRecoveryState;
|
|
792
|
+
return {
|
|
793
|
+
schemaVersion: 1,
|
|
794
|
+
outcome: "candidate-contract-invalid",
|
|
795
|
+
origin: {
|
|
796
|
+
kind: "frontend-prewrite-gate",
|
|
797
|
+
parentRunId: recovery?.parentRunId ?? state.runId,
|
|
798
|
+
...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
|
|
799
|
+
requestId: recovery?.requestId ?? "",
|
|
800
|
+
failedNodeId: result.selectedPlanNodeId,
|
|
801
|
+
},
|
|
802
|
+
failureClass: {
|
|
803
|
+
code: "candidate-contract-invalid",
|
|
804
|
+
classification: result.classification,
|
|
805
|
+
reason: result.failureReason ?? "candidate contract invalid",
|
|
806
|
+
},
|
|
807
|
+
evidenceRefs: [
|
|
808
|
+
{
|
|
809
|
+
runId: state.runId,
|
|
810
|
+
relativePath: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
|
|
811
|
+
sha256: artifactHash,
|
|
812
|
+
},
|
|
813
|
+
],
|
|
814
|
+
};
|
|
815
|
+
}
|
|
816
|
+
function buildWriterPartialWriteBlockedResult(state, failedNodeId, reason) {
|
|
817
|
+
const recovery = state.frontendRecoveryState;
|
|
818
|
+
return {
|
|
819
|
+
schemaVersion: 1,
|
|
820
|
+
outcome: "auto-recovery-blocked",
|
|
821
|
+
origin: {
|
|
822
|
+
kind: "frontend-writer",
|
|
823
|
+
parentRunId: recovery?.parentRunId ?? state.runId,
|
|
824
|
+
...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
|
|
825
|
+
requestId: recovery?.requestId ?? "",
|
|
826
|
+
failedNodeId,
|
|
827
|
+
},
|
|
828
|
+
failureClass: {
|
|
829
|
+
code: "writer-transient-partial-write",
|
|
830
|
+
classification: "transient",
|
|
831
|
+
reason,
|
|
832
|
+
},
|
|
833
|
+
evidenceRefs: [],
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
function settleFrontendRecovery(state, outcome) {
|
|
837
|
+
const recovery = state.frontendRecoveryState;
|
|
838
|
+
if (!recovery)
|
|
839
|
+
return;
|
|
840
|
+
recovery.phase = "settled";
|
|
841
|
+
if (!state.frontendRecoveryResult) {
|
|
842
|
+
state.frontendRecoveryResult = {
|
|
843
|
+
schemaVersion: 1,
|
|
844
|
+
outcome,
|
|
845
|
+
origin: {
|
|
846
|
+
kind: "frontend-prewrite-gate",
|
|
847
|
+
parentRunId: recovery.parentRunId,
|
|
848
|
+
...(recovery.childRunId ? { childRunId: recovery.childRunId } : {}),
|
|
849
|
+
requestId: recovery.requestId,
|
|
850
|
+
failedNodeId: "",
|
|
851
|
+
},
|
|
852
|
+
evidenceRefs: [],
|
|
853
|
+
};
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
|
|
857
|
+
const repoRoot = cwd ?? path.resolve(runDir, "..", "..", "..", "..");
|
|
858
|
+
// Terminal override: a denied frontend writer must never surface as a
|
|
859
|
+
// mechanical partial_failed just because prewrite FINISHED while the writer
|
|
860
|
+
// was SKIPPED. Fail closed on retryable-invalid/blocked before aggregation.
|
|
861
|
+
let recoveryPending = false;
|
|
862
|
+
const hasFrontendWriter = Object.keys(state.nodes).some((id) => FRONTEND_WRITER_NODE_IDS.includes(id));
|
|
863
|
+
if (hasFrontendWriter) {
|
|
864
|
+
const admission = await readFrontendPrewriteResult(runDir);
|
|
865
|
+
if (admission.ok) {
|
|
866
|
+
if (admission.result.classification === "blocked") {
|
|
867
|
+
state.status = "failed";
|
|
868
|
+
return { recoveryPending: false };
|
|
869
|
+
}
|
|
870
|
+
if (admission.result.classification === "retryable-invalid") {
|
|
871
|
+
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
872
|
+
if (continuationCount >= 1) {
|
|
873
|
+
// Continuation quota exhausted: a second retryable-invalid candidate
|
|
874
|
+
// is authoritative candidate-contract-invalid → forced failed.
|
|
875
|
+
state.frontendRecoveryResult = buildCandidateContractInvalidResult(state, admission.result, admission.artifactHash);
|
|
876
|
+
state.status = "failed";
|
|
877
|
+
return { recoveryPending: false };
|
|
878
|
+
}
|
|
879
|
+
// Root continuation still available: record the recovery intent and let
|
|
880
|
+
// mechanical aggregation settle partial_failed/failed. The parent keeps
|
|
881
|
+
// phase != settled (recovery in progress) and no final root manifest.
|
|
882
|
+
state.frontendRecoveryState = buildFrontendRecoveryIntent(state);
|
|
883
|
+
recoveryPending = true;
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
// Phase 5: writer transient partial write → rollback + recovery intent. Only
|
|
888
|
+
// for frontend-implementation writers with a remaining root continuation, and
|
|
889
|
+
// only when the rollback journal (captured before the provider call) restores
|
|
890
|
+
// cleanly; otherwise the run stays failed with auto-recovery-blocked.
|
|
891
|
+
if (hasFrontendWriter && !recoveryPending) {
|
|
892
|
+
const writerNodeId = FRONTEND_WRITER_NODE_IDS.find((id) => {
|
|
893
|
+
const node = state.nodes[id];
|
|
894
|
+
return (node &&
|
|
895
|
+
node.status === "ERROR" &&
|
|
896
|
+
isFrontendWriterTransientPartialWrite({
|
|
897
|
+
failureCategory: node.failureCategory,
|
|
898
|
+
}));
|
|
899
|
+
});
|
|
900
|
+
if (writerNodeId) {
|
|
901
|
+
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
902
|
+
if (continuationCount < 1) {
|
|
903
|
+
const { rollbackFrontendWriter } = await import("./frontend-writer-recovery.js");
|
|
904
|
+
const rollback = await rollbackFrontendWriter({
|
|
905
|
+
cwd: repoRoot,
|
|
906
|
+
parentRunId: state.runId,
|
|
907
|
+
});
|
|
908
|
+
if (rollback.ok) {
|
|
909
|
+
state.frontendRecoveryState = buildFrontendRecoveryIntent(state, "frontend-implement-pi");
|
|
910
|
+
recoveryPending = true;
|
|
911
|
+
}
|
|
912
|
+
else {
|
|
913
|
+
state.frontendRecoveryResult = buildWriterPartialWriteBlockedResult(state, writerNodeId, rollback.reason);
|
|
914
|
+
state.status = "failed";
|
|
915
|
+
return { recoveryPending: false };
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
}
|
|
699
920
|
const finishedCount = Object.values(state.nodes).filter((n) => n.status === "FINISHED").length;
|
|
700
921
|
const { supersededIds, supersededFailures } = collectSupersededIntermediateFailures(state);
|
|
701
922
|
if (state.convergence && supersededFailures.length > 0) {
|
|
@@ -719,6 +940,19 @@ function finalizeTerminalRunStatus(state, taskCount) {
|
|
|
719
940
|
else {
|
|
720
941
|
state.status = "failed";
|
|
721
942
|
}
|
|
943
|
+
// Terminal recovery bookkeeping for a child attempt: a child that finished
|
|
944
|
+
// recovered settles its own lineage; any other child terminal settles as
|
|
945
|
+
// auto-recovery-blocked (phase 3 performs no further recovery).
|
|
946
|
+
const recovery = state.frontendRecoveryState;
|
|
947
|
+
if (recovery && recovery.attemptIndex >= 1) {
|
|
948
|
+
if (state.status === "finished") {
|
|
949
|
+
settleFrontendRecovery(state, "recovered");
|
|
950
|
+
}
|
|
951
|
+
else if (!state.frontendRecoveryResult) {
|
|
952
|
+
settleFrontendRecovery(state, "auto-recovery-blocked");
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
return { recoveryPending };
|
|
722
956
|
}
|
|
723
957
|
function concurrentSiblingWriteSetsForNode(rankWriterNodeIds, nodeId, tasksById) {
|
|
724
958
|
if (rankWriterNodeIds.length <= 1)
|
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
1
3
|
import { isPauseOnHumanDecisionGate } from "./decision-envelope.js";
|
|
2
4
|
import { evaluateConditionExpression } from "./dynamic-runtime/condition.js";
|
|
5
|
+
import { sha256Hex } from "./frontend-implementation-contract.js";
|
|
6
|
+
import { frontendPrewriteResultV1Schema, FRONTEND_PREWRITE_RESULT_ARTIFACT_NAME, } from "./frontend-prewrite-gate.js";
|
|
7
|
+
import { FRONTEND_RECOVERY_IMPORT_MANIFEST_REL_PATH, FRONTEND_RECOVERY_INTENT_REL_DIR, } from "./frontend-recovery-plan.js";
|
|
3
8
|
export function isConditionSkippedReason(reason) {
|
|
4
9
|
return Boolean(reason?.startsWith("condition "));
|
|
5
10
|
}
|
|
@@ -104,6 +109,58 @@ async function mapConcurrent(items, limit, fn) {
|
|
|
104
109
|
}
|
|
105
110
|
await Promise.all(executing);
|
|
106
111
|
}
|
|
112
|
+
export const FRONTEND_WRITER_NODE_IDS = [
|
|
113
|
+
"frontend-implement-pi",
|
|
114
|
+
"frontend-repair-pi",
|
|
115
|
+
];
|
|
116
|
+
/** Run-relative artifact location written by the prewrite gate generator. */
|
|
117
|
+
export const FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT = path.posix.join("contracts", FRONTEND_PREWRITE_RESULT_ARTIFACT_NAME);
|
|
118
|
+
/**
|
|
119
|
+
* Read and validate the frontend-prewrite-result-v1 artifact. Fail-closed:
|
|
120
|
+
* a missing file or invalid payload returns ok:false and never throws.
|
|
121
|
+
*/
|
|
122
|
+
export async function readFrontendPrewriteResult(runDir) {
|
|
123
|
+
const artifactPath = path.join(runDir, "contracts", FRONTEND_PREWRITE_RESULT_ARTIFACT_NAME);
|
|
124
|
+
let raw;
|
|
125
|
+
try {
|
|
126
|
+
raw = await readFile(artifactPath, "utf8");
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
if (error.code === "ENOENT") {
|
|
130
|
+
return {
|
|
131
|
+
ok: false,
|
|
132
|
+
reason: `frontend prewrite result artifact missing: ${artifactPath}`,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
throw error;
|
|
136
|
+
}
|
|
137
|
+
const artifactHash = sha256Hex(raw);
|
|
138
|
+
let parsed;
|
|
139
|
+
try {
|
|
140
|
+
parsed = JSON.parse(raw);
|
|
141
|
+
}
|
|
142
|
+
catch (error) {
|
|
143
|
+
return {
|
|
144
|
+
ok: false,
|
|
145
|
+
reason: `frontend prewrite result artifact is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
const result = frontendPrewriteResultV1Schema.safeParse(parsed);
|
|
149
|
+
if (!result.success) {
|
|
150
|
+
return {
|
|
151
|
+
ok: false,
|
|
152
|
+
reason: "frontend prewrite result artifact failed schema validation",
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
return { ok: true, result: result.data, artifactHash };
|
|
156
|
+
}
|
|
157
|
+
/** Authorize only accepted / accepted-normalized classifications. */
|
|
158
|
+
export function isFrontendWriterAuthorized(result) {
|
|
159
|
+
return result.classification === "accepted" ||
|
|
160
|
+
result.classification === "accepted-normalized"
|
|
161
|
+
? "authorized"
|
|
162
|
+
: "denied";
|
|
163
|
+
}
|
|
107
164
|
/** Fail-closed: leave no PENDING nodes that look "still scheduled" after abort. */
|
|
108
165
|
export function markPendingNodesControllerInterrupted(state, reason = "run aborted by controller (abortSignal)") {
|
|
109
166
|
const affected = [];
|
|
@@ -122,8 +179,166 @@ export function markPendingNodesControllerInterrupted(state, reason = "run abort
|
|
|
122
179
|
}
|
|
123
180
|
return affected;
|
|
124
181
|
}
|
|
182
|
+
/**
|
|
183
|
+
* True when this run is a candidate-continuation child: it has a recovery
|
|
184
|
+
* lineage whose attemptIndex is >= 1 and whose recovery root is another run.
|
|
185
|
+
* The root/parent keeps `recoveryRootRunId === state.runId`.
|
|
186
|
+
*/
|
|
187
|
+
export function isFrontendRecoveryChild(state) {
|
|
188
|
+
const recovery = state.frontendRecoveryState;
|
|
189
|
+
return Boolean(recovery &&
|
|
190
|
+
recovery.attemptIndex >= 1 &&
|
|
191
|
+
recovery.recoveryRootRunId !== state.runId);
|
|
192
|
+
}
|
|
193
|
+
function isValidFrontendRecoveryActivationMarker(raw) {
|
|
194
|
+
return (raw.schemaVersion === 1 &&
|
|
195
|
+
typeof raw.requestId === "string" &&
|
|
196
|
+
raw.requestId.length > 0 &&
|
|
197
|
+
typeof raw.parentRunId === "string" &&
|
|
198
|
+
raw.parentRunId.length > 0 &&
|
|
199
|
+
typeof raw.recoveryRootRunId === "string" &&
|
|
200
|
+
raw.recoveryRootRunId.length > 0 &&
|
|
201
|
+
typeof raw.childRunId === "string" &&
|
|
202
|
+
raw.childRunId.length > 0 &&
|
|
203
|
+
typeof raw.importManifestSha256 === "string" &&
|
|
204
|
+
/^[a-f0-9]{64}$/.test(raw.importManifestSha256));
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Activation marker gate (phase 3c AC-1). Fail-closed: an active child is only
|
|
208
|
+
* executable when its parent is `child-running`, points at this child, carries
|
|
209
|
+
* a structurally valid activation marker whose lineage matches, and the marker
|
|
210
|
+
* hash matches the child's import manifest. Read-only and never throws.
|
|
211
|
+
*/
|
|
212
|
+
export async function checkFrontendRecoveryActivation(state, childRunDir) {
|
|
213
|
+
if (!isFrontendRecoveryChild(state)) {
|
|
214
|
+
return { applicable: false };
|
|
215
|
+
}
|
|
216
|
+
const recovery = state.frontendRecoveryState;
|
|
217
|
+
const parentRunId = recovery.parentRunId;
|
|
218
|
+
const parentRunDir = path.join(path.dirname(childRunDir), parentRunId);
|
|
219
|
+
let parentState;
|
|
220
|
+
try {
|
|
221
|
+
parentState = JSON.parse(await readFile(path.join(parentRunDir, "state.json"), "utf8"));
|
|
222
|
+
}
|
|
223
|
+
catch {
|
|
224
|
+
return { applicable: true, ok: false, reason: "parent state unreadable" };
|
|
225
|
+
}
|
|
226
|
+
const parentRecovery = parentState.frontendRecoveryState;
|
|
227
|
+
if (parentRecovery?.phase !== "child-running") {
|
|
228
|
+
return {
|
|
229
|
+
applicable: true,
|
|
230
|
+
ok: false,
|
|
231
|
+
reason: "parent phase not child-running",
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
if (parentRecovery.childRunId !== state.runId) {
|
|
235
|
+
return {
|
|
236
|
+
applicable: true,
|
|
237
|
+
ok: false,
|
|
238
|
+
reason: "parent childRunId mismatch",
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
const markerPath = path.join(parentRunDir, FRONTEND_RECOVERY_INTENT_REL_DIR, `${recovery.requestId}.json`);
|
|
242
|
+
let markerRaw;
|
|
243
|
+
try {
|
|
244
|
+
markerRaw = JSON.parse(await readFile(markerPath, "utf8"));
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
return {
|
|
248
|
+
applicable: true,
|
|
249
|
+
ok: false,
|
|
250
|
+
reason: "activation marker missing",
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
if (typeof markerRaw !== "object" ||
|
|
254
|
+
markerRaw === null ||
|
|
255
|
+
!isValidFrontendRecoveryActivationMarker(markerRaw)) {
|
|
256
|
+
return {
|
|
257
|
+
applicable: true,
|
|
258
|
+
ok: false,
|
|
259
|
+
reason: "activation marker invalid",
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
const marker = markerRaw;
|
|
263
|
+
if (marker.requestId !== recovery.requestId) {
|
|
264
|
+
return {
|
|
265
|
+
applicable: true,
|
|
266
|
+
ok: false,
|
|
267
|
+
reason: "marker requestId mismatch",
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
if (marker.parentRunId !== parentRunId) {
|
|
271
|
+
return {
|
|
272
|
+
applicable: true,
|
|
273
|
+
ok: false,
|
|
274
|
+
reason: "marker parentRunId mismatch",
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
if (marker.recoveryRootRunId !== recovery.recoveryRootRunId) {
|
|
278
|
+
return {
|
|
279
|
+
applicable: true,
|
|
280
|
+
ok: false,
|
|
281
|
+
reason: "marker recoveryRootRunId mismatch",
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
if (marker.childRunId !== state.runId) {
|
|
285
|
+
return {
|
|
286
|
+
applicable: true,
|
|
287
|
+
ok: false,
|
|
288
|
+
reason: "marker childRunId mismatch",
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
let manifestSha256;
|
|
292
|
+
try {
|
|
293
|
+
manifestSha256 = sha256Hex(await readFile(path.join(childRunDir, FRONTEND_RECOVERY_IMPORT_MANIFEST_REL_PATH), "utf8"));
|
|
294
|
+
}
|
|
295
|
+
catch {
|
|
296
|
+
return {
|
|
297
|
+
applicable: true,
|
|
298
|
+
ok: false,
|
|
299
|
+
reason: "import manifest unreadable",
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
if (manifestSha256 !== marker.importManifestSha256) {
|
|
303
|
+
return {
|
|
304
|
+
applicable: true,
|
|
305
|
+
ok: false,
|
|
306
|
+
reason: "import manifest sha256 mismatch",
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
return {
|
|
310
|
+
applicable: true,
|
|
311
|
+
ok: true,
|
|
312
|
+
requestId: recovery.requestId,
|
|
313
|
+
childRunId: state.runId,
|
|
314
|
+
};
|
|
315
|
+
}
|
|
125
316
|
export async function executeDagRanksOnce(input) {
|
|
126
317
|
let pausedByNodeId;
|
|
318
|
+
// Activation marker gate (phase 3c AC-1): a recovery child without a valid
|
|
319
|
+
// activation marker must never be scheduled or executed. Fail-closed before
|
|
320
|
+
// any executeScheduledNode call, so zero writer/provider invocations happen.
|
|
321
|
+
if (input.runDir) {
|
|
322
|
+
const hasPending = Object.values(input.state.nodes).some((node) => node.status === "PENDING");
|
|
323
|
+
if (hasPending) {
|
|
324
|
+
const activation = await checkFrontendRecoveryActivation(input.state, input.runDir);
|
|
325
|
+
if (activation.applicable && !activation.ok) {
|
|
326
|
+
const finishedAt = new Date().toISOString();
|
|
327
|
+
let affected = 0;
|
|
328
|
+
for (const node of Object.values(input.state.nodes)) {
|
|
329
|
+
if (node.status !== "PENDING")
|
|
330
|
+
continue;
|
|
331
|
+
node.status = "SKIPPED";
|
|
332
|
+
node.skippedReason = "frontend-recovery-child-not-activated";
|
|
333
|
+
node.finishedAt = finishedAt;
|
|
334
|
+
affected += 1;
|
|
335
|
+
}
|
|
336
|
+
if (affected > 0)
|
|
337
|
+
await input.persistState();
|
|
338
|
+
return undefined;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
127
342
|
for (const rank of input.ranks) {
|
|
128
343
|
if (input.abortSignal?.aborted) {
|
|
129
344
|
const marked = markPendingNodesControllerInterrupted(input.state, input.abortSignal.reason
|
|
@@ -168,6 +383,45 @@ export async function executeDagRanksOnce(input) {
|
|
|
168
383
|
await input.persistState();
|
|
169
384
|
const conditionSkippedSet = new Set(conditionSettled);
|
|
170
385
|
const actuallyRunnable = runnable.filter((id) => !conditionSkippedSet.has(id));
|
|
386
|
+
const frontendAdmissionSettled = [];
|
|
387
|
+
if (input.runDir) {
|
|
388
|
+
for (const id of actuallyRunnable) {
|
|
389
|
+
if (!FRONTEND_WRITER_NODE_IDS.includes(id))
|
|
390
|
+
continue;
|
|
391
|
+
const node = input.state.nodes[id];
|
|
392
|
+
const checkedAt = new Date().toISOString();
|
|
393
|
+
const admission = await readFrontendPrewriteResult(input.runDir);
|
|
394
|
+
if (!admission.ok) {
|
|
395
|
+
node.status = "SKIPPED";
|
|
396
|
+
node.skippedReason = "frontend-prewrite-not-authorized";
|
|
397
|
+
node.finishedAt = checkedAt;
|
|
398
|
+
frontendAdmissionSettled.push(id);
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
const decision = isFrontendWriterAuthorized(admission.result);
|
|
402
|
+
node.frontendWriterAdmission = {
|
|
403
|
+
schemaVersion: 1,
|
|
404
|
+
writerNodeId: id,
|
|
405
|
+
decision,
|
|
406
|
+
sourceArtifact: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
|
|
407
|
+
artifactHash: admission.artifactHash,
|
|
408
|
+
checkedAt,
|
|
409
|
+
reason: decision === "denied"
|
|
410
|
+
? `classification: ${admission.result.classification}`
|
|
411
|
+
: null,
|
|
412
|
+
};
|
|
413
|
+
if (decision === "denied") {
|
|
414
|
+
node.status = "SKIPPED";
|
|
415
|
+
node.skippedReason = "frontend-prewrite-not-authorized";
|
|
416
|
+
node.finishedAt = checkedAt;
|
|
417
|
+
frontendAdmissionSettled.push(id);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
if (frontendAdmissionSettled.length > 0)
|
|
421
|
+
await input.persistState();
|
|
422
|
+
}
|
|
423
|
+
const frontendAdmissionSkippedSet = new Set(frontendAdmissionSettled);
|
|
424
|
+
const runnableAfterAdmission = actuallyRunnable.filter((id) => !frontendAdmissionSkippedSet.has(id));
|
|
171
425
|
const blocked = pending.filter((id) => {
|
|
172
426
|
const task = input.tasksById.get(id);
|
|
173
427
|
return (dependencyReadiness(task, input.state.nodes, input.tasksById) === "skip");
|
|
@@ -182,12 +436,12 @@ export async function executeDagRanksOnce(input) {
|
|
|
182
436
|
if (blocked.length > 0) {
|
|
183
437
|
await input.persistState();
|
|
184
438
|
}
|
|
185
|
-
const pauseGateRunnable =
|
|
439
|
+
const pauseGateRunnable = runnableAfterAdmission.filter((id) => {
|
|
186
440
|
const task = input.tasksById.get(id);
|
|
187
441
|
return isPauseOnHumanDecisionGate(task);
|
|
188
442
|
});
|
|
189
|
-
const regularRunnable =
|
|
190
|
-
const rankWriterNodeIds =
|
|
443
|
+
const regularRunnable = runnableAfterAdmission.filter((id) => !pauseGateRunnable.includes(id));
|
|
444
|
+
const rankWriterNodeIds = runnableAfterAdmission.filter((id) => {
|
|
191
445
|
const task = input.tasksById.get(id);
|
|
192
446
|
return (task?.executor === "pi" &&
|
|
193
447
|
task.toolProfile === "write" &&
|