harnery 0.44.0 → 0.45.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +44 -0
- package/dist/core/agents/coord-client.d.ts.map +1 -1
- package/dist/core/agents/coord-client.js +18 -0
- package/dist/core/agents/live-authority-v3.d.ts +0 -1
- package/dist/core/agents/live-authority-v3.d.ts.map +1 -1
- package/dist/core/agents/live-authority-v3.js +0 -11
- package/dist/core/agents/live-lifecycle-v3.d.ts +2 -4
- package/dist/core/agents/live-lifecycle-v3.d.ts.map +1 -1
- package/dist/core/agents/live-lifecycle-v3.js +7 -28
- package/dist/core/agents/state/heartbeat-writer.d.ts.map +1 -1
- package/dist/core/agents/state/heartbeat-writer.js +14 -25
- package/dist/core/agents/state/live-coordination-writer.d.ts.map +1 -1
- package/dist/core/agents/state/live-coordination-writer.js +0 -5
- package/dist/core/agents/state/pidmap.d.ts.map +1 -1
- package/dist/core/agents/state/pidmap.js +0 -5
- package/dist/core/agents/state/proc-start.d.ts +5 -6
- package/dist/core/agents/state/proc-start.d.ts.map +1 -1
- package/dist/core/agents/state/proc-start.js +13 -9
- package/dist/core/agents/state/process-handle.d.ts +26 -0
- package/dist/core/agents/state/process-handle.d.ts.map +1 -0
- package/dist/core/agents/state/process-handle.js +121 -0
- package/dist/core/agents/state/stale-sweep.d.ts.map +1 -1
- package/dist/core/agents/state/stale-sweep.js +70 -180
- package/dist/core/artifacts/activity.d.ts +4 -1
- package/dist/core/artifacts/activity.d.ts.map +1 -1
- package/dist/core/artifacts/activity.js +8 -2
- package/dist/core/artifacts/index.d.ts.map +1 -1
- package/dist/core/artifacts/index.js +38 -12
- package/dist/core/events/v3/authority-outbox.d.ts.map +1 -1
- package/dist/core/events/v3/authority-outbox.js +0 -29
- package/dist/core/events/v3/producers/coordination-recorder.d.ts +0 -1
- package/dist/core/events/v3/producers/coordination-recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/coordination-recorder.js +0 -7
- package/dist/core/events/v3/producers/recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/recorder.js +64 -77
- package/dist/core/hooks/cli.js +78 -48
- package/dist/core/resources/service.d.ts +2 -16
- package/dist/core/resources/service.d.ts.map +1 -1
- package/dist/core/resources/service.js +1 -78
- package/dist/core/semantic/service.d.ts.map +1 -1
- package/dist/core/semantic/service.js +2 -9
- package/dist/core/servers/index.d.ts +4 -0
- package/dist/core/servers/index.d.ts.map +1 -1
- package/dist/core/servers/index.js +21 -3
- package/dist/core/storage/durable-history.js +56 -22
- package/dist/core/storage/segments.d.ts.map +1 -1
- package/dist/core/storage/segments.js +73 -17
- package/dist/core/supervisor/service.d.ts +2 -4
- package/dist/core/supervisor/service.d.ts.map +1 -1
- package/dist/core/work/state.js +7 -7
- package/dist/core/workflow/engine.d.ts.map +1 -1
- package/dist/core/workflow/engine.js +21 -65
- package/dist/core/workflow/index.d.ts +4 -2
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +2 -1
- package/dist/core/workflow/live-session-v3.d.ts +0 -6
- package/dist/core/workflow/live-session-v3.d.ts.map +1 -1
- package/dist/core/workflow/live-session-v3.js +5 -126
- package/dist/core/workflow/proof.d.ts +4 -4
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +4 -12
- package/dist/core/workflow/recovery-audit.d.ts +30 -0
- package/dist/core/workflow/recovery-audit.d.ts.map +1 -0
- package/dist/core/workflow/recovery-audit.js +116 -0
- package/dist/core/workflow/run-state.d.ts +19 -2
- package/dist/core/workflow/run-state.d.ts.map +1 -1
- package/dist/core/workflow/run-state.js +113 -8
- package/dist/core/workflow/spawn-claude.d.ts.map +1 -1
- package/dist/core/workflow/spawn-claude.js +0 -19
- package/dist/core/workflow/spawn-codex.d.ts.map +1 -1
- package/dist/core/workflow/spawn-codex.js +0 -13
- package/dist/core/workflow/spawn-cursor.d.ts.map +1 -1
- package/dist/core/workflow/spawn-cursor.js +0 -19
- package/dist/core/workflow/spawn-opencode.d.ts.map +1 -1
- package/dist/core/workflow/spawn-opencode.js +0 -14
- package/dist/core/workflow/types.d.ts +11 -54
- package/dist/core/workflow/types.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/local-git.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/local-git.js +70 -14
- package/dist/core/workflow/workspaces/reclaim.d.ts +6 -1
- package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.d.ts +2 -0
- package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.js +24 -0
- package/dist/lib/identities/assume.d.ts.map +1 -1
- package/dist/lib/identities/assume.js +0 -4
- package/package.json +1 -1
- package/src/commands/workflow.ts +59 -0
- package/src/core/agents/coord-client.ts +20 -0
- package/src/core/agents/live-authority-v3.ts +0 -17
- package/src/core/agents/live-lifecycle-v3.ts +14 -52
- package/src/core/agents/state/heartbeat-writer.ts +12 -29
- package/src/core/agents/state/live-coordination-writer.ts +0 -29
- package/src/core/agents/state/pidmap.ts +0 -13
- package/src/core/agents/state/proc-start.ts +14 -11
- package/src/core/agents/state/process-handle.ts +165 -0
- package/src/core/agents/state/stale-sweep.ts +82 -247
- package/src/core/artifacts/activity.ts +9 -2
- package/src/core/artifacts/index.ts +45 -12
- package/src/core/events/v3/authority-outbox.ts +0 -56
- package/src/core/events/v3/producers/coordination-recorder.ts +0 -15
- package/src/core/events/v3/producers/recorder.ts +68 -94
- package/src/core/hooks/cli.ts +85 -47
- package/src/core/resources/service.ts +3 -88
- package/src/core/semantic/service.ts +2 -8
- package/src/core/servers/index.ts +30 -3
- package/src/core/storage/durable-history.ts +52 -20
- package/src/core/storage/segments.ts +79 -17
- package/src/core/supervisor/service.ts +2 -3
- package/src/core/work/state.ts +7 -7
- package/src/core/workflow/engine.ts +21 -73
- package/src/core/workflow/index.ts +12 -1
- package/src/core/workflow/live-session-v3.ts +5 -146
- package/src/core/workflow/proof.ts +5 -21
- package/src/core/workflow/recovery-audit.ts +181 -0
- package/src/core/workflow/run-state.ts +137 -8
- package/src/core/workflow/spawn-claude.ts +0 -18
- package/src/core/workflow/spawn-codex.ts +0 -13
- package/src/core/workflow/spawn-cursor.ts +0 -18
- package/src/core/workflow/spawn-opencode.ts +0 -14
- package/src/core/workflow/types.ts +11 -41
- package/src/core/workflow/workspaces/local-git.ts +86 -15
- package/src/core/workflow/workspaces/reclaim.ts +4 -0
- package/src/core/workflow/workspaces/recovery.ts +23 -0
- package/src/lib/identities/assume.ts +0 -4
- package/dist/core/agents/state/unchanged-file.d.ts +0 -44
- package/dist/core/agents/state/unchanged-file.d.ts.map +0 -1
- package/dist/core/agents/state/unchanged-file.js +0 -390
- package/dist/core/events/v3/generation-custody.d.ts +0 -5
- package/dist/core/events/v3/generation-custody.d.ts.map +0 -1
- package/dist/core/events/v3/generation-custody.js +0 -50
- package/dist/core/workflow/failure-evidence.d.ts +0 -16
- package/dist/core/workflow/failure-evidence.d.ts.map +0 -1
- package/dist/core/workflow/failure-evidence.js +0 -114
- package/src/core/agents/state/unchanged-file.ts +0 -456
- package/src/core/events/v3/generation-custody.ts +0 -59
- package/src/core/workflow/failure-evidence.ts +0 -141
|
@@ -37,8 +37,7 @@ import { freezeWorkflowAttemptContext } from "./attempt-context.js";
|
|
|
37
37
|
import { probeBilling } from "./billing.js";
|
|
38
38
|
import { stableDigest } from "./durable-record.js";
|
|
39
39
|
import { evidencePreflightError } from "./evidence-preflight.js";
|
|
40
|
-
import {
|
|
41
|
-
import { endWorkflowChildSessionV3, recordWorkflowChildTerminalFailureV3, startWorkflowChildSessionV3, } from "./live-session-v3.js";
|
|
40
|
+
import { endWorkflowChildSessionV3, startWorkflowChildSessionV3 } from "./live-session-v3.js";
|
|
42
41
|
import { buildWorkflowProof, createEvidenceRecord, digestResult, normalizeWorkflowMeta, writeWorkflowProof, } from "./proof.js";
|
|
43
42
|
import { acquireWorkflowResumeLease, assertWorkflowRunResumable, assertWorkflowScriptUnchanged, WORKFLOW_RUN_MANIFEST_SCHEMA_VERSION, workflowScriptDigest, writeWorkflowRunManifest, } from "./run-state.js";
|
|
44
43
|
import { assertProjectionWithinWorkspace, resolveGitGrantRoots } from "./sandbox-projection.js";
|
|
@@ -865,40 +864,23 @@ async function executeWorkflow(scriptPath, absScript, opts, resumeState, workCon
|
|
|
865
864
|
},
|
|
866
865
|
});
|
|
867
866
|
agentProof.access = access;
|
|
868
|
-
let childGenerationId;
|
|
869
867
|
let recordAgentEnd;
|
|
870
868
|
const finalizeChild = () => {
|
|
871
869
|
try {
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
// Result publication is the resume-cache boundary, after canonical custody.
|
|
870
|
+
// Record the canonical terminal first. agent.end is the resume-cache
|
|
871
|
+
// boundary, so a child whose end was not recorded never reports success.
|
|
872
|
+
endWorkflowChildSessionV3({
|
|
873
|
+
coordRoot: opts.coordRoot,
|
|
874
|
+
instanceId: childInstanceId(runId, id),
|
|
875
|
+
runId,
|
|
876
|
+
agentId: id,
|
|
877
|
+
adapter,
|
|
878
|
+
cleanExit: agentProof.status === "succeeded",
|
|
879
|
+
});
|
|
883
880
|
recordAgentEnd?.();
|
|
884
881
|
}
|
|
885
882
|
catch (error) {
|
|
886
|
-
|
|
887
|
-
try {
|
|
888
|
-
if (childGenerationId)
|
|
889
|
-
recordWorkflowChildTerminalFailureV3({
|
|
890
|
-
coordRoot: opts.coordRoot,
|
|
891
|
-
instanceId: childInstanceId(runId, id),
|
|
892
|
-
runId,
|
|
893
|
-
agentId: id,
|
|
894
|
-
adapter,
|
|
895
|
-
expectedGenerationId: childGenerationId,
|
|
896
|
-
reason,
|
|
897
|
-
});
|
|
898
|
-
}
|
|
899
|
-
catch (visibilityError) {
|
|
900
|
-
reason += `; blocked lifecycle recording failed: ${visibilityError.message}`;
|
|
901
|
-
}
|
|
883
|
+
const reason = `V3 terminal recording failed: ${error.message}`;
|
|
902
884
|
const failure = new Error(agentProof.error ? `${agentProof.error}; ${reason}` : reason, {
|
|
903
885
|
cause: error,
|
|
904
886
|
});
|
|
@@ -930,7 +912,7 @@ async function executeWorkflow(scriptPath, absScript, opts, resumeState, workCon
|
|
|
930
912
|
// Headless children do not reliably fire adapter hooks. The engine owns
|
|
931
913
|
// their canonical V3 start so every dispatched child has an auditable
|
|
932
914
|
// generation before it begins work.
|
|
933
|
-
|
|
915
|
+
startWorkflowChildSessionV3({
|
|
934
916
|
coordRoot: opts.coordRoot,
|
|
935
917
|
instanceId: childInstanceId(runId, id),
|
|
936
918
|
runId,
|
|
@@ -939,13 +921,9 @@ async function executeWorkflow(scriptPath, absScript, opts, resumeState, workCon
|
|
|
939
921
|
label,
|
|
940
922
|
model: agentOpts.model,
|
|
941
923
|
});
|
|
942
|
-
if (!child.v3_generation_id)
|
|
943
|
-
throw new Error("workflow_child_v3_generation_missing");
|
|
944
|
-
childGenerationId = child.v3_generation_id;
|
|
945
924
|
let attemptPrompt = dispatchPrompt;
|
|
946
925
|
let last = null;
|
|
947
926
|
let agentCostUsd = 0;
|
|
948
|
-
let informativeAttempt;
|
|
949
927
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
950
928
|
last = await spawner({
|
|
951
929
|
prompt: attemptPrompt,
|
|
@@ -967,29 +945,15 @@ async function executeWorkflow(scriptPath, absScript, opts, resumeState, workCon
|
|
|
967
945
|
agentCostUsd += last.costUsd ?? 0;
|
|
968
946
|
costUsd += last.costUsd ?? 0;
|
|
969
947
|
if (!last.ok) {
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
effect: { state: "unknown" },
|
|
977
|
-
partial_output: informativeAttempt.partial_output,
|
|
978
|
-
}
|
|
979
|
-
: observed;
|
|
980
|
-
transcript("agent.attempt_failed", {
|
|
981
|
-
id,
|
|
982
|
-
attempt,
|
|
983
|
-
error: last.error,
|
|
984
|
-
failure_evidence: observed,
|
|
985
|
-
result_digest: digestResult(last.text, "text"),
|
|
986
|
-
text: last.text,
|
|
987
|
-
});
|
|
988
|
-
if (!canRetryFailedSpawn({ ...last, failureEvidence: agentProof.failure_evidence }))
|
|
948
|
+
transcript("agent.attempt_failed", { id, attempt, error: last.error });
|
|
949
|
+
// ADR 0046: an environment failure (the binary was absent) cannot be
|
|
950
|
+
// helped by retrying an unchanged environment, so stop the in-agent
|
|
951
|
+
// retry too — not just the outer attempt/replan budget. An upstream
|
|
952
|
+
// refusal keeps retrying here: the vendor may recover mid-loop.
|
|
953
|
+
if (last.class === "environment")
|
|
989
954
|
break;
|
|
990
|
-
continue;
|
|
955
|
+
continue; // spawn-level failure: retry with the original prompt
|
|
991
956
|
}
|
|
992
|
-
informativeAttempt = observeSpawnFailure({}, last.text, "final_message");
|
|
993
957
|
if (!agentOpts.schema) {
|
|
994
958
|
agentProof.status = "succeeded";
|
|
995
959
|
agentProof.cost_usd =
|
|
@@ -1030,14 +994,7 @@ async function executeWorkflow(scriptPath, absScript, opts, resumeState, workCon
|
|
|
1030
994
|
}, "json", parsed.value);
|
|
1031
995
|
return parsed.value;
|
|
1032
996
|
}
|
|
1033
|
-
|
|
1034
|
-
transcript("agent.schema_retry", {
|
|
1035
|
-
id,
|
|
1036
|
-
attempt,
|
|
1037
|
-
problems,
|
|
1038
|
-
rejected_text: last.text,
|
|
1039
|
-
result_digest: digestResult(last.text, "text"),
|
|
1040
|
-
});
|
|
997
|
+
transcript("agent.schema_retry", { id, attempt, problems });
|
|
1041
998
|
// Keep the original cache identity while making the actual output
|
|
1042
999
|
// contract explicit. A full bounded copy of the rejected value lets the
|
|
1043
1000
|
// child repair omissions instead of reconstructing an unseen object.
|
|
@@ -1076,7 +1033,6 @@ async function executeWorkflow(scriptPath, absScript, opts, resumeState, workCon
|
|
|
1076
1033
|
finalizeChild();
|
|
1077
1034
|
}
|
|
1078
1035
|
finally {
|
|
1079
|
-
// A slot is disposable; unresolved custody remains visible and failed.
|
|
1080
1036
|
release();
|
|
1081
1037
|
}
|
|
1082
1038
|
}
|
|
@@ -4,8 +4,10 @@ export { assertWorkflowRunId, createWorkflowApproval, listWorkflowApprovals, rea
|
|
|
4
4
|
export { freezeWorkflowAttemptContext, isCanonicalWorkflowAttemptContext, normalizeWorkflowAttemptContext, } from "./attempt-context.js";
|
|
5
5
|
export { runWorkflow, WorkflowParkedError, WorkflowRunError } from "./engine.js";
|
|
6
6
|
export { buildWorkflowProof, createEvidenceRecord, deriveRunFailureClass, digestResult, normalizeWorkflowMeta, readWorkflowProof, renderWorkflowProof, rollupAcceptance, writeWorkflowProof, } from "./proof.js";
|
|
7
|
-
export type {
|
|
8
|
-
export {
|
|
7
|
+
export type { WorkflowRecoveryAuthorization, WorkflowRecoveryIntent, WorkflowRecoveryRecord, } from "./recovery-audit.js";
|
|
8
|
+
export { beginWorkflowRecovery, readWorkflowRecoveryLedger } from "./recovery-audit.js";
|
|
9
|
+
export type { WorkflowResumeRecoveryOptions, WorkflowResumeRecoveryResult, WorkflowRunManifest, } from "./run-state.js";
|
|
10
|
+
export { acquireWorkflowResumeLease, assertWorkflowRunResumable, assertWorkflowScriptUnchanged, readWorkflowRunManifest, recoverWorkflowResumeLease, WORKFLOW_RUN_MANIFEST_SCHEMA_VERSION, workflowRunManifestPath, workflowScriptDigest, writeWorkflowRunManifest, } from "./run-state.js";
|
|
9
11
|
export { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.js";
|
|
10
12
|
export type { AcceptanceCriterion, AcceptanceResult, AcceptanceStatus, AcceptanceSummary, AdapterEvidenceCapability, AdapterEvidenceCoverage, AdapterName, AgentOpts, EngineOpts, EvidenceKind, EvidenceSource, EvidenceStatus, ResultDigest, RunReport, Spawner, SpawnFailureClass, SpawnRequest, SpawnResult, StageSchema, WorkflowAgentProof, WorkflowApprovalMode, WorkflowAttemptContext, WorkflowAttemptFailureCause, WorkflowAttemptPriorContext, WorkflowAttemptUnresolvedCriterion, WorkflowContext, WorkflowDiagnosticAdmissionConfig, WorkflowDiagnosticAdmissionObservation, WorkflowDiagnosticAdmissionProof, WorkflowEvidenceInput, WorkflowEvidenceRecord, WorkflowMeta, WorkflowModule, WorkflowPolicyProof, WorkflowProof, WorkflowProofUnknown, WorkflowRepoEvidence, WorkflowRepoSnapshot, WorkflowRunStatus, WorkflowSpecialistProfile, WorkflowWorkContext, } from "./types.js";
|
|
11
13
|
export { WORKFLOW_ATTEMPT_CONTEXT_SCHEMA_VERSION, WORKFLOW_DIAGNOSTIC_ADMISSION_SCHEMA_VERSION, WORKFLOW_PROOF_SCHEMA_VERSION, WORKFLOW_WORK_CONTEXT_SCHEMA_VERSION, } from "./types.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/workflow/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,4CAA4C,EAC5C,KAAK,uCAAuC,EAC5C,kCAAkC,EAClC,KAAK,kCAAkC,GACxC,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,2BAA2B,EAC3B,4BAA4B,EAC5B,gBAAgB,EAChB,wBAAwB,EACxB,uBAAuB,EACvB,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,mBAAmB,EACnB,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,gCAAgC,EAChC,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,4BAA4B,EAC5B,iCAAiC,EACjC,+BAA+B,GAChC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,EACrB,YAAY,EACZ,qBAAqB,EACrB,iBAAiB,EACjB,mBAAmB,EACnB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/workflow/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,4CAA4C,EAC5C,KAAK,uCAAuC,EAC5C,kCAAkC,EAClC,KAAK,kCAAkC,GACxC,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,2BAA2B,EAC3B,4BAA4B,EAC5B,gBAAgB,EAChB,wBAAwB,EACxB,uBAAuB,EACvB,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,mBAAmB,EACnB,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,gCAAgC,EAChC,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,4BAA4B,EAC5B,iCAAiC,EACjC,+BAA+B,GAChC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,EACrB,YAAY,EACZ,qBAAqB,EACrB,iBAAiB,EACjB,mBAAmB,EACnB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,6BAA6B,EAC7B,sBAAsB,EACtB,sBAAsB,GACvB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AACxF,YAAY,EACV,6BAA6B,EAC7B,4BAA4B,EAC5B,mBAAmB,GACpB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,0BAA0B,EAC1B,0BAA0B,EAC1B,6BAA6B,EAC7B,uBAAuB,EACvB,0BAA0B,EAC1B,oCAAoC,EACpC,uBAAuB,EACvB,oBAAoB,EACpB,wBAAwB,GACzB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,yBAAyB,EACzB,uBAAuB,EACvB,WAAW,EACX,SAAS,EACT,UAAU,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,YAAY,EACZ,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,YAAY,EACZ,WAAW,EACX,WAAW,EACX,kBAAkB,EAClB,oBAAoB,EACpB,sBAAsB,EACtB,2BAA2B,EAC3B,2BAA2B,EAC3B,kCAAkC,EAClC,eAAe,EACf,iCAAiC,EACjC,sCAAsC,EACtC,gCAAgC,EAChC,qBAAqB,EACrB,sBAAsB,EACtB,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,oBAAoB,EACpB,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,uCAAuC,EACvC,4CAA4C,EAC5C,6BAA6B,EAC7B,oCAAoC,GACrC,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,qBAAqB,EACrB,yBAAyB,EACzB,kCAAkC,EAClC,oBAAoB,EACpB,qBAAqB,EACrB,kBAAkB,EAClB,uBAAuB,EACvB,uBAAuB,EACvB,wBAAwB,EACxB,eAAe,EACf,uBAAuB,EACvB,iBAAiB,EACjB,+BAA+B,EAC/B,uBAAuB,EACvB,wBAAwB,EACxB,0BAA0B,EAC1B,yBAAyB,EACzB,eAAe,EACf,yBAAyB,EACzB,iBAAiB,EACjB,uBAAuB,EACvB,2BAA2B,EAC3B,0BAA0B,EAC1B,uBAAuB,EACvB,6BAA6B,EAC7B,0BAA0B,EAC1B,oBAAoB,EACpB,gBAAgB,EAChB,+BAA+B,EAC/B,4BAA4B,EAC5B,2BAA2B,EAC3B,uBAAuB,EACvB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,uCAAuC,EACvC,0BAA0B,EAC1B,yBAAyB,EACzB,kBAAkB,EAClB,4BAA4B,EAC5B,iCAAiC,EACjC,uBAAuB,EACvB,oBAAoB,EACpB,cAAc,EACd,mBAAmB,EACnB,oBAAoB,EACpB,qBAAqB,EACrB,iBAAiB,EACjB,6BAA6B,EAC7B,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,gBAAgB,EAChB,kCAAkC,EAClC,eAAe,EACf,gBAAgB,EAChB,8BAA8B,EAC9B,wBAAwB,EACxB,6BAA6B,EAC7B,wBAAwB,EACxB,yBAAyB,EACzB,gCAAgC,EAChC,kBAAkB,EAClB,6BAA6B,EAC7B,2BAA2B,EAC3B,6BAA6B,EAC7B,0BAA0B,EAC1B,gCAAgC,EAChC,gCAAgC,GACjC,MAAM,uBAAuB,CAAC"}
|
|
@@ -3,7 +3,8 @@ export { assertWorkflowRunId, createWorkflowApproval, listWorkflowApprovals, rea
|
|
|
3
3
|
export { freezeWorkflowAttemptContext, isCanonicalWorkflowAttemptContext, normalizeWorkflowAttemptContext, } from "./attempt-context.js";
|
|
4
4
|
export { runWorkflow, WorkflowParkedError, WorkflowRunError } from "./engine.js";
|
|
5
5
|
export { buildWorkflowProof, createEvidenceRecord, deriveRunFailureClass, digestResult, normalizeWorkflowMeta, readWorkflowProof, renderWorkflowProof, rollupAcceptance, writeWorkflowProof, } from "./proof.js";
|
|
6
|
-
export {
|
|
6
|
+
export { beginWorkflowRecovery, readWorkflowRecoveryLedger } from "./recovery-audit.js";
|
|
7
|
+
export { acquireWorkflowResumeLease, assertWorkflowRunResumable, assertWorkflowScriptUnchanged, readWorkflowRunManifest, recoverWorkflowResumeLease, WORKFLOW_RUN_MANIFEST_SCHEMA_VERSION, workflowRunManifestPath, workflowScriptDigest, writeWorkflowRunManifest, } from "./run-state.js";
|
|
7
8
|
export { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.js";
|
|
8
9
|
export { WORKFLOW_ATTEMPT_CONTEXT_SCHEMA_VERSION, WORKFLOW_DIAGNOSTIC_ADMISSION_SCHEMA_VERSION, WORKFLOW_PROOF_SCHEMA_VERSION, WORKFLOW_WORK_CONTEXT_SCHEMA_VERSION, } from "./types.js";
|
|
9
10
|
export { applyIntegration, assertWorkspaceLifecycleTransition, cancelWorkspace, cleanupWorkspace, createLocalGitWorktreeProvider, deriveWorkspaceLifecycle, IntegrationPrepareParkedError, inspectWorkflowWorkspace, isWorkspaceLifecycleState, listWorkflowWorkspaceInspections, prepareIntegration, probeLocalGitWorktreeProvider, readWorkflowWorkspaceStatus, renderWorkflowWorkspaceStatus, validateProviderEventChain, WORKSPACE_BINDING_SCHEMA_VERSION, WORKSPACE_RECEIPT_SCHEMA_VERSION, } from "./workspaces/index.js";
|
|
@@ -14,12 +14,6 @@ export declare function startWorkflowChildSessionV3(input: WorkflowChildSessionV
|
|
|
14
14
|
/** Record the authoritative terminal before removing the disposable coordination cache. */
|
|
15
15
|
export declare function endWorkflowChildSessionV3(input: WorkflowChildSessionV3Input & {
|
|
16
16
|
cleanExit: boolean;
|
|
17
|
-
expectedGenerationId: string;
|
|
18
|
-
}): void;
|
|
19
|
-
/** Persist unresolved terminal custody only for the generation this dispatch owns. */
|
|
20
|
-
export declare function recordWorkflowChildTerminalFailureV3(input: WorkflowChildSessionV3Input & {
|
|
21
|
-
expectedGenerationId: string;
|
|
22
|
-
reason: string;
|
|
23
17
|
}): void;
|
|
24
18
|
export { stableScopeId } from "./scope-id.js";
|
|
25
19
|
//# sourceMappingURL=live-session-v3.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"live-session-v3.d.ts","sourceRoot":"","sources":["../../../src/core/workflow/live-session-v3.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"live-session-v3.d.ts","sourceRoot":"","sources":["../../../src/core/workflow/live-session-v3.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAY7C,MAAM,WAAW,2BAA2B;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,8FAA8F;AAC9F,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,2BAA2B,GAAG,SAAS,CAiDzF;AAED,2FAA2F;AAC3F,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,2BAA2B,GAAG;IAAE,SAAS,EAAE,OAAO,CAAA;CAAE,GAC1D,IAAI,CAmBN;AAgBD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC"}
|
|
@@ -1,21 +1,13 @@
|
|
|
1
|
-
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { existsSync, linkSync, readdirSync, readFileSync, renameSync, unlinkSync } from "node:fs";
|
|
3
|
-
import { basename, dirname } from "node:path";
|
|
4
1
|
import { adapterFromPlatform } from "../adapter.js";
|
|
5
|
-
import {
|
|
2
|
+
import { recordLiveTaskChangeV3 } from "../agents/live-authority-v3.js";
|
|
6
3
|
import { endSessionExplicitV3 } from "../agents/session-finalizer-v3.js";
|
|
7
|
-
import {
|
|
8
|
-
import { readHeartbeat, setIdentityCache, } from "../agents/state/heartbeat-writer.js";
|
|
4
|
+
import { clearCoordinationCache, setIdentityCache, } from "../agents/state/heartbeat-writer.js";
|
|
9
5
|
import { readLiveCoordinationRow } from "../agents/state/live-coordination-view.js";
|
|
10
6
|
import { ensureLiveCoordinationHeartbeat } from "../agents/state/live-coordination-writer.js";
|
|
11
7
|
import { assignName } from "../agents/state/names.js";
|
|
12
8
|
import { coordFreshnessSeconds } from "../config.js";
|
|
13
9
|
import { ensureEventLedgerV3 } from "../events/v3/bootstrap.js";
|
|
14
|
-
import {
|
|
15
|
-
import { withGenerationCustodyV3 } from "../events/v3/generation-custody.js";
|
|
16
|
-
import { liveInstanceIdV3, recordLiveHookSignalV3, resolveLiveEventLedgerRouteV3, } from "../events/v3/live-routing.js";
|
|
17
|
-
import { readHookProducerStateV3 } from "../events/v3/producers/recorder.js";
|
|
18
|
-
import { fsyncParentDirectory } from "./durable-record.js";
|
|
10
|
+
import { recordLiveHookSignalV3, resolveLiveEventLedgerRouteV3, } from "../events/v3/live-routing.js";
|
|
19
11
|
import { stableScopeId } from "./scope-id.js";
|
|
20
12
|
/** Start a headless workflow child through the same canonical V3 producer as native hooks. */
|
|
21
13
|
export function startWorkflowChildSessionV3(input) {
|
|
@@ -62,38 +54,13 @@ export function startWorkflowChildSessionV3(input) {
|
|
|
62
54
|
const projected = readLiveCoordinationRow(input.coordRoot, input.instanceId);
|
|
63
55
|
if (!projected)
|
|
64
56
|
throw new Error("workflow_child_v3_projection_missing");
|
|
65
|
-
const own = readHookProducerStateV3(input.coordRoot, adapter, sessionId);
|
|
66
|
-
if (!own || own.terminal || own.generation_id !== projected.v3_generation_id) {
|
|
67
|
-
throw new Error("workflow_child_v3_start_generation_changed");
|
|
68
|
-
}
|
|
69
57
|
return projected;
|
|
70
58
|
}
|
|
71
59
|
/** Record the authoritative terminal before removing the disposable coordination cache. */
|
|
72
60
|
export function endWorkflowChildSessionV3(input) {
|
|
73
|
-
withGenerationCustodyV3(input.coordRoot, [liveInstanceIdV3(input.instanceId)], () => endExpectedChild(input));
|
|
74
|
-
}
|
|
75
|
-
function endExpectedChild(input) {
|
|
76
61
|
requireV3Route(input.coordRoot);
|
|
77
|
-
const view = requireAuthoritySafeCoordinationViewV3(readCoordinationViewV3(input.coordRoot));
|
|
78
|
-
const instanceId = liveInstanceIdV3(input.instanceId);
|
|
79
|
-
const current = view.instances[instanceId];
|
|
80
|
-
const terminal = view.terminal_generations[input.expectedGenerationId];
|
|
81
|
-
if (terminal?.instance_id === instanceId)
|
|
82
|
-
recoverHeldCaches(input);
|
|
83
|
-
if (current && current.generation_id !== input.expectedGenerationId) {
|
|
84
|
-
throw new Error("workflow_child_v3_generation_changed");
|
|
85
|
-
}
|
|
86
|
-
if (!current) {
|
|
87
|
-
if (!terminal || terminal.instance_id !== instanceId)
|
|
88
|
-
throw new Error("workflow_child_v3_generation_missing");
|
|
89
|
-
if (terminal.terminal?.outcome !== (input.cleanExit ? "succeeded" : "failed")) {
|
|
90
|
-
throw new Error("workflow_child_v3_terminal_outcome_changed");
|
|
91
|
-
}
|
|
92
|
-
clearExpectedCache(input);
|
|
93
|
-
return;
|
|
94
|
-
}
|
|
95
62
|
const row = readLiveCoordinationRow(input.coordRoot, input.instanceId);
|
|
96
|
-
if (!row?.v3_instance_id || row.v3_generation_id
|
|
63
|
+
if (!row?.v3_instance_id || !row.v3_generation_id) {
|
|
97
64
|
throw new Error("workflow_child_v3_generation_missing");
|
|
98
65
|
}
|
|
99
66
|
const result = endSessionExplicitV3({
|
|
@@ -106,95 +73,7 @@ function endExpectedChild(input) {
|
|
|
106
73
|
if (result.state !== "recorded" && result.state !== "already_ended") {
|
|
107
74
|
throw new Error(`workflow_child_v3_end_failed:${result.state}${"reason" in result ? `:${result.reason}` : ""}`);
|
|
108
75
|
}
|
|
109
|
-
|
|
110
|
-
}
|
|
111
|
-
/** Persist unresolved terminal custody only for the generation this dispatch owns. */
|
|
112
|
-
export function recordWorkflowChildTerminalFailureV3(input) {
|
|
113
|
-
withGenerationCustodyV3(input.coordRoot, [liveInstanceIdV3(input.instanceId)], () => {
|
|
114
|
-
const row = readLiveCoordinationRow(input.coordRoot, input.instanceId);
|
|
115
|
-
if (!row || row.v3_generation_id !== input.expectedGenerationId)
|
|
116
|
-
return;
|
|
117
|
-
recordLiveLifecycleChangeV3({
|
|
118
|
-
coordRoot: input.coordRoot,
|
|
119
|
-
owner: input.instanceId,
|
|
120
|
-
nativeSessionId: input.sessionId ?? input.instanceId,
|
|
121
|
-
adapter: workflowAdapter(input.adapter),
|
|
122
|
-
expectedGenerationId: input.expectedGenerationId,
|
|
123
|
-
state: "blocked",
|
|
124
|
-
reason: input.reason,
|
|
125
|
-
});
|
|
126
|
-
});
|
|
127
|
-
}
|
|
128
|
-
function heldPrefix(input) {
|
|
129
|
-
if (!/^gen_[a-zA-Z0-9._-]{1,128}$/.test(input.expectedGenerationId))
|
|
130
|
-
throw new Error("workflow_child_v3_generation_invalid");
|
|
131
|
-
return `${basename(heartbeatPath(input.coordRoot, input.instanceId))}.terminal-${input.expectedGenerationId}-`;
|
|
132
|
-
}
|
|
133
|
-
function recoverHeldCaches(input) {
|
|
134
|
-
const path = heartbeatPath(input.coordRoot, input.instanceId);
|
|
135
|
-
if (!existsSync(dirname(path)))
|
|
136
|
-
return;
|
|
137
|
-
const prefix = heldPrefix(input);
|
|
138
|
-
for (const name of readdirSync(dirname(path)).filter((value) => value.startsWith(prefix))) {
|
|
139
|
-
finishHeldCache(`${dirname(path)}/${name}`, path, input.expectedGenerationId);
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
function clearExpectedCache(input) {
|
|
143
|
-
recoverHeldCaches(input);
|
|
144
|
-
if (readHeartbeat(input.coordRoot, input.instanceId)?.v3_generation_id !==
|
|
145
|
-
input.expectedGenerationId)
|
|
146
|
-
return;
|
|
147
|
-
const path = heartbeatPath(input.coordRoot, input.instanceId);
|
|
148
|
-
const held = `${dirname(path)}/${heldPrefix(input)}${randomUUID()}`;
|
|
149
|
-
try {
|
|
150
|
-
// A unique held name pins the actual bytes atomically. Checking before
|
|
151
|
-
// unlink alone could consume a replacement published at this boundary.
|
|
152
|
-
renameSync(path, held);
|
|
153
|
-
fsyncParentDirectory(path);
|
|
154
|
-
}
|
|
155
|
-
catch (error) {
|
|
156
|
-
if (error.code === "ENOENT")
|
|
157
|
-
return;
|
|
158
|
-
throw error;
|
|
159
|
-
}
|
|
160
|
-
finishHeldCache(held, path, input.expectedGenerationId);
|
|
161
|
-
}
|
|
162
|
-
function finishHeldCache(held, path, expectedGenerationId) {
|
|
163
|
-
let removed = false;
|
|
164
|
-
let cleanupFailed = false;
|
|
165
|
-
let cleanupError;
|
|
166
|
-
try {
|
|
167
|
-
const cache = JSON.parse(readFileSync(held, "utf8"));
|
|
168
|
-
if (cache.v3_generation_id === expectedGenerationId) {
|
|
169
|
-
unlinkSync(held);
|
|
170
|
-
fsyncParentDirectory(held);
|
|
171
|
-
removed = true;
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
catch (error) {
|
|
175
|
-
cleanupFailed = true;
|
|
176
|
-
cleanupError = error;
|
|
177
|
-
}
|
|
178
|
-
if (!removed) {
|
|
179
|
-
try {
|
|
180
|
-
// Never overwrite a later publication. Keep the held evidence when
|
|
181
|
-
// another cache already owns the live name.
|
|
182
|
-
linkSync(held, path);
|
|
183
|
-
fsyncParentDirectory(path);
|
|
184
|
-
unlinkSync(held);
|
|
185
|
-
fsyncParentDirectory(held);
|
|
186
|
-
}
|
|
187
|
-
catch (error) {
|
|
188
|
-
if (error.code !== "EEXIST") {
|
|
189
|
-
if (cleanupFailed) {
|
|
190
|
-
throw new AggregateError([cleanupError, error], `workflow_child_v3_cache_cleanup_and_restore_failed: ${String(cleanupError)}; restoration: ${String(error)}`);
|
|
191
|
-
}
|
|
192
|
-
throw error;
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
if (cleanupFailed)
|
|
197
|
-
throw cleanupError;
|
|
76
|
+
clearCoordinationCache(input.coordRoot, input.instanceId);
|
|
198
77
|
}
|
|
199
78
|
function requireV3Route(coordRoot) {
|
|
200
79
|
let route = resolveLiveEventLedgerRouteV3(coordRoot);
|
|
@@ -72,12 +72,12 @@ export declare function digestResult(value: unknown, kind?: "text" | "json"): Re
|
|
|
72
72
|
* attempt was informative about the work — charge it. This also keeps a
|
|
73
73
|
* resumed run whose earlier segment did real work from being written off by
|
|
74
74
|
* a later environment failure.
|
|
75
|
-
* 3.
|
|
76
|
-
*
|
|
77
|
-
*
|
|
75
|
+
* 3. Otherwise, among the failed agents, environment wins over upstream: a
|
|
76
|
+
* missing binary means nothing ran at all, and it is the operator-chosen
|
|
77
|
+
* hard stop.
|
|
78
78
|
* 4. Anything else is undefined ⇒ a charged work failure, exactly as today.
|
|
79
79
|
*/
|
|
80
|
-
export declare function deriveRunFailureClass(status: "succeeded" | "failed", agents: readonly Pick<WorkflowAgentProof, "status" | "class"
|
|
80
|
+
export declare function deriveRunFailureClass(status: "succeeded" | "failed", agents: readonly Pick<WorkflowAgentProof, "status" | "class">[]): SpawnFailureClass | undefined;
|
|
81
81
|
export declare function buildWorkflowProof(input: BuildWorkflowProofInput): WorkflowProof;
|
|
82
82
|
export declare function writeWorkflowProof(path: string, proof: WorkflowProof): void;
|
|
83
83
|
export declare function readWorkflowProof(coordRoot: string, runId: string): WorkflowProof;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proof.d.ts","sourceRoot":"","sources":["../../../src/core/workflow/proof.ts"],"names":[],"mappings":"AAGA,OAAO,EAA0B,KAAK,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAEtF,OAAO,KAAK,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,mBAAmB,EACpB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"proof.d.ts","sourceRoot":"","sources":["../../../src/core/workflow/proof.ts"],"names":[],"mappings":"AAGA,OAAO,EAA0B,KAAK,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAEtF,OAAO,KAAK,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,mBAAmB,EACpB,MAAM,oBAAoB,CAAC;AAK5B,OAAO,KAAK,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,yBAAyB,EAEzB,YAAY,EAEZ,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EACtB,gCAAgC,EAChC,qBAAqB,EACrB,sBAAsB,EACtB,YAAY,EAEZ,aAAa,EAIb,iCAAiC,EACjC,mBAAmB,EACpB,MAAM,YAAY,CAAC;AAOpB,OAAO,KAAK,EACV,oBAAoB,EACpB,gBAAgB,EAChB,uCAAuC,EAExC,MAAM,uBAAuB,CAAC;AAoB/B,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,mBAAmB,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAC5C,cAAc,CAAC,EAAE,QAAQ,CAAC,sBAAsB,CAAC,CAAC;IAClD,IAAI,EAAE,sBAAsB,CAAC;IAC7B,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,kBAAkB,EAAE,CAAC;IAC7B,QAAQ,EAAE,sBAAsB,EAAE,CAAC;IACnC,mBAAmB,CAAC,EAAE,gCAAgC,CAAC;IACvD,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,yBAAyB,GAAG,SAAS,CAAC,CAAC,CAAC;IAClF,qEAAqE;IACrE,iBAAiB,CAAC,EAAE,iCAAiC,CAAC;IACtD;+DAC2D;IAC3D,mBAAmB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,0BAA0B,GAAG,SAAS,CAAC,CAAC,CAAC;IACvF,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CAAC;QACnC,SAAS,EAAE,SAAS,cAAc,EAAE,CAAC;QACrC,SAAS,EAAE,eAAe,CAAC;QAC3B,aAAa,EAAE,mBAAmB,CAAC;KACpC,CAAC;IACF;mFAC+E;IAC/E,OAAO,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAClD,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,iBAAiB,CAAC,EAAE,uCAAuC,CAAC;IAC5D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,YAAY,GAAG,SAAS,EAC9B,YAAY,EAAE,MAAM,GACnB,sBAAsB,CAqCxB;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,KAAK,EAAE,qBAAqB,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,sBAAsB,CA8BzB;AAED,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,mBAAmB,EAAE,EAC/B,QAAQ,EAAE,sBAAsB,EAAE,GACjC;IAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,CAuB9D;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,YAAY,CAQjF;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,WAAW,GAAG,QAAQ,EAC9B,MAAM,EAAE,SAAS,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,CAAC,EAAE,GAC9D,iBAAiB,GAAG,SAAS,CAS/B;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,uBAAuB,GAAG,aAAa,CA8DhF;AAgCD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,IAAI,CAO3E;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,aAAa,CAqEjF;AA+DD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM,CA+DhE"}
|
|
@@ -6,7 +6,6 @@ import { DIAGNOSTIC_ADVICE_SCHEMA_VERSION } from "../diagnostics/contract.js";
|
|
|
6
6
|
import { policyDigest } from "../policy/index.js";
|
|
7
7
|
import { isCanonicalWorkflowAttemptContext } from "./attempt-context.js";
|
|
8
8
|
import { stableDigest, writeImmutableJson } from "./durable-record.js";
|
|
9
|
-
import { failureEvidenceAllowsUncharged, isSpawnFailureEvidence } from "./failure-evidence.js";
|
|
10
9
|
import { readWorkflowRunManifest } from "./run-state.js";
|
|
11
10
|
import { EVIDENCE_KINDS, WORKFLOW_DIAGNOSTIC_ADMISSION_SCHEMA_VERSION, WORKFLOW_PROOF_SCHEMA_VERSION, } from "./types.js";
|
|
12
11
|
import { isCanonicalWorkflowWorkContext } from "./work-context.js";
|
|
@@ -127,18 +126,15 @@ export function digestResult(value, kind) {
|
|
|
127
126
|
* attempt was informative about the work — charge it. This also keeps a
|
|
128
127
|
* resumed run whose earlier segment did real work from being written off by
|
|
129
128
|
* a later environment failure.
|
|
130
|
-
* 3.
|
|
131
|
-
*
|
|
132
|
-
*
|
|
129
|
+
* 3. Otherwise, among the failed agents, environment wins over upstream: a
|
|
130
|
+
* missing binary means nothing ran at all, and it is the operator-chosen
|
|
131
|
+
* hard stop.
|
|
133
132
|
* 4. Anything else is undefined ⇒ a charged work failure, exactly as today.
|
|
134
133
|
*/
|
|
135
134
|
export function deriveRunFailureClass(status, agents) {
|
|
136
135
|
if (status !== "failed")
|
|
137
136
|
return undefined;
|
|
138
|
-
if (agents.
|
|
139
|
-
agents.some((agent) => agent.status !== "failed" ||
|
|
140
|
-
(agent.class !== "environment" && agent.class !== "upstream") ||
|
|
141
|
-
!failureEvidenceAllowsUncharged(agent.failure_evidence))) {
|
|
137
|
+
if (agents.some((agent) => agent.status === "succeeded" || agent.status === "cached")) {
|
|
142
138
|
return undefined;
|
|
143
139
|
}
|
|
144
140
|
const failed = agents.filter((agent) => agent.status === "failed");
|
|
@@ -267,10 +263,6 @@ export function readWorkflowProof(coordRoot, runId) {
|
|
|
267
263
|
proof.run.class !== "environment" &&
|
|
268
264
|
proof.run.class !== "upstream" &&
|
|
269
265
|
proof.run.class !== "decision") ||
|
|
270
|
-
!Array.isArray(proof.agents) ||
|
|
271
|
-
proof.agents.some((agent) => !agent ||
|
|
272
|
-
typeof agent !== "object" ||
|
|
273
|
-
(agent.failure_evidence !== undefined && !isSpawnFailureEvidence(agent.failure_evidence))) ||
|
|
274
266
|
(proof.run.work_context !== undefined &&
|
|
275
267
|
(!proof.run.work_item_id ||
|
|
276
268
|
proof.run.work_context.id !== proof.run.work_item_id ||
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export interface WorkflowRecoveryAuthorization {
|
|
2
|
+
actor: string;
|
|
3
|
+
reason: string;
|
|
4
|
+
}
|
|
5
|
+
export interface WorkflowRecoveryIntent extends WorkflowRecoveryAuthorization {
|
|
6
|
+
schema_version: 1;
|
|
7
|
+
operation_id: string;
|
|
8
|
+
run_id: string;
|
|
9
|
+
action: "reclaim_unknown_liveness" | "unlock_resume";
|
|
10
|
+
target_sha256: string;
|
|
11
|
+
target: Record<string, unknown>;
|
|
12
|
+
recorded_at: string;
|
|
13
|
+
pid: number;
|
|
14
|
+
host: string;
|
|
15
|
+
outcome: "unknown";
|
|
16
|
+
}
|
|
17
|
+
export interface WorkflowRecoveryRecord {
|
|
18
|
+
intent: WorkflowRecoveryIntent;
|
|
19
|
+
outcome: "unknown" | "completed";
|
|
20
|
+
}
|
|
21
|
+
export declare function validateWorkflowRecoveryAuthorization(authorization: WorkflowRecoveryAuthorization): void;
|
|
22
|
+
/** Canonical recovery ledger: immutable intent is durable before effects.
|
|
23
|
+
* Completion cites that exact intent; absence always means unknown. */
|
|
24
|
+
export declare function beginWorkflowRecovery(coordRoot: string, input: Omit<WorkflowRecoveryIntent, "schema_version" | "operation_id" | "recorded_at" | "pid" | "host" | "outcome">): {
|
|
25
|
+
intent: WorkflowRecoveryIntent;
|
|
26
|
+
assertCurrent(): void;
|
|
27
|
+
complete(): void;
|
|
28
|
+
};
|
|
29
|
+
export declare function readWorkflowRecoveryLedger(coordRoot: string, runId: string): WorkflowRecoveryRecord[];
|
|
30
|
+
//# sourceMappingURL=recovery-audit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recovery-audit.d.ts","sourceRoot":"","sources":["../../../src/core/workflow/recovery-audit.ts"],"names":[],"mappings":"AAaA,MAAM,WAAW,6BAA6B;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,sBAAuB,SAAQ,6BAA6B;IAC3E,cAAc,EAAE,CAAC,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,0BAA0B,GAAG,eAAe,CAAC;IACrD,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,SAAS,CAAC;CACpB;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,sBAAsB,CAAC;IAC/B,OAAO,EAAE,SAAS,GAAG,WAAW,CAAC;CAClC;AAED,wBAAgB,qCAAqC,CACnD,aAAa,EAAE,6BAA6B,GAC3C,IAAI,CAmBN;AAED;uEACuE;AACvE,wBAAgB,qBAAqB,CACnC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,IAAI,CACT,sBAAsB,EACtB,gBAAgB,GAAG,cAAc,GAAG,aAAa,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,CAC/E,GACA;IAAE,MAAM,EAAE,sBAAsB,CAAC;IAAC,aAAa,IAAI,IAAI,CAAC;IAAC,QAAQ,IAAI,IAAI,CAAA;CAAE,CAmD7E;AAkBD,wBAAgB,0BAA0B,CACxC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,EAAE,CAyC1B"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { existsSync, lstatSync, mkdirSync, readdirSync } from "node:fs";
|
|
3
|
+
import { hostname } from "node:os";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
|
+
import { stateDirMode } from "../storage/modes.js";
|
|
6
|
+
import { assertWorkflowRunId } from "./approvals.js";
|
|
7
|
+
import { fsyncParentDirectory, readJsonRecord, stableDigest, writeImmutableJson, } from "./durable-record.js";
|
|
8
|
+
export function validateWorkflowRecoveryAuthorization(authorization) {
|
|
9
|
+
for (const [field, limit] of [
|
|
10
|
+
["actor", 200],
|
|
11
|
+
["reason", 2_000],
|
|
12
|
+
]) {
|
|
13
|
+
const value = authorization[field];
|
|
14
|
+
if (typeof value !== "string" ||
|
|
15
|
+
!value.trim() ||
|
|
16
|
+
value.length > limit ||
|
|
17
|
+
Array.from(value).some((character) => character.charCodeAt(0) < 32 || character.charCodeAt(0) === 127)) {
|
|
18
|
+
throw new Error(`workflow recovery requires a nonempty ${field} of at most ${limit} characters without control characters`);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
/** Canonical recovery ledger: immutable intent is durable before effects.
|
|
23
|
+
* Completion cites that exact intent; absence always means unknown. */
|
|
24
|
+
export function beginWorkflowRecovery(coordRoot, input) {
|
|
25
|
+
assertWorkflowRunId(input.run_id);
|
|
26
|
+
validateWorkflowRecoveryAuthorization(input);
|
|
27
|
+
if (!/^[a-f0-9]{64}$/.test(input.target_sha256))
|
|
28
|
+
throw new Error("workflow recovery requires an exact preview digest");
|
|
29
|
+
const intent = {
|
|
30
|
+
...input,
|
|
31
|
+
schema_version: 1,
|
|
32
|
+
operation_id: `recovery-${randomUUID()}`,
|
|
33
|
+
recorded_at: new Date().toISOString(),
|
|
34
|
+
pid: process.pid,
|
|
35
|
+
host: hostname(),
|
|
36
|
+
outcome: "unknown",
|
|
37
|
+
};
|
|
38
|
+
const directory = join(coordRoot, ".harnery", "workflows", input.run_id, "recovery", intent.operation_id);
|
|
39
|
+
// Sync each newly published directory name too. Syncing only the intent's
|
|
40
|
+
// immediate parent cannot make a newly created ancestor durable.
|
|
41
|
+
const runDirectory = join(coordRoot, ".harnery", "workflows", input.run_id);
|
|
42
|
+
ensureDurableDirectory(runDirectory);
|
|
43
|
+
const ledgerDirectory = join(runDirectory, "recovery");
|
|
44
|
+
ensureDurableDirectory(ledgerDirectory);
|
|
45
|
+
fsyncParentDirectory(ledgerDirectory);
|
|
46
|
+
mkdirSync(directory, { mode: stateDirMode() });
|
|
47
|
+
fsyncParentDirectory(directory);
|
|
48
|
+
const intentPath = join(directory, "intent.json");
|
|
49
|
+
writeImmutableJson(intentPath, intent);
|
|
50
|
+
const assertCurrent = () => {
|
|
51
|
+
if (stableDigest(readJsonRecord(intentPath, "recovery intent")) !== stableDigest(intent))
|
|
52
|
+
throw new Error("workflow recovery audit authority changed");
|
|
53
|
+
};
|
|
54
|
+
return {
|
|
55
|
+
intent,
|
|
56
|
+
assertCurrent,
|
|
57
|
+
complete: () => {
|
|
58
|
+
assertCurrent();
|
|
59
|
+
writeImmutableJson(join(directory, "completion.json"), {
|
|
60
|
+
schema_version: 1,
|
|
61
|
+
operation_id: intent.operation_id,
|
|
62
|
+
intent_sha256: stableDigest(intent),
|
|
63
|
+
outcome: "completed",
|
|
64
|
+
recorded_at: new Date().toISOString(),
|
|
65
|
+
});
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
function ensureDurableDirectory(path) {
|
|
70
|
+
if (existsSync(path)) {
|
|
71
|
+
if (!lstatSync(path).isDirectory())
|
|
72
|
+
throw new Error("workflow recovery ledger is not a directory");
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
ensureDurableDirectory(dirname(path));
|
|
76
|
+
try {
|
|
77
|
+
mkdirSync(path, { mode: stateDirMode() });
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
if (error.code !== "EEXIST" || !lstatSync(path).isDirectory())
|
|
81
|
+
throw error;
|
|
82
|
+
}
|
|
83
|
+
fsyncParentDirectory(path);
|
|
84
|
+
}
|
|
85
|
+
export function readWorkflowRecoveryLedger(coordRoot, runId) {
|
|
86
|
+
assertWorkflowRunId(runId);
|
|
87
|
+
const directory = join(coordRoot, ".harnery", "workflows", runId, "recovery");
|
|
88
|
+
if (!existsSync(directory))
|
|
89
|
+
return [];
|
|
90
|
+
return readdirSync(directory)
|
|
91
|
+
.sort()
|
|
92
|
+
.map((entry) => {
|
|
93
|
+
if (!/^recovery-[a-f0-9-]{36}$/.test(entry))
|
|
94
|
+
throw new Error("invalid workflow recovery ledger entry");
|
|
95
|
+
const intent = readJsonRecord(join(directory, entry, "intent.json"), "recovery intent");
|
|
96
|
+
if (intent.schema_version !== 1 ||
|
|
97
|
+
intent.operation_id !== entry ||
|
|
98
|
+
intent.run_id !== runId ||
|
|
99
|
+
intent.outcome !== "unknown" ||
|
|
100
|
+
!["reclaim_unknown_liveness", "unlock_resume"].includes(intent.action) ||
|
|
101
|
+
!/^[a-f0-9]{64}$/.test(intent.target_sha256)) {
|
|
102
|
+
throw new Error("invalid workflow recovery intent authority");
|
|
103
|
+
}
|
|
104
|
+
validateWorkflowRecoveryAuthorization(intent);
|
|
105
|
+
const completionPath = join(directory, entry, "completion.json");
|
|
106
|
+
if (!existsSync(completionPath))
|
|
107
|
+
return { intent, outcome: "unknown" };
|
|
108
|
+
const completion = readJsonRecord(completionPath, "recovery completion");
|
|
109
|
+
if (completion.schema_version !== 1 ||
|
|
110
|
+
completion.operation_id !== entry ||
|
|
111
|
+
completion.intent_sha256 !== stableDigest(intent) ||
|
|
112
|
+
completion.outcome !== "completed")
|
|
113
|
+
throw new Error("invalid workflow recovery completion authority");
|
|
114
|
+
return { intent, outcome: "completed" };
|
|
115
|
+
});
|
|
116
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { RepoSnapshot } from "../context/index.js";
|
|
2
2
|
import { type NormalizedPolicy, type PolicyIsolation, type PolicyNetworkAccess } from "../policy/index.js";
|
|
3
|
+
import { type WorkflowRecoveryAuthorization } from "./recovery-audit.js";
|
|
3
4
|
import type { WorkflowAttemptContext, WorkflowDiagnosticAdmissionConfig, WorkflowSpecialistProfile, WorkflowWorkContext } from "./types.js";
|
|
4
5
|
import type { WorkspaceBinding, WorkspaceCompatibilityExecutionEvidence } from "./workspaces/index.js";
|
|
5
6
|
export declare const WORKFLOW_RUN_MANIFEST_SCHEMA_VERSION: 2;
|
|
@@ -48,7 +49,23 @@ export declare function assertWorkflowRunResumable(coordRoot: string, runId: str
|
|
|
48
49
|
export declare function assertWorkflowScriptUnchanged(manifest: WorkflowRunManifest): void;
|
|
49
50
|
export declare function workflowRunManifestPath(coordRoot: string, runId: string): string;
|
|
50
51
|
/** Acquire an exclusive lease shared by resume and reclaim. Existing locks
|
|
51
|
-
* always refuse
|
|
52
|
-
* Crash recovery requires an audited ownership check outside this operation. */
|
|
52
|
+
* always refuse; explicit audited recovery is a separate operation. */
|
|
53
53
|
export declare function acquireWorkflowResumeLease(coordRoot: string, runId: string): () => void;
|
|
54
|
+
export interface WorkflowResumeRecoveryOptions extends Partial<WorkflowRecoveryAuthorization> {
|
|
55
|
+
yes?: boolean;
|
|
56
|
+
lockSha256?: string;
|
|
57
|
+
}
|
|
58
|
+
export interface WorkflowResumeRecoveryResult {
|
|
59
|
+
action: "preview" | "unlocked";
|
|
60
|
+
run_id: string;
|
|
61
|
+
lock_sha256: string | null;
|
|
62
|
+
owner_state: "dead" | "live" | "unknown" | "absent";
|
|
63
|
+
owner: {
|
|
64
|
+
pid: number;
|
|
65
|
+
host: string;
|
|
66
|
+
created_at: string;
|
|
67
|
+
} | null;
|
|
68
|
+
recovery_id?: string;
|
|
69
|
+
}
|
|
70
|
+
export declare function recoverWorkflowResumeLease(coordRoot: string, runId: string, options: WorkflowResumeRecoveryOptions): WorkflowResumeRecoveryResult;
|
|
54
71
|
//# sourceMappingURL=run-state.d.ts.map
|