harnery 0.44.0 → 0.45.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +44 -0
- package/dist/core/agents/coord-client.d.ts.map +1 -1
- package/dist/core/agents/coord-client.js +18 -0
- package/dist/core/agents/live-authority-v3.d.ts +0 -1
- package/dist/core/agents/live-authority-v3.d.ts.map +1 -1
- package/dist/core/agents/live-authority-v3.js +0 -11
- package/dist/core/agents/live-lifecycle-v3.d.ts +2 -4
- package/dist/core/agents/live-lifecycle-v3.d.ts.map +1 -1
- package/dist/core/agents/live-lifecycle-v3.js +7 -28
- package/dist/core/agents/state/heartbeat-writer.d.ts.map +1 -1
- package/dist/core/agents/state/heartbeat-writer.js +14 -25
- package/dist/core/agents/state/live-coordination-writer.d.ts.map +1 -1
- package/dist/core/agents/state/live-coordination-writer.js +0 -5
- package/dist/core/agents/state/pidmap.d.ts.map +1 -1
- package/dist/core/agents/state/pidmap.js +0 -5
- package/dist/core/agents/state/proc-start.d.ts +5 -6
- package/dist/core/agents/state/proc-start.d.ts.map +1 -1
- package/dist/core/agents/state/proc-start.js +13 -9
- package/dist/core/agents/state/process-handle.d.ts +26 -0
- package/dist/core/agents/state/process-handle.d.ts.map +1 -0
- package/dist/core/agents/state/process-handle.js +121 -0
- package/dist/core/agents/state/stale-sweep.d.ts.map +1 -1
- package/dist/core/agents/state/stale-sweep.js +70 -180
- package/dist/core/artifacts/activity.d.ts +4 -1
- package/dist/core/artifacts/activity.d.ts.map +1 -1
- package/dist/core/artifacts/activity.js +8 -2
- package/dist/core/artifacts/index.d.ts.map +1 -1
- package/dist/core/artifacts/index.js +38 -12
- package/dist/core/events/v3/authority-outbox.d.ts.map +1 -1
- package/dist/core/events/v3/authority-outbox.js +0 -29
- package/dist/core/events/v3/producers/coordination-recorder.d.ts +0 -1
- package/dist/core/events/v3/producers/coordination-recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/coordination-recorder.js +0 -7
- package/dist/core/events/v3/producers/recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/recorder.js +64 -77
- package/dist/core/hooks/cli.js +78 -48
- package/dist/core/resources/service.d.ts +2 -16
- package/dist/core/resources/service.d.ts.map +1 -1
- package/dist/core/resources/service.js +1 -78
- package/dist/core/semantic/service.d.ts.map +1 -1
- package/dist/core/semantic/service.js +2 -9
- package/dist/core/servers/index.d.ts +4 -0
- package/dist/core/servers/index.d.ts.map +1 -1
- package/dist/core/servers/index.js +21 -3
- package/dist/core/storage/durable-history.js +56 -22
- package/dist/core/storage/segments.d.ts.map +1 -1
- package/dist/core/storage/segments.js +73 -17
- package/dist/core/supervisor/service.d.ts +2 -4
- package/dist/core/supervisor/service.d.ts.map +1 -1
- package/dist/core/work/state.js +7 -7
- package/dist/core/workflow/engine.d.ts.map +1 -1
- package/dist/core/workflow/engine.js +21 -65
- package/dist/core/workflow/index.d.ts +4 -2
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +2 -1
- package/dist/core/workflow/live-session-v3.d.ts +0 -6
- package/dist/core/workflow/live-session-v3.d.ts.map +1 -1
- package/dist/core/workflow/live-session-v3.js +5 -126
- package/dist/core/workflow/proof.d.ts +4 -4
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +4 -12
- package/dist/core/workflow/recovery-audit.d.ts +30 -0
- package/dist/core/workflow/recovery-audit.d.ts.map +1 -0
- package/dist/core/workflow/recovery-audit.js +116 -0
- package/dist/core/workflow/run-state.d.ts +19 -2
- package/dist/core/workflow/run-state.d.ts.map +1 -1
- package/dist/core/workflow/run-state.js +113 -8
- package/dist/core/workflow/spawn-claude.d.ts.map +1 -1
- package/dist/core/workflow/spawn-claude.js +0 -19
- package/dist/core/workflow/spawn-codex.d.ts.map +1 -1
- package/dist/core/workflow/spawn-codex.js +0 -13
- package/dist/core/workflow/spawn-cursor.d.ts.map +1 -1
- package/dist/core/workflow/spawn-cursor.js +0 -19
- package/dist/core/workflow/spawn-opencode.d.ts.map +1 -1
- package/dist/core/workflow/spawn-opencode.js +0 -14
- package/dist/core/workflow/types.d.ts +11 -54
- package/dist/core/workflow/types.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/local-git.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/local-git.js +70 -14
- package/dist/core/workflow/workspaces/reclaim.d.ts +6 -1
- package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.d.ts +2 -0
- package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.js +24 -0
- package/dist/lib/identities/assume.d.ts.map +1 -1
- package/dist/lib/identities/assume.js +0 -4
- package/package.json +1 -1
- package/src/commands/workflow.ts +59 -0
- package/src/core/agents/coord-client.ts +20 -0
- package/src/core/agents/live-authority-v3.ts +0 -17
- package/src/core/agents/live-lifecycle-v3.ts +14 -52
- package/src/core/agents/state/heartbeat-writer.ts +12 -29
- package/src/core/agents/state/live-coordination-writer.ts +0 -29
- package/src/core/agents/state/pidmap.ts +0 -13
- package/src/core/agents/state/proc-start.ts +14 -11
- package/src/core/agents/state/process-handle.ts +165 -0
- package/src/core/agents/state/stale-sweep.ts +82 -247
- package/src/core/artifacts/activity.ts +9 -2
- package/src/core/artifacts/index.ts +45 -12
- package/src/core/events/v3/authority-outbox.ts +0 -56
- package/src/core/events/v3/producers/coordination-recorder.ts +0 -15
- package/src/core/events/v3/producers/recorder.ts +68 -94
- package/src/core/hooks/cli.ts +85 -47
- package/src/core/resources/service.ts +3 -88
- package/src/core/semantic/service.ts +2 -8
- package/src/core/servers/index.ts +30 -3
- package/src/core/storage/durable-history.ts +52 -20
- package/src/core/storage/segments.ts +79 -17
- package/src/core/supervisor/service.ts +2 -3
- package/src/core/work/state.ts +7 -7
- package/src/core/workflow/engine.ts +21 -73
- package/src/core/workflow/index.ts +12 -1
- package/src/core/workflow/live-session-v3.ts +5 -146
- package/src/core/workflow/proof.ts +5 -21
- package/src/core/workflow/recovery-audit.ts +181 -0
- package/src/core/workflow/run-state.ts +137 -8
- package/src/core/workflow/spawn-claude.ts +0 -18
- package/src/core/workflow/spawn-codex.ts +0 -13
- package/src/core/workflow/spawn-cursor.ts +0 -18
- package/src/core/workflow/spawn-opencode.ts +0 -14
- package/src/core/workflow/types.ts +11 -41
- package/src/core/workflow/workspaces/local-git.ts +86 -15
- package/src/core/workflow/workspaces/reclaim.ts +4 -0
- package/src/core/workflow/workspaces/recovery.ts +23 -0
- package/src/lib/identities/assume.ts +0 -4
- package/dist/core/agents/state/unchanged-file.d.ts +0 -44
- package/dist/core/agents/state/unchanged-file.d.ts.map +0 -1
- package/dist/core/agents/state/unchanged-file.js +0 -390
- package/dist/core/events/v3/generation-custody.d.ts +0 -5
- package/dist/core/events/v3/generation-custody.d.ts.map +0 -1
- package/dist/core/events/v3/generation-custody.js +0 -50
- package/dist/core/workflow/failure-evidence.d.ts +0 -16
- package/dist/core/workflow/failure-evidence.d.ts.map +0 -1
- package/dist/core/workflow/failure-evidence.js +0 -114
- package/src/core/agents/state/unchanged-file.ts +0 -456
- package/src/core/events/v3/generation-custody.ts +0 -59
- package/src/core/workflow/failure-evidence.ts +0 -141
|
@@ -12,7 +12,6 @@ import type {
|
|
|
12
12
|
import { policyDigest } from "../policy/index.ts";
|
|
13
13
|
import { isCanonicalWorkflowAttemptContext } from "./attempt-context.ts";
|
|
14
14
|
import { stableDigest, writeImmutableJson } from "./durable-record.ts";
|
|
15
|
-
import { failureEvidenceAllowsUncharged, isSpawnFailureEvidence } from "./failure-evidence.ts";
|
|
16
15
|
import { readWorkflowRunManifest } from "./run-state.ts";
|
|
17
16
|
import type {
|
|
18
17
|
AcceptanceCriterion,
|
|
@@ -243,25 +242,17 @@ export function digestResult(value: unknown, kind?: "text" | "json"): ResultDige
|
|
|
243
242
|
* attempt was informative about the work — charge it. This also keeps a
|
|
244
243
|
* resumed run whose earlier segment did real work from being written off by
|
|
245
244
|
* a later environment failure.
|
|
246
|
-
* 3.
|
|
247
|
-
*
|
|
248
|
-
*
|
|
245
|
+
* 3. Otherwise, among the failed agents, environment wins over upstream: a
|
|
246
|
+
* missing binary means nothing ran at all, and it is the operator-chosen
|
|
247
|
+
* hard stop.
|
|
249
248
|
* 4. Anything else is undefined ⇒ a charged work failure, exactly as today.
|
|
250
249
|
*/
|
|
251
250
|
export function deriveRunFailureClass(
|
|
252
251
|
status: "succeeded" | "failed",
|
|
253
|
-
agents: readonly Pick<WorkflowAgentProof, "status" | "class"
|
|
252
|
+
agents: readonly Pick<WorkflowAgentProof, "status" | "class">[],
|
|
254
253
|
): SpawnFailureClass | undefined {
|
|
255
254
|
if (status !== "failed") return undefined;
|
|
256
|
-
if (
|
|
257
|
-
agents.length === 0 ||
|
|
258
|
-
agents.some(
|
|
259
|
-
(agent) =>
|
|
260
|
-
agent.status !== "failed" ||
|
|
261
|
-
(agent.class !== "environment" && agent.class !== "upstream") ||
|
|
262
|
-
!failureEvidenceAllowsUncharged(agent.failure_evidence),
|
|
263
|
-
)
|
|
264
|
-
) {
|
|
255
|
+
if (agents.some((agent) => agent.status === "succeeded" || agent.status === "cached")) {
|
|
265
256
|
return undefined;
|
|
266
257
|
}
|
|
267
258
|
const failed = agents.filter((agent) => agent.status === "failed");
|
|
@@ -394,13 +385,6 @@ export function readWorkflowProof(coordRoot: string, runId: string): WorkflowPro
|
|
|
394
385
|
proof.run.class !== "environment" &&
|
|
395
386
|
proof.run.class !== "upstream" &&
|
|
396
387
|
proof.run.class !== "decision") ||
|
|
397
|
-
!Array.isArray(proof.agents) ||
|
|
398
|
-
proof.agents.some(
|
|
399
|
-
(agent) =>
|
|
400
|
-
!agent ||
|
|
401
|
-
typeof agent !== "object" ||
|
|
402
|
-
(agent.failure_evidence !== undefined && !isSpawnFailureEvidence(agent.failure_evidence)),
|
|
403
|
-
) ||
|
|
404
388
|
(proof.run.work_context !== undefined &&
|
|
405
389
|
(!proof.run.work_item_id ||
|
|
406
390
|
proof.run.work_context.id !== proof.run.work_item_id ||
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { existsSync, lstatSync, mkdirSync, readdirSync } from "node:fs";
|
|
3
|
+
import { hostname } from "node:os";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
|
+
import { stateDirMode } from "../storage/modes.ts";
|
|
6
|
+
import { assertWorkflowRunId } from "./approvals.ts";
|
|
7
|
+
import {
|
|
8
|
+
fsyncParentDirectory,
|
|
9
|
+
readJsonRecord,
|
|
10
|
+
stableDigest,
|
|
11
|
+
writeImmutableJson,
|
|
12
|
+
} from "./durable-record.ts";
|
|
13
|
+
|
|
14
|
+
export interface WorkflowRecoveryAuthorization {
|
|
15
|
+
actor: string;
|
|
16
|
+
reason: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface WorkflowRecoveryIntent extends WorkflowRecoveryAuthorization {
|
|
20
|
+
schema_version: 1;
|
|
21
|
+
operation_id: string;
|
|
22
|
+
run_id: string;
|
|
23
|
+
action: "reclaim_unknown_liveness" | "unlock_resume";
|
|
24
|
+
target_sha256: string;
|
|
25
|
+
target: Record<string, unknown>;
|
|
26
|
+
recorded_at: string;
|
|
27
|
+
pid: number;
|
|
28
|
+
host: string;
|
|
29
|
+
outcome: "unknown";
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface WorkflowRecoveryRecord {
|
|
33
|
+
intent: WorkflowRecoveryIntent;
|
|
34
|
+
outcome: "unknown" | "completed";
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function validateWorkflowRecoveryAuthorization(
|
|
38
|
+
authorization: WorkflowRecoveryAuthorization,
|
|
39
|
+
): void {
|
|
40
|
+
for (const [field, limit] of [
|
|
41
|
+
["actor", 200],
|
|
42
|
+
["reason", 2_000],
|
|
43
|
+
] as const) {
|
|
44
|
+
const value = authorization[field];
|
|
45
|
+
if (
|
|
46
|
+
typeof value !== "string" ||
|
|
47
|
+
!value.trim() ||
|
|
48
|
+
value.length > limit ||
|
|
49
|
+
Array.from(value).some(
|
|
50
|
+
(character) => character.charCodeAt(0) < 32 || character.charCodeAt(0) === 127,
|
|
51
|
+
)
|
|
52
|
+
) {
|
|
53
|
+
throw new Error(
|
|
54
|
+
`workflow recovery requires a nonempty ${field} of at most ${limit} characters without control characters`,
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Canonical recovery ledger: immutable intent is durable before effects.
|
|
61
|
+
* Completion cites that exact intent; absence always means unknown. */
|
|
62
|
+
export function beginWorkflowRecovery(
|
|
63
|
+
coordRoot: string,
|
|
64
|
+
input: Omit<
|
|
65
|
+
WorkflowRecoveryIntent,
|
|
66
|
+
"schema_version" | "operation_id" | "recorded_at" | "pid" | "host" | "outcome"
|
|
67
|
+
>,
|
|
68
|
+
): { intent: WorkflowRecoveryIntent; assertCurrent(): void; complete(): void } {
|
|
69
|
+
assertWorkflowRunId(input.run_id);
|
|
70
|
+
validateWorkflowRecoveryAuthorization(input);
|
|
71
|
+
if (!/^[a-f0-9]{64}$/.test(input.target_sha256))
|
|
72
|
+
throw new Error("workflow recovery requires an exact preview digest");
|
|
73
|
+
const intent: WorkflowRecoveryIntent = {
|
|
74
|
+
...input,
|
|
75
|
+
schema_version: 1,
|
|
76
|
+
operation_id: `recovery-${randomUUID()}`,
|
|
77
|
+
recorded_at: new Date().toISOString(),
|
|
78
|
+
pid: process.pid,
|
|
79
|
+
host: hostname(),
|
|
80
|
+
outcome: "unknown",
|
|
81
|
+
};
|
|
82
|
+
const directory = join(
|
|
83
|
+
coordRoot,
|
|
84
|
+
".harnery",
|
|
85
|
+
"workflows",
|
|
86
|
+
input.run_id,
|
|
87
|
+
"recovery",
|
|
88
|
+
intent.operation_id,
|
|
89
|
+
);
|
|
90
|
+
// Sync each newly published directory name too. Syncing only the intent's
|
|
91
|
+
// immediate parent cannot make a newly created ancestor durable.
|
|
92
|
+
const runDirectory = join(coordRoot, ".harnery", "workflows", input.run_id);
|
|
93
|
+
ensureDurableDirectory(runDirectory);
|
|
94
|
+
const ledgerDirectory = join(runDirectory, "recovery");
|
|
95
|
+
ensureDurableDirectory(ledgerDirectory);
|
|
96
|
+
fsyncParentDirectory(ledgerDirectory);
|
|
97
|
+
mkdirSync(directory, { mode: stateDirMode() });
|
|
98
|
+
fsyncParentDirectory(directory);
|
|
99
|
+
const intentPath = join(directory, "intent.json");
|
|
100
|
+
writeImmutableJson(intentPath, intent);
|
|
101
|
+
const assertCurrent = () => {
|
|
102
|
+
if (stableDigest(readJsonRecord(intentPath, "recovery intent")) !== stableDigest(intent))
|
|
103
|
+
throw new Error("workflow recovery audit authority changed");
|
|
104
|
+
};
|
|
105
|
+
return {
|
|
106
|
+
intent,
|
|
107
|
+
assertCurrent,
|
|
108
|
+
complete: () => {
|
|
109
|
+
assertCurrent();
|
|
110
|
+
writeImmutableJson(join(directory, "completion.json"), {
|
|
111
|
+
schema_version: 1,
|
|
112
|
+
operation_id: intent.operation_id,
|
|
113
|
+
intent_sha256: stableDigest(intent),
|
|
114
|
+
outcome: "completed",
|
|
115
|
+
recorded_at: new Date().toISOString(),
|
|
116
|
+
});
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function ensureDurableDirectory(path: string): void {
|
|
122
|
+
if (existsSync(path)) {
|
|
123
|
+
if (!lstatSync(path).isDirectory())
|
|
124
|
+
throw new Error("workflow recovery ledger is not a directory");
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
ensureDurableDirectory(dirname(path));
|
|
128
|
+
try {
|
|
129
|
+
mkdirSync(path, { mode: stateDirMode() });
|
|
130
|
+
} catch (error) {
|
|
131
|
+
if ((error as NodeJS.ErrnoException).code !== "EEXIST" || !lstatSync(path).isDirectory())
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
134
|
+
fsyncParentDirectory(path);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export function readWorkflowRecoveryLedger(
|
|
138
|
+
coordRoot: string,
|
|
139
|
+
runId: string,
|
|
140
|
+
): WorkflowRecoveryRecord[] {
|
|
141
|
+
assertWorkflowRunId(runId);
|
|
142
|
+
const directory = join(coordRoot, ".harnery", "workflows", runId, "recovery");
|
|
143
|
+
if (!existsSync(directory)) return [];
|
|
144
|
+
return readdirSync(directory)
|
|
145
|
+
.sort()
|
|
146
|
+
.map((entry) => {
|
|
147
|
+
if (!/^recovery-[a-f0-9-]{36}$/.test(entry))
|
|
148
|
+
throw new Error("invalid workflow recovery ledger entry");
|
|
149
|
+
const intent = readJsonRecord<WorkflowRecoveryIntent>(
|
|
150
|
+
join(directory, entry, "intent.json"),
|
|
151
|
+
"recovery intent",
|
|
152
|
+
);
|
|
153
|
+
if (
|
|
154
|
+
intent.schema_version !== 1 ||
|
|
155
|
+
intent.operation_id !== entry ||
|
|
156
|
+
intent.run_id !== runId ||
|
|
157
|
+
intent.outcome !== "unknown" ||
|
|
158
|
+
!["reclaim_unknown_liveness", "unlock_resume"].includes(intent.action) ||
|
|
159
|
+
!/^[a-f0-9]{64}$/.test(intent.target_sha256)
|
|
160
|
+
) {
|
|
161
|
+
throw new Error("invalid workflow recovery intent authority");
|
|
162
|
+
}
|
|
163
|
+
validateWorkflowRecoveryAuthorization(intent);
|
|
164
|
+
const completionPath = join(directory, entry, "completion.json");
|
|
165
|
+
if (!existsSync(completionPath)) return { intent, outcome: "unknown" };
|
|
166
|
+
const completion = readJsonRecord<{
|
|
167
|
+
schema_version: number;
|
|
168
|
+
operation_id: string;
|
|
169
|
+
intent_sha256: string;
|
|
170
|
+
outcome: string;
|
|
171
|
+
}>(completionPath, "recovery completion");
|
|
172
|
+
if (
|
|
173
|
+
completion.schema_version !== 1 ||
|
|
174
|
+
completion.operation_id !== entry ||
|
|
175
|
+
completion.intent_sha256 !== stableDigest(intent) ||
|
|
176
|
+
completion.outcome !== "completed"
|
|
177
|
+
)
|
|
178
|
+
throw new Error("invalid workflow recovery completion authority");
|
|
179
|
+
return { intent, outcome: "completed" };
|
|
180
|
+
});
|
|
181
|
+
}
|
|
@@ -3,6 +3,8 @@ import {
|
|
|
3
3
|
chmodSync,
|
|
4
4
|
closeSync,
|
|
5
5
|
existsSync,
|
|
6
|
+
fsyncSync,
|
|
7
|
+
lstatSync,
|
|
6
8
|
mkdirSync,
|
|
7
9
|
openSync,
|
|
8
10
|
readFileSync,
|
|
@@ -23,7 +25,13 @@ import {
|
|
|
23
25
|
import { stateFileMode } from "../storage/modes.ts";
|
|
24
26
|
import { assertWorkflowRunId, readWorkflowApproval } from "./approvals.ts";
|
|
25
27
|
import { isCanonicalWorkflowAttemptContext } from "./attempt-context.ts";
|
|
26
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
fsyncParentDirectory,
|
|
30
|
+
readJsonRecord,
|
|
31
|
+
stableDigest,
|
|
32
|
+
writeImmutableJson,
|
|
33
|
+
} from "./durable-record.ts";
|
|
34
|
+
import { beginWorkflowRecovery, type WorkflowRecoveryAuthorization } from "./recovery-audit.ts";
|
|
27
35
|
import { normalizeWorkflowSpecialists } from "./specialists.ts";
|
|
28
36
|
import type {
|
|
29
37
|
WorkflowAttemptContext,
|
|
@@ -38,6 +46,7 @@ import type {
|
|
|
38
46
|
WorkspaceBinding,
|
|
39
47
|
WorkspaceCompatibilityExecutionEvidence,
|
|
40
48
|
} from "./workspaces/index.ts";
|
|
49
|
+
import { acquireNoClobberLease } from "./workspaces/leases.ts";
|
|
41
50
|
import {
|
|
42
51
|
assertWorkspaceManifestAuthority,
|
|
43
52
|
isWorkspaceBinding,
|
|
@@ -204,8 +213,7 @@ export function workflowRunManifestPath(coordRoot: string, runId: string): strin
|
|
|
204
213
|
}
|
|
205
214
|
|
|
206
215
|
/** Acquire an exclusive lease shared by resume and reclaim. Existing locks
|
|
207
|
-
* always refuse
|
|
208
|
-
* Crash recovery requires an audited ownership check outside this operation. */
|
|
216
|
+
* always refuse; explicit audited recovery is a separate operation. */
|
|
209
217
|
export function acquireWorkflowResumeLease(coordRoot: string, runId: string): () => void {
|
|
210
218
|
assertWorkflowRunId(runId);
|
|
211
219
|
const path = join(coordRoot, ".harnery", "workflows", runId, "resume.lock");
|
|
@@ -218,23 +226,27 @@ export function acquireWorkflowResumeLease(coordRoot: string, runId: string): ()
|
|
|
218
226
|
const ownerBytes = `${JSON.stringify(owner)}\n`;
|
|
219
227
|
mkdirSync(dirname(path), { recursive: true });
|
|
220
228
|
|
|
221
|
-
const acquire = ():
|
|
229
|
+
const acquire = (): { device: number; inode: number } | false => {
|
|
222
230
|
try {
|
|
223
231
|
const fd = openSync(path, "wx", stateFileMode());
|
|
224
232
|
try {
|
|
225
233
|
writeFileSync(fd, ownerBytes, "utf8");
|
|
234
|
+
fsyncSync(fd);
|
|
226
235
|
} finally {
|
|
227
236
|
closeSync(fd);
|
|
228
237
|
}
|
|
229
238
|
chmodSync(path, stateFileMode());
|
|
230
|
-
|
|
239
|
+
fsyncParentDirectory(path);
|
|
240
|
+
const identity = lstatSync(path);
|
|
241
|
+
return { device: identity.dev, inode: identity.ino };
|
|
231
242
|
} catch (error) {
|
|
232
243
|
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
233
244
|
return false;
|
|
234
245
|
}
|
|
235
246
|
};
|
|
236
247
|
|
|
237
|
-
|
|
248
|
+
const acquired = withResumeLeaseMutation(coordRoot, runId, acquire);
|
|
249
|
+
if (!acquired) {
|
|
238
250
|
const existing = readResumeLease(path);
|
|
239
251
|
const holder = existing
|
|
240
252
|
? ` by pid ${existing.pid} on ${existing.host}; its liveness has not been verified`
|
|
@@ -244,13 +256,129 @@ export function acquireWorkflowResumeLease(coordRoot: string, runId: string): ()
|
|
|
244
256
|
|
|
245
257
|
return () => {
|
|
246
258
|
try {
|
|
247
|
-
|
|
259
|
+
// Only this live process can release this nonce. Acquisition never
|
|
260
|
+
// replaces a present lock, and unlock requires ESRCH for its owner, so
|
|
261
|
+
// neither supported writer can replace this lock during synchronous
|
|
262
|
+
// release. Taking their mutation gate would strand a live owner's lock
|
|
263
|
+
// on ordinary contention. Pin both identity and bytes for stale closures.
|
|
264
|
+
const identity = lstatSync(path);
|
|
265
|
+
if (
|
|
266
|
+
identity.isFile() &&
|
|
267
|
+
identity.dev === acquired.device &&
|
|
268
|
+
identity.ino === acquired.inode &&
|
|
269
|
+
readFileSync(path, "utf8") === ownerBytes
|
|
270
|
+
) {
|
|
271
|
+
unlinkSync(path);
|
|
272
|
+
fsyncParentDirectory(path);
|
|
273
|
+
}
|
|
248
274
|
} catch {
|
|
249
275
|
// Leave an unreadable or replaced lock untouched.
|
|
250
276
|
}
|
|
251
277
|
};
|
|
252
278
|
}
|
|
253
279
|
|
|
280
|
+
export interface WorkflowResumeRecoveryOptions extends Partial<WorkflowRecoveryAuthorization> {
|
|
281
|
+
yes?: boolean;
|
|
282
|
+
lockSha256?: string;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
export interface WorkflowResumeRecoveryResult {
|
|
286
|
+
action: "preview" | "unlocked";
|
|
287
|
+
run_id: string;
|
|
288
|
+
lock_sha256: string | null;
|
|
289
|
+
owner_state: "dead" | "live" | "unknown" | "absent";
|
|
290
|
+
owner: { pid: number; host: string; created_at: string } | null;
|
|
291
|
+
recovery_id?: string;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
export function recoverWorkflowResumeLease(
|
|
295
|
+
coordRoot: string,
|
|
296
|
+
runId: string,
|
|
297
|
+
options: WorkflowResumeRecoveryOptions,
|
|
298
|
+
): WorkflowResumeRecoveryResult {
|
|
299
|
+
assertWorkflowRunId(runId);
|
|
300
|
+
const path = join(coordRoot, ".harnery", "workflows", runId, "resume.lock");
|
|
301
|
+
const inspect = (): WorkflowResumeRecoveryResult => {
|
|
302
|
+
let lockSha256: string;
|
|
303
|
+
try {
|
|
304
|
+
const stat = lstatSync(path);
|
|
305
|
+
if (!stat.isFile() || stat.size > 8 * 1024)
|
|
306
|
+
throw new Error("resume lock is not a bounded regular file");
|
|
307
|
+
lockSha256 = stableDigest({
|
|
308
|
+
bytes: readFileSync(path, "utf8"),
|
|
309
|
+
device: stat.dev,
|
|
310
|
+
inode: stat.ino,
|
|
311
|
+
});
|
|
312
|
+
} catch (error) {
|
|
313
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
314
|
+
return {
|
|
315
|
+
action: "preview",
|
|
316
|
+
run_id: runId,
|
|
317
|
+
lock_sha256: null,
|
|
318
|
+
owner_state: "absent",
|
|
319
|
+
owner: null,
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
const owner = readResumeLease(path);
|
|
323
|
+
let ownerState: WorkflowResumeRecoveryResult["owner_state"] = "unknown";
|
|
324
|
+
if (owner?.host === hostname()) {
|
|
325
|
+
try {
|
|
326
|
+
process.kill(owner.pid, 0);
|
|
327
|
+
ownerState = "live";
|
|
328
|
+
} catch (error) {
|
|
329
|
+
if ((error as NodeJS.ErrnoException).code === "ESRCH") ownerState = "dead";
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return {
|
|
333
|
+
action: "preview",
|
|
334
|
+
run_id: runId,
|
|
335
|
+
lock_sha256: lockSha256,
|
|
336
|
+
owner_state: ownerState,
|
|
337
|
+
owner,
|
|
338
|
+
};
|
|
339
|
+
};
|
|
340
|
+
if (!options.yes) return inspect();
|
|
341
|
+
return withResumeLeaseMutation(coordRoot, runId, () => {
|
|
342
|
+
const observed = inspect();
|
|
343
|
+
if (observed.owner_state !== "dead")
|
|
344
|
+
throw new Error(`workflow resume unlock refused: owner is ${observed.owner_state}`);
|
|
345
|
+
if (!options.lockSha256 || options.lockSha256 !== observed.lock_sha256)
|
|
346
|
+
throw new Error("workflow resume unlock requires the exact current preview digest");
|
|
347
|
+
const audit = beginWorkflowRecovery(coordRoot, {
|
|
348
|
+
run_id: runId,
|
|
349
|
+
action: "unlock_resume",
|
|
350
|
+
target_sha256: options.lockSha256,
|
|
351
|
+
target: { lock: "resume.lock", owner: observed.owner },
|
|
352
|
+
actor: options.actor!,
|
|
353
|
+
reason: options.reason!,
|
|
354
|
+
});
|
|
355
|
+
const current = inspect();
|
|
356
|
+
if (current.lock_sha256 !== observed.lock_sha256 || current.owner_state !== "dead")
|
|
357
|
+
throw new Error("workflow resume lock authority changed before unlock");
|
|
358
|
+
audit.assertCurrent();
|
|
359
|
+
unlinkSync(path);
|
|
360
|
+
fsyncParentDirectory(path);
|
|
361
|
+
audit.complete();
|
|
362
|
+
return { ...observed, action: "unlocked", recovery_id: audit.intent.operation_id };
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
function withResumeLeaseMutation<T>(coordRoot: string, runId: string, action: () => T): T {
|
|
367
|
+
const authority = stableDigest({ run_id: runId, lock: "resume.lock" });
|
|
368
|
+
const lease = acquireNoClobberLease({
|
|
369
|
+
path: join(coordRoot, ".harnery", "workflows", runId, "resume-mutation.lease"),
|
|
370
|
+
scope: "workflow-resume-mutation",
|
|
371
|
+
authoritySha256: authority,
|
|
372
|
+
staleAfterMs: 300_000,
|
|
373
|
+
validateStaleOwner: (owner) => owner.authority_sha256 === authority,
|
|
374
|
+
});
|
|
375
|
+
try {
|
|
376
|
+
return action();
|
|
377
|
+
} finally {
|
|
378
|
+
lease.release();
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
254
382
|
function readResumeLease(path: string): { pid: number; host: string; created_at: string } | null {
|
|
255
383
|
try {
|
|
256
384
|
const value = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>;
|
|
@@ -258,7 +386,8 @@ function readResumeLease(path: string): { pid: number; host: string; created_at:
|
|
|
258
386
|
Number.isSafeInteger(value.pid) &&
|
|
259
387
|
value.pid > 0 &&
|
|
260
388
|
typeof value.host === "string" &&
|
|
261
|
-
|
|
389
|
+
value.host.length > 0 &&
|
|
390
|
+
validTimestamp(value.created_at)
|
|
262
391
|
? { pid: value.pid, host: value.host, created_at: value.created_at }
|
|
263
392
|
: null;
|
|
264
393
|
} catch {
|
|
@@ -23,7 +23,6 @@ import { builtinAdapterProfile, validateAdapterEffort } from "../adapters/profil
|
|
|
23
23
|
import type { AdapterInvocation, AdapterRawResult } from "../adapters/types.ts";
|
|
24
24
|
import { notFoundError } from "./adapters.ts";
|
|
25
25
|
import { buildChildEnv } from "./child-env.ts";
|
|
26
|
-
import { observeSpawnFailure } from "./failure-evidence.ts";
|
|
27
26
|
import { resolveFullAccessArgv, resolveSandboxProjection } from "./sandbox-projection.ts";
|
|
28
27
|
import { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.ts";
|
|
29
28
|
import type { Spawner, SpawnRequest, SpawnResult } from "./types.ts";
|
|
@@ -67,23 +66,6 @@ export function buildClaudeInvocation(req: SpawnRequest): AdapterInvocation {
|
|
|
67
66
|
}
|
|
68
67
|
|
|
69
68
|
export function normalizeClaudeResult(raw: AdapterRawResult): SpawnResult {
|
|
70
|
-
const result = normalizeClaudeTransportResult(raw);
|
|
71
|
-
if (result.ok) return result;
|
|
72
|
-
let text = raw.stdout.trim();
|
|
73
|
-
let source: "stdout" | "vendor_envelope" = "stdout";
|
|
74
|
-
try {
|
|
75
|
-
const envelope = JSON.parse(raw.stdout) as ClaudeEnvelope;
|
|
76
|
-
if (typeof envelope.result === "string" && envelope.result.length > 0) {
|
|
77
|
-
text = envelope.result;
|
|
78
|
-
source = "vendor_envelope";
|
|
79
|
-
}
|
|
80
|
-
} catch {
|
|
81
|
-
// Retain unstructured output without treating it as native authority.
|
|
82
|
-
}
|
|
83
|
-
return { ...result, text, failureEvidence: observeSpawnFailure(raw, text, source) };
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
function normalizeClaudeTransportResult(raw: AdapterRawResult): SpawnResult {
|
|
87
69
|
if (raw.timedOut) {
|
|
88
70
|
return {
|
|
89
71
|
ok: false,
|
|
@@ -26,7 +26,6 @@ import { builtinAdapterProfile, validateAdapterEffort } from "../adapters/profil
|
|
|
26
26
|
import type { AdapterInvocation, AdapterRawResult } from "../adapters/types.ts";
|
|
27
27
|
import { notFoundError } from "./adapters.ts";
|
|
28
28
|
import { buildChildEnv } from "./child-env.ts";
|
|
29
|
-
import { observeSpawnFailure } from "./failure-evidence.ts";
|
|
30
29
|
import { resolveFullAccessArgv, resolveSandboxProjection } from "./sandbox-projection.ts";
|
|
31
30
|
import { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.ts";
|
|
32
31
|
import type { Spawner, SpawnRequest, SpawnResult } from "./types.ts";
|
|
@@ -79,18 +78,6 @@ export function buildCodexInvocation(req: SpawnRequest, resultFile?: string): Ad
|
|
|
79
78
|
}
|
|
80
79
|
|
|
81
80
|
export function normalizeCodexResult(raw: AdapterRawResult): SpawnResult {
|
|
82
|
-
const result = normalizeCodexTransportResult(raw);
|
|
83
|
-
if (result.ok) return result;
|
|
84
|
-
const finalText = raw.resultFileText?.trim();
|
|
85
|
-
const text = finalText || raw.stdout.trim();
|
|
86
|
-
return {
|
|
87
|
-
...result,
|
|
88
|
-
text,
|
|
89
|
-
failureEvidence: observeSpawnFailure(raw, text, finalText ? "final_message" : "stdout"),
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
function normalizeCodexTransportResult(raw: AdapterRawResult): SpawnResult {
|
|
94
81
|
if (raw.timedOut) {
|
|
95
82
|
return {
|
|
96
83
|
ok: false,
|
|
@@ -24,7 +24,6 @@ import { builtinAdapterProfile, validateAdapterEffort } from "../adapters/profil
|
|
|
24
24
|
import type { AdapterInvocation, AdapterRawResult } from "../adapters/types.ts";
|
|
25
25
|
import { notFoundError } from "./adapters.ts";
|
|
26
26
|
import { buildChildEnv } from "./child-env.ts";
|
|
27
|
-
import { observeSpawnFailure } from "./failure-evidence.ts";
|
|
28
27
|
import { resolveFullAccessArgv, resolveSandboxProjection } from "./sandbox-projection.ts";
|
|
29
28
|
import { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.ts";
|
|
30
29
|
import type { Spawner, SpawnRequest, SpawnResult } from "./types.ts";
|
|
@@ -79,23 +78,6 @@ export function buildCursorInvocation(req: SpawnRequest): AdapterInvocation {
|
|
|
79
78
|
}
|
|
80
79
|
|
|
81
80
|
export function normalizeCursorResult(raw: AdapterRawResult): SpawnResult {
|
|
82
|
-
const result = normalizeCursorTransportResult(raw);
|
|
83
|
-
if (result.ok) return result;
|
|
84
|
-
let text = raw.stdout;
|
|
85
|
-
let source: "stdout" | "vendor_envelope" = "stdout";
|
|
86
|
-
try {
|
|
87
|
-
const envelope = JSON.parse(raw.stdout) as CursorEnvelope;
|
|
88
|
-
if (typeof envelope.result === "string" && envelope.result.length > 0) {
|
|
89
|
-
text = envelope.result;
|
|
90
|
-
source = "vendor_envelope";
|
|
91
|
-
}
|
|
92
|
-
} catch {
|
|
93
|
-
// Raw output remains evidence, not an inferred refusal or effect result.
|
|
94
|
-
}
|
|
95
|
-
return { ...result, text, failureEvidence: observeSpawnFailure(raw, text, source) };
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
function normalizeCursorTransportResult(raw: AdapterRawResult): SpawnResult {
|
|
99
81
|
if (raw.timedOut) {
|
|
100
82
|
return {
|
|
101
83
|
ok: false,
|
|
@@ -22,7 +22,6 @@ import { builtinAdapterProfile, validateAdapterEffort } from "../adapters/profil
|
|
|
22
22
|
import type { AdapterInvocation, AdapterRawResult } from "../adapters/types.ts";
|
|
23
23
|
import { notFoundError } from "./adapters.ts";
|
|
24
24
|
import { buildChildEnv } from "./child-env.ts";
|
|
25
|
-
import { observeSpawnFailure } from "./failure-evidence.ts";
|
|
26
25
|
import { resolveFullAccessArgv, resolveSandboxProjection } from "./sandbox-projection.ts";
|
|
27
26
|
import { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.ts";
|
|
28
27
|
import type { Spawner, SpawnRequest, SpawnResult } from "./types.ts";
|
|
@@ -89,19 +88,6 @@ export function buildOpenCodeInvocation(req: SpawnRequest): AdapterInvocation {
|
|
|
89
88
|
}
|
|
90
89
|
|
|
91
90
|
export function normalizeOpenCodeResult(raw: AdapterRawResult): SpawnResult {
|
|
92
|
-
const result = normalizeOpenCodeTransportResult(raw);
|
|
93
|
-
if (result.ok) return result;
|
|
94
|
-
const parsed = parseOpenCodeStream(raw.stdout);
|
|
95
|
-
const text = parsed.text || raw.stdout.trim();
|
|
96
|
-
return {
|
|
97
|
-
...result,
|
|
98
|
-
text,
|
|
99
|
-
sessionId: parsed.sessionId,
|
|
100
|
-
failureEvidence: observeSpawnFailure(raw, text, parsed.text ? "vendor_envelope" : "stdout"),
|
|
101
|
-
};
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
function normalizeOpenCodeTransportResult(raw: AdapterRawResult): SpawnResult {
|
|
105
91
|
if (raw.timedOut) {
|
|
106
92
|
return {
|
|
107
93
|
ok: false,
|
|
@@ -137,12 +137,11 @@ export interface WorkflowAgentProof {
|
|
|
137
137
|
session_id?: string;
|
|
138
138
|
result?: ResultDigest;
|
|
139
139
|
error?: string;
|
|
140
|
-
/**
|
|
141
|
-
*
|
|
140
|
+
/** Set on a failed agent whose spawn was classified uninformative about the
|
|
141
|
+
* work (ADR 0046): environment (binary absent) or upstream (vendor refused).
|
|
142
|
+
* Absent ⇒ a work failure. Recorded even when a script's parallel() swallows
|
|
143
|
+
* the rejection, so the run-level class can still be derived from proof. */
|
|
142
144
|
class?: SpawnFailureClass;
|
|
143
|
-
/** Failure observations carried from the adapter. Missing observations are
|
|
144
|
-
* unknown and cannot authorize an uncharged attempt. */
|
|
145
|
-
failure_evidence?: SpawnFailureEvidence;
|
|
146
145
|
/** Which vendor sandbox mode this agent launched with and why (ADR 0192).
|
|
147
146
|
* Absent on a cached agent, which launched nothing. */
|
|
148
147
|
access?: WorkerAccessDecision;
|
|
@@ -363,16 +362,15 @@ export interface WorkflowSpecialistProfile {
|
|
|
363
362
|
}
|
|
364
363
|
|
|
365
364
|
/**
|
|
366
|
-
*
|
|
367
|
-
*
|
|
365
|
+
* Why a failed run was uninformative about the work (ADR 0046). Absent means
|
|
366
|
+
* the attempt produced information about the work (or succeeded), which is the
|
|
367
|
+
* default and is charged against the attempt budget exactly as before.
|
|
368
368
|
*
|
|
369
369
|
* - `environment`: the run never started — a precondition was missing (the
|
|
370
|
-
* vendor binary was absent).
|
|
371
|
-
* uncharged and not retried: retrying an unchanged
|
|
370
|
+
* vendor binary was absent). Uncharged AND not retried: retrying an unchanged
|
|
372
371
|
* environment cannot help, so the work item stops and names the precondition.
|
|
373
372
|
* - `upstream`: the vendor was reached and refused (5xx, 429, circuit open).
|
|
374
|
-
*
|
|
375
|
-
* observed absent output, and no refusal. Unsupported evidence stays unknown.
|
|
373
|
+
* Uncharged, but retry stays available (bounded by max_uncharged_attempts).
|
|
376
374
|
*/
|
|
377
375
|
export type SpawnFailureClass = "environment" | "upstream";
|
|
378
376
|
|
|
@@ -394,30 +392,6 @@ export type SpawnFailureClass = "environment" | "upstream";
|
|
|
394
392
|
*/
|
|
395
393
|
export type RunFailureClass = SpawnFailureClass | "decision";
|
|
396
394
|
|
|
397
|
-
/** Positive claims name the observation that supports them. A transport error
|
|
398
|
-
* or an empty reply is never evidence that no effects occurred. */
|
|
399
|
-
export type SpawnEffectEvidence =
|
|
400
|
-
| { state: "unknown" }
|
|
401
|
-
| { state: "none"; source: "spawn_errno"; reference: "ENOENT" }
|
|
402
|
-
| { state: "none" | "observed"; source: "adapter_evidence"; reference: string };
|
|
403
|
-
|
|
404
|
-
export type SpawnRefusalEvidence =
|
|
405
|
-
| { state: "unknown" }
|
|
406
|
-
| { state: "refused"; source: "vendor_envelope"; reference: string };
|
|
407
|
-
|
|
408
|
-
export type SpawnOutputSource = "final_message" | "vendor_envelope" | "stdout";
|
|
409
|
-
|
|
410
|
-
export type SpawnPartialOutputEvidence =
|
|
411
|
-
| { state: "unknown" }
|
|
412
|
-
| { state: "absent"; source: SpawnOutputSource }
|
|
413
|
-
| { state: "present"; source: SpawnOutputSource; bytes: number; sha256: string };
|
|
414
|
-
|
|
415
|
-
export interface SpawnFailureEvidence {
|
|
416
|
-
effect: SpawnEffectEvidence;
|
|
417
|
-
refusal: SpawnRefusalEvidence;
|
|
418
|
-
partial_output: SpawnPartialOutputEvidence;
|
|
419
|
-
}
|
|
420
|
-
|
|
421
395
|
/** What a spawn adapter returns for one subagent run. */
|
|
422
396
|
export interface SpawnResult {
|
|
423
397
|
ok: boolean;
|
|
@@ -429,13 +403,9 @@ export interface SpawnResult {
|
|
|
429
403
|
durationMs: number;
|
|
430
404
|
/** Populated when ok=false. */
|
|
431
405
|
error?: string;
|
|
432
|
-
/**
|
|
433
|
-
*
|
|
406
|
+
/** Set when ok=false and the failure was positively identified as
|
|
407
|
+
* uninformative about the work. Absent ⇒ a work failure (charged). */
|
|
434
408
|
class?: SpawnFailureClass;
|
|
435
|
-
/** Transport classification alone grants neither a retry nor free work.
|
|
436
|
-
* Absent evidence is explicitly unknown. Positive adapter_evidence claims
|
|
437
|
-
* require a documented producer observation, never an HTTP/text heuristic. */
|
|
438
|
-
failureEvidence?: SpawnFailureEvidence;
|
|
439
409
|
}
|
|
440
410
|
|
|
441
411
|
export interface SpawnRequest {
|