harnery 0.44.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commander.js +1 -0
- package/dist/commands/files.d.ts +2 -2
- package/dist/commands/files.d.ts.map +1 -1
- package/dist/commands/files.js +7 -4
- package/dist/commands/servers.d.ts.map +1 -1
- package/dist/commands/servers.js +11 -17
- package/dist/commands/tunnel.d.ts +15 -0
- package/dist/commands/tunnel.d.ts.map +1 -1
- package/dist/commands/tunnel.js +453 -153
- package/dist/commands/turn-rollback.d.ts +5 -0
- package/dist/commands/turn-rollback.d.ts.map +1 -0
- package/dist/commands/turn-rollback.js +76 -0
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +44 -0
- package/dist/core/agents/coord-client.d.ts.map +1 -1
- package/dist/core/agents/coord-client.js +18 -0
- package/dist/core/agents/state/proc-start.d.ts +5 -6
- package/dist/core/agents/state/proc-start.d.ts.map +1 -1
- package/dist/core/agents/state/proc-start.js +13 -9
- package/dist/core/agents/state/process-handle.d.ts +26 -0
- package/dist/core/agents/state/process-handle.d.ts.map +1 -0
- package/dist/core/agents/state/process-handle.js +121 -0
- package/dist/core/artifacts/delivery-card.d.ts.map +1 -1
- package/dist/core/artifacts/delivery-card.js +2 -1
- package/dist/core/resources/service.d.ts +2 -16
- package/dist/core/resources/service.d.ts.map +1 -1
- package/dist/core/resources/service.js +1 -78
- package/dist/core/semantic/service.d.ts.map +1 -1
- package/dist/core/semantic/service.js +2 -9
- package/dist/core/servers/index.d.ts +12 -11
- package/dist/core/servers/index.d.ts.map +1 -1
- package/dist/core/servers/index.js +194 -81
- package/dist/core/servers/tunnels.d.ts +5 -3
- package/dist/core/servers/tunnels.d.ts.map +1 -1
- package/dist/core/servers/tunnels.js +45 -2
- package/dist/core/supervisor/service.d.ts +2 -4
- package/dist/core/supervisor/service.d.ts.map +1 -1
- package/dist/core/turn-rollback/content.d.ts +53 -0
- package/dist/core/turn-rollback/content.d.ts.map +1 -0
- package/dist/core/turn-rollback/content.js +396 -0
- package/dist/core/turn-rollback/index.d.ts +72 -0
- package/dist/core/turn-rollback/index.d.ts.map +1 -0
- package/dist/core/turn-rollback/index.js +443 -0
- package/dist/core/workflow/index.d.ts +4 -2
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +2 -1
- package/dist/core/workflow/recovery-audit.d.ts +30 -0
- package/dist/core/workflow/recovery-audit.d.ts.map +1 -0
- package/dist/core/workflow/recovery-audit.js +116 -0
- package/dist/core/workflow/run-state.d.ts +19 -2
- package/dist/core/workflow/run-state.d.ts.map +1 -1
- package/dist/core/workflow/run-state.js +113 -8
- package/dist/core/workflow/workspaces/local-git.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/local-git.js +70 -14
- package/dist/core/workflow/workspaces/reclaim.d.ts +6 -1
- package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.d.ts +2 -0
- package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.js +24 -0
- package/dist/lib/tunnel/state.d.ts +13 -6
- package/dist/lib/tunnel/state.d.ts.map +1 -1
- package/dist/lib/tunnel/state.js +58 -12
- package/package.json +6 -1
- package/src/commander.ts +10 -0
- package/src/commands/files.ts +13 -5
- package/src/commands/servers.ts +12 -23
- package/src/commands/tunnel.ts +533 -166
- package/src/commands/turn-rollback.ts +111 -0
- package/src/commands/workflow.ts +59 -0
- package/src/core/agents/coord-client.ts +20 -0
- package/src/core/agents/state/proc-start.ts +14 -11
- package/src/core/agents/state/process-handle.ts +165 -0
- package/src/core/artifacts/delivery-card.ts +7 -1
- package/src/core/resources/service.ts +3 -88
- package/src/core/semantic/service.ts +2 -8
- package/src/core/servers/index.ts +234 -85
- package/src/core/servers/tunnels.ts +55 -3
- package/src/core/supervisor/service.ts +2 -3
- package/src/core/turn-rollback/content.ts +499 -0
- package/src/core/turn-rollback/index.ts +580 -0
- package/src/core/workflow/index.ts +12 -1
- package/src/core/workflow/recovery-audit.ts +181 -0
- package/src/core/workflow/run-state.ts +137 -8
- package/src/core/workflow/workspaces/local-git.ts +86 -15
- package/src/core/workflow/workspaces/reclaim.ts +4 -0
- package/src/core/workflow/workspaces/recovery.ts +23 -0
- package/src/lib/tunnel/state.ts +70 -15
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { existsSync, lstatSync, mkdirSync, readdirSync } from "node:fs";
|
|
3
|
+
import { hostname } from "node:os";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
|
+
import { stateDirMode } from "../storage/modes.ts";
|
|
6
|
+
import { assertWorkflowRunId } from "./approvals.ts";
|
|
7
|
+
import {
|
|
8
|
+
fsyncParentDirectory,
|
|
9
|
+
readJsonRecord,
|
|
10
|
+
stableDigest,
|
|
11
|
+
writeImmutableJson,
|
|
12
|
+
} from "./durable-record.ts";
|
|
13
|
+
|
|
14
|
+
export interface WorkflowRecoveryAuthorization {
|
|
15
|
+
actor: string;
|
|
16
|
+
reason: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface WorkflowRecoveryIntent extends WorkflowRecoveryAuthorization {
|
|
20
|
+
schema_version: 1;
|
|
21
|
+
operation_id: string;
|
|
22
|
+
run_id: string;
|
|
23
|
+
action: "reclaim_unknown_liveness" | "unlock_resume";
|
|
24
|
+
target_sha256: string;
|
|
25
|
+
target: Record<string, unknown>;
|
|
26
|
+
recorded_at: string;
|
|
27
|
+
pid: number;
|
|
28
|
+
host: string;
|
|
29
|
+
outcome: "unknown";
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface WorkflowRecoveryRecord {
|
|
33
|
+
intent: WorkflowRecoveryIntent;
|
|
34
|
+
outcome: "unknown" | "completed";
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function validateWorkflowRecoveryAuthorization(
|
|
38
|
+
authorization: WorkflowRecoveryAuthorization,
|
|
39
|
+
): void {
|
|
40
|
+
for (const [field, limit] of [
|
|
41
|
+
["actor", 200],
|
|
42
|
+
["reason", 2_000],
|
|
43
|
+
] as const) {
|
|
44
|
+
const value = authorization[field];
|
|
45
|
+
if (
|
|
46
|
+
typeof value !== "string" ||
|
|
47
|
+
!value.trim() ||
|
|
48
|
+
value.length > limit ||
|
|
49
|
+
Array.from(value).some(
|
|
50
|
+
(character) => character.charCodeAt(0) < 32 || character.charCodeAt(0) === 127,
|
|
51
|
+
)
|
|
52
|
+
) {
|
|
53
|
+
throw new Error(
|
|
54
|
+
`workflow recovery requires a nonempty ${field} of at most ${limit} characters without control characters`,
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Canonical recovery ledger: immutable intent is durable before effects.
|
|
61
|
+
* Completion cites that exact intent; absence always means unknown. */
|
|
62
|
+
export function beginWorkflowRecovery(
|
|
63
|
+
coordRoot: string,
|
|
64
|
+
input: Omit<
|
|
65
|
+
WorkflowRecoveryIntent,
|
|
66
|
+
"schema_version" | "operation_id" | "recorded_at" | "pid" | "host" | "outcome"
|
|
67
|
+
>,
|
|
68
|
+
): { intent: WorkflowRecoveryIntent; assertCurrent(): void; complete(): void } {
|
|
69
|
+
assertWorkflowRunId(input.run_id);
|
|
70
|
+
validateWorkflowRecoveryAuthorization(input);
|
|
71
|
+
if (!/^[a-f0-9]{64}$/.test(input.target_sha256))
|
|
72
|
+
throw new Error("workflow recovery requires an exact preview digest");
|
|
73
|
+
const intent: WorkflowRecoveryIntent = {
|
|
74
|
+
...input,
|
|
75
|
+
schema_version: 1,
|
|
76
|
+
operation_id: `recovery-${randomUUID()}`,
|
|
77
|
+
recorded_at: new Date().toISOString(),
|
|
78
|
+
pid: process.pid,
|
|
79
|
+
host: hostname(),
|
|
80
|
+
outcome: "unknown",
|
|
81
|
+
};
|
|
82
|
+
const directory = join(
|
|
83
|
+
coordRoot,
|
|
84
|
+
".harnery",
|
|
85
|
+
"workflows",
|
|
86
|
+
input.run_id,
|
|
87
|
+
"recovery",
|
|
88
|
+
intent.operation_id,
|
|
89
|
+
);
|
|
90
|
+
// Sync each newly published directory name too. Syncing only the intent's
|
|
91
|
+
// immediate parent cannot make a newly created ancestor durable.
|
|
92
|
+
const runDirectory = join(coordRoot, ".harnery", "workflows", input.run_id);
|
|
93
|
+
ensureDurableDirectory(runDirectory);
|
|
94
|
+
const ledgerDirectory = join(runDirectory, "recovery");
|
|
95
|
+
ensureDurableDirectory(ledgerDirectory);
|
|
96
|
+
fsyncParentDirectory(ledgerDirectory);
|
|
97
|
+
mkdirSync(directory, { mode: stateDirMode() });
|
|
98
|
+
fsyncParentDirectory(directory);
|
|
99
|
+
const intentPath = join(directory, "intent.json");
|
|
100
|
+
writeImmutableJson(intentPath, intent);
|
|
101
|
+
const assertCurrent = () => {
|
|
102
|
+
if (stableDigest(readJsonRecord(intentPath, "recovery intent")) !== stableDigest(intent))
|
|
103
|
+
throw new Error("workflow recovery audit authority changed");
|
|
104
|
+
};
|
|
105
|
+
return {
|
|
106
|
+
intent,
|
|
107
|
+
assertCurrent,
|
|
108
|
+
complete: () => {
|
|
109
|
+
assertCurrent();
|
|
110
|
+
writeImmutableJson(join(directory, "completion.json"), {
|
|
111
|
+
schema_version: 1,
|
|
112
|
+
operation_id: intent.operation_id,
|
|
113
|
+
intent_sha256: stableDigest(intent),
|
|
114
|
+
outcome: "completed",
|
|
115
|
+
recorded_at: new Date().toISOString(),
|
|
116
|
+
});
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function ensureDurableDirectory(path: string): void {
|
|
122
|
+
if (existsSync(path)) {
|
|
123
|
+
if (!lstatSync(path).isDirectory())
|
|
124
|
+
throw new Error("workflow recovery ledger is not a directory");
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
ensureDurableDirectory(dirname(path));
|
|
128
|
+
try {
|
|
129
|
+
mkdirSync(path, { mode: stateDirMode() });
|
|
130
|
+
} catch (error) {
|
|
131
|
+
if ((error as NodeJS.ErrnoException).code !== "EEXIST" || !lstatSync(path).isDirectory())
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
134
|
+
fsyncParentDirectory(path);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export function readWorkflowRecoveryLedger(
|
|
138
|
+
coordRoot: string,
|
|
139
|
+
runId: string,
|
|
140
|
+
): WorkflowRecoveryRecord[] {
|
|
141
|
+
assertWorkflowRunId(runId);
|
|
142
|
+
const directory = join(coordRoot, ".harnery", "workflows", runId, "recovery");
|
|
143
|
+
if (!existsSync(directory)) return [];
|
|
144
|
+
return readdirSync(directory)
|
|
145
|
+
.sort()
|
|
146
|
+
.map((entry) => {
|
|
147
|
+
if (!/^recovery-[a-f0-9-]{36}$/.test(entry))
|
|
148
|
+
throw new Error("invalid workflow recovery ledger entry");
|
|
149
|
+
const intent = readJsonRecord<WorkflowRecoveryIntent>(
|
|
150
|
+
join(directory, entry, "intent.json"),
|
|
151
|
+
"recovery intent",
|
|
152
|
+
);
|
|
153
|
+
if (
|
|
154
|
+
intent.schema_version !== 1 ||
|
|
155
|
+
intent.operation_id !== entry ||
|
|
156
|
+
intent.run_id !== runId ||
|
|
157
|
+
intent.outcome !== "unknown" ||
|
|
158
|
+
!["reclaim_unknown_liveness", "unlock_resume"].includes(intent.action) ||
|
|
159
|
+
!/^[a-f0-9]{64}$/.test(intent.target_sha256)
|
|
160
|
+
) {
|
|
161
|
+
throw new Error("invalid workflow recovery intent authority");
|
|
162
|
+
}
|
|
163
|
+
validateWorkflowRecoveryAuthorization(intent);
|
|
164
|
+
const completionPath = join(directory, entry, "completion.json");
|
|
165
|
+
if (!existsSync(completionPath)) return { intent, outcome: "unknown" };
|
|
166
|
+
const completion = readJsonRecord<{
|
|
167
|
+
schema_version: number;
|
|
168
|
+
operation_id: string;
|
|
169
|
+
intent_sha256: string;
|
|
170
|
+
outcome: string;
|
|
171
|
+
}>(completionPath, "recovery completion");
|
|
172
|
+
if (
|
|
173
|
+
completion.schema_version !== 1 ||
|
|
174
|
+
completion.operation_id !== entry ||
|
|
175
|
+
completion.intent_sha256 !== stableDigest(intent) ||
|
|
176
|
+
completion.outcome !== "completed"
|
|
177
|
+
)
|
|
178
|
+
throw new Error("invalid workflow recovery completion authority");
|
|
179
|
+
return { intent, outcome: "completed" };
|
|
180
|
+
});
|
|
181
|
+
}
|
|
@@ -3,6 +3,8 @@ import {
|
|
|
3
3
|
chmodSync,
|
|
4
4
|
closeSync,
|
|
5
5
|
existsSync,
|
|
6
|
+
fsyncSync,
|
|
7
|
+
lstatSync,
|
|
6
8
|
mkdirSync,
|
|
7
9
|
openSync,
|
|
8
10
|
readFileSync,
|
|
@@ -23,7 +25,13 @@ import {
|
|
|
23
25
|
import { stateFileMode } from "../storage/modes.ts";
|
|
24
26
|
import { assertWorkflowRunId, readWorkflowApproval } from "./approvals.ts";
|
|
25
27
|
import { isCanonicalWorkflowAttemptContext } from "./attempt-context.ts";
|
|
26
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
fsyncParentDirectory,
|
|
30
|
+
readJsonRecord,
|
|
31
|
+
stableDigest,
|
|
32
|
+
writeImmutableJson,
|
|
33
|
+
} from "./durable-record.ts";
|
|
34
|
+
import { beginWorkflowRecovery, type WorkflowRecoveryAuthorization } from "./recovery-audit.ts";
|
|
27
35
|
import { normalizeWorkflowSpecialists } from "./specialists.ts";
|
|
28
36
|
import type {
|
|
29
37
|
WorkflowAttemptContext,
|
|
@@ -38,6 +46,7 @@ import type {
|
|
|
38
46
|
WorkspaceBinding,
|
|
39
47
|
WorkspaceCompatibilityExecutionEvidence,
|
|
40
48
|
} from "./workspaces/index.ts";
|
|
49
|
+
import { acquireNoClobberLease } from "./workspaces/leases.ts";
|
|
41
50
|
import {
|
|
42
51
|
assertWorkspaceManifestAuthority,
|
|
43
52
|
isWorkspaceBinding,
|
|
@@ -204,8 +213,7 @@ export function workflowRunManifestPath(coordRoot: string, runId: string): strin
|
|
|
204
213
|
}
|
|
205
214
|
|
|
206
215
|
/** Acquire an exclusive lease shared by resume and reclaim. Existing locks
|
|
207
|
-
* always refuse
|
|
208
|
-
* Crash recovery requires an audited ownership check outside this operation. */
|
|
216
|
+
* always refuse; explicit audited recovery is a separate operation. */
|
|
209
217
|
export function acquireWorkflowResumeLease(coordRoot: string, runId: string): () => void {
|
|
210
218
|
assertWorkflowRunId(runId);
|
|
211
219
|
const path = join(coordRoot, ".harnery", "workflows", runId, "resume.lock");
|
|
@@ -218,23 +226,27 @@ export function acquireWorkflowResumeLease(coordRoot: string, runId: string): ()
|
|
|
218
226
|
const ownerBytes = `${JSON.stringify(owner)}\n`;
|
|
219
227
|
mkdirSync(dirname(path), { recursive: true });
|
|
220
228
|
|
|
221
|
-
const acquire = ():
|
|
229
|
+
const acquire = (): { device: number; inode: number } | false => {
|
|
222
230
|
try {
|
|
223
231
|
const fd = openSync(path, "wx", stateFileMode());
|
|
224
232
|
try {
|
|
225
233
|
writeFileSync(fd, ownerBytes, "utf8");
|
|
234
|
+
fsyncSync(fd);
|
|
226
235
|
} finally {
|
|
227
236
|
closeSync(fd);
|
|
228
237
|
}
|
|
229
238
|
chmodSync(path, stateFileMode());
|
|
230
|
-
|
|
239
|
+
fsyncParentDirectory(path);
|
|
240
|
+
const identity = lstatSync(path);
|
|
241
|
+
return { device: identity.dev, inode: identity.ino };
|
|
231
242
|
} catch (error) {
|
|
232
243
|
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
233
244
|
return false;
|
|
234
245
|
}
|
|
235
246
|
};
|
|
236
247
|
|
|
237
|
-
|
|
248
|
+
const acquired = withResumeLeaseMutation(coordRoot, runId, acquire);
|
|
249
|
+
if (!acquired) {
|
|
238
250
|
const existing = readResumeLease(path);
|
|
239
251
|
const holder = existing
|
|
240
252
|
? ` by pid ${existing.pid} on ${existing.host}; its liveness has not been verified`
|
|
@@ -244,13 +256,129 @@ export function acquireWorkflowResumeLease(coordRoot: string, runId: string): ()
|
|
|
244
256
|
|
|
245
257
|
return () => {
|
|
246
258
|
try {
|
|
247
|
-
|
|
259
|
+
// Only this live process can release this nonce. Acquisition never
|
|
260
|
+
// replaces a present lock, and unlock requires ESRCH for its owner, so
|
|
261
|
+
// neither supported writer can replace this lock during synchronous
|
|
262
|
+
// release. Taking their mutation gate would strand a live owner's lock
|
|
263
|
+
// on ordinary contention. Pin both identity and bytes for stale closures.
|
|
264
|
+
const identity = lstatSync(path);
|
|
265
|
+
if (
|
|
266
|
+
identity.isFile() &&
|
|
267
|
+
identity.dev === acquired.device &&
|
|
268
|
+
identity.ino === acquired.inode &&
|
|
269
|
+
readFileSync(path, "utf8") === ownerBytes
|
|
270
|
+
) {
|
|
271
|
+
unlinkSync(path);
|
|
272
|
+
fsyncParentDirectory(path);
|
|
273
|
+
}
|
|
248
274
|
} catch {
|
|
249
275
|
// Leave an unreadable or replaced lock untouched.
|
|
250
276
|
}
|
|
251
277
|
};
|
|
252
278
|
}
|
|
253
279
|
|
|
280
|
+
export interface WorkflowResumeRecoveryOptions extends Partial<WorkflowRecoveryAuthorization> {
|
|
281
|
+
yes?: boolean;
|
|
282
|
+
lockSha256?: string;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
export interface WorkflowResumeRecoveryResult {
|
|
286
|
+
action: "preview" | "unlocked";
|
|
287
|
+
run_id: string;
|
|
288
|
+
lock_sha256: string | null;
|
|
289
|
+
owner_state: "dead" | "live" | "unknown" | "absent";
|
|
290
|
+
owner: { pid: number; host: string; created_at: string } | null;
|
|
291
|
+
recovery_id?: string;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
export function recoverWorkflowResumeLease(
|
|
295
|
+
coordRoot: string,
|
|
296
|
+
runId: string,
|
|
297
|
+
options: WorkflowResumeRecoveryOptions,
|
|
298
|
+
): WorkflowResumeRecoveryResult {
|
|
299
|
+
assertWorkflowRunId(runId);
|
|
300
|
+
const path = join(coordRoot, ".harnery", "workflows", runId, "resume.lock");
|
|
301
|
+
const inspect = (): WorkflowResumeRecoveryResult => {
|
|
302
|
+
let lockSha256: string;
|
|
303
|
+
try {
|
|
304
|
+
const stat = lstatSync(path);
|
|
305
|
+
if (!stat.isFile() || stat.size > 8 * 1024)
|
|
306
|
+
throw new Error("resume lock is not a bounded regular file");
|
|
307
|
+
lockSha256 = stableDigest({
|
|
308
|
+
bytes: readFileSync(path, "utf8"),
|
|
309
|
+
device: stat.dev,
|
|
310
|
+
inode: stat.ino,
|
|
311
|
+
});
|
|
312
|
+
} catch (error) {
|
|
313
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
314
|
+
return {
|
|
315
|
+
action: "preview",
|
|
316
|
+
run_id: runId,
|
|
317
|
+
lock_sha256: null,
|
|
318
|
+
owner_state: "absent",
|
|
319
|
+
owner: null,
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
const owner = readResumeLease(path);
|
|
323
|
+
let ownerState: WorkflowResumeRecoveryResult["owner_state"] = "unknown";
|
|
324
|
+
if (owner?.host === hostname()) {
|
|
325
|
+
try {
|
|
326
|
+
process.kill(owner.pid, 0);
|
|
327
|
+
ownerState = "live";
|
|
328
|
+
} catch (error) {
|
|
329
|
+
if ((error as NodeJS.ErrnoException).code === "ESRCH") ownerState = "dead";
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return {
|
|
333
|
+
action: "preview",
|
|
334
|
+
run_id: runId,
|
|
335
|
+
lock_sha256: lockSha256,
|
|
336
|
+
owner_state: ownerState,
|
|
337
|
+
owner,
|
|
338
|
+
};
|
|
339
|
+
};
|
|
340
|
+
if (!options.yes) return inspect();
|
|
341
|
+
return withResumeLeaseMutation(coordRoot, runId, () => {
|
|
342
|
+
const observed = inspect();
|
|
343
|
+
if (observed.owner_state !== "dead")
|
|
344
|
+
throw new Error(`workflow resume unlock refused: owner is ${observed.owner_state}`);
|
|
345
|
+
if (!options.lockSha256 || options.lockSha256 !== observed.lock_sha256)
|
|
346
|
+
throw new Error("workflow resume unlock requires the exact current preview digest");
|
|
347
|
+
const audit = beginWorkflowRecovery(coordRoot, {
|
|
348
|
+
run_id: runId,
|
|
349
|
+
action: "unlock_resume",
|
|
350
|
+
target_sha256: options.lockSha256,
|
|
351
|
+
target: { lock: "resume.lock", owner: observed.owner },
|
|
352
|
+
actor: options.actor!,
|
|
353
|
+
reason: options.reason!,
|
|
354
|
+
});
|
|
355
|
+
const current = inspect();
|
|
356
|
+
if (current.lock_sha256 !== observed.lock_sha256 || current.owner_state !== "dead")
|
|
357
|
+
throw new Error("workflow resume lock authority changed before unlock");
|
|
358
|
+
audit.assertCurrent();
|
|
359
|
+
unlinkSync(path);
|
|
360
|
+
fsyncParentDirectory(path);
|
|
361
|
+
audit.complete();
|
|
362
|
+
return { ...observed, action: "unlocked", recovery_id: audit.intent.operation_id };
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
function withResumeLeaseMutation<T>(coordRoot: string, runId: string, action: () => T): T {
|
|
367
|
+
const authority = stableDigest({ run_id: runId, lock: "resume.lock" });
|
|
368
|
+
const lease = acquireNoClobberLease({
|
|
369
|
+
path: join(coordRoot, ".harnery", "workflows", runId, "resume-mutation.lease"),
|
|
370
|
+
scope: "workflow-resume-mutation",
|
|
371
|
+
authoritySha256: authority,
|
|
372
|
+
staleAfterMs: 300_000,
|
|
373
|
+
validateStaleOwner: (owner) => owner.authority_sha256 === authority,
|
|
374
|
+
});
|
|
375
|
+
try {
|
|
376
|
+
return action();
|
|
377
|
+
} finally {
|
|
378
|
+
lease.release();
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
254
382
|
function readResumeLease(path: string): { pid: number; host: string; created_at: string } | null {
|
|
255
383
|
try {
|
|
256
384
|
const value = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>;
|
|
@@ -258,7 +386,8 @@ function readResumeLease(path: string): { pid: number; host: string; created_at:
|
|
|
258
386
|
Number.isSafeInteger(value.pid) &&
|
|
259
387
|
value.pid > 0 &&
|
|
260
388
|
typeof value.host === "string" &&
|
|
261
|
-
|
|
389
|
+
value.host.length > 0 &&
|
|
390
|
+
validTimestamp(value.created_at)
|
|
262
391
|
? { pid: value.pid, host: value.host, created_at: value.created_at }
|
|
263
392
|
: null;
|
|
264
393
|
} catch {
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
} from "node:fs";
|
|
10
10
|
import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
|
|
11
11
|
import { stateDirMode, stateFileMode } from "../../storage/modes.ts";
|
|
12
|
+
import { beginWorkflowRecovery, validateWorkflowRecoveryAuthorization } from "../recovery-audit.ts";
|
|
12
13
|
import { acquireWorkflowResumeLease } from "../run-state.ts";
|
|
13
14
|
import { WorkspaceAttestationError } from "./attestation-error.ts";
|
|
14
15
|
import { cleanupWorkspace } from "./cleanup.ts";
|
|
@@ -53,7 +54,7 @@ import {
|
|
|
53
54
|
type WorkspaceReclaimPreview,
|
|
54
55
|
type WorkspaceReclaimResult,
|
|
55
56
|
} from "./reclaim.ts";
|
|
56
|
-
import { readWorkflowLiveness } from "./recovery.ts";
|
|
57
|
+
import { hasActiveWorkflowEvidence, readWorkflowLiveness } from "./recovery.ts";
|
|
57
58
|
import {
|
|
58
59
|
appendWorkspaceEvent,
|
|
59
60
|
fileSha256,
|
|
@@ -820,6 +821,7 @@ function reclaimPreview(
|
|
|
820
821
|
run_id: binding.run_id,
|
|
821
822
|
mode,
|
|
822
823
|
liveness: readWorkflowLiveness(coordRoot, binding.run_id),
|
|
824
|
+
evidence_sha256: reclaimEvidenceDigest(coordRoot, binding),
|
|
823
825
|
ownership: "unproven",
|
|
824
826
|
dirty_paths: null,
|
|
825
827
|
lease_holder: reclaimLeaseHolders(coordRoot, binding),
|
|
@@ -839,14 +841,45 @@ function reclaimPreview(
|
|
|
839
841
|
return preview;
|
|
840
842
|
}
|
|
841
843
|
|
|
842
|
-
function
|
|
844
|
+
function reclaimEvidenceDigest(coordRoot: string, binding: WorkspaceBinding): string {
|
|
845
|
+
const evidence = (name: string): string | null => {
|
|
846
|
+
try {
|
|
847
|
+
return readFileSync(join(coordRoot, ".harnery", "workflows", binding.run_id, name), "utf8");
|
|
848
|
+
} catch (error) {
|
|
849
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
850
|
+
throw error;
|
|
851
|
+
}
|
|
852
|
+
};
|
|
853
|
+
return stableDigest({
|
|
854
|
+
binding,
|
|
855
|
+
proof: evidence("proof.json"),
|
|
856
|
+
transcript: evidence("transcript.jsonl"),
|
|
857
|
+
});
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
function assertReclaimLiveness(
|
|
861
|
+
coordRoot: string,
|
|
862
|
+
binding: WorkspaceBinding,
|
|
863
|
+
recovery?: WorkspaceReclaimOptions["unknownLiveness"],
|
|
864
|
+
): void {
|
|
843
865
|
const liveness = readWorkflowLiveness(coordRoot, binding.run_id);
|
|
866
|
+
if (liveness === "active" || (recovery && hasActiveWorkflowEvidence(coordRoot, binding.run_id))) {
|
|
867
|
+
throw new WorkflowReclaimError(
|
|
868
|
+
"workflow_reclaim_run_live",
|
|
869
|
+
`run ${binding.run_id} has active evidence; its workspace was left untouched`,
|
|
870
|
+
);
|
|
871
|
+
}
|
|
872
|
+
if (recovery && reclaimEvidenceDigest(coordRoot, binding) !== recovery.evidenceSha256) {
|
|
873
|
+
throw new WorkflowReclaimError(
|
|
874
|
+
"workflow_reclaim_recovery_mismatch",
|
|
875
|
+
"workflow reclaim requires the exact current preview evidence digest",
|
|
876
|
+
);
|
|
877
|
+
}
|
|
844
878
|
if (liveness === "inactive") return;
|
|
879
|
+
if (liveness === "unknown" && recovery) return;
|
|
845
880
|
throw new WorkflowReclaimError(
|
|
846
|
-
|
|
847
|
-
liveness
|
|
848
|
-
? `run ${binding.run_id} is active; its workspace was left untouched`
|
|
849
|
-
: `run ${binding.run_id} has missing, corrupt, or ambiguous liveness evidence; its workspace was left untouched`,
|
|
881
|
+
"workflow_reclaim_liveness_unknown",
|
|
882
|
+
`run ${binding.run_id} has missing, corrupt, or ambiguous liveness evidence; its workspace was left untouched`,
|
|
850
883
|
);
|
|
851
884
|
}
|
|
852
885
|
|
|
@@ -892,10 +925,11 @@ function reclaimAuthorityGuard(
|
|
|
892
925
|
binding: WorkspaceBinding,
|
|
893
926
|
claim: WorkspaceClaim,
|
|
894
927
|
reservation: WorkspaceReclaimReservation,
|
|
928
|
+
recovery?: WorkspaceReclaimOptions["unknownLiveness"],
|
|
895
929
|
): void {
|
|
896
930
|
try {
|
|
897
931
|
reservation.assertCurrent();
|
|
898
|
-
assertReclaimLiveness(coordRoot, binding);
|
|
932
|
+
assertReclaimLiveness(coordRoot, binding, recovery);
|
|
899
933
|
assertReclaimOwnership(coordRoot, binding);
|
|
900
934
|
assertPeerWorkspaceAuthority(coordRoot, claim);
|
|
901
935
|
} catch (error) {
|
|
@@ -914,10 +948,11 @@ async function reclaim(
|
|
|
914
948
|
throw new Error("unsupported workspace reclaim mode");
|
|
915
949
|
}
|
|
916
950
|
if (!options.yes) return reclaimPreview(coordRoot, binding, options.mode);
|
|
951
|
+
if (options.unknownLiveness) validateWorkflowRecoveryAuthorization(options.unknownLiveness);
|
|
917
952
|
|
|
918
953
|
// Liveness comes before terminal-proof cleanup helpers. A live run may not
|
|
919
954
|
// have a proof yet, and a missing proof must never be discovered after reset.
|
|
920
|
-
assertReclaimLiveness(coordRoot, binding);
|
|
955
|
+
assertReclaimLiveness(coordRoot, binding, options.unknownLiveness);
|
|
921
956
|
const claim = assertReclaimOwnership(coordRoot, binding);
|
|
922
957
|
let reservation: WorkspaceReclaimReservation;
|
|
923
958
|
try {
|
|
@@ -926,7 +961,7 @@ async function reclaim(
|
|
|
926
961
|
throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
|
|
927
962
|
}
|
|
928
963
|
try {
|
|
929
|
-
reclaimAuthorityGuard(coordRoot, binding, claim, reservation);
|
|
964
|
+
reclaimAuthorityGuard(coordRoot, binding, claim, reservation, options.unknownLiveness);
|
|
930
965
|
let releaseResume: () => void;
|
|
931
966
|
try {
|
|
932
967
|
releaseResume = acquireWorkflowResumeLease(coordRoot, binding.run_id);
|
|
@@ -953,9 +988,24 @@ async function reclaim(
|
|
|
953
988
|
throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
|
|
954
989
|
}
|
|
955
990
|
try {
|
|
956
|
-
assertReclaimLiveness(coordRoot, binding);
|
|
991
|
+
assertReclaimLiveness(coordRoot, binding, options.unknownLiveness);
|
|
957
992
|
assertReclaimOwnership(coordRoot, binding);
|
|
958
993
|
const preview = reclaimPreview(coordRoot, binding, options.mode);
|
|
994
|
+
const audit = options.unknownLiveness
|
|
995
|
+
? beginWorkflowRecovery(coordRoot, {
|
|
996
|
+
run_id: binding.run_id,
|
|
997
|
+
action: "reclaim_unknown_liveness",
|
|
998
|
+
actor: options.unknownLiveness.actor,
|
|
999
|
+
reason: options.unknownLiveness.reason,
|
|
1000
|
+
target_sha256: options.unknownLiveness.evidenceSha256,
|
|
1001
|
+
target: {
|
|
1002
|
+
binding_id: binding.binding_id,
|
|
1003
|
+
workspace_root: binding.workspace_root,
|
|
1004
|
+
mode: options.mode,
|
|
1005
|
+
liveness: preview.liveness,
|
|
1006
|
+
},
|
|
1007
|
+
})
|
|
1008
|
+
: undefined;
|
|
959
1009
|
let preparation: ReclaimPreparation | undefined;
|
|
960
1010
|
// Keep ordinary cleanup's terminal-evidence checks, intent and receipts.
|
|
961
1011
|
// Its provider callback uses the leases already held here instead of
|
|
@@ -967,12 +1017,28 @@ async function reclaim(
|
|
|
967
1017
|
provider: {
|
|
968
1018
|
...provider,
|
|
969
1019
|
cleanup: async (candidate, intent) => {
|
|
970
|
-
assertReclaimLiveness(coordRoot, candidate);
|
|
1020
|
+
assertReclaimLiveness(coordRoot, candidate, options.unknownLiveness);
|
|
971
1021
|
assertReclaimOwnership(coordRoot, candidate);
|
|
972
1022
|
return cleanupUnderLease(coordRoot, candidate, claim, intent, {
|
|
973
1023
|
mode: options.mode,
|
|
974
|
-
|
|
975
|
-
|
|
1024
|
+
recovery: options.unknownLiveness,
|
|
1025
|
+
beforeEffect: () => {
|
|
1026
|
+
audit?.assertCurrent();
|
|
1027
|
+
reclaimAuthorityGuard(
|
|
1028
|
+
coordRoot,
|
|
1029
|
+
candidate,
|
|
1030
|
+
claim,
|
|
1031
|
+
reservation,
|
|
1032
|
+
options.unknownLiveness,
|
|
1033
|
+
);
|
|
1034
|
+
},
|
|
1035
|
+
beforeRefEffect: () => {
|
|
1036
|
+
audit?.assertCurrent();
|
|
1037
|
+
reservation.assertCurrent();
|
|
1038
|
+
assertReclaimLiveness(coordRoot, candidate, options.unknownLiveness);
|
|
1039
|
+
requiredClaim(coordRoot, candidate);
|
|
1040
|
+
assertPeerWorkspaceAuthority(coordRoot, claim);
|
|
1041
|
+
},
|
|
976
1042
|
prepared: (value) => {
|
|
977
1043
|
preparation = value;
|
|
978
1044
|
},
|
|
@@ -991,6 +1057,8 @@ async function reclaim(
|
|
|
991
1057
|
"workspace cleanup preconditions could not be proved",
|
|
992
1058
|
);
|
|
993
1059
|
}
|
|
1060
|
+
if (cleanupResult.status === "released" || cleanupResult.status === "already_released")
|
|
1061
|
+
audit?.complete();
|
|
994
1062
|
return {
|
|
995
1063
|
...preview,
|
|
996
1064
|
action: "reclaimed",
|
|
@@ -1104,7 +1172,9 @@ function cleanupUnderLease(
|
|
|
1104
1172
|
intent: WorkspaceCleanupIntent,
|
|
1105
1173
|
reclaimRequest?: {
|
|
1106
1174
|
mode: ReclaimMode;
|
|
1175
|
+
recovery?: WorkspaceReclaimOptions["unknownLiveness"];
|
|
1107
1176
|
beforeEffect: () => void;
|
|
1177
|
+
beforeRefEffect: () => void;
|
|
1108
1178
|
prepared: (value: ReclaimPreparation) => void;
|
|
1109
1179
|
},
|
|
1110
1180
|
beforeRemoval?: () => void,
|
|
@@ -1152,7 +1222,7 @@ function cleanupUnderLease(
|
|
|
1152
1222
|
};
|
|
1153
1223
|
}
|
|
1154
1224
|
const liveness = readWorkflowLiveness(coordRoot, binding.run_id);
|
|
1155
|
-
if (liveness !== "inactive") {
|
|
1225
|
+
if (liveness !== "inactive" && !(liveness === "unknown" && reclaimRequest?.recovery)) {
|
|
1156
1226
|
return {
|
|
1157
1227
|
...base,
|
|
1158
1228
|
status: "blocked",
|
|
@@ -1245,7 +1315,7 @@ function cleanupUnderLease(
|
|
|
1245
1315
|
}
|
|
1246
1316
|
|
|
1247
1317
|
if (reclaimRequest) {
|
|
1248
|
-
assertReclaimLiveness(coordRoot, binding);
|
|
1318
|
+
assertReclaimLiveness(coordRoot, binding, reclaimRequest.recovery);
|
|
1249
1319
|
assertReclaimOwnership(coordRoot, binding);
|
|
1250
1320
|
revalidateClaimResources(claim, root);
|
|
1251
1321
|
reclaimRequest.prepared(
|
|
@@ -1293,6 +1363,7 @@ function cleanupUnderLease(
|
|
|
1293
1363
|
return { ...base, status: "blocked", reason: "workspace ref changed during cleanup" };
|
|
1294
1364
|
}
|
|
1295
1365
|
revalidateClaimResources(claim, root);
|
|
1366
|
+
reclaimRequest?.beforeRefEffect();
|
|
1296
1367
|
const deleted = gitMaybe(repository.source_root.realpath, [
|
|
1297
1368
|
"update-ref",
|
|
1298
1369
|
"-d",
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
18
|
import { existsSync } from "node:fs";
|
|
19
|
+
import type { WorkflowRecoveryAuthorization } from "../recovery-audit.ts";
|
|
19
20
|
import { git, gitMaybe } from "./git.ts";
|
|
20
21
|
import type { WorkflowLiveness } from "./recovery.ts";
|
|
21
22
|
import type { WorkspaceCleanupAttempt, WorkspaceCleanupReceipt } from "./types.ts";
|
|
@@ -25,6 +26,7 @@ export type ReclaimMode = "salvage" | "discard";
|
|
|
25
26
|
export type WorkflowReclaimRefusalCode =
|
|
26
27
|
| "workflow_reclaim_run_live"
|
|
27
28
|
| "workflow_reclaim_liveness_unknown"
|
|
29
|
+
| "workflow_reclaim_recovery_mismatch"
|
|
28
30
|
| "workflow_reclaim_ownership_unproven"
|
|
29
31
|
| "workflow_reclaim_lease_held";
|
|
30
32
|
|
|
@@ -41,6 +43,7 @@ export class WorkflowReclaimError extends Error {
|
|
|
41
43
|
export interface WorkspaceReclaimOptions {
|
|
42
44
|
mode: ReclaimMode;
|
|
43
45
|
yes?: boolean;
|
|
46
|
+
unknownLiveness?: WorkflowRecoveryAuthorization & { evidenceSha256: string };
|
|
44
47
|
}
|
|
45
48
|
|
|
46
49
|
export interface ReclaimLeaseHolder {
|
|
@@ -56,6 +59,7 @@ export interface WorkspaceReclaimPreview {
|
|
|
56
59
|
run_id: string;
|
|
57
60
|
mode: ReclaimMode;
|
|
58
61
|
liveness: WorkflowLiveness;
|
|
62
|
+
evidence_sha256: string;
|
|
59
63
|
ownership: "ok" | "unproven";
|
|
60
64
|
ownership_detail?: string;
|
|
61
65
|
dirty_paths: string[] | null;
|
|
@@ -4,6 +4,29 @@ import { readWorkflowProof } from "../proof.ts";
|
|
|
4
4
|
|
|
5
5
|
export type WorkflowLiveness = "active" | "inactive" | "unknown";
|
|
6
6
|
|
|
7
|
+
/** Corruption must not hide a readable active marker from an audited override. */
|
|
8
|
+
export function hasActiveWorkflowEvidence(coordRoot: string, runId: string): boolean {
|
|
9
|
+
const path = join(resolve(coordRoot), ".harnery", "workflows", runId, "transcript.jsonl");
|
|
10
|
+
if (!existsSync(path)) return false;
|
|
11
|
+
let active = false;
|
|
12
|
+
for (const line of readFileSync(path, "utf8").split("\n")) {
|
|
13
|
+
try {
|
|
14
|
+
const event = JSON.parse(line);
|
|
15
|
+
if (event?.run_id !== runId) continue;
|
|
16
|
+
if (["run.start", "run.resume", "agent.start"].includes(event.event)) active = true;
|
|
17
|
+
if (
|
|
18
|
+
event.event === "run.parked" ||
|
|
19
|
+
(event.event === "run.end" && typeof event.ok === "boolean") ||
|
|
20
|
+
(event.event === "workspace.reattach.failed" && ["blocked", "lost"].includes(event.status))
|
|
21
|
+
)
|
|
22
|
+
active = false;
|
|
23
|
+
} catch {
|
|
24
|
+
/* Unreadable evidence cannot establish inactivity. */
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return active;
|
|
28
|
+
}
|
|
29
|
+
|
|
7
30
|
export function readWorkflowLiveness(coordRoot: string, runId: string): WorkflowLiveness {
|
|
8
31
|
const root = resolve(coordRoot);
|
|
9
32
|
const transcriptPath = join(root, ".harnery", "workflows", runId, "transcript.jsonl");
|