harnery 0.44.0 → 0.45.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/dist/commander.js +1 -0
  2. package/dist/commands/files.d.ts +2 -2
  3. package/dist/commands/files.d.ts.map +1 -1
  4. package/dist/commands/files.js +7 -4
  5. package/dist/commands/servers.d.ts.map +1 -1
  6. package/dist/commands/servers.js +11 -17
  7. package/dist/commands/tunnel.d.ts +15 -0
  8. package/dist/commands/tunnel.d.ts.map +1 -1
  9. package/dist/commands/tunnel.js +453 -153
  10. package/dist/commands/turn-rollback.d.ts +5 -0
  11. package/dist/commands/turn-rollback.d.ts.map +1 -0
  12. package/dist/commands/turn-rollback.js +76 -0
  13. package/dist/commands/workflow.d.ts.map +1 -1
  14. package/dist/commands/workflow.js +44 -0
  15. package/dist/core/agents/coord-client.d.ts.map +1 -1
  16. package/dist/core/agents/coord-client.js +18 -0
  17. package/dist/core/agents/state/proc-start.d.ts +5 -6
  18. package/dist/core/agents/state/proc-start.d.ts.map +1 -1
  19. package/dist/core/agents/state/proc-start.js +13 -9
  20. package/dist/core/agents/state/process-handle.d.ts +26 -0
  21. package/dist/core/agents/state/process-handle.d.ts.map +1 -0
  22. package/dist/core/agents/state/process-handle.js +121 -0
  23. package/dist/core/artifacts/delivery-card.d.ts.map +1 -1
  24. package/dist/core/artifacts/delivery-card.js +2 -1
  25. package/dist/core/resources/service.d.ts +2 -16
  26. package/dist/core/resources/service.d.ts.map +1 -1
  27. package/dist/core/resources/service.js +1 -78
  28. package/dist/core/semantic/service.d.ts.map +1 -1
  29. package/dist/core/semantic/service.js +2 -9
  30. package/dist/core/servers/index.d.ts +12 -11
  31. package/dist/core/servers/index.d.ts.map +1 -1
  32. package/dist/core/servers/index.js +194 -81
  33. package/dist/core/servers/tunnels.d.ts +5 -3
  34. package/dist/core/servers/tunnels.d.ts.map +1 -1
  35. package/dist/core/servers/tunnels.js +45 -2
  36. package/dist/core/supervisor/service.d.ts +2 -4
  37. package/dist/core/supervisor/service.d.ts.map +1 -1
  38. package/dist/core/turn-rollback/content.d.ts +53 -0
  39. package/dist/core/turn-rollback/content.d.ts.map +1 -0
  40. package/dist/core/turn-rollback/content.js +396 -0
  41. package/dist/core/turn-rollback/index.d.ts +72 -0
  42. package/dist/core/turn-rollback/index.d.ts.map +1 -0
  43. package/dist/core/turn-rollback/index.js +443 -0
  44. package/dist/core/workflow/index.d.ts +4 -2
  45. package/dist/core/workflow/index.d.ts.map +1 -1
  46. package/dist/core/workflow/index.js +2 -1
  47. package/dist/core/workflow/recovery-audit.d.ts +30 -0
  48. package/dist/core/workflow/recovery-audit.d.ts.map +1 -0
  49. package/dist/core/workflow/recovery-audit.js +116 -0
  50. package/dist/core/workflow/run-state.d.ts +19 -2
  51. package/dist/core/workflow/run-state.d.ts.map +1 -1
  52. package/dist/core/workflow/run-state.js +113 -8
  53. package/dist/core/workflow/workspaces/local-git.d.ts.map +1 -1
  54. package/dist/core/workflow/workspaces/local-git.js +70 -14
  55. package/dist/core/workflow/workspaces/reclaim.d.ts +6 -1
  56. package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
  57. package/dist/core/workflow/workspaces/recovery.d.ts +2 -0
  58. package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
  59. package/dist/core/workflow/workspaces/recovery.js +24 -0
  60. package/dist/lib/tunnel/state.d.ts +13 -6
  61. package/dist/lib/tunnel/state.d.ts.map +1 -1
  62. package/dist/lib/tunnel/state.js +58 -12
  63. package/package.json +6 -1
  64. package/src/commander.ts +10 -0
  65. package/src/commands/files.ts +13 -5
  66. package/src/commands/servers.ts +12 -23
  67. package/src/commands/tunnel.ts +533 -166
  68. package/src/commands/turn-rollback.ts +111 -0
  69. package/src/commands/workflow.ts +59 -0
  70. package/src/core/agents/coord-client.ts +20 -0
  71. package/src/core/agents/state/proc-start.ts +14 -11
  72. package/src/core/agents/state/process-handle.ts +165 -0
  73. package/src/core/artifacts/delivery-card.ts +7 -1
  74. package/src/core/resources/service.ts +3 -88
  75. package/src/core/semantic/service.ts +2 -8
  76. package/src/core/servers/index.ts +234 -85
  77. package/src/core/servers/tunnels.ts +55 -3
  78. package/src/core/supervisor/service.ts +2 -3
  79. package/src/core/turn-rollback/content.ts +499 -0
  80. package/src/core/turn-rollback/index.ts +580 -0
  81. package/src/core/workflow/index.ts +12 -1
  82. package/src/core/workflow/recovery-audit.ts +181 -0
  83. package/src/core/workflow/run-state.ts +137 -8
  84. package/src/core/workflow/workspaces/local-git.ts +86 -15
  85. package/src/core/workflow/workspaces/reclaim.ts +4 -0
  86. package/src/core/workflow/workspaces/recovery.ts +23 -0
  87. package/src/lib/tunnel/state.ts +70 -15
@@ -0,0 +1,181 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { existsSync, lstatSync, mkdirSync, readdirSync } from "node:fs";
3
+ import { hostname } from "node:os";
4
+ import { dirname, join } from "node:path";
5
+ import { stateDirMode } from "../storage/modes.ts";
6
+ import { assertWorkflowRunId } from "./approvals.ts";
7
+ import {
8
+ fsyncParentDirectory,
9
+ readJsonRecord,
10
+ stableDigest,
11
+ writeImmutableJson,
12
+ } from "./durable-record.ts";
13
+
14
+ export interface WorkflowRecoveryAuthorization {
15
+ actor: string;
16
+ reason: string;
17
+ }
18
+
19
+ export interface WorkflowRecoveryIntent extends WorkflowRecoveryAuthorization {
20
+ schema_version: 1;
21
+ operation_id: string;
22
+ run_id: string;
23
+ action: "reclaim_unknown_liveness" | "unlock_resume";
24
+ target_sha256: string;
25
+ target: Record<string, unknown>;
26
+ recorded_at: string;
27
+ pid: number;
28
+ host: string;
29
+ outcome: "unknown";
30
+ }
31
+
32
+ export interface WorkflowRecoveryRecord {
33
+ intent: WorkflowRecoveryIntent;
34
+ outcome: "unknown" | "completed";
35
+ }
36
+
37
+ export function validateWorkflowRecoveryAuthorization(
38
+ authorization: WorkflowRecoveryAuthorization,
39
+ ): void {
40
+ for (const [field, limit] of [
41
+ ["actor", 200],
42
+ ["reason", 2_000],
43
+ ] as const) {
44
+ const value = authorization[field];
45
+ if (
46
+ typeof value !== "string" ||
47
+ !value.trim() ||
48
+ value.length > limit ||
49
+ Array.from(value).some(
50
+ (character) => character.charCodeAt(0) < 32 || character.charCodeAt(0) === 127,
51
+ )
52
+ ) {
53
+ throw new Error(
54
+ `workflow recovery requires a nonempty ${field} of at most ${limit} characters without control characters`,
55
+ );
56
+ }
57
+ }
58
+ }
59
+
60
+ /** Canonical recovery ledger: immutable intent is durable before effects.
61
+ * Completion cites that exact intent; absence always means unknown. */
62
+ export function beginWorkflowRecovery(
63
+ coordRoot: string,
64
+ input: Omit<
65
+ WorkflowRecoveryIntent,
66
+ "schema_version" | "operation_id" | "recorded_at" | "pid" | "host" | "outcome"
67
+ >,
68
+ ): { intent: WorkflowRecoveryIntent; assertCurrent(): void; complete(): void } {
69
+ assertWorkflowRunId(input.run_id);
70
+ validateWorkflowRecoveryAuthorization(input);
71
+ if (!/^[a-f0-9]{64}$/.test(input.target_sha256))
72
+ throw new Error("workflow recovery requires an exact preview digest");
73
+ const intent: WorkflowRecoveryIntent = {
74
+ ...input,
75
+ schema_version: 1,
76
+ operation_id: `recovery-${randomUUID()}`,
77
+ recorded_at: new Date().toISOString(),
78
+ pid: process.pid,
79
+ host: hostname(),
80
+ outcome: "unknown",
81
+ };
82
+ const directory = join(
83
+ coordRoot,
84
+ ".harnery",
85
+ "workflows",
86
+ input.run_id,
87
+ "recovery",
88
+ intent.operation_id,
89
+ );
90
+ // Sync each newly published directory name too. Syncing only the intent's
91
+ // immediate parent cannot make a newly created ancestor durable.
92
+ const runDirectory = join(coordRoot, ".harnery", "workflows", input.run_id);
93
+ ensureDurableDirectory(runDirectory);
94
+ const ledgerDirectory = join(runDirectory, "recovery");
95
+ ensureDurableDirectory(ledgerDirectory);
96
+ fsyncParentDirectory(ledgerDirectory);
97
+ mkdirSync(directory, { mode: stateDirMode() });
98
+ fsyncParentDirectory(directory);
99
+ const intentPath = join(directory, "intent.json");
100
+ writeImmutableJson(intentPath, intent);
101
+ const assertCurrent = () => {
102
+ if (stableDigest(readJsonRecord(intentPath, "recovery intent")) !== stableDigest(intent))
103
+ throw new Error("workflow recovery audit authority changed");
104
+ };
105
+ return {
106
+ intent,
107
+ assertCurrent,
108
+ complete: () => {
109
+ assertCurrent();
110
+ writeImmutableJson(join(directory, "completion.json"), {
111
+ schema_version: 1,
112
+ operation_id: intent.operation_id,
113
+ intent_sha256: stableDigest(intent),
114
+ outcome: "completed",
115
+ recorded_at: new Date().toISOString(),
116
+ });
117
+ },
118
+ };
119
+ }
120
+
121
+ function ensureDurableDirectory(path: string): void {
122
+ if (existsSync(path)) {
123
+ if (!lstatSync(path).isDirectory())
124
+ throw new Error("workflow recovery ledger is not a directory");
125
+ return;
126
+ }
127
+ ensureDurableDirectory(dirname(path));
128
+ try {
129
+ mkdirSync(path, { mode: stateDirMode() });
130
+ } catch (error) {
131
+ if ((error as NodeJS.ErrnoException).code !== "EEXIST" || !lstatSync(path).isDirectory())
132
+ throw error;
133
+ }
134
+ fsyncParentDirectory(path);
135
+ }
136
+
137
+ export function readWorkflowRecoveryLedger(
138
+ coordRoot: string,
139
+ runId: string,
140
+ ): WorkflowRecoveryRecord[] {
141
+ assertWorkflowRunId(runId);
142
+ const directory = join(coordRoot, ".harnery", "workflows", runId, "recovery");
143
+ if (!existsSync(directory)) return [];
144
+ return readdirSync(directory)
145
+ .sort()
146
+ .map((entry) => {
147
+ if (!/^recovery-[a-f0-9-]{36}$/.test(entry))
148
+ throw new Error("invalid workflow recovery ledger entry");
149
+ const intent = readJsonRecord<WorkflowRecoveryIntent>(
150
+ join(directory, entry, "intent.json"),
151
+ "recovery intent",
152
+ );
153
+ if (
154
+ intent.schema_version !== 1 ||
155
+ intent.operation_id !== entry ||
156
+ intent.run_id !== runId ||
157
+ intent.outcome !== "unknown" ||
158
+ !["reclaim_unknown_liveness", "unlock_resume"].includes(intent.action) ||
159
+ !/^[a-f0-9]{64}$/.test(intent.target_sha256)
160
+ ) {
161
+ throw new Error("invalid workflow recovery intent authority");
162
+ }
163
+ validateWorkflowRecoveryAuthorization(intent);
164
+ const completionPath = join(directory, entry, "completion.json");
165
+ if (!existsSync(completionPath)) return { intent, outcome: "unknown" };
166
+ const completion = readJsonRecord<{
167
+ schema_version: number;
168
+ operation_id: string;
169
+ intent_sha256: string;
170
+ outcome: string;
171
+ }>(completionPath, "recovery completion");
172
+ if (
173
+ completion.schema_version !== 1 ||
174
+ completion.operation_id !== entry ||
175
+ completion.intent_sha256 !== stableDigest(intent) ||
176
+ completion.outcome !== "completed"
177
+ )
178
+ throw new Error("invalid workflow recovery completion authority");
179
+ return { intent, outcome: "completed" };
180
+ });
181
+ }
@@ -3,6 +3,8 @@ import {
3
3
  chmodSync,
4
4
  closeSync,
5
5
  existsSync,
6
+ fsyncSync,
7
+ lstatSync,
6
8
  mkdirSync,
7
9
  openSync,
8
10
  readFileSync,
@@ -23,7 +25,13 @@ import {
23
25
  import { stateFileMode } from "../storage/modes.ts";
24
26
  import { assertWorkflowRunId, readWorkflowApproval } from "./approvals.ts";
25
27
  import { isCanonicalWorkflowAttemptContext } from "./attempt-context.ts";
26
- import { readJsonRecord, writeImmutableJson } from "./durable-record.ts";
28
+ import {
29
+ fsyncParentDirectory,
30
+ readJsonRecord,
31
+ stableDigest,
32
+ writeImmutableJson,
33
+ } from "./durable-record.ts";
34
+ import { beginWorkflowRecovery, type WorkflowRecoveryAuthorization } from "./recovery-audit.ts";
27
35
  import { normalizeWorkflowSpecialists } from "./specialists.ts";
28
36
  import type {
29
37
  WorkflowAttemptContext,
@@ -38,6 +46,7 @@ import type {
38
46
  WorkspaceBinding,
39
47
  WorkspaceCompatibilityExecutionEvidence,
40
48
  } from "./workspaces/index.ts";
49
+ import { acquireNoClobberLease } from "./workspaces/leases.ts";
41
50
  import {
42
51
  assertWorkspaceManifestAuthority,
43
52
  isWorkspaceBinding,
@@ -204,8 +213,7 @@ export function workflowRunManifestPath(coordRoot: string, runId: string): strin
204
213
  }
205
214
 
206
215
  /** Acquire an exclusive lease shared by resume and reclaim. Existing locks
207
- * always refuse: a stale observation cannot authorize unlinking a new owner.
208
- * Crash recovery requires an audited ownership check outside this operation. */
216
+ * always refuse; explicit audited recovery is a separate operation. */
209
217
  export function acquireWorkflowResumeLease(coordRoot: string, runId: string): () => void {
210
218
  assertWorkflowRunId(runId);
211
219
  const path = join(coordRoot, ".harnery", "workflows", runId, "resume.lock");
@@ -218,23 +226,27 @@ export function acquireWorkflowResumeLease(coordRoot: string, runId: string): ()
218
226
  const ownerBytes = `${JSON.stringify(owner)}\n`;
219
227
  mkdirSync(dirname(path), { recursive: true });
220
228
 
221
- const acquire = (): boolean => {
229
+ const acquire = (): { device: number; inode: number } | false => {
222
230
  try {
223
231
  const fd = openSync(path, "wx", stateFileMode());
224
232
  try {
225
233
  writeFileSync(fd, ownerBytes, "utf8");
234
+ fsyncSync(fd);
226
235
  } finally {
227
236
  closeSync(fd);
228
237
  }
229
238
  chmodSync(path, stateFileMode());
230
- return true;
239
+ fsyncParentDirectory(path);
240
+ const identity = lstatSync(path);
241
+ return { device: identity.dev, inode: identity.ino };
231
242
  } catch (error) {
232
243
  if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
233
244
  return false;
234
245
  }
235
246
  };
236
247
 
237
- if (!acquire()) {
248
+ const acquired = withResumeLeaseMutation(coordRoot, runId, acquire);
249
+ if (!acquired) {
238
250
  const existing = readResumeLease(path);
239
251
  const holder = existing
240
252
  ? ` by pid ${existing.pid} on ${existing.host}; its liveness has not been verified`
@@ -244,13 +256,129 @@ export function acquireWorkflowResumeLease(coordRoot: string, runId: string): ()
244
256
 
245
257
  return () => {
246
258
  try {
247
- if (readFileSync(path, "utf8") === ownerBytes) unlinkSync(path);
259
+ // Only this live process can release this nonce. Acquisition never
260
+ // replaces a present lock, and unlock requires ESRCH for its owner, so
261
+ // neither supported writer can replace this lock during synchronous
262
+ // release. Taking their mutation gate would strand a live owner's lock
263
+ // on ordinary contention. Pin both identity and bytes for stale closures.
264
+ const identity = lstatSync(path);
265
+ if (
266
+ identity.isFile() &&
267
+ identity.dev === acquired.device &&
268
+ identity.ino === acquired.inode &&
269
+ readFileSync(path, "utf8") === ownerBytes
270
+ ) {
271
+ unlinkSync(path);
272
+ fsyncParentDirectory(path);
273
+ }
248
274
  } catch {
249
275
  // Leave an unreadable or replaced lock untouched.
250
276
  }
251
277
  };
252
278
  }
253
279
 
280
+ export interface WorkflowResumeRecoveryOptions extends Partial<WorkflowRecoveryAuthorization> {
281
+ yes?: boolean;
282
+ lockSha256?: string;
283
+ }
284
+
285
+ export interface WorkflowResumeRecoveryResult {
286
+ action: "preview" | "unlocked";
287
+ run_id: string;
288
+ lock_sha256: string | null;
289
+ owner_state: "dead" | "live" | "unknown" | "absent";
290
+ owner: { pid: number; host: string; created_at: string } | null;
291
+ recovery_id?: string;
292
+ }
293
+
294
+ export function recoverWorkflowResumeLease(
295
+ coordRoot: string,
296
+ runId: string,
297
+ options: WorkflowResumeRecoveryOptions,
298
+ ): WorkflowResumeRecoveryResult {
299
+ assertWorkflowRunId(runId);
300
+ const path = join(coordRoot, ".harnery", "workflows", runId, "resume.lock");
301
+ const inspect = (): WorkflowResumeRecoveryResult => {
302
+ let lockSha256: string;
303
+ try {
304
+ const stat = lstatSync(path);
305
+ if (!stat.isFile() || stat.size > 8 * 1024)
306
+ throw new Error("resume lock is not a bounded regular file");
307
+ lockSha256 = stableDigest({
308
+ bytes: readFileSync(path, "utf8"),
309
+ device: stat.dev,
310
+ inode: stat.ino,
311
+ });
312
+ } catch (error) {
313
+ if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
314
+ return {
315
+ action: "preview",
316
+ run_id: runId,
317
+ lock_sha256: null,
318
+ owner_state: "absent",
319
+ owner: null,
320
+ };
321
+ }
322
+ const owner = readResumeLease(path);
323
+ let ownerState: WorkflowResumeRecoveryResult["owner_state"] = "unknown";
324
+ if (owner?.host === hostname()) {
325
+ try {
326
+ process.kill(owner.pid, 0);
327
+ ownerState = "live";
328
+ } catch (error) {
329
+ if ((error as NodeJS.ErrnoException).code === "ESRCH") ownerState = "dead";
330
+ }
331
+ }
332
+ return {
333
+ action: "preview",
334
+ run_id: runId,
335
+ lock_sha256: lockSha256,
336
+ owner_state: ownerState,
337
+ owner,
338
+ };
339
+ };
340
+ if (!options.yes) return inspect();
341
+ return withResumeLeaseMutation(coordRoot, runId, () => {
342
+ const observed = inspect();
343
+ if (observed.owner_state !== "dead")
344
+ throw new Error(`workflow resume unlock refused: owner is ${observed.owner_state}`);
345
+ if (!options.lockSha256 || options.lockSha256 !== observed.lock_sha256)
346
+ throw new Error("workflow resume unlock requires the exact current preview digest");
347
+ const audit = beginWorkflowRecovery(coordRoot, {
348
+ run_id: runId,
349
+ action: "unlock_resume",
350
+ target_sha256: options.lockSha256,
351
+ target: { lock: "resume.lock", owner: observed.owner },
352
+ actor: options.actor!,
353
+ reason: options.reason!,
354
+ });
355
+ const current = inspect();
356
+ if (current.lock_sha256 !== observed.lock_sha256 || current.owner_state !== "dead")
357
+ throw new Error("workflow resume lock authority changed before unlock");
358
+ audit.assertCurrent();
359
+ unlinkSync(path);
360
+ fsyncParentDirectory(path);
361
+ audit.complete();
362
+ return { ...observed, action: "unlocked", recovery_id: audit.intent.operation_id };
363
+ });
364
+ }
365
+
366
+ function withResumeLeaseMutation<T>(coordRoot: string, runId: string, action: () => T): T {
367
+ const authority = stableDigest({ run_id: runId, lock: "resume.lock" });
368
+ const lease = acquireNoClobberLease({
369
+ path: join(coordRoot, ".harnery", "workflows", runId, "resume-mutation.lease"),
370
+ scope: "workflow-resume-mutation",
371
+ authoritySha256: authority,
372
+ staleAfterMs: 300_000,
373
+ validateStaleOwner: (owner) => owner.authority_sha256 === authority,
374
+ });
375
+ try {
376
+ return action();
377
+ } finally {
378
+ lease.release();
379
+ }
380
+ }
381
+
254
382
  function readResumeLease(path: string): { pid: number; host: string; created_at: string } | null {
255
383
  try {
256
384
  const value = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>;
@@ -258,7 +386,8 @@ function readResumeLease(path: string): { pid: number; host: string; created_at:
258
386
  Number.isSafeInteger(value.pid) &&
259
387
  value.pid > 0 &&
260
388
  typeof value.host === "string" &&
261
- typeof value.created_at === "string"
389
+ value.host.length > 0 &&
390
+ validTimestamp(value.created_at)
262
391
  ? { pid: value.pid, host: value.host, created_at: value.created_at }
263
392
  : null;
264
393
  } catch {
@@ -9,6 +9,7 @@ import {
9
9
  } from "node:fs";
10
10
  import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
11
11
  import { stateDirMode, stateFileMode } from "../../storage/modes.ts";
12
+ import { beginWorkflowRecovery, validateWorkflowRecoveryAuthorization } from "../recovery-audit.ts";
12
13
  import { acquireWorkflowResumeLease } from "../run-state.ts";
13
14
  import { WorkspaceAttestationError } from "./attestation-error.ts";
14
15
  import { cleanupWorkspace } from "./cleanup.ts";
@@ -53,7 +54,7 @@ import {
53
54
  type WorkspaceReclaimPreview,
54
55
  type WorkspaceReclaimResult,
55
56
  } from "./reclaim.ts";
56
- import { readWorkflowLiveness } from "./recovery.ts";
57
+ import { hasActiveWorkflowEvidence, readWorkflowLiveness } from "./recovery.ts";
57
58
  import {
58
59
  appendWorkspaceEvent,
59
60
  fileSha256,
@@ -820,6 +821,7 @@ function reclaimPreview(
820
821
  run_id: binding.run_id,
821
822
  mode,
822
823
  liveness: readWorkflowLiveness(coordRoot, binding.run_id),
824
+ evidence_sha256: reclaimEvidenceDigest(coordRoot, binding),
823
825
  ownership: "unproven",
824
826
  dirty_paths: null,
825
827
  lease_holder: reclaimLeaseHolders(coordRoot, binding),
@@ -839,14 +841,45 @@ function reclaimPreview(
839
841
  return preview;
840
842
  }
841
843
 
842
- function assertReclaimLiveness(coordRoot: string, binding: WorkspaceBinding): void {
844
+ function reclaimEvidenceDigest(coordRoot: string, binding: WorkspaceBinding): string {
845
+ const evidence = (name: string): string | null => {
846
+ try {
847
+ return readFileSync(join(coordRoot, ".harnery", "workflows", binding.run_id, name), "utf8");
848
+ } catch (error) {
849
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
850
+ throw error;
851
+ }
852
+ };
853
+ return stableDigest({
854
+ binding,
855
+ proof: evidence("proof.json"),
856
+ transcript: evidence("transcript.jsonl"),
857
+ });
858
+ }
859
+
860
+ function assertReclaimLiveness(
861
+ coordRoot: string,
862
+ binding: WorkspaceBinding,
863
+ recovery?: WorkspaceReclaimOptions["unknownLiveness"],
864
+ ): void {
843
865
  const liveness = readWorkflowLiveness(coordRoot, binding.run_id);
866
+ if (liveness === "active" || (recovery && hasActiveWorkflowEvidence(coordRoot, binding.run_id))) {
867
+ throw new WorkflowReclaimError(
868
+ "workflow_reclaim_run_live",
869
+ `run ${binding.run_id} has active evidence; its workspace was left untouched`,
870
+ );
871
+ }
872
+ if (recovery && reclaimEvidenceDigest(coordRoot, binding) !== recovery.evidenceSha256) {
873
+ throw new WorkflowReclaimError(
874
+ "workflow_reclaim_recovery_mismatch",
875
+ "workflow reclaim requires the exact current preview evidence digest",
876
+ );
877
+ }
844
878
  if (liveness === "inactive") return;
879
+ if (liveness === "unknown" && recovery) return;
845
880
  throw new WorkflowReclaimError(
846
- liveness === "active" ? "workflow_reclaim_run_live" : "workflow_reclaim_liveness_unknown",
847
- liveness === "active"
848
- ? `run ${binding.run_id} is active; its workspace was left untouched`
849
- : `run ${binding.run_id} has missing, corrupt, or ambiguous liveness evidence; its workspace was left untouched`,
881
+ "workflow_reclaim_liveness_unknown",
882
+ `run ${binding.run_id} has missing, corrupt, or ambiguous liveness evidence; its workspace was left untouched`,
850
883
  );
851
884
  }
852
885
 
@@ -892,10 +925,11 @@ function reclaimAuthorityGuard(
892
925
  binding: WorkspaceBinding,
893
926
  claim: WorkspaceClaim,
894
927
  reservation: WorkspaceReclaimReservation,
928
+ recovery?: WorkspaceReclaimOptions["unknownLiveness"],
895
929
  ): void {
896
930
  try {
897
931
  reservation.assertCurrent();
898
- assertReclaimLiveness(coordRoot, binding);
932
+ assertReclaimLiveness(coordRoot, binding, recovery);
899
933
  assertReclaimOwnership(coordRoot, binding);
900
934
  assertPeerWorkspaceAuthority(coordRoot, claim);
901
935
  } catch (error) {
@@ -914,10 +948,11 @@ async function reclaim(
914
948
  throw new Error("unsupported workspace reclaim mode");
915
949
  }
916
950
  if (!options.yes) return reclaimPreview(coordRoot, binding, options.mode);
951
+ if (options.unknownLiveness) validateWorkflowRecoveryAuthorization(options.unknownLiveness);
917
952
 
918
953
  // Liveness comes before terminal-proof cleanup helpers. A live run may not
919
954
  // have a proof yet, and a missing proof must never be discovered after reset.
920
- assertReclaimLiveness(coordRoot, binding);
955
+ assertReclaimLiveness(coordRoot, binding, options.unknownLiveness);
921
956
  const claim = assertReclaimOwnership(coordRoot, binding);
922
957
  let reservation: WorkspaceReclaimReservation;
923
958
  try {
@@ -926,7 +961,7 @@ async function reclaim(
926
961
  throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
927
962
  }
928
963
  try {
929
- reclaimAuthorityGuard(coordRoot, binding, claim, reservation);
964
+ reclaimAuthorityGuard(coordRoot, binding, claim, reservation, options.unknownLiveness);
930
965
  let releaseResume: () => void;
931
966
  try {
932
967
  releaseResume = acquireWorkflowResumeLease(coordRoot, binding.run_id);
@@ -953,9 +988,24 @@ async function reclaim(
953
988
  throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
954
989
  }
955
990
  try {
956
- assertReclaimLiveness(coordRoot, binding);
991
+ assertReclaimLiveness(coordRoot, binding, options.unknownLiveness);
957
992
  assertReclaimOwnership(coordRoot, binding);
958
993
  const preview = reclaimPreview(coordRoot, binding, options.mode);
994
+ const audit = options.unknownLiveness
995
+ ? beginWorkflowRecovery(coordRoot, {
996
+ run_id: binding.run_id,
997
+ action: "reclaim_unknown_liveness",
998
+ actor: options.unknownLiveness.actor,
999
+ reason: options.unknownLiveness.reason,
1000
+ target_sha256: options.unknownLiveness.evidenceSha256,
1001
+ target: {
1002
+ binding_id: binding.binding_id,
1003
+ workspace_root: binding.workspace_root,
1004
+ mode: options.mode,
1005
+ liveness: preview.liveness,
1006
+ },
1007
+ })
1008
+ : undefined;
959
1009
  let preparation: ReclaimPreparation | undefined;
960
1010
  // Keep ordinary cleanup's terminal-evidence checks, intent and receipts.
961
1011
  // Its provider callback uses the leases already held here instead of
@@ -967,12 +1017,28 @@ async function reclaim(
967
1017
  provider: {
968
1018
  ...provider,
969
1019
  cleanup: async (candidate, intent) => {
970
- assertReclaimLiveness(coordRoot, candidate);
1020
+ assertReclaimLiveness(coordRoot, candidate, options.unknownLiveness);
971
1021
  assertReclaimOwnership(coordRoot, candidate);
972
1022
  return cleanupUnderLease(coordRoot, candidate, claim, intent, {
973
1023
  mode: options.mode,
974
- beforeEffect: () =>
975
- reclaimAuthorityGuard(coordRoot, candidate, claim, reservation),
1024
+ recovery: options.unknownLiveness,
1025
+ beforeEffect: () => {
1026
+ audit?.assertCurrent();
1027
+ reclaimAuthorityGuard(
1028
+ coordRoot,
1029
+ candidate,
1030
+ claim,
1031
+ reservation,
1032
+ options.unknownLiveness,
1033
+ );
1034
+ },
1035
+ beforeRefEffect: () => {
1036
+ audit?.assertCurrent();
1037
+ reservation.assertCurrent();
1038
+ assertReclaimLiveness(coordRoot, candidate, options.unknownLiveness);
1039
+ requiredClaim(coordRoot, candidate);
1040
+ assertPeerWorkspaceAuthority(coordRoot, claim);
1041
+ },
976
1042
  prepared: (value) => {
977
1043
  preparation = value;
978
1044
  },
@@ -991,6 +1057,8 @@ async function reclaim(
991
1057
  "workspace cleanup preconditions could not be proved",
992
1058
  );
993
1059
  }
1060
+ if (cleanupResult.status === "released" || cleanupResult.status === "already_released")
1061
+ audit?.complete();
994
1062
  return {
995
1063
  ...preview,
996
1064
  action: "reclaimed",
@@ -1104,7 +1172,9 @@ function cleanupUnderLease(
1104
1172
  intent: WorkspaceCleanupIntent,
1105
1173
  reclaimRequest?: {
1106
1174
  mode: ReclaimMode;
1175
+ recovery?: WorkspaceReclaimOptions["unknownLiveness"];
1107
1176
  beforeEffect: () => void;
1177
+ beforeRefEffect: () => void;
1108
1178
  prepared: (value: ReclaimPreparation) => void;
1109
1179
  },
1110
1180
  beforeRemoval?: () => void,
@@ -1152,7 +1222,7 @@ function cleanupUnderLease(
1152
1222
  };
1153
1223
  }
1154
1224
  const liveness = readWorkflowLiveness(coordRoot, binding.run_id);
1155
- if (liveness !== "inactive") {
1225
+ if (liveness !== "inactive" && !(liveness === "unknown" && reclaimRequest?.recovery)) {
1156
1226
  return {
1157
1227
  ...base,
1158
1228
  status: "blocked",
@@ -1245,7 +1315,7 @@ function cleanupUnderLease(
1245
1315
  }
1246
1316
 
1247
1317
  if (reclaimRequest) {
1248
- assertReclaimLiveness(coordRoot, binding);
1318
+ assertReclaimLiveness(coordRoot, binding, reclaimRequest.recovery);
1249
1319
  assertReclaimOwnership(coordRoot, binding);
1250
1320
  revalidateClaimResources(claim, root);
1251
1321
  reclaimRequest.prepared(
@@ -1293,6 +1363,7 @@ function cleanupUnderLease(
1293
1363
  return { ...base, status: "blocked", reason: "workspace ref changed during cleanup" };
1294
1364
  }
1295
1365
  revalidateClaimResources(claim, root);
1366
+ reclaimRequest?.beforeRefEffect();
1296
1367
  const deleted = gitMaybe(repository.source_root.realpath, [
1297
1368
  "update-ref",
1298
1369
  "-d",
@@ -16,6 +16,7 @@
16
16
  */
17
17
 
18
18
  import { existsSync } from "node:fs";
19
+ import type { WorkflowRecoveryAuthorization } from "../recovery-audit.ts";
19
20
  import { git, gitMaybe } from "./git.ts";
20
21
  import type { WorkflowLiveness } from "./recovery.ts";
21
22
  import type { WorkspaceCleanupAttempt, WorkspaceCleanupReceipt } from "./types.ts";
@@ -25,6 +26,7 @@ export type ReclaimMode = "salvage" | "discard";
25
26
  export type WorkflowReclaimRefusalCode =
26
27
  | "workflow_reclaim_run_live"
27
28
  | "workflow_reclaim_liveness_unknown"
29
+ | "workflow_reclaim_recovery_mismatch"
28
30
  | "workflow_reclaim_ownership_unproven"
29
31
  | "workflow_reclaim_lease_held";
30
32
 
@@ -41,6 +43,7 @@ export class WorkflowReclaimError extends Error {
41
43
  export interface WorkspaceReclaimOptions {
42
44
  mode: ReclaimMode;
43
45
  yes?: boolean;
46
+ unknownLiveness?: WorkflowRecoveryAuthorization & { evidenceSha256: string };
44
47
  }
45
48
 
46
49
  export interface ReclaimLeaseHolder {
@@ -56,6 +59,7 @@ export interface WorkspaceReclaimPreview {
56
59
  run_id: string;
57
60
  mode: ReclaimMode;
58
61
  liveness: WorkflowLiveness;
62
+ evidence_sha256: string;
59
63
  ownership: "ok" | "unproven";
60
64
  ownership_detail?: string;
61
65
  dirty_paths: string[] | null;
@@ -4,6 +4,29 @@ import { readWorkflowProof } from "../proof.ts";
4
4
 
5
5
  export type WorkflowLiveness = "active" | "inactive" | "unknown";
6
6
 
7
+ /** Corruption must not hide a readable active marker from an audited override. */
8
+ export function hasActiveWorkflowEvidence(coordRoot: string, runId: string): boolean {
9
+ const path = join(resolve(coordRoot), ".harnery", "workflows", runId, "transcript.jsonl");
10
+ if (!existsSync(path)) return false;
11
+ let active = false;
12
+ for (const line of readFileSync(path, "utf8").split("\n")) {
13
+ try {
14
+ const event = JSON.parse(line);
15
+ if (event?.run_id !== runId) continue;
16
+ if (["run.start", "run.resume", "agent.start"].includes(event.event)) active = true;
17
+ if (
18
+ event.event === "run.parked" ||
19
+ (event.event === "run.end" && typeof event.ok === "boolean") ||
20
+ (event.event === "workspace.reattach.failed" && ["blocked", "lost"].includes(event.status))
21
+ )
22
+ active = false;
23
+ } catch {
24
+ /* Unreadable evidence cannot establish inactivity. */
25
+ }
26
+ }
27
+ return active;
28
+ }
29
+
7
30
  export function readWorkflowLiveness(coordRoot: string, runId: string): WorkflowLiveness {
8
31
  const root = resolve(coordRoot);
9
32
  const transcriptPath = join(root, ".harnery", "workflows", runId, "transcript.jsonl");