harnery 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/commander.js +1 -0
- package/dist/commands/agents.d.ts.map +1 -1
- package/dist/commands/agents.js +41 -48
- package/dist/commands/browse.d.ts.map +1 -1
- package/dist/commands/browse.js +38 -11
- package/dist/commands/diagnostics.d.ts.map +1 -1
- package/dist/commands/diagnostics.js +10 -0
- package/dist/commands/doctor.d.ts +12 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +22 -7
- package/dist/commands/files.d.ts +2 -2
- package/dist/commands/files.d.ts.map +1 -1
- package/dist/commands/files.js +7 -4
- package/dist/commands/governor.js +22 -10
- package/dist/commands/inbox.d.ts +1 -1
- package/dist/commands/inbox.d.ts.map +1 -1
- package/dist/commands/inbox.js +55 -2
- package/dist/commands/rm.d.ts.map +1 -1
- package/dist/commands/rm.js +28 -4
- package/dist/commands/servers.d.ts.map +1 -1
- package/dist/commands/servers.js +26 -17
- package/dist/commands/supervisor.d.ts.map +1 -1
- package/dist/commands/supervisor.js +7 -2
- package/dist/commands/toc.d.ts +14 -0
- package/dist/commands/toc.d.ts.map +1 -1
- package/dist/commands/toc.js +7 -4
- package/dist/commands/tunnel.d.ts +15 -0
- package/dist/commands/tunnel.d.ts.map +1 -1
- package/dist/commands/tunnel.js +453 -153
- package/dist/commands/turn-rollback.d.ts +5 -0
- package/dist/commands/turn-rollback.d.ts.map +1 -0
- package/dist/commands/turn-rollback.js +76 -0
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +68 -30
- package/dist/core/agents/coord-client.d.ts.map +1 -1
- package/dist/core/agents/coord-client.js +18 -0
- package/dist/core/agents/live-authority-v3.d.ts +1 -0
- package/dist/core/agents/live-authority-v3.d.ts.map +1 -1
- package/dist/core/agents/live-authority-v3.js +11 -0
- package/dist/core/agents/live-lifecycle-v3.d.ts +4 -2
- package/dist/core/agents/live-lifecycle-v3.d.ts.map +1 -1
- package/dist/core/agents/live-lifecycle-v3.js +28 -7
- package/dist/core/agents/mailbox.d.ts +9 -8
- package/dist/core/agents/mailbox.d.ts.map +1 -1
- package/dist/core/agents/mailbox.js +24 -17
- package/dist/core/agents/render/prompt-context.d.ts +2 -0
- package/dist/core/agents/render/prompt-context.d.ts.map +1 -1
- package/dist/core/agents/render/prompt-context.js +8 -12
- package/dist/core/agents/render/session-context.d.ts +2 -8
- package/dist/core/agents/render/session-context.d.ts.map +1 -1
- package/dist/core/agents/render/session-context.js +7 -30
- package/dist/core/agents/rules/stop-hook.js +3 -3
- package/dist/core/agents/state/heartbeat-writer.d.ts.map +1 -1
- package/dist/core/agents/state/heartbeat-writer.js +25 -14
- package/dist/core/agents/state/live-coordination-writer.d.ts.map +1 -1
- package/dist/core/agents/state/live-coordination-writer.js +5 -0
- package/dist/core/agents/state/pidmap.d.ts.map +1 -1
- package/dist/core/agents/state/pidmap.js +5 -0
- package/dist/core/agents/state/proc-start.d.ts +19 -1
- package/dist/core/agents/state/proc-start.d.ts.map +1 -1
- package/dist/core/agents/state/proc-start.js +46 -1
- package/dist/core/agents/state/process-handle.d.ts +26 -0
- package/dist/core/agents/state/process-handle.d.ts.map +1 -0
- package/dist/core/agents/state/process-handle.js +121 -0
- package/dist/core/agents/state/stale-sweep.d.ts.map +1 -1
- package/dist/core/agents/state/stale-sweep.js +180 -70
- package/dist/core/agents/state/unchanged-file.d.ts +44 -0
- package/dist/core/agents/state/unchanged-file.d.ts.map +1 -0
- package/dist/core/agents/state/unchanged-file.js +390 -0
- package/dist/core/artifacts/delivery-card.d.ts.map +1 -1
- package/dist/core/artifacts/delivery-card.js +2 -1
- package/dist/core/context/index.d.ts +22 -10
- package/dist/core/context/index.d.ts.map +1 -1
- package/dist/core/context/index.js +381 -119
- package/dist/core/context/observation.d.ts +26 -0
- package/dist/core/context/observation.d.ts.map +1 -0
- package/dist/core/context/observation.js +11 -0
- package/dist/core/diagnostics/bundle.d.ts.map +1 -1
- package/dist/core/diagnostics/bundle.js +25 -14
- package/dist/core/diagnostics/path-identity.d.ts +3 -0
- package/dist/core/diagnostics/path-identity.d.ts.map +1 -0
- package/dist/core/diagnostics/path-identity.js +12 -0
- package/dist/core/diagnostics/sanitize.d.ts.map +1 -1
- package/dist/core/diagnostics/sanitize.js +4 -3
- package/dist/core/events/v3/authority-outbox.d.ts.map +1 -1
- package/dist/core/events/v3/authority-outbox.js +29 -0
- package/dist/core/events/v3/generation-custody.d.ts +5 -0
- package/dist/core/events/v3/generation-custody.d.ts.map +1 -0
- package/dist/core/events/v3/generation-custody.js +50 -0
- package/dist/core/events/v3/producers/coordination-recorder.d.ts +1 -0
- package/dist/core/events/v3/producers/coordination-recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/coordination-recorder.js +7 -0
- package/dist/core/events/v3/producers/recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/recorder.js +77 -64
- package/dist/core/governor/planning.d.ts.map +1 -1
- package/dist/core/governor/planning.js +14 -1
- package/dist/core/governor/service-read.d.ts +10 -2
- package/dist/core/governor/service-read.d.ts.map +1 -1
- package/dist/core/governor/service-read.js +105 -22
- package/dist/core/governor/service.d.ts +20 -2
- package/dist/core/governor/service.d.ts.map +1 -1
- package/dist/core/governor/service.js +72 -47
- package/dist/core/hooks/cli.js +33 -5
- package/dist/core/inbox/contract.d.ts +2 -1
- package/dist/core/inbox/contract.d.ts.map +1 -1
- package/dist/core/inbox/index.d.ts +2 -0
- package/dist/core/inbox/index.d.ts.map +1 -1
- package/dist/core/inbox/index.js +2 -0
- package/dist/core/inbox/peer-messages.d.ts +37 -0
- package/dist/core/inbox/peer-messages.d.ts.map +1 -0
- package/dist/core/inbox/peer-messages.js +72 -0
- package/dist/core/inbox/service.d.ts +8 -1
- package/dist/core/inbox/service.d.ts.map +1 -1
- package/dist/core/inbox/service.js +67 -21
- package/dist/core/inbox/targets.d.ts +22 -0
- package/dist/core/inbox/targets.d.ts.map +1 -0
- package/dist/core/inbox/targets.js +108 -0
- package/dist/core/inbox/watcher.js +7 -3
- package/dist/core/journal/index.d.ts.map +1 -1
- package/dist/core/journal/index.js +17 -2
- package/dist/core/resources/contract.d.ts +3 -0
- package/dist/core/resources/contract.d.ts.map +1 -1
- package/dist/core/resources/service-status.d.ts +14 -1
- package/dist/core/resources/service-status.d.ts.map +1 -1
- package/dist/core/resources/service-status.js +23 -13
- package/dist/core/resources/service.d.ts +7 -13
- package/dist/core/resources/service.d.ts.map +1 -1
- package/dist/core/resources/service.js +34 -91
- package/dist/core/resources/status.d.ts +2 -5
- package/dist/core/resources/status.d.ts.map +1 -1
- package/dist/core/resources/status.js +36 -7
- package/dist/core/semantic/service.d.ts.map +1 -1
- package/dist/core/semantic/service.js +2 -9
- package/dist/core/servers/index.d.ts +34 -22
- package/dist/core/servers/index.d.ts.map +1 -1
- package/dist/core/servers/index.js +297 -144
- package/dist/core/servers/tunnels.d.ts +7 -7
- package/dist/core/servers/tunnels.d.ts.map +1 -1
- package/dist/core/servers/tunnels.js +47 -10
- package/dist/core/supervisor/contract.d.ts +12 -0
- package/dist/core/supervisor/contract.d.ts.map +1 -1
- package/dist/core/supervisor/explanations.d.ts.map +1 -1
- package/dist/core/supervisor/explanations.js +2 -1
- package/dist/core/supervisor/findings.d.ts.map +1 -1
- package/dist/core/supervisor/findings.js +54 -9
- package/dist/core/supervisor/service.d.ts +7 -1
- package/dist/core/supervisor/service.d.ts.map +1 -1
- package/dist/core/supervisor/service.js +59 -29
- package/dist/core/supervisor/services.d.ts.map +1 -1
- package/dist/core/supervisor/services.js +22 -1
- package/dist/core/supervisor/status.d.ts +17 -2
- package/dist/core/supervisor/status.d.ts.map +1 -1
- package/dist/core/supervisor/status.js +70 -12
- package/dist/core/supervisor/timeline.d.ts.map +1 -1
- package/dist/core/supervisor/timeline.js +2 -1
- package/dist/core/turn-rollback/content.d.ts +53 -0
- package/dist/core/turn-rollback/content.d.ts.map +1 -0
- package/dist/core/turn-rollback/content.js +396 -0
- package/dist/core/turn-rollback/index.d.ts +72 -0
- package/dist/core/turn-rollback/index.d.ts.map +1 -0
- package/dist/core/turn-rollback/index.js +443 -0
- package/dist/core/work/state.js +7 -7
- package/dist/core/workflow/engine.d.ts.map +1 -1
- package/dist/core/workflow/engine.js +105 -32
- package/dist/core/workflow/failure-evidence.d.ts +16 -0
- package/dist/core/workflow/failure-evidence.d.ts.map +1 -0
- package/dist/core/workflow/failure-evidence.js +114 -0
- package/dist/core/workflow/index.d.ts +5 -3
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +3 -2
- package/dist/core/workflow/live-session-v3.d.ts +6 -0
- package/dist/core/workflow/live-session-v3.d.ts.map +1 -1
- package/dist/core/workflow/live-session-v3.js +126 -5
- package/dist/core/workflow/proof.d.ts +5 -5
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +115 -20
- package/dist/core/workflow/recovery-audit.d.ts +30 -0
- package/dist/core/workflow/recovery-audit.d.ts.map +1 -0
- package/dist/core/workflow/recovery-audit.js +116 -0
- package/dist/core/workflow/run-state.d.ts +20 -1
- package/dist/core/workflow/run-state.d.ts.map +1 -1
- package/dist/core/workflow/run-state.js +141 -32
- package/dist/core/workflow/spawn-claude.d.ts.map +1 -1
- package/dist/core/workflow/spawn-claude.js +19 -0
- package/dist/core/workflow/spawn-codex.d.ts.map +1 -1
- package/dist/core/workflow/spawn-codex.js +13 -0
- package/dist/core/workflow/spawn-cursor.d.ts.map +1 -1
- package/dist/core/workflow/spawn-cursor.js +19 -0
- package/dist/core/workflow/spawn-opencode.d.ts.map +1 -1
- package/dist/core/workflow/spawn-opencode.js +14 -0
- package/dist/core/workflow/types.d.ts +64 -17
- package/dist/core/workflow/types.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/index.d.ts +0 -1
- package/dist/core/workflow/workspaces/index.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/index.js +0 -1
- package/dist/core/workflow/workspaces/leases.d.ts +3 -1
- package/dist/core/workflow/workspaces/leases.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/leases.js +30 -16
- package/dist/core/workflow/workspaces/local-git.d.ts +6 -2
- package/dist/core/workflow/workspaces/local-git.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/local-git.js +328 -8
- package/dist/core/workflow/workspaces/ownership.d.ts +11 -0
- package/dist/core/workflow/workspaces/ownership.d.ts.map +1 -0
- package/dist/core/workflow/workspaces/ownership.js +194 -0
- package/dist/core/workflow/workspaces/reclaim.d.ts +40 -0
- package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/reclaim.js +14 -0
- package/dist/core/workflow/workspaces/recovery.d.ts +2 -0
- package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.js +42 -10
- package/dist/core/workflow/workspaces/state.d.ts +1 -0
- package/dist/core/workflow/workspaces/state.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/state.js +21 -1
- package/dist/lib/browser/windows-chrome.d.ts +100 -12
- package/dist/lib/browser/windows-chrome.d.ts.map +1 -1
- package/dist/lib/browser/windows-chrome.js +262 -73
- package/dist/lib/docs-links.d.ts.map +1 -1
- package/dist/lib/docs-links.js +45 -2
- package/dist/lib/guarded-remove.d.ts +7 -0
- package/dist/lib/guarded-remove.d.ts.map +1 -1
- package/dist/lib/guarded-remove.js +106 -12
- package/dist/lib/identities/assume.d.ts.map +1 -1
- package/dist/lib/identities/assume.js +4 -0
- package/dist/lib/tunnel/state.d.ts +13 -6
- package/dist/lib/tunnel/state.d.ts.map +1 -1
- package/dist/lib/tunnel/state.js +58 -12
- package/dist/lib/zip/index.d.ts +41 -0
- package/dist/lib/zip/index.d.ts.map +1 -0
- package/dist/lib/zip/index.js +398 -0
- package/package.json +12 -1
- package/src/commander.ts +10 -0
- package/src/commands/agents.ts +54 -63
- package/src/commands/browse.ts +50 -14
- package/src/commands/diagnostics.ts +10 -0
- package/src/commands/doctor.ts +26 -7
- package/src/commands/files.ts +13 -5
- package/src/commands/governor.ts +23 -10
- package/src/commands/inbox.ts +52 -2
- package/src/commands/rm.ts +43 -4
- package/src/commands/servers.ts +27 -26
- package/src/commands/supervisor.ts +7 -2
- package/src/commands/toc.ts +14 -6
- package/src/commands/tunnel.ts +533 -166
- package/src/commands/turn-rollback.ts +111 -0
- package/src/commands/workflow.ts +89 -30
- package/src/core/agents/coord-client.ts +20 -0
- package/src/core/agents/live-authority-v3.ts +17 -0
- package/src/core/agents/live-lifecycle-v3.ts +52 -14
- package/src/core/agents/mailbox.ts +26 -17
- package/src/core/agents/render/prompt-context.ts +10 -12
- package/src/core/agents/render/session-context.ts +9 -32
- package/src/core/agents/rules/stop-hook.ts +3 -3
- package/src/core/agents/state/heartbeat-writer.ts +29 -12
- package/src/core/agents/state/live-coordination-writer.ts +29 -0
- package/src/core/agents/state/pidmap.ts +13 -0
- package/src/core/agents/state/proc-start.ts +62 -2
- package/src/core/agents/state/process-handle.ts +165 -0
- package/src/core/agents/state/stale-sweep.ts +247 -82
- package/src/core/agents/state/unchanged-file.ts +456 -0
- package/src/core/artifacts/delivery-card.ts +7 -1
- package/src/core/context/index.ts +446 -136
- package/src/core/context/observation.ts +29 -0
- package/src/core/diagnostics/bundle.ts +23 -18
- package/src/core/diagnostics/path-identity.ts +15 -0
- package/src/core/diagnostics/sanitize.ts +4 -3
- package/src/core/events/v3/authority-outbox.ts +56 -0
- package/src/core/events/v3/generation-custody.ts +59 -0
- package/src/core/events/v3/producers/coordination-recorder.ts +15 -0
- package/src/core/events/v3/producers/recorder.ts +94 -68
- package/src/core/governor/planning.ts +20 -8
- package/src/core/governor/service-read.ts +128 -20
- package/src/core/governor/service.ts +119 -42
- package/src/core/hooks/cli.ts +33 -5
- package/src/core/inbox/contract.ts +2 -1
- package/src/core/inbox/index.ts +2 -0
- package/src/core/inbox/peer-messages.ts +94 -0
- package/src/core/inbox/service.ts +88 -22
- package/src/core/inbox/targets.ts +148 -0
- package/src/core/inbox/watcher.ts +8 -9
- package/src/core/journal/index.ts +19 -2
- package/src/core/resources/contract.ts +4 -0
- package/src/core/resources/service-status.ts +35 -12
- package/src/core/resources/service.ts +57 -95
- package/src/core/resources/status.ts +57 -8
- package/src/core/semantic/service.ts +2 -8
- package/src/core/servers/index.ts +376 -141
- package/src/core/servers/tunnels.ts +57 -13
- package/src/core/supervisor/contract.ts +13 -0
- package/src/core/supervisor/explanations.ts +4 -1
- package/src/core/supervisor/findings.ts +68 -10
- package/src/core/supervisor/service.ts +87 -26
- package/src/core/supervisor/services.ts +26 -8
- package/src/core/supervisor/status.ts +89 -12
- package/src/core/supervisor/timeline.ts +2 -1
- package/src/core/turn-rollback/content.ts +499 -0
- package/src/core/turn-rollback/index.ts +580 -0
- package/src/core/work/state.ts +7 -7
- package/src/core/workflow/engine.ts +131 -49
- package/src/core/workflow/failure-evidence.ts +141 -0
- package/src/core/workflow/index.ts +12 -2
- package/src/core/workflow/live-session-v3.ts +146 -5
- package/src/core/workflow/proof.ts +134 -26
- package/src/core/workflow/recovery-audit.ts +181 -0
- package/src/core/workflow/run-state.ts +165 -34
- package/src/core/workflow/spawn-claude.ts +18 -0
- package/src/core/workflow/spawn-codex.ts +13 -0
- package/src/core/workflow/spawn-cursor.ts +18 -0
- package/src/core/workflow/spawn-opencode.ts +14 -0
- package/src/core/workflow/types.ts +51 -17
- package/src/core/workflow/workspaces/index.ts +0 -1
- package/src/core/workflow/workspaces/leases.ts +48 -17
- package/src/core/workflow/workspaces/local-git.ts +388 -7
- package/src/core/workflow/workspaces/ownership.ts +232 -0
- package/src/core/workflow/workspaces/reclaim.ts +62 -0
- package/src/core/workflow/workspaces/recovery.ts +45 -10
- package/src/core/workflow/workspaces/state.ts +20 -0
- package/src/lib/browser/windows-chrome.ts +346 -86
- package/src/lib/docs-links.ts +49 -2
- package/src/lib/guarded-remove.ts +132 -14
- package/src/lib/identities/assume.ts +4 -0
- package/src/lib/tunnel/state.ts +70 -15
- package/src/lib/zip/index.ts +483 -0
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import {
|
|
3
3
|
chmodSync,
|
|
4
4
|
closeSync,
|
|
5
5
|
existsSync,
|
|
6
|
+
fsyncSync,
|
|
7
|
+
lstatSync,
|
|
6
8
|
mkdirSync,
|
|
7
9
|
openSync,
|
|
8
10
|
readFileSync,
|
|
@@ -23,7 +25,13 @@ import {
|
|
|
23
25
|
import { stateFileMode } from "../storage/modes.ts";
|
|
24
26
|
import { assertWorkflowRunId, readWorkflowApproval } from "./approvals.ts";
|
|
25
27
|
import { isCanonicalWorkflowAttemptContext } from "./attempt-context.ts";
|
|
26
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
fsyncParentDirectory,
|
|
30
|
+
readJsonRecord,
|
|
31
|
+
stableDigest,
|
|
32
|
+
writeImmutableJson,
|
|
33
|
+
} from "./durable-record.ts";
|
|
34
|
+
import { beginWorkflowRecovery, type WorkflowRecoveryAuthorization } from "./recovery-audit.ts";
|
|
27
35
|
import { normalizeWorkflowSpecialists } from "./specialists.ts";
|
|
28
36
|
import type {
|
|
29
37
|
WorkflowAttemptContext,
|
|
@@ -38,6 +46,7 @@ import type {
|
|
|
38
46
|
WorkspaceBinding,
|
|
39
47
|
WorkspaceCompatibilityExecutionEvidence,
|
|
40
48
|
} from "./workspaces/index.ts";
|
|
49
|
+
import { acquireNoClobberLease } from "./workspaces/leases.ts";
|
|
41
50
|
import {
|
|
42
51
|
assertWorkspaceManifestAuthority,
|
|
43
52
|
isWorkspaceBinding,
|
|
@@ -47,7 +56,6 @@ import {
|
|
|
47
56
|
export const WORKFLOW_RUN_MANIFEST_SCHEMA_VERSION = 2 as const;
|
|
48
57
|
|
|
49
58
|
const MANIFEST_LIMIT_BYTES = 256 * 1024;
|
|
50
|
-
const FOREIGN_LEASE_STALE_MS = 24 * 60 * 60 * 1_000;
|
|
51
59
|
|
|
52
60
|
export interface WorkflowRunManifest {
|
|
53
61
|
schema_version: typeof WORKFLOW_RUN_MANIFEST_SCHEMA_VERSION;
|
|
@@ -144,6 +152,13 @@ export function readWorkflowRunManifest(coordRoot: string, runId: string): Workf
|
|
|
144
152
|
) {
|
|
145
153
|
throw new Error(`workflow run manifest at ${path} has an unsupported or mismatched schema`);
|
|
146
154
|
}
|
|
155
|
+
if (manifest.repository_before.observation === undefined) {
|
|
156
|
+
manifest.repository_before = {
|
|
157
|
+
...manifest.repository_before,
|
|
158
|
+
observation: "unavailable",
|
|
159
|
+
observation_detail: "No repository observation was recorded.",
|
|
160
|
+
};
|
|
161
|
+
}
|
|
147
162
|
assertWorkspaceManifestAuthority(manifest, readManifestWorkspaceRequest(coordRoot, runId));
|
|
148
163
|
return manifest;
|
|
149
164
|
}
|
|
@@ -197,51 +212,173 @@ export function workflowRunManifestPath(coordRoot: string, runId: string): strin
|
|
|
197
212
|
return join(coordRoot, ".harnery", "workflows", runId, "run.json");
|
|
198
213
|
}
|
|
199
214
|
|
|
200
|
-
/** Acquire an exclusive
|
|
215
|
+
/** Acquire an exclusive lease shared by resume and reclaim. Existing locks
|
|
216
|
+
* always refuse; explicit audited recovery is a separate operation. */
|
|
201
217
|
export function acquireWorkflowResumeLease(coordRoot: string, runId: string): () => void {
|
|
202
218
|
assertWorkflowRunId(runId);
|
|
203
219
|
const path = join(coordRoot, ".harnery", "workflows", runId, "resume.lock");
|
|
204
|
-
const owner = {
|
|
220
|
+
const owner = {
|
|
221
|
+
owner_id: randomUUID(),
|
|
222
|
+
pid: process.pid,
|
|
223
|
+
host: hostname(),
|
|
224
|
+
created_at: new Date().toISOString(),
|
|
225
|
+
};
|
|
226
|
+
const ownerBytes = `${JSON.stringify(owner)}\n`;
|
|
205
227
|
mkdirSync(dirname(path), { recursive: true });
|
|
206
228
|
|
|
207
|
-
const acquire = ():
|
|
229
|
+
const acquire = (): { device: number; inode: number } | false => {
|
|
208
230
|
try {
|
|
209
231
|
const fd = openSync(path, "wx", stateFileMode());
|
|
210
232
|
try {
|
|
211
|
-
writeFileSync(fd,
|
|
233
|
+
writeFileSync(fd, ownerBytes, "utf8");
|
|
234
|
+
fsyncSync(fd);
|
|
212
235
|
} finally {
|
|
213
236
|
closeSync(fd);
|
|
214
237
|
}
|
|
215
238
|
chmodSync(path, stateFileMode());
|
|
216
|
-
|
|
239
|
+
fsyncParentDirectory(path);
|
|
240
|
+
const identity = lstatSync(path);
|
|
241
|
+
return { device: identity.dev, inode: identity.ino };
|
|
217
242
|
} catch (error) {
|
|
218
243
|
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
219
244
|
return false;
|
|
220
245
|
}
|
|
221
246
|
};
|
|
222
247
|
|
|
223
|
-
|
|
248
|
+
const acquired = withResumeLeaseMutation(coordRoot, runId, acquire);
|
|
249
|
+
if (!acquired) {
|
|
224
250
|
const existing = readResumeLease(path);
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
}
|
|
230
|
-
unlinkSync(path);
|
|
231
|
-
if (!acquire())
|
|
232
|
-
throw new Error(`workflow run ${runId} resume lease raced with another process`);
|
|
251
|
+
const holder = existing
|
|
252
|
+
? ` by pid ${existing.pid} on ${existing.host}; its liveness has not been verified`
|
|
253
|
+
: "; its owner could not be verified";
|
|
254
|
+
throw new Error(`workflow run ${runId} resume lease is already held${holder}`);
|
|
233
255
|
}
|
|
234
256
|
|
|
235
257
|
return () => {
|
|
236
258
|
try {
|
|
237
|
-
|
|
238
|
-
|
|
259
|
+
// Only this live process can release this nonce. Acquisition never
|
|
260
|
+
// replaces a present lock, and unlock requires ESRCH for its owner, so
|
|
261
|
+
// neither supported writer can replace this lock during synchronous
|
|
262
|
+
// release. Taking their mutation gate would strand a live owner's lock
|
|
263
|
+
// on ordinary contention. Pin both identity and bytes for stale closures.
|
|
264
|
+
const identity = lstatSync(path);
|
|
265
|
+
if (
|
|
266
|
+
identity.isFile() &&
|
|
267
|
+
identity.dev === acquired.device &&
|
|
268
|
+
identity.ino === acquired.inode &&
|
|
269
|
+
readFileSync(path, "utf8") === ownerBytes
|
|
270
|
+
) {
|
|
271
|
+
unlinkSync(path);
|
|
272
|
+
fsyncParentDirectory(path);
|
|
273
|
+
}
|
|
239
274
|
} catch {
|
|
240
|
-
//
|
|
275
|
+
// Leave an unreadable or replaced lock untouched.
|
|
241
276
|
}
|
|
242
277
|
};
|
|
243
278
|
}
|
|
244
279
|
|
|
280
|
+
export interface WorkflowResumeRecoveryOptions extends Partial<WorkflowRecoveryAuthorization> {
|
|
281
|
+
yes?: boolean;
|
|
282
|
+
lockSha256?: string;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
export interface WorkflowResumeRecoveryResult {
|
|
286
|
+
action: "preview" | "unlocked";
|
|
287
|
+
run_id: string;
|
|
288
|
+
lock_sha256: string | null;
|
|
289
|
+
owner_state: "dead" | "live" | "unknown" | "absent";
|
|
290
|
+
owner: { pid: number; host: string; created_at: string } | null;
|
|
291
|
+
recovery_id?: string;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
export function recoverWorkflowResumeLease(
|
|
295
|
+
coordRoot: string,
|
|
296
|
+
runId: string,
|
|
297
|
+
options: WorkflowResumeRecoveryOptions,
|
|
298
|
+
): WorkflowResumeRecoveryResult {
|
|
299
|
+
assertWorkflowRunId(runId);
|
|
300
|
+
const path = join(coordRoot, ".harnery", "workflows", runId, "resume.lock");
|
|
301
|
+
const inspect = (): WorkflowResumeRecoveryResult => {
|
|
302
|
+
let lockSha256: string;
|
|
303
|
+
try {
|
|
304
|
+
const stat = lstatSync(path);
|
|
305
|
+
if (!stat.isFile() || stat.size > 8 * 1024)
|
|
306
|
+
throw new Error("resume lock is not a bounded regular file");
|
|
307
|
+
lockSha256 = stableDigest({
|
|
308
|
+
bytes: readFileSync(path, "utf8"),
|
|
309
|
+
device: stat.dev,
|
|
310
|
+
inode: stat.ino,
|
|
311
|
+
});
|
|
312
|
+
} catch (error) {
|
|
313
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
314
|
+
return {
|
|
315
|
+
action: "preview",
|
|
316
|
+
run_id: runId,
|
|
317
|
+
lock_sha256: null,
|
|
318
|
+
owner_state: "absent",
|
|
319
|
+
owner: null,
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
const owner = readResumeLease(path);
|
|
323
|
+
let ownerState: WorkflowResumeRecoveryResult["owner_state"] = "unknown";
|
|
324
|
+
if (owner?.host === hostname()) {
|
|
325
|
+
try {
|
|
326
|
+
process.kill(owner.pid, 0);
|
|
327
|
+
ownerState = "live";
|
|
328
|
+
} catch (error) {
|
|
329
|
+
if ((error as NodeJS.ErrnoException).code === "ESRCH") ownerState = "dead";
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return {
|
|
333
|
+
action: "preview",
|
|
334
|
+
run_id: runId,
|
|
335
|
+
lock_sha256: lockSha256,
|
|
336
|
+
owner_state: ownerState,
|
|
337
|
+
owner,
|
|
338
|
+
};
|
|
339
|
+
};
|
|
340
|
+
if (!options.yes) return inspect();
|
|
341
|
+
return withResumeLeaseMutation(coordRoot, runId, () => {
|
|
342
|
+
const observed = inspect();
|
|
343
|
+
if (observed.owner_state !== "dead")
|
|
344
|
+
throw new Error(`workflow resume unlock refused: owner is ${observed.owner_state}`);
|
|
345
|
+
if (!options.lockSha256 || options.lockSha256 !== observed.lock_sha256)
|
|
346
|
+
throw new Error("workflow resume unlock requires the exact current preview digest");
|
|
347
|
+
const audit = beginWorkflowRecovery(coordRoot, {
|
|
348
|
+
run_id: runId,
|
|
349
|
+
action: "unlock_resume",
|
|
350
|
+
target_sha256: options.lockSha256,
|
|
351
|
+
target: { lock: "resume.lock", owner: observed.owner },
|
|
352
|
+
actor: options.actor!,
|
|
353
|
+
reason: options.reason!,
|
|
354
|
+
});
|
|
355
|
+
const current = inspect();
|
|
356
|
+
if (current.lock_sha256 !== observed.lock_sha256 || current.owner_state !== "dead")
|
|
357
|
+
throw new Error("workflow resume lock authority changed before unlock");
|
|
358
|
+
audit.assertCurrent();
|
|
359
|
+
unlinkSync(path);
|
|
360
|
+
fsyncParentDirectory(path);
|
|
361
|
+
audit.complete();
|
|
362
|
+
return { ...observed, action: "unlocked", recovery_id: audit.intent.operation_id };
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
function withResumeLeaseMutation<T>(coordRoot: string, runId: string, action: () => T): T {
|
|
367
|
+
const authority = stableDigest({ run_id: runId, lock: "resume.lock" });
|
|
368
|
+
const lease = acquireNoClobberLease({
|
|
369
|
+
path: join(coordRoot, ".harnery", "workflows", runId, "resume-mutation.lease"),
|
|
370
|
+
scope: "workflow-resume-mutation",
|
|
371
|
+
authoritySha256: authority,
|
|
372
|
+
staleAfterMs: 300_000,
|
|
373
|
+
validateStaleOwner: (owner) => owner.authority_sha256 === authority,
|
|
374
|
+
});
|
|
375
|
+
try {
|
|
376
|
+
return action();
|
|
377
|
+
} finally {
|
|
378
|
+
lease.release();
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
245
382
|
function readResumeLease(path: string): { pid: number; host: string; created_at: string } | null {
|
|
246
383
|
try {
|
|
247
384
|
const value = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>;
|
|
@@ -249,7 +386,8 @@ function readResumeLease(path: string): { pid: number; host: string; created_at:
|
|
|
249
386
|
Number.isSafeInteger(value.pid) &&
|
|
250
387
|
value.pid > 0 &&
|
|
251
388
|
typeof value.host === "string" &&
|
|
252
|
-
|
|
389
|
+
value.host.length > 0 &&
|
|
390
|
+
validTimestamp(value.created_at)
|
|
253
391
|
? { pid: value.pid, host: value.host, created_at: value.created_at }
|
|
254
392
|
: null;
|
|
255
393
|
} catch {
|
|
@@ -257,19 +395,6 @@ function readResumeLease(path: string): { pid: number; host: string; created_at:
|
|
|
257
395
|
}
|
|
258
396
|
}
|
|
259
397
|
|
|
260
|
-
function resumeLeaseIsLive(lease: { pid: number; host: string; created_at: string }): boolean {
|
|
261
|
-
if (lease.host !== hostname()) {
|
|
262
|
-
const age = Date.now() - Date.parse(lease.created_at);
|
|
263
|
-
return Number.isFinite(age) && age < FOREIGN_LEASE_STALE_MS;
|
|
264
|
-
}
|
|
265
|
-
try {
|
|
266
|
-
process.kill(lease.pid, 0);
|
|
267
|
-
return true;
|
|
268
|
-
} catch (error) {
|
|
269
|
-
return (error as NodeJS.ErrnoException).code === "EPERM";
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
|
|
273
398
|
function validExecution(value: WorkflowRunManifest["execution"]): boolean {
|
|
274
399
|
if (!value || typeof value !== "object") return false;
|
|
275
400
|
return (
|
|
@@ -352,6 +477,12 @@ function validRepoSnapshot(value: RepoSnapshot): boolean {
|
|
|
352
477
|
return (
|
|
353
478
|
Boolean(value) &&
|
|
354
479
|
typeof value === "object" &&
|
|
480
|
+
(value.observation === undefined ||
|
|
481
|
+
value.observation === "ok" ||
|
|
482
|
+
value.observation === "partial" ||
|
|
483
|
+
value.observation === "unavailable") &&
|
|
484
|
+
(value.observation_detail === undefined ||
|
|
485
|
+
(typeof value.observation_detail === "string" && value.observation_detail.length <= 500)) &&
|
|
355
486
|
isAbsolute(value.cwd) &&
|
|
356
487
|
(value.root === undefined || isAbsolute(value.root)) &&
|
|
357
488
|
(value.branch === undefined || typeof value.branch === "string") &&
|
|
@@ -23,6 +23,7 @@ import { builtinAdapterProfile, validateAdapterEffort } from "../adapters/profil
|
|
|
23
23
|
import type { AdapterInvocation, AdapterRawResult } from "../adapters/types.ts";
|
|
24
24
|
import { notFoundError } from "./adapters.ts";
|
|
25
25
|
import { buildChildEnv } from "./child-env.ts";
|
|
26
|
+
import { observeSpawnFailure } from "./failure-evidence.ts";
|
|
26
27
|
import { resolveFullAccessArgv, resolveSandboxProjection } from "./sandbox-projection.ts";
|
|
27
28
|
import { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.ts";
|
|
28
29
|
import type { Spawner, SpawnRequest, SpawnResult } from "./types.ts";
|
|
@@ -66,6 +67,23 @@ export function buildClaudeInvocation(req: SpawnRequest): AdapterInvocation {
|
|
|
66
67
|
}
|
|
67
68
|
|
|
68
69
|
export function normalizeClaudeResult(raw: AdapterRawResult): SpawnResult {
|
|
70
|
+
const result = normalizeClaudeTransportResult(raw);
|
|
71
|
+
if (result.ok) return result;
|
|
72
|
+
let text = raw.stdout.trim();
|
|
73
|
+
let source: "stdout" | "vendor_envelope" = "stdout";
|
|
74
|
+
try {
|
|
75
|
+
const envelope = JSON.parse(raw.stdout) as ClaudeEnvelope;
|
|
76
|
+
if (typeof envelope.result === "string" && envelope.result.length > 0) {
|
|
77
|
+
text = envelope.result;
|
|
78
|
+
source = "vendor_envelope";
|
|
79
|
+
}
|
|
80
|
+
} catch {
|
|
81
|
+
// Retain unstructured output without treating it as native authority.
|
|
82
|
+
}
|
|
83
|
+
return { ...result, text, failureEvidence: observeSpawnFailure(raw, text, source) };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function normalizeClaudeTransportResult(raw: AdapterRawResult): SpawnResult {
|
|
69
87
|
if (raw.timedOut) {
|
|
70
88
|
return {
|
|
71
89
|
ok: false,
|
|
@@ -26,6 +26,7 @@ import { builtinAdapterProfile, validateAdapterEffort } from "../adapters/profil
|
|
|
26
26
|
import type { AdapterInvocation, AdapterRawResult } from "../adapters/types.ts";
|
|
27
27
|
import { notFoundError } from "./adapters.ts";
|
|
28
28
|
import { buildChildEnv } from "./child-env.ts";
|
|
29
|
+
import { observeSpawnFailure } from "./failure-evidence.ts";
|
|
29
30
|
import { resolveFullAccessArgv, resolveSandboxProjection } from "./sandbox-projection.ts";
|
|
30
31
|
import { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.ts";
|
|
31
32
|
import type { Spawner, SpawnRequest, SpawnResult } from "./types.ts";
|
|
@@ -78,6 +79,18 @@ export function buildCodexInvocation(req: SpawnRequest, resultFile?: string): Ad
|
|
|
78
79
|
}
|
|
79
80
|
|
|
80
81
|
export function normalizeCodexResult(raw: AdapterRawResult): SpawnResult {
|
|
82
|
+
const result = normalizeCodexTransportResult(raw);
|
|
83
|
+
if (result.ok) return result;
|
|
84
|
+
const finalText = raw.resultFileText?.trim();
|
|
85
|
+
const text = finalText || raw.stdout.trim();
|
|
86
|
+
return {
|
|
87
|
+
...result,
|
|
88
|
+
text,
|
|
89
|
+
failureEvidence: observeSpawnFailure(raw, text, finalText ? "final_message" : "stdout"),
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function normalizeCodexTransportResult(raw: AdapterRawResult): SpawnResult {
|
|
81
94
|
if (raw.timedOut) {
|
|
82
95
|
return {
|
|
83
96
|
ok: false,
|
|
@@ -24,6 +24,7 @@ import { builtinAdapterProfile, validateAdapterEffort } from "../adapters/profil
|
|
|
24
24
|
import type { AdapterInvocation, AdapterRawResult } from "../adapters/types.ts";
|
|
25
25
|
import { notFoundError } from "./adapters.ts";
|
|
26
26
|
import { buildChildEnv } from "./child-env.ts";
|
|
27
|
+
import { observeSpawnFailure } from "./failure-evidence.ts";
|
|
27
28
|
import { resolveFullAccessArgv, resolveSandboxProjection } from "./sandbox-projection.ts";
|
|
28
29
|
import { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.ts";
|
|
29
30
|
import type { Spawner, SpawnRequest, SpawnResult } from "./types.ts";
|
|
@@ -78,6 +79,23 @@ export function buildCursorInvocation(req: SpawnRequest): AdapterInvocation {
|
|
|
78
79
|
}
|
|
79
80
|
|
|
80
81
|
export function normalizeCursorResult(raw: AdapterRawResult): SpawnResult {
|
|
82
|
+
const result = normalizeCursorTransportResult(raw);
|
|
83
|
+
if (result.ok) return result;
|
|
84
|
+
let text = raw.stdout;
|
|
85
|
+
let source: "stdout" | "vendor_envelope" = "stdout";
|
|
86
|
+
try {
|
|
87
|
+
const envelope = JSON.parse(raw.stdout) as CursorEnvelope;
|
|
88
|
+
if (typeof envelope.result === "string" && envelope.result.length > 0) {
|
|
89
|
+
text = envelope.result;
|
|
90
|
+
source = "vendor_envelope";
|
|
91
|
+
}
|
|
92
|
+
} catch {
|
|
93
|
+
// Raw output remains evidence, not an inferred refusal or effect result.
|
|
94
|
+
}
|
|
95
|
+
return { ...result, text, failureEvidence: observeSpawnFailure(raw, text, source) };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function normalizeCursorTransportResult(raw: AdapterRawResult): SpawnResult {
|
|
81
99
|
if (raw.timedOut) {
|
|
82
100
|
return {
|
|
83
101
|
ok: false,
|
|
@@ -22,6 +22,7 @@ import { builtinAdapterProfile, validateAdapterEffort } from "../adapters/profil
|
|
|
22
22
|
import type { AdapterInvocation, AdapterRawResult } from "../adapters/types.ts";
|
|
23
23
|
import { notFoundError } from "./adapters.ts";
|
|
24
24
|
import { buildChildEnv } from "./child-env.ts";
|
|
25
|
+
import { observeSpawnFailure } from "./failure-evidence.ts";
|
|
25
26
|
import { resolveFullAccessArgv, resolveSandboxProjection } from "./sandbox-projection.ts";
|
|
26
27
|
import { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.ts";
|
|
27
28
|
import type { Spawner, SpawnRequest, SpawnResult } from "./types.ts";
|
|
@@ -88,6 +89,19 @@ export function buildOpenCodeInvocation(req: SpawnRequest): AdapterInvocation {
|
|
|
88
89
|
}
|
|
89
90
|
|
|
90
91
|
export function normalizeOpenCodeResult(raw: AdapterRawResult): SpawnResult {
|
|
92
|
+
const result = normalizeOpenCodeTransportResult(raw);
|
|
93
|
+
if (result.ok) return result;
|
|
94
|
+
const parsed = parseOpenCodeStream(raw.stdout);
|
|
95
|
+
const text = parsed.text || raw.stdout.trim();
|
|
96
|
+
return {
|
|
97
|
+
...result,
|
|
98
|
+
text,
|
|
99
|
+
sessionId: parsed.sessionId,
|
|
100
|
+
failureEvidence: observeSpawnFailure(raw, text, parsed.text ? "vendor_envelope" : "stdout"),
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function normalizeOpenCodeTransportResult(raw: AdapterRawResult): SpawnResult {
|
|
91
105
|
if (raw.timedOut) {
|
|
92
106
|
return {
|
|
93
107
|
ok: false,
|
|
@@ -137,11 +137,12 @@ export interface WorkflowAgentProof {
|
|
|
137
137
|
session_id?: string;
|
|
138
138
|
result?: ResultDigest;
|
|
139
139
|
error?: string;
|
|
140
|
-
/**
|
|
141
|
-
*
|
|
142
|
-
* Absent ⇒ a work failure. Recorded even when a script's parallel() swallows
|
|
143
|
-
* the rejection, so the run-level class can still be derived from proof. */
|
|
140
|
+
/** Failed-spawn transport class. It is retained even when a script swallows
|
|
141
|
+
* the rejection; charging additionally requires validated failure evidence. */
|
|
144
142
|
class?: SpawnFailureClass;
|
|
143
|
+
/** Failure observations carried from the adapter. Missing observations are
|
|
144
|
+
* unknown and cannot authorize an uncharged attempt. */
|
|
145
|
+
failure_evidence?: SpawnFailureEvidence;
|
|
145
146
|
/** Which vendor sandbox mode this agent launched with and why (ADR 0192).
|
|
146
147
|
* Absent on a cached agent, which launched nothing. */
|
|
147
148
|
access?: WorkerAccessDecision;
|
|
@@ -149,6 +150,9 @@ export interface WorkflowAgentProof {
|
|
|
149
150
|
|
|
150
151
|
export interface WorkflowRepoSnapshot {
|
|
151
152
|
cwd: string;
|
|
153
|
+
observation: "ok" | "unavailable" | "partial";
|
|
154
|
+
/** Bounded explanation of an unavailable or incomplete observation. */
|
|
155
|
+
observation_detail?: string;
|
|
152
156
|
root?: string;
|
|
153
157
|
branch?: string;
|
|
154
158
|
head?: string;
|
|
@@ -161,12 +165,13 @@ export interface WorkflowRepoEvidence {
|
|
|
161
165
|
before: WorkflowRepoSnapshot;
|
|
162
166
|
after: WorkflowRepoSnapshot;
|
|
163
167
|
drift: {
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
168
|
+
/** Null means the snapshots do not establish this comparison. */
|
|
169
|
+
branch_changed: boolean | null;
|
|
170
|
+
head_changed: boolean | null;
|
|
171
|
+
dirty_paths_added: string[] | null;
|
|
172
|
+
dirty_paths_cleared: string[] | null;
|
|
173
|
+
dirty_paths_retained: string[] | null;
|
|
174
|
+
/** True when an observation, comparison, or retained-path change is unknown. */
|
|
170
175
|
incomplete: boolean;
|
|
171
176
|
note?: string;
|
|
172
177
|
};
|
|
@@ -358,15 +363,16 @@ export interface WorkflowSpecialistProfile {
|
|
|
358
363
|
}
|
|
359
364
|
|
|
360
365
|
/**
|
|
361
|
-
*
|
|
362
|
-
*
|
|
363
|
-
* default and is charged against the attempt budget exactly as before.
|
|
366
|
+
* The positively identified transport cause of a failed spawn (ADR 0046).
|
|
367
|
+
* This class alone proves neither absence of effects nor retry eligibility.
|
|
364
368
|
*
|
|
365
369
|
* - `environment`: the run never started — a precondition was missing (the
|
|
366
|
-
* vendor binary was absent).
|
|
370
|
+
* vendor binary was absent). With positive no-effect/output evidence it is
|
|
371
|
+
* uncharged and not retried: retrying an unchanged
|
|
367
372
|
* environment cannot help, so the work item stops and names the precondition.
|
|
368
373
|
* - `upstream`: the vendor was reached and refused (5xx, 429, circuit open).
|
|
369
|
-
*
|
|
374
|
+
* Retry/free-attempt eligibility also requires positive no-effect evidence,
|
|
375
|
+
* observed absent output, and no refusal. Unsupported evidence stays unknown.
|
|
370
376
|
*/
|
|
371
377
|
export type SpawnFailureClass = "environment" | "upstream";
|
|
372
378
|
|
|
@@ -388,6 +394,30 @@ export type SpawnFailureClass = "environment" | "upstream";
|
|
|
388
394
|
*/
|
|
389
395
|
export type RunFailureClass = SpawnFailureClass | "decision";
|
|
390
396
|
|
|
397
|
+
/** Positive claims name the observation that supports them. A transport error
|
|
398
|
+
* or an empty reply is never evidence that no effects occurred. */
|
|
399
|
+
export type SpawnEffectEvidence =
|
|
400
|
+
| { state: "unknown" }
|
|
401
|
+
| { state: "none"; source: "spawn_errno"; reference: "ENOENT" }
|
|
402
|
+
| { state: "none" | "observed"; source: "adapter_evidence"; reference: string };
|
|
403
|
+
|
|
404
|
+
export type SpawnRefusalEvidence =
|
|
405
|
+
| { state: "unknown" }
|
|
406
|
+
| { state: "refused"; source: "vendor_envelope"; reference: string };
|
|
407
|
+
|
|
408
|
+
export type SpawnOutputSource = "final_message" | "vendor_envelope" | "stdout";
|
|
409
|
+
|
|
410
|
+
export type SpawnPartialOutputEvidence =
|
|
411
|
+
| { state: "unknown" }
|
|
412
|
+
| { state: "absent"; source: SpawnOutputSource }
|
|
413
|
+
| { state: "present"; source: SpawnOutputSource; bytes: number; sha256: string };
|
|
414
|
+
|
|
415
|
+
export interface SpawnFailureEvidence {
|
|
416
|
+
effect: SpawnEffectEvidence;
|
|
417
|
+
refusal: SpawnRefusalEvidence;
|
|
418
|
+
partial_output: SpawnPartialOutputEvidence;
|
|
419
|
+
}
|
|
420
|
+
|
|
391
421
|
/** What a spawn adapter returns for one subagent run. */
|
|
392
422
|
export interface SpawnResult {
|
|
393
423
|
ok: boolean;
|
|
@@ -399,9 +429,13 @@ export interface SpawnResult {
|
|
|
399
429
|
durationMs: number;
|
|
400
430
|
/** Populated when ok=false. */
|
|
401
431
|
error?: string;
|
|
402
|
-
/**
|
|
403
|
-
*
|
|
432
|
+
/** Transport classification when ok=false; never sufficient by itself to
|
|
433
|
+
* grant free work or automatic redispatch. */
|
|
404
434
|
class?: SpawnFailureClass;
|
|
435
|
+
/** Transport classification alone grants neither a retry nor free work.
|
|
436
|
+
* Absent evidence is explicitly unknown. Positive adapter_evidence claims
|
|
437
|
+
* require a documented producer observation, never an HTTP/text heuristic. */
|
|
438
|
+
failureEvidence?: SpawnFailureEvidence;
|
|
405
439
|
}
|
|
406
440
|
|
|
407
441
|
export interface SpawnRequest {
|
|
@@ -36,7 +36,6 @@ export {
|
|
|
36
36
|
probe as probeLocalGitWorktreeProvider,
|
|
37
37
|
} from "./local-git.ts";
|
|
38
38
|
export type { PrepareReclaimInput, ReclaimMode, ReclaimPreparation } from "./reclaim.ts";
|
|
39
|
-
export { prepareReclaim } from "./reclaim.ts";
|
|
40
39
|
export type {
|
|
41
40
|
AuthorizedIntegrationPlan,
|
|
42
41
|
AuthorizedProviderIntegrationInput,
|
|
@@ -39,7 +39,8 @@ export type LeaseRecoveryStep =
|
|
|
39
39
|
| "claim_created"
|
|
40
40
|
| "stale_pinned"
|
|
41
41
|
| "current_removed"
|
|
42
|
-
| "new_current_linked"
|
|
42
|
+
| "new_current_linked"
|
|
43
|
+
| "recovery_retired";
|
|
43
44
|
|
|
44
45
|
export interface AcquireNoClobberLeaseInput {
|
|
45
46
|
path: string;
|
|
@@ -51,6 +52,8 @@ export interface AcquireNoClobberLeaseInput {
|
|
|
51
52
|
now?: () => number;
|
|
52
53
|
pid?: number;
|
|
53
54
|
host?: string;
|
|
55
|
+
/** Probe dependency for deterministic liveness-failure tests. */
|
|
56
|
+
probePid?: (pid: number) => void;
|
|
54
57
|
/** Fault-injection hook immediately before the lease directory is created. */
|
|
55
58
|
onLeaseDirectoryCreate?: () => void;
|
|
56
59
|
/** Fault-injection hook after contention is observed and before owner inspection. */
|
|
@@ -137,8 +140,10 @@ export function acquireNoClobberLease(input: AcquireNoClobberLeaseInput): NoClob
|
|
|
137
140
|
if (observed === undefined) {
|
|
138
141
|
throw new Error(`lease ${input.scope} changed repeatedly during contention`);
|
|
139
142
|
}
|
|
140
|
-
if (leaseIsLive(observed.owner, now(), input.staleAfterMs)) {
|
|
141
|
-
throw new Error(
|
|
143
|
+
if (leaseIsLive(observed.owner, now(), input.staleAfterMs, input.probePid)) {
|
|
144
|
+
throw new Error(
|
|
145
|
+
`lease ${input.scope} is held by a live or unexpired owner, or its liveness is unknown`,
|
|
146
|
+
);
|
|
142
147
|
}
|
|
143
148
|
if (
|
|
144
149
|
observed.owner.scope !== input.scope ||
|
|
@@ -156,7 +161,7 @@ export function acquireNoClobberLease(input: AcquireNoClobberLeaseInput): NoClob
|
|
|
156
161
|
linkSync(currentPath, pinPath);
|
|
157
162
|
assertSameInodeAndBytes(currentPath, pinPath, observed.bytes);
|
|
158
163
|
} catch {
|
|
159
|
-
abortUnchangedRecoveryClaim(leaseDir, recoveryPath,
|
|
164
|
+
abortUnchangedRecoveryClaim(leaseDir, recoveryPath, owner);
|
|
160
165
|
recoveryClaimed = false;
|
|
161
166
|
throw new Error(`lease ${input.scope} recovery raced with another contender`);
|
|
162
167
|
}
|
|
@@ -174,10 +179,9 @@ export function acquireNoClobberLease(input: AcquireNoClobberLeaseInput): NoClob
|
|
|
174
179
|
input.onRecoveryStep?.("new_current_linked");
|
|
175
180
|
|
|
176
181
|
removePinnedOwner(leaseDir, pinPath, observed.owner);
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
fsyncDirectory(leaseDir);
|
|
182
|
+
retireRecoveryClaim(leaseDir, recoveryPath, owner, () =>
|
|
183
|
+
input.onRecoveryStep?.("recovery_retired"),
|
|
184
|
+
);
|
|
181
185
|
return {
|
|
182
186
|
...leaseHandle(leaseDir, ownerPath, ownerBytes, owner),
|
|
183
187
|
recovered_owner: observed.owner,
|
|
@@ -266,7 +270,7 @@ function reclaimAbandonedRecovery(
|
|
|
266
270
|
now: number,
|
|
267
271
|
): void {
|
|
268
272
|
const claim = readRecoveryClaim(recoveryPath);
|
|
269
|
-
if (leaseIsLive(claim.claimant, now, input.staleAfterMs)) {
|
|
273
|
+
if (leaseIsLive(claim.claimant, now, input.staleAfterMs, input.probePid)) {
|
|
270
274
|
throw new Error(`lease ${input.scope} recovery is already in progress`);
|
|
271
275
|
}
|
|
272
276
|
if (
|
|
@@ -326,16 +330,34 @@ function reclaimAbandonedRecovery(
|
|
|
326
330
|
function abortUnchangedRecoveryClaim(
|
|
327
331
|
leaseDir: string,
|
|
328
332
|
recoveryPath: string,
|
|
329
|
-
pinPath: string,
|
|
330
333
|
claimant: NoClobberLeaseOwner,
|
|
331
334
|
): void {
|
|
332
335
|
const claim = readRecoveryClaim(recoveryPath);
|
|
333
336
|
if (claim.claimant.owner_id !== claimant.owner_id) {
|
|
334
337
|
throw new Error(`lease ${claimant.scope} recovery ownership changed`);
|
|
335
338
|
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
+
retireRecoveryClaim(leaseDir, recoveryPath, claimant);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Move this claimant's recovery directory to a private name before emptying
|
|
344
|
+
* it. Renaming a directory onto an existing empty directory replaces it, so
|
|
345
|
+
* emptying `recovery` in place would let a contender's claim land there and
|
|
346
|
+
* fail this claimant's final rmdir after it already holds the lease.
|
|
347
|
+
*/
|
|
348
|
+
function retireRecoveryClaim(
|
|
349
|
+
leaseDir: string,
|
|
350
|
+
recoveryPath: string,
|
|
351
|
+
claimant: NoClobberLeaseOwner,
|
|
352
|
+
onRetired?: () => void,
|
|
353
|
+
): void {
|
|
354
|
+
const retired = join(leaseDir, `recovery-retired-${claimant.owner_id}`);
|
|
355
|
+
renameSync(recoveryPath, retired);
|
|
356
|
+
fsyncDirectory(leaseDir);
|
|
357
|
+
onRetired?.();
|
|
358
|
+
safeUnlink(join(retired, "stale-pin"));
|
|
359
|
+
unlinkSync(join(retired, "claim.json"));
|
|
360
|
+
rmdirSync(retired);
|
|
339
361
|
fsyncDirectory(leaseDir);
|
|
340
362
|
}
|
|
341
363
|
|
|
@@ -466,7 +488,14 @@ function isStringRecord(value: unknown): value is Record<string, string> {
|
|
|
466
488
|
);
|
|
467
489
|
}
|
|
468
490
|
|
|
469
|
-
function leaseIsLive(
|
|
491
|
+
function leaseIsLive(
|
|
492
|
+
owner: NoClobberLeaseOwner,
|
|
493
|
+
now: number,
|
|
494
|
+
staleAfterMs: number,
|
|
495
|
+
probePid: (pid: number) => void = (pid) => {
|
|
496
|
+
process.kill(pid, 0);
|
|
497
|
+
},
|
|
498
|
+
): boolean {
|
|
470
499
|
const created = Date.parse(owner.created_at);
|
|
471
500
|
const expires = Date.parse(owner.expires_at);
|
|
472
501
|
if (
|
|
@@ -478,12 +507,14 @@ function leaseIsLive(owner: NoClobberLeaseOwner, now: number, staleAfterMs: numb
|
|
|
478
507
|
) {
|
|
479
508
|
return true;
|
|
480
509
|
}
|
|
481
|
-
|
|
510
|
+
// Expiry only makes a same-host owner eligible for a process check. Neither
|
|
511
|
+
// a remote owner nor an unreadable probe proves that its work has stopped.
|
|
512
|
+
if (owner.host !== hostname()) return true;
|
|
482
513
|
try {
|
|
483
|
-
|
|
514
|
+
probePid(owner.pid);
|
|
484
515
|
return true;
|
|
485
516
|
} catch (error) {
|
|
486
|
-
return (error as NodeJS.ErrnoException).code
|
|
517
|
+
return (error as NodeJS.ErrnoException).code !== "ESRCH";
|
|
487
518
|
}
|
|
488
519
|
}
|
|
489
520
|
|