harnery 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/commander.js +1 -0
- package/dist/commands/agents.d.ts.map +1 -1
- package/dist/commands/agents.js +41 -48
- package/dist/commands/browse.d.ts.map +1 -1
- package/dist/commands/browse.js +38 -11
- package/dist/commands/diagnostics.d.ts.map +1 -1
- package/dist/commands/diagnostics.js +10 -0
- package/dist/commands/doctor.d.ts +12 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +22 -7
- package/dist/commands/files.d.ts +2 -2
- package/dist/commands/files.d.ts.map +1 -1
- package/dist/commands/files.js +7 -4
- package/dist/commands/governor.js +22 -10
- package/dist/commands/inbox.d.ts +1 -1
- package/dist/commands/inbox.d.ts.map +1 -1
- package/dist/commands/inbox.js +55 -2
- package/dist/commands/rm.d.ts.map +1 -1
- package/dist/commands/rm.js +28 -4
- package/dist/commands/servers.d.ts.map +1 -1
- package/dist/commands/servers.js +26 -17
- package/dist/commands/supervisor.d.ts.map +1 -1
- package/dist/commands/supervisor.js +7 -2
- package/dist/commands/toc.d.ts +14 -0
- package/dist/commands/toc.d.ts.map +1 -1
- package/dist/commands/toc.js +7 -4
- package/dist/commands/tunnel.d.ts +15 -0
- package/dist/commands/tunnel.d.ts.map +1 -1
- package/dist/commands/tunnel.js +453 -153
- package/dist/commands/turn-rollback.d.ts +5 -0
- package/dist/commands/turn-rollback.d.ts.map +1 -0
- package/dist/commands/turn-rollback.js +76 -0
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +68 -30
- package/dist/core/agents/coord-client.d.ts.map +1 -1
- package/dist/core/agents/coord-client.js +18 -0
- package/dist/core/agents/live-authority-v3.d.ts +1 -0
- package/dist/core/agents/live-authority-v3.d.ts.map +1 -1
- package/dist/core/agents/live-authority-v3.js +11 -0
- package/dist/core/agents/live-lifecycle-v3.d.ts +4 -2
- package/dist/core/agents/live-lifecycle-v3.d.ts.map +1 -1
- package/dist/core/agents/live-lifecycle-v3.js +28 -7
- package/dist/core/agents/mailbox.d.ts +9 -8
- package/dist/core/agents/mailbox.d.ts.map +1 -1
- package/dist/core/agents/mailbox.js +24 -17
- package/dist/core/agents/render/prompt-context.d.ts +2 -0
- package/dist/core/agents/render/prompt-context.d.ts.map +1 -1
- package/dist/core/agents/render/prompt-context.js +8 -12
- package/dist/core/agents/render/session-context.d.ts +2 -8
- package/dist/core/agents/render/session-context.d.ts.map +1 -1
- package/dist/core/agents/render/session-context.js +7 -30
- package/dist/core/agents/rules/stop-hook.js +3 -3
- package/dist/core/agents/state/heartbeat-writer.d.ts.map +1 -1
- package/dist/core/agents/state/heartbeat-writer.js +25 -14
- package/dist/core/agents/state/live-coordination-writer.d.ts.map +1 -1
- package/dist/core/agents/state/live-coordination-writer.js +5 -0
- package/dist/core/agents/state/pidmap.d.ts.map +1 -1
- package/dist/core/agents/state/pidmap.js +5 -0
- package/dist/core/agents/state/proc-start.d.ts +19 -1
- package/dist/core/agents/state/proc-start.d.ts.map +1 -1
- package/dist/core/agents/state/proc-start.js +46 -1
- package/dist/core/agents/state/process-handle.d.ts +26 -0
- package/dist/core/agents/state/process-handle.d.ts.map +1 -0
- package/dist/core/agents/state/process-handle.js +121 -0
- package/dist/core/agents/state/stale-sweep.d.ts.map +1 -1
- package/dist/core/agents/state/stale-sweep.js +180 -70
- package/dist/core/agents/state/unchanged-file.d.ts +44 -0
- package/dist/core/agents/state/unchanged-file.d.ts.map +1 -0
- package/dist/core/agents/state/unchanged-file.js +390 -0
- package/dist/core/artifacts/delivery-card.d.ts.map +1 -1
- package/dist/core/artifacts/delivery-card.js +2 -1
- package/dist/core/context/index.d.ts +22 -10
- package/dist/core/context/index.d.ts.map +1 -1
- package/dist/core/context/index.js +381 -119
- package/dist/core/context/observation.d.ts +26 -0
- package/dist/core/context/observation.d.ts.map +1 -0
- package/dist/core/context/observation.js +11 -0
- package/dist/core/diagnostics/bundle.d.ts.map +1 -1
- package/dist/core/diagnostics/bundle.js +25 -14
- package/dist/core/diagnostics/path-identity.d.ts +3 -0
- package/dist/core/diagnostics/path-identity.d.ts.map +1 -0
- package/dist/core/diagnostics/path-identity.js +12 -0
- package/dist/core/diagnostics/sanitize.d.ts.map +1 -1
- package/dist/core/diagnostics/sanitize.js +4 -3
- package/dist/core/events/v3/authority-outbox.d.ts.map +1 -1
- package/dist/core/events/v3/authority-outbox.js +29 -0
- package/dist/core/events/v3/generation-custody.d.ts +5 -0
- package/dist/core/events/v3/generation-custody.d.ts.map +1 -0
- package/dist/core/events/v3/generation-custody.js +50 -0
- package/dist/core/events/v3/producers/coordination-recorder.d.ts +1 -0
- package/dist/core/events/v3/producers/coordination-recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/coordination-recorder.js +7 -0
- package/dist/core/events/v3/producers/recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/recorder.js +77 -64
- package/dist/core/governor/planning.d.ts.map +1 -1
- package/dist/core/governor/planning.js +14 -1
- package/dist/core/governor/service-read.d.ts +10 -2
- package/dist/core/governor/service-read.d.ts.map +1 -1
- package/dist/core/governor/service-read.js +105 -22
- package/dist/core/governor/service.d.ts +20 -2
- package/dist/core/governor/service.d.ts.map +1 -1
- package/dist/core/governor/service.js +72 -47
- package/dist/core/hooks/cli.js +33 -5
- package/dist/core/inbox/contract.d.ts +2 -1
- package/dist/core/inbox/contract.d.ts.map +1 -1
- package/dist/core/inbox/index.d.ts +2 -0
- package/dist/core/inbox/index.d.ts.map +1 -1
- package/dist/core/inbox/index.js +2 -0
- package/dist/core/inbox/peer-messages.d.ts +37 -0
- package/dist/core/inbox/peer-messages.d.ts.map +1 -0
- package/dist/core/inbox/peer-messages.js +72 -0
- package/dist/core/inbox/service.d.ts +8 -1
- package/dist/core/inbox/service.d.ts.map +1 -1
- package/dist/core/inbox/service.js +67 -21
- package/dist/core/inbox/targets.d.ts +22 -0
- package/dist/core/inbox/targets.d.ts.map +1 -0
- package/dist/core/inbox/targets.js +108 -0
- package/dist/core/inbox/watcher.js +7 -3
- package/dist/core/journal/index.d.ts.map +1 -1
- package/dist/core/journal/index.js +17 -2
- package/dist/core/resources/contract.d.ts +3 -0
- package/dist/core/resources/contract.d.ts.map +1 -1
- package/dist/core/resources/service-status.d.ts +14 -1
- package/dist/core/resources/service-status.d.ts.map +1 -1
- package/dist/core/resources/service-status.js +23 -13
- package/dist/core/resources/service.d.ts +7 -13
- package/dist/core/resources/service.d.ts.map +1 -1
- package/dist/core/resources/service.js +34 -91
- package/dist/core/resources/status.d.ts +2 -5
- package/dist/core/resources/status.d.ts.map +1 -1
- package/dist/core/resources/status.js +36 -7
- package/dist/core/semantic/service.d.ts.map +1 -1
- package/dist/core/semantic/service.js +2 -9
- package/dist/core/servers/index.d.ts +34 -22
- package/dist/core/servers/index.d.ts.map +1 -1
- package/dist/core/servers/index.js +297 -144
- package/dist/core/servers/tunnels.d.ts +7 -7
- package/dist/core/servers/tunnels.d.ts.map +1 -1
- package/dist/core/servers/tunnels.js +47 -10
- package/dist/core/supervisor/contract.d.ts +12 -0
- package/dist/core/supervisor/contract.d.ts.map +1 -1
- package/dist/core/supervisor/explanations.d.ts.map +1 -1
- package/dist/core/supervisor/explanations.js +2 -1
- package/dist/core/supervisor/findings.d.ts.map +1 -1
- package/dist/core/supervisor/findings.js +54 -9
- package/dist/core/supervisor/service.d.ts +7 -1
- package/dist/core/supervisor/service.d.ts.map +1 -1
- package/dist/core/supervisor/service.js +59 -29
- package/dist/core/supervisor/services.d.ts.map +1 -1
- package/dist/core/supervisor/services.js +22 -1
- package/dist/core/supervisor/status.d.ts +17 -2
- package/dist/core/supervisor/status.d.ts.map +1 -1
- package/dist/core/supervisor/status.js +70 -12
- package/dist/core/supervisor/timeline.d.ts.map +1 -1
- package/dist/core/supervisor/timeline.js +2 -1
- package/dist/core/turn-rollback/content.d.ts +53 -0
- package/dist/core/turn-rollback/content.d.ts.map +1 -0
- package/dist/core/turn-rollback/content.js +396 -0
- package/dist/core/turn-rollback/index.d.ts +72 -0
- package/dist/core/turn-rollback/index.d.ts.map +1 -0
- package/dist/core/turn-rollback/index.js +443 -0
- package/dist/core/work/state.js +7 -7
- package/dist/core/workflow/engine.d.ts.map +1 -1
- package/dist/core/workflow/engine.js +105 -32
- package/dist/core/workflow/failure-evidence.d.ts +16 -0
- package/dist/core/workflow/failure-evidence.d.ts.map +1 -0
- package/dist/core/workflow/failure-evidence.js +114 -0
- package/dist/core/workflow/index.d.ts +5 -3
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +3 -2
- package/dist/core/workflow/live-session-v3.d.ts +6 -0
- package/dist/core/workflow/live-session-v3.d.ts.map +1 -1
- package/dist/core/workflow/live-session-v3.js +126 -5
- package/dist/core/workflow/proof.d.ts +5 -5
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +115 -20
- package/dist/core/workflow/recovery-audit.d.ts +30 -0
- package/dist/core/workflow/recovery-audit.d.ts.map +1 -0
- package/dist/core/workflow/recovery-audit.js +116 -0
- package/dist/core/workflow/run-state.d.ts +20 -1
- package/dist/core/workflow/run-state.d.ts.map +1 -1
- package/dist/core/workflow/run-state.js +141 -32
- package/dist/core/workflow/spawn-claude.d.ts.map +1 -1
- package/dist/core/workflow/spawn-claude.js +19 -0
- package/dist/core/workflow/spawn-codex.d.ts.map +1 -1
- package/dist/core/workflow/spawn-codex.js +13 -0
- package/dist/core/workflow/spawn-cursor.d.ts.map +1 -1
- package/dist/core/workflow/spawn-cursor.js +19 -0
- package/dist/core/workflow/spawn-opencode.d.ts.map +1 -1
- package/dist/core/workflow/spawn-opencode.js +14 -0
- package/dist/core/workflow/types.d.ts +64 -17
- package/dist/core/workflow/types.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/index.d.ts +0 -1
- package/dist/core/workflow/workspaces/index.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/index.js +0 -1
- package/dist/core/workflow/workspaces/leases.d.ts +3 -1
- package/dist/core/workflow/workspaces/leases.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/leases.js +30 -16
- package/dist/core/workflow/workspaces/local-git.d.ts +6 -2
- package/dist/core/workflow/workspaces/local-git.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/local-git.js +328 -8
- package/dist/core/workflow/workspaces/ownership.d.ts +11 -0
- package/dist/core/workflow/workspaces/ownership.d.ts.map +1 -0
- package/dist/core/workflow/workspaces/ownership.js +194 -0
- package/dist/core/workflow/workspaces/reclaim.d.ts +40 -0
- package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/reclaim.js +14 -0
- package/dist/core/workflow/workspaces/recovery.d.ts +2 -0
- package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.js +42 -10
- package/dist/core/workflow/workspaces/state.d.ts +1 -0
- package/dist/core/workflow/workspaces/state.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/state.js +21 -1
- package/dist/lib/browser/windows-chrome.d.ts +100 -12
- package/dist/lib/browser/windows-chrome.d.ts.map +1 -1
- package/dist/lib/browser/windows-chrome.js +262 -73
- package/dist/lib/docs-links.d.ts.map +1 -1
- package/dist/lib/docs-links.js +45 -2
- package/dist/lib/guarded-remove.d.ts +7 -0
- package/dist/lib/guarded-remove.d.ts.map +1 -1
- package/dist/lib/guarded-remove.js +106 -12
- package/dist/lib/identities/assume.d.ts.map +1 -1
- package/dist/lib/identities/assume.js +4 -0
- package/dist/lib/tunnel/state.d.ts +13 -6
- package/dist/lib/tunnel/state.d.ts.map +1 -1
- package/dist/lib/tunnel/state.js +58 -12
- package/dist/lib/zip/index.d.ts +41 -0
- package/dist/lib/zip/index.d.ts.map +1 -0
- package/dist/lib/zip/index.js +398 -0
- package/package.json +12 -1
- package/src/commander.ts +10 -0
- package/src/commands/agents.ts +54 -63
- package/src/commands/browse.ts +50 -14
- package/src/commands/diagnostics.ts +10 -0
- package/src/commands/doctor.ts +26 -7
- package/src/commands/files.ts +13 -5
- package/src/commands/governor.ts +23 -10
- package/src/commands/inbox.ts +52 -2
- package/src/commands/rm.ts +43 -4
- package/src/commands/servers.ts +27 -26
- package/src/commands/supervisor.ts +7 -2
- package/src/commands/toc.ts +14 -6
- package/src/commands/tunnel.ts +533 -166
- package/src/commands/turn-rollback.ts +111 -0
- package/src/commands/workflow.ts +89 -30
- package/src/core/agents/coord-client.ts +20 -0
- package/src/core/agents/live-authority-v3.ts +17 -0
- package/src/core/agents/live-lifecycle-v3.ts +52 -14
- package/src/core/agents/mailbox.ts +26 -17
- package/src/core/agents/render/prompt-context.ts +10 -12
- package/src/core/agents/render/session-context.ts +9 -32
- package/src/core/agents/rules/stop-hook.ts +3 -3
- package/src/core/agents/state/heartbeat-writer.ts +29 -12
- package/src/core/agents/state/live-coordination-writer.ts +29 -0
- package/src/core/agents/state/pidmap.ts +13 -0
- package/src/core/agents/state/proc-start.ts +62 -2
- package/src/core/agents/state/process-handle.ts +165 -0
- package/src/core/agents/state/stale-sweep.ts +247 -82
- package/src/core/agents/state/unchanged-file.ts +456 -0
- package/src/core/artifacts/delivery-card.ts +7 -1
- package/src/core/context/index.ts +446 -136
- package/src/core/context/observation.ts +29 -0
- package/src/core/diagnostics/bundle.ts +23 -18
- package/src/core/diagnostics/path-identity.ts +15 -0
- package/src/core/diagnostics/sanitize.ts +4 -3
- package/src/core/events/v3/authority-outbox.ts +56 -0
- package/src/core/events/v3/generation-custody.ts +59 -0
- package/src/core/events/v3/producers/coordination-recorder.ts +15 -0
- package/src/core/events/v3/producers/recorder.ts +94 -68
- package/src/core/governor/planning.ts +20 -8
- package/src/core/governor/service-read.ts +128 -20
- package/src/core/governor/service.ts +119 -42
- package/src/core/hooks/cli.ts +33 -5
- package/src/core/inbox/contract.ts +2 -1
- package/src/core/inbox/index.ts +2 -0
- package/src/core/inbox/peer-messages.ts +94 -0
- package/src/core/inbox/service.ts +88 -22
- package/src/core/inbox/targets.ts +148 -0
- package/src/core/inbox/watcher.ts +8 -9
- package/src/core/journal/index.ts +19 -2
- package/src/core/resources/contract.ts +4 -0
- package/src/core/resources/service-status.ts +35 -12
- package/src/core/resources/service.ts +57 -95
- package/src/core/resources/status.ts +57 -8
- package/src/core/semantic/service.ts +2 -8
- package/src/core/servers/index.ts +376 -141
- package/src/core/servers/tunnels.ts +57 -13
- package/src/core/supervisor/contract.ts +13 -0
- package/src/core/supervisor/explanations.ts +4 -1
- package/src/core/supervisor/findings.ts +68 -10
- package/src/core/supervisor/service.ts +87 -26
- package/src/core/supervisor/services.ts +26 -8
- package/src/core/supervisor/status.ts +89 -12
- package/src/core/supervisor/timeline.ts +2 -1
- package/src/core/turn-rollback/content.ts +499 -0
- package/src/core/turn-rollback/index.ts +580 -0
- package/src/core/work/state.ts +7 -7
- package/src/core/workflow/engine.ts +131 -49
- package/src/core/workflow/failure-evidence.ts +141 -0
- package/src/core/workflow/index.ts +12 -2
- package/src/core/workflow/live-session-v3.ts +146 -5
- package/src/core/workflow/proof.ts +134 -26
- package/src/core/workflow/recovery-audit.ts +181 -0
- package/src/core/workflow/run-state.ts +165 -34
- package/src/core/workflow/spawn-claude.ts +18 -0
- package/src/core/workflow/spawn-codex.ts +13 -0
- package/src/core/workflow/spawn-cursor.ts +18 -0
- package/src/core/workflow/spawn-opencode.ts +14 -0
- package/src/core/workflow/types.ts +51 -17
- package/src/core/workflow/workspaces/index.ts +0 -1
- package/src/core/workflow/workspaces/leases.ts +48 -17
- package/src/core/workflow/workspaces/local-git.ts +388 -7
- package/src/core/workflow/workspaces/ownership.ts +232 -0
- package/src/core/workflow/workspaces/reclaim.ts +62 -0
- package/src/core/workflow/workspaces/recovery.ts +45 -10
- package/src/core/workflow/workspaces/state.ts +20 -0
- package/src/lib/browser/windows-chrome.ts +346 -86
- package/src/lib/docs-links.ts +49 -2
- package/src/lib/guarded-remove.ts +132 -14
- package/src/lib/identities/assume.ts +4 -0
- package/src/lib/tunnel/state.ts +70 -15
- package/src/lib/zip/index.ts +483 -0
|
@@ -1,26 +1,71 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Tunnels as registry services.
|
|
3
3
|
*
|
|
4
|
-
* `tunnel up` and `tunnel reload` register the
|
|
5
|
-
*
|
|
6
|
-
* time the registry is read, so a long-lived tunnel never shows as an
|
|
7
|
-
* unregistered listener.
|
|
4
|
+
* `tunnel up` and `tunnel reload` register the new gate; `tunnel down` removes
|
|
5
|
+
* its record. Missing original witnesses remain unknown and cannot authorize actions.
|
|
8
6
|
*/
|
|
9
7
|
|
|
8
|
+
import { hostname } from "node:os";
|
|
10
9
|
import { resolve } from "node:path";
|
|
11
|
-
import {
|
|
12
|
-
|
|
13
|
-
|
|
10
|
+
import {
|
|
11
|
+
gateLogFile,
|
|
12
|
+
type TunnelReadinessVerifier,
|
|
13
|
+
type TunnelState,
|
|
14
|
+
} from "../../lib/tunnel/state.ts";
|
|
15
|
+
import {
|
|
16
|
+
type RegisterServerInput,
|
|
17
|
+
readServer,
|
|
18
|
+
type ServerOptions,
|
|
19
|
+
verifyProcessGeneration,
|
|
20
|
+
withServerRegistrationCustody,
|
|
21
|
+
} from "./index.ts";
|
|
22
|
+
|
|
23
|
+
/** Verify only witnesses saved by the original tunnel registration. */
|
|
24
|
+
export function createTunnelReadinessVerifier(options: ServerOptions): TunnelReadinessVerifier {
|
|
25
|
+
return {
|
|
26
|
+
observeProcess(pid, originalToken) {
|
|
27
|
+
try {
|
|
28
|
+
process.kill(pid, 0);
|
|
29
|
+
} catch (error) {
|
|
30
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
31
|
+
if (code === "ESRCH") return "absent";
|
|
32
|
+
return "unknown";
|
|
33
|
+
}
|
|
34
|
+
return verifyProcessGeneration(pid, originalToken);
|
|
35
|
+
},
|
|
36
|
+
withRegistration(state, inspect) {
|
|
37
|
+
try {
|
|
38
|
+
const original = readServer(tunnelServerId(state.name), options);
|
|
39
|
+
if (
|
|
40
|
+
!original ||
|
|
41
|
+
original.host !== hostname() ||
|
|
42
|
+
original.kind !== "service" ||
|
|
43
|
+
original.type !== "tunnel" ||
|
|
44
|
+
original.registration_id !== state.registration_id ||
|
|
45
|
+
original.pid !== state.gate_pid ||
|
|
46
|
+
original.start_token !== state.gate_start_token ||
|
|
47
|
+
original.port !== state.gate_port ||
|
|
48
|
+
original.url !== state.url
|
|
49
|
+
)
|
|
50
|
+
return "unknown";
|
|
51
|
+
return withServerRegistrationCustody(original, options, inspect);
|
|
52
|
+
} catch {
|
|
53
|
+
return "unknown";
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
}
|
|
14
58
|
|
|
15
59
|
export function tunnelServerId(name: string): string {
|
|
16
60
|
return `tunnel-${name.toLowerCase().replace(/[^a-z0-9._-]+/g, "-")}`;
|
|
17
61
|
}
|
|
18
62
|
|
|
19
|
-
/**
|
|
63
|
+
/** Register the new gate; saved helper PIDs have no generation witnesses. */
|
|
20
64
|
export function tunnelServerInput(state: TunnelState, coordRoot: string): RegisterServerInput {
|
|
21
|
-
|
|
22
|
-
(
|
|
23
|
-
)
|
|
65
|
+
if (Object.hasOwn(state, "provider_pid") || Object.hasOwn(state, "provider_start_token"))
|
|
66
|
+
throw new Error("obsolete tunnel provider authority is unknown; state retained");
|
|
67
|
+
if (!state.gate_start_token)
|
|
68
|
+
throw new Error("tunnel gate generation witness is unavailable; state retained");
|
|
24
69
|
return {
|
|
25
70
|
id: tunnelServerId(state.name),
|
|
26
71
|
kind: "service",
|
|
@@ -29,10 +74,9 @@ export function tunnelServerInput(state: TunnelState, coordRoot: string): Regist
|
|
|
29
74
|
url: state.url,
|
|
30
75
|
port: state.gate_port,
|
|
31
76
|
pid: state.gate_pid,
|
|
32
|
-
|
|
77
|
+
start_token: state.gate_start_token,
|
|
33
78
|
log: resolve(coordRoot, ".cache", "tunnel", gateLogFile(state.name)),
|
|
34
79
|
cwd: coordRoot,
|
|
35
|
-
stop_argv: [resolveBinName(coordRoot), "tunnel", "down", "--name", state.name],
|
|
36
80
|
started_at: state.started_at,
|
|
37
81
|
owner: null,
|
|
38
82
|
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ProcessGenerationVerdict } from "../agents/state/proc-start.ts";
|
|
1
2
|
import type {
|
|
2
3
|
PressureAssessment,
|
|
3
4
|
PressureHysteresisState,
|
|
@@ -47,10 +48,16 @@ export const SUPERVISOR_RESOURCE_BUDGET = {
|
|
|
47
48
|
|
|
48
49
|
export type SupervisorProcessState = "starting" | "running" | "stopping" | "stopped" | "error";
|
|
49
50
|
|
|
51
|
+
export interface SupervisorBuild {
|
|
52
|
+
version: string;
|
|
53
|
+
revision: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
50
56
|
export interface SupervisorServiceStatusRecord {
|
|
51
57
|
schema_version: typeof SUPERVISOR_STATUS_SCHEMA_VERSION;
|
|
52
58
|
pid: number;
|
|
53
59
|
start_token?: string;
|
|
60
|
+
running_build?: SupervisorBuild;
|
|
54
61
|
host: string;
|
|
55
62
|
nonce: string;
|
|
56
63
|
state: SupervisorProcessState;
|
|
@@ -70,6 +77,12 @@ export interface SupervisorServiceStatusRecord {
|
|
|
70
77
|
export interface SupervisorStatus {
|
|
71
78
|
running: boolean;
|
|
72
79
|
stale: boolean;
|
|
80
|
+
generation: ProcessGenerationVerdict;
|
|
81
|
+
liveness: "running" | "stopped" | "unknown";
|
|
82
|
+
running_build: SupervisorBuild | null;
|
|
83
|
+
installed_build: SupervisorBuild;
|
|
84
|
+
restart_needed: boolean;
|
|
85
|
+
build_status: string;
|
|
73
86
|
record?: SupervisorServiceStatusRecord;
|
|
74
87
|
status_path: string;
|
|
75
88
|
snapshot_path: string;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { canonicalizeDiagnosticIdentity } from "../diagnostics/path-identity.ts";
|
|
2
3
|
import {
|
|
3
4
|
SUPERVISOR_DIAGNOSTIC_LIMITS,
|
|
4
5
|
SUPERVISOR_EXPLANATION_SCHEMA_VERSION,
|
|
@@ -94,7 +95,9 @@ function statement(
|
|
|
94
95
|
): SupervisorExplanationStatement {
|
|
95
96
|
return {
|
|
96
97
|
id: `ex_${createHash("sha256")
|
|
97
|
-
.update(
|
|
98
|
+
.update(
|
|
99
|
+
canonicalizeDiagnosticIdentity(`${code}\u0000${summary}\u0000${evidenceIds.join(",")}`),
|
|
100
|
+
)
|
|
98
101
|
.digest("hex")
|
|
99
102
|
.slice(0, 24)}`,
|
|
100
103
|
code,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import type { CoordinationHealthSnapshot } from "../agents/health.ts";
|
|
3
|
+
import { canonicalizeDiagnosticIdentity } from "../diagnostics/path-identity.ts";
|
|
3
4
|
import type {
|
|
4
5
|
ResourceProcessGroup,
|
|
5
6
|
ResourceProcessSample,
|
|
@@ -96,15 +97,17 @@ export function updateSupervisorFindings(
|
|
|
96
97
|
const candidates = evaluateCandidates(input)
|
|
97
98
|
.sort((left, right) => severityRank(right.severity) - severityRank(left.severity))
|
|
98
99
|
.slice(0, SUPERVISOR_DIAGNOSTIC_LIMITS.max_active_findings);
|
|
99
|
-
const priorActive = new Map(previous.active.map((entry) => [entry
|
|
100
|
+
const priorActive = new Map(previous.active.map((entry) => [findingFingerprint(entry), entry]));
|
|
100
101
|
const current = new Set<string>();
|
|
101
102
|
const active: SupervisorFinding[] = [];
|
|
102
103
|
const transitions = new Map(previous.transitions.map((entry) => [entry.id, entry]));
|
|
103
104
|
|
|
104
|
-
for (const
|
|
105
|
-
const
|
|
106
|
-
current.add(
|
|
107
|
-
const prior = priorActive.get(
|
|
105
|
+
for (const observed of candidates) {
|
|
106
|
+
const identity = findingFingerprint(observed);
|
|
107
|
+
current.add(identity);
|
|
108
|
+
const prior = priorActive.get(identity) ?? recentResolved(previous, identity, observedAt);
|
|
109
|
+
const candidate = preserveEpisodeReferences(observed, prior);
|
|
110
|
+
const fingerprint = prior?.fingerprint ?? identity;
|
|
108
111
|
const openedAt = prior?.opened_at ?? candidate.primary_source.observed_at;
|
|
109
112
|
const occurrenceCount = prior
|
|
110
113
|
? prior.state === "resolved"
|
|
@@ -114,7 +117,7 @@ export function updateSupervisorFindings(
|
|
|
114
117
|
const peak = peakEvidence(prior, candidate.evidence, observedAt);
|
|
115
118
|
const finding: SupervisorFinding = {
|
|
116
119
|
schema_version: SUPERVISOR_FINDING_SCHEMA_VERSION,
|
|
117
|
-
id: deterministicId("find", `${fingerprint}\u0000${openedAt}`),
|
|
120
|
+
id: prior?.id ?? deterministicId("find", `${fingerprint}\u0000${openedAt}`),
|
|
118
121
|
fingerprint,
|
|
119
122
|
source_kind: candidate.source_kind,
|
|
120
123
|
finding_kind: candidate.finding_kind,
|
|
@@ -139,7 +142,7 @@ export function updateSupervisorFindings(
|
|
|
139
142
|
}
|
|
140
143
|
|
|
141
144
|
for (const prior of priorActive.values()) {
|
|
142
|
-
if (current.has(prior
|
|
145
|
+
if (current.has(findingFingerprint(prior))) continue;
|
|
143
146
|
transitions.set(prior.id, {
|
|
144
147
|
...prior,
|
|
145
148
|
state: "resolved",
|
|
@@ -859,7 +862,7 @@ function recentResolved(
|
|
|
859
862
|
return findings.transitions
|
|
860
863
|
.filter(
|
|
861
864
|
(entry) =>
|
|
862
|
-
entry
|
|
865
|
+
findingFingerprint(entry) === fingerprint &&
|
|
863
866
|
entry.state === "resolved" &&
|
|
864
867
|
entry.resolved_at !== undefined &&
|
|
865
868
|
observedMs - Date.parse(entry.resolved_at) >= 0 &&
|
|
@@ -898,7 +901,12 @@ function peakEvidence(
|
|
|
898
901
|
: {};
|
|
899
902
|
}
|
|
900
903
|
|
|
901
|
-
function findingFingerprint(
|
|
904
|
+
function findingFingerprint(
|
|
905
|
+
candidate: Pick<
|
|
906
|
+
Candidate,
|
|
907
|
+
"source_kind" | "finding_kind" | "scope_kind" | "scope_id" | "primary_source"
|
|
908
|
+
>,
|
|
909
|
+
): string {
|
|
902
910
|
return digest(
|
|
903
911
|
[
|
|
904
912
|
candidate.source_kind,
|
|
@@ -915,7 +923,57 @@ function deterministicId(prefix: string, value: string): string {
|
|
|
915
923
|
}
|
|
916
924
|
|
|
917
925
|
function digest(value: string): string {
|
|
918
|
-
return createHash("sha256").update(value).digest("hex");
|
|
926
|
+
return createHash("sha256").update(canonicalizeDiagnosticIdentity(value)).digest("hex");
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
/** Keep opaque episode links when the same source observation is replayed privately. */
|
|
930
|
+
function preserveEpisodeReferences(candidate: Candidate, prior?: SupervisorFinding): Candidate {
|
|
931
|
+
if (!prior) return candidate;
|
|
932
|
+
const sources = [
|
|
933
|
+
prior.primary_source,
|
|
934
|
+
...prior.evidence.map((entry) => entry.source),
|
|
935
|
+
...(prior.workload_context ? [prior.workload_context.source] : []),
|
|
936
|
+
];
|
|
937
|
+
const byIdentity = new Map(sources.map((source) => [sourceIdentity(source), source]));
|
|
938
|
+
const retainSource = (source: SupervisorSourceReference): SupervisorSourceReference => {
|
|
939
|
+
const captured = byIdentity.get(sourceIdentity(source));
|
|
940
|
+
return captured ? { ...source, id: captured.id } : source;
|
|
941
|
+
};
|
|
942
|
+
return {
|
|
943
|
+
...candidate,
|
|
944
|
+
primary_source: retainSource(candidate.primary_source),
|
|
945
|
+
evidence: candidate.evidence.map((entry) => {
|
|
946
|
+
const captured = prior.evidence.find(
|
|
947
|
+
(previous) =>
|
|
948
|
+
sourceIdentity(previous.source) === sourceIdentity(entry.source) &&
|
|
949
|
+
previous.unit === entry.unit &&
|
|
950
|
+
previous.observed_value === entry.observed_value &&
|
|
951
|
+
canonicalizeDiagnosticIdentity(previous.summary) ===
|
|
952
|
+
canonicalizeDiagnosticIdentity(entry.summary),
|
|
953
|
+
);
|
|
954
|
+
return {
|
|
955
|
+
...entry,
|
|
956
|
+
...(captured ? { id: captured.id } : {}),
|
|
957
|
+
source: retainSource(entry.source),
|
|
958
|
+
};
|
|
959
|
+
}),
|
|
960
|
+
...(candidate.workload_context
|
|
961
|
+
? {
|
|
962
|
+
workload_context: {
|
|
963
|
+
...candidate.workload_context,
|
|
964
|
+
source: retainSource(candidate.workload_context.source),
|
|
965
|
+
},
|
|
966
|
+
}
|
|
967
|
+
: {}),
|
|
968
|
+
};
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
function sourceIdentity(source: SupervisorSourceReference): string {
|
|
972
|
+
return canonicalizeDiagnosticIdentity(
|
|
973
|
+
[source.source_kind, source.source_id, source.record_id ?? "", source.observed_at].join(
|
|
974
|
+
"\u0000",
|
|
975
|
+
),
|
|
976
|
+
);
|
|
919
977
|
}
|
|
920
978
|
|
|
921
979
|
function validFindings(value: SupervisorFindings | undefined): value is SupervisorFindings {
|
|
@@ -16,7 +16,13 @@ import { resolve } from "node:path";
|
|
|
16
16
|
import { performance } from "node:perf_hooks";
|
|
17
17
|
import { fileURLToPath } from "node:url";
|
|
18
18
|
import { collectCoordinationHealthSnapshot } from "../agents/health.ts";
|
|
19
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
type ProcessGenerationSignalDependencies,
|
|
21
|
+
type ProcessGenerationVerdict,
|
|
22
|
+
processStartToken,
|
|
23
|
+
signalIfSameGeneration,
|
|
24
|
+
} from "../agents/state/proc-start.ts";
|
|
25
|
+
import { resolveBinName } from "../config.ts";
|
|
20
26
|
import { pressureHistoryFromSupervisor } from "../diagnostics/advice.ts";
|
|
21
27
|
import { assessPressure } from "../diagnostics/pressure.ts";
|
|
22
28
|
import { readEventV3ControlState } from "../events/v3/control.ts";
|
|
@@ -52,7 +58,12 @@ import type {
|
|
|
52
58
|
} from "./ledger-state.ts";
|
|
53
59
|
import { SupervisorLogCollector } from "./log-feed.ts";
|
|
54
60
|
import { collectServiceHealth } from "./services.ts";
|
|
55
|
-
import {
|
|
61
|
+
import {
|
|
62
|
+
installedSupervisorBuild,
|
|
63
|
+
readSupervisorStatus,
|
|
64
|
+
type SupervisorProcessDependencies,
|
|
65
|
+
supervisorProcessLiveness,
|
|
66
|
+
} from "./status.ts";
|
|
56
67
|
import {
|
|
57
68
|
readSupervisorFindings,
|
|
58
69
|
readSupervisorHistory,
|
|
@@ -97,15 +108,16 @@ export async function spawnSupervisor(
|
|
|
97
108
|
): Promise<SupervisorStatus> {
|
|
98
109
|
const coordRoot = resolve(coordRootRaw);
|
|
99
110
|
const current = readSupervisorStatus(coordRoot);
|
|
100
|
-
if (current.
|
|
111
|
+
if (current.liveness === "unknown") throw unknownOwnerError();
|
|
112
|
+
if (current.liveness === "running") {
|
|
101
113
|
throw new Error(`local supervisor is already running under pid ${current.record?.pid}`);
|
|
102
114
|
}
|
|
115
|
+
const paths = supervisorPaths(coordRoot);
|
|
116
|
+
assertLeaseAvailable(paths.lease);
|
|
103
117
|
await stopLegacyResourceObserver(coordRoot);
|
|
104
118
|
const intervalMs = normalizeInterval(options.intervalMs);
|
|
105
119
|
const idleExitMs = normalizeIdleExit(options.idleExitMs);
|
|
106
|
-
const paths = supervisorPaths(coordRoot);
|
|
107
120
|
mkdirSync(paths.root, { recursive: true, mode: stateDirMode() });
|
|
108
|
-
rmSync(paths.stop, { force: true });
|
|
109
121
|
const cliPath = fileURLToPath(
|
|
110
122
|
new URL(import.meta.url.endsWith(".ts") ? "../../cli.ts" : "../../cli.js", import.meta.url),
|
|
111
123
|
);
|
|
@@ -167,29 +179,55 @@ export async function ensureSupervisorRunning(coordRoot: string): Promise<Ensure
|
|
|
167
179
|
}
|
|
168
180
|
}
|
|
169
181
|
|
|
170
|
-
export
|
|
182
|
+
export interface SupervisorStopResult extends SupervisorStatus {
|
|
183
|
+
signal_sent: boolean;
|
|
184
|
+
signal_verdict: ProcessGenerationVerdict;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export function requestSupervisorStop(
|
|
188
|
+
coordRootRaw: string,
|
|
189
|
+
dependencies: SupervisorProcessDependencies & ProcessGenerationSignalDependencies = {},
|
|
190
|
+
): SupervisorStopResult {
|
|
171
191
|
const coordRoot = resolve(coordRootRaw);
|
|
172
192
|
const paths = supervisorPaths(coordRoot);
|
|
173
|
-
const status = readSupervisorStatus(coordRoot);
|
|
193
|
+
const status = readSupervisorStatus(coordRoot, Date.now(), dependencies);
|
|
174
194
|
mkdirSync(paths.root, { recursive: true, mode: stateDirMode() });
|
|
175
195
|
writePrivateJsonAtomic(paths.stop, {
|
|
176
196
|
requested_at: new Date().toISOString(),
|
|
177
197
|
requested_by_pid: process.pid,
|
|
178
198
|
});
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
199
|
+
const legacyPaths = resourcePaths(coordRoot);
|
|
200
|
+
if (existsSync(legacyPaths.service) || existsSync(legacyPaths.lease))
|
|
201
|
+
requestResourceServiceStop(coordRoot, dependencies);
|
|
202
|
+
let signal = { sent: false, verdict: status.generation };
|
|
203
|
+
if (
|
|
204
|
+
status.liveness === "running" &&
|
|
205
|
+
status.record?.host === hostname() &&
|
|
206
|
+
process.platform !== "win32"
|
|
207
|
+
) {
|
|
208
|
+
signal = signalIfSameGeneration(
|
|
209
|
+
status.record.pid,
|
|
210
|
+
status.record.start_token,
|
|
211
|
+
"SIGTERM",
|
|
212
|
+
dependencies,
|
|
213
|
+
);
|
|
185
214
|
}
|
|
186
|
-
return
|
|
215
|
+
return {
|
|
216
|
+
...readSupervisorStatus(coordRoot, Date.now(), dependencies),
|
|
217
|
+
signal_sent: signal.sent,
|
|
218
|
+
signal_verdict: signal.verdict,
|
|
219
|
+
};
|
|
187
220
|
}
|
|
188
221
|
|
|
189
222
|
export async function runSupervisor(
|
|
190
223
|
input: RunSupervisorInput,
|
|
191
224
|
): Promise<SupervisorServiceStatusRecord> {
|
|
192
225
|
const coordRoot = resolve(input.coordRoot);
|
|
226
|
+
const current = readSupervisorStatus(coordRoot);
|
|
227
|
+
if (current.liveness === "unknown") throw unknownOwnerError();
|
|
228
|
+
if (current.liveness === "running") {
|
|
229
|
+
throw new Error(`local supervisor is already running under pid ${current.record?.pid}`);
|
|
230
|
+
}
|
|
193
231
|
const paths = supervisorPaths(coordRoot);
|
|
194
232
|
const intervalMs = normalizeInterval(input.intervalMs);
|
|
195
233
|
const idleExitMs = normalizeIdleExit(input.idleExitMs);
|
|
@@ -204,6 +242,7 @@ export async function runSupervisor(
|
|
|
204
242
|
schema_version: SUPERVISOR_STATUS_SCHEMA_VERSION,
|
|
205
243
|
pid: process.pid,
|
|
206
244
|
...(startToken ? { start_token: startToken } : {}),
|
|
245
|
+
running_build: installedSupervisorBuild(),
|
|
207
246
|
host: hostname(),
|
|
208
247
|
nonce: lease.nonce,
|
|
209
248
|
state: "starting",
|
|
@@ -517,7 +556,9 @@ function acquireLease(coordRoot: string, now: Date): SupervisorLease {
|
|
|
517
556
|
} catch (error) {
|
|
518
557
|
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
519
558
|
const existing = readLease(paths.lease);
|
|
520
|
-
|
|
559
|
+
const liveness = existing ? leaseIsLive(existing) : "unknown";
|
|
560
|
+
if (liveness === "unknown") throw unknownOwnerError();
|
|
561
|
+
if (existing && liveness === "running") {
|
|
521
562
|
throw new Error(`local supervisor lease is held by pid ${existing.pid}`);
|
|
522
563
|
}
|
|
523
564
|
try {
|
|
@@ -554,7 +595,13 @@ function releaseLease(path: string, nonce: string): void {
|
|
|
554
595
|
function readLease(path: string): SupervisorLease | undefined {
|
|
555
596
|
try {
|
|
556
597
|
const value = JSON.parse(readFileSync(path, "utf8")) as SupervisorLease;
|
|
557
|
-
return value &&
|
|
598
|
+
return value &&
|
|
599
|
+
Number.isSafeInteger(value.pid) &&
|
|
600
|
+
value.pid > 0 &&
|
|
601
|
+
typeof value.host === "string" &&
|
|
602
|
+
value.host.length > 0 &&
|
|
603
|
+
typeof value.nonce === "string" &&
|
|
604
|
+
value.nonce.length > 0
|
|
558
605
|
? value
|
|
559
606
|
: undefined;
|
|
560
607
|
} catch {
|
|
@@ -562,22 +609,36 @@ function readLease(path: string): SupervisorLease | undefined {
|
|
|
562
609
|
}
|
|
563
610
|
}
|
|
564
611
|
|
|
565
|
-
function leaseIsLive(lease: SupervisorLease):
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
612
|
+
function leaseIsLive(lease: SupervisorLease): SupervisorStatus["liveness"] {
|
|
613
|
+
return supervisorProcessLiveness(lease).liveness;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
function assertLeaseAvailable(path: string): void {
|
|
617
|
+
if (!existsSync(path)) return;
|
|
618
|
+
const lease = readLease(path);
|
|
619
|
+
const liveness = lease ? leaseIsLive(lease) : "unknown";
|
|
620
|
+
if (liveness === "unknown") throw unknownOwnerError();
|
|
621
|
+
if (liveness === "running")
|
|
622
|
+
throw new Error(`local supervisor lease is held by pid ${lease?.pid}`);
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
function unknownOwnerError(): Error {
|
|
626
|
+
return new Error(
|
|
627
|
+
`local supervisor process identity is unknown; run ${resolveBinName()} supervisor stop to write the durable stop request before starting again`,
|
|
628
|
+
);
|
|
573
629
|
}
|
|
574
630
|
|
|
575
631
|
async function stopLegacyResourceObserver(coordRoot: string): Promise<void> {
|
|
576
632
|
const legacy = readResourceServiceStatus(coordRoot);
|
|
577
|
-
if (
|
|
633
|
+
if (legacy.liveness === "stopped") return;
|
|
578
634
|
requestResourceServiceStop(coordRoot);
|
|
579
635
|
const deadline = Date.now() + 2_000;
|
|
580
|
-
while (readResourceServiceStatus(coordRoot).
|
|
636
|
+
while (readResourceServiceStatus(coordRoot).liveness !== "stopped" && Date.now() < deadline)
|
|
637
|
+
await delay(50);
|
|
638
|
+
if (readResourceServiceStatus(coordRoot).liveness !== "stopped")
|
|
639
|
+
throw new Error(
|
|
640
|
+
"legacy resource observer has not stopped after its durable stop request; its process identity may be unknown",
|
|
641
|
+
);
|
|
581
642
|
}
|
|
582
643
|
|
|
583
644
|
function log(
|
|
@@ -2,6 +2,7 @@ import { existsSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs
|
|
|
2
2
|
import { hostname } from "node:os";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { checkPidToken, processStartToken } from "../agents/state/proc-start.ts";
|
|
5
|
+
import { readGovernorServiceStatus } from "../governor/service-read.ts";
|
|
5
6
|
import { writePrivateJsonAtomic } from "../storage/atomic-json.ts";
|
|
6
7
|
import {
|
|
7
8
|
type ObservedServiceHealth,
|
|
@@ -41,10 +42,7 @@ export function collectServiceHealth(
|
|
|
41
42
|
join(coordRoot, ".harnery", "semantic", "v2", "service.json"),
|
|
42
43
|
"semantic-reader",
|
|
43
44
|
);
|
|
44
|
-
const governor =
|
|
45
|
-
join(coordRoot, ".harnery", "governor-service", "service.json"),
|
|
46
|
-
"governor",
|
|
47
|
-
);
|
|
45
|
+
const governor = readGovernorHealth(coordRoot);
|
|
48
46
|
const relay = readPresenceRelay(join(coordRoot, ".harnery", "presence", "relay-daemon.json"));
|
|
49
47
|
const dashboard = consumers.find((consumer) => consumer.id === "dashboard");
|
|
50
48
|
const services: ObservedServiceHealth[] = [
|
|
@@ -118,10 +116,7 @@ interface DetachedServiceRecord {
|
|
|
118
116
|
heartbeat_at?: string;
|
|
119
117
|
}
|
|
120
118
|
|
|
121
|
-
function readDetachedService(
|
|
122
|
-
path: string,
|
|
123
|
-
id: "semantic-reader" | "governor",
|
|
124
|
-
): ObservedServiceHealth {
|
|
119
|
+
function readDetachedService(path: string, id: "semantic-reader"): ObservedServiceHealth {
|
|
125
120
|
const record = readBoundedJson<DetachedServiceRecord>(path);
|
|
126
121
|
if (!record || !Number.isSafeInteger(record.pid) || record.pid < 1) {
|
|
127
122
|
return { id, state: "stopped" };
|
|
@@ -150,6 +145,29 @@ function readDetachedService(
|
|
|
150
145
|
);
|
|
151
146
|
}
|
|
152
147
|
|
|
148
|
+
function readGovernorHealth(coordRoot: string): ObservedServiceHealth {
|
|
149
|
+
const status = readGovernorServiceStatus(coordRoot);
|
|
150
|
+
const state =
|
|
151
|
+
status.liveness === "unknown"
|
|
152
|
+
? "unknown"
|
|
153
|
+
: status.running
|
|
154
|
+
? "running"
|
|
155
|
+
: status.stale
|
|
156
|
+
? "stale"
|
|
157
|
+
: "stopped";
|
|
158
|
+
const reason =
|
|
159
|
+
state === "unknown"
|
|
160
|
+
? "governor-process-generation-unknown"
|
|
161
|
+
: status.generation === "different"
|
|
162
|
+
? "recorded-process-generation-different"
|
|
163
|
+
: status.stale
|
|
164
|
+
? "recorded-running-process-missing"
|
|
165
|
+
: undefined;
|
|
166
|
+
return status.record
|
|
167
|
+
? projection("governor", state, status.record, reason)
|
|
168
|
+
: { id: "governor", state, ...(reason ? { reason } : {}) };
|
|
169
|
+
}
|
|
170
|
+
|
|
153
171
|
function readPresenceRelay(path: string): ObservedServiceHealth {
|
|
154
172
|
const record = readBoundedJson<{ pid: number; started_at?: string }>(path);
|
|
155
173
|
if (!record || !Number.isSafeInteger(record.pid) || record.pid < 1) {
|
|
@@ -1,7 +1,14 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
1
2
|
import { hostname } from "node:os";
|
|
2
|
-
import {
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import {
|
|
5
|
+
type ProcessGenerationVerdict,
|
|
6
|
+
verifyProcessGeneration,
|
|
7
|
+
} from "../agents/state/proc-start.ts";
|
|
8
|
+
import { currentHarneryRuntimeBuild, harneryPackageRoot } from "../events/v3/runtime-build.ts";
|
|
3
9
|
import {
|
|
4
10
|
SUPERVISOR_STATUS_SCHEMA_VERSION,
|
|
11
|
+
type SupervisorBuild,
|
|
5
12
|
type SupervisorServiceStatusRecord,
|
|
6
13
|
type SupervisorStatus,
|
|
7
14
|
} from "./contract.ts";
|
|
@@ -9,27 +16,71 @@ import { readSupervisorServiceRecord, supervisorPaths } from "./storage.ts";
|
|
|
9
16
|
|
|
10
17
|
const STALE_HEARTBEAT_MS = 15_000;
|
|
11
18
|
|
|
12
|
-
export
|
|
19
|
+
export interface SupervisorProcessDependencies {
|
|
20
|
+
readToken?: (pid: number) => string | null;
|
|
21
|
+
probePid?: (pid: number) => unknown;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function installedSupervisorBuild(): SupervisorBuild {
|
|
25
|
+
let version = "unknown";
|
|
26
|
+
let revision = "unknown";
|
|
27
|
+
try {
|
|
28
|
+
const pkg = JSON.parse(readFileSync(join(harneryPackageRoot(), "package.json"), "utf8"));
|
|
29
|
+
if (typeof pkg.version === "string") version = pkg.version;
|
|
30
|
+
revision = currentHarneryRuntimeBuild();
|
|
31
|
+
} catch {
|
|
32
|
+
// Missing build evidence must still produce an inspectable status.
|
|
33
|
+
}
|
|
34
|
+
return { version, revision };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function readSupervisorStatus(
|
|
38
|
+
coordRoot: string,
|
|
39
|
+
nowMs = Date.now(),
|
|
40
|
+
dependencies: SupervisorProcessDependencies = {},
|
|
41
|
+
): SupervisorStatus {
|
|
13
42
|
const paths = supervisorPaths(coordRoot);
|
|
14
43
|
const record = readSupervisorServiceRecord(coordRoot);
|
|
44
|
+
const installedBuild = installedSupervisorBuild();
|
|
45
|
+
const runningBuild = validBuild(record?.running_build) ? record.running_build : null;
|
|
46
|
+
const restartNeeded =
|
|
47
|
+
!runningBuild ||
|
|
48
|
+
installedBuild.version === "unknown" ||
|
|
49
|
+
installedBuild.revision === "unknown" ||
|
|
50
|
+
runningBuild.version !== installedBuild.version ||
|
|
51
|
+
runningBuild.revision !== installedBuild.revision;
|
|
52
|
+
const build = {
|
|
53
|
+
running_build: runningBuild,
|
|
54
|
+
installed_build: installedBuild,
|
|
55
|
+
restart_needed: restartNeeded,
|
|
56
|
+
build_status: !runningBuild
|
|
57
|
+
? "running build unknown; restart needed"
|
|
58
|
+
: restartNeeded
|
|
59
|
+
? "restart needed"
|
|
60
|
+
: "current",
|
|
61
|
+
};
|
|
15
62
|
if (!validRecord(record)) {
|
|
16
63
|
return {
|
|
17
64
|
running: false,
|
|
18
65
|
stale: false,
|
|
66
|
+
generation: "unknown",
|
|
67
|
+
liveness: existsSync(paths.service) ? "unknown" : "stopped",
|
|
68
|
+
...build,
|
|
19
69
|
status_path: paths.service,
|
|
20
70
|
snapshot_path: paths.snapshot,
|
|
21
71
|
};
|
|
22
72
|
}
|
|
23
73
|
const heartbeatAt = Date.parse(record.heartbeat_at);
|
|
24
74
|
const stale = !Number.isFinite(heartbeatAt) || nowMs - heartbeatAt > STALE_HEARTBEAT_MS;
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
(record.state === "starting" || record.state === "running" || record.state === "stopping") &&
|
|
29
|
-
pidStillMatches(record.pid, record.start_token);
|
|
75
|
+
const owner = supervisorProcessLiveness(record, dependencies);
|
|
76
|
+
const liveness =
|
|
77
|
+
owner.generation === "same" && record.state === "stopped" ? "stopped" : owner.liveness;
|
|
30
78
|
return {
|
|
31
|
-
running,
|
|
79
|
+
running: liveness === "running" && !stale && record.state !== "stopped",
|
|
32
80
|
stale,
|
|
81
|
+
generation: owner.generation,
|
|
82
|
+
liveness,
|
|
83
|
+
...build,
|
|
33
84
|
record,
|
|
34
85
|
status_path: paths.service,
|
|
35
86
|
snapshot_path: paths.snapshot,
|
|
@@ -49,11 +100,37 @@ function validRecord(
|
|
|
49
100
|
);
|
|
50
101
|
}
|
|
51
102
|
|
|
52
|
-
|
|
103
|
+
/** A stale heartbeat does not prove that a process has exited. */
|
|
104
|
+
export function supervisorProcessLiveness(
|
|
105
|
+
owner: { pid: number; host: string; start_token?: string },
|
|
106
|
+
dependencies: SupervisorProcessDependencies = {},
|
|
107
|
+
): { liveness: SupervisorStatus["liveness"]; generation: ProcessGenerationVerdict } {
|
|
108
|
+
if (owner.host !== hostname()) return { liveness: "unknown", generation: "unknown" };
|
|
53
109
|
try {
|
|
54
|
-
process.kill(pid, 0);
|
|
110
|
+
(dependencies.probePid ?? ((pid) => process.kill(pid, 0)))(owner.pid);
|
|
55
111
|
} catch (error) {
|
|
56
|
-
|
|
112
|
+
return {
|
|
113
|
+
liveness: (error as NodeJS.ErrnoException).code === "ESRCH" ? "stopped" : "unknown",
|
|
114
|
+
generation: "unknown",
|
|
115
|
+
};
|
|
57
116
|
}
|
|
58
|
-
|
|
117
|
+
const generation = verifyProcessGeneration(owner.pid, owner.start_token, dependencies.readToken);
|
|
118
|
+
return {
|
|
119
|
+
generation,
|
|
120
|
+
liveness:
|
|
121
|
+
generation === "same" ? "running" : generation === "different" ? "stopped" : "unknown",
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function validBuild(value: unknown): value is SupervisorBuild {
|
|
126
|
+
if (!value || typeof value !== "object") return false;
|
|
127
|
+
const build = value as Partial<SupervisorBuild>;
|
|
128
|
+
return (
|
|
129
|
+
typeof build.version === "string" &&
|
|
130
|
+
!!build.version &&
|
|
131
|
+
build.version !== "unknown" &&
|
|
132
|
+
typeof build.revision === "string" &&
|
|
133
|
+
!!build.revision &&
|
|
134
|
+
build.revision !== "unknown"
|
|
135
|
+
);
|
|
59
136
|
}
|