harnery 0.45.1 → 0.45.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +6 -45
- package/dist/core/agents/coord-client.d.ts.map +1 -1
- package/dist/core/agents/coord-client.js +0 -18
- package/dist/core/agents/state/pidmap.js +2 -2
- package/dist/core/agents/state/proc-start.d.ts +20 -31
- package/dist/core/agents/state/proc-start.d.ts.map +1 -1
- package/dist/core/agents/state/proc-start.js +37 -70
- package/dist/core/agents/state/service-process.d.ts +61 -0
- package/dist/core/agents/state/service-process.d.ts.map +1 -0
- package/dist/core/agents/state/service-process.js +143 -0
- package/dist/core/context/index.d.ts +6 -4
- package/dist/core/context/index.d.ts.map +1 -1
- package/dist/core/context/index.js +194 -256
- package/dist/core/context/observation.d.ts +9 -9
- package/dist/core/context/observation.d.ts.map +1 -1
- package/dist/core/context/observation.js +37 -6
- package/dist/core/events/v3/runtime-build.d.ts +14 -0
- package/dist/core/events/v3/runtime-build.d.ts.map +1 -1
- package/dist/core/events/v3/runtime-build.js +44 -0
- package/dist/core/governor/index.d.ts +2 -1
- package/dist/core/governor/index.d.ts.map +1 -1
- package/dist/core/governor/index.js +2 -1
- package/dist/core/governor/service-read.d.ts +3 -10
- package/dist/core/governor/service-read.d.ts.map +1 -1
- package/dist/core/governor/service-read.js +10 -94
- package/dist/core/governor/service.d.ts +3 -7
- package/dist/core/governor/service.d.ts.map +1 -1
- package/dist/core/governor/service.js +17 -182
- package/dist/core/hooks/resolve/owner.js +2 -2
- package/dist/core/resources/sampler.js +2 -2
- package/dist/core/resources/service-status.d.ts +2 -14
- package/dist/core/resources/service-status.d.ts.map +1 -1
- package/dist/core/resources/service-status.js +3 -23
- package/dist/core/resources/service.d.ts +3 -7
- package/dist/core/resources/service.d.ts.map +1 -1
- package/dist/core/resources/service.js +12 -101
- package/dist/core/resources/status.d.ts.map +1 -1
- package/dist/core/resources/status.js +3 -2
- package/dist/core/semantic/service.d.ts.map +1 -1
- package/dist/core/semantic/service.js +9 -2
- package/dist/core/servers/index.d.ts +0 -2
- package/dist/core/servers/index.d.ts.map +1 -1
- package/dist/core/servers/index.js +3 -18
- package/dist/core/supervisor/service.d.ts +4 -7
- package/dist/core/supervisor/service.d.ts.map +1 -1
- package/dist/core/supervisor/service.js +23 -122
- package/dist/core/supervisor/services.js +2 -2
- package/dist/core/supervisor/status.d.ts +3 -17
- package/dist/core/supervisor/status.d.ts.map +1 -1
- package/dist/core/supervisor/status.js +7 -70
- package/dist/core/workflow/engine.js +2 -1
- package/dist/core/workflow/index.d.ts +3 -4
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +2 -2
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +30 -74
- package/dist/core/workflow/resume-lock.d.ts +17 -0
- package/dist/core/workflow/resume-lock.d.ts.map +1 -0
- package/dist/core/workflow/resume-lock.js +158 -0
- package/dist/core/workflow/run-state.d.ts +0 -21
- package/dist/core/workflow/run-state.d.ts.map +1 -1
- package/dist/core/workflow/run-state.js +3 -174
- package/dist/core/workflow/workspaces/leases.d.ts +4 -0
- package/dist/core/workflow/workspaces/leases.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/leases.js +1 -1
- package/dist/core/workflow/workspaces/local-git.js +91 -210
- package/dist/core/workflow/workspaces/ownership.d.ts +0 -7
- package/dist/core/workflow/workspaces/ownership.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/ownership.js +3 -130
- package/dist/core/workflow/workspaces/reclaim.d.ts +4 -7
- package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/reclaim.js +0 -4
- package/dist/core/workflow/workspaces/recovery.d.ts +1 -1
- package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.js +1 -1
- package/dist/core/workflow/workspaces/state.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/state.js +1 -4
- package/package.json +1 -1
- package/src/commands/workflow.ts +8 -58
- package/src/core/agents/coord-client.ts +0 -20
- package/src/core/agents/state/pidmap.ts +2 -2
- package/src/core/agents/state/proc-start.ts +41 -69
- package/src/core/agents/state/service-process.ts +217 -0
- package/src/core/context/index.ts +207 -272
- package/src/core/context/observation.ts +40 -9
- package/src/core/events/v3/runtime-build.ts +62 -0
- package/src/core/governor/index.ts +1 -2
- package/src/core/governor/service-read.ts +15 -114
- package/src/core/governor/service.ts +36 -220
- package/src/core/hooks/resolve/owner.ts +2 -2
- package/src/core/resources/sampler.ts +3 -2
- package/src/core/resources/service-status.ts +6 -35
- package/src/core/resources/service.ts +23 -133
- package/src/core/resources/status.ts +3 -2
- package/src/core/semantic/service.ts +8 -2
- package/src/core/servers/index.ts +3 -23
- package/src/core/supervisor/service.ts +33 -155
- package/src/core/supervisor/services.ts +2 -2
- package/src/core/supervisor/status.ts +10 -85
- package/src/core/workflow/engine.ts +1 -1
- package/src/core/workflow/index.ts +2 -13
- package/src/core/workflow/proof.ts +35 -83
- package/src/core/workflow/resume-lock.ts +179 -0
- package/src/core/workflow/run-state.ts +3 -207
- package/src/core/workflow/workspaces/leases.ts +1 -1
- package/src/core/workflow/workspaces/local-git.ts +120 -239
- package/src/core/workflow/workspaces/ownership.ts +3 -158
- package/src/core/workflow/workspaces/reclaim.ts +3 -9
- package/src/core/workflow/workspaces/recovery.ts +1 -1
- package/src/core/workflow/workspaces/state.ts +1 -5
- package/dist/core/agents/state/process-handle.d.ts +0 -26
- package/dist/core/agents/state/process-handle.d.ts.map +0 -1
- package/dist/core/agents/state/process-handle.js +0 -121
- package/dist/core/workflow/recovery-audit.d.ts +0 -30
- package/dist/core/workflow/recovery-audit.d.ts.map +0 -1
- package/dist/core/workflow/recovery-audit.js +0 -116
- package/src/core/agents/state/process-handle.ts +0 -165
- package/src/core/workflow/recovery-audit.ts +0 -181
|
@@ -1,32 +1,26 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
closeSync,
|
|
5
|
-
existsSync,
|
|
6
|
-
mkdirSync,
|
|
7
|
-
openSync,
|
|
8
|
-
readdirSync,
|
|
9
|
-
readFileSync,
|
|
10
|
-
rmSync,
|
|
11
|
-
unlinkSync,
|
|
12
|
-
writeFileSync,
|
|
13
|
-
} from "node:fs";
|
|
2
|
+
import { existsSync, mkdirSync, readdirSync, rmSync } from "node:fs";
|
|
14
3
|
import { hostname } from "node:os";
|
|
15
4
|
import { resolve } from "node:path";
|
|
16
5
|
import { performance } from "node:perf_hooks";
|
|
17
6
|
import { fileURLToPath } from "node:url";
|
|
18
7
|
import { collectCoordinationHealthSnapshot } from "../agents/health.ts";
|
|
8
|
+
import { processStartToken } from "../agents/state/proc-start.ts";
|
|
19
9
|
import {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
10
|
+
acquireServiceLease,
|
|
11
|
+
requestServiceStop,
|
|
12
|
+
type ServiceLiveness,
|
|
13
|
+
type ServiceProcessDependencies,
|
|
14
|
+
type ServiceStopFields,
|
|
15
|
+
serviceLeaseOwner,
|
|
16
|
+
serviceStartRefusal,
|
|
17
|
+
} from "../agents/state/service-process.ts";
|
|
25
18
|
import { resolveBinName } from "../config.ts";
|
|
26
19
|
import { pressureHistoryFromSupervisor } from "../diagnostics/advice.ts";
|
|
27
20
|
import { assessPressure } from "../diagnostics/pressure.ts";
|
|
28
21
|
import { readEventV3ControlState } from "../events/v3/control.ts";
|
|
29
22
|
import { janitorLiveDisplayV3 } from "../events/v3/live-feed.ts";
|
|
23
|
+
import { installedHarneryBuild } from "../events/v3/runtime-build.ts";
|
|
30
24
|
import type { ResourceSamplerState } from "../resources/contract.ts";
|
|
31
25
|
import { sampleResources } from "../resources/sampler.ts";
|
|
32
26
|
import { requestResourceServiceStop } from "../resources/service.ts";
|
|
@@ -34,7 +28,7 @@ import { readResourceServiceStatus } from "../resources/service-status.ts";
|
|
|
34
28
|
import { resourcePaths } from "../resources/storage.ts";
|
|
35
29
|
import { writePrivateJsonAtomic } from "../storage/atomic-json.ts";
|
|
36
30
|
import { closeProcessLoggers, legacyLogFields, processLogger } from "../storage/logger.ts";
|
|
37
|
-
import { stateDirMode
|
|
31
|
+
import { stateDirMode } from "../storage/modes.ts";
|
|
38
32
|
import { collectSupervisorActivitySnapshot } from "./activity.ts";
|
|
39
33
|
import {
|
|
40
34
|
SUPERVISOR_PRESSURE_SCHEMA_VERSION,
|
|
@@ -58,12 +52,7 @@ import type {
|
|
|
58
52
|
} from "./ledger-state.ts";
|
|
59
53
|
import { SupervisorLogCollector } from "./log-feed.ts";
|
|
60
54
|
import { collectServiceHealth } from "./services.ts";
|
|
61
|
-
import {
|
|
62
|
-
installedSupervisorBuild,
|
|
63
|
-
readSupervisorStatus,
|
|
64
|
-
type SupervisorProcessDependencies,
|
|
65
|
-
supervisorProcessLiveness,
|
|
66
|
-
} from "./status.ts";
|
|
55
|
+
import { readSupervisorStatus } from "./status.ts";
|
|
67
56
|
import {
|
|
68
57
|
readSupervisorFindings,
|
|
69
58
|
readSupervisorHistory,
|
|
@@ -80,14 +69,6 @@ const SUPERVISOR_MIN_IDLE_EXIT_MS = 5_000;
|
|
|
80
69
|
const SUPERVISOR_MAX_IDLE_EXIT_MS = 24 * 60 * 60_000;
|
|
81
70
|
const LIVE_DISPLAY_SWEEP_INTERVAL_MS = 5 * 60_000;
|
|
82
71
|
|
|
83
|
-
interface SupervisorLease {
|
|
84
|
-
pid: number;
|
|
85
|
-
start_token?: string;
|
|
86
|
-
host: string;
|
|
87
|
-
nonce: string;
|
|
88
|
-
created_at: string;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
72
|
export interface RunSupervisorInput {
|
|
92
73
|
coordRoot: string;
|
|
93
74
|
intervalMs?: number;
|
|
@@ -108,12 +89,10 @@ export async function spawnSupervisor(
|
|
|
108
89
|
): Promise<SupervisorStatus> {
|
|
109
90
|
const coordRoot = resolve(coordRootRaw);
|
|
110
91
|
const current = readSupervisorStatus(coordRoot);
|
|
111
|
-
if (current.liveness
|
|
112
|
-
if (current.liveness === "running") {
|
|
113
|
-
throw new Error(`local supervisor is already running under pid ${current.record?.pid}`);
|
|
114
|
-
}
|
|
92
|
+
if (current.liveness !== "stopped") throw startRefusal(current.record?.pid, current.liveness);
|
|
115
93
|
const paths = supervisorPaths(coordRoot);
|
|
116
|
-
|
|
94
|
+
const lease = serviceLeaseOwner(paths.lease);
|
|
95
|
+
if (lease.liveness !== "stopped") throw startRefusal(lease.lease?.pid, lease.liveness);
|
|
117
96
|
await stopLegacyResourceObserver(coordRoot);
|
|
118
97
|
const intervalMs = normalizeInterval(options.intervalMs);
|
|
119
98
|
const idleExitMs = normalizeIdleExit(options.idleExitMs);
|
|
@@ -179,44 +158,21 @@ export async function ensureSupervisorRunning(coordRoot: string): Promise<Ensure
|
|
|
179
158
|
}
|
|
180
159
|
}
|
|
181
160
|
|
|
182
|
-
export
|
|
183
|
-
signal_sent: boolean;
|
|
184
|
-
signal_verdict: ProcessGenerationVerdict;
|
|
185
|
-
}
|
|
161
|
+
export type SupervisorStopResult = SupervisorStatus & ServiceStopFields;
|
|
186
162
|
|
|
163
|
+
/** Windows has no graceful SIGTERM, so the durable request alone stops it there. */
|
|
187
164
|
export function requestSupervisorStop(
|
|
188
165
|
coordRootRaw: string,
|
|
189
|
-
dependencies:
|
|
166
|
+
dependencies: ServiceProcessDependencies = {},
|
|
190
167
|
): SupervisorStopResult {
|
|
191
168
|
const coordRoot = resolve(coordRootRaw);
|
|
192
|
-
const paths = supervisorPaths(coordRoot);
|
|
193
|
-
const status = readSupervisorStatus(coordRoot, Date.now(), dependencies);
|
|
194
|
-
mkdirSync(paths.root, { recursive: true, mode: stateDirMode() });
|
|
195
|
-
writePrivateJsonAtomic(paths.stop, {
|
|
196
|
-
requested_at: new Date().toISOString(),
|
|
197
|
-
requested_by_pid: process.pid,
|
|
198
|
-
});
|
|
199
169
|
const legacyPaths = resourcePaths(coordRoot);
|
|
200
170
|
if (existsSync(legacyPaths.service) || existsSync(legacyPaths.lease))
|
|
201
171
|
requestResourceServiceStop(coordRoot, dependencies);
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
process.platform !== "win32"
|
|
207
|
-
) {
|
|
208
|
-
signal = signalIfSameGeneration(
|
|
209
|
-
status.record.pid,
|
|
210
|
-
status.record.start_token,
|
|
211
|
-
"SIGTERM",
|
|
212
|
-
dependencies,
|
|
213
|
-
);
|
|
214
|
-
}
|
|
215
|
-
return {
|
|
216
|
-
...readSupervisorStatus(coordRoot, Date.now(), dependencies),
|
|
217
|
-
signal_sent: signal.sent,
|
|
218
|
-
signal_verdict: signal.verdict,
|
|
219
|
-
};
|
|
172
|
+
const read = () => readSupervisorStatus(coordRoot, Date.now(), dependencies);
|
|
173
|
+
return requestServiceStop(read(), read, supervisorPaths(coordRoot).stop, dependencies, {
|
|
174
|
+
signal: process.platform !== "win32",
|
|
175
|
+
});
|
|
220
176
|
}
|
|
221
177
|
|
|
222
178
|
export async function runSupervisor(
|
|
@@ -224,17 +180,14 @@ export async function runSupervisor(
|
|
|
224
180
|
): Promise<SupervisorServiceStatusRecord> {
|
|
225
181
|
const coordRoot = resolve(input.coordRoot);
|
|
226
182
|
const current = readSupervisorStatus(coordRoot);
|
|
227
|
-
if (current.liveness
|
|
228
|
-
if (current.liveness === "running") {
|
|
229
|
-
throw new Error(`local supervisor is already running under pid ${current.record?.pid}`);
|
|
230
|
-
}
|
|
183
|
+
if (current.liveness !== "stopped") throw startRefusal(current.record?.pid, current.liveness);
|
|
231
184
|
const paths = supervisorPaths(coordRoot);
|
|
232
185
|
const intervalMs = normalizeInterval(input.intervalMs);
|
|
233
186
|
const idleExitMs = normalizeIdleExit(input.idleExitMs);
|
|
234
187
|
const keepAlive = input.keepAlive ?? false;
|
|
235
188
|
const now = input.now ?? (() => new Date());
|
|
236
189
|
const wait = input.wait ?? delay;
|
|
237
|
-
const lease =
|
|
190
|
+
const lease = acquireServiceLease(paths.lease, startRefusal);
|
|
238
191
|
rmSync(paths.stop, { force: true });
|
|
239
192
|
const startedAt = now().toISOString();
|
|
240
193
|
const startToken = processStartToken(process.pid);
|
|
@@ -242,7 +195,7 @@ export async function runSupervisor(
|
|
|
242
195
|
schema_version: SUPERVISOR_STATUS_SCHEMA_VERSION,
|
|
243
196
|
pid: process.pid,
|
|
244
197
|
...(startToken ? { start_token: startToken } : {}),
|
|
245
|
-
running_build:
|
|
198
|
+
running_build: installedHarneryBuild(),
|
|
246
199
|
host: hostname(),
|
|
247
200
|
nonce: lease.nonce,
|
|
248
201
|
state: "starting",
|
|
@@ -488,7 +441,7 @@ export async function runSupervisor(
|
|
|
488
441
|
try {
|
|
489
442
|
await closeProcessLoggers();
|
|
490
443
|
} finally {
|
|
491
|
-
|
|
444
|
+
lease.release();
|
|
492
445
|
}
|
|
493
446
|
}
|
|
494
447
|
return status;
|
|
@@ -533,44 +486,6 @@ function observeLedgerControlState(
|
|
|
533
486
|
}
|
|
534
487
|
}
|
|
535
488
|
|
|
536
|
-
function acquireLease(coordRoot: string, now: Date): SupervisorLease {
|
|
537
|
-
const paths = supervisorPaths(coordRoot);
|
|
538
|
-
mkdirSync(paths.root, { recursive: true, mode: stateDirMode() });
|
|
539
|
-
for (let attempt = 0; attempt < 2; attempt++) {
|
|
540
|
-
const startToken = processStartToken(process.pid);
|
|
541
|
-
const lease: SupervisorLease = {
|
|
542
|
-
pid: process.pid,
|
|
543
|
-
...(startToken ? { start_token: startToken } : {}),
|
|
544
|
-
host: hostname(),
|
|
545
|
-
nonce: randomUUID(),
|
|
546
|
-
created_at: now.toISOString(),
|
|
547
|
-
};
|
|
548
|
-
try {
|
|
549
|
-
const fd = openSync(paths.lease, "wx", stateFileMode());
|
|
550
|
-
try {
|
|
551
|
-
writeFileSync(fd, `${JSON.stringify(lease)}\n`, "utf8");
|
|
552
|
-
} finally {
|
|
553
|
-
closeSync(fd);
|
|
554
|
-
}
|
|
555
|
-
return lease;
|
|
556
|
-
} catch (error) {
|
|
557
|
-
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
558
|
-
const existing = readLease(paths.lease);
|
|
559
|
-
const liveness = existing ? leaseIsLive(existing) : "unknown";
|
|
560
|
-
if (liveness === "unknown") throw unknownOwnerError();
|
|
561
|
-
if (existing && liveness === "running") {
|
|
562
|
-
throw new Error(`local supervisor lease is held by pid ${existing.pid}`);
|
|
563
|
-
}
|
|
564
|
-
try {
|
|
565
|
-
unlinkSync(paths.lease);
|
|
566
|
-
} catch {
|
|
567
|
-
// A concurrent starter may have replaced or removed the stale lease.
|
|
568
|
-
}
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
throw new Error("local supervisor could not acquire its singleton lease");
|
|
572
|
-
}
|
|
573
|
-
|
|
574
489
|
function pruneProjectionDirectory(
|
|
575
490
|
directory: string,
|
|
576
491
|
retainedFindingIds: ReadonlySet<string>,
|
|
@@ -582,49 +497,12 @@ function pruneProjectionDirectory(
|
|
|
582
497
|
}
|
|
583
498
|
}
|
|
584
499
|
|
|
585
|
-
function
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
// A missing lease is already released.
|
|
592
|
-
}
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
function readLease(path: string): SupervisorLease | undefined {
|
|
596
|
-
try {
|
|
597
|
-
const value = JSON.parse(readFileSync(path, "utf8")) as SupervisorLease;
|
|
598
|
-
return value &&
|
|
599
|
-
Number.isSafeInteger(value.pid) &&
|
|
600
|
-
value.pid > 0 &&
|
|
601
|
-
typeof value.host === "string" &&
|
|
602
|
-
value.host.length > 0 &&
|
|
603
|
-
typeof value.nonce === "string" &&
|
|
604
|
-
value.nonce.length > 0
|
|
605
|
-
? value
|
|
606
|
-
: undefined;
|
|
607
|
-
} catch {
|
|
608
|
-
return undefined;
|
|
609
|
-
}
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
function leaseIsLive(lease: SupervisorLease): SupervisorStatus["liveness"] {
|
|
613
|
-
return supervisorProcessLiveness(lease).liveness;
|
|
614
|
-
}
|
|
615
|
-
|
|
616
|
-
function assertLeaseAvailable(path: string): void {
|
|
617
|
-
if (!existsSync(path)) return;
|
|
618
|
-
const lease = readLease(path);
|
|
619
|
-
const liveness = lease ? leaseIsLive(lease) : "unknown";
|
|
620
|
-
if (liveness === "unknown") throw unknownOwnerError();
|
|
621
|
-
if (liveness === "running")
|
|
622
|
-
throw new Error(`local supervisor lease is held by pid ${lease?.pid}`);
|
|
623
|
-
}
|
|
624
|
-
|
|
625
|
-
function unknownOwnerError(): Error {
|
|
626
|
-
return new Error(
|
|
627
|
-
`local supervisor process identity is unknown; run ${resolveBinName()} supervisor stop to write the durable stop request before starting again`,
|
|
500
|
+
function startRefusal(pid: number | undefined, liveness: ServiceLiveness): Error {
|
|
501
|
+
return serviceStartRefusal(
|
|
502
|
+
"local supervisor",
|
|
503
|
+
`${resolveBinName()} supervisor stop`,
|
|
504
|
+
pid,
|
|
505
|
+
liveness,
|
|
628
506
|
);
|
|
629
507
|
}
|
|
630
508
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs";
|
|
2
2
|
import { hostname } from "node:os";
|
|
3
3
|
import { join } from "node:path";
|
|
4
|
-
import {
|
|
4
|
+
import { processStartToken, verifyProcessGeneration } from "../agents/state/proc-start.ts";
|
|
5
5
|
import { readGovernorServiceStatus } from "../governor/service-read.ts";
|
|
6
6
|
import { writePrivateJsonAtomic } from "../storage/atomic-json.ts";
|
|
7
7
|
import {
|
|
@@ -105,7 +105,7 @@ function consumerAlive(record: SupervisorConsumerRecord): boolean {
|
|
|
105
105
|
} catch (error) {
|
|
106
106
|
if ((error as NodeJS.ErrnoException).code !== "EPERM") return false;
|
|
107
107
|
}
|
|
108
|
-
return
|
|
108
|
+
return verifyProcessGeneration(record.pid, record.start_token) !== "different";
|
|
109
109
|
}
|
|
110
110
|
|
|
111
111
|
interface DetachedServiceRecord {
|
|
@@ -1,14 +1,11 @@
|
|
|
1
|
-
import { existsSync
|
|
2
|
-
import { hostname } from "node:os";
|
|
3
|
-
import { join } from "node:path";
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
4
2
|
import {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
} from "../agents/state/
|
|
8
|
-
import {
|
|
3
|
+
observeServiceProcess,
|
|
4
|
+
type ServiceProcessDependencies,
|
|
5
|
+
} from "../agents/state/service-process.ts";
|
|
6
|
+
import { serviceBuildStatus } from "../events/v3/runtime-build.ts";
|
|
9
7
|
import {
|
|
10
8
|
SUPERVISOR_STATUS_SCHEMA_VERSION,
|
|
11
|
-
type SupervisorBuild,
|
|
12
9
|
type SupervisorServiceStatusRecord,
|
|
13
10
|
type SupervisorStatus,
|
|
14
11
|
} from "./contract.ts";
|
|
@@ -16,49 +13,14 @@ import { readSupervisorServiceRecord, supervisorPaths } from "./storage.ts";
|
|
|
16
13
|
|
|
17
14
|
const STALE_HEARTBEAT_MS = 15_000;
|
|
18
15
|
|
|
19
|
-
export interface SupervisorProcessDependencies {
|
|
20
|
-
readToken?: (pid: number) => string | null;
|
|
21
|
-
probePid?: (pid: number) => unknown;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
export function installedSupervisorBuild(): SupervisorBuild {
|
|
25
|
-
let version = "unknown";
|
|
26
|
-
let revision = "unknown";
|
|
27
|
-
try {
|
|
28
|
-
const pkg = JSON.parse(readFileSync(join(harneryPackageRoot(), "package.json"), "utf8"));
|
|
29
|
-
if (typeof pkg.version === "string") version = pkg.version;
|
|
30
|
-
revision = currentHarneryRuntimeBuild();
|
|
31
|
-
} catch {
|
|
32
|
-
// Missing build evidence must still produce an inspectable status.
|
|
33
|
-
}
|
|
34
|
-
return { version, revision };
|
|
35
|
-
}
|
|
36
|
-
|
|
37
16
|
export function readSupervisorStatus(
|
|
38
17
|
coordRoot: string,
|
|
39
18
|
nowMs = Date.now(),
|
|
40
|
-
dependencies:
|
|
19
|
+
dependencies: ServiceProcessDependencies = {},
|
|
41
20
|
): SupervisorStatus {
|
|
42
21
|
const paths = supervisorPaths(coordRoot);
|
|
43
22
|
const record = readSupervisorServiceRecord(coordRoot);
|
|
44
|
-
const
|
|
45
|
-
const runningBuild = validBuild(record?.running_build) ? record.running_build : null;
|
|
46
|
-
const restartNeeded =
|
|
47
|
-
!runningBuild ||
|
|
48
|
-
installedBuild.version === "unknown" ||
|
|
49
|
-
installedBuild.revision === "unknown" ||
|
|
50
|
-
runningBuild.version !== installedBuild.version ||
|
|
51
|
-
runningBuild.revision !== installedBuild.revision;
|
|
52
|
-
const build = {
|
|
53
|
-
running_build: runningBuild,
|
|
54
|
-
installed_build: installedBuild,
|
|
55
|
-
restart_needed: restartNeeded,
|
|
56
|
-
build_status: !runningBuild
|
|
57
|
-
? "running build unknown; restart needed"
|
|
58
|
-
: restartNeeded
|
|
59
|
-
? "restart needed"
|
|
60
|
-
: "current",
|
|
61
|
-
};
|
|
23
|
+
const build = serviceBuildStatus(record?.running_build);
|
|
62
24
|
if (!validRecord(record)) {
|
|
63
25
|
return {
|
|
64
26
|
running: false,
|
|
@@ -72,13 +34,11 @@ export function readSupervisorStatus(
|
|
|
72
34
|
}
|
|
73
35
|
const heartbeatAt = Date.parse(record.heartbeat_at);
|
|
74
36
|
const stale = !Number.isFinite(heartbeatAt) || nowMs - heartbeatAt > STALE_HEARTBEAT_MS;
|
|
75
|
-
const
|
|
76
|
-
const liveness =
|
|
77
|
-
owner.generation === "same" && record.state === "stopped" ? "stopped" : owner.liveness;
|
|
37
|
+
const { generation, liveness } = observeServiceProcess(record, dependencies);
|
|
78
38
|
return {
|
|
79
|
-
running: liveness === "running" && !stale
|
|
39
|
+
running: liveness === "running" && !stale,
|
|
80
40
|
stale,
|
|
81
|
-
generation
|
|
41
|
+
generation,
|
|
82
42
|
liveness,
|
|
83
43
|
...build,
|
|
84
44
|
record,
|
|
@@ -99,38 +59,3 @@ function validRecord(
|
|
|
99
59
|
typeof value.keep_alive === "boolean"
|
|
100
60
|
);
|
|
101
61
|
}
|
|
102
|
-
|
|
103
|
-
/** A stale heartbeat does not prove that a process has exited. */
|
|
104
|
-
export function supervisorProcessLiveness(
|
|
105
|
-
owner: { pid: number; host: string; start_token?: string },
|
|
106
|
-
dependencies: SupervisorProcessDependencies = {},
|
|
107
|
-
): { liveness: SupervisorStatus["liveness"]; generation: ProcessGenerationVerdict } {
|
|
108
|
-
if (owner.host !== hostname()) return { liveness: "unknown", generation: "unknown" };
|
|
109
|
-
try {
|
|
110
|
-
(dependencies.probePid ?? ((pid) => process.kill(pid, 0)))(owner.pid);
|
|
111
|
-
} catch (error) {
|
|
112
|
-
return {
|
|
113
|
-
liveness: (error as NodeJS.ErrnoException).code === "ESRCH" ? "stopped" : "unknown",
|
|
114
|
-
generation: "unknown",
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
const generation = verifyProcessGeneration(owner.pid, owner.start_token, dependencies.readToken);
|
|
118
|
-
return {
|
|
119
|
-
generation,
|
|
120
|
-
liveness:
|
|
121
|
-
generation === "same" ? "running" : generation === "different" ? "stopped" : "unknown",
|
|
122
|
-
};
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
function validBuild(value: unknown): value is SupervisorBuild {
|
|
126
|
-
if (!value || typeof value !== "object") return false;
|
|
127
|
-
const build = value as Partial<SupervisorBuild>;
|
|
128
|
-
return (
|
|
129
|
-
typeof build.version === "string" &&
|
|
130
|
-
!!build.version &&
|
|
131
|
-
build.version !== "unknown" &&
|
|
132
|
-
typeof build.revision === "string" &&
|
|
133
|
-
!!build.revision &&
|
|
134
|
-
build.revision !== "unknown"
|
|
135
|
-
);
|
|
136
|
-
}
|
|
@@ -53,8 +53,8 @@ import {
|
|
|
53
53
|
normalizeWorkflowMeta,
|
|
54
54
|
writeWorkflowProof,
|
|
55
55
|
} from "./proof.ts";
|
|
56
|
+
import { acquireWorkflowResumeLease } from "./resume-lock.ts";
|
|
56
57
|
import {
|
|
57
|
-
acquireWorkflowResumeLease,
|
|
58
58
|
assertWorkflowRunResumable,
|
|
59
59
|
assertWorkflowScriptUnchanged,
|
|
60
60
|
WORKFLOW_RUN_MANIFEST_SCHEMA_VERSION,
|
|
@@ -39,23 +39,12 @@ export {
|
|
|
39
39
|
rollupAcceptance,
|
|
40
40
|
writeWorkflowProof,
|
|
41
41
|
} from "./proof.ts";
|
|
42
|
-
export
|
|
43
|
-
|
|
44
|
-
WorkflowRecoveryIntent,
|
|
45
|
-
WorkflowRecoveryRecord,
|
|
46
|
-
} from "./recovery-audit.ts";
|
|
47
|
-
export { beginWorkflowRecovery, readWorkflowRecoveryLedger } from "./recovery-audit.ts";
|
|
48
|
-
export type {
|
|
49
|
-
WorkflowResumeRecoveryOptions,
|
|
50
|
-
WorkflowResumeRecoveryResult,
|
|
51
|
-
WorkflowRunManifest,
|
|
52
|
-
} from "./run-state.ts";
|
|
42
|
+
export { acquireWorkflowResumeLease } from "./resume-lock.ts";
|
|
43
|
+
export type { WorkflowRunManifest } from "./run-state.ts";
|
|
53
44
|
export {
|
|
54
|
-
acquireWorkflowResumeLease,
|
|
55
45
|
assertWorkflowRunResumable,
|
|
56
46
|
assertWorkflowScriptUnchanged,
|
|
57
47
|
readWorkflowRunManifest,
|
|
58
|
-
recoverWorkflowResumeLease,
|
|
59
48
|
WORKFLOW_RUN_MANIFEST_SCHEMA_VERSION,
|
|
60
49
|
workflowRunManifestPath,
|
|
61
50
|
workflowScriptDigest,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { type RepoSnapshot, repoComparisonProblems } from "../context/observation.ts";
|
|
5
5
|
import { DIAGNOSTIC_ADVICE_SCHEMA_VERSION } from "../diagnostics/contract.ts";
|
|
6
6
|
import type {
|
|
7
7
|
NormalizedPolicy,
|
|
@@ -399,15 +399,8 @@ export function readWorkflowProof(coordRoot: string, runId: string): WorkflowPro
|
|
|
399
399
|
) {
|
|
400
400
|
throw new Error(`workflow proof at ${path} has an unsupported or mismatched schema`);
|
|
401
401
|
}
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
proof.repository.before.dirty_paths_truncated ||
|
|
405
|
-
proof.repository.after.dirty_paths_truncated ||
|
|
406
|
-
!proof.repository.before.branch?.trim() ||
|
|
407
|
-
!proof.repository.after.branch?.trim() ||
|
|
408
|
-
!proof.repository.before.head?.trim() ||
|
|
409
|
-
!proof.repository.after.head?.trim()
|
|
410
|
-
) {
|
|
402
|
+
// Packets written before observation checks may carry unproven drift.
|
|
403
|
+
if (repoProblems(proof.repository).length > 0) {
|
|
411
404
|
proof.repository = buildRepoEvidence(proof.repository.before, proof.repository.after);
|
|
412
405
|
proof.unknowns = proof.unknowns.filter((item) => item.code !== "repository_drift_incomplete");
|
|
413
406
|
proof.unknowns.push({
|
|
@@ -528,29 +521,16 @@ export function renderWorkflowProof(proof: WorkflowProof): string {
|
|
|
528
521
|
: "admission: shadow not needed; no real child dispatch",
|
|
529
522
|
);
|
|
530
523
|
}
|
|
531
|
-
const
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
comparable &&
|
|
535
|
-
!proof.repository.before.dirty_paths_truncated &&
|
|
536
|
-
!proof.repository.after.dirty_paths_truncated &&
|
|
537
|
-
proof.repository.before.branch?.trim() &&
|
|
538
|
-
proof.repository.after.branch?.trim() &&
|
|
539
|
-
proof.repository.before.head?.trim() &&
|
|
540
|
-
proof.repository.after.head?.trim()
|
|
541
|
-
? proof.repository
|
|
542
|
-
: buildRepoEvidence(proof.repository.before, proof.repository.after);
|
|
543
|
-
const drift = repo.drift;
|
|
544
|
-
if (!comparable) {
|
|
545
|
-
lines.push(`repository: drift unknown. ${drift.note}`);
|
|
524
|
+
const problems = repoProblems(proof.repository);
|
|
525
|
+
if (problems.length > 0) {
|
|
526
|
+
lines.push(`repository: drift unknown. ${unverifiedDrift(problems)}`);
|
|
546
527
|
} else {
|
|
547
|
-
const
|
|
548
|
-
drift.dirty_paths_added === null || drift.dirty_paths_cleared === null
|
|
549
|
-
? "dirty-path drift unknown"
|
|
550
|
-
: `${drift.dirty_paths_added.length} dirty added, ${drift.dirty_paths_cleared.length} cleared`;
|
|
528
|
+
const { before, after, drift } = proof.repository;
|
|
551
529
|
lines.push(
|
|
552
|
-
`repository: branch ${
|
|
553
|
-
`HEAD ${short(
|
|
530
|
+
`repository: branch ${before.branch} -> ${after.branch}; ` +
|
|
531
|
+
`HEAD ${short(before.head)} -> ${short(after.head)}; ` +
|
|
532
|
+
`${drift.dirty_paths_added?.length ?? "unknown"} dirty added, ` +
|
|
533
|
+
`${drift.dirty_paths_cleared?.length ?? "unknown"} cleared`,
|
|
554
534
|
);
|
|
555
535
|
}
|
|
556
536
|
if (proof.unknowns.length > 0) {
|
|
@@ -564,8 +544,8 @@ export function renderWorkflowProof(proof: WorkflowProof): string {
|
|
|
564
544
|
function buildRepoEvidence(beforeRaw: RepoSnapshot, afterRaw: RepoSnapshot): WorkflowRepoEvidence {
|
|
565
545
|
const before = normalizeRepoSnapshot(beforeRaw);
|
|
566
546
|
const after = normalizeRepoSnapshot(afterRaw);
|
|
567
|
-
const
|
|
568
|
-
if (
|
|
547
|
+
const problems = repoProblems({ before, after });
|
|
548
|
+
if (problems.length > 0) {
|
|
569
549
|
return {
|
|
570
550
|
source: "engine",
|
|
571
551
|
before,
|
|
@@ -577,47 +557,43 @@ function buildRepoEvidence(beforeRaw: RepoSnapshot, afterRaw: RepoSnapshot): Wor
|
|
|
577
557
|
dirty_paths_cleared: null,
|
|
578
558
|
dirty_paths_retained: null,
|
|
579
559
|
incomplete: true,
|
|
580
|
-
note:
|
|
560
|
+
note: unverifiedDrift(problems),
|
|
581
561
|
},
|
|
582
562
|
};
|
|
583
563
|
}
|
|
584
|
-
const pathsComparable = !before.dirty_paths_truncated && !after.dirty_paths_truncated;
|
|
585
|
-
const branchKnown = Boolean(before.branch && after.branch);
|
|
586
|
-
const headKnown = Boolean(before.head && after.head);
|
|
587
564
|
const beforeDirty = new Set(before.dirty_paths);
|
|
588
565
|
const afterDirty = new Set(after.dirty_paths);
|
|
589
566
|
const retained = after.dirty_paths.filter((path) => beforeDirty.has(path));
|
|
590
|
-
const
|
|
591
|
-
...(!pathsComparable
|
|
592
|
-
? ["Dirty-path drift is unknown because a snapshot list was truncated."]
|
|
593
|
-
: []),
|
|
594
|
-
...(!branchKnown ? ["Branch drift is unknown because a snapshot has no branch."] : []),
|
|
595
|
-
...(!headKnown ? ["HEAD drift is unknown because a snapshot has no HEAD."] : []),
|
|
596
|
-
...(pathsComparable && retained.length > 0
|
|
597
|
-
? ["Snapshots cannot prove whether retained dirty paths changed during the run."]
|
|
598
|
-
: []),
|
|
599
|
-
];
|
|
600
|
-
const incomplete = notes.length > 0;
|
|
567
|
+
const incomplete = retained.length > 0;
|
|
601
568
|
return {
|
|
602
569
|
source: "engine",
|
|
603
570
|
before,
|
|
604
571
|
after,
|
|
605
572
|
drift: {
|
|
606
|
-
branch_changed:
|
|
607
|
-
head_changed:
|
|
608
|
-
dirty_paths_added:
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
dirty_paths_cleared: pathsComparable
|
|
612
|
-
? before.dirty_paths.filter((path) => !afterDirty.has(path))
|
|
613
|
-
: null,
|
|
614
|
-
dirty_paths_retained: pathsComparable ? retained : null,
|
|
573
|
+
branch_changed: before.branch !== after.branch,
|
|
574
|
+
head_changed: before.head !== after.head,
|
|
575
|
+
dirty_paths_added: after.dirty_paths.filter((path) => !beforeDirty.has(path)),
|
|
576
|
+
dirty_paths_cleared: before.dirty_paths.filter((path) => !afterDirty.has(path)),
|
|
577
|
+
dirty_paths_retained: retained,
|
|
615
578
|
incomplete,
|
|
616
|
-
note: incomplete
|
|
579
|
+
note: incomplete
|
|
580
|
+
? "Snapshots cannot prove whether retained dirty paths changed during the run."
|
|
581
|
+
: undefined,
|
|
617
582
|
},
|
|
618
583
|
};
|
|
619
584
|
}
|
|
620
585
|
|
|
586
|
+
function repoProblems(repository: {
|
|
587
|
+
before: WorkflowRepoSnapshot;
|
|
588
|
+
after: WorkflowRepoSnapshot;
|
|
589
|
+
}): string[] {
|
|
590
|
+
return repoComparisonProblems(repository.before, repository.after, ["before", "after"]);
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
function unverifiedDrift(problems: string[]): string {
|
|
594
|
+
return `Repository drift could not be verified: ${problems.join("; ")}. Re-check before editing.`;
|
|
595
|
+
}
|
|
596
|
+
|
|
621
597
|
function normalizeRepoSnapshot(snapshot: RepoSnapshot): WorkflowRepoSnapshot {
|
|
622
598
|
return {
|
|
623
599
|
cwd: snapshot.cwd,
|
|
@@ -633,30 +609,6 @@ function normalizeRepoSnapshot(snapshot: RepoSnapshot): WorkflowRepoSnapshot {
|
|
|
633
609
|
};
|
|
634
610
|
}
|
|
635
611
|
|
|
636
|
-
function snapshotFailure(label: string, snapshot: WorkflowRepoSnapshot): string {
|
|
637
|
-
return `${label} snapshot is ${snapshot.observation}${snapshot.observation_detail ? ` (${snapshot.observation_detail})` : ""}`;
|
|
638
|
-
}
|
|
639
|
-
|
|
640
|
-
function repoComparisonProblem(
|
|
641
|
-
before: WorkflowRepoSnapshot,
|
|
642
|
-
after: WorkflowRepoSnapshot,
|
|
643
|
-
): string | undefined {
|
|
644
|
-
if (!observationsComparable(before, after)) {
|
|
645
|
-
return [
|
|
646
|
-
...(before.observation !== "ok" ? [snapshotFailure("before", before)] : []),
|
|
647
|
-
...(after.observation !== "ok" ? [snapshotFailure("after", after)] : []),
|
|
648
|
-
].join("; ");
|
|
649
|
-
}
|
|
650
|
-
const unknownRoots = [
|
|
651
|
-
...(!before.root || !isAbsolute(before.root) ? ["before"] : []),
|
|
652
|
-
...(!after.root || !isAbsolute(after.root) ? ["after"] : []),
|
|
653
|
-
];
|
|
654
|
-
if (unknownRoots.length > 0)
|
|
655
|
-
return `${unknownRoots.join(" and ")} snapshot repository root is unknown`;
|
|
656
|
-
if (before.root !== after.root) return "snapshots name different repository roots";
|
|
657
|
-
return undefined;
|
|
658
|
-
}
|
|
659
|
-
|
|
660
612
|
function buildAdapterCoverage(
|
|
661
613
|
agents: WorkflowAgentProof[],
|
|
662
614
|
claims: Readonly<Record<string, AdapterEvidenceCapability | undefined>> | undefined,
|