harnery 0.45.1 → 0.45.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +6 -45
- package/dist/core/agents/coord-client.d.ts.map +1 -1
- package/dist/core/agents/coord-client.js +0 -18
- package/dist/core/agents/state/pidmap.js +2 -2
- package/dist/core/agents/state/proc-start.d.ts +20 -31
- package/dist/core/agents/state/proc-start.d.ts.map +1 -1
- package/dist/core/agents/state/proc-start.js +37 -70
- package/dist/core/agents/state/service-process.d.ts +61 -0
- package/dist/core/agents/state/service-process.d.ts.map +1 -0
- package/dist/core/agents/state/service-process.js +143 -0
- package/dist/core/context/index.d.ts +6 -4
- package/dist/core/context/index.d.ts.map +1 -1
- package/dist/core/context/index.js +194 -256
- package/dist/core/context/observation.d.ts +9 -9
- package/dist/core/context/observation.d.ts.map +1 -1
- package/dist/core/context/observation.js +37 -6
- package/dist/core/events/v3/runtime-build.d.ts +14 -0
- package/dist/core/events/v3/runtime-build.d.ts.map +1 -1
- package/dist/core/events/v3/runtime-build.js +44 -0
- package/dist/core/governor/index.d.ts +2 -1
- package/dist/core/governor/index.d.ts.map +1 -1
- package/dist/core/governor/index.js +2 -1
- package/dist/core/governor/service-read.d.ts +3 -10
- package/dist/core/governor/service-read.d.ts.map +1 -1
- package/dist/core/governor/service-read.js +10 -94
- package/dist/core/governor/service.d.ts +3 -7
- package/dist/core/governor/service.d.ts.map +1 -1
- package/dist/core/governor/service.js +17 -182
- package/dist/core/hooks/resolve/owner.js +2 -2
- package/dist/core/resources/sampler.js +2 -2
- package/dist/core/resources/service-status.d.ts +2 -14
- package/dist/core/resources/service-status.d.ts.map +1 -1
- package/dist/core/resources/service-status.js +3 -23
- package/dist/core/resources/service.d.ts +3 -7
- package/dist/core/resources/service.d.ts.map +1 -1
- package/dist/core/resources/service.js +12 -101
- package/dist/core/resources/status.d.ts.map +1 -1
- package/dist/core/resources/status.js +3 -2
- package/dist/core/semantic/service.d.ts.map +1 -1
- package/dist/core/semantic/service.js +9 -2
- package/dist/core/servers/index.d.ts +0 -2
- package/dist/core/servers/index.d.ts.map +1 -1
- package/dist/core/servers/index.js +3 -18
- package/dist/core/supervisor/service.d.ts +4 -7
- package/dist/core/supervisor/service.d.ts.map +1 -1
- package/dist/core/supervisor/service.js +23 -122
- package/dist/core/supervisor/services.js +2 -2
- package/dist/core/supervisor/status.d.ts +3 -17
- package/dist/core/supervisor/status.d.ts.map +1 -1
- package/dist/core/supervisor/status.js +7 -70
- package/dist/core/workflow/engine.js +2 -1
- package/dist/core/workflow/index.d.ts +3 -4
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +2 -2
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +30 -74
- package/dist/core/workflow/resume-lock.d.ts +17 -0
- package/dist/core/workflow/resume-lock.d.ts.map +1 -0
- package/dist/core/workflow/resume-lock.js +158 -0
- package/dist/core/workflow/run-state.d.ts +0 -21
- package/dist/core/workflow/run-state.d.ts.map +1 -1
- package/dist/core/workflow/run-state.js +3 -174
- package/dist/core/workflow/workspaces/leases.d.ts +4 -0
- package/dist/core/workflow/workspaces/leases.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/leases.js +1 -1
- package/dist/core/workflow/workspaces/local-git.js +91 -210
- package/dist/core/workflow/workspaces/ownership.d.ts +0 -7
- package/dist/core/workflow/workspaces/ownership.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/ownership.js +3 -130
- package/dist/core/workflow/workspaces/reclaim.d.ts +4 -7
- package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/reclaim.js +0 -4
- package/dist/core/workflow/workspaces/recovery.d.ts +1 -1
- package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.js +1 -1
- package/dist/core/workflow/workspaces/state.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/state.js +1 -4
- package/package.json +1 -1
- package/src/commands/workflow.ts +8 -58
- package/src/core/agents/coord-client.ts +0 -20
- package/src/core/agents/state/pidmap.ts +2 -2
- package/src/core/agents/state/proc-start.ts +41 -69
- package/src/core/agents/state/service-process.ts +217 -0
- package/src/core/context/index.ts +207 -272
- package/src/core/context/observation.ts +40 -9
- package/src/core/events/v3/runtime-build.ts +62 -0
- package/src/core/governor/index.ts +1 -2
- package/src/core/governor/service-read.ts +15 -114
- package/src/core/governor/service.ts +36 -220
- package/src/core/hooks/resolve/owner.ts +2 -2
- package/src/core/resources/sampler.ts +3 -2
- package/src/core/resources/service-status.ts +6 -35
- package/src/core/resources/service.ts +23 -133
- package/src/core/resources/status.ts +3 -2
- package/src/core/semantic/service.ts +8 -2
- package/src/core/servers/index.ts +3 -23
- package/src/core/supervisor/service.ts +33 -155
- package/src/core/supervisor/services.ts +2 -2
- package/src/core/supervisor/status.ts +10 -85
- package/src/core/workflow/engine.ts +1 -1
- package/src/core/workflow/index.ts +2 -13
- package/src/core/workflow/proof.ts +35 -83
- package/src/core/workflow/resume-lock.ts +179 -0
- package/src/core/workflow/run-state.ts +3 -207
- package/src/core/workflow/workspaces/leases.ts +1 -1
- package/src/core/workflow/workspaces/local-git.ts +120 -239
- package/src/core/workflow/workspaces/ownership.ts +3 -158
- package/src/core/workflow/workspaces/reclaim.ts +3 -9
- package/src/core/workflow/workspaces/recovery.ts +1 -1
- package/src/core/workflow/workspaces/state.ts +1 -5
- package/dist/core/agents/state/process-handle.d.ts +0 -26
- package/dist/core/agents/state/process-handle.d.ts.map +0 -1
- package/dist/core/agents/state/process-handle.js +0 -121
- package/dist/core/workflow/recovery-audit.d.ts +0 -30
- package/dist/core/workflow/recovery-audit.d.ts.map +0 -1
- package/dist/core/workflow/recovery-audit.js +0 -116
- package/src/core/agents/state/process-handle.ts +0 -165
- package/src/core/workflow/recovery-audit.ts +0 -181
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Repository observation contract shared by context recovery and workflow proof.
|
|
3
3
|
*
|
|
4
|
-
* This module
|
|
5
|
-
* dashboard, proof readers) can use it without reaching the
|
|
6
|
-
* snapshot code in ./index.ts.
|
|
4
|
+
* This module runs no Git and touches no files, so read-only consumers (the
|
|
5
|
+
* Codec dashboard, proof readers) can use it without reaching the
|
|
6
|
+
* Git-running snapshot code in ./index.ts.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
+
import { isAbsolute } from "node:path";
|
|
10
|
+
|
|
9
11
|
export type RepoObservation = "ok" | "unavailable" | "partial";
|
|
10
12
|
|
|
11
13
|
export interface RepoSnapshot {
|
|
@@ -20,10 +22,39 @@ export interface RepoSnapshot {
|
|
|
20
22
|
dirty_paths_truncated?: boolean;
|
|
21
23
|
}
|
|
22
24
|
|
|
23
|
-
/**
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
25
|
+
/**
|
|
26
|
+
* Every reason two snapshots cannot establish repository drift. Only an empty
|
|
27
|
+
* list lets a caller compare branch, HEAD and dirty paths; anything else
|
|
28
|
+
* leaves the whole comparison unknown.
|
|
29
|
+
*/
|
|
30
|
+
export function repoComparisonProblems(
|
|
31
|
+
a: RepoSnapshot,
|
|
32
|
+
b: RepoSnapshot,
|
|
33
|
+
labels: readonly [string, string],
|
|
34
|
+
): string[] {
|
|
35
|
+
const sides = [
|
|
36
|
+
[labels[0], a],
|
|
37
|
+
[labels[1], b],
|
|
38
|
+
] as const;
|
|
39
|
+
const failed = sides.flatMap(([label, repo]) =>
|
|
40
|
+
repo.observation === "ok"
|
|
41
|
+
? []
|
|
42
|
+
: [
|
|
43
|
+
// Snapshots recorded before observations existed carry none.
|
|
44
|
+
`${label} snapshot is ${repo.observation ?? "unavailable"}${repo.observation_detail ? ` (${repo.observation_detail})` : ""}`,
|
|
45
|
+
],
|
|
46
|
+
);
|
|
47
|
+
if (failed.length > 0) return failed;
|
|
48
|
+
const problems: string[] = [];
|
|
49
|
+
if (a.dirty_paths_truncated || b.dirty_paths_truncated) {
|
|
50
|
+
problems.push("dirty path list is truncated");
|
|
51
|
+
}
|
|
52
|
+
if (!a.root || !b.root || !isAbsolute(a.root) || !isAbsolute(b.root)) {
|
|
53
|
+
problems.push("repository root is unknown");
|
|
54
|
+
} else if (a.root !== b.root) {
|
|
55
|
+
problems.push("repository roots differ");
|
|
56
|
+
}
|
|
57
|
+
if (!a.branch?.trim() || !b.branch?.trim()) problems.push("branch is unknown");
|
|
58
|
+
if (!a.head?.trim() || !b.head?.trim()) problems.push("HEAD is unknown");
|
|
59
|
+
return problems;
|
|
29
60
|
}
|
|
@@ -40,3 +40,65 @@ export function currentHarneryRuntimeBuild(): string {
|
|
|
40
40
|
export function harneryPackageRoot(): string {
|
|
41
41
|
return resolve(dirname(fileURLToPath(import.meta.url)), "../../../..");
|
|
42
42
|
}
|
|
43
|
+
|
|
44
|
+
export interface HarneryBuild {
|
|
45
|
+
version: string;
|
|
46
|
+
revision: string;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The installed package version and runtime revision; "unknown" when unreadable. */
|
|
50
|
+
export function installedHarneryBuild(): HarneryBuild {
|
|
51
|
+
let version = "unknown";
|
|
52
|
+
let revision = "unknown";
|
|
53
|
+
try {
|
|
54
|
+
const manifest = JSON.parse(
|
|
55
|
+
readFileSync(resolve(harneryPackageRoot(), "package.json"), "utf8"),
|
|
56
|
+
) as { version?: unknown };
|
|
57
|
+
if (typeof manifest.version === "string" && manifest.version.length <= 100)
|
|
58
|
+
version = manifest.version;
|
|
59
|
+
} catch {
|
|
60
|
+
// Missing package metadata cannot establish that a running build is current.
|
|
61
|
+
}
|
|
62
|
+
try {
|
|
63
|
+
revision = currentHarneryRuntimeBuild();
|
|
64
|
+
} catch {
|
|
65
|
+
// Status still reports the unavailable build instead of guessing a match.
|
|
66
|
+
}
|
|
67
|
+
return { version, revision };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export interface ServiceBuildStatus {
|
|
71
|
+
running_build: HarneryBuild | null;
|
|
72
|
+
installed_build: HarneryBuild;
|
|
73
|
+
restart_needed: boolean;
|
|
74
|
+
build_status: string;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** A missing or unknown running build, or any difference, means restart. */
|
|
78
|
+
export function serviceBuildStatus(recorded: unknown): ServiceBuildStatus {
|
|
79
|
+
const installed_build = installedHarneryBuild();
|
|
80
|
+
const candidate = recorded as Partial<HarneryBuild> | undefined;
|
|
81
|
+
const known = (value: unknown): value is string =>
|
|
82
|
+
typeof value === "string" && value.length > 0 && value.length <= 100 && value !== "unknown";
|
|
83
|
+
const running_build =
|
|
84
|
+
candidate && known(candidate.version) && known(candidate.revision)
|
|
85
|
+
? { version: candidate.version, revision: candidate.revision }
|
|
86
|
+
: null;
|
|
87
|
+
const restart_needed =
|
|
88
|
+
running_build === null ||
|
|
89
|
+
installed_build.version === "unknown" ||
|
|
90
|
+
installed_build.revision === "unknown" ||
|
|
91
|
+
running_build.version !== installed_build.version ||
|
|
92
|
+
running_build.revision !== installed_build.revision;
|
|
93
|
+
return {
|
|
94
|
+
running_build,
|
|
95
|
+
installed_build,
|
|
96
|
+
restart_needed,
|
|
97
|
+
build_status:
|
|
98
|
+
running_build === null
|
|
99
|
+
? "running build unknown; restart needed"
|
|
100
|
+
: restart_needed
|
|
101
|
+
? "restart needed"
|
|
102
|
+
: "current",
|
|
103
|
+
};
|
|
104
|
+
}
|
|
@@ -63,13 +63,12 @@ export {
|
|
|
63
63
|
type RunGovernorServiceDaemonInput,
|
|
64
64
|
type RunGovernorServiceSweepInput,
|
|
65
65
|
readGovernorServiceConfig,
|
|
66
|
-
readGovernorServiceRuntime,
|
|
67
|
-
readGovernorServiceStatus,
|
|
68
66
|
requestGovernorServiceStop,
|
|
69
67
|
runGovernorServiceDaemon,
|
|
70
68
|
runGovernorServiceSweep,
|
|
71
69
|
spawnGovernorService,
|
|
72
70
|
} from "./service.ts";
|
|
71
|
+
export { readGovernorServiceRuntime, readGovernorServiceStatus } from "./service-read.ts";
|
|
73
72
|
export {
|
|
74
73
|
assertGovernorId,
|
|
75
74
|
type CreateGovernorInput,
|
|
@@ -1,21 +1,20 @@
|
|
|
1
1
|
import { closeSync, existsSync, openSync, readFileSync, readSync, statSync } from "node:fs";
|
|
2
|
-
import { hostname } from "node:os";
|
|
3
2
|
import { join, resolve } from "node:path";
|
|
4
3
|
import { gunzipSync } from "node:zlib";
|
|
5
4
|
import { coordEnv } from "../../lib/env.ts";
|
|
5
|
+
import type { ProcessGenerationVerdict } from "../agents/state/proc-start.ts";
|
|
6
6
|
import {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
7
|
+
observeServiceProcess,
|
|
8
|
+
type ServiceLiveness,
|
|
9
|
+
type ServiceProcessDependencies,
|
|
10
|
+
} from "../agents/state/service-process.ts";
|
|
11
|
+
import { serviceBuildStatus } from "../events/v3/runtime-build.ts";
|
|
11
12
|
import { createStorageCatalog } from "../storage/catalog.ts";
|
|
12
13
|
import type { HarneryLogRecordV1 } from "../storage/jsonl.ts";
|
|
13
14
|
import { parseLogRecord } from "../storage/jsonl.ts";
|
|
14
15
|
import { familyLogDirectory, readSegmentManifest } from "../storage/segments.ts";
|
|
15
16
|
import type {
|
|
16
|
-
GovernorServiceBuild,
|
|
17
17
|
GovernorServiceConfig,
|
|
18
|
-
GovernorServiceProcessDependencies,
|
|
19
18
|
GovernorServiceRuntime,
|
|
20
19
|
GovernorServiceStatus,
|
|
21
20
|
GovernorServiceStatusRecord,
|
|
@@ -25,106 +24,6 @@ const MAX_FILE_BYTES = 512 * 1024;
|
|
|
25
24
|
const MAX_LOG_RECORDS = 10_000;
|
|
26
25
|
const MAX_GOALS = 100;
|
|
27
26
|
|
|
28
|
-
export function currentGovernorServiceBuild(): GovernorServiceBuild {
|
|
29
|
-
let version = "unknown";
|
|
30
|
-
let revision = "unknown";
|
|
31
|
-
try {
|
|
32
|
-
const manifest = JSON.parse(
|
|
33
|
-
readFileSync(new URL("../../../package.json", import.meta.url), "utf8"),
|
|
34
|
-
) as {
|
|
35
|
-
version?: unknown;
|
|
36
|
-
};
|
|
37
|
-
if (typeof manifest.version === "string" && manifest.version.length <= 100) {
|
|
38
|
-
version = manifest.version;
|
|
39
|
-
}
|
|
40
|
-
} catch {
|
|
41
|
-
// Missing package metadata cannot establish that the running build is current.
|
|
42
|
-
}
|
|
43
|
-
try {
|
|
44
|
-
revision = currentHarneryRuntimeBuild();
|
|
45
|
-
} catch {
|
|
46
|
-
// Status still reports the unavailable build instead of guessing a match.
|
|
47
|
-
}
|
|
48
|
-
return { version, revision };
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
export function governorServiceBuildStatus(
|
|
52
|
-
record: GovernorServiceStatusRecord | undefined,
|
|
53
|
-
): Pick<
|
|
54
|
-
GovernorServiceStatus,
|
|
55
|
-
"running_build" | "installed_build" | "restart_needed" | "build_status"
|
|
56
|
-
> {
|
|
57
|
-
const installed_build = currentGovernorServiceBuild();
|
|
58
|
-
const candidate = record?.running_build;
|
|
59
|
-
const running_build =
|
|
60
|
-
candidate &&
|
|
61
|
-
typeof candidate.version === "string" &&
|
|
62
|
-
candidate.version.length > 0 &&
|
|
63
|
-
candidate.version.length <= 100 &&
|
|
64
|
-
typeof candidate.revision === "string" &&
|
|
65
|
-
candidate.revision.length > 0 &&
|
|
66
|
-
candidate.revision.length <= 100 &&
|
|
67
|
-
candidate.version !== "unknown" &&
|
|
68
|
-
candidate.revision !== "unknown"
|
|
69
|
-
? candidate
|
|
70
|
-
: null;
|
|
71
|
-
const restart_needed =
|
|
72
|
-
running_build === null ||
|
|
73
|
-
installed_build.version === "unknown" ||
|
|
74
|
-
installed_build.revision === "unknown" ||
|
|
75
|
-
running_build.version !== installed_build.version ||
|
|
76
|
-
running_build.revision !== installed_build.revision;
|
|
77
|
-
return {
|
|
78
|
-
running_build,
|
|
79
|
-
installed_build,
|
|
80
|
-
restart_needed,
|
|
81
|
-
build_status:
|
|
82
|
-
running_build === null
|
|
83
|
-
? "running build unknown; restart needed"
|
|
84
|
-
: restart_needed
|
|
85
|
-
? "restart needed"
|
|
86
|
-
: "current",
|
|
87
|
-
};
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/** An unknown token or process probe cannot establish a running or stopped owner. */
|
|
91
|
-
export function observeGovernorServiceProcess(
|
|
92
|
-
record: Pick<GovernorServiceStatusRecord, "pid" | "host" | "state" | "start_token"> | undefined,
|
|
93
|
-
dependencies: GovernorServiceProcessDependencies = {},
|
|
94
|
-
): { generation: ProcessGenerationVerdict; liveness: GovernorServiceStatus["liveness"] } {
|
|
95
|
-
if (!record) return { generation: "unknown", liveness: "stopped" };
|
|
96
|
-
const generation =
|
|
97
|
-
record.host === hostname()
|
|
98
|
-
? verifyProcessGeneration(record.pid, record.start_token, dependencies.readToken)
|
|
99
|
-
: "unknown";
|
|
100
|
-
if (record.host !== hostname()) return { generation, liveness: "unknown" };
|
|
101
|
-
try {
|
|
102
|
-
const present = (dependencies.probePid ?? probePid)(record.pid);
|
|
103
|
-
if (!present) return { generation, liveness: "stopped" };
|
|
104
|
-
} catch (error) {
|
|
105
|
-
return {
|
|
106
|
-
generation,
|
|
107
|
-
liveness: (error as NodeJS.ErrnoException).code === "ESRCH" ? "stopped" : "unknown",
|
|
108
|
-
};
|
|
109
|
-
}
|
|
110
|
-
return {
|
|
111
|
-
generation,
|
|
112
|
-
liveness:
|
|
113
|
-
generation === "same"
|
|
114
|
-
? record.state === "stopped"
|
|
115
|
-
? "stopped"
|
|
116
|
-
: "running"
|
|
117
|
-
: generation === "different"
|
|
118
|
-
? "stopped"
|
|
119
|
-
: "unknown",
|
|
120
|
-
};
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
function probePid(pid: number): boolean {
|
|
124
|
-
process.kill(pid, 0);
|
|
125
|
-
return true;
|
|
126
|
-
}
|
|
127
|
-
|
|
128
27
|
export function readGovernorServiceConfig(coordRootRaw: string): GovernorServiceConfig {
|
|
129
28
|
const coordRoot = resolve(coordRootRaw);
|
|
130
29
|
const config = readBoundedJson<GovernorServiceConfig>(
|
|
@@ -175,7 +74,7 @@ export function readGovernorServiceRuntime(
|
|
|
175
74
|
|
|
176
75
|
export function readGovernorServiceStatus(
|
|
177
76
|
coordRootRaw: string,
|
|
178
|
-
dependencies:
|
|
77
|
+
dependencies: ServiceProcessDependencies = {},
|
|
179
78
|
): GovernorServiceStatus {
|
|
180
79
|
const coordRoot = resolve(coordRootRaw);
|
|
181
80
|
const record = readStatusRecord(coordRoot);
|
|
@@ -191,15 +90,17 @@ export function readGovernorServiceStatus(
|
|
|
191
90
|
} catch {
|
|
192
91
|
// Runtime is recoverable and must not make the dashboard unreadable.
|
|
193
92
|
}
|
|
194
|
-
const observation =
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
93
|
+
const observation: { generation: ProcessGenerationVerdict; liveness: ServiceLiveness } = record
|
|
94
|
+
? observeServiceProcess(record, dependencies)
|
|
95
|
+
: {
|
|
96
|
+
generation: "unknown",
|
|
97
|
+
liveness: existsSync(join(serviceDir(coordRoot), "status.json")) ? "unknown" : "stopped",
|
|
98
|
+
};
|
|
198
99
|
return {
|
|
199
|
-
running: observation.liveness === "running"
|
|
100
|
+
running: observation.liveness === "running",
|
|
200
101
|
stale: observation.liveness === "stopped" && record !== undefined && record.state !== "stopped",
|
|
201
102
|
...observation,
|
|
202
|
-
...
|
|
103
|
+
...serviceBuildStatus(record?.running_build),
|
|
203
104
|
record,
|
|
204
105
|
config,
|
|
205
106
|
runtime,
|
|
@@ -10,30 +10,28 @@ import {
|
|
|
10
10
|
renameSync,
|
|
11
11
|
rmSync,
|
|
12
12
|
statSync,
|
|
13
|
-
unlinkSync,
|
|
14
13
|
writeFileSync,
|
|
15
14
|
} from "node:fs";
|
|
16
15
|
import { hostname } from "node:os";
|
|
17
16
|
import { dirname, join, resolve } from "node:path";
|
|
18
17
|
import { coordEnv } from "../../lib/env.ts";
|
|
18
|
+
import { type ProcessGenerationVerdict, processStartToken } from "../agents/state/proc-start.ts";
|
|
19
19
|
import {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
20
|
+
acquireServiceLease,
|
|
21
|
+
requestServiceStop,
|
|
22
|
+
type ServiceLiveness,
|
|
23
|
+
type ServiceProcessDependencies,
|
|
24
|
+
serviceStartRefusal,
|
|
25
|
+
} from "../agents/state/service-process.ts";
|
|
25
26
|
import { resolveBinName } from "../config.ts";
|
|
27
|
+
import { installedHarneryBuild } from "../events/v3/runtime-build.ts";
|
|
26
28
|
import { type NormalizedPolicy, normalizePolicy, type PolicyIsolation } from "../policy/index.ts";
|
|
27
29
|
import { closeProcessLoggers, legacyLogFields, processLogger } from "../storage/logger.ts";
|
|
28
30
|
import { stateDirMode, stateFileMode } from "../storage/modes.ts";
|
|
29
31
|
import { RotatingTextSink } from "../storage/process-log.ts";
|
|
30
32
|
import type { RunWorkItemInput } from "../work/index.ts";
|
|
31
33
|
import { type GovernorRunReport, type GovernorStopReason, runGovernor } from "./runner.ts";
|
|
32
|
-
import {
|
|
33
|
-
currentGovernorServiceBuild,
|
|
34
|
-
governorServiceBuildStatus,
|
|
35
|
-
observeGovernorServiceProcess,
|
|
36
|
-
} from "./service-read.ts";
|
|
34
|
+
import { readGovernorServiceRuntime, readGovernorServiceStatus } from "./service-read.ts";
|
|
37
35
|
import { type GovernorRecord, readGovernor } from "./state.ts";
|
|
38
36
|
|
|
39
37
|
export const GOVERNOR_SERVICE_CONFIG_SCHEMA_VERSION = 1 as const;
|
|
@@ -101,10 +99,6 @@ export interface GovernorServiceBuild {
|
|
|
101
99
|
revision: string;
|
|
102
100
|
}
|
|
103
101
|
|
|
104
|
-
export interface GovernorServiceProcessDependencies extends ProcessGenerationSignalDependencies {
|
|
105
|
-
probePid?: (pid: number) => boolean;
|
|
106
|
-
}
|
|
107
|
-
|
|
108
102
|
export interface GovernorServiceStatusRecord {
|
|
109
103
|
schema_version: typeof GOVERNOR_SERVICE_STATUS_SCHEMA_VERSION;
|
|
110
104
|
pid: number;
|
|
@@ -173,27 +167,13 @@ export interface RunGovernorServiceDaemonInput {
|
|
|
173
167
|
onLog?: (line: string) => void;
|
|
174
168
|
}
|
|
175
169
|
|
|
176
|
-
interface ServiceLease {
|
|
177
|
-
pid: number;
|
|
178
|
-
start_token?: string;
|
|
179
|
-
running_build?: GovernorServiceBuild;
|
|
180
|
-
host: string;
|
|
181
|
-
nonce: string;
|
|
182
|
-
created_at: string;
|
|
183
|
-
}
|
|
184
|
-
|
|
185
170
|
export function configureGovernorService(
|
|
186
171
|
input: ConfigureGovernorServiceInput,
|
|
187
172
|
): GovernorServiceConfig {
|
|
188
173
|
const coordRoot = resolve(input.coordRoot);
|
|
189
|
-
const
|
|
190
|
-
if (
|
|
191
|
-
throw
|
|
192
|
-
}
|
|
193
|
-
if (active) {
|
|
194
|
-
const liveness = statusOwnerIsLive(active);
|
|
195
|
-
if (liveness !== "stopped") throw serviceStartRefusal(coordRoot, active.pid, liveness);
|
|
196
|
-
}
|
|
174
|
+
const current = readGovernorServiceStatus(coordRoot);
|
|
175
|
+
if (current.liveness !== "stopped")
|
|
176
|
+
throw startRefusal(coordRoot, current.record?.pid, current.liveness);
|
|
197
177
|
const goalIds = unique(input.goalIds.map((value) => value.trim()).filter(Boolean));
|
|
198
178
|
if (goalIds.length < 1 || goalIds.length > MAX_GOALS) {
|
|
199
179
|
throw new Error(`governor service requires from 1 to ${MAX_GOALS} unique goal ids`);
|
|
@@ -249,65 +229,12 @@ export function readGovernorServiceConfig(coordRootRaw: string): GovernorService
|
|
|
249
229
|
return config;
|
|
250
230
|
}
|
|
251
231
|
|
|
252
|
-
export function readGovernorServiceRuntime(
|
|
253
|
-
coordRootRaw: string,
|
|
254
|
-
): GovernorServiceRuntime | undefined {
|
|
255
|
-
const coordRoot = resolve(coordRootRaw);
|
|
256
|
-
if (!existsSync(serviceRuntimePath(coordRoot))) return undefined;
|
|
257
|
-
const runtime = readBoundedJson<GovernorServiceRuntime>(
|
|
258
|
-
serviceRuntimePath(coordRoot),
|
|
259
|
-
"governor service runtime",
|
|
260
|
-
);
|
|
261
|
-
if (
|
|
262
|
-
runtime.schema_version !== GOVERNOR_SERVICE_RUNTIME_SCHEMA_VERSION ||
|
|
263
|
-
!validTimestamp(runtime.config_created_at) ||
|
|
264
|
-
!validTimestamp(runtime.updated_at) ||
|
|
265
|
-
!runtime.goals ||
|
|
266
|
-
typeof runtime.goals !== "object" ||
|
|
267
|
-
Array.isArray(runtime.goals)
|
|
268
|
-
) {
|
|
269
|
-
throw new Error("governor service runtime has an unsupported schema");
|
|
270
|
-
}
|
|
271
|
-
return runtime;
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
export function readGovernorServiceStatus(
|
|
275
|
-
coordRootRaw: string,
|
|
276
|
-
dependencies: GovernorServiceProcessDependencies = {},
|
|
277
|
-
): GovernorServiceStatus {
|
|
278
|
-
const coordRoot = resolve(coordRootRaw);
|
|
279
|
-
const record = readStatusRecord(coordRoot);
|
|
280
|
-
let config: GovernorServiceConfig | undefined;
|
|
281
|
-
let runtime: GovernorServiceRuntime | undefined;
|
|
282
|
-
try {
|
|
283
|
-
config = readGovernorServiceConfig(coordRoot);
|
|
284
|
-
} catch {
|
|
285
|
-
// A service has no usable configuration until explicitly configured.
|
|
286
|
-
}
|
|
287
|
-
try {
|
|
288
|
-
runtime = readGovernorServiceRuntime(coordRoot);
|
|
289
|
-
} catch {
|
|
290
|
-
// Status remains readable when recoverable runtime state is corrupt.
|
|
291
|
-
}
|
|
292
|
-
const observation = observeGovernorServiceProcess(record, dependencies);
|
|
293
|
-
if (!record && existsSync(serviceStatusPath(coordRoot))) observation.liveness = "unknown";
|
|
294
|
-
return {
|
|
295
|
-
running: observation.liveness === "running" && record?.state !== "stopped",
|
|
296
|
-
stale: observation.liveness === "stopped" && record !== undefined && record.state !== "stopped",
|
|
297
|
-
...observation,
|
|
298
|
-
...governorServiceBuildStatus(record),
|
|
299
|
-
record,
|
|
300
|
-
config,
|
|
301
|
-
runtime,
|
|
302
|
-
};
|
|
303
|
-
}
|
|
304
|
-
|
|
305
232
|
export async function spawnGovernorService(coordRootRaw: string): Promise<GovernorServiceStatus> {
|
|
306
233
|
const coordRoot = resolve(coordRootRaw);
|
|
307
234
|
readGovernorServiceConfig(coordRoot);
|
|
308
235
|
const current = readGovernorServiceStatus(coordRoot);
|
|
309
236
|
if (current.liveness !== "stopped") {
|
|
310
|
-
throw
|
|
237
|
+
throw startRefusal(coordRoot, current.record?.pid, current.liveness);
|
|
311
238
|
}
|
|
312
239
|
mkdirSync(serviceDir(coordRoot), { recursive: true, mode: stateDirMode() });
|
|
313
240
|
const sharedLogs = coordEnv("SHARED_LOGS") !== "0";
|
|
@@ -347,31 +274,19 @@ export async function spawnGovernorService(coordRootRaw: string): Promise<Govern
|
|
|
347
274
|
|
|
348
275
|
export function requestGovernorServiceStop(
|
|
349
276
|
coordRootRaw: string,
|
|
350
|
-
dependencies:
|
|
277
|
+
dependencies: ServiceProcessDependencies = {},
|
|
351
278
|
): GovernorServiceStatus {
|
|
352
279
|
const coordRoot = resolve(coordRootRaw);
|
|
353
280
|
const status = readGovernorServiceStatus(coordRoot, dependencies);
|
|
354
281
|
if ((status.record?.state === "stopped" || !status.record) && status.liveness === "stopped") {
|
|
355
282
|
return { ...status, signal_sent: false, signal_verdict: status.generation };
|
|
356
283
|
}
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
? signalIfSameGeneration(
|
|
364
|
-
status.record.pid,
|
|
365
|
-
status.record.start_token,
|
|
366
|
-
"SIGTERM",
|
|
367
|
-
dependencies,
|
|
368
|
-
)
|
|
369
|
-
: { sent: false, verdict: status.generation };
|
|
370
|
-
return {
|
|
371
|
-
...readGovernorServiceStatus(coordRoot, dependencies),
|
|
372
|
-
signal_sent: signal.sent,
|
|
373
|
-
signal_verdict: signal.verdict,
|
|
374
|
-
};
|
|
284
|
+
return requestServiceStop(
|
|
285
|
+
status,
|
|
286
|
+
() => readGovernorServiceStatus(coordRoot, dependencies),
|
|
287
|
+
serviceStopPath(coordRoot),
|
|
288
|
+
dependencies,
|
|
289
|
+
);
|
|
375
290
|
}
|
|
376
291
|
|
|
377
292
|
export async function runGovernorServiceSweep(
|
|
@@ -519,22 +434,21 @@ export async function runGovernorServiceDaemon(
|
|
|
519
434
|
): Promise<GovernorServiceStatusRecord> {
|
|
520
435
|
const coordRoot = resolve(input.coordRoot);
|
|
521
436
|
const config = readGovernorServiceConfig(coordRoot);
|
|
522
|
-
const
|
|
523
|
-
if (
|
|
524
|
-
throw
|
|
525
|
-
}
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
const release = acquireServiceLease(coordRoot);
|
|
437
|
+
const current = readGovernorServiceStatus(coordRoot);
|
|
438
|
+
if (current.liveness !== "stopped")
|
|
439
|
+
throw startRefusal(coordRoot, current.record?.pid, current.liveness);
|
|
440
|
+
const { release } = acquireServiceLease(
|
|
441
|
+
serviceLeasePath(coordRoot),
|
|
442
|
+
(pid, liveness) => startRefusal(coordRoot, pid, liveness),
|
|
443
|
+
{ running_build: installedHarneryBuild() },
|
|
444
|
+
);
|
|
531
445
|
rmSync(serviceStopPath(coordRoot), { force: true });
|
|
532
446
|
const now = new Date().toISOString();
|
|
533
447
|
const status: GovernorServiceStatusRecord = {
|
|
534
448
|
schema_version: GOVERNOR_SERVICE_STATUS_SCHEMA_VERSION,
|
|
535
449
|
pid: process.pid,
|
|
536
450
|
start_token: processStartToken(process.pid) ?? undefined,
|
|
537
|
-
running_build:
|
|
451
|
+
running_build: installedHarneryBuild(),
|
|
538
452
|
host: hostname(),
|
|
539
453
|
nonce: randomUUID(),
|
|
540
454
|
state: "starting",
|
|
@@ -781,114 +695,16 @@ function normalizeIsolation(value: unknown): PolicyIsolation | undefined {
|
|
|
781
695
|
throw new Error("service isolation must be shared, worktree, sandbox, or remote");
|
|
782
696
|
}
|
|
783
697
|
|
|
784
|
-
function
|
|
785
|
-
const path = serviceLeasePath(coordRoot);
|
|
786
|
-
mkdirSync(dirname(path), { recursive: true, mode: stateDirMode() });
|
|
787
|
-
const owner: ServiceLease = {
|
|
788
|
-
pid: process.pid,
|
|
789
|
-
start_token: processStartToken(process.pid) ?? undefined,
|
|
790
|
-
running_build: currentGovernorServiceBuild(),
|
|
791
|
-
host: hostname(),
|
|
792
|
-
nonce: randomUUID(),
|
|
793
|
-
created_at: new Date().toISOString(),
|
|
794
|
-
};
|
|
795
|
-
const acquire = (): boolean => {
|
|
796
|
-
try {
|
|
797
|
-
const fd = openSync(path, "wx", stateFileMode());
|
|
798
|
-
try {
|
|
799
|
-
writeFileSync(fd, `${JSON.stringify(owner)}\n`, "utf8");
|
|
800
|
-
} finally {
|
|
801
|
-
closeSync(fd);
|
|
802
|
-
}
|
|
803
|
-
return true;
|
|
804
|
-
} catch (error) {
|
|
805
|
-
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
806
|
-
return false;
|
|
807
|
-
}
|
|
808
|
-
};
|
|
809
|
-
if (!acquire()) {
|
|
810
|
-
const existing = readLease(path);
|
|
811
|
-
const liveness = existing ? leaseOwnerIsLive(existing) : "unknown";
|
|
812
|
-
if (liveness !== "stopped") {
|
|
813
|
-
throw serviceStartRefusal(coordRoot, existing?.pid, liveness);
|
|
814
|
-
}
|
|
815
|
-
unlinkSync(path);
|
|
816
|
-
if (!acquire()) throw new Error("governor service lease raced with another process");
|
|
817
|
-
}
|
|
818
|
-
return () => {
|
|
819
|
-
try {
|
|
820
|
-
const existing = readLease(path);
|
|
821
|
-
if (existing?.nonce === owner.nonce) unlinkSync(path);
|
|
822
|
-
} catch {
|
|
823
|
-
// A stale lease is recoverable by the next explicit service start.
|
|
824
|
-
}
|
|
825
|
-
};
|
|
826
|
-
}
|
|
827
|
-
|
|
828
|
-
function readLease(path: string): ServiceLease | null {
|
|
829
|
-
try {
|
|
830
|
-
const value = JSON.parse(readFileSync(path, "utf8")) as Partial<ServiceLease>;
|
|
831
|
-
return Number.isSafeInteger(value.pid) &&
|
|
832
|
-
(value.pid ?? 0) > 0 &&
|
|
833
|
-
typeof value.host === "string" &&
|
|
834
|
-
typeof value.nonce === "string" &&
|
|
835
|
-
typeof value.created_at === "string"
|
|
836
|
-
? ({
|
|
837
|
-
...value,
|
|
838
|
-
start_token: typeof value.start_token === "string" ? value.start_token : undefined,
|
|
839
|
-
} as ServiceLease)
|
|
840
|
-
: null;
|
|
841
|
-
} catch {
|
|
842
|
-
return null;
|
|
843
|
-
}
|
|
844
|
-
}
|
|
845
|
-
|
|
846
|
-
function leaseOwnerIsLive(lease: ServiceLease): GovernorServiceStatus["liveness"] {
|
|
847
|
-
return observeGovernorServiceProcess({ ...lease, state: "running" }).liveness;
|
|
848
|
-
}
|
|
849
|
-
|
|
850
|
-
function readStatusRecord(coordRoot: string): GovernorServiceStatusRecord | undefined {
|
|
851
|
-
if (!existsSync(serviceStatusPath(coordRoot))) return undefined;
|
|
852
|
-
try {
|
|
853
|
-
const value = readBoundedJson<GovernorServiceStatusRecord>(
|
|
854
|
-
serviceStatusPath(coordRoot),
|
|
855
|
-
"governor service status",
|
|
856
|
-
);
|
|
857
|
-
if (
|
|
858
|
-
value.schema_version !== GOVERNOR_SERVICE_STATUS_SCHEMA_VERSION ||
|
|
859
|
-
!Number.isSafeInteger(value.pid) ||
|
|
860
|
-
value.pid < 1 ||
|
|
861
|
-
typeof value.host !== "string" ||
|
|
862
|
-
typeof value.nonce !== "string" ||
|
|
863
|
-
!validTimestamp(value.started_at) ||
|
|
864
|
-
!validTimestamp(value.heartbeat_at)
|
|
865
|
-
) {
|
|
866
|
-
return undefined;
|
|
867
|
-
}
|
|
868
|
-
return {
|
|
869
|
-
...value,
|
|
870
|
-
start_token: typeof value.start_token === "string" ? value.start_token : undefined,
|
|
871
|
-
};
|
|
872
|
-
} catch {
|
|
873
|
-
return undefined;
|
|
874
|
-
}
|
|
875
|
-
}
|
|
876
|
-
|
|
877
|
-
function statusOwnerIsLive(record: GovernorServiceStatusRecord): GovernorServiceStatus["liveness"] {
|
|
878
|
-
return observeGovernorServiceProcess(record).liveness;
|
|
879
|
-
}
|
|
880
|
-
|
|
881
|
-
function serviceStartRefusal(
|
|
698
|
+
function startRefusal(
|
|
882
699
|
coordRoot: string,
|
|
883
700
|
pid: number | undefined,
|
|
884
|
-
liveness:
|
|
701
|
+
liveness: ServiceLiveness,
|
|
885
702
|
): Error {
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
`${reason}; run \`${resolveBinName(coordRoot)} governor service stop\` to write a durable stop request before starting or reconfiguring it`,
|
|
703
|
+
return serviceStartRefusal(
|
|
704
|
+
"governor service",
|
|
705
|
+
`${resolveBinName(coordRoot)} governor service stop`,
|
|
706
|
+
pid,
|
|
707
|
+
liveness,
|
|
892
708
|
);
|
|
893
709
|
}
|
|
894
710
|
|
|
@@ -4,7 +4,7 @@ import { join } from "node:path";
|
|
|
4
4
|
import { coordEnv } from "../../../lib/env.ts";
|
|
5
5
|
import { resolveOwnerBySessionEnv } from "../../agents/coord-client.ts";
|
|
6
6
|
import { readLiveCoordinationRow } from "../../agents/state/live-coordination-view.ts";
|
|
7
|
-
import {
|
|
7
|
+
import { verifyProcessGeneration } from "../../agents/state/proc-start.ts";
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* Resolve the canonical `instance_id` for the current hook invocation.
|
|
@@ -98,7 +98,7 @@ export function resolveOwner(opts: {
|
|
|
98
98
|
// A row whose start token disagrees with the process now holding this
|
|
99
99
|
// pid is about a different, earlier process. Believing it hands this
|
|
100
100
|
// session another agent's identity, so walk past it.
|
|
101
|
-
const recycled =
|
|
101
|
+
const recycled = verifyProcessGeneration(pid, startToken) === "different";
|
|
102
102
|
if (owner && owner.length > 0 && !recycled) {
|
|
103
103
|
return {
|
|
104
104
|
instance_id: owner,
|