harnery 0.44.0 → 0.45.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +44 -0
- package/dist/core/agents/coord-client.d.ts.map +1 -1
- package/dist/core/agents/coord-client.js +18 -0
- package/dist/core/agents/live-authority-v3.d.ts +0 -1
- package/dist/core/agents/live-authority-v3.d.ts.map +1 -1
- package/dist/core/agents/live-authority-v3.js +0 -11
- package/dist/core/agents/live-lifecycle-v3.d.ts +2 -4
- package/dist/core/agents/live-lifecycle-v3.d.ts.map +1 -1
- package/dist/core/agents/live-lifecycle-v3.js +7 -28
- package/dist/core/agents/state/heartbeat-writer.d.ts.map +1 -1
- package/dist/core/agents/state/heartbeat-writer.js +14 -25
- package/dist/core/agents/state/live-coordination-writer.d.ts.map +1 -1
- package/dist/core/agents/state/live-coordination-writer.js +0 -5
- package/dist/core/agents/state/pidmap.d.ts.map +1 -1
- package/dist/core/agents/state/pidmap.js +0 -5
- package/dist/core/agents/state/proc-start.d.ts +5 -6
- package/dist/core/agents/state/proc-start.d.ts.map +1 -1
- package/dist/core/agents/state/proc-start.js +13 -9
- package/dist/core/agents/state/process-handle.d.ts +26 -0
- package/dist/core/agents/state/process-handle.d.ts.map +1 -0
- package/dist/core/agents/state/process-handle.js +121 -0
- package/dist/core/agents/state/stale-sweep.d.ts.map +1 -1
- package/dist/core/agents/state/stale-sweep.js +70 -180
- package/dist/core/artifacts/activity.d.ts +4 -1
- package/dist/core/artifacts/activity.d.ts.map +1 -1
- package/dist/core/artifacts/activity.js +8 -2
- package/dist/core/artifacts/index.d.ts.map +1 -1
- package/dist/core/artifacts/index.js +38 -12
- package/dist/core/events/v3/authority-outbox.d.ts.map +1 -1
- package/dist/core/events/v3/authority-outbox.js +0 -29
- package/dist/core/events/v3/producers/coordination-recorder.d.ts +0 -1
- package/dist/core/events/v3/producers/coordination-recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/coordination-recorder.js +0 -7
- package/dist/core/events/v3/producers/recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/recorder.js +64 -77
- package/dist/core/hooks/cli.js +78 -48
- package/dist/core/resources/service.d.ts +2 -16
- package/dist/core/resources/service.d.ts.map +1 -1
- package/dist/core/resources/service.js +1 -78
- package/dist/core/semantic/service.d.ts.map +1 -1
- package/dist/core/semantic/service.js +2 -9
- package/dist/core/servers/index.d.ts +4 -0
- package/dist/core/servers/index.d.ts.map +1 -1
- package/dist/core/servers/index.js +21 -3
- package/dist/core/storage/durable-history.js +56 -22
- package/dist/core/storage/segments.d.ts.map +1 -1
- package/dist/core/storage/segments.js +73 -17
- package/dist/core/supervisor/service.d.ts +2 -4
- package/dist/core/supervisor/service.d.ts.map +1 -1
- package/dist/core/work/state.js +7 -7
- package/dist/core/workflow/engine.d.ts.map +1 -1
- package/dist/core/workflow/engine.js +21 -65
- package/dist/core/workflow/index.d.ts +4 -2
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +2 -1
- package/dist/core/workflow/live-session-v3.d.ts +0 -6
- package/dist/core/workflow/live-session-v3.d.ts.map +1 -1
- package/dist/core/workflow/live-session-v3.js +5 -126
- package/dist/core/workflow/proof.d.ts +4 -4
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +4 -12
- package/dist/core/workflow/recovery-audit.d.ts +30 -0
- package/dist/core/workflow/recovery-audit.d.ts.map +1 -0
- package/dist/core/workflow/recovery-audit.js +116 -0
- package/dist/core/workflow/run-state.d.ts +19 -2
- package/dist/core/workflow/run-state.d.ts.map +1 -1
- package/dist/core/workflow/run-state.js +113 -8
- package/dist/core/workflow/spawn-claude.d.ts.map +1 -1
- package/dist/core/workflow/spawn-claude.js +0 -19
- package/dist/core/workflow/spawn-codex.d.ts.map +1 -1
- package/dist/core/workflow/spawn-codex.js +0 -13
- package/dist/core/workflow/spawn-cursor.d.ts.map +1 -1
- package/dist/core/workflow/spawn-cursor.js +0 -19
- package/dist/core/workflow/spawn-opencode.d.ts.map +1 -1
- package/dist/core/workflow/spawn-opencode.js +0 -14
- package/dist/core/workflow/types.d.ts +11 -54
- package/dist/core/workflow/types.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/local-git.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/local-git.js +70 -14
- package/dist/core/workflow/workspaces/reclaim.d.ts +6 -1
- package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.d.ts +2 -0
- package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.js +24 -0
- package/dist/lib/identities/assume.d.ts.map +1 -1
- package/dist/lib/identities/assume.js +0 -4
- package/package.json +1 -1
- package/src/commands/workflow.ts +59 -0
- package/src/core/agents/coord-client.ts +20 -0
- package/src/core/agents/live-authority-v3.ts +0 -17
- package/src/core/agents/live-lifecycle-v3.ts +14 -52
- package/src/core/agents/state/heartbeat-writer.ts +12 -29
- package/src/core/agents/state/live-coordination-writer.ts +0 -29
- package/src/core/agents/state/pidmap.ts +0 -13
- package/src/core/agents/state/proc-start.ts +14 -11
- package/src/core/agents/state/process-handle.ts +165 -0
- package/src/core/agents/state/stale-sweep.ts +82 -247
- package/src/core/artifacts/activity.ts +9 -2
- package/src/core/artifacts/index.ts +45 -12
- package/src/core/events/v3/authority-outbox.ts +0 -56
- package/src/core/events/v3/producers/coordination-recorder.ts +0 -15
- package/src/core/events/v3/producers/recorder.ts +68 -94
- package/src/core/hooks/cli.ts +85 -47
- package/src/core/resources/service.ts +3 -88
- package/src/core/semantic/service.ts +2 -8
- package/src/core/servers/index.ts +30 -3
- package/src/core/storage/durable-history.ts +52 -20
- package/src/core/storage/segments.ts +79 -17
- package/src/core/supervisor/service.ts +2 -3
- package/src/core/work/state.ts +7 -7
- package/src/core/workflow/engine.ts +21 -73
- package/src/core/workflow/index.ts +12 -1
- package/src/core/workflow/live-session-v3.ts +5 -146
- package/src/core/workflow/proof.ts +5 -21
- package/src/core/workflow/recovery-audit.ts +181 -0
- package/src/core/workflow/run-state.ts +137 -8
- package/src/core/workflow/spawn-claude.ts +0 -18
- package/src/core/workflow/spawn-codex.ts +0 -13
- package/src/core/workflow/spawn-cursor.ts +0 -18
- package/src/core/workflow/spawn-opencode.ts +0 -14
- package/src/core/workflow/types.ts +11 -41
- package/src/core/workflow/workspaces/local-git.ts +86 -15
- package/src/core/workflow/workspaces/reclaim.ts +4 -0
- package/src/core/workflow/workspaces/recovery.ts +23 -0
- package/src/lib/identities/assume.ts +0 -4
- package/dist/core/agents/state/unchanged-file.d.ts +0 -44
- package/dist/core/agents/state/unchanged-file.d.ts.map +0 -1
- package/dist/core/agents/state/unchanged-file.js +0 -390
- package/dist/core/events/v3/generation-custody.d.ts +0 -5
- package/dist/core/events/v3/generation-custody.d.ts.map +0 -1
- package/dist/core/events/v3/generation-custody.js +0 -50
- package/dist/core/workflow/failure-evidence.d.ts +0 -16
- package/dist/core/workflow/failure-evidence.d.ts.map +0 -1
- package/dist/core/workflow/failure-evidence.js +0 -114
- package/src/core/agents/state/unchanged-file.ts +0 -456
- package/src/core/events/v3/generation-custody.ts +0 -59
- package/src/core/workflow/failure-evidence.ts +0 -141
|
@@ -10,21 +10,12 @@
|
|
|
10
10
|
* `coord.freshness_seconds` (see `coordFreshnessSeconds`).
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { existsSync,
|
|
14
|
-
import {
|
|
13
|
+
import { existsSync, readdirSync, readFileSync, statSync, unlinkSync } from "node:fs";
|
|
14
|
+
import { join } from "node:path";
|
|
15
15
|
import { coordFreshnessSeconds } from "../../config.ts";
|
|
16
16
|
import { type EventAdapterIdV3, eventAdapterIdV3FromPlatform } from "../../events/v3/adapter-id.ts";
|
|
17
|
-
import { withGenerationCustodyV3 } from "../../events/v3/generation-custody.ts";
|
|
18
|
-
import { liveInstanceIdV3 } from "../../events/v3/live-routing.ts";
|
|
19
17
|
import { writeProducerDiagnosticV3 } from "../../events/v3/producers/intake.ts";
|
|
20
18
|
import { recordLiveSweepObservationV3 } from "../live-lifecycle-v3.ts";
|
|
21
|
-
import {
|
|
22
|
-
captureFileRevision,
|
|
23
|
-
type DeleteFileResult,
|
|
24
|
-
deleteFileIfUnchanged,
|
|
25
|
-
type FileRecoveryResult,
|
|
26
|
-
recoverInterruptedFileDeletions,
|
|
27
|
-
} from "./unchanged-file.ts";
|
|
28
19
|
|
|
29
20
|
/** platform → adapter, for the swept-event envelope (mirrors heartbeat-writer's adapterOf). */
|
|
30
21
|
/**
|
|
@@ -41,7 +32,6 @@ function emitSwept(
|
|
|
41
32
|
instanceId: string,
|
|
42
33
|
adapter: EventAdapterIdV3,
|
|
43
34
|
sessionId: string,
|
|
44
|
-
expectedGenerationId: `gen_${string}` | null,
|
|
45
35
|
reason: "stale" | "unparseable" | "missing_ts",
|
|
46
36
|
ageSecs?: number,
|
|
47
37
|
): boolean {
|
|
@@ -51,7 +41,6 @@ function emitSwept(
|
|
|
51
41
|
owner: instanceId,
|
|
52
42
|
nativeSessionId: sessionId,
|
|
53
43
|
adapter,
|
|
54
|
-
expectedGenerationId,
|
|
55
44
|
observation:
|
|
56
45
|
reason === "stale"
|
|
57
46
|
? "stale_heartbeat"
|
|
@@ -76,73 +65,13 @@ function emitSwept(
|
|
|
76
65
|
}
|
|
77
66
|
}
|
|
78
67
|
|
|
79
|
-
/**
|
|
80
|
-
function
|
|
81
|
-
coordRoot: string,
|
|
82
|
-
activeDir: string,
|
|
83
|
-
owner: string,
|
|
84
|
-
uncertainOwners: Set<string>,
|
|
85
|
-
): boolean {
|
|
86
|
-
if (uncertainOwners.has("*") || uncertainOwners.has(owner)) return true;
|
|
87
|
-
if (!/^[A-Za-z0-9_-]{1,128}$/.test(owner)) return true;
|
|
88
|
-
const heartbeat = join(activeDir, `${owner}.json`);
|
|
89
|
-
try {
|
|
90
|
-
lstatSync(heartbeat);
|
|
91
|
-
return true;
|
|
92
|
-
} catch (error) {
|
|
93
|
-
if ((error as NodeJS.ErrnoException).code !== "ENOENT") return true;
|
|
94
|
-
}
|
|
68
|
+
/** File mtime in epoch-seconds, or +Infinity if it can't be read (treat as fresh → don't reap). */
|
|
69
|
+
function mtimeSecs(path: string): number {
|
|
95
70
|
try {
|
|
96
|
-
|
|
97
|
-
} catch
|
|
98
|
-
return
|
|
99
|
-
}
|
|
100
|
-
// Callers hold this owner's custody. A sweeper that died after the initial
|
|
101
|
-
// directory scan may have left its heartbeat hidden under a private pin.
|
|
102
|
-
for (const recovery of recoverInterruptedFileDeletions(
|
|
103
|
-
activeDir,
|
|
104
|
-
(source) => source === heartbeat,
|
|
105
|
-
)) {
|
|
106
|
-
if (recovery.sourcePath && recovery.sourcePath !== heartbeat) continue;
|
|
107
|
-
if (recovery.state !== "restored") {
|
|
108
|
-
uncertainOwners.add(recovery.sourcePath ? owner : "*");
|
|
109
|
-
recordCleanupFailure(coordRoot, recovery.sourcePath ?? activeDir, recovery);
|
|
110
|
-
}
|
|
111
|
-
return true;
|
|
112
|
-
}
|
|
113
|
-
return false;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
function recordCleanupFailure(
|
|
117
|
-
coordRoot: string,
|
|
118
|
-
sourcePath: string,
|
|
119
|
-
result: DeleteFileResult | FileRecoveryResult,
|
|
120
|
-
): void {
|
|
121
|
-
if (
|
|
122
|
-
result.state !== "error" &&
|
|
123
|
-
result.state !== "recovery-required" &&
|
|
124
|
-
result.state !== "retained"
|
|
125
|
-
) {
|
|
126
|
-
return;
|
|
71
|
+
return Math.floor(statSync(path).mtimeMs / 1000);
|
|
72
|
+
} catch {
|
|
73
|
+
return Number.POSITIVE_INFINITY;
|
|
127
74
|
}
|
|
128
|
-
writeProducerDiagnosticV3(coordRoot, "heartbeat_sweep_unrecorded", {
|
|
129
|
-
signal: "cache.cleanup",
|
|
130
|
-
state: "cache_cleanup_pending",
|
|
131
|
-
reason: `${result.state}:${result.error ?? "file_cleanup_pending"}; source=${sourcePath}; recovery=${result.recoveryPath ?? "unavailable"}`,
|
|
132
|
-
});
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
function recordHeartbeatCleanupResult(
|
|
136
|
-
coordRoot: string,
|
|
137
|
-
sourcePath: string,
|
|
138
|
-
owner: string,
|
|
139
|
-
result: DeleteFileResult,
|
|
140
|
-
uncertainOwners: Set<string>,
|
|
141
|
-
): void {
|
|
142
|
-
if (result.state === "recovery-required" || result.state === "error") {
|
|
143
|
-
uncertainOwners.add(owner);
|
|
144
|
-
}
|
|
145
|
-
recordCleanupFailure(coordRoot, sourcePath, result);
|
|
146
75
|
}
|
|
147
76
|
|
|
148
77
|
export function staleSweep(coordRoot: string): {
|
|
@@ -176,171 +105,98 @@ export function staleSweep(coordRoot: string): {
|
|
|
176
105
|
// lifecycle.sweep_observed; a terminal or unavailable generation gets a
|
|
177
106
|
// heartbeat_sweep_unrecorded producer diagnostic without being resurrected.
|
|
178
107
|
const liveInstanceIds = new Set<string>();
|
|
179
|
-
const
|
|
180
|
-
const d = resolve(coordRoot, ".harnery", "active");
|
|
181
|
-
for (const recovery of existsSync(d)
|
|
182
|
-
? recoverInterruptedFileDeletions(
|
|
183
|
-
d,
|
|
184
|
-
(source) => dirname(source) === d && /^[A-Za-z0-9_-]{1,128}\.json$/.test(basename(source)),
|
|
185
|
-
)
|
|
186
|
-
: []) {
|
|
187
|
-
if (recovery.state !== "restored") {
|
|
188
|
-
uncertainOwners.add(recovery.sourcePath ? basename(recovery.sourcePath, ".json") : "*");
|
|
189
|
-
recordCleanupFailure(coordRoot, recovery.sourcePath ?? d, recovery);
|
|
190
|
-
}
|
|
191
|
-
}
|
|
108
|
+
const d = join(coordRoot, ".harnery", "active");
|
|
192
109
|
if (existsSync(d)) {
|
|
193
110
|
for (const f of readdirSync(d)) {
|
|
194
111
|
if (!f.endsWith(".json")) continue;
|
|
195
112
|
const path = join(d, f);
|
|
196
113
|
const idFromFile = f.replace(/\.json$/, "");
|
|
197
|
-
|
|
114
|
+
let parsed: {
|
|
115
|
+
instance_id?: string;
|
|
116
|
+
last_heartbeat?: string;
|
|
117
|
+
platform?: unknown;
|
|
118
|
+
session_id?: string;
|
|
119
|
+
} | null = null;
|
|
198
120
|
try {
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
let parsed: {
|
|
204
|
-
instance_id?: string;
|
|
205
|
-
last_heartbeat?: string;
|
|
206
|
-
platform?: unknown;
|
|
207
|
-
session_id?: string;
|
|
208
|
-
v3_generation_id?: unknown;
|
|
209
|
-
} | null = null;
|
|
210
|
-
try {
|
|
211
|
-
const value = JSON.parse(revision.bytes.toString("utf8"));
|
|
212
|
-
if (value && typeof value === "object" && !Array.isArray(value)) parsed = value;
|
|
213
|
-
} catch {
|
|
214
|
-
parsed = null;
|
|
215
|
-
}
|
|
121
|
+
parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
122
|
+
} catch {
|
|
123
|
+
parsed = null;
|
|
124
|
+
}
|
|
216
125
|
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
null,
|
|
227
|
-
"unparseable",
|
|
228
|
-
nowSec - mtimeSecs,
|
|
229
|
-
),
|
|
230
|
-
);
|
|
231
|
-
recordHeartbeatCleanupResult(coordRoot, path, idFromFile, result, uncertainOwners);
|
|
232
|
-
if (result.state === "deleted") {
|
|
233
|
-
heartbeatsRemoved.push(f);
|
|
234
|
-
}
|
|
126
|
+
if (parsed === null) {
|
|
127
|
+
// Unparseable: only reap if the file itself is mtime-old.
|
|
128
|
+
if (mtimeSecs(path) < cutoff) {
|
|
129
|
+
try {
|
|
130
|
+
const ageSecs = nowSec - mtimeSecs(path);
|
|
131
|
+
if (
|
|
132
|
+
!emitSwept(coordRoot, idFromFile, "claude-code", idFromFile, "unparseable", ageSecs)
|
|
133
|
+
) {
|
|
134
|
+
continue;
|
|
235
135
|
}
|
|
236
|
-
|
|
136
|
+
unlinkSync(path);
|
|
137
|
+
heartbeatsRemoved.push(f);
|
|
138
|
+
} catch {
|
|
139
|
+
/* swallow */
|
|
237
140
|
}
|
|
141
|
+
}
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
238
144
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
const generationId =
|
|
246
|
-
typeof parsed.v3_generation_id === "string" &&
|
|
247
|
-
/^gen_[A-Za-z0-9._-]+$/.test(parsed.v3_generation_id)
|
|
248
|
-
? (parsed.v3_generation_id as `gen_${string}`)
|
|
249
|
-
: null;
|
|
250
|
-
const ts = parsed.last_heartbeat
|
|
251
|
-
? Math.floor(Date.parse(parsed.last_heartbeat) / 1000)
|
|
252
|
-
: Number.NaN;
|
|
253
|
-
|
|
254
|
-
if (!parsed.last_heartbeat || !Number.isFinite(ts)) {
|
|
255
|
-
// No / NaN last_heartbeat: can't trust content; gate on mtime.
|
|
256
|
-
if (mtimeSecs < cutoff) {
|
|
257
|
-
const result = deleteFileIfUnchanged(path, revision, () =>
|
|
258
|
-
emitSwept(
|
|
259
|
-
coordRoot,
|
|
260
|
-
instanceId,
|
|
261
|
-
adapter,
|
|
262
|
-
sessionId,
|
|
263
|
-
generationId,
|
|
264
|
-
"missing_ts",
|
|
265
|
-
nowSec - mtimeSecs,
|
|
266
|
-
),
|
|
267
|
-
);
|
|
268
|
-
recordHeartbeatCleanupResult(coordRoot, path, idFromFile, result, uncertainOwners);
|
|
269
|
-
if (result.state === "deleted") {
|
|
270
|
-
heartbeatsRemoved.push(f);
|
|
271
|
-
}
|
|
272
|
-
} else if (parsed.instance_id) {
|
|
273
|
-
liveInstanceIds.add(parsed.instance_id);
|
|
274
|
-
}
|
|
275
|
-
return;
|
|
276
|
-
}
|
|
145
|
+
const instanceId = parsed.instance_id ?? idFromFile;
|
|
146
|
+
const adapter = eventAdapterIdV3FromPlatform(parsed.platform, { context: "stale-sweep" });
|
|
147
|
+
const sessionId = parsed.session_id ?? instanceId;
|
|
148
|
+
const ts = parsed.last_heartbeat
|
|
149
|
+
? Math.floor(Date.parse(parsed.last_heartbeat) / 1000)
|
|
150
|
+
: Number.NaN;
|
|
277
151
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
if (result.state === "deleted") {
|
|
293
|
-
heartbeatsRemoved.push(f);
|
|
294
|
-
}
|
|
295
|
-
return;
|
|
152
|
+
if (!parsed.last_heartbeat || !Number.isFinite(ts)) {
|
|
153
|
+
// No / NaN last_heartbeat: can't trust content; gate on mtime.
|
|
154
|
+
if (mtimeSecs(path) < cutoff) {
|
|
155
|
+
if (
|
|
156
|
+
!emitSwept(
|
|
157
|
+
coordRoot,
|
|
158
|
+
instanceId,
|
|
159
|
+
adapter,
|
|
160
|
+
sessionId,
|
|
161
|
+
"missing_ts",
|
|
162
|
+
nowSec - mtimeSecs(path),
|
|
163
|
+
)
|
|
164
|
+
) {
|
|
165
|
+
continue;
|
|
296
166
|
}
|
|
167
|
+
unlinkSync(path);
|
|
168
|
+
heartbeatsRemoved.push(f);
|
|
169
|
+
} else if (parsed.instance_id) {
|
|
170
|
+
liveInstanceIds.add(parsed.instance_id);
|
|
171
|
+
}
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
297
174
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
175
|
+
if (ts < cutoff) {
|
|
176
|
+
// Legitimate stale prune (valid timestamp, past the freshness cutoff).
|
|
177
|
+
if (!emitSwept(coordRoot, instanceId, adapter, sessionId, "stale", nowSec - ts)) {
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
unlinkSync(path);
|
|
181
|
+
heartbeatsRemoved.push(f);
|
|
182
|
+
continue;
|
|
303
183
|
}
|
|
184
|
+
|
|
185
|
+
if (parsed.instance_id) liveInstanceIds.add(parsed.instance_id);
|
|
304
186
|
}
|
|
305
187
|
}
|
|
306
188
|
|
|
307
189
|
// 2. Prune pid-map entries whose instance has no live heartbeat.
|
|
308
|
-
const pidmapDir =
|
|
309
|
-
for (const recovery of existsSync(pidmapDir)
|
|
310
|
-
? recoverInterruptedFileDeletions(
|
|
311
|
-
pidmapDir,
|
|
312
|
-
(source) => dirname(source) === pidmapDir && /^\d+$/.test(basename(source)),
|
|
313
|
-
)
|
|
314
|
-
: [])
|
|
315
|
-
recordCleanupFailure(coordRoot, recovery.sourcePath ?? pidmapDir, recovery);
|
|
190
|
+
const pidmapDir = join(coordRoot, ".harnery", "pid-map");
|
|
316
191
|
if (existsSync(pidmapDir)) {
|
|
317
192
|
for (const f of readdirSync(pidmapDir)) {
|
|
318
|
-
if (f.startsWith(".harnery-delete-")) continue;
|
|
319
193
|
const path = join(pidmapDir, f);
|
|
320
194
|
try {
|
|
321
|
-
const
|
|
322
|
-
if (!revision) continue;
|
|
323
|
-
const row = revision.bytes.toString("utf8").trim();
|
|
195
|
+
const row = readFileSync(path, "utf8").trim();
|
|
324
196
|
const ownerId = row.split("\t")[0]?.trim() ?? "";
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
(!liveInstanceIds.has(ownerId) &&
|
|
329
|
-
!retainsHeartbeat(coordRoot, d, ownerId, uncertainOwners))
|
|
330
|
-
) {
|
|
331
|
-
const result = deleteFileIfUnchanged(
|
|
332
|
-
path,
|
|
333
|
-
revision,
|
|
334
|
-
() => !ownerId || !retainsHeartbeat(coordRoot, d, ownerId, uncertainOwners),
|
|
335
|
-
);
|
|
336
|
-
recordCleanupFailure(coordRoot, path, result);
|
|
337
|
-
if (result.state === "deleted") pidmapsRemoved += 1;
|
|
338
|
-
}
|
|
339
|
-
};
|
|
340
|
-
if (/^[A-Za-z0-9_-]{1,128}$/.test(ownerId)) {
|
|
341
|
-
withGenerationCustodyV3(coordRoot, [liveInstanceIdV3(ownerId)], prune);
|
|
342
|
-
} else if (!ownerId) {
|
|
343
|
-
prune();
|
|
197
|
+
if (!ownerId || !liveInstanceIds.has(ownerId)) {
|
|
198
|
+
unlinkSync(path);
|
|
199
|
+
pidmapsRemoved += 1;
|
|
344
200
|
}
|
|
345
201
|
} catch {
|
|
346
202
|
/* swallow */
|
|
@@ -350,16 +206,7 @@ export function staleSweep(coordRoot: string): {
|
|
|
350
206
|
|
|
351
207
|
// 3. Prune prompt hashes for dead owners. Resource hashes retain their own
|
|
352
208
|
// count so existing peer-hash telemetry keeps its original meaning.
|
|
353
|
-
const agentsDir =
|
|
354
|
-
for (const recovery of existsSync(agentsDir)
|
|
355
|
-
? recoverInterruptedFileDeletions(
|
|
356
|
-
agentsDir,
|
|
357
|
-
(source) =>
|
|
358
|
-
dirname(source) === agentsDir &&
|
|
359
|
-
/^\.last-(?:resource|peer)-hash\.[A-Za-z0-9_-]{1,128}$/.test(basename(source)),
|
|
360
|
-
)
|
|
361
|
-
: [])
|
|
362
|
-
recordCleanupFailure(coordRoot, recovery.sourcePath ?? agentsDir, recovery);
|
|
209
|
+
const agentsDir = join(coordRoot, ".harnery");
|
|
363
210
|
if (existsSync(agentsDir)) {
|
|
364
211
|
for (const f of readdirSync(agentsDir)) {
|
|
365
212
|
const resourceHash = f.startsWith(".last-resource-hash.");
|
|
@@ -370,25 +217,13 @@ export function staleSweep(coordRoot: string): {
|
|
|
370
217
|
// Ignore atomic-write temporary files and preserve hashes when a
|
|
371
218
|
// retained heartbeat's liveness could not be determined safely.
|
|
372
219
|
if (!/^[A-Za-z0-9_-]{1,128}$/.test(owner)) continue;
|
|
220
|
+
if (existsSync(join(d, `${owner}.json`))) continue;
|
|
373
221
|
}
|
|
374
|
-
if (!liveInstanceIds.has(owner)
|
|
222
|
+
if (!liveInstanceIds.has(owner)) {
|
|
375
223
|
try {
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
const revision = captureFileRevision(path);
|
|
380
|
-
if (!revision) return;
|
|
381
|
-
const result = deleteFileIfUnchanged(
|
|
382
|
-
path,
|
|
383
|
-
revision,
|
|
384
|
-
() => !retainsHeartbeat(coordRoot, d, owner, uncertainOwners),
|
|
385
|
-
);
|
|
386
|
-
recordCleanupFailure(coordRoot, path, result);
|
|
387
|
-
if (result.state === "deleted") {
|
|
388
|
-
if (resourceHash) resourceHashesRemoved += 1;
|
|
389
|
-
else peerHashesRemoved += 1;
|
|
390
|
-
}
|
|
391
|
-
});
|
|
224
|
+
unlinkSync(join(agentsDir, f));
|
|
225
|
+
if (resourceHash) resourceHashesRemoved += 1;
|
|
226
|
+
else peerHashesRemoved += 1;
|
|
392
227
|
} catch {
|
|
393
228
|
/* swallow */
|
|
394
229
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { lstatSync, readdirSync } from "node:fs";
|
|
2
|
+
import { lstatSync, readdirSync, type Stats } from "node:fs";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { ARTIFACT_MANIFEST } from "./constants.ts";
|
|
5
5
|
|
|
@@ -36,11 +36,14 @@ export function artifactRootEntries(path: string): string {
|
|
|
36
36
|
* Keep the prior root activity while its payload entries match the checkpoint.
|
|
37
37
|
* A root deletion or rename changes that fingerprint; nested changes retain
|
|
38
38
|
* their ordinary filesystem timestamps. Symlinks are inspected, never followed.
|
|
39
|
+
* `visit` sees every entry's lstat exactly once, including a root manifest the
|
|
40
|
+
* activity rules skip, so a caller can size the tree in the same walk.
|
|
39
41
|
*/
|
|
40
42
|
export function readArtifactActivity(
|
|
41
43
|
path: string,
|
|
42
44
|
now: Date,
|
|
43
45
|
previous?: ArtifactActivity,
|
|
46
|
+
visit?: (entry: string, stat: Stats) => void,
|
|
44
47
|
): ArtifactActivity {
|
|
45
48
|
const entries = artifactRootEntries(path);
|
|
46
49
|
const nowMs = now.getTime();
|
|
@@ -50,6 +53,7 @@ export function readArtifactActivity(
|
|
|
50
53
|
: 0;
|
|
51
54
|
const walk = (entry: string, root = false): number => {
|
|
52
55
|
const stat = lstatSync(entry);
|
|
56
|
+
visit?.(entry, stat);
|
|
53
57
|
let latest =
|
|
54
58
|
root && previous?.root_entries_sha256 === entries
|
|
55
59
|
? accept(Date.parse(previous.last_changed_at))
|
|
@@ -58,7 +62,10 @@ export function readArtifactActivity(
|
|
|
58
62
|
for (const name of readdirSync(entry)) {
|
|
59
63
|
// Only this unit's own control manifest is metadata. Nested artifact
|
|
60
64
|
// manifests are payload from the containing workspace's point of view.
|
|
61
|
-
if (root && previous && name === ARTIFACT_MANIFEST)
|
|
65
|
+
if (root && previous && name === ARTIFACT_MANIFEST) {
|
|
66
|
+
if (visit) visit(join(entry, name), lstatSync(join(entry, name)));
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
62
69
|
latest = Math.max(latest, walk(join(entry, name)));
|
|
63
70
|
}
|
|
64
71
|
}
|
|
@@ -794,11 +794,13 @@ function cleanArtifactsUnlocked(
|
|
|
794
794
|
timeBudgetMs?: number;
|
|
795
795
|
},
|
|
796
796
|
): ArtifactInventoryEntry[] {
|
|
797
|
-
const started = performance.now();
|
|
798
797
|
const now = opts.now ?? new Date();
|
|
799
798
|
const freshnessSeconds = opts.freshnessSeconds ?? 600;
|
|
800
799
|
const rows = inventoryArtifacts(repoRoot, { now, freshnessSeconds });
|
|
801
800
|
if (!opts.yes) return rows;
|
|
801
|
+
// The budget bounds deletion work. Inventory cost scales with the store, so
|
|
802
|
+
// counting it here would leave room for only the one guaranteed deletion.
|
|
803
|
+
const started = performance.now();
|
|
802
804
|
|
|
803
805
|
let attempted = 0;
|
|
804
806
|
return rows.map((entry) => {
|
|
@@ -1276,7 +1278,13 @@ function autoCleanArtifactsUnlocked(
|
|
|
1276
1278
|
failures,
|
|
1277
1279
|
...(status === "completed"
|
|
1278
1280
|
? { last_completed_at: now.toISOString() }
|
|
1279
|
-
: {
|
|
1281
|
+
: {
|
|
1282
|
+
// A slice that deleted something continues soon. One that made no
|
|
1283
|
+
// progress would only repeat the same walk, so it waits a full interval.
|
|
1284
|
+
retry_after: new Date(
|
|
1285
|
+
now.getTime() + (deleted > 0 ? 60_000 : intervalMs),
|
|
1286
|
+
).toISOString(),
|
|
1287
|
+
}),
|
|
1280
1288
|
});
|
|
1281
1289
|
return { ran: true, reason: status === "completed" ? "swept" : status, deleted, bytes };
|
|
1282
1290
|
} catch (error) {
|
|
@@ -1340,8 +1348,7 @@ function classifyArtifactPath(
|
|
|
1340
1348
|
return rowFor(path, name, repoRoot, classification, parsed.reason);
|
|
1341
1349
|
}
|
|
1342
1350
|
const manifest = parsed.manifest;
|
|
1343
|
-
const usage =
|
|
1344
|
-
const lastModifiedMs = safeTreeLastModified(path, now, manifest.activity);
|
|
1351
|
+
const { usage, lastModifiedMs } = scanManagedTree(path, now, manifest.activity);
|
|
1345
1352
|
const retentionAnchorMs = Date.parse(manifest.retention.renewed_at ?? manifest.created_at);
|
|
1346
1353
|
const retentionWindowMs = Date.parse(manifest.retention.expires_at) - retentionAnchorMs;
|
|
1347
1354
|
const effectiveLastModifiedMs = Math.max(retentionAnchorMs, lastModifiedMs ?? 0);
|
|
@@ -2033,23 +2040,49 @@ function safeTreeUsage(path: string, seen = new Set<string>()): TreeUsage | null
|
|
|
2033
2040
|
}
|
|
2034
2041
|
}
|
|
2035
2042
|
|
|
2036
|
-
function
|
|
2037
|
-
|
|
2043
|
+
function addEntryUsage(total: TreeUsage, st: Stats, seen: Set<string>): void {
|
|
2044
|
+
if (st.isDirectory()) {
|
|
2045
|
+
total.disk += allocatedBytes(st);
|
|
2046
|
+
total.apparent += st.size;
|
|
2047
|
+
return;
|
|
2048
|
+
}
|
|
2049
|
+
if (!st.isFile() && !st.isSymbolicLink()) return;
|
|
2050
|
+
const key = `${st.dev}:${st.ino}`;
|
|
2051
|
+
if (st.nlink > 1 && seen.has(key)) {
|
|
2052
|
+
total.apparent += st.size;
|
|
2053
|
+
return;
|
|
2054
|
+
}
|
|
2055
|
+
if (st.nlink > 1) seen.add(key);
|
|
2056
|
+
total.disk += allocatedBytes(st);
|
|
2057
|
+
total.apparent += st.size;
|
|
2038
2058
|
}
|
|
2039
2059
|
|
|
2040
2060
|
/**
|
|
2041
|
-
*
|
|
2042
|
-
*
|
|
2043
|
-
* scan
|
|
2061
|
+
* Size a managed tree and find its newest change in one walk, so inventory
|
|
2062
|
+
* reads each entry's metadata once instead of once per question. If the
|
|
2063
|
+
* activity scan fails, usage is measured on its own, as before.
|
|
2044
2064
|
*/
|
|
2045
|
-
function
|
|
2065
|
+
function scanManagedTree(
|
|
2066
|
+
path: string,
|
|
2067
|
+
now: Date,
|
|
2068
|
+
activity?: ArtifactActivity,
|
|
2069
|
+
): { usage: TreeUsage | null; lastModifiedMs: number | null } {
|
|
2070
|
+
const usage: TreeUsage = { disk: 0, apparent: 0 };
|
|
2071
|
+
const seen = new Set<string>();
|
|
2046
2072
|
try {
|
|
2047
|
-
|
|
2073
|
+
const result = readArtifactActivity(path, now, activity, (_entry, st) =>
|
|
2074
|
+
addEntryUsage(usage, st, seen),
|
|
2075
|
+
);
|
|
2076
|
+
return { usage, lastModifiedMs: Date.parse(result.last_changed_at) };
|
|
2048
2077
|
} catch {
|
|
2049
|
-
return null;
|
|
2078
|
+
return { usage: safeTreeUsage(path), lastModifiedMs: null };
|
|
2050
2079
|
}
|
|
2051
2080
|
}
|
|
2052
2081
|
|
|
2082
|
+
function safeTreeSize(path: string): number | null {
|
|
2083
|
+
return safeTreeUsage(path)?.disk ?? null;
|
|
2084
|
+
}
|
|
2085
|
+
|
|
2053
2086
|
function rowFor(
|
|
2054
2087
|
path: string,
|
|
2055
2088
|
name: string,
|
|
@@ -16,7 +16,6 @@ import { fsyncParentDirectory } from "../../workflow/durable-record.ts";
|
|
|
16
16
|
import { canonicalJsonV3, sha256V3 } from "./canonical.ts";
|
|
17
17
|
import type { EventV3 } from "./contract.ts";
|
|
18
18
|
import { liveGenesisIdV3 } from "./control.ts";
|
|
19
|
-
import { assertCurrentGenerationV3, withGenerationCustodyV3 } from "./generation-custody.ts";
|
|
20
19
|
import { assertEventV3 } from "./validate.ts";
|
|
21
20
|
import { ensureEventV3Layout, eventV3Paths, writeEventV3 } from "./writer.ts";
|
|
22
21
|
|
|
@@ -155,32 +154,9 @@ export function publishAuthorityTransactionV3(
|
|
|
155
154
|
coordRoot: string,
|
|
156
155
|
transaction: AuthorityTransactionV3,
|
|
157
156
|
fence: AuthorityEpochFenceV3 = {},
|
|
158
|
-
): AuthorityTransactionV3 {
|
|
159
|
-
return withGenerationCustodyV3(
|
|
160
|
-
coordRoot,
|
|
161
|
-
[transaction.actor_instance_id, transaction.subject_instance_id],
|
|
162
|
-
() => publishAuthorityTransactionWithCustodyV3(coordRoot, transaction, fence),
|
|
163
|
-
);
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
function publishAuthorityTransactionWithCustodyV3(
|
|
167
|
-
coordRoot: string,
|
|
168
|
-
transaction: AuthorityTransactionV3,
|
|
169
|
-
fence: AuthorityEpochFenceV3,
|
|
170
157
|
): AuthorityTransactionV3 {
|
|
171
158
|
validateTransaction(transaction);
|
|
172
159
|
const paths = ensureEventV3Layout(coordRoot);
|
|
173
|
-
const committedPath = authorityCommittedPath(paths.authorityOutbox, transaction.transaction_id);
|
|
174
|
-
if (existsSync(committedPath)) {
|
|
175
|
-
if (
|
|
176
|
-
readAuthorityReceiptV3(committedPath).transaction_digest !==
|
|
177
|
-
sha256V3(`${canonicalJsonV3(transaction)}\n`)
|
|
178
|
-
) {
|
|
179
|
-
throw new Error("authority committed transaction conflicts with this transaction");
|
|
180
|
-
}
|
|
181
|
-
return transaction;
|
|
182
|
-
}
|
|
183
|
-
assertTransactionGeneration(coordRoot, transaction);
|
|
184
160
|
if (fence.expectedGenesisId && liveGenesisIdV3(coordRoot) !== fence.expectedGenesisId) {
|
|
185
161
|
throw new Error("authority transaction epoch was replaced before publication");
|
|
186
162
|
}
|
|
@@ -209,26 +185,6 @@ export function reconcileAuthorityTransactionV3(
|
|
|
209
185
|
transactionId: string,
|
|
210
186
|
reconciler: AuthorityReconcilerV3,
|
|
211
187
|
fence: AuthorityEpochFenceV3 = {},
|
|
212
|
-
): AuthorityReceiptV3 {
|
|
213
|
-
assertTransactionId(transactionId);
|
|
214
|
-
const paths = ensureEventV3Layout(coordRoot);
|
|
215
|
-
const committedPath = authorityCommittedPath(paths.authorityOutbox, transactionId);
|
|
216
|
-
if (existsSync(committedPath)) return readAuthorityReceiptV3(committedPath);
|
|
217
|
-
const transaction = readAuthorityTransactionV3(
|
|
218
|
-
authorityReadyPath(paths.authorityOutbox, transactionId),
|
|
219
|
-
);
|
|
220
|
-
return withGenerationCustodyV3(
|
|
221
|
-
coordRoot,
|
|
222
|
-
[transaction.actor_instance_id, transaction.subject_instance_id],
|
|
223
|
-
() => reconcileAuthorityTransactionWithCustodyV3(coordRoot, transactionId, reconciler, fence),
|
|
224
|
-
);
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
function reconcileAuthorityTransactionWithCustodyV3(
|
|
228
|
-
coordRoot: string,
|
|
229
|
-
transactionId: string,
|
|
230
|
-
reconciler: AuthorityReconcilerV3,
|
|
231
|
-
fence: AuthorityEpochFenceV3,
|
|
232
188
|
): AuthorityReceiptV3 {
|
|
233
189
|
assertTransactionId(transactionId);
|
|
234
190
|
const paths = ensureEventV3Layout(coordRoot);
|
|
@@ -236,10 +192,8 @@ function reconcileAuthorityTransactionWithCustodyV3(
|
|
|
236
192
|
if (existsSync(committedPath)) return readAuthorityReceiptV3(committedPath);
|
|
237
193
|
const readyPath = authorityReadyPath(paths.authorityOutbox, transactionId);
|
|
238
194
|
const transaction = readAuthorityTransactionV3(readyPath);
|
|
239
|
-
assertTransactionGeneration(coordRoot, transaction);
|
|
240
195
|
const current = reconciler.readStateDigest();
|
|
241
196
|
if (current === transaction.expected_prior_state_digest) {
|
|
242
|
-
assertTransactionGeneration(coordRoot, transaction);
|
|
243
197
|
reconciler.apply(transaction.mutation, transaction.transaction_id);
|
|
244
198
|
} else if (current !== transaction.desired_state_digest) {
|
|
245
199
|
throw new Error("authority transaction prior state conflicts with current state");
|
|
@@ -247,7 +201,6 @@ function reconcileAuthorityTransactionWithCustodyV3(
|
|
|
247
201
|
if (reconciler.readStateDigest() !== transaction.desired_state_digest) {
|
|
248
202
|
throw new Error("authority transaction did not reach its declared desired state");
|
|
249
203
|
}
|
|
250
|
-
assertTransactionGeneration(coordRoot, transaction);
|
|
251
204
|
const event = JSON.parse(transaction.event_row) as unknown;
|
|
252
205
|
assertEventV3(event);
|
|
253
206
|
const durability = writeEventV3(coordRoot, event, {
|
|
@@ -277,15 +230,6 @@ function reconcileAuthorityTransactionWithCustodyV3(
|
|
|
277
230
|
return receipt;
|
|
278
231
|
}
|
|
279
232
|
|
|
280
|
-
function assertTransactionGeneration(coordRoot: string, transaction: AuthorityTransactionV3): void {
|
|
281
|
-
const event = JSON.parse(transaction.event_row) as EventV3;
|
|
282
|
-
assertCurrentGenerationV3(
|
|
283
|
-
coordRoot,
|
|
284
|
-
transaction.actor_instance_id,
|
|
285
|
-
"generation_id" in event.scope ? event.scope.generation_id : undefined,
|
|
286
|
-
);
|
|
287
|
-
}
|
|
288
|
-
|
|
289
233
|
export function listPendingAuthorityTransactionsV3(coordRoot: string): AuthorityTransactionV3[] {
|
|
290
234
|
const outbox = eventV3Paths(coordRoot).authorityOutbox;
|
|
291
235
|
if (!existsSync(outbox)) return [];
|