@claudexor/daemon 1.0.1 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.d.ts +29 -1
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +24 -4
- package/dist/client.js.map +1 -1
- package/dist/command-authority.d.ts +11 -0
- package/dist/command-authority.d.ts.map +1 -0
- package/dist/command-authority.js +15 -0
- package/dist/command-authority.js.map +1 -0
- package/dist/command-retention.d.ts +5 -0
- package/dist/command-retention.d.ts.map +1 -0
- package/dist/command-retention.js +18 -0
- package/dist/command-retention.js.map +1 -0
- package/dist/command-store.d.ts +41 -0
- package/dist/command-store.d.ts.map +1 -0
- package/dist/command-store.js +197 -0
- package/dist/command-store.js.map +1 -0
- package/dist/events.d.ts +2 -3
- package/dist/events.d.ts.map +1 -1
- package/dist/events.js +2 -3
- package/dist/events.js.map +1 -1
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -0
- package/dist/index.js.map +1 -1
- package/dist/interactions.d.ts +32 -22
- package/dist/interactions.d.ts.map +1 -1
- package/dist/interactions.js +183 -56
- package/dist/interactions.js.map +1 -1
- package/dist/journal-events.d.ts +4 -0
- package/dist/journal-events.d.ts.map +1 -0
- package/dist/journal-events.js +23 -0
- package/dist/journal-events.js.map +1 -0
- package/dist/journal-manager.d.ts +72 -0
- package/dist/journal-manager.d.ts.map +1 -0
- package/dist/journal-manager.js +463 -0
- package/dist/journal-manager.js.map +1 -0
- package/dist/journal-projection.d.ts +11 -0
- package/dist/journal-projection.d.ts.map +1 -0
- package/dist/journal-projection.js +2 -0
- package/dist/journal-projection.js.map +1 -0
- package/dist/journal-recovery-files.d.ts +27 -0
- package/dist/journal-recovery-files.d.ts.map +1 -0
- package/dist/journal-recovery-files.js +209 -0
- package/dist/journal-recovery-files.js.map +1 -0
- package/dist/operator-decisions.d.ts +30 -0
- package/dist/operator-decisions.d.ts.map +1 -0
- package/dist/operator-decisions.js +137 -0
- package/dist/operator-decisions.js.map +1 -0
- package/dist/project-partitions.d.ts +167 -0
- package/dist/project-partitions.d.ts.map +1 -0
- package/dist/project-partitions.js +365 -0
- package/dist/project-partitions.js.map +1 -0
- package/dist/projects.d.ts +29 -0
- package/dist/projects.d.ts.map +1 -0
- package/dist/projects.js +180 -0
- package/dist/projects.js.map +1 -0
- package/dist/quota-registry.d.ts +75 -0
- package/dist/quota-registry.d.ts.map +1 -0
- package/dist/quota-registry.js +208 -0
- package/dist/quota-registry.js.map +1 -0
- package/dist/resource-store.d.ts +22 -0
- package/dist/resource-store.d.ts.map +1 -0
- package/dist/resource-store.js +272 -0
- package/dist/resource-store.js.map +1 -0
- package/dist/run-events.d.ts +15 -0
- package/dist/run-events.d.ts.map +1 -0
- package/dist/run-events.js +29 -0
- package/dist/run-events.js.map +1 -0
- package/dist/server.d.ts +33 -33
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +231 -238
- package/dist/server.js.map +1 -1
- package/dist/spec-sessions.d.ts +72 -0
- package/dist/spec-sessions.d.ts.map +1 -0
- package/dist/spec-sessions.js +301 -0
- package/dist/spec-sessions.js.map +1 -0
- package/dist/terminate.d.ts +52 -0
- package/dist/terminate.d.ts.map +1 -0
- package/dist/terminate.js +111 -0
- package/dist/terminate.js.map +1 -0
- package/dist/thread-head-ping.d.ts +35 -0
- package/dist/thread-head-ping.d.ts.map +1 -0
- package/dist/thread-head-ping.js +67 -0
- package/dist/thread-head-ping.js.map +1 -0
- package/dist/thread-lifecycle.d.ts +5 -0
- package/dist/thread-lifecycle.d.ts.map +1 -0
- package/dist/thread-lifecycle.js +57 -0
- package/dist/thread-lifecycle.js.map +1 -0
- package/dist/thread-store-support.d.ts +37 -0
- package/dist/thread-store-support.d.ts.map +1 -0
- package/dist/thread-store-support.js +94 -0
- package/dist/thread-store-support.js.map +1 -0
- package/dist/threads.d.ts +59 -13
- package/dist/threads.d.ts.map +1 -1
- package/dist/threads.js +254 -141
- package/dist/threads.js.map +1 -1
- package/dist/token.d.ts +13 -4
- package/dist/token.d.ts.map +1 -1
- package/dist/token.js +134 -29
- package/dist/token.js.map +1 -1
- package/dist/writer-lease.d.ts +21 -0
- package/dist/writer-lease.d.ts.map +1 -0
- package/dist/writer-lease.js +94 -0
- package/dist/writer-lease.js.map +1 -0
- package/package.json +5 -4
package/dist/server.js
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { connect, createServer } from "node:net";
|
|
2
2
|
import { timingSafeEqual } from "node:crypto";
|
|
3
|
-
import { chmodSync,
|
|
4
|
-
import { dirname } from "node:path";
|
|
3
|
+
import { chmodSync, lstatSync, unlinkSync } from "node:fs";
|
|
5
4
|
import { createInterface } from "node:readline";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
5
|
+
import { assertNoInlineSecretValues, errorCode, newId, nowIso, pathExists, redactSecrets, } from "@claudexor/util";
|
|
6
|
+
import { commandStoreForId, commandStoreForRequest, commandStores, } from "./command-authority.js";
|
|
7
|
+
import { productCommandRecords, prunableCommandIds } from "./command-retention.js";
|
|
9
8
|
export const JOB_STATES = [
|
|
10
9
|
"queued",
|
|
11
10
|
"running",
|
|
@@ -16,81 +15,78 @@ export const JOB_STATES = [
|
|
|
16
15
|
"review_not_run",
|
|
17
16
|
"failed",
|
|
18
17
|
"cancelled",
|
|
19
|
-
"
|
|
18
|
+
"interrupted_unknown",
|
|
19
|
+
"cost_unverifiable",
|
|
20
|
+
"exhausted_overshoot",
|
|
20
21
|
"exhausted",
|
|
21
22
|
"not_converged",
|
|
22
23
|
"stuck_no_progress",
|
|
23
24
|
];
|
|
24
|
-
/**
|
|
25
|
-
* Per-record validation for the persisted registry: one hand-edited or
|
|
26
|
-
* version-skewed record must not wipe the whole run history, and a record
|
|
27
|
-
* with a state outside the enum must never reach the strict control-api
|
|
28
|
-
* DTOs (where it would 500 the entire GET /runs list).
|
|
29
|
-
*/
|
|
30
|
-
function salvageJobRecord(raw) {
|
|
31
|
-
if (!raw || typeof raw !== "object")
|
|
32
|
-
return null;
|
|
33
|
-
const rec = raw;
|
|
34
|
-
if (typeof rec["id"] !== "string" || rec["id"].length === 0)
|
|
35
|
-
return null;
|
|
36
|
-
if (typeof rec["state"] !== "string" || !JOB_STATES.includes(rec["state"]))
|
|
37
|
-
return null;
|
|
38
|
-
if (typeof rec["createdAt"] !== "string")
|
|
39
|
-
return null;
|
|
40
|
-
const optionalString = (key) => typeof rec[key] === "string" ? rec[key] : undefined;
|
|
41
|
-
return {
|
|
42
|
-
id: rec["id"],
|
|
43
|
-
state: rec["state"],
|
|
44
|
-
params: rec["params"],
|
|
45
|
-
error: optionalString("error"),
|
|
46
|
-
errorCode: optionalString("errorCode"),
|
|
47
|
-
createdAt: rec["createdAt"],
|
|
48
|
-
runId: optionalString("runId"),
|
|
49
|
-
taskId: optionalString("taskId"),
|
|
50
|
-
runDir: optionalString("runDir"),
|
|
51
|
-
startedAt: optionalString("startedAt"),
|
|
52
|
-
finishedAt: optionalString("finishedAt"),
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
/**
|
|
56
|
-
* Local daemon: Unix-socket JSON-RPC with token auth + a bounded-concurrency
|
|
57
|
-
* worker pool (up to maxConcurrent jobs in parallel) backed by an optional
|
|
58
|
-
* durable, atomically-written job registry. It does NOT contain a second
|
|
59
|
-
* scheduler — it calls the injected runner (the same Orchestrator the CLI uses).
|
|
60
|
-
*/
|
|
25
|
+
/** Unix-socket worker pool; scheduling stays in the injected Orchestrator. */
|
|
61
26
|
export class DaemonServer {
|
|
62
27
|
opts;
|
|
63
28
|
server;
|
|
29
|
+
sockets = new Set();
|
|
64
30
|
queue = [];
|
|
65
|
-
records = new Map();
|
|
66
31
|
cancelled = new Set();
|
|
67
32
|
controllers = new Map();
|
|
33
|
+
activeTasks = new Set();
|
|
34
|
+
taskFailures = [];
|
|
68
35
|
active = 0;
|
|
69
36
|
startedAt = Date.now();
|
|
70
|
-
|
|
37
|
+
stopping = false;
|
|
38
|
+
startPromise;
|
|
39
|
+
stopPromise;
|
|
40
|
+
resolveShutdown;
|
|
41
|
+
shutdownPromise = new Promise((resolve) => {
|
|
42
|
+
this.resolveShutdown = resolve;
|
|
43
|
+
});
|
|
71
44
|
constructor(opts) {
|
|
72
45
|
this.opts = opts;
|
|
73
46
|
}
|
|
74
47
|
async start() {
|
|
75
|
-
|
|
76
|
-
|
|
48
|
+
if (this.stopping) {
|
|
49
|
+
throw Object.assign(new Error("daemon is stopping and cannot be started"), {
|
|
50
|
+
code: "daemon_stopping",
|
|
51
|
+
status: 503,
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
this.startPromise ??= this.startOnce();
|
|
55
|
+
await this.startPromise;
|
|
56
|
+
if (this.stopping) {
|
|
57
|
+
await this.stop();
|
|
58
|
+
throw Object.assign(new Error("daemon startup was cancelled by shutdown"), {
|
|
59
|
+
code: "daemon_stopping",
|
|
60
|
+
status: 503,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
async startOnce() {
|
|
77
65
|
if (pathExists(this.opts.socketPath) && (await socketAlive(this.opts.socketPath))) {
|
|
78
66
|
throw new Error(`a claudexor daemon is already listening on ${this.opts.socketPath}; stop it first`);
|
|
79
67
|
}
|
|
80
|
-
this.
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
68
|
+
await this.opts.startupBarrier?.("before_registry_load");
|
|
69
|
+
if (this.stopping)
|
|
70
|
+
throw this.stoppingError("daemon startup was cancelled before listen");
|
|
71
|
+
commandStores(this.opts.commands);
|
|
72
|
+
await this.opts.startupBarrier?.("after_registry_load");
|
|
73
|
+
if (this.stopping)
|
|
74
|
+
throw this.stoppingError("daemon startup was cancelled after registry load");
|
|
75
|
+
if (pathExists(this.opts.socketPath)) {
|
|
76
|
+
const stale = lstatSync(this.opts.socketPath);
|
|
77
|
+
if (!stale.isSocket() || (process.getuid && stale.uid !== process.getuid())) {
|
|
78
|
+
throw Object.assign(new Error(`refusing to replace non-owned Unix socket path`), {
|
|
79
|
+
code: "unsafe_daemon_socket_path",
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
unlinkSync(this.opts.socketPath);
|
|
86
83
|
}
|
|
84
|
+
if (this.stopping)
|
|
85
|
+
throw this.stoppingError("daemon startup was cancelled before listen");
|
|
87
86
|
await new Promise((resolve, reject) => {
|
|
88
87
|
this.server = createServer((sock) => this.onConnection(sock));
|
|
89
88
|
this.server.once("error", reject);
|
|
90
89
|
this.server.listen(this.opts.socketPath, () => {
|
|
91
|
-
// Owner-only socket: the bearer token is the auth layer, but a
|
|
92
|
-
// world-connectable socket needlessly exposes the RPC surface to every
|
|
93
|
-
// local user; chmod narrows it to the owning account.
|
|
94
90
|
try {
|
|
95
91
|
chmodSync(this.opts.socketPath, 0o600);
|
|
96
92
|
}
|
|
@@ -100,47 +96,76 @@ export class DaemonServer {
|
|
|
100
96
|
resolve();
|
|
101
97
|
});
|
|
102
98
|
});
|
|
103
|
-
// Resume any queued jobs re-enqueued from a previous session (see load()).
|
|
104
|
-
void this.drain();
|
|
105
99
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
100
|
+
stop() {
|
|
101
|
+
this.stopping = true;
|
|
102
|
+
this.stopPromise ??= this.stopOnce();
|
|
103
|
+
return this.stopPromise;
|
|
104
|
+
}
|
|
105
|
+
async stopOnce() {
|
|
112
106
|
for (const controller of this.controllers.values()) {
|
|
113
107
|
try {
|
|
114
|
-
controller.abort();
|
|
108
|
+
controller.abort(new Error("daemon shutdown"));
|
|
115
109
|
}
|
|
116
110
|
catch {
|
|
117
111
|
/* already gone */
|
|
118
112
|
}
|
|
119
113
|
}
|
|
120
|
-
const
|
|
121
|
-
|
|
122
|
-
|
|
114
|
+
const settled = await Promise.allSettled([...this.activeTasks]);
|
|
115
|
+
const rejected = settled.filter((entry) => entry.status === "rejected");
|
|
116
|
+
if (rejected.length > 0 || this.taskFailures.length > 0 || this.active !== 0) {
|
|
117
|
+
const first = rejected[0]?.reason ??
|
|
118
|
+
this.taskFailures[0] ??
|
|
119
|
+
new Error(`daemon still owns ${this.active} active runner(s)`);
|
|
120
|
+
throw Object.assign(new Error(`daemon shutdown drain failed: ${first instanceof Error ? first.message : String(first)}`), {
|
|
121
|
+
code: "daemon_shutdown_unconfirmed",
|
|
122
|
+
status: 503,
|
|
123
|
+
cause: first,
|
|
124
|
+
});
|
|
123
125
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
+
// A signal may have fenced shutdown while listen() was still resolving.
|
|
127
|
+
// Wait for that raw startup attempt, then close whatever listener exists;
|
|
128
|
+
// start() observes `stopping` and refuses to advertise readiness.
|
|
129
|
+
if (this.startPromise) {
|
|
130
|
+
try {
|
|
131
|
+
await this.startPromise;
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
/* a failed startup has no usable listener to preserve */
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
const serverClosed = new Promise((resolve, reject) => {
|
|
126
138
|
if (!this.server)
|
|
127
139
|
return resolve();
|
|
128
|
-
|
|
140
|
+
try {
|
|
141
|
+
this.server.close((error) => (error ? reject(error) : resolve()));
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
if (error?.code === "ERR_SERVER_NOT_RUNNING")
|
|
145
|
+
resolve();
|
|
146
|
+
else
|
|
147
|
+
reject(error);
|
|
148
|
+
}
|
|
129
149
|
});
|
|
130
|
-
|
|
150
|
+
// Existing local RPC sockets can otherwise keep server.close() pending
|
|
151
|
+
// forever. They are destroyed only after every accepted command settled.
|
|
152
|
+
for (const socket of this.sockets)
|
|
153
|
+
socket.destroy();
|
|
154
|
+
await serverClosed;
|
|
155
|
+
this.resolveShutdown();
|
|
131
156
|
}
|
|
132
157
|
/** Resolves when the daemon is shut down via RPC. */
|
|
133
158
|
waitForShutdown() {
|
|
134
|
-
return
|
|
135
|
-
this.onClosed = resolve;
|
|
136
|
-
});
|
|
159
|
+
return this.shutdownPromise;
|
|
137
160
|
}
|
|
138
161
|
onConnection(sock) {
|
|
162
|
+
this.sockets.add(sock);
|
|
139
163
|
const rl = createInterface({ input: sock });
|
|
140
164
|
rl.on("line", (line) => {
|
|
141
165
|
void this.handle(line, sock);
|
|
142
166
|
});
|
|
143
167
|
sock.on("error", () => rl.close());
|
|
168
|
+
sock.on("close", () => this.sockets.delete(sock));
|
|
144
169
|
}
|
|
145
170
|
send(sock, obj) {
|
|
146
171
|
try {
|
|
@@ -170,10 +195,17 @@ export class DaemonServer {
|
|
|
170
195
|
this.send(sock, { id, result: await this.dispatch(method, params) });
|
|
171
196
|
}
|
|
172
197
|
catch (err) {
|
|
173
|
-
// Carry the machine-readable error code (e.g. inline_secret_rejected)
|
|
174
|
-
// alongside the redacted message so socket clients get the typed class.
|
|
175
198
|
const code = errorCode(err);
|
|
176
|
-
this.send(sock, {
|
|
199
|
+
this.send(sock, {
|
|
200
|
+
id,
|
|
201
|
+
error: {
|
|
202
|
+
message: redactSecrets(err instanceof Error ? err.message : String(err)),
|
|
203
|
+
...(code ? { code } : {}),
|
|
204
|
+
...(err && typeof err === "object" && "status" in err
|
|
205
|
+
? { status: Number(err.status) }
|
|
206
|
+
: {}),
|
|
207
|
+
},
|
|
208
|
+
});
|
|
177
209
|
}
|
|
178
210
|
}
|
|
179
211
|
async dispatch(method, params) {
|
|
@@ -185,43 +217,64 @@ export class DaemonServer {
|
|
|
185
217
|
queue: this.queue.length,
|
|
186
218
|
running: this.active > 0,
|
|
187
219
|
active: this.active,
|
|
188
|
-
jobs: this.
|
|
220
|
+
jobs: this.allRecords().length,
|
|
221
|
+
stopping: this.stopping,
|
|
189
222
|
};
|
|
190
223
|
case "claudexor.enqueue": {
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
224
|
+
if (this.stopping) {
|
|
225
|
+
throw Object.assign(new Error("daemon is stopping; retry after reconnect"), {
|
|
226
|
+
code: "daemon_stopping",
|
|
227
|
+
status: 503,
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
const envelope = params;
|
|
231
|
+
const request = envelope?.request;
|
|
232
|
+
const idempotencyKey = String(envelope?.idempotencyKey ?? "");
|
|
233
|
+
const clientId = String(envelope?.clientId ?? "daemon-client");
|
|
234
|
+
assertNoInlineSecretValues(request, "$", "daemon job params");
|
|
235
|
+
const accepted = this.acceptCommand(request, idempotencyKey, clientId, envelope.idempotencyRequest, typeof envelope.operation === "string" ? envelope.operation : undefined);
|
|
236
|
+
if (!accepted.reused)
|
|
237
|
+
this.queue.push(accepted.record.id);
|
|
196
238
|
void this.drain();
|
|
197
|
-
return { id, state:
|
|
239
|
+
return { id: accepted.record.id, state: accepted.record.state };
|
|
198
240
|
}
|
|
199
241
|
case "claudexor.status": {
|
|
200
|
-
const rec = this.
|
|
242
|
+
const rec = this.getRecord(String(params?.id));
|
|
201
243
|
if (!rec)
|
|
202
244
|
throw new Error(`no such job: ${params?.id}`);
|
|
203
245
|
return publicJobRecord(rec);
|
|
204
246
|
}
|
|
247
|
+
case "claudexor.findAccepted": {
|
|
248
|
+
const store = commandStoreForRequest(this.opts.commands, params?.request);
|
|
249
|
+
const record = store.find({
|
|
250
|
+
params: params?.request,
|
|
251
|
+
idempotencyKey: String(params?.idempotencyKey ?? ""),
|
|
252
|
+
clientId: String(params?.clientId ?? "daemon-client"),
|
|
253
|
+
operation: typeof params?.operation === "string" ? params.operation : undefined,
|
|
254
|
+
});
|
|
255
|
+
return record ? publicJobRecord(record) : null;
|
|
256
|
+
}
|
|
205
257
|
case "claudexor.list":
|
|
206
|
-
return
|
|
258
|
+
return productCommandRecords(this.allRecords()).map(publicJobRecord);
|
|
207
259
|
case "claudexor.cancel": {
|
|
208
260
|
const jid = String(params?.id);
|
|
209
|
-
|
|
210
|
-
// never claim `{cancelled:true}` for a job that does not exist.
|
|
211
|
-
const rec = this.records.get(jid);
|
|
261
|
+
const rec = this.getRecord(jid);
|
|
212
262
|
if (!rec)
|
|
213
263
|
throw new Error(`no such job: ${jid}`);
|
|
214
264
|
this.cancelled.add(jid);
|
|
215
265
|
if (rec.state === "queued")
|
|
216
|
-
rec
|
|
217
|
-
// Abort the in-flight run; the runner (Orchestrator) honors the signal,
|
|
218
|
-
// cancels the harness, then settles this job as cancelled.
|
|
266
|
+
this.updateRecord(rec, { state: "cancelled" });
|
|
219
267
|
this.controllers.get(jid)?.abort();
|
|
220
|
-
this.persist();
|
|
221
268
|
return { id: jid, cancelled: true };
|
|
222
269
|
}
|
|
223
270
|
case "claudexor.shutdown":
|
|
224
|
-
setTimeout(() =>
|
|
271
|
+
setTimeout(() => {
|
|
272
|
+
const operation = this.opts.onShutdownRequested?.() ?? this.stop();
|
|
273
|
+
void operation.catch(() => {
|
|
274
|
+
// Fail closed: the process and ownership lease remain alive. The
|
|
275
|
+
// composition root records the detailed failure in its private log.
|
|
276
|
+
});
|
|
277
|
+
}, 10);
|
|
225
278
|
return { ok: true };
|
|
226
279
|
default:
|
|
227
280
|
throw new Error(`unknown method: ${method}`);
|
|
@@ -230,104 +283,39 @@ export class DaemonServer {
|
|
|
230
283
|
get maxConcurrent() {
|
|
231
284
|
return this.opts.maxConcurrent ?? 4;
|
|
232
285
|
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
* (temp file + rename) so a crash mid-write cannot corrupt/drop the registry.
|
|
236
|
-
* The raw run `result` is intentionally NOT persisted: canonical output lives
|
|
237
|
-
* in .claudexor/runs (redacted), and result.summary can contain raw model text —
|
|
238
|
-
* keeping it out of jobs.json upholds the redaction-at-persistence invariant.
|
|
239
|
-
*/
|
|
240
|
-
persist() {
|
|
241
|
-
const path = this.opts.persistPath;
|
|
242
|
-
if (!path)
|
|
243
|
-
return;
|
|
244
|
-
try {
|
|
245
|
-
const view = [...this.records.values()].map(persistedJobRecord);
|
|
246
|
-
mkdirSync(dirname(path), { recursive: true });
|
|
247
|
-
const tmp = `${path}.tmp`;
|
|
248
|
-
writeFileSync(tmp, JSON.stringify(view, null, 2) + "\n", { mode: 0o600 });
|
|
249
|
-
chmodSync(tmp, 0o600);
|
|
250
|
-
renameSync(tmp, path);
|
|
251
|
-
chmodSync(path, 0o600);
|
|
252
|
-
}
|
|
253
|
-
catch {
|
|
254
|
-
/* best-effort; never break a run on a persistence failure */
|
|
255
|
-
}
|
|
286
|
+
stoppingError(message) {
|
|
287
|
+
return Object.assign(new Error(message), { code: "daemon_stopping", status: 503 });
|
|
256
288
|
}
|
|
257
|
-
/** Bound memory/disk: prune the oldest terminal jobs beyond maxHistory.
|
|
258
|
-
* `blocked` runs are NEVER pruned — they are the needs-human inbox awaiting an
|
|
259
|
-
* operator decision; dropping one would silently lose a pending action. */
|
|
260
289
|
pruneHistory() {
|
|
261
|
-
const
|
|
262
|
-
const
|
|
263
|
-
|
|
264
|
-
return;
|
|
265
|
-
terminal.sort((a, b) => (a.createdAt < b.createdAt ? -1 : 1));
|
|
266
|
-
for (const r of terminal.slice(0, terminal.length - cap)) {
|
|
267
|
-
this.records.delete(r.id);
|
|
268
|
-
this.cancelled.delete(r.id);
|
|
290
|
+
const removed = prunableCommandIds(this.allRecords(), this.opts.maxHistory ?? 500, this.opts.idempotencyRetentionMs ?? 30 * 24 * 60 * 60 * 1_000, (this.opts.now ?? (() => new Date()))().getTime());
|
|
291
|
+
for (const store of commandStores(this.opts.commands)) {
|
|
292
|
+
store.prune(removed.filter((id) => store.get(id)));
|
|
269
293
|
}
|
|
294
|
+
for (const id of removed)
|
|
295
|
+
this.cancelled.delete(id);
|
|
270
296
|
}
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
this.backupCorruptRegistry(path, "registry file is not a JSON array");
|
|
282
|
-
return;
|
|
283
|
-
}
|
|
284
|
-
let dropped = 0;
|
|
285
|
-
for (const raw of saved) {
|
|
286
|
-
const rec = salvageJobRecord(raw);
|
|
287
|
-
if (!rec) {
|
|
288
|
-
dropped += 1;
|
|
289
|
-
continue;
|
|
290
|
-
}
|
|
291
|
-
// A fresh process cannot resume an in-memory RUN, so a `running` job becomes
|
|
292
|
-
// interrupted (honest). `blocked` is a TERMINAL outcome (NEEDS_HUMAN / web
|
|
293
|
-
// policy) the review queue must keep across restarts.
|
|
294
|
-
if (rec.state === "running") {
|
|
295
|
-
rec.state = "interrupted";
|
|
296
|
-
// Stamp the orphaned event log with a TERMINAL event: the
|
|
297
|
-
// canonical events.jsonl must agree with jobs.json, or SSE tailers and
|
|
298
|
-
// `follow` wait forever on a log that will never terminate.
|
|
299
|
-
if (rec.runDir && rec.runId) {
|
|
300
|
-
try {
|
|
301
|
-
appendRunEvent(join(rec.runDir, "events.jsonl"), rec.runId, rec.taskId ?? "", "run.failed", {
|
|
302
|
-
status: "interrupted",
|
|
303
|
-
error: "daemon restarted while the run was in flight",
|
|
304
|
-
});
|
|
305
|
-
}
|
|
306
|
-
catch {
|
|
307
|
-
/* best-effort: a missing/corrupt log must not block startup */
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
this.records.set(rec.id, rec);
|
|
312
|
-
// A `queued` job never started; its params are persisted, so RE-ENQUEUE it
|
|
313
|
-
// on restart (drain() runs after start()) instead of silently dropping
|
|
314
|
-
// pending work to interrupted.
|
|
315
|
-
if (rec.state === "queued")
|
|
316
|
-
this.queue.push(rec.id);
|
|
317
|
-
}
|
|
318
|
-
if (dropped > 0) {
|
|
319
|
-
this.backupCorruptRegistry(path, `${dropped} unparseable job record(s) dropped`);
|
|
320
|
-
}
|
|
297
|
+
acceptCommand(params, idempotencyKey, clientId, idempotencyParams, operation) {
|
|
298
|
+
const store = commandStoreForRequest(this.opts.commands, params);
|
|
299
|
+
return store.accept({
|
|
300
|
+
id: newId("job"),
|
|
301
|
+
params,
|
|
302
|
+
idempotencyKey,
|
|
303
|
+
clientId,
|
|
304
|
+
idempotencyParams,
|
|
305
|
+
operation,
|
|
306
|
+
});
|
|
321
307
|
}
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
308
|
+
allRecords() {
|
|
309
|
+
return commandStores(this.opts.commands).flatMap((store) => store.records());
|
|
310
|
+
}
|
|
311
|
+
getRecord(id) {
|
|
312
|
+
return commandStoreForId(this.opts.commands, id)?.get(id);
|
|
313
|
+
}
|
|
314
|
+
updateRecord(record, patch) {
|
|
315
|
+
const store = commandStoreForId(this.opts.commands, record.id);
|
|
316
|
+
if (!store)
|
|
317
|
+
throw new Error(`command authority lost job ${record.id}`);
|
|
318
|
+
return store.update(record.id, patch);
|
|
331
319
|
}
|
|
332
320
|
threadIdOf(rec) {
|
|
333
321
|
const p = rec.params;
|
|
@@ -344,14 +332,16 @@ export class DaemonServer {
|
|
|
344
332
|
* turn starts as soon as its previous one settles.
|
|
345
333
|
*/
|
|
346
334
|
drain() {
|
|
335
|
+
if (this.stopping)
|
|
336
|
+
return;
|
|
347
337
|
while (this.active < this.maxConcurrent && this.queue.length > 0) {
|
|
348
|
-
const busyThreads = new Set(
|
|
338
|
+
const busyThreads = new Set(this.allRecords()
|
|
349
339
|
.filter((r) => r.state === "running")
|
|
350
340
|
.map((r) => this.threadIdOf(r))
|
|
351
341
|
.filter((t) => !!t));
|
|
352
342
|
let pickIdx = -1;
|
|
353
343
|
for (let i = 0; i < this.queue.length; i++) {
|
|
354
|
-
const rec = this.
|
|
344
|
+
const rec = this.getRecord(this.queue[i]);
|
|
355
345
|
const tid = rec ? this.threadIdOf(rec) : undefined;
|
|
356
346
|
if (!rec || !tid || !busyThreads.has(tid)) {
|
|
357
347
|
pickIdx = i;
|
|
@@ -361,59 +351,75 @@ export class DaemonServer {
|
|
|
361
351
|
if (pickIdx === -1)
|
|
362
352
|
break; // every queued job waits on a busy thread
|
|
363
353
|
const id = this.queue.splice(pickIdx, 1)[0];
|
|
364
|
-
const rec = this.
|
|
354
|
+
const rec = this.getRecord(id);
|
|
365
355
|
if (!rec)
|
|
366
356
|
continue;
|
|
367
357
|
if (this.cancelled.has(id)) {
|
|
368
|
-
rec
|
|
358
|
+
this.updateRecord(rec, { state: "cancelled", finishedAt: nowIso() });
|
|
369
359
|
continue;
|
|
370
360
|
}
|
|
371
361
|
this.active += 1;
|
|
372
|
-
|
|
362
|
+
const task = this.runJob(id, rec);
|
|
363
|
+
this.activeTasks.add(task);
|
|
364
|
+
void task.then(() => this.activeTasks.delete(task), (error) => {
|
|
365
|
+
this.activeTasks.delete(task);
|
|
366
|
+
this.taskFailures.push(error);
|
|
367
|
+
});
|
|
373
368
|
}
|
|
374
369
|
}
|
|
375
370
|
async runJob(id, rec) {
|
|
376
371
|
const controller = new AbortController();
|
|
377
372
|
this.controllers.set(id, controller);
|
|
378
|
-
rec.state
|
|
379
|
-
rec.startedAt = nowIso();
|
|
380
|
-
this.persist();
|
|
373
|
+
rec = this.updateRecord(rec, { state: "running", startedAt: nowIso() });
|
|
381
374
|
try {
|
|
382
|
-
|
|
375
|
+
const result = await this.opts.runner(rec.params, {
|
|
383
376
|
signal: controller.signal,
|
|
384
377
|
onRunStart: (info) => {
|
|
385
|
-
rec
|
|
386
|
-
rec.taskId = info.taskId;
|
|
387
|
-
rec.runDir = info.runDir;
|
|
388
|
-
// Persist the pointer immediately so a mid-run crash still reloads with
|
|
389
|
-
// runId/runDir to locate .claudexor/runs/<runId> (the recovery path).
|
|
390
|
-
this.persist();
|
|
378
|
+
rec = this.updateRecord(rec, info);
|
|
391
379
|
},
|
|
392
380
|
});
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
381
|
+
const state = jobStateFromResult(result, controller.signal.aborted);
|
|
382
|
+
if (state === "failed" ||
|
|
383
|
+
state === "cost_unverifiable" ||
|
|
384
|
+
state === "exhausted_overshoot" ||
|
|
385
|
+
state === "exhausted" ||
|
|
386
|
+
state === "not_converged" ||
|
|
387
|
+
state === "stuck_no_progress") {
|
|
388
|
+
rec = this.updateRecord(rec, {
|
|
389
|
+
state,
|
|
390
|
+
result,
|
|
391
|
+
error: resultSummary(result) ?? `run ended with status ${state}`,
|
|
392
|
+
finishedAt: nowIso(),
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
else {
|
|
396
|
+
rec = this.updateRecord(rec, { state, result, finishedAt: nowIso() });
|
|
399
397
|
}
|
|
400
398
|
}
|
|
401
399
|
catch (err) {
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
400
|
+
const code = err && typeof err === "object" && "code" in err
|
|
401
|
+
? err.code
|
|
402
|
+
: undefined;
|
|
403
|
+
const status = err && typeof err === "object" && "status" in err
|
|
404
|
+
? err.status
|
|
405
|
+
: undefined;
|
|
406
|
+
const typedStatus = typeof status === "number" && Number.isInteger(status) && status >= 400 && status <= 599
|
|
407
|
+
? status
|
|
408
|
+
: undefined;
|
|
409
|
+
rec = this.updateRecord(rec, {
|
|
410
|
+
state: controller.signal.aborted ? "cancelled" : "failed",
|
|
411
|
+
error: redactSecrets(err instanceof Error ? err.message : String(err)),
|
|
412
|
+
...(typeof code === "string" && code ? { errorCode: code } : {}),
|
|
413
|
+
...(typedStatus !== undefined ? { errorStatus: typedStatus } : {}),
|
|
414
|
+
finishedAt: nowIso(),
|
|
415
|
+
});
|
|
409
416
|
}
|
|
410
417
|
finally {
|
|
411
|
-
rec.finishedAt = nowIso();
|
|
412
418
|
this.controllers.delete(id);
|
|
413
419
|
this.active -= 1;
|
|
414
420
|
if (rec.runId) {
|
|
415
421
|
try {
|
|
416
|
-
this.opts.onRunTerminal?.(rec.runId);
|
|
422
|
+
this.opts.onRunTerminal?.(rec.runId, this.threadIdOf(rec));
|
|
417
423
|
}
|
|
418
424
|
catch {
|
|
419
425
|
/* observer failure must not corrupt terminal bookkeeping */
|
|
@@ -434,8 +440,8 @@ export class DaemonServer {
|
|
|
434
440
|
}
|
|
435
441
|
}
|
|
436
442
|
this.pruneHistory();
|
|
437
|
-
this.
|
|
438
|
-
|
|
443
|
+
if (!this.stopping)
|
|
444
|
+
this.drain();
|
|
439
445
|
}
|
|
440
446
|
}
|
|
441
447
|
}
|
|
@@ -457,7 +463,9 @@ function jobStateFromResult(result, aborted) {
|
|
|
457
463
|
case "cancelled":
|
|
458
464
|
return "cancelled";
|
|
459
465
|
case "exhausted":
|
|
460
|
-
|
|
466
|
+
case "exhausted_overshoot":
|
|
467
|
+
case "cost_unverifiable":
|
|
468
|
+
return status;
|
|
461
469
|
case "not_converged":
|
|
462
470
|
return "not_converged";
|
|
463
471
|
case "stuck_no_progress":
|
|
@@ -478,7 +486,6 @@ function tokenMatches(candidate, expected) {
|
|
|
478
486
|
return false;
|
|
479
487
|
return timingSafeEqual(a, b);
|
|
480
488
|
}
|
|
481
|
-
/** True when something is actively accepting connections on the socket path. */
|
|
482
489
|
/** Is a daemon already listening on this socket? Exported so the claudexord
|
|
483
490
|
* entrypoint can refuse BEFORE running crash GC — a second daemon must never
|
|
484
491
|
* reap the live daemon's children or sweep envelopes its jobs still own. */
|
|
@@ -513,21 +520,6 @@ function publicJobRecord(rec) {
|
|
|
513
520
|
params: redactParams(rec.params),
|
|
514
521
|
};
|
|
515
522
|
}
|
|
516
|
-
function persistedJobRecord(rec) {
|
|
517
|
-
return {
|
|
518
|
-
id: rec.id,
|
|
519
|
-
state: rec.state,
|
|
520
|
-
params: redactParams(rec.params),
|
|
521
|
-
error: rec.error ? redactSecrets(rec.error) : undefined,
|
|
522
|
-
errorCode: rec.errorCode,
|
|
523
|
-
createdAt: rec.createdAt,
|
|
524
|
-
runId: rec.runId,
|
|
525
|
-
taskId: rec.taskId,
|
|
526
|
-
runDir: rec.runDir,
|
|
527
|
-
startedAt: rec.startedAt,
|
|
528
|
-
finishedAt: rec.finishedAt,
|
|
529
|
-
};
|
|
530
|
-
}
|
|
531
523
|
function redactParams(value) {
|
|
532
524
|
if (typeof value === "string")
|
|
533
525
|
return redactSecrets(value);
|
|
@@ -537,7 +529,8 @@ function redactParams(value) {
|
|
|
537
529
|
return value;
|
|
538
530
|
const out = {};
|
|
539
531
|
for (const [key, child] of Object.entries(value)) {
|
|
540
|
-
out[key] =
|
|
532
|
+
out[key] =
|
|
533
|
+
key === "prompt" && typeof child === "string" ? redactSecrets(child) : redactParams(child);
|
|
541
534
|
}
|
|
542
535
|
return out;
|
|
543
536
|
}
|