@claudexor/daemon 1.0.1 → 2.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/client.d.ts +29 -1
  2. package/dist/client.d.ts.map +1 -1
  3. package/dist/client.js +24 -4
  4. package/dist/client.js.map +1 -1
  5. package/dist/command-authority.d.ts +11 -0
  6. package/dist/command-authority.d.ts.map +1 -0
  7. package/dist/command-authority.js +15 -0
  8. package/dist/command-authority.js.map +1 -0
  9. package/dist/command-retention.d.ts +5 -0
  10. package/dist/command-retention.d.ts.map +1 -0
  11. package/dist/command-retention.js +18 -0
  12. package/dist/command-retention.js.map +1 -0
  13. package/dist/command-store.d.ts +41 -0
  14. package/dist/command-store.d.ts.map +1 -0
  15. package/dist/command-store.js +197 -0
  16. package/dist/command-store.js.map +1 -0
  17. package/dist/events.d.ts +2 -3
  18. package/dist/events.d.ts.map +1 -1
  19. package/dist/events.js +2 -3
  20. package/dist/events.js.map +1 -1
  21. package/dist/index.d.ts +13 -0
  22. package/dist/index.d.ts.map +1 -1
  23. package/dist/index.js +13 -0
  24. package/dist/index.js.map +1 -1
  25. package/dist/interactions.d.ts +32 -22
  26. package/dist/interactions.d.ts.map +1 -1
  27. package/dist/interactions.js +183 -56
  28. package/dist/interactions.js.map +1 -1
  29. package/dist/journal-events.d.ts +4 -0
  30. package/dist/journal-events.d.ts.map +1 -0
  31. package/dist/journal-events.js +23 -0
  32. package/dist/journal-events.js.map +1 -0
  33. package/dist/journal-manager.d.ts +72 -0
  34. package/dist/journal-manager.d.ts.map +1 -0
  35. package/dist/journal-manager.js +463 -0
  36. package/dist/journal-manager.js.map +1 -0
  37. package/dist/journal-projection.d.ts +11 -0
  38. package/dist/journal-projection.d.ts.map +1 -0
  39. package/dist/journal-projection.js +2 -0
  40. package/dist/journal-projection.js.map +1 -0
  41. package/dist/journal-recovery-files.d.ts +27 -0
  42. package/dist/journal-recovery-files.d.ts.map +1 -0
  43. package/dist/journal-recovery-files.js +209 -0
  44. package/dist/journal-recovery-files.js.map +1 -0
  45. package/dist/operator-decisions.d.ts +30 -0
  46. package/dist/operator-decisions.d.ts.map +1 -0
  47. package/dist/operator-decisions.js +137 -0
  48. package/dist/operator-decisions.js.map +1 -0
  49. package/dist/project-partitions.d.ts +167 -0
  50. package/dist/project-partitions.d.ts.map +1 -0
  51. package/dist/project-partitions.js +365 -0
  52. package/dist/project-partitions.js.map +1 -0
  53. package/dist/projects.d.ts +29 -0
  54. package/dist/projects.d.ts.map +1 -0
  55. package/dist/projects.js +180 -0
  56. package/dist/projects.js.map +1 -0
  57. package/dist/quota-registry.d.ts +75 -0
  58. package/dist/quota-registry.d.ts.map +1 -0
  59. package/dist/quota-registry.js +208 -0
  60. package/dist/quota-registry.js.map +1 -0
  61. package/dist/resource-store.d.ts +22 -0
  62. package/dist/resource-store.d.ts.map +1 -0
  63. package/dist/resource-store.js +272 -0
  64. package/dist/resource-store.js.map +1 -0
  65. package/dist/run-events.d.ts +15 -0
  66. package/dist/run-events.d.ts.map +1 -0
  67. package/dist/run-events.js +29 -0
  68. package/dist/run-events.js.map +1 -0
  69. package/dist/server.d.ts +33 -33
  70. package/dist/server.d.ts.map +1 -1
  71. package/dist/server.js +231 -238
  72. package/dist/server.js.map +1 -1
  73. package/dist/spec-sessions.d.ts +72 -0
  74. package/dist/spec-sessions.d.ts.map +1 -0
  75. package/dist/spec-sessions.js +301 -0
  76. package/dist/spec-sessions.js.map +1 -0
  77. package/dist/terminate.d.ts +52 -0
  78. package/dist/terminate.d.ts.map +1 -0
  79. package/dist/terminate.js +111 -0
  80. package/dist/terminate.js.map +1 -0
  81. package/dist/thread-head-ping.d.ts +35 -0
  82. package/dist/thread-head-ping.d.ts.map +1 -0
  83. package/dist/thread-head-ping.js +67 -0
  84. package/dist/thread-head-ping.js.map +1 -0
  85. package/dist/thread-lifecycle.d.ts +5 -0
  86. package/dist/thread-lifecycle.d.ts.map +1 -0
  87. package/dist/thread-lifecycle.js +57 -0
  88. package/dist/thread-lifecycle.js.map +1 -0
  89. package/dist/thread-store-support.d.ts +37 -0
  90. package/dist/thread-store-support.d.ts.map +1 -0
  91. package/dist/thread-store-support.js +94 -0
  92. package/dist/thread-store-support.js.map +1 -0
  93. package/dist/threads.d.ts +59 -13
  94. package/dist/threads.d.ts.map +1 -1
  95. package/dist/threads.js +254 -141
  96. package/dist/threads.js.map +1 -1
  97. package/dist/token.d.ts +13 -4
  98. package/dist/token.d.ts.map +1 -1
  99. package/dist/token.js +134 -29
  100. package/dist/token.js.map +1 -1
  101. package/dist/writer-lease.d.ts +21 -0
  102. package/dist/writer-lease.d.ts.map +1 -0
  103. package/dist/writer-lease.js +94 -0
  104. package/dist/writer-lease.js.map +1 -0
  105. package/package.json +5 -4
package/dist/server.js CHANGED
@@ -1,11 +1,10 @@
1
1
  import { connect, createServer } from "node:net";
2
2
  import { timingSafeEqual } from "node:crypto";
3
- import { chmodSync, copyFileSync, mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
4
- import { dirname } from "node:path";
3
+ import { chmodSync, lstatSync, unlinkSync } from "node:fs";
5
4
  import { createInterface } from "node:readline";
6
- import { join } from "node:path";
7
- import { appendRunEvent } from "@claudexor/event-log";
8
- import { assertNoInlineSecretValues, errorCode, newId, nowIso, pathExists, readJsonSafe, redactSecrets } from "@claudexor/util";
5
+ import { assertNoInlineSecretValues, errorCode, newId, nowIso, pathExists, redactSecrets, } from "@claudexor/util";
6
+ import { commandStoreForId, commandStoreForRequest, commandStores, } from "./command-authority.js";
7
+ import { productCommandRecords, prunableCommandIds } from "./command-retention.js";
9
8
  export const JOB_STATES = [
10
9
  "queued",
11
10
  "running",
@@ -16,81 +15,78 @@ export const JOB_STATES = [
16
15
  "review_not_run",
17
16
  "failed",
18
17
  "cancelled",
19
- "interrupted",
18
+ "interrupted_unknown",
19
+ "cost_unverifiable",
20
+ "exhausted_overshoot",
20
21
  "exhausted",
21
22
  "not_converged",
22
23
  "stuck_no_progress",
23
24
  ];
24
- /**
25
- * Per-record validation for the persisted registry: one hand-edited or
26
- * version-skewed record must not wipe the whole run history, and a record
27
- * with a state outside the enum must never reach the strict control-api
28
- * DTOs (where it would 500 the entire GET /runs list).
29
- */
30
- function salvageJobRecord(raw) {
31
- if (!raw || typeof raw !== "object")
32
- return null;
33
- const rec = raw;
34
- if (typeof rec["id"] !== "string" || rec["id"].length === 0)
35
- return null;
36
- if (typeof rec["state"] !== "string" || !JOB_STATES.includes(rec["state"]))
37
- return null;
38
- if (typeof rec["createdAt"] !== "string")
39
- return null;
40
- const optionalString = (key) => typeof rec[key] === "string" ? rec[key] : undefined;
41
- return {
42
- id: rec["id"],
43
- state: rec["state"],
44
- params: rec["params"],
45
- error: optionalString("error"),
46
- errorCode: optionalString("errorCode"),
47
- createdAt: rec["createdAt"],
48
- runId: optionalString("runId"),
49
- taskId: optionalString("taskId"),
50
- runDir: optionalString("runDir"),
51
- startedAt: optionalString("startedAt"),
52
- finishedAt: optionalString("finishedAt"),
53
- };
54
- }
55
- /**
56
- * Local daemon: Unix-socket JSON-RPC with token auth + a bounded-concurrency
57
- * worker pool (up to maxConcurrent jobs in parallel) backed by an optional
58
- * durable, atomically-written job registry. It does NOT contain a second
59
- * scheduler — it calls the injected runner (the same Orchestrator the CLI uses).
60
- */
25
+ /** Unix-socket worker pool; scheduling stays in the injected Orchestrator. */
61
26
  export class DaemonServer {
62
27
  opts;
63
28
  server;
29
+ sockets = new Set();
64
30
  queue = [];
65
- records = new Map();
66
31
  cancelled = new Set();
67
32
  controllers = new Map();
33
+ activeTasks = new Set();
34
+ taskFailures = [];
68
35
  active = 0;
69
36
  startedAt = Date.now();
70
- onClosed;
37
+ stopping = false;
38
+ startPromise;
39
+ stopPromise;
40
+ resolveShutdown;
41
+ shutdownPromise = new Promise((resolve) => {
42
+ this.resolveShutdown = resolve;
43
+ });
71
44
  constructor(opts) {
72
45
  this.opts = opts;
73
46
  }
74
47
  async start() {
75
- // Refuse to clobber a LIVE daemon: deleting its socket would orphan it and
76
- // turn jobs.json into a last-writer-wins race between two processes.
48
+ if (this.stopping) {
49
+ throw Object.assign(new Error("daemon is stopping and cannot be started"), {
50
+ code: "daemon_stopping",
51
+ status: 503,
52
+ });
53
+ }
54
+ this.startPromise ??= this.startOnce();
55
+ await this.startPromise;
56
+ if (this.stopping) {
57
+ await this.stop();
58
+ throw Object.assign(new Error("daemon startup was cancelled by shutdown"), {
59
+ code: "daemon_stopping",
60
+ status: 503,
61
+ });
62
+ }
63
+ }
64
+ async startOnce() {
77
65
  if (pathExists(this.opts.socketPath) && (await socketAlive(this.opts.socketPath))) {
78
66
  throw new Error(`a claudexor daemon is already listening on ${this.opts.socketPath}; stop it first`);
79
67
  }
80
- this.load();
81
- try {
82
- rmSync(this.opts.socketPath, { force: true });
83
- }
84
- catch {
85
- /* nothing to clean */
68
+ await this.opts.startupBarrier?.("before_registry_load");
69
+ if (this.stopping)
70
+ throw this.stoppingError("daemon startup was cancelled before listen");
71
+ commandStores(this.opts.commands);
72
+ await this.opts.startupBarrier?.("after_registry_load");
73
+ if (this.stopping)
74
+ throw this.stoppingError("daemon startup was cancelled after registry load");
75
+ if (pathExists(this.opts.socketPath)) {
76
+ const stale = lstatSync(this.opts.socketPath);
77
+ if (!stale.isSocket() || (process.getuid && stale.uid !== process.getuid())) {
78
+ throw Object.assign(new Error(`refusing to replace non-owned Unix socket path`), {
79
+ code: "unsafe_daemon_socket_path",
80
+ });
81
+ }
82
+ unlinkSync(this.opts.socketPath);
86
83
  }
84
+ if (this.stopping)
85
+ throw this.stoppingError("daemon startup was cancelled before listen");
87
86
  await new Promise((resolve, reject) => {
88
87
  this.server = createServer((sock) => this.onConnection(sock));
89
88
  this.server.once("error", reject);
90
89
  this.server.listen(this.opts.socketPath, () => {
91
- // Owner-only socket: the bearer token is the auth layer, but a
92
- // world-connectable socket needlessly exposes the RPC surface to every
93
- // local user; chmod narrows it to the owning account.
94
90
  try {
95
91
  chmodSync(this.opts.socketPath, 0o600);
96
92
  }
@@ -100,47 +96,76 @@ export class DaemonServer {
100
96
  resolve();
101
97
  });
102
98
  });
103
- // Resume any queued jobs re-enqueued from a previous session (see load()).
104
- void this.drain();
105
99
  }
106
- async stop() {
107
- // Graceful shutdown: abort in-flight runs so the runner cancels their harness
108
- // children and settles each job (no orphaned processes / "running" zombies in
109
- // jobs.json), then WAIT (bounded) for the cancellations to settle. Without
110
- // the wait, the process could exit before the SIGKILL escalation timers
111
- // fire, leaving a SIGTERM-ignoring harness child alive in its group.
100
+ stop() {
101
+ this.stopping = true;
102
+ this.stopPromise ??= this.stopOnce();
103
+ return this.stopPromise;
104
+ }
105
+ async stopOnce() {
112
106
  for (const controller of this.controllers.values()) {
113
107
  try {
114
- controller.abort();
108
+ controller.abort(new Error("daemon shutdown"));
115
109
  }
116
110
  catch {
117
111
  /* already gone */
118
112
  }
119
113
  }
120
- const deadline = Date.now() + 5_000;
121
- while (this.active > 0 && Date.now() < deadline) {
122
- await new Promise((resolve) => setTimeout(resolve, 50));
114
+ const settled = await Promise.allSettled([...this.activeTasks]);
115
+ const rejected = settled.filter((entry) => entry.status === "rejected");
116
+ if (rejected.length > 0 || this.taskFailures.length > 0 || this.active !== 0) {
117
+ const first = rejected[0]?.reason ??
118
+ this.taskFailures[0] ??
119
+ new Error(`daemon still owns ${this.active} active runner(s)`);
120
+ throw Object.assign(new Error(`daemon shutdown drain failed: ${first instanceof Error ? first.message : String(first)}`), {
121
+ code: "daemon_shutdown_unconfirmed",
122
+ status: 503,
123
+ cause: first,
124
+ });
123
125
  }
124
- this.persist();
125
- await new Promise((resolve) => {
126
+ // A signal may have fenced shutdown while listen() was still resolving.
127
+ // Wait for that raw startup attempt, then close whatever listener exists;
128
+ // start() observes `stopping` and refuses to advertise readiness.
129
+ if (this.startPromise) {
130
+ try {
131
+ await this.startPromise;
132
+ }
133
+ catch {
134
+ /* a failed startup has no usable listener to preserve */
135
+ }
136
+ }
137
+ const serverClosed = new Promise((resolve, reject) => {
126
138
  if (!this.server)
127
139
  return resolve();
128
- this.server.close(() => resolve());
140
+ try {
141
+ this.server.close((error) => (error ? reject(error) : resolve()));
142
+ }
143
+ catch (error) {
144
+ if (error?.code === "ERR_SERVER_NOT_RUNNING")
145
+ resolve();
146
+ else
147
+ reject(error);
148
+ }
129
149
  });
130
- this.onClosed?.();
150
+ // Existing local RPC sockets can otherwise keep server.close() pending
151
+ // forever. They are destroyed only after every accepted command settled.
152
+ for (const socket of this.sockets)
153
+ socket.destroy();
154
+ await serverClosed;
155
+ this.resolveShutdown();
131
156
  }
132
157
  /** Resolves when the daemon is shut down via RPC. */
133
158
  waitForShutdown() {
134
- return new Promise((resolve) => {
135
- this.onClosed = resolve;
136
- });
159
+ return this.shutdownPromise;
137
160
  }
138
161
  onConnection(sock) {
162
+ this.sockets.add(sock);
139
163
  const rl = createInterface({ input: sock });
140
164
  rl.on("line", (line) => {
141
165
  void this.handle(line, sock);
142
166
  });
143
167
  sock.on("error", () => rl.close());
168
+ sock.on("close", () => this.sockets.delete(sock));
144
169
  }
145
170
  send(sock, obj) {
146
171
  try {
@@ -170,10 +195,17 @@ export class DaemonServer {
170
195
  this.send(sock, { id, result: await this.dispatch(method, params) });
171
196
  }
172
197
  catch (err) {
173
- // Carry the machine-readable error code (e.g. inline_secret_rejected)
174
- // alongside the redacted message so socket clients get the typed class.
175
198
  const code = errorCode(err);
176
- this.send(sock, { id, error: { message: redactSecrets(err instanceof Error ? err.message : String(err)), ...(code ? { code } : {}) } });
199
+ this.send(sock, {
200
+ id,
201
+ error: {
202
+ message: redactSecrets(err instanceof Error ? err.message : String(err)),
203
+ ...(code ? { code } : {}),
204
+ ...(err && typeof err === "object" && "status" in err
205
+ ? { status: Number(err.status) }
206
+ : {}),
207
+ },
208
+ });
177
209
  }
178
210
  }
179
211
  async dispatch(method, params) {
@@ -185,43 +217,64 @@ export class DaemonServer {
185
217
  queue: this.queue.length,
186
218
  running: this.active > 0,
187
219
  active: this.active,
188
- jobs: this.records.size,
220
+ jobs: this.allRecords().length,
221
+ stopping: this.stopping,
189
222
  };
190
223
  case "claudexor.enqueue": {
191
- assertNoInlineSecretValues(params, "$", "daemon job params");
192
- const id = newId("job");
193
- this.records.set(id, { id, state: "queued", params, createdAt: nowIso() });
194
- this.queue.push(id);
195
- this.persist();
224
+ if (this.stopping) {
225
+ throw Object.assign(new Error("daemon is stopping; retry after reconnect"), {
226
+ code: "daemon_stopping",
227
+ status: 503,
228
+ });
229
+ }
230
+ const envelope = params;
231
+ const request = envelope?.request;
232
+ const idempotencyKey = String(envelope?.idempotencyKey ?? "");
233
+ const clientId = String(envelope?.clientId ?? "daemon-client");
234
+ assertNoInlineSecretValues(request, "$", "daemon job params");
235
+ const accepted = this.acceptCommand(request, idempotencyKey, clientId, envelope.idempotencyRequest, typeof envelope.operation === "string" ? envelope.operation : undefined);
236
+ if (!accepted.reused)
237
+ this.queue.push(accepted.record.id);
196
238
  void this.drain();
197
- return { id, state: "queued" };
239
+ return { id: accepted.record.id, state: accepted.record.state };
198
240
  }
199
241
  case "claudexor.status": {
200
- const rec = this.records.get(String(params?.id));
242
+ const rec = this.getRecord(String(params?.id));
201
243
  if (!rec)
202
244
  throw new Error(`no such job: ${params?.id}`);
203
245
  return publicJobRecord(rec);
204
246
  }
247
+ case "claudexor.findAccepted": {
248
+ const store = commandStoreForRequest(this.opts.commands, params?.request);
249
+ const record = store.find({
250
+ params: params?.request,
251
+ idempotencyKey: String(params?.idempotencyKey ?? ""),
252
+ clientId: String(params?.clientId ?? "daemon-client"),
253
+ operation: typeof params?.operation === "string" ? params.operation : undefined,
254
+ });
255
+ return record ? publicJobRecord(record) : null;
256
+ }
205
257
  case "claudexor.list":
206
- return [...this.records.values()].map(publicJobRecord);
258
+ return productCommandRecords(this.allRecords()).map(publicJobRecord);
207
259
  case "claudexor.cancel": {
208
260
  const jid = String(params?.id);
209
- // Honesty: cancelling an unknown id must fail loudly (like status),
210
- // never claim `{cancelled:true}` for a job that does not exist.
211
- const rec = this.records.get(jid);
261
+ const rec = this.getRecord(jid);
212
262
  if (!rec)
213
263
  throw new Error(`no such job: ${jid}`);
214
264
  this.cancelled.add(jid);
215
265
  if (rec.state === "queued")
216
- rec.state = "cancelled";
217
- // Abort the in-flight run; the runner (Orchestrator) honors the signal,
218
- // cancels the harness, then settles this job as cancelled.
266
+ this.updateRecord(rec, { state: "cancelled" });
219
267
  this.controllers.get(jid)?.abort();
220
- this.persist();
221
268
  return { id: jid, cancelled: true };
222
269
  }
223
270
  case "claudexor.shutdown":
224
- setTimeout(() => void this.stop(), 10);
271
+ setTimeout(() => {
272
+ const operation = this.opts.onShutdownRequested?.() ?? this.stop();
273
+ void operation.catch(() => {
274
+ // Fail closed: the process and ownership lease remain alive. The
275
+ // composition root records the detailed failure in its private log.
276
+ });
277
+ }, 10);
225
278
  return { ok: true };
226
279
  default:
227
280
  throw new Error(`unknown method: ${method}`);
@@ -230,104 +283,39 @@ export class DaemonServer {
230
283
  get maxConcurrent() {
231
284
  return this.opts.maxConcurrent ?? 4;
232
285
  }
233
- /**
234
- * Best-effort durable persistence of the job registry. Writes atomically
235
- * (temp file + rename) so a crash mid-write cannot corrupt/drop the registry.
236
- * The raw run `result` is intentionally NOT persisted: canonical output lives
237
- * in .claudexor/runs (redacted), and result.summary can contain raw model text —
238
- * keeping it out of jobs.json upholds the redaction-at-persistence invariant.
239
- */
240
- persist() {
241
- const path = this.opts.persistPath;
242
- if (!path)
243
- return;
244
- try {
245
- const view = [...this.records.values()].map(persistedJobRecord);
246
- mkdirSync(dirname(path), { recursive: true });
247
- const tmp = `${path}.tmp`;
248
- writeFileSync(tmp, JSON.stringify(view, null, 2) + "\n", { mode: 0o600 });
249
- chmodSync(tmp, 0o600);
250
- renameSync(tmp, path);
251
- chmodSync(path, 0o600);
252
- }
253
- catch {
254
- /* best-effort; never break a run on a persistence failure */
255
- }
286
+ stoppingError(message) {
287
+ return Object.assign(new Error(message), { code: "daemon_stopping", status: 503 });
256
288
  }
257
- /** Bound memory/disk: prune the oldest terminal jobs beyond maxHistory.
258
- * `blocked` runs are NEVER pruned — they are the needs-human inbox awaiting an
259
- * operator decision; dropping one would silently lose a pending action. */
260
289
  pruneHistory() {
261
- const cap = this.opts.maxHistory ?? 500;
262
- const terminal = [...this.records.values()].filter((r) => r.state !== "running" && r.state !== "queued" && r.state !== "blocked");
263
- if (terminal.length <= cap)
264
- return;
265
- terminal.sort((a, b) => (a.createdAt < b.createdAt ? -1 : 1));
266
- for (const r of terminal.slice(0, terminal.length - cap)) {
267
- this.records.delete(r.id);
268
- this.cancelled.delete(r.id);
290
+ const removed = prunableCommandIds(this.allRecords(), this.opts.maxHistory ?? 500, this.opts.idempotencyRetentionMs ?? 30 * 24 * 60 * 60 * 1_000, (this.opts.now ?? (() => new Date()))().getTime());
291
+ for (const store of commandStores(this.opts.commands)) {
292
+ store.prune(removed.filter((id) => store.get(id)));
269
293
  }
294
+ for (const id of removed)
295
+ this.cancelled.delete(id);
270
296
  }
271
- /** Reload the registry on startup; a fresh process cannot resume in-memory runs. */
272
- load() {
273
- const path = this.opts.persistPath;
274
- if (!path || !pathExists(path))
275
- return;
276
- const saved = readJsonSafe(path);
277
- if (saved === null || !Array.isArray(saved)) {
278
- // A corrupt registry must not be silently wiped: the run history includes
279
- // the blocked needs-human inbox. Back the raw bytes up, start empty, and
280
- // say so loudly (ThreadStore discipline).
281
- this.backupCorruptRegistry(path, "registry file is not a JSON array");
282
- return;
283
- }
284
- let dropped = 0;
285
- for (const raw of saved) {
286
- const rec = salvageJobRecord(raw);
287
- if (!rec) {
288
- dropped += 1;
289
- continue;
290
- }
291
- // A fresh process cannot resume an in-memory RUN, so a `running` job becomes
292
- // interrupted (honest). `blocked` is a TERMINAL outcome (NEEDS_HUMAN / web
293
- // policy) the review queue must keep across restarts.
294
- if (rec.state === "running") {
295
- rec.state = "interrupted";
296
- // Stamp the orphaned event log with a TERMINAL event: the
297
- // canonical events.jsonl must agree with jobs.json, or SSE tailers and
298
- // `follow` wait forever on a log that will never terminate.
299
- if (rec.runDir && rec.runId) {
300
- try {
301
- appendRunEvent(join(rec.runDir, "events.jsonl"), rec.runId, rec.taskId ?? "", "run.failed", {
302
- status: "interrupted",
303
- error: "daemon restarted while the run was in flight",
304
- });
305
- }
306
- catch {
307
- /* best-effort: a missing/corrupt log must not block startup */
308
- }
309
- }
310
- }
311
- this.records.set(rec.id, rec);
312
- // A `queued` job never started; its params are persisted, so RE-ENQUEUE it
313
- // on restart (drain() runs after start()) instead of silently dropping
314
- // pending work to interrupted.
315
- if (rec.state === "queued")
316
- this.queue.push(rec.id);
317
- }
318
- if (dropped > 0) {
319
- this.backupCorruptRegistry(path, `${dropped} unparseable job record(s) dropped`);
320
- }
297
+ acceptCommand(params, idempotencyKey, clientId, idempotencyParams, operation) {
298
+ const store = commandStoreForRequest(this.opts.commands, params);
299
+ return store.accept({
300
+ id: newId("job"),
301
+ params,
302
+ idempotencyKey,
303
+ clientId,
304
+ idempotencyParams,
305
+ operation,
306
+ });
321
307
  }
322
- /** Preserve the raw bytes of a damaged registry and report the loss loudly. */
323
- backupCorruptRegistry(path, reason) {
324
- try {
325
- copyFileSync(path, `${path}.bak`);
326
- console.error(`[claudexor] jobs store: ${reason}; original backed up to ${path}.bak`);
327
- }
328
- catch {
329
- console.error(`[claudexor] jobs store: ${reason}; backup to ${path}.bak FAILED`);
330
- }
308
+ allRecords() {
309
+ return commandStores(this.opts.commands).flatMap((store) => store.records());
310
+ }
311
+ getRecord(id) {
312
+ return commandStoreForId(this.opts.commands, id)?.get(id);
313
+ }
314
+ updateRecord(record, patch) {
315
+ const store = commandStoreForId(this.opts.commands, record.id);
316
+ if (!store)
317
+ throw new Error(`command authority lost job ${record.id}`);
318
+ return store.update(record.id, patch);
331
319
  }
332
320
  threadIdOf(rec) {
333
321
  const p = rec.params;
@@ -344,14 +332,16 @@ export class DaemonServer {
344
332
  * turn starts as soon as its previous one settles.
345
333
  */
346
334
  drain() {
335
+ if (this.stopping)
336
+ return;
347
337
  while (this.active < this.maxConcurrent && this.queue.length > 0) {
348
- const busyThreads = new Set([...this.records.values()]
338
+ const busyThreads = new Set(this.allRecords()
349
339
  .filter((r) => r.state === "running")
350
340
  .map((r) => this.threadIdOf(r))
351
341
  .filter((t) => !!t));
352
342
  let pickIdx = -1;
353
343
  for (let i = 0; i < this.queue.length; i++) {
354
- const rec = this.records.get(this.queue[i]);
344
+ const rec = this.getRecord(this.queue[i]);
355
345
  const tid = rec ? this.threadIdOf(rec) : undefined;
356
346
  if (!rec || !tid || !busyThreads.has(tid)) {
357
347
  pickIdx = i;
@@ -361,59 +351,75 @@ export class DaemonServer {
361
351
  if (pickIdx === -1)
362
352
  break; // every queued job waits on a busy thread
363
353
  const id = this.queue.splice(pickIdx, 1)[0];
364
- const rec = this.records.get(id);
354
+ const rec = this.getRecord(id);
365
355
  if (!rec)
366
356
  continue;
367
357
  if (this.cancelled.has(id)) {
368
- rec.state = "cancelled";
358
+ this.updateRecord(rec, { state: "cancelled", finishedAt: nowIso() });
369
359
  continue;
370
360
  }
371
361
  this.active += 1;
372
- void this.runJob(id, rec);
362
+ const task = this.runJob(id, rec);
363
+ this.activeTasks.add(task);
364
+ void task.then(() => this.activeTasks.delete(task), (error) => {
365
+ this.activeTasks.delete(task);
366
+ this.taskFailures.push(error);
367
+ });
373
368
  }
374
369
  }
375
370
  async runJob(id, rec) {
376
371
  const controller = new AbortController();
377
372
  this.controllers.set(id, controller);
378
- rec.state = "running";
379
- rec.startedAt = nowIso();
380
- this.persist();
373
+ rec = this.updateRecord(rec, { state: "running", startedAt: nowIso() });
381
374
  try {
382
- rec.result = await this.opts.runner(rec.params, {
375
+ const result = await this.opts.runner(rec.params, {
383
376
  signal: controller.signal,
384
377
  onRunStart: (info) => {
385
- rec.runId = info.runId;
386
- rec.taskId = info.taskId;
387
- rec.runDir = info.runDir;
388
- // Persist the pointer immediately so a mid-run crash still reloads with
389
- // runId/runDir to locate .claudexor/runs/<runId> (the recovery path).
390
- this.persist();
378
+ rec = this.updateRecord(rec, info);
391
379
  },
392
380
  });
393
- rec.state = jobStateFromResult(rec.result, controller.signal.aborted);
394
- // Only failure-shaped terminals carry an error string. no_op / ungated /
395
- // review_not_run / blocked are HONEST terminals: fabricating an error here
396
- // would make the control facade render a failure that never happened.
397
- if (rec.state === "failed" || rec.state === "exhausted" || rec.state === "not_converged" || rec.state === "stuck_no_progress") {
398
- rec.error = resultSummary(rec.result) ?? `run ended with status ${rec.state}`;
381
+ const state = jobStateFromResult(result, controller.signal.aborted);
382
+ if (state === "failed" ||
383
+ state === "cost_unverifiable" ||
384
+ state === "exhausted_overshoot" ||
385
+ state === "exhausted" ||
386
+ state === "not_converged" ||
387
+ state === "stuck_no_progress") {
388
+ rec = this.updateRecord(rec, {
389
+ state,
390
+ result,
391
+ error: resultSummary(result) ?? `run ended with status ${state}`,
392
+ finishedAt: nowIso(),
393
+ });
394
+ }
395
+ else {
396
+ rec = this.updateRecord(rec, { state, result, finishedAt: nowIso() });
399
397
  }
400
398
  }
401
399
  catch (err) {
402
- rec.state = controller.signal.aborted ? "cancelled" : "failed";
403
- rec.error = redactSecrets(err instanceof Error ? err.message : String(err));
404
- // Preserve a typed throw's machine code (e.g. the trust gate) so
405
- // consumers can key remedies on it instead of parsing the message.
406
- const code = err && typeof err === "object" && "code" in err ? err.code : undefined;
407
- if (typeof code === "string" && code)
408
- rec.errorCode = code;
400
+ const code = err && typeof err === "object" && "code" in err
401
+ ? err.code
402
+ : undefined;
403
+ const status = err && typeof err === "object" && "status" in err
404
+ ? err.status
405
+ : undefined;
406
+ const typedStatus = typeof status === "number" && Number.isInteger(status) && status >= 400 && status <= 599
407
+ ? status
408
+ : undefined;
409
+ rec = this.updateRecord(rec, {
410
+ state: controller.signal.aborted ? "cancelled" : "failed",
411
+ error: redactSecrets(err instanceof Error ? err.message : String(err)),
412
+ ...(typeof code === "string" && code ? { errorCode: code } : {}),
413
+ ...(typedStatus !== undefined ? { errorStatus: typedStatus } : {}),
414
+ finishedAt: nowIso(),
415
+ });
409
416
  }
410
417
  finally {
411
- rec.finishedAt = nowIso();
412
418
  this.controllers.delete(id);
413
419
  this.active -= 1;
414
420
  if (rec.runId) {
415
421
  try {
416
- this.opts.onRunTerminal?.(rec.runId);
422
+ this.opts.onRunTerminal?.(rec.runId, this.threadIdOf(rec));
417
423
  }
418
424
  catch {
419
425
  /* observer failure must not corrupt terminal bookkeeping */
@@ -434,8 +440,8 @@ export class DaemonServer {
434
440
  }
435
441
  }
436
442
  this.pruneHistory();
437
- this.persist();
438
- this.drain();
443
+ if (!this.stopping)
444
+ this.drain();
439
445
  }
440
446
  }
441
447
  }
@@ -457,7 +463,9 @@ function jobStateFromResult(result, aborted) {
457
463
  case "cancelled":
458
464
  return "cancelled";
459
465
  case "exhausted":
460
- return "exhausted";
466
+ case "exhausted_overshoot":
467
+ case "cost_unverifiable":
468
+ return status;
461
469
  case "not_converged":
462
470
  return "not_converged";
463
471
  case "stuck_no_progress":
@@ -478,7 +486,6 @@ function tokenMatches(candidate, expected) {
478
486
  return false;
479
487
  return timingSafeEqual(a, b);
480
488
  }
481
- /** True when something is actively accepting connections on the socket path. */
482
489
  /** Is a daemon already listening on this socket? Exported so the claudexord
483
490
  * entrypoint can refuse BEFORE running crash GC — a second daemon must never
484
491
  * reap the live daemon's children or sweep envelopes its jobs still own. */
@@ -513,21 +520,6 @@ function publicJobRecord(rec) {
513
520
  params: redactParams(rec.params),
514
521
  };
515
522
  }
516
- function persistedJobRecord(rec) {
517
- return {
518
- id: rec.id,
519
- state: rec.state,
520
- params: redactParams(rec.params),
521
- error: rec.error ? redactSecrets(rec.error) : undefined,
522
- errorCode: rec.errorCode,
523
- createdAt: rec.createdAt,
524
- runId: rec.runId,
525
- taskId: rec.taskId,
526
- runDir: rec.runDir,
527
- startedAt: rec.startedAt,
528
- finishedAt: rec.finishedAt,
529
- };
530
- }
531
523
  function redactParams(value) {
532
524
  if (typeof value === "string")
533
525
  return redactSecrets(value);
@@ -537,7 +529,8 @@ function redactParams(value) {
537
529
  return value;
538
530
  const out = {};
539
531
  for (const [key, child] of Object.entries(value)) {
540
- out[key] = key === "prompt" && typeof child === "string" ? redactSecrets(child) : redactParams(child);
532
+ out[key] =
533
+ key === "prompt" && typeof child === "string" ? redactSecrets(child) : redactParams(child);
541
534
  }
542
535
  return out;
543
536
  }