@aron98/load-lock 1.2.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/opencode/plugin.d.ts +2 -1
- package/dist/adapters/opencode/plugin.js +85 -5
- package/dist/adapters/opencode/tui.js +5 -1
- package/dist/cli/copies.d.ts +28 -0
- package/dist/cli/copies.js +207 -0
- package/dist/cli/main.js +298 -25
- package/dist/core/daemon.d.ts +42 -0
- package/dist/core/daemon.js +305 -3
- package/dist/core/persist.js +5 -3
- package/dist/core/state.d.ts +12 -4
- package/dist/core/state.js +80 -10
- package/dist/dashboard/frame.js +5 -1
- package/dist/dashboard/view.d.ts +4 -0
- package/dist/dashboard/view.js +5 -0
- package/dist/mcp/server.js +107 -7
- package/dist/policy/text.d.ts +1 -1
- package/dist/policy/text.js +6 -0
- package/dist/shared/client.js +4 -1
- package/dist/shared/protocol.d.ts +72 -2
- package/dist/shared/protocol.js +5 -0
- package/dist/shared/version.d.ts +10 -0
- package/dist/shared/version.js +57 -0
- package/dist/shared/wait-completion.d.ts +67 -0
- package/dist/shared/wait-completion.js +505 -0
- package/dist/test/mcp.test.js +115 -1
- package/dist/test/update.test.d.ts +1 -0
- package/dist/test/update.test.js +455 -0
- package/dist/test/wait-completion.test.d.ts +1 -0
- package/dist/test/wait-completion.test.js +418 -0
- package/docs/ADAPTERS.md +3 -2
- package/docs/AGENT_GUIDE.md +49 -3
- package/docs/ARCHITECTURE.md +25 -6
- package/docs/COMPATIBILITY.md +32 -0
- package/docs/OPERATIONS.md +54 -6
- package/package.json +1 -1
package/dist/core/daemon.js
CHANGED
|
@@ -20,11 +20,21 @@ import { createServer } from "node:net";
|
|
|
20
20
|
import { existsSync, mkdirSync, unlinkSync, writeFileSync, readFileSync, rmSync } from "node:fs";
|
|
21
21
|
import { readdirSync } from "node:fs";
|
|
22
22
|
import { join } from "node:path";
|
|
23
|
+
import { spawn } from "node:child_process";
|
|
23
24
|
import { Machine } from "./state.js";
|
|
24
25
|
import { loadState, saveState, loadJournal, saveJournal, socketPath as defaultSocket, stateDir as defaultStateDir } from "./persist.js";
|
|
25
26
|
import { inspectMachine } from "./sysinfo.js";
|
|
26
27
|
import { ProtocolError } from "../shared/protocol.js";
|
|
27
28
|
import { isWindows, probeTransport } from "../shared/transport.js";
|
|
29
|
+
import { packageVersion, compareVersions } from "../shared/version.js";
|
|
30
|
+
/** Package version of the copy of load-lock this daemon process runs from. */
|
|
31
|
+
const DAEMON_VERSION = packageVersion();
|
|
32
|
+
/**
|
|
33
|
+
* Env marker set on the respawned daemon: the version this process already
|
|
34
|
+
* restarted for. Prevents restart loops when a newer client pings twice —
|
|
35
|
+
* only one respawn per version bump.
|
|
36
|
+
*/
|
|
37
|
+
const RESTARTED_FOR_ENV = "LOAD_LOCK_RESTARTED_FOR";
|
|
28
38
|
export class Daemon {
|
|
29
39
|
machine;
|
|
30
40
|
server = null;
|
|
@@ -34,11 +44,14 @@ export class Daemon {
|
|
|
34
44
|
* restart via the event cursor) and the process can exit promptly. */
|
|
35
45
|
conns = new Set();
|
|
36
46
|
waiters = new Map();
|
|
47
|
+
/** wait_for_completion calls blocked on tracked jobs; multiple waiters per job are fine. */
|
|
48
|
+
completionWaiters = new Map();
|
|
37
49
|
sentEventId;
|
|
38
50
|
stateDirPath;
|
|
39
51
|
socketPath;
|
|
40
52
|
sweepTimer = null;
|
|
41
53
|
expiryTimer = null;
|
|
54
|
+
restartTimer = null;
|
|
42
55
|
stopped = false;
|
|
43
56
|
constructor(opts = {}) {
|
|
44
57
|
this.stateDirPath = defaultStateDir(opts.stateDir);
|
|
@@ -104,10 +117,28 @@ export class Daemon {
|
|
|
104
117
|
if (this.stopped)
|
|
105
118
|
return;
|
|
106
119
|
this.stopped = true;
|
|
120
|
+
// Resolve blocked wait_for_completion calls first so no promise dangles.
|
|
121
|
+
// Responses may not make it out (conns are destroyed below) — that is
|
|
122
|
+
// fine: clients see the disconnect, reconnect, and re-issue the wait.
|
|
123
|
+
for (const [, waiters] of this.completionWaiters) {
|
|
124
|
+
for (const w of waiters) {
|
|
125
|
+
clearTimeout(w.timer);
|
|
126
|
+
w.resolve({
|
|
127
|
+
state: "job_unknown",
|
|
128
|
+
requestId: w.requestId,
|
|
129
|
+
waitedMs: this.machine.now() - w.registeredAt,
|
|
130
|
+
nextAction: "daemon is stopping — call wait_for_completion again after it is back",
|
|
131
|
+
detail: "daemon stopping",
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
this.completionWaiters.clear();
|
|
107
136
|
if (this.sweepTimer)
|
|
108
137
|
clearInterval(this.sweepTimer);
|
|
109
138
|
if (this.expiryTimer)
|
|
110
139
|
clearTimeout(this.expiryTimer);
|
|
140
|
+
if (this.restartTimer)
|
|
141
|
+
clearTimeout(this.restartTimer);
|
|
111
142
|
for (const w of this.watchers)
|
|
112
143
|
w.socket.destroy();
|
|
113
144
|
this.watchers.clear();
|
|
@@ -171,6 +202,8 @@ export class Daemon {
|
|
|
171
202
|
this.send(sock, { id: null, ok: false, error: { code: "invalid", message: "malformed JSON line" } });
|
|
172
203
|
return;
|
|
173
204
|
}
|
|
205
|
+
// Version handshake: remember the client's version for skew detection.
|
|
206
|
+
const clientVersion = typeof msg.clientVersion === "string" ? msg.clientVersion : undefined;
|
|
174
207
|
let result;
|
|
175
208
|
let agentId;
|
|
176
209
|
try {
|
|
@@ -187,6 +220,9 @@ export class Daemon {
|
|
|
187
220
|
else {
|
|
188
221
|
this.send(sock, { id: msg.id, ok: false, error: { code: "internal", message: String(err?.message ?? err) } });
|
|
189
222
|
}
|
|
223
|
+
// Respond first, then consider a self-restart — even on errors, a
|
|
224
|
+
// newer client means this daemon is running stale on-disk code.
|
|
225
|
+
this.maybeScheduleSelfRestart(clientVersion);
|
|
190
226
|
return;
|
|
191
227
|
}
|
|
192
228
|
// Tool-boundary delivery: surface this agent's outstanding offers/queue
|
|
@@ -195,6 +231,63 @@ export class Daemon {
|
|
|
195
231
|
result.pending = this.machine.pending(agentId);
|
|
196
232
|
}
|
|
197
233
|
this.send(sock, { id: msg.id, ok: true, result });
|
|
234
|
+
this.maybeScheduleSelfRestart(clientVersion);
|
|
235
|
+
}
|
|
236
|
+
// -------------------------------------------------------- version handshake
|
|
237
|
+
/**
|
|
238
|
+
* Self-restart handshake (issue #34): if a client identifies a NEWER
|
|
239
|
+
* package version than this daemon runs, schedule a respawn so the daemon
|
|
240
|
+
* picks up the updated on-disk code (e.g. after `npm i -g
|
|
241
|
+
* @aron98/load-lock` while the daemon kept running). Skew detection is
|
|
242
|
+
* entirely daemon-side and best-effort: pre-handshake clients that send
|
|
243
|
+
* no version never trigger a restart.
|
|
244
|
+
*/
|
|
245
|
+
maybeScheduleSelfRestart(clientVersion) {
|
|
246
|
+
if (this.stopped || this.restartTimer || !clientVersion)
|
|
247
|
+
return;
|
|
248
|
+
if (compareVersions(clientVersion, DAEMON_VERSION) <= 0)
|
|
249
|
+
return;
|
|
250
|
+
const restartedFor = process.env[RESTARTED_FOR_ENV];
|
|
251
|
+
if (restartedFor && compareVersions(clientVersion, restartedFor) <= 0)
|
|
252
|
+
return; // already tried
|
|
253
|
+
this.restartTimer = setTimeout(() => {
|
|
254
|
+
this.restartTimer = null;
|
|
255
|
+
this.selfRestart(clientVersion);
|
|
256
|
+
}, 250);
|
|
257
|
+
this.restartTimer.unref?.();
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* Drain + respawn onto the updated on-disk code. Persists state and the
|
|
261
|
+
* journal FIRST, then stops (destroying all client connections — they
|
|
262
|
+
* reconnect to the fresh daemon), then spawns the replacement with the
|
|
263
|
+
* exact same argv so a daemon started via the old copy runs the new copy's
|
|
264
|
+
* code, and exits.
|
|
265
|
+
*/
|
|
266
|
+
selfRestart(clientVersion) {
|
|
267
|
+
if (this.stopped)
|
|
268
|
+
return;
|
|
269
|
+
// Best-effort persistence; a failed save must not block the restart.
|
|
270
|
+
try {
|
|
271
|
+
saveState(this.stateDirPath, this.machine.config, this.machine.history);
|
|
272
|
+
saveJournal(this.stateDirPath, this.machine.journalEntries());
|
|
273
|
+
}
|
|
274
|
+
catch {
|
|
275
|
+
/* journal is best-effort */
|
|
276
|
+
}
|
|
277
|
+
this.stop();
|
|
278
|
+
try {
|
|
279
|
+
const child = spawn(process.execPath, process.argv.slice(1), {
|
|
280
|
+
detached: true,
|
|
281
|
+
stdio: "ignore",
|
|
282
|
+
cwd: process.cwd(),
|
|
283
|
+
env: { ...process.env, [RESTARTED_FOR_ENV]: clientVersion },
|
|
284
|
+
});
|
|
285
|
+
child.unref();
|
|
286
|
+
}
|
|
287
|
+
catch {
|
|
288
|
+
/* if the respawn fails the next client contact re-triggers it */
|
|
289
|
+
}
|
|
290
|
+
process.exit(0);
|
|
198
291
|
}
|
|
199
292
|
// --------------------------------------------------------------- dispatch
|
|
200
293
|
/** Returns [result, agentIdForPendingAttachment]. */
|
|
@@ -202,7 +295,10 @@ export class Daemon {
|
|
|
202
295
|
const m = this.machine;
|
|
203
296
|
switch (method) {
|
|
204
297
|
case "hello":
|
|
205
|
-
return [
|
|
298
|
+
return [
|
|
299
|
+
{ protocolVersion: 1, pid: process.pid, startedAt: m.startedAt, configVersion: m.config.version, daemonVersion: DAEMON_VERSION },
|
|
300
|
+
undefined,
|
|
301
|
+
];
|
|
206
302
|
case "acquire": {
|
|
207
303
|
const params = p;
|
|
208
304
|
let res = m.acquire(params);
|
|
@@ -265,8 +361,11 @@ export class Daemon {
|
|
|
265
361
|
const res = m.pending(String(p.agentId));
|
|
266
362
|
return [res, String(p.agentId)];
|
|
267
363
|
}
|
|
268
|
-
case "status":
|
|
269
|
-
|
|
364
|
+
case "status": {
|
|
365
|
+
const res = m.status(p.pool ? String(p.pool) : undefined);
|
|
366
|
+
res.daemon.daemonVersion = DAEMON_VERSION;
|
|
367
|
+
return [res, undefined];
|
|
368
|
+
}
|
|
270
369
|
case "get_config":
|
|
271
370
|
return [m.get_config(), undefined];
|
|
272
371
|
case "configure": {
|
|
@@ -314,6 +413,24 @@ export class Daemon {
|
|
|
314
413
|
m.adminRelease(String(p.requestId));
|
|
315
414
|
return [{ ok: true }, undefined];
|
|
316
415
|
}
|
|
416
|
+
case "wait_for_completion": {
|
|
417
|
+
// Bounded "sleep until the job finishes" for a daemon-tracked job
|
|
418
|
+
// (issue #36) — the execution-phase counterpart of wait_for_turn.
|
|
419
|
+
const requestId = String(p.requestId ?? "");
|
|
420
|
+
if (!requestId)
|
|
421
|
+
throw new ProtocolError("invalid", "wait_for_completion: requestId is required");
|
|
422
|
+
const requested = Number(p.timeoutMs ?? 0);
|
|
423
|
+
if (!Number.isFinite(requested) || requested < 0) {
|
|
424
|
+
throw new ProtocolError("invalid", "wait_for_completion: timeoutMs must be a non-negative number of milliseconds");
|
|
425
|
+
}
|
|
426
|
+
// Same bounded-sleep contract as wait_for_turn: one call is capped by
|
|
427
|
+
// maxTurnWaitMs; longer waits are checkpointed (still_running) and re-issued.
|
|
428
|
+
const timeoutMs = Math.min(requested, m.config.maxTurnWaitMs);
|
|
429
|
+
const res = await this.waitForJobCompletion(requestId, timeoutMs);
|
|
430
|
+
if (timeoutMs !== requested)
|
|
431
|
+
res.effectiveWaitMs = timeoutMs;
|
|
432
|
+
return [res, undefined];
|
|
433
|
+
}
|
|
317
434
|
default:
|
|
318
435
|
throw new ProtocolError("invalid", `unknown method "${method}"`);
|
|
319
436
|
}
|
|
@@ -333,6 +450,181 @@ export class Daemon {
|
|
|
333
450
|
this.waiters.set(requestId, { requestId, resolve, timer });
|
|
334
451
|
});
|
|
335
452
|
}
|
|
453
|
+
// ---------------------------------------------- wait_for_completion (issue #36)
|
|
454
|
+
/**
|
|
455
|
+
* Bounded blocking wait for a daemon-tracked job's completion. Mirrors
|
|
456
|
+
* waitForAdmission: the call is held server-side and woken by
|
|
457
|
+
* resolveCompletionWaiters() when the job is observed done — which always
|
|
458
|
+
* happens AFTER the permit has been released (finish() frees capacity
|
|
459
|
+
* before the event is emitted, and waiters are resolved after that).
|
|
460
|
+
* timeoutMs 0 is check-only. Never blocks unbounded.
|
|
461
|
+
*/
|
|
462
|
+
async waitForJobCompletion(requestId, timeoutMs) {
|
|
463
|
+
const m = this.machine;
|
|
464
|
+
let entry;
|
|
465
|
+
try {
|
|
466
|
+
entry = m.entryById(requestId);
|
|
467
|
+
}
|
|
468
|
+
catch {
|
|
469
|
+
return {
|
|
470
|
+
state: "job_unknown",
|
|
471
|
+
requestId,
|
|
472
|
+
waitedMs: 0,
|
|
473
|
+
nextAction: "no request with this id exists (yet) — check the id, or acquire first",
|
|
474
|
+
detail: "unknown request id",
|
|
475
|
+
};
|
|
476
|
+
}
|
|
477
|
+
if (entry.state === "held") {
|
|
478
|
+
if (entry.phase !== "running_tracked") {
|
|
479
|
+
const why = entry.phase === "claimed"
|
|
480
|
+
? "permit is held but no process is attached yet — start the job (run_with_permit / attach_pid / start_job) first"
|
|
481
|
+
: "permit is running untracked (no process association) — completion is not observable; attach a pid or poll your job directly";
|
|
482
|
+
return { state: "job_unknown", requestId, waitedMs: 0, startedAt: entry.startedAt, nextAction: why, detail: `held permit in phase "${entry.phase}"` };
|
|
483
|
+
}
|
|
484
|
+
if (timeoutMs === 0) {
|
|
485
|
+
return {
|
|
486
|
+
state: "still_running",
|
|
487
|
+
requestId,
|
|
488
|
+
waitedMs: 0,
|
|
489
|
+
startedAt: entry.startedAt,
|
|
490
|
+
nextAction: "job still running — call wait_for_completion again (with a timeout) to sleep until it finishes",
|
|
491
|
+
detail: "check-only",
|
|
492
|
+
};
|
|
493
|
+
}
|
|
494
|
+
// Register a completion waiter; woken by resolveCompletionWaiters()
|
|
495
|
+
// when the job ends, or checkpointed by the bounded timer below.
|
|
496
|
+
return await new Promise((resolve) => {
|
|
497
|
+
const registeredAt = m.now();
|
|
498
|
+
const waiter = {
|
|
499
|
+
requestId,
|
|
500
|
+
resolve,
|
|
501
|
+
timer: null,
|
|
502
|
+
registeredAt,
|
|
503
|
+
};
|
|
504
|
+
waiter.timer = setTimeout(() => {
|
|
505
|
+
const arr = this.completionWaiters.get(requestId);
|
|
506
|
+
if (arr) {
|
|
507
|
+
const idx = arr.indexOf(waiter);
|
|
508
|
+
if (idx >= 0)
|
|
509
|
+
arr.splice(idx, 1);
|
|
510
|
+
if (arr.length === 0)
|
|
511
|
+
this.completionWaiters.delete(requestId);
|
|
512
|
+
}
|
|
513
|
+
resolve({
|
|
514
|
+
state: "still_running",
|
|
515
|
+
requestId,
|
|
516
|
+
waitedMs: m.now() - registeredAt,
|
|
517
|
+
startedAt: entry.startedAt,
|
|
518
|
+
nextAction: "job still running — call wait_for_completion again to keep sleeping honestly",
|
|
519
|
+
detail: "wait budget elapsed",
|
|
520
|
+
});
|
|
521
|
+
}, timeoutMs);
|
|
522
|
+
waiter.timer.unref?.();
|
|
523
|
+
const arr = this.completionWaiters.get(requestId) ?? [];
|
|
524
|
+
arr.push(waiter);
|
|
525
|
+
this.completionWaiters.set(requestId, arr);
|
|
526
|
+
});
|
|
527
|
+
}
|
|
528
|
+
// Terminal or pre-execution states: classify from the entry's last event.
|
|
529
|
+
return this.classifyEntryCompletion(requestId, 0);
|
|
530
|
+
}
|
|
531
|
+
/**
|
|
532
|
+
* Classify a request whose entry is no longer held (released, reclaimed,
|
|
533
|
+
* cancelled, or never started) into a structured completion result, using
|
|
534
|
+
* the most recent event for the request.
|
|
535
|
+
*/
|
|
536
|
+
classifyEntryCompletion(requestId, waitedMs) {
|
|
537
|
+
const m = this.machine;
|
|
538
|
+
const ev = m.lastEventForRequest(requestId, 0);
|
|
539
|
+
const startedAt = ev?.entry?.startedAt;
|
|
540
|
+
if (!ev) {
|
|
541
|
+
return {
|
|
542
|
+
state: "job_unknown",
|
|
543
|
+
requestId,
|
|
544
|
+
waitedMs,
|
|
545
|
+
nextAction: "no completion is observable for this request id — verify the id or re-acquire",
|
|
546
|
+
detail: "entry is no longer held and no matching event was found",
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
switch (ev.type) {
|
|
550
|
+
case "released":
|
|
551
|
+
case "auto_released":
|
|
552
|
+
return {
|
|
553
|
+
state: "finished",
|
|
554
|
+
requestId,
|
|
555
|
+
durationMs: startedAt !== undefined ? ev.ts - startedAt : undefined,
|
|
556
|
+
permitReleased: true,
|
|
557
|
+
waitedMs,
|
|
558
|
+
startedAt,
|
|
559
|
+
nextAction: "job finished and the permit is released — capacity is free; proceed with the next step",
|
|
560
|
+
detail: ev.detail,
|
|
561
|
+
};
|
|
562
|
+
case "reclaimed":
|
|
563
|
+
if ((ev.detail ?? "").includes("admin")) {
|
|
564
|
+
return {
|
|
565
|
+
state: "stopped_by_request",
|
|
566
|
+
requestId,
|
|
567
|
+
waitedMs,
|
|
568
|
+
startedAt,
|
|
569
|
+
nextAction: "the permit was force-released by an admin while you were waiting — re-acquire before running more heavy work",
|
|
570
|
+
detail: ev.detail,
|
|
571
|
+
};
|
|
572
|
+
}
|
|
573
|
+
return {
|
|
574
|
+
state: "job_unknown",
|
|
575
|
+
requestId,
|
|
576
|
+
waitedMs,
|
|
577
|
+
startedAt,
|
|
578
|
+
nextAction: "the permit was reclaimed before completion could be observed — re-acquire before running more heavy work",
|
|
579
|
+
detail: ev.detail,
|
|
580
|
+
};
|
|
581
|
+
case "cancelled":
|
|
582
|
+
return {
|
|
583
|
+
state: "job_unknown",
|
|
584
|
+
requestId,
|
|
585
|
+
waitedMs,
|
|
586
|
+
startedAt,
|
|
587
|
+
nextAction: "the request was cancelled — re-acquire before running more heavy work",
|
|
588
|
+
detail: ev.detail,
|
|
589
|
+
};
|
|
590
|
+
default:
|
|
591
|
+
return {
|
|
592
|
+
state: "job_unknown",
|
|
593
|
+
requestId,
|
|
594
|
+
waitedMs,
|
|
595
|
+
startedAt,
|
|
596
|
+
nextAction: "no completion is observable for this request id — verify the id or re-acquire",
|
|
597
|
+
detail: `entry is in state "${ev.entry?.state ?? "unknown"}" (last event: ${ev.type})`,
|
|
598
|
+
};
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
/**
|
|
602
|
+
* Wake blocked wait_for_completion calls whose tracked job has ended.
|
|
603
|
+
* Called after every machine transition (afterOp, sweep, expiry timer) —
|
|
604
|
+
* always AFTER finish() has freed capacity and after the event flush, so
|
|
605
|
+
* a woken agent observes the pool as free. Entries still held are skipped.
|
|
606
|
+
*/
|
|
607
|
+
resolveCompletionWaiters() {
|
|
608
|
+
if (this.completionWaiters.size === 0)
|
|
609
|
+
return;
|
|
610
|
+
const m = this.machine;
|
|
611
|
+
for (const [requestId, waiters] of [...this.completionWaiters]) {
|
|
612
|
+
let entry;
|
|
613
|
+
try {
|
|
614
|
+
entry = m.entryById(requestId);
|
|
615
|
+
}
|
|
616
|
+
catch {
|
|
617
|
+
entry = undefined;
|
|
618
|
+
}
|
|
619
|
+
if (entry && entry.state === "held")
|
|
620
|
+
continue; // job still running
|
|
621
|
+
this.completionWaiters.delete(requestId);
|
|
622
|
+
for (const w of waiters) {
|
|
623
|
+
clearTimeout(w.timer);
|
|
624
|
+
w.resolve(this.classifyEntryCompletion(requestId, m.now() - w.registeredAt));
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
}
|
|
336
628
|
// ------------------------------------------------------------- post-op pass
|
|
337
629
|
/** Runs after every dispatch: sweep deadlines, auto-claim for waiters, flush events, journal. */
|
|
338
630
|
afterOp() {
|
|
@@ -367,6 +659,10 @@ export class Daemon {
|
|
|
367
659
|
for (const ev of fresh)
|
|
368
660
|
this.broadcast(ev);
|
|
369
661
|
}
|
|
662
|
+
// Wake completion waiters whose job ended in this op — after the events
|
|
663
|
+
// flushed, i.e. after finish() already freed the capacity (ordering
|
|
664
|
+
// guarantee: a woken agent sees the pool as free).
|
|
665
|
+
this.resolveCompletionWaiters();
|
|
370
666
|
// Persist held permits for restart reconciliation.
|
|
371
667
|
try {
|
|
372
668
|
saveJournal(this.stateDirPath, m.journalEntries());
|
|
@@ -393,6 +689,8 @@ export class Daemon {
|
|
|
393
689
|
for (const ev of fresh)
|
|
394
690
|
this.broadcast(ev);
|
|
395
691
|
}
|
|
692
|
+
// Reclaims performed by the tick above must also wake waiters.
|
|
693
|
+
this.resolveCompletionWaiters();
|
|
396
694
|
try {
|
|
397
695
|
saveJournal(this.stateDirPath, this.machine.journalEntries());
|
|
398
696
|
}
|
|
@@ -452,6 +750,10 @@ export class Daemon {
|
|
|
452
750
|
for (const ev of fresh)
|
|
453
751
|
this.broadcast(ev);
|
|
454
752
|
}
|
|
753
|
+
// Wake completion waiters woken by this sweep (tracked job confirmed
|
|
754
|
+
// dead => auto-released above) — after capacity was freed and events
|
|
755
|
+
// flushed, per the wake ordering guarantee.
|
|
756
|
+
this.resolveCompletionWaiters();
|
|
455
757
|
}
|
|
456
758
|
}
|
|
457
759
|
// --------------------------------------------------------------------- helpers
|
package/dist/core/persist.js
CHANGED
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
* State dir: $XDG_STATE_HOME/load-lock (default ~/.local/state/load-lock;
|
|
4
4
|
* Windows: %LOCALAPPDATA%\load-lock — see shared/transport.ts).
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* Configuration persists across daemon restarts, plus the journal
|
|
7
|
+
* (journal.json): held permits AND queued/offered requests. Held permits
|
|
8
|
+
* are restored conservatively (tracked jobs re-checked for liveness, same
|
|
9
|
+
* permit tokens so late releases stay idempotent); queued requests are
|
|
10
|
+
* re-queued FIFO with the same requestIds (see docs/ARCHITECTURE.md).
|
|
9
11
|
*/
|
|
10
12
|
import { mkdirSync, writeFileSync, renameSync, readFileSync, existsSync } from "node:fs";
|
|
11
13
|
import { dirname, join } from "node:path";
|
package/dist/core/state.d.ts
CHANGED
|
@@ -75,6 +75,11 @@ export declare class Machine {
|
|
|
75
75
|
/** Events with id > since, newest last. */
|
|
76
76
|
eventsSince(since: number, agentId?: string, limit?: number): MachineEvent[];
|
|
77
77
|
latestEventId(): number;
|
|
78
|
+
/**
|
|
79
|
+
* Newest event with id > `since` whose entry is the given request, or null.
|
|
80
|
+
* Scans backwards, so the first match is the most recent one.
|
|
81
|
+
*/
|
|
82
|
+
lastEventForRequest(requestId: string, since: number): MachineEvent | null;
|
|
78
83
|
private emit;
|
|
79
84
|
snapshot(e: Entry): EntrySnapshot;
|
|
80
85
|
private ownerSnapshot;
|
|
@@ -180,16 +185,18 @@ export declare class Machine {
|
|
|
180
185
|
get_config(): GetConfigResult;
|
|
181
186
|
configure(p: ConfigureParams): ConfigureResult;
|
|
182
187
|
history: ConfigHistoryEntry[];
|
|
183
|
-
/** Held permits to persist
|
|
188
|
+
/** Held permits (first) and queued/offered requests to persist across restarts. */
|
|
184
189
|
journalEntries(): JournalEntry[];
|
|
185
190
|
/**
|
|
186
|
-
* Restore
|
|
191
|
+
* Restore state after a daemon restart (conservative reconciliation):
|
|
192
|
+
* - queued (incl. previously offered): re-queued FIFO by original enqueue
|
|
193
|
+
* time, same requestIds so idempotent re-acquire still works; pools no
|
|
194
|
+
* longer in the config are dropped. Offered entries restart as queued
|
|
195
|
+
* and get a fresh offer (old offer tokens are invalid after restart).
|
|
187
196
|
* - tracked: re-check liveness via the supplied probe; dead => release,
|
|
188
197
|
* alive => keep (same token, so a late explicit release still works).
|
|
189
198
|
* - claimed: keep the start deadline (reclaimed by tick if already past).
|
|
190
199
|
* - untracked: keep, with a FRESH lease window from restore time.
|
|
191
|
-
* Queued/offered requests are deliberately NOT restored — queued agents
|
|
192
|
-
* re-acquire (documented restart behavior).
|
|
193
200
|
*/
|
|
194
201
|
restore(journal: JournalEntry[], liveness: (e: {
|
|
195
202
|
pid?: number;
|
|
@@ -198,6 +205,7 @@ export declare class Machine {
|
|
|
198
205
|
}) => boolean | null): {
|
|
199
206
|
kept: number;
|
|
200
207
|
releasedDead: number;
|
|
208
|
+
queued: number;
|
|
201
209
|
};
|
|
202
210
|
}
|
|
203
211
|
export {};
|
package/dist/core/state.js
CHANGED
|
@@ -87,6 +87,20 @@ export class Machine {
|
|
|
87
87
|
latestEventId() {
|
|
88
88
|
return this.nextEventId - 1;
|
|
89
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* Newest event with id > `since` whose entry is the given request, or null.
|
|
92
|
+
* Scans backwards, so the first match is the most recent one.
|
|
93
|
+
*/
|
|
94
|
+
lastEventForRequest(requestId, since) {
|
|
95
|
+
for (let i = this.events.length - 1; i >= 0; i--) {
|
|
96
|
+
const ev = this.events[i];
|
|
97
|
+
if (ev.id <= since)
|
|
98
|
+
break;
|
|
99
|
+
if (ev.entry?.requestId === requestId)
|
|
100
|
+
return ev;
|
|
101
|
+
}
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
90
104
|
emit(type, entry, extra) {
|
|
91
105
|
const targetAgent = entry ? entry.agentId : "*";
|
|
92
106
|
const ev = {
|
|
@@ -134,6 +148,8 @@ export class Machine {
|
|
|
134
148
|
s.leaseDeadline = e.leaseDeadline;
|
|
135
149
|
if (e.lastSeen)
|
|
136
150
|
s.lastSeen = e.lastSeen;
|
|
151
|
+
if (e.startedAt !== undefined)
|
|
152
|
+
s.startedAt = e.startedAt;
|
|
137
153
|
return s;
|
|
138
154
|
}
|
|
139
155
|
ownerSnapshot(e) {
|
|
@@ -830,7 +846,7 @@ export class Machine {
|
|
|
830
846
|
}
|
|
831
847
|
history = [];
|
|
832
848
|
// ----------------------------------------------------------------- journal
|
|
833
|
-
/** Held permits to persist
|
|
849
|
+
/** Held permits (first) and queued/offered requests to persist across restarts. */
|
|
834
850
|
journalEntries() {
|
|
835
851
|
const out = [];
|
|
836
852
|
for (const e of this.entries.values()) {
|
|
@@ -845,6 +861,7 @@ export class Machine {
|
|
|
845
861
|
opKey: e.opKey,
|
|
846
862
|
reason: e.reason,
|
|
847
863
|
weight: e.weight,
|
|
864
|
+
state: "held",
|
|
848
865
|
permitToken: e.permitToken,
|
|
849
866
|
phase: e.phase ?? "claimed",
|
|
850
867
|
pid: e.pid,
|
|
@@ -855,21 +872,45 @@ export class Machine {
|
|
|
855
872
|
leaseDeadline: e.leaseDeadline,
|
|
856
873
|
});
|
|
857
874
|
}
|
|
875
|
+
for (const e of this.entries.values()) {
|
|
876
|
+
if (e.state !== "queued" && e.state !== "offered")
|
|
877
|
+
continue;
|
|
878
|
+
out.push({
|
|
879
|
+
requestId: e.id,
|
|
880
|
+
pool: e.pool,
|
|
881
|
+
agentId: e.agentId,
|
|
882
|
+
agentLabel: e.agentLabel,
|
|
883
|
+
cwd: e.cwd,
|
|
884
|
+
opKey: e.opKey,
|
|
885
|
+
reason: e.reason,
|
|
886
|
+
weight: e.weight,
|
|
887
|
+
state: "queued",
|
|
888
|
+
enqueuedAt: e.enqueuedAt ?? e.since,
|
|
889
|
+
claimedAt: e.since,
|
|
890
|
+
});
|
|
891
|
+
}
|
|
858
892
|
return out;
|
|
859
893
|
}
|
|
860
894
|
/**
|
|
861
|
-
* Restore
|
|
895
|
+
* Restore state after a daemon restart (conservative reconciliation):
|
|
896
|
+
* - queued (incl. previously offered): re-queued FIFO by original enqueue
|
|
897
|
+
* time, same requestIds so idempotent re-acquire still works; pools no
|
|
898
|
+
* longer in the config are dropped. Offered entries restart as queued
|
|
899
|
+
* and get a fresh offer (old offer tokens are invalid after restart).
|
|
862
900
|
* - tracked: re-check liveness via the supplied probe; dead => release,
|
|
863
901
|
* alive => keep (same token, so a late explicit release still works).
|
|
864
902
|
* - claimed: keep the start deadline (reclaimed by tick if already past).
|
|
865
903
|
* - untracked: keep, with a FRESH lease window from restore time.
|
|
866
|
-
* Queued/offered requests are deliberately NOT restored — queued agents
|
|
867
|
-
* re-acquire (documented restart behavior).
|
|
868
904
|
*/
|
|
869
905
|
restore(journal, liveness) {
|
|
870
906
|
let kept = 0;
|
|
871
907
|
let releasedDead = 0;
|
|
908
|
+
const queuedEntries = [];
|
|
872
909
|
for (const j of journal) {
|
|
910
|
+
if (j.state === "queued") {
|
|
911
|
+
queuedEntries.push({ j, at: j.enqueuedAt ?? j.claimedAt });
|
|
912
|
+
continue;
|
|
913
|
+
}
|
|
873
914
|
const e = {
|
|
874
915
|
id: j.requestId,
|
|
875
916
|
pool: j.pool,
|
|
@@ -882,20 +923,20 @@ export class Machine {
|
|
|
882
923
|
state: "held",
|
|
883
924
|
since: j.claimedAt,
|
|
884
925
|
permitToken: j.permitToken,
|
|
885
|
-
phase: j.phase,
|
|
926
|
+
phase: j.phase ?? "claimed",
|
|
886
927
|
pid: j.pid,
|
|
887
928
|
pgid: j.pgid,
|
|
888
929
|
pidStartTime: j.pidStartTime,
|
|
889
930
|
startedAt: j.startedAt,
|
|
890
931
|
lastSeen: this.now(),
|
|
891
932
|
};
|
|
892
|
-
if (
|
|
933
|
+
if (e.phase === "claimed") {
|
|
893
934
|
e.startDeadline = j.claimedAt + this.config.startTimeoutMs;
|
|
894
935
|
}
|
|
895
|
-
else if (
|
|
936
|
+
else if (e.phase === "running_untracked") {
|
|
896
937
|
e.leaseDeadline = this.now() + this.config.untrackedLeaseMs; // fresh window
|
|
897
938
|
}
|
|
898
|
-
else if (
|
|
939
|
+
else if (e.phase === "running_tracked") {
|
|
899
940
|
const alive = liveness(e);
|
|
900
941
|
if (alive === false) {
|
|
901
942
|
// Confirmed dead while we were down: release, but keep a terminal
|
|
@@ -915,11 +956,40 @@ export class Machine {
|
|
|
915
956
|
this.byKey.set(`${e.agentId}\u0000${e.opKey}`, e.id);
|
|
916
957
|
kept++;
|
|
917
958
|
}
|
|
959
|
+
// Re-queue queued requests, oldest first, preserving FIFO fairness.
|
|
960
|
+
queuedEntries.sort((a, b) => a.at - b.at);
|
|
961
|
+
let queued = 0;
|
|
962
|
+
for (const { j } of queuedEntries) {
|
|
963
|
+
if (!this.config.pools[j.pool])
|
|
964
|
+
continue; // pool removed while down
|
|
965
|
+
const e = {
|
|
966
|
+
id: j.requestId,
|
|
967
|
+
pool: j.pool,
|
|
968
|
+
agentId: j.agentId,
|
|
969
|
+
agentLabel: j.agentLabel,
|
|
970
|
+
cwd: j.cwd,
|
|
971
|
+
opKey: j.opKey,
|
|
972
|
+
reason: j.reason,
|
|
973
|
+
weight: j.weight,
|
|
974
|
+
state: "queued",
|
|
975
|
+
since: this.now(),
|
|
976
|
+
enqueuedAt: j.enqueuedAt ?? j.claimedAt,
|
|
977
|
+
};
|
|
978
|
+
this.entries.set(e.id, e);
|
|
979
|
+
this.byKey.set(`${e.agentId}\u0000${e.opKey}`, e.id);
|
|
980
|
+
if (!this.queues.has(e.pool))
|
|
981
|
+
this.queues.set(e.pool, []);
|
|
982
|
+
this.queues.get(e.pool).push(e.id);
|
|
983
|
+
queued++;
|
|
984
|
+
}
|
|
985
|
+
// Admit newly-eligible waiters (capacity freed while we were down).
|
|
986
|
+
for (const name of this.poolNames())
|
|
987
|
+
this.admit(name);
|
|
918
988
|
this.pushEvent({
|
|
919
989
|
type: "daemon_recovered",
|
|
920
990
|
targetAgent: "*",
|
|
921
|
-
detail: `restored ${kept} held permit(s) from journal (${releasedDead} confirmed dead and released)`,
|
|
991
|
+
detail: `restored ${kept} held permit(s) from journal (${releasedDead} confirmed dead and released); re-queued ${queued} waiting request(s)`,
|
|
922
992
|
});
|
|
923
|
-
return { kept, releasedDead };
|
|
993
|
+
return { kept, releasedDead, queued };
|
|
924
994
|
}
|
|
925
995
|
}
|
package/dist/dashboard/frame.js
CHANGED
|
@@ -59,13 +59,17 @@ export function renderFrame(view, width = 100, height = 40) {
|
|
|
59
59
|
const syncAge = view.lastSyncAt ? fmtDuration(Math.max(0, view.now - view.lastSyncAt)) : "never";
|
|
60
60
|
push(`load-lock dashboard — machine-wide heavy-work budget ${state} synced ${syncAge} ago`);
|
|
61
61
|
if (view.daemon) {
|
|
62
|
-
push(`daemon pid ${view.daemon.pid} · uptime ${fmtDuration(view.daemon.uptimeMs)} · config v${view.daemon.configVersion} · protocol v${view.daemon.protocolVersion}
|
|
62
|
+
push(`daemon pid ${view.daemon.pid} · uptime ${fmtDuration(view.daemon.uptimeMs)} · config v${view.daemon.configVersion} · protocol v${view.daemon.protocolVersion}` +
|
|
63
|
+
(view.daemon.daemonVersion ? ` · daemon v${view.daemon.daemonVersion}` : "") +
|
|
64
|
+
` · events through #${view.latestEventId}`);
|
|
63
65
|
}
|
|
64
66
|
else {
|
|
65
67
|
push(`daemon: unreachable — it auto-spawns on the next load-lock command`);
|
|
66
68
|
}
|
|
67
69
|
if (view.daemonRestartNote)
|
|
68
70
|
push(`note: ${view.daemonRestartNote}`);
|
|
71
|
+
if (view.versionSkewNote)
|
|
72
|
+
push(`note: ${view.versionSkewNote}`);
|
|
69
73
|
if (view.machine) {
|
|
70
74
|
const m = view.machine;
|
|
71
75
|
const cg = m.cgroupCpuQuota != null ? ` · cgroup ${m.cgroupVersion ?? ""} quota ${m.cgroupCpuQuota / 100} CPU` : "";
|
package/dist/dashboard/view.d.ts
CHANGED
|
@@ -105,6 +105,8 @@ export interface DaemonView {
|
|
|
105
105
|
uptimeMs: number;
|
|
106
106
|
protocolVersion: number;
|
|
107
107
|
configVersion: number;
|
|
108
|
+
/** Package version of the daemon's code (pre-handshake daemons omit it). */
|
|
109
|
+
daemonVersion?: string;
|
|
108
110
|
}
|
|
109
111
|
/** The complete render-ready snapshot of the machine-wide budget. */
|
|
110
112
|
export interface DashboardView {
|
|
@@ -112,6 +114,8 @@ export interface DashboardView {
|
|
|
112
114
|
state: FeedState;
|
|
113
115
|
/** False right after a daemon restart was detected (history was reset). */
|
|
114
116
|
daemonRestartNote?: string;
|
|
117
|
+
/** Set when the running daemon is older than the installed copy. */
|
|
118
|
+
versionSkewNote?: string;
|
|
115
119
|
daemon: DaemonView | null;
|
|
116
120
|
pools: PoolView[];
|
|
117
121
|
events: EventLine[];
|