@aron98/load-lock 1.2.1 → 1.4.0

package/dist/test/wait-completion.test.js

@@ -0,0 +1,418 @@
+/**
+ * Integration tests for wait_for_completion (issue #36): honest sleeping
+ * through the EXECUTION phase — the counterpart of wait_for_turn. Covers:
+ * - daemon-tracked jobs: the daemon holds the wait server-side and wakes it
+ *   from its liveness sweep (real daemons, real processes), with the permit
+ *   freed BEFORE the waiter is woken (ordering),
+ * - adapter-parented jobs: waitForLocalJob wakes on the child exit event
+ *   (pure unit tests with real timers),
+ * - the old-daemon polling fallback (client that does not know the method),
+ * - daemon restart transparency mid-wait,
+ * - all structured paths: finished / still_running / timeout / job_unknown /
+ *   stopped_by_request,
+ * - the CLI `wait --job` surface.
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { ProtocolError } from "../shared/protocol.js";
+import { waitForLocalJob, waitForTrackedJob, } from "../shared/wait-completion.js";
+import { startDaemon, delay, until, probe, runCli, daemonMainPath, } from "./testutil.js";
+const ACQ = (agentId, opKey) => ({
+    agentId,
+    opKey,
+    reason: "integration test",
+});
+/** A wait expected to stay pending for at least `ms` (race sentinel). */
+function expectPending(ms) {
+    return delay(ms).then(() => "pending");
+}
+/**
+ * Acquire a permit and attach a REAL detached process to it (daemon-tracked,
+ * exactly what an adapter does on behalf of an agent).
+ */
+async function startTrackedJob(c, agentId, opKey) {
+    const acq = (await c.request("acquire", ACQ(agentId, opKey)));
+    assert.equal(acq.state, "granted");
+    const child = spawn("sleep", ["30"], { detached: true, stdio: "ignore" });
+    child.unref();
+    const sj = (await c.request("start_job", { permitToken: acq.entry.permitToken, pid: child.pid }));
+    assert.ok(sj.entry.requestId, "start_job returns the entry snapshot");
+    return { requestId: sj.entry.requestId, permitToken: acq.entry.permitToken, child };
+}
+test("wait_for_completion: liveness sweep wakes the sleeper when the tracked process dies (ordering: permit freed first)", async () => {
+    const d = await startDaemon();
+    const clients = [];
+    try {
+        const cA = d.client();
+        await cA.connect();
+        clients.push(cA);
+        const cB = d.client();
+        await cB.connect();
+        clients.push(cB);
+        const job = await startTrackedJob(cA, "holderA", "opA");
+        // Two independent waiters on the same job — multiple waiters are fine.
+        const waitA = waitForTrackedJob({ client: cA, requestId: job.requestId, timeoutMs: 30_000 });
+        const waitB = waitForTrackedJob({ client: cB, requestId: job.requestId, timeoutMs: 30_000 });
+        const early = await Promise.race([waitA.then(() => "done"), expectPending(400)]);
+        assert.equal(early, "pending", "the wait blocks while the tracked process runs");
+        const t0 = Date.now();
+        job.child.kill("SIGKILL");
+        const [rA, rB] = await Promise.all([waitA, waitB]);
+        const elapsed = Date.now() - t0;
+        assert.ok(elapsed < 7_000, `woke promptly after process death (took ${elapsed}ms; sweep interval is 5s)`);
+        assert.equal(rA.state, "finished");
+        assert.equal(rB.state, "finished");
+        assert.equal(rA.requestId, job.requestId);
+        assert.equal(rA.permitReleased, true, "the permit is released by the time we wake");
+        assert.ok(rA.durationMs === undefined || rA.durationMs >= 0);
+        assert.ok(rA.waitedMs >= 300, `waitedMs reported (got ${rA.waitedMs})`);
+        // ORDERING: capacity was freed BEFORE the waiters were woken.
+        const st = (await cA.request("status", {}));
+        assert.equal(st.pools[0].used, 0, "pool is empty the moment the wait resolves");
+    }
+    finally {
+        for (const cl of clients)
+            cl.close();
+        await d.stop();
+    }
+});
+test("wait_for_completion: budget elapsing is a still_running checkpoint, not an error; timeout_ms 0 is check-only", async () => {
+    const d = await startDaemon();
+    const clients = [];
+    try {
+        const cA = d.client();
+        await cA.connect();
+        clients.push(cA);
+        const job = await startTrackedJob(cA, "holderA", "opA");
+        const t0 = Date.now();
+        const r = await waitForTrackedJob({ client: cA, requestId: job.requestId, timeoutMs: 800 });
+        assert.ok(Date.now() - t0 < 5_000, "checkpoint returns promptly after the budget");
+        assert.equal(r.state, "still_running", "timeout is a structured checkpoint result");
+        assert.ok(r.waitedMs >= 700, `waited the full budget (waitedMs=${r.waitedMs})`);
+        assert.ok(r.startedAt, "startedAt reported so callers can reason about duration");
+        assert.match(r.nextAction ?? "", /wait_for_completion|call again/i);
+        // Check-only: no budget spent, immediate structured answer.
+        const t1 = Date.now();
+        const check = await waitForTrackedJob({ client: cA, requestId: job.requestId, timeoutMs: 0 });
+        assert.ok(Date.now() - t1 < 1_000, "check-only does not sleep");
+        assert.equal(check.state, "still_running");
+        assert.ok(check.waitedMs < 1_000, "check-only spends no budget waiting");
+        assert.match(check.detail ?? "", /check-only/);
+        job.child.kill("SIGKILL");
+    }
+    finally {
+        for (const cl of clients)
+            cl.close();
+        await d.stop();
+    }
+});
+test("wait_for_completion: job_unknown for unknown ids, queued requests and untracked permits", async () => {
+    const d = await startDaemon();
+    const clients = [];
+    try {
+        const cA = d.client();
+        await cA.connect();
+        clients.push(cA);
+        // Unknown id resolves immediately with a structured result.
+        const t0 = Date.now();
+        const unknown = await waitForTrackedJob({ client: cA, requestId: "no-such-request", timeoutMs: 5_000 });
+        assert.ok(Date.now() - t0 < 1_000, "unknown id does not busy-wait");
+        assert.equal(unknown.state, "job_unknown");
+        assert.match(unknown.detail ?? "", /unknown request id/);
+        // An untracked (mark_started) permit has no observable completion.
+        const u = (await cA.request("acquire", ACQ("untrackedA", "opU")));
+        assert.equal(u.state, "granted");
+        await cA.request("mark_started", { permitToken: u.entry.permitToken });
+        const untracked = await waitForTrackedJob({ client: cA, requestId: u.entry.requestId, timeoutMs: 1_000 });
+        assert.equal(untracked.state, "job_unknown");
+        assert.match(untracked.detail ?? "", /untracked|not observable|phase/);
+        await cA.request("release", { permitToken: u.entry.permitToken });
+        // A queued request is not a running job either.
+        const a = (await cA.request("acquire", ACQ("holderA", "opA")));
+        assert.equal(a.state, "granted");
+        const cB = d.client();
+        await cB.connect();
+        clients.push(cB);
+        const b = (await cB.request("acquire", ACQ("waitB", "opB")));
+        assert.equal(b.state, "queued");
+        const queued = await waitForTrackedJob({ client: cB, requestId: b.entry.requestId, timeoutMs: 1_000 });
+        assert.equal(queued.state, "job_unknown", "a queued request has no completion to wait for");
+    }
+    finally {
+        for (const cl of clients)
+            cl.close();
+        await d.stop();
+    }
+});
+test("wait_for_completion: explicit release mid-wait wakes the sleeper finished", async () => {
+    const d = await startDaemon();
+    const clients = [];
+    try {
+        const cA = d.client();
+        await cA.connect();
+        clients.push(cA);
+        const cB = d.client();
+        await cB.connect();
+        clients.push(cB);
+        const job = await startTrackedJob(cA, "holderA", "opA");
+        const wait = waitForTrackedJob({ client: cB, requestId: job.requestId, timeoutMs: 30_000 });
+        const early = await Promise.race([wait.then(() => "done"), expectPending(400)]);
+        assert.equal(early, "pending");
+        // The owner releases explicitly while someone sleeps on the job.
+        await cA.request("release", { permitToken: job.permitToken });
+        const r = await wait;
+        assert.equal(r.state, "finished");
+        assert.equal(r.permitReleased, true);
+        assert.ok(r.waitedMs >= 300);
+        const st = (await cB.request("status", {}));
+        assert.equal(st.pools[0].used, 0);
+        job.child.kill("SIGKILL");
+    }
+    finally {
+        for (const cl of clients)
+            cl.close();
+        await d.stop();
+    }
+});
+test("wait_for_completion: admin force-release mid-wait wakes the sleeper stopped_by_request", async () => {
+    const d = await startDaemon();
+    const clients = [];
+    try {
+        const cA = d.client();
+        await cA.connect();
+        clients.push(cA);
+        const cB = d.client();
+        await cB.connect();
+        clients.push(cB);
+        const job = await startTrackedJob(cA, "holderA", "opA");
+        const wait = waitForTrackedJob({ client: cB, requestId: job.requestId, timeoutMs: 30_000 });
+        const early = await Promise.race([wait.then(() => "done"), expectPending(400)]);
+        assert.equal(early, "pending");
+        await cB.request("admin_release", { requestId: job.requestId });
+        const r = await wait;
+        assert.equal(r.state, "stopped_by_request", "admin force-release classifies as stopped_by_request");
+        assert.match(r.nextAction ?? "", /re-acquire|force-released/i);
+        const st = (await cB.request("status", {}));
+        assert.equal(st.pools[0].used, 0);
+        job.child.kill("SIGKILL");
+    }
+    finally {
+        for (const cl of clients)
+            cl.close();
+        await d.stop();
+    }
+});
+test("wait_for_completion: daemon restart mid-wait is transparent — the wait continues against the recovered state", async () => {
+    const d = await startDaemon();
+    const clients = [];
+    let child2 = null;
+    let job = null;
+    try {
+        const cA = d.client();
+        await cA.connect();
+        clients.push(cA);
+        job = await startTrackedJob(cA, "holderA", "opA");
+        const wait = waitForTrackedJob({ client: cA, requestId: job.requestId, timeoutMs: 30_000 });
+        const early = await Promise.race([wait.then(() => "done"), expectPending(400)]);
+        assert.equal(early, "pending");
+        // Restart the daemon mid-wait. Tracked permits survive via journal restore.
+        d.child.kill("SIGTERM");
+        await new Promise((r) => d.child.once("exit", r));
+        child2 = spawn(process.execPath, [daemonMainPath, "--state-dir", d.dir, "--socket", d.sock], {
+            stdio: ["ignore", "pipe", "pipe"],
+            env: { ...process.env, LOAD_LOCK_STATE_DIR: d.dir, LOAD_LOCK_SOCKET: d.sock },
+        });
+        await until("restarted daemon listens", async () => await probe(d.sock));
+        // The wait survives (re-issued transparently): killing the process now
+        // wakes it finished via the restarted daemon's sweep.
+        job.child.kill("SIGKILL");
+        const r = await wait;
+        assert.equal(r.state, "finished", "the wait continued through the restart and woke on completion");
+        assert.equal(r.permitReleased, true);
+        const st = (await cA.request("status", {}));
+        assert.equal(st.pools[0].used, 0);
+    }
+    finally {
+        if (job && job.child.exitCode === null && !job.child.killed)
+            job.child.kill("SIGKILL");
+        if (child2 && child2.exitCode === null && !child2.killed)
+            child2.kill("SIGKILL");
+        for (const cl of clients)
+            cl.close();
+        d.child.kill("SIGKILL");
+        await delay(100);
+        await d.stop();
+    }
+});
+test("wait_for_completion: old daemons without the method degrade to a light events-based poll (fallback)", async () => {
+    const d = await startDaemon();
+    const clients = [];
+    try {
+        const real = d.client();
+        await real.connect();
+        clients.push(real);
+        const job = await startTrackedJob(real, "holderA", "opA");
+        // A client facing an OLD daemon: wait_for_completion is "unknown method".
+        const fake = Object.create(real);
+        fake.request = (method, params, timeoutMs) => {
+            if (method === "wait_for_completion") {
+                return Promise.reject(new ProtocolError("invalid", `unknown method "${method}"`));
+            }
+            return real.request(method, params, timeoutMs);
+        };
+        const t0 = Date.now();
+        const waitP = waitForTrackedJob({ client: fake, requestId: job.requestId, timeoutMs: 30_000 });
+        const early = await Promise.race([waitP.then(() => "done"), expectPending(400)]);
+        assert.equal(early, "pending", "the fallback polls quietly");
+        job.child.kill("SIGKILL");
+        const r = await waitP;
+        const elapsed = Date.now() - t0;
+        assert.ok(elapsed < 8_000, `fallback woke on completion (took ${elapsed}ms)`);
+        assert.equal(r.state, "finished");
+        assert.equal(r.permitReleased, true);
+        // Fallback also classifies job_unknown ids without spinning.
+        const unknown = await waitForTrackedJob({ client: fake, requestId: "no-such-request", timeoutMs: 2_000 });
+        assert.equal(unknown.state, "job_unknown");
+        assert.match(unknown.detail ?? "", /absent|unknown/i);
+    }
+    finally {
+        for (const cl of clients)
+            cl.close();
+        await d.stop();
+    }
+});
+test("waitForLocalJob: adapter-parented job wakes on the child exit event with exit code, tail and duration", async () => {
+    let resolveExit;
+    let resolveRelease;
+    const job = {
+        id: "job_unit",
+        startedAt: Date.now(),
+        exited: false,
+        output: [],
+        // Adapter contract: exitPromise resolves only AFTER releasePromise exists.
+        exitPromise: new Promise((r) => (resolveExit = r)),
+    };
+    job.releasePromise = new Promise((r) => (resolveRelease = r));
+    const t0 = Date.now();
+    const waitP = waitForLocalJob(job, { timeoutMs: 10_000 });
+    setTimeout(() => {
+        // The child exit handler runs: record exit, then release, then wake.
+        job.exited = true;
+        job.exitedAt = Date.now();
+        job.exitCode = 0;
+        job.output.push("building...\n", "done\n");
+        resolveRelease();
+        job.permitReleased = true;
+        resolveExit(0);
+    }, 300);
+    const r = await waitP;
+    const elapsed = Date.now() - t0;
+    assert.ok(elapsed < 2_000, `woke promptly on the exit event (took ${elapsed}ms)`);
+    assert.equal(r.state, "finished");
+    assert.equal(r.exitCode, 0);
+    assert.ok(r.outputTail.includes("done"), "output tail included");
+    assert.ok(r.durationMs >= 250, `duration reported (got ${r.durationMs}ms)`);
+    assert.equal(r.permitReleased, true);
+});
+test("waitForLocalJob: already-exited job resolves immediately (waits for the release round-trip)", async () => {
+    const job = {
+        id: "job_done",
+        startedAt: Date.now() - 500,
+        exited: true,
+        exitedAt: Date.now() - 100,
+        exitCode: 0,
+        output: ["done\n"],
+        permitReleased: false,
+    };
+    const t0 = Date.now();
+    const r = await waitForLocalJob(job, { timeoutMs: 5_000 });
+    assert.ok(Date.now() - t0 < 1_000, "no busy-wait for an already-exited job");
+    assert.equal(r.state, "finished");
+    assert.equal(r.exitCode, 0);
+    assert.equal(r.permitReleased, false, "honest: release round-trip had not completed");
+    assert.match(r.detail ?? "", /verify with status/);
+});
+test("waitForLocalJob: budget elapsing is a still_running checkpoint; abort resolves timeout with the job untouched", async () => {
+    // Budget checkpoint.
+    const slow = {
+        id: "job_slow",
+        startedAt: Date.now(),
+        exited: false,
+        output: [],
+        exitPromise: new Promise(() => { }),
+    };
+    const r = await waitForLocalJob(slow, { timeoutMs: 250 });
+    assert.equal(r.state, "still_running");
+    assert.ok(r.waitedMs >= 200, `waited the budget (waitedMs=${r.waitedMs})`);
+    assert.equal(slow.exited, false, "the job keeps running after a checkpoint");
+    // AbortSignal (harness tool timeout) — the call resolves, the job untouched.
+    const ac = new AbortController();
+    const waitP = waitForLocalJob(slow, { timeoutMs: 10_000, signal: ac.signal });
+    await delay(200);
+    ac.abort();
+    const aborted = await waitP;
+    assert.equal(aborted.state, "timeout", "an aborted call is a structured timeout, not an error");
+    assert.match(aborted.detail ?? "", /aborted by caller/);
+    assert.match(aborted.nextAction ?? "", /wait_for_completion|again/);
+    assert.equal(slow.exited, false, "aborting the wait never touches the job");
+});
+test("waitForLocalJob: job_stop before exit classifies stopped_by_request; missing exitPromise is job_unknown", async () => {
+    let resolveExit;
+    const job = {
+        id: "job_stop",
+        startedAt: Date.now(),
+        exited: false,
+        output: [],
+        stopRequested: false,
+        exitPromise: new Promise((r) => (resolveExit = r)),
+    };
+    const waitP = waitForLocalJob(job, { timeoutMs: 10_000 });
+    await delay(150);
+    job.stopRequested = true; // job_stop flags the job before killing the group
+    job.exited = true;
+    job.exitedAt = Date.now();
+    job.exitCode = null;
+    resolveExit(null);
+    const r = await waitP;
+    assert.equal(r.state, "stopped_by_request");
+    assert.equal(r.exitCode, null);
+    assert.match(r.nextAction ?? "", /job_stop|re-run/i);
+    const broken = { id: "job_broken", startedAt: Date.now(), exited: false, output: [] };
+    const u = await waitForLocalJob(broken, { timeoutMs: 100 });
+    assert.equal(u.state, "job_unknown");
+    assert.match(u.detail ?? "", /exitPromise/);
+});
+test("cli: wait --job sleeps through a tracked job, exits 0 on finish and reports structured states otherwise", async () => {
+    const d = await startDaemon();
+    const clients = [];
+    try {
+        const cA = d.client();
+        await cA.connect();
+        clients.push(cA);
+        const job = await startTrackedJob(cA, "holderA", "opA");
+        // Checkpoint path: budget elapses -> still_running, exit code 68.
+        const ck = await runCli(d, ["wait", "--job", job.requestId, "--timeout", "400"]);
+        assert.equal(ck.code, 68);
+        assert.equal(JSON.parse(ck.stdout).state, "still_running");
+        // Sleep-through path: kill the process mid-wait -> finished, exit 0.
+        const cliP = runCli(d, ["wait", "--job", job.requestId, "--timeout", "25000"]);
+        await delay(500);
+        job.child.kill("SIGKILL");
+        const done = await cliP;
+        assert.equal(done.code, 0, `stdout=${done.stdout} stderr=${done.stderr}`);
+        const out = JSON.parse(done.stdout);
+        assert.equal(out.state, "finished");
+        assert.equal(out.permit_released, true);
+        assert.equal(out.request_id, job.requestId);
+        // Unknown id -> job_unknown, exit 69.
+        const unknown = await runCli(d, ["wait", "--job", "no-such-request", "--timeout", "1000"]);
+        assert.equal(unknown.code, 69);
+        assert.equal(JSON.parse(unknown.stdout).state, "job_unknown");
+    }
+    finally {
+        for (const cl of clients)
+            cl.close();
+        await d.stop();
+    }
+});

package/docs/ADAPTERS.md

@@ -72,8 +72,9 @@ whatever global instructions file the harness reads (GEMINI.md / rules /
 hints...). Offers are surfaced in every tool result (tool-boundary
 delivery), so correctness is preserved; what you lose is idle wakeup —
 steer agents to `wait_for_turn` (bounded blocking sleep that auto-claims
-and returns a `permit_token`) as the primary idle mechanism, and document
-the remaining gap honestly (see docs/COMPATIBILITY.md).
+and returns a `permit_token`) as the primary idle mechanism, and to
+`wait_for_completion` (bounded execution-phase sleep) once a job runs.
+Document the remaining gap honestly (see docs/COMPATIBILITY.md).
 
 ## Protocol quick reference
 

package/docs/AGENT_GUIDE.md

@@ -103,6 +103,50 @@ defaults to 600000 (10 minutes) and is clamped to the machine's
 range 1000–3600000 ms). `timeout_ms: 0` means check-only. The CLI form is
 `load-lock wait --id <request-id> --timeout <ms>`.
 
+### Sleeping through execution (`wait_for_completion`)
+
+The queue phase has `wait_for_turn`; the execution phase has its counterpart.
+After you start a heavy job, do not guess sleeps or poll `job_status` in a
+loop — sleep honestly: one bounded call that **wakes promptly when the job
+finishes**. There are two mechanisms, and the identifier picks the right one:
+
+- **`job_id` of a `run_with_permit` job** (the job belongs to your adapter
+  process): the wake is the child's exact `exit` event — no polling, prompt,
+  with the exact exit code and an output tail. The same thing in one call:
+  `run_with_permit(..., wait: true)` starts the job and sleeps through it.
+- **`request_id` of a daemon-tracked permit** (`attach_pid` etc. — the daemon
+  is not the parent): the daemon holds the wait server-side and wakes it when
+  its liveness sweep observes the process gone (worst case one sweep interval,
+  default 5s). The permit is **released before you are woken**, so a woken
+  agent sees the capacity freed. The CLI form is
+  `load-lock wait --job <request-id> --timeout <ms>`.
+
+Every resolution is a structured result, never an opaque error:
+
+- `finished` — exit code, output tail, duration, and the permit-release
+  state (`permit_released`; if the release round-trip was still in flight
+  the `detail` says so — verify with `status` if in doubt).
+- `still_running` — the bound expired mid-execution; this is a **checkpoint**,
+  not a failure. Call again; a long job is a handful of long sleeps, not a
+  poll loop.
+- `timeout` — this wait call was aborted/cancelled (harness tool timeout,
+  AbortSignal). The job is untouched; call again.
+- `job_unknown` — no such job/request, or the permit was reclaimed/expired
+  meanwhile (e.g. the daemon or adapter restarted); re-check before
+  proceeding.
+- `stopped_by_request` — the job or permit was stopped by an explicit
+  (admin) action; re-acquire before running more heavy work.
+
+The sleep is bounded and safe by construction: it holds no permit and no
+capacity itself, and for daemon-tracked waits `timeout_ms` is clamped to the
+machine's `maxTurnWaitMs` cap (same as `wait_for_turn`; adapter-parented
+waits default to 600000). `timeout_ms: 0` means check-only. A daemon restart
+mid-wait is handled transparently (tracked permits survive restarts via the
+journal); against an older daemon the client falls back to a light
+events-based poll with the same structured results. Cancelling the call
+never kills the job. If someone calls `job_stop` while you are waiting, you
+wake with `stopped_by_request`.
+
 ## 3. Running heavy work (lifecycle options)
 
 How you start the work determines how reliably the permit cleans up after
@@ -111,9 +155,11 @@ you — pick the best available:
 1. **`run_with_permit(permit_token, command)`** — the helper spawns your
    command in its own process group, atomically associates it with the
    permit, and auto-releases when the process group *actually finishes*
-   (including background children). Check `job_status(job_id)` for
-   completion/exit code/tail of output; `job_stop(job_id)` to kill it.
-   Do **not** manually release a permit you handed to `run_with_permit`.
+   (including background children). Sleep through the execution with
+   `wait_for_completion(job_id)` (or pass `wait: true` to fire-and-wait in
+   one call); quick-check with `job_status(job_id)`; kill with
+   `job_stop(job_id)`. Do **not** manually release a permit you handed to
+   `run_with_permit`.
 2. **`attach_pid(permit_token, pid)`** — you spawned the process yourself;
    the daemon now tracks that pid (and its process group) and releases when
    it is confirmed dead. Works for `... &` background jobs whose parent

package/docs/ARCHITECTURE.md

@@ -112,11 +112,15 @@ permits are journaled: on daemon restart, tracked permits are reconciled
 against real process liveness (dead ⇒ released, alive ⇒ kept, same tokens
 still valid), claimed permits keep their original start deadline, untracked
 permits get a fresh lease window, and a release arriving late for an entry
-that was reconciled is idempotent. Queues and offers are *not* journaled:
-they are ephemeral by design — queued agents are still running, notice the
-missed offer via their next tool call, and re-acquire; the daemon
-auto-spawns on the next client connection, so an agent never needs to know
-the daemon restarted. This is documented, deliberate behavior, not a bug.
+that was reconciled is idempotent. Queued *and offered* requests are journaled
+too (issue #34): they are re-queued FIFO by original enqueue time with the
+same request ids (idempotent re-acquire keeps working), entries offered at
+crash time restart as queued and are re-offered with fresh tokens — so a
+restart no longer silently drops a queue position. The daemon auto-spawns on
+the next client connection, and a *self-ordered* restart (version handshake:
+a newer client than the daemon) respawns with the same argv after persisting
+state — see the update lifecycle in
+[OPERATIONS.md](OPERATIONS.md#update-lifecycle).
 
 ## Notification honesty
 
@@ -144,6 +148,21 @@ mechanisms, and labels exactly which apply per harness:
    can never orphan a daemon-side waiter when the harness cancels the
    tool mid-sleep. Daemon restarts are detected via the event cursor and
    resolve bounded. No model-authored busy-polling required.
+   The execution phase has its mirror, `wait_for_completion`
+   (`src/shared/wait-completion.ts`), surfaced as the same CLI `wait`
+   command (`--job`), the MCP/OpenCode `wait_for_completion` tool and
+   `run_with_permit(wait: true)`. Two wake mechanisms: for
+   adapter-parented jobs (`run_with_permit`) the adapter awaits the
+   child's `exit` event directly — exact exit code, no polling; for
+   daemon-tracked permits (`attach_pid`) the daemon holds the wait
+   server-side and wakes it when the liveness sweep observes the pid
+   gone — mirroring `wait_for_turn`'s bounded-sleep shape (checkpoint
+   results on every path: finished / still_running / timeout /
+   job_unknown / stopped_by_request). Ordering guarantee: the permit is
+   released (capacity freed) *before* waiters are woken. Restarts are
+   transparent for tracked permits (journal restore), and against older
+   daemons the client degrades to a light events-based poll with the
+   same structured results.
 3. **Idle wakeup where the harness genuinely supports it.**
    - OpenCode (live-verified): the native plugin watches daemon events and
      calls `ctx.session.synthetic(...)`, delivering the offer (with token)
@@ -191,7 +210,7 @@ prompts.
 | Agent disconnects mid-heavy-work | Permit retained; tracked job keeps capacity until the real process finishes; untracked job's lease survives until heartbeats stop |
 | Agent process crashes | Tracked permit auto-releases on confirmed process death; untracked reclaims after lease expiry |
 | Background job outlives the tool call | pgid tracking retains the permit until the whole group exits |
-| Daemon crash/restart | Auto-respawn on next connect; journal reconciliation of held permits (conservative: dead ⇒ released, alive ⇒ kept); queues/offers ephemeral by design |
+| Daemon crash/restart | Auto-respawn on next connect; journal reconciliation of held permits (conservative: dead ⇒ released, alive ⇒ kept); queued/offered requests re-queued FIFO with same ids, offered entries re-offered with fresh tokens |
 | Stale/forged token | Opaque tokens; wrong-owner claim/release rejected; expired offers rejected with a precise error |
 | Forgotten release | start deadline (claimed), process death (tracked), lease expiry (untracked), plus human `recover force-release` |
 | PID reuse | `/proc` start-time comparison |

package/docs/COMPATIBILITY.md

@@ -73,6 +73,38 @@ tool round-trip (acquire → queued → wakeup → claim → release), execution
 boundary delivery without interrupting a running `sleep`, skill visible,
 and graceful handling of a deliberately missed offer (paused → cancel).
 
+#### Multiple copies of load-lock (opencode)
+
+`install --harness opencode` registers the **bare package name** in
+`~/.config/opencode/opencode.json`, so opencode maintains its own private
+copy of load-lock under
+`~/.cache/opencode/npm/@aron98/load-lock@latest/<timestamp>/node_modules/@aron98/load-lock/`,
+refreshed on opencode's schedule. That copy is *separate* from a globally
+installed CLI on PATH, and the two can drift (this was observed on a real
+machine with cache copies at 0.1.2, 1.0.0, 1.1.0 and 1.2.1 — issue #34).
+
+Which copy is authoritative for what:
+
+- **opencode plugin behavior** — the cache copy. Updated by opencode itself
+  (it refreshes the cache entry); you cannot reliably update it by hand.
+- **CLI commands** (`load-lock status`, `service`, `doctor`, …) — the copy on
+  PATH (typically a global install via npm/bun). Update it with your usual
+  global install command.
+- **The daemon's code** — whichever copy spawned it (usually the CLI copy,
+  since auto-spawn and `service start` run from the CLI client code). The
+  version handshake makes a daemon self-restart onto updated code when a
+  newer client contacts it — but the respawn reuses the daemon's original
+  argv, so it only picks up an update when its spawning copy was updated
+  *in place* (the normal global-upgrade case). A daemon spawned from a
+  different copy than the one you updated keeps running the old code until
+  `load-lock service stop` + any command re-spawns it from the new one.
+
+`load-lock doctor` lists every copy it can find (PATH-resolved CLI, this
+running copy, opencode cache copies) with versions and warns when they
+drift. Detection is best-effort (never crashes on missing paths) and was
+contract-tested with planted copies; the drift scenario itself was
+observed live on this machine before the feature existed.
+
 ### Tier 2 — Claude Code / Codex CLI (hooks + MCP, live-verified)
 
 `load-lock install --harness claude-code|codex` configures the MCP server

package/docs/OPERATIONS.md

@@ -95,15 +95,62 @@ uninstall — see issue #19); not live-verified inside the real CLIs.
   [Sandboxes](#sandboxes)).
 - Also honored by the shared client: `LOAD_LOCK_LABEL_PREFIX` (prefix for
   holder labels; see Sandboxes).
-- Files: `config.json` (settings + history), `journal.json` (held permits),
-  `daemon.json` (pid), `spawn.lock` (bootstrap race guard).
+- Files: `config.json` (settings + history), `journal.json` (held permits +
+  queued requests), `daemon.json` (pid), `spawn.lock` (bootstrap race guard).
 
 ```sh
-load-lock service status   # hello + pid
-load-lock service start    # ensure running (idempotent)
-load-lock service stop     # SIGTERM; next tool call respawns
+load-lock service status   # hello + pid + daemon version
+load-lock service start    # ensure running (idempotent); refusals name the conflicting pid
+load-lock service stop     # SIGTERM → bounded wait → SIGKILL; verifies the socket is free
 ```
 
+`service stop` is honest, not fire-and-forget: it signals the daemon
+(`--timeout <ms>` bounds the wait, default 5s), escalates to SIGKILL if the
+socket still answers, and only reports success once the endpoint is
+verifiably free (stale `daemon.json`/socket files are cleaned up either way).
+`service start` refuses loudly when something else owns the endpoint —
+naming the conflicting pid — instead of a bare "did not come up within 10s".
+
+## Update lifecycle
+
+The daemon is long-lived, so an update (`npm i -g @aron98/load-lock@latest`,
+or opencode refreshing its plugin copy) leaves the running daemon on stale
+code. The **version handshake** closes that gap (v1.3.0+):
+
+- Every client envelope carries `clientVersion`; `hello`/`status` replies
+  carry `daemonVersion` (the package version of the code the daemon runs).
+- When a **newer** client contacts an older daemon, the daemon finishes
+  answering, then **self-restarts**: it persists state + journal, drains all
+  connections, respawns itself with the same argv (`process.execPath` +
+  original arguments, so it runs whatever is now on disk at the same path),
+  and exits. The respawn inherits a `LOAD_LOCK_RESTARTED_FOR` marker so
+  repeated contact with the same version cannot cause a restart loop — one
+  respawn attempt per version.
+- Clients that were connected see a disconnect and simply reconnect to the
+  fresh daemon.
+- **What survives a restart:** held permits are journaled and reconciled
+  conservatively (tracked jobs re-checked for liveness, same permit tokens
+  valid; dead jobs released); queued *and offered* requests are journaled and
+  re-queued FIFO with the same request ids (idempotent re-acquire keeps
+  working). Offered entries restart as queued and are re-offered with fresh
+  tokens — pre-restart offer tokens are invalid, exactly as for any restart.
+- Skew is visible before it heals: `load-lock status`, `service status`,
+  `doctor`, and the dashboards show "daemon vX, installed vY" and note the
+  pending self-restart; `doctor` actively waits a few seconds to confirm the
+  daemon came back on the new version.
+
+Limits, stated honestly: the respawn re-runs the **same argv** — it picks up
+an updated copy only if the daemon's launching copy was updated *in place*
+(the normal `npm i -g` case). A daemon spawned from a different copy (e.g.
+opencode's private cache copy) keeps running that copy's code after a
+restart; see the multi-copy notes in
+[COMPATIBILITY.md](COMPATIBILITY.md#multiple-copies-of-load-lock-opencode).
+Pre-1.3.0 daemons do not answer the handshake and never self-restart —
+`service stop` followed by any load-lock command moves them onto current
+code. The restart behavior itself is live-verified on Linux by the test
+suite (`src/test/update.test.ts`); the in-place-update path beyond
+same-argv respawn is inspected, not live-tested.
+
 ## Configuration
 
 All configuration is agent-editable through tools (no file editing for
@@ -119,7 +166,8 @@ load-lock get-config
 ```
 
 Defaults: one pool `heavy` capacity 1 · offer window 30s · blocking wait 5s ·
-turn-wait cap (`maxTurnWaitMs`, bounds `wait_for_turn`/`load-lock wait`) 10min ·
+turn-wait cap (`maxTurnWaitMs`, bounds `wait_for_turn`, `wait_for_completion`
+and `load-lock wait --id/--job`) 10min ·
 start deadline 120s · untracked lease 15min. Valid ranges:
 offer window 1s–10min, wait 0–10min, turn-wait cap 1s–1h, start deadline 1s–1h, lease 10s–24h.