@noopolis/daimon 0.2.0 → 0.2.2

package/dist/runtime/grokEngineBrokerTurn.js

@@ -1,25 +1,36 @@
+import { brokerTaskClockFileFor } from "./grokBrokerSlot.js";
+import { brokerClockObservation, parseBrokerProcessClock, prepareBrokerTaskClock, writeBrokerTaskClock } from "./grokBrokerTaskClock.js";
+import { GrokBrokerProviderClock } from "./grokBrokerProviderClock.js";
 import { createHash, randomUUID } from "node:crypto";
+import { engineBrokerCapabilityTtlMs } from "./engineBrokerCapabilities.js";
 import { decodeGrokHeadlessTurn } from "../pi/grokHeadlessResult.js";
 import { decodeGrokStreamUsage } from "../pi/grokStreamUsage.js";
-import { ENGINE_BROKER_VERSION } from "./engineBrokerProtocol.js";
+import { ENGINE_BROKER_VERSION, ENGINE_BROKER_TASK_CLOCK_RESET_FAILED } from "./engineBrokerProtocol.js";
 import { lowerEngineBrokerTurnLimits, mapGrokReportedModel } from "./engineBrokerTurnAccounting.js";
 import { NativeBrokerTurnFailure } from "./engineBrokerNativeClient.js";
 import { engineBrokerRequestLedgerPathFor, engineBrokerSealLedgerPathFor } from "./engineBrokerServiceConfig.js";
 import { ensureBrokerTurnLedgered } from "./grokEngineBrokerLedger.js";
 import { finishBrokerTurnWithUsage } from "./grokEngineBrokerMetering.js";
 import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js";
+import { awaitGrokWorkerIdentityEmpty, findGrokWorkerProcess, GROK_WORKER_REAP_WAIT_MS, grokWorkerIdentityEmpty, watchGrokBrokerTurnEnd } from "./grokEngineBrokerTurnEnd.js";
 import { GrokWorkerAttestationFailure } from "./grokWorkerAttestation.js";
 export class EngineBrokerTurnFailure extends Error {
     code;
     diagnostic;
     accounting;
     mcpCalls;
-    constructor(code, diagnostic, accounting, mcpCalls) {
+    shellClock;
+    providerClock;
+    cleanupDiagnostic;
+    constructor(code, diagnostic, accounting, mcpCalls, shellClock, providerClock, cleanupDiagnostic) {
         super("engine broker turn failed");
         this.code = code;
         this.diagnostic = diagnostic;
         this.accounting = accounting;
         this.mcpCalls = mcpCalls;
+        this.shellClock = shellClock;
+        this.providerClock = providerClock;
+        this.cleanupDiagnostic = cleanupDiagnostic;
     }
 }
 const limitReasonFor = { tokens: "token_ceiling", requests: "request_ceiling", timeout: "wake_timeout" };
@@ -38,37 +49,91 @@ const usageOf = (usage) => ({ input: usage.input, cacheRead: usage.cacheRead, ca
  * metered through {@link finishBrokerTurnWithUsage}; a replayed turn returns its
  * sealed accounting before any of this runs and never meters again.
  */
-export async function runGrokEngineBrokerTurn(deps, registration, wakeId, prompt, mcpEndpoint, signal, overrides) {
+export async function runGrokEngineBrokerTurn(deps, registration, wakeId, prompt, mcpEndpoint, signal, overrides, processClock) {
     const { agentId } = registration, declared = registration.model.model;
+    const writeClock = deps.writeTaskClock ?? ((value) => writeBrokerTaskClock(value, brokerTaskClockFileFor(registration.slot)));
     let limits;
     try {
         limits = lowerEngineBrokerTurnLimits(registration.limits, overrides);
+        if (processClock !== undefined)
+            parseBrokerProcessClock(processClock);
     }
     catch {
         throw new EngineBrokerTurnFailure("invalid_request", undefined, { outcome: "failed", usage: null, model: declared, requests: 0, limitReason: "none" });
     }
-    const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex");
-    const request = { version: ENGINE_BROKER_VERSION, kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt, mcpEndpoint, ...(overrides === undefined ? {} : { limits: overrides }) };
+    const turnId = createHash("sha256").update(`${registration.slot}\0${agentId}\0${wakeId}`).digest("hex");
+    const request = { version: ENGINE_BROKER_VERSION, kind: "start_turn", requestId: randomUUID(), turnId, slot: registration.slot, agentId, wakeId, prompt, mcpEndpoint, ...(overrides === undefined ? {} : { limits: overrides }), ...(processClock === undefined ? {} : { processClock }) };
     const begun = await deps.turns.begin(request, declared);
     const metering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), sealLedgerPath: engineBrokerSealLedgerPathFor(registration.usageLedgerPath), agentId, wakeId };
     if (begun !== "start") {
         await ensureBrokerTurnLedgered(begun.ledger, turnId, metering);
         return replay(begun.replay);
     }
+    const clock = processClock === undefined ? undefined : new GrokBrokerProviderClock(processClock);
     const controller = new AbortController();
+    let worker;
     const meter = new GrokBrokerTurnMeter(limits, () => controller.abort());
-    const onAbort = () => controller.abort();
+    let cancelledByCaller = false;
+    const onAbort = () => { cancelledByCaller = true; controller.abort(); };
     signal?.addEventListener("abort", onAbort, { once: true });
-    if (signal?.aborted)
+    if (signal?.aborted) {
+        cancelledByCaller = true;
         controller.abort();
+    }
     const timer = setTimeout(() => meter.trip("timeout"), limits.timeoutMs);
     timer.unref?.();
+    // What was in flight, and which worker held the final reply, at the instant the
+    // turn ended — read before killing the worker can close its calls or empty its
+    // identity. A final reply the 1 s watchdog poll never saw is caught here too.
+    let inFlightAtEnd = false;
+    controller.signal.addEventListener("abort", () => {
+        let mcp = 0;
+        try {
+            mcp = deps.mcp.activity?.(turnId)?.inFlight ?? 0;
+        }
+        catch {
+            mcp = 0;
+        }
+        inFlightAtEnd = mcp > 0 || meter.busy();
+        if (meter.finalReply() !== undefined)
+            worker ??= (deps.observeWorker ?? findGrokWorkerProcess)(registration.workerUid).catch(() => undefined);
+    }, { once: true });
+    let shellClock;
+    let resetClock = deps.clockedSlot === true;
+    let nativeStarted = false;
+    let workerReaped = false;
+    let sealedFailure;
     let nativeDiagnostic, attested = false, output, rejected = false, sealed;
+    // Set when the broker itself ended the turn (`grokEngineBrokerTurnEnd.ts`); the worker seen at the final reply names the turn it ended.
+    let endedBy;
+    let watchdog, isolationGuard;
     try {
-        const isolationGuard = await deps.prepareIsolation(registration);
+        const isolation = await deps.prepareIsolation(registration);
+        isolationGuard = isolation.guard;
+        // Publication may throw after rename. Claim cleanup before attempting it.
+        resetClock ||= isolation.configVariant === "clocked";
+        const coverage = await prepareBrokerTaskClock(isolation.configVariant, processClock, writeClock);
+        if (isolation.configVariant === "clocked" || processClock !== undefined)
+            shellClock = coverage;
         deps.proxy.registerIsolationGuard(turnId, isolationGuard);
-        deps.proxy.registerTurn(turnId, { policy: registration.model, meter });
-        const providerCapability = deps.proxy.capabilities.issue(agentId, turnId), mcpCapability = deps.mcp.register(agentId, turnId, mcpEndpoint);
+        deps.proxy.registerTurn(turnId, { policy: registration.model, meter, clock });
+        const capabilityTtlMs = engineBrokerCapabilityTtlMs(limits.timeoutMs);
+        const providerCapability = deps.proxy.capabilities.issue(agentId, turnId, capabilityTtlMs), mcpCapability = deps.mcp.register(agentId, turnId, mcpEndpoint, capabilityTtlMs);
+        watchdog = watchGrokBrokerTurnEnd({
+            meter, activity: () => deps.mcp.activity?.(turnId), signal: controller.signal, ...(deps.turnEnd === undefined ? {} : { timing: deps.turnEnd }),
+            // The worker is read once, as soon as it has made a request — long before
+            // anything can end the turn and reap it — so a final reply is never left
+            // without the worker that gave it.
+            onPoll: () => { if (worker === undefined && meter.snapshot().requests > 0)
+                worker = (deps.observeWorker ?? findGrokWorkerProcess)(registration.workerUid).catch(() => undefined); },
+            onFinalReply: () => { worker ??= (deps.observeWorker ?? findGrokWorkerProcess)(registration.workerUid).catch(() => undefined); },
+            // An idle stall is a deadline of its own: it trips the meter as a timeout, so it seals like one.
+            onEnd: (reason) => { endedBy = reason; if (reason === "idle")
+                meter.trip("timeout");
+            else
+                controller.abort(); }
+        });
+        nativeStarted = true;
         const result = await deps.runNative({ slot: registration.slot, requestId: request.requestId, turnId, agentId, wakeId, prompt, providerCapability, mcpCapability }, controller.signal);
         nativeDiagnostic = result.diagnostic;
         output = result.text;
@@ -86,8 +151,16 @@ export async function runGrokEngineBrokerTurn(deps, registration, wakeId, prompt
             throw new Error("engine broker turn limit reached");
         const usage = decoded.usage === undefined ? streamOrMeterUsage(stream, snapshot) : usageOf(decoded.usage);
         const accounting = { outcome: "completed", usage, model: declared, requests: requestCount(stream, snapshot), limitReason: "none" };
-        const completed = { version: request.version, kind: "completed", requestId: request.requestId, turnId, text: decoded.text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString(), ...accounting };
-        const result_ = { text: completed.text, workerPid: completed.workerPid, workerUid: completed.workerUid, workerStartTime: completed.workerStartTime, ...accounting };
+        const observation = brokerClockObservation(shellClock, clock?.snapshot());
+        const completed = {
+            version: request.version, kind: "completed", requestId: request.requestId, turnId,
+            text: decoded.text, workerPid: result.workerPid, workerUid: result.workerUid,
+            workerStartTime: result.startTicks.toString(), ...observation, ...accounting
+        };
+        const result_ = {
+            text: completed.text, workerPid: completed.workerPid, workerUid: completed.workerUid,
+            workerStartTime: completed.workerStartTime, ...observation, ...accounting
+        };
         await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }, () => { sealed = result_; });
         return result_;
     }
@@ -95,6 +168,34 @@ export async function runGrokEngineBrokerTurn(deps, registration, wakeId, prompt
         // Once the completed record is published it is the durable truth: anything
         // failing after that (metering) must neither re-seal the turn as failed nor
         // append a second row.
+        if (sealed !== undefined)
+            return sealed;
+        watchdog?.stop();
+        // A turn this broker aborted is sealed only once its worker identity is
+        // empty (bounded): the agent's next turn must not start a worker the
+        // previous launcher handler's identity-wide reap would kill.
+        if (controller.signal.aborted) {
+            workerReaped = await awaitGrokWorkerIdentityEmpty(registration.workerUid, deps.workerReapWaitMs ?? GROK_WORKER_REAP_WAIT_MS, deps.workerIdentityEmpty ?? ((uid) => grokWorkerIdentityEmpty(uid, undefined, resetClock)));
+        }
+        // Not aborted (an output-limit failure, say): nothing was killed, so the live view is the true one.
+        if (!controller.signal.aborted) {
+            let mcp = 0;
+            try {
+                mcp = deps.mcp.activity?.(turnId)?.inFlight ?? 0;
+            }
+            catch {
+                mcp = 0;
+            }
+            inFlightAtEnd = mcp > 0 || meter.busy();
+        }
+        const vetoed = cancelledByCaller || rejected || (output !== undefined && !attested) || inFlightAtEnd;
+        // Finished work is completed work. A worker that gave its final reply and was
+        // then ended by the broker, refused its stdout by the launcher's output bound,
+        // or left idle into the deadline did its job; only its exit went wrong.
+        const finished = vetoed ? undefined : await completedFromFinalReply(deps, registration, request, meter, error, endedBy, worker, isolationGuard, metering, (value) => { sealed = value; }, shellClock, clock).catch(() => undefined);
+        if (finished !== undefined)
+            return finished;
+        // Sealed completed but its ledger append failed: the published record is the truth, exactly as on the success path.
         if (sealed !== undefined)
             return sealed;
         const snapshot = meter.snapshot();
@@ -112,27 +213,109 @@ export async function runGrokEngineBrokerTurn(deps, registration, wakeId, prompt
         catch {
             mcpCalls = undefined;
         }
-        const failed = { version: request.version, kind: "failed", requestId: request.requestId, turnId, code, ...(diagnostic ? { diagnostic } : {}), ...(mcpCalls === undefined ? {} : { mcpCalls }), ...accounting };
+        const observation = brokerClockObservation(shellClock, clock?.snapshot());
+        const failed = {
+            version: request.version, kind: "failed", requestId: request.requestId, turnId, code,
+            ...observation, ...(diagnostic ? { diagnostic } : {}),
+            ...(mcpCalls === undefined ? {} : { mcpCalls }), ...accounting
+        };
         const reason = snapshot.limitReason !== "none" ? limitReasonFor[snapshot.limitReason] : rejected ? "turn_rejected" : "unknown";
-        await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) });
-        throw new EngineBrokerTurnFailure(code, diagnostic, accounting, mcpCalls);
+        const failure = new EngineBrokerTurnFailure(code, diagnostic, accounting, mcpCalls, shellClock, observation.providerClock);
+        await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, {
+            notionalUsd: 0, complete: false, reason, estimatedRequests: snapshot.estimatedRequests,
+            requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId })
+        }, () => { sealedFailure = failure; });
+        throw failure;
     }
     finally {
+        watchdog?.stop();
         clearTimeout(timer);
         signal?.removeEventListener("abort", onAbort);
         meter.abortInFlight();
-        deps.proxy.revokeTurn(turnId);
-        deps.proxy.revokeIsolationGuard(turnId);
-        deps.proxy.capabilities.revoke(turnId);
-        deps.mcp.revoke(turnId);
+        try {
+            deps.proxy.revokeTurn(turnId);
+            deps.proxy.revokeIsolationGuard(turnId);
+            deps.proxy.capabilities.revoke(turnId);
+            deps.mcp.revoke(turnId);
+        }
+        finally {
+            if (resetClock) {
+                try {
+                    // A disconnected launcher client can settle before its worker is reaped.
+                    // Never change the clock of a surviving shell, or release the slot as healthy.
+                    if (nativeStarted && !workerReaped) {
+                        workerReaped = await awaitGrokWorkerIdentityEmpty(registration.workerUid, deps.workerReapWaitMs ?? GROK_WORKER_REAP_WAIT_MS, deps.workerIdentityEmpty ?? ((uid) => grokWorkerIdentityEmpty(uid, undefined, true)));
+                        if (!workerReaped)
+                            throw new Error("Grok broker task clock worker did not terminate");
+                    }
+                    await writeClock(undefined);
+                }
+                catch (error) {
+                    deps.taskClockFailed?.();
+                    // Cleanup cannot replace a published outcome. Report the live fence
+                    // on the wire without rewriting the seal or its accounting.
+                    if (sealed !== undefined)
+                        return { ...sealed, cleanupDiagnostic: ENGINE_BROKER_TASK_CLOCK_RESET_FAILED };
+                    if (sealedFailure !== undefined)
+                        throw new EngineBrokerTurnFailure(sealedFailure.code, sealedFailure.diagnostic, sealedFailure.accounting, sealedFailure.mcpCalls, sealedFailure.shellClock, sealedFailure.providerClock, ENGINE_BROKER_TASK_CLOCK_RESET_FAILED);
+                    throw new Error("Grok broker task clock cleanup failed; restart required", { cause: error });
+                }
+            }
+        }
     }
 }
 function replay(response) {
     const accounting = { outcome: response.outcome, usage: response.usage, model: response.model, requests: response.requests, limitReason: response.limitReason };
     if (response.kind === "completed")
-        return { text: response.text, workerPid: response.workerPid, workerUid: response.workerUid, workerStartTime: response.workerStartTime, ...accounting, outcome: "completed" };
+        return {
+            text: response.text, workerPid: response.workerPid, workerUid: response.workerUid,
+            workerStartTime: response.workerStartTime,
+            ...brokerClockObservation(response.shellClock, response.providerClock), ...accounting, outcome: "completed"
+        };
     const code = response.code === "turn_conflict" || response.code === "unavailable" ? "engine_failed" : response.code;
-    throw new EngineBrokerTurnFailure(code, response.diagnostic, accounting, response.mcpCalls);
+    throw new EngineBrokerTurnFailure(code, response.diagnostic, accounting, response.mcpCalls, response.shellClock, response.providerClock);
+}
+/**
+ * The completed seal for a turn whose model gave its final reply but whose
+ * worker never delivered stdout: the broker ended it (`final_reply`), the
+ * launcher's output bound refused its stream (`output_limit`, which publishes
+ * nothing), or it idled into the wall-clock deadline. The reply is the one the
+ * proxy saw, already bounded with a marker (`boundGrokFinalReply`), the usage is
+ * what the proxy metered, and the worker is the one the launcher reported or
+ * the broker observed while it was alive. `undefined` — and the ordinary
+ * failure seal — whenever any of that is missing, the isolation attestation
+ * fails, or the failure was anything else (a token or request limit, a client
+ * cancellation, an engine fault before the reply). The caller vetoes it outright
+ * for a caller cancellation, a rejected or unattested stdout, and a model
+ * request or MCP tool call still in flight.
+ */
+async function completedFromFinalReply(deps, registration, request, meter, error, endedBy, worker, isolationGuard, metering, onSealed, shellClock, clock) {
+    const final = meter.finalReply(), snapshot = meter.snapshot();
+    if (final === undefined || isolationGuard === undefined || snapshot.usage === null)
+        return undefined;
+    // A token or request limit is never finished work, whatever else failed beside it.
+    if (snapshot.limitReason === "tokens" || snapshot.limitReason === "requests")
+        return undefined;
+    const outputLimited = error instanceof NativeBrokerTurnFailure && error.diagnostic.failureClass === "output_limit" && error.diagnostic.workerPid > 0;
+    if (!(endedBy === "final_reply" || outputLimited || (snapshot.limitReason === "timeout" && endedBy !== "idle")))
+        return undefined;
+    const identity = outputLimited ? { pid: error.diagnostic.workerPid, uid: error.diagnostic.workerUid, startTicks: error.diagnostic.startTicks } : await (async () => { const seen = (await worker) ?? await (deps.observeWorker ?? findGrokWorkerProcess)(registration.workerUid).catch(() => undefined); return seen === undefined ? undefined : { ...seen, uid: registration.workerUid }; })();
+    if (identity === undefined || identity.uid !== registration.workerUid)
+        return undefined;
+    await isolationGuard();
+    const accounting = { outcome: "completed", usage: snapshot.usage, model: registration.model.model, requests: snapshot.requests, limitReason: "none" };
+    const observation = brokerClockObservation(shellClock, clock?.snapshot());
+    const completed = {
+        version: request.version, kind: "completed", requestId: request.requestId, turnId: request.turnId,
+        text: final.text, workerPid: identity.pid, workerUid: identity.uid, workerStartTime: identity.startTicks,
+        ...observation, ...accounting
+    };
+    const result = {
+        text: completed.text, workerPid: completed.workerPid, workerUid: completed.workerUid,
+        workerStartTime: completed.workerStartTime, ...observation, ...accounting
+    };
+    await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: 0, complete: false, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(undefined, snapshot) }, () => onSealed(result));
+    return result;
 }
 /** The proxy saw every forwarded request; the stream is the fallback when no request crossed this proxy. */
 const requestCount = (stream, snapshot) => snapshot.requests > 0 ? snapshot.requests : stream?.requests.length ?? 0;

package/dist/runtime/grokEngineBrokerTurnEnd.d.ts

@@ -0,0 +1,88 @@
+import type { EngineBrokerMcpActivity } from "./engineBrokerMcpCallLog.js";
+import type { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js";
+/**
+ * When a brokered turn is over, decided by the broker rather than by the
+ * worker's exit.
+ *
+ * Grok 1.0.34 answers, logs `handle_prompt.done ok:true`, ends its session
+ * `turn_ended completed` — and then does not exit. Production measured it: 61
+ * of 128 turns over two days sat 8–27 minutes after their last model request
+ * until the wall-clock limit killed them, and each was then sealed
+ * `failed/timeout` with its work already done. Every one still had its MCP GET
+ * tunnel open, but that is a symptom: against the real 1.0.34 binary the tunnel
+ * held open, closed, refused (405) or absent all exit ~55 ms after `result`,
+ * and Grok reopens a closed tunnel within milliseconds, so closing it cannot
+ * end a worker. The launcher only publishes a worker's output when the worker
+ * exits, so waiting for the exit made the deadline the only way a turn could
+ * end.
+ *
+ * The proxy sees every model answer, so the broker knows when the model has
+ * given its final reply: a successful response that called no tool and
+ * stopped. From that point:
+ *
+ * 1. `onFinalReply` runs once — the broker notes the worker it will end while
+ *    that worker is certainly still alive.
+ * 2. After {@link GROK_BROKER_TURN_END}`.finalGraceMs` with no new model request
+ *    and no tool call in flight, `onEnd("final_reply")` runs: the broker ends
+ *    the worker and seals the turn completed with that reply.
+ *
+ * A new model request clears the final reply (the meter forgets it on admit),
+ * so a model that keeps going is never cut off. Independently, a turn with no
+ * model request and no tool call in flight and no activity of either kind for
+ * `idleMs` ends as `onEnd("idle")` — the backstop for a worker that stalls
+ * without ever answering. Shell tools Grok runs itself are invisible here, so
+ * the idle bound is minutes, not seconds.
+ */
+export type GrokBrokerTurnEndTiming = Readonly<{
+    finalGraceMs: number;
+    idleMs: number;
+    pollMs: number;
+}>;
+export declare const GROK_BROKER_TURN_END: GrokBrokerTurnEndTiming;
+/** How long a turn the broker aborted waits for its worker identity to empty before it is sealed. */
+export declare const GROK_WORKER_REAP_WAIT_MS = 10000;
+export type GrokBrokerTurnEndReason = "final_reply" | "idle";
+export declare function watchGrokBrokerTurnEnd(input: Readonly<{
+    meter: Pick<GrokBrokerTurnMeter, "finalReply" | "busy" | "lastActivityAt">;
+    activity: () => EngineBrokerMcpActivity | undefined;
+    onFinalReply: () => void;
+    /** Every poll, before any decision: the broker records the worker while it is certainly alive. */
+    onPoll?: () => void;
+    onEnd: (reason: GrokBrokerTurnEndReason) => void;
+    timing?: GrokBrokerTurnEndTiming;
+    now?: () => number;
+    /** The turn's own abort: once anything else ended the turn, the watchdog never claims it. */
+    signal?: AbortSignal;
+}>): Readonly<{
+    stop(): void;
+}>;
+/** The launcher-forked root of a worker identity's process tree: its pid and `/proc/<pid>/stat` start time, as the launcher itself reports them. */
+export type GrokWorkerProcess = Readonly<{
+    pid: number;
+    startTicks: string;
+}>;
+/**
+ * Finds the worker the launcher started for `uid`, read-only from `/proc`.
+ *
+ * A turn the broker ends early never receives the launcher's result frame (a
+ * cancelled client is not written to), yet a completed response must name the
+ * worker it came from. One identity carries at most one turn at a time
+ * (`native/AGENTS.md`), so a process owned by `uid` whose parent is *not* owned
+ * by `uid` is a worker the launcher forked. A previous turn's worker can still
+ * be alive beside it — a hung worker is exactly what this module exists for, and
+ * its kill is the launcher's — so the newest such root (the latest start time)
+ * is this turn's: turns of one identity are sequential.
+ */
+export declare function findGrokWorkerProcess(uid: number, procRoot?: string): Promise<GrokWorkerProcess | undefined>;
+/** No process at all, zombies included, owned by `uid`. */
+export declare function grokWorkerIdentityEmpty(uid: number, procRoot?: string, requireReadable?: boolean): Promise<boolean>;
+/**
+ * Waits, bounded, for the worker identity to empty after the broker ended a
+ * turn. Ending a turn kills the launcher's client; the launcher handler then
+ * kills and reaps the identity on its own schedule. Sealing first would let
+ * the agent's next turn start a worker on the same identity that the previous
+ * handler's identity-wide reap then kills. The bound keeps a wedged launcher
+ * from holding the agent: past it the turn is sealed anyway, as before.
+ */
+export declare function awaitGrokWorkerIdentityEmpty(uid: number, timeoutMs: number, empty?: (uid: number) => Promise<boolean>, pollMs?: number, stableMs?: number): Promise<boolean>;
+//# sourceMappingURL=grokEngineBrokerTurnEnd.d.ts.map

package/dist/runtime/grokEngineBrokerTurnEnd.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grokEngineBrokerTurnEnd.d.ts","sourceRoot":"","sources":["../../src/runtime/grokEngineBrokerTurnEnd.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,6BAA6B,CAAC;AAC3E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAEpE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,MAAM,uBAAuB,GAAG,QAAQ,CAAC;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AACzG,eAAO,MAAM,oBAAoB,EAAE,uBAAqG,CAAC;AACzI,qGAAqG;AACrG,eAAO,MAAM,wBAAwB,QAAS,CAAC;AAC/C,MAAM,MAAM,uBAAuB,GAAG,aAAa,GAAG,MAAM,CAAC;AAE7D,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,QAAQ,CAAC;IACrD,KAAK,EAAE,IAAI,CAAC,mBAAmB,EAAE,YAAY,GAAG,MAAM,GAAG,gBAAgB,CAAC,CAAC;IAC3E,QAAQ,EAAE,MAAM,uBAAuB,GAAG,SAAS,CAAC;IACpD,YAAY,EAAE,MAAM,IAAI,CAAC;IACzB,kGAAkG;IAClG,MAAM,CAAC,EAAE,MAAM,IAAI,CAAC;IACpB,KAAK,EAAE,CAAC,MAAM,EAAE,uBAAuB,KAAK,IAAI,CAAC;IACjD,MAAM,CAAC,EAAE,uBAAuB,CAAC;IACjC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,6FAA6F;IAC7F,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC,GAAG,QAAQ,CAAC;IAAE,IAAI,IAAI,IAAI,CAAA;CAAE,CAAC,CAyB9B;AAED,oJAAoJ;AACpJ,MAAM,MAAM,iBAAiB,GAAG,QAAQ,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,SAAU,GAAG,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC,CAqBnH;AAED,2DAA2D;AAC3D,wBAAsB,uBAAuB,CAC3C,GAAG,EAAE,MAAM,EAAE,QAAQ,SAAU,EAAE,eAAe,UAAQ,GACvD,OAAO,CAAC,OAAO,CAAC,CAgBlB;AAED;;;;;;;GAOG;AACH,wBAAsB,4BAA4B,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,GAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAA2B,EAAE,MAAM,SAAM,EAAE,QAAQ,SAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAarM"}

package/dist/runtime/grokEngineBrokerTurnEnd.js

@@ -0,0 +1,148 @@
+import { readdir, readFile } from "node:fs/promises";
+import { performance } from "node:perf_hooks";
+export const GROK_BROKER_TURN_END = Object.freeze({ finalGraceMs: 15_000, idleMs: 10 * 60_000, pollMs: 1_000 });
+/** How long a turn the broker aborted waits for its worker identity to empty before it is sealed. */
+export const GROK_WORKER_REAP_WAIT_MS = 10_000;
+export function watchGrokBrokerTurnEnd(input) {
+    const timing = input.timing ?? GROK_BROKER_TURN_END, now = input.now ?? Date.now;
+    let notified, done = false;
+    const check = () => {
+        if (done)
+            return;
+        if (input.signal?.aborted === true) {
+            stop();
+            return;
+        }
+        try {
+            input.onPoll?.();
+        }
+        catch { /* observation only */ }
+        const final = input.meter.finalReply();
+        if (final !== undefined && notified !== final.index) {
+            notified = final.index;
+            try {
+                input.onFinalReply();
+            }
+            catch { /* closing streams is best effort; the grace below still ends the turn */ }
+        }
+        let activity;
+        try {
+            activity = input.activity();
+        }
+        catch {
+            activity = undefined;
+        }
+        if (input.meter.busy() || (activity?.inFlight ?? 0) > 0)
+            return;
+        const quietSince = Math.max(input.meter.lastActivityAt(), activity?.lastActivityAt ?? 0);
+        const reason = final !== undefined && now() - Math.max(final.at, quietSince) >= timing.finalGraceMs ? "final_reply"
+            : now() - quietSince >= timing.idleMs ? "idle" : undefined;
+        if (reason === undefined)
+            return;
+        stop();
+        input.onEnd(reason);
+    };
+    const timer = setInterval(check, timing.pollMs);
+    timer.unref?.();
+    const stop = () => { done = true; clearInterval(timer); };
+    return { stop };
+}
+/**
+ * Finds the worker the launcher started for `uid`, read-only from `/proc`.
+ *
+ * A turn the broker ends early never receives the launcher's result frame (a
+ * cancelled client is not written to), yet a completed response must name the
+ * worker it came from. One identity carries at most one turn at a time
+ * (`native/AGENTS.md`), so a process owned by `uid` whose parent is *not* owned
+ * by `uid` is a worker the launcher forked. A previous turn's worker can still
+ * be alive beside it — a hung worker is exactly what this module exists for, and
+ * its kill is the launcher's — so the newest such root (the latest start time)
+ * is this turn's: turns of one identity are sequential.
+ */
+export async function findGrokWorkerProcess(uid, procRoot = "/proc") {
+    let entries;
+    try {
+        entries = await readdir(procRoot);
+    }
+    catch {
+        return undefined;
+    }
+    const owned = new Map();
+    for (const entry of entries) {
+        if (!/^\d+$/u.test(entry))
+            continue;
+        try {
+            const status = await readFile(`${procRoot}/${entry}/status`, "utf8");
+            const real = /^Uid:\s+(\d+)/mu.exec(status)?.[1];
+            if (real === undefined || Number(real) !== uid)
+                continue;
+            const stat = await readFile(`${procRoot}/${entry}/stat`, "utf8");
+            const fields = stat.slice(stat.lastIndexOf(")") + 1).trim().split(/\s+/u);
+            // After the command: state(0) ppid(1) … starttime is field 22 of stat, index 19 here.
+            const parent = Number(fields[1]), startTicks = fields[19];
+            if (fields[0] === "Z" || !Number.isSafeInteger(parent) || startTicks === undefined || !/^[1-9][0-9]*$/u.test(startTicks))
+                continue;
+            owned.set(Number(entry), { parent, startTicks });
+        }
+        catch { /* a process exiting mid-scan is not part of the answer */ }
+    }
+    const roots = [...owned].filter(([, value]) => !owned.has(value.parent)).sort(([, left], [, right]) => BigInt(right.startTicks) > BigInt(left.startTicks) ? 1 : BigInt(right.startTicks) < BigInt(left.startTicks) ? -1 : 0);
+    const newest = roots[0];
+    return newest === undefined ? undefined : { pid: newest[0], startTicks: newest[1].startTicks };
+}
+/** No process at all, zombies included, owned by `uid`. */
+export async function grokWorkerIdentityEmpty(uid, procRoot = "/proc", requireReadable = false) {
+    let entries;
+    try {
+        entries = await readdir(procRoot);
+    }
+    catch {
+        return !requireReadable;
+    }
+    for (const entry of entries) {
+        if (!/^\d+$/u.test(entry))
+            continue;
+        try {
+            const status = await readFile(`${procRoot}/${entry}/status`, "utf8");
+            // Zombies count, exactly as the launcher's own reap counts them.
+            if (Number(/^Uid:\s+(\d+)/mu.exec(status)?.[1]) !== uid)
+                continue;
+            return false;
+        }
+        catch (error) {
+            // ENOENT is a process exiting mid-scan; an unreadable identity is not empty.
+            if (requireReadable && error.code !== "ENOENT")
+                return false;
+        }
+    }
+    return true;
+}
+/**
+ * Waits, bounded, for the worker identity to empty after the broker ended a
+ * turn. Ending a turn kills the launcher's client; the launcher handler then
+ * kills and reaps the identity on its own schedule. Sealing first would let
+ * the agent's next turn start a worker on the same identity that the previous
+ * handler's identity-wide reap then kills. The bound keeps a wedged launcher
+ * from holding the agent: past it the turn is sealed anyway, as before.
+ */
+export async function awaitGrokWorkerIdentityEmpty(uid, timeoutMs, empty = grokWorkerIdentityEmpty, pollMs = 100, stableMs = 300) {
+    // Empty must hold across a window longer than the launcher's own reap pass
+    // (10 ms between scans): one empty glimpse can fall between its kill and its
+    // re-scan, and that re-scan would kill the agent's next worker.
+    // Cleanup must measure elapsed real time even if the wall clock changes.
+    const deadline = performance.now() + timeoutMs;
+    let emptySince;
+    for (;;) {
+        const now = performance.now();
+        if (await empty(uid).catch(() => false)) {
+            emptySince ??= now;
+            if (now - emptySince >= stableMs)
+                return true;
+        }
+        else
+            emptySince = undefined;
+        if (now >= deadline)
+            return false;
+        await new Promise((resolve) => setTimeout(resolve, pollMs));
+    }
+}

package/dist/runtime/grokInferenceProxy.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"grokInferenceProxy.d.ts","sourceRoot":"","sources":["../../src/runtime/grokInferenceProxy.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAGhD,OAAO,KAAK,EAAE,6BAA6B,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE9F,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAGpE,MAAM,MAAM,uBAAuB,GAAG,QAAQ,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAEzK,yGAAyG;AACzG,eAAO,MAAM,8BAA8B,+BAA2B,CAAC;AAEvE;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,uBAAuB,CAAC,KAAK,EAAE,uBAAuB,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAE,6BAA6B,EAAE,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiC1N"}
+{"version":3,"file":"grokInferenceProxy.d.ts","sourceRoot":"","sources":["../../src/runtime/grokInferenceProxy.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAGhD,OAAO,KAAK,EAAE,6BAA6B,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE9F,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAGpE,MAAM,MAAM,uBAAuB,GAAG,QAAQ,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAEzK,yGAAyG;AACzG,eAAO,MAAM,8BAA8B,+BAA2B,CAAC;AAEvE;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,uBAAuB,CAAC,KAAK,EAAE,uBAAuB,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAE,6BAA6B,EAAE,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAkC1N"}

package/dist/runtime/grokInferenceProxy.js

@@ -1,3 +1,4 @@
+import { GrokBrokerRunLimit } from "./grokBrokerRunBudget.js";
 import { createHash } from "node:crypto";
 import { GrokBrokerProxyRefusal } from "./grokBrokerProxy.js";
 import { parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js";
@@ -57,6 +58,8 @@ export async function serveGrokInferenceGrant(input, response, grants, authority
     }
     catch (error) {
         settle?.(undefined);
+        if (error instanceof GrokBrokerRunLimit)
+            return json(response, 429, '{"error":"run_request_limit"}');
         // Same rule as the subject path: a policy miss is non-retryable (400), because a
         // retryable 503 makes the client re-send a request the broker will never accept,
         // charging estimated usage for every attempt. 503 stays for transient faults only.

package/dist/runtime/grokSubscriptionRealm.js

@@ -1,6 +1,6 @@
 import { createHash, randomUUID } from "node:crypto";
 import { constants } from "node:fs";
-import { spawn } from "node:child_process";
+import { spawnRealTime as spawn } from "./taskClockProcess.js";
 import { lstat, open, rename, unlink } from "node:fs/promises";
 import path from "node:path";
 import { terminateChild, trackCliChild } from "../pi/cliProcess.js";

package/dist/runtime/grokWorkerAttestation.d.ts

@@ -1,3 +1,5 @@
+import type { GrokWorkerConfigVariant } from "../contracts/grokBrokerTaskClock.js";
+import { type GrokWorkerConfigPins } from "./grokWorkerHomeAttestation.js";
 import { type GrokWorkerTmpOptions } from "./grokWorkerTmpAttestation.js";
 /**
  * The per-turn freshness watermark taken before the worker is launched: the
@@ -10,6 +12,7 @@ export type GrokWorkerAttestationSnapshot = Readonly<{
     ino: number;
     size: number;
     denyPaths: readonly string[];
+    configVariant?: GrokWorkerConfigVariant;
 }>;
 type Snapshot = GrokWorkerAttestationSnapshot;
 export declare class GrokWorkerAttestationFailure extends Error {
@@ -50,7 +53,8 @@ export declare function parseGrokWorkerSandboxProfile(bytes: Uint8Array, profile
  * the events watermark it requires the 1.0.34 layout: events under
  * `$GROK_HOME/sessions/` (the root `sandbox-events.jsonl` stays empty on
  * 1.0.34), and a root-owned read-only worker home whose `config.toml` hashes to
- * the declared renderer output (`grokWorkerHomeAttestation.ts`).
+ * the declared renderer output and whose `skills/` is exactly the declared
+ * skills (`skillsSha256`; absent for none) (`grokWorkerHomeAttestation.ts`).
  *
  * It also refuses a deny list bubblewrap could not materialize
  * (`grokWorkerDenyPlacement.ts`), naming the entry and the ancestor that stops
@@ -69,7 +73,8 @@ export declare function prepareGrokWorkerAttestation(input: Readonly<{
     workerUid: number;
     workerGid?: number;
     brokerGid: number;
-    configSha256: string;
+    configSha256: GrokWorkerConfigPins;
+    skillsSha256?: string;
     registeredWorkers?: readonly Readonly<{
         profilePath: string;
         workerUid: number;
@@ -78,7 +83,9 @@ export declare function prepareGrokWorkerAttestation(input: Readonly<{
     uid: number;
     gid: number;
     tmp?: GrokWorkerTmpOptions;
-}>): Promise<Snapshot>;
+}>): Promise<Snapshot & {
+    configVariant: GrokWorkerConfigVariant;
+}>;
 /** The per-turn attestation input for one registration, carrying every registered worker so sibling temp is attested too. */
 export declare const grokBrokerAttestationInput: <T extends Readonly<{
     profilePath: string;
@@ -88,9 +95,9 @@ export declare const grokBrokerAttestationInput: <T extends Readonly<{
 }>>(registration: T, registrations: readonly Readonly<{
     profilePath: string;
     workerUid: number;
-}>[], configSha256: string) => T & {
+}>[], configSha256: GrokWorkerConfigPins) => T & {
     brokerGid: number;
-    configSha256: string;
+    configSha256: GrokWorkerConfigPins;
     registeredWorkers: {
         profilePath: string;
         workerUid: number;