@north-light/api 0.3.580 → 0.3.588

package/dist/daemon/reconcilers/node-lifecycle/remote-wake.js

@@ -0,0 +1,124 @@
+// remote-wake.ts — the wake sequence's steps for a node whose environment is kind `blaxel` (design-fleet, Wake call):
+// the funding-bill check (step 2), the wake of a resident, possibly frozen broker (step 3), the no-built-image wait (D95), and the
+// recovery of a `launching` execution whose starter is gone. A node in a `machine` environment never reaches any of it.
+import { liveExecutionForNode, endExecution, recordExecutionAwake, endLaunchingExecution, requestExecutionStop } from '../../../core/canvas/executions.js';
+import { openDb } from '../../../core/canvas/db.js';
+import { isCloudRole } from '../../../core/canvas/person-state.js';
+import { emitEvent } from '../../../core/events/emit.js';
+import { findRefusal, refusalError, refusedMark, writeCharge } from '../../../core/ledger/index.js';
+import { transition } from '../../../core/runtime/lifecycle.js';
+import { runnerForExecution } from '../../../core/runtime/runner/index.js';
+import { machineEnvironmentOfProfile } from '../../../core/runtime/runner/machine/environment.js';
+import { startCandidateOf } from '../../../core/runtime/start-check.js';
+import { settleSandboxBeforeWake } from '../ledger-sandbox.js';
+import { crtrVersion } from '../../../shared/crtr-version.js';
+import { resolveEnvironmentImage } from '../recipe-image.js';
+/** A `launching` execution running past this since `started_at` is stopped: the 30-minute recipe setup limit plus boot. */
+export const LAUNCH_DEADLINE_MS = 45 * 60_000;
+/** A `launching` execution with no process is given this long before it is called lost, so the starter's own call is not raced. */
+export const LAUNCH_LOST_GRACE_MS = 120_000;
+/** The release the control plane runs, which is the release a wake names. */
+export const acceptedRelease = () => crtrVersion();
+/** The machine environment the node's profile points at when this process is a cloud role (the machine is another host, reached
+ *  through the remote machine runner), else null: a Mac's own crtrd runs its machine environment's nodes as processes. */
+export async function remoteMachineEnvironmentOf(profileId) {
+    if (profileId === null || !isCloudRole())
+        return null;
+    return await machineEnvironmentOfProfile(profileId);
+}
+/** Step 2: the `funding bill` check. A refusal writes `nodes.spend_refusal` and one refused `sandbox` row per node, limit and month,
+ *  then throws `402 cap_exhausted`; no execution row is written and no runner is called. A pass clears `spend_refusal`. */
+export async function checkBilledWake(meta) {
+    const start = await startCandidateOf(meta);
+    if (start === null)
+        return;
+    const candidate = { scope: 'call', payer: start.payer, funding: 'bill', sub: start.sub, grantee: start.grantee, provider: 'blaxel' };
+    const refusal = await findRefusal(candidate);
+    const db = await openDb();
+    if (refusal === null) {
+        await db.prepare('UPDATE nodes SET spend_refusal = NULL WHERE node_id = ? AND spend_refusal IS NOT NULL').run(meta.node_id);
+        return;
+    }
+    await db.prepare('UPDATE nodes SET spend_refusal = ? WHERE node_id = ?')
+        .run(JSON.stringify({ limit: refusal.limit.id, usd: refusal.usd, reset_at: refusal.reset_at, at: new Date().toISOString() }), meta.node_id);
+    await writeCharge({ id: `refused:sandbox:${meta.node_id}:${refusal.limit.id}:${refusal.month}`, kind: 'sandbox', what: 'wake', usd: null,
+        payer: candidate.payer, funding: 'bill', sub: candidate.sub, grantee: candidate.grantee, node: meta.node_id, run: meta.run_id ?? null,
+        provider: 'blaxel', refused: refusedMark(refusal) });
+    throw refusalError(refusal);
+}
+/** A recipe environment with no built image: the wake is not started, the mail stays pending, and the image-built tick wakes it. */
+export class EnvironmentImagePending extends Error {
+    environmentId;
+    imageError;
+    constructor(environmentId, imageError) {
+        super(`environment ${environmentId} has no built image yet${imageError === null ? '' : ` (${imageError})`}; the wake waits for the build.`);
+        this.environmentId = environmentId;
+        this.imageError = imageError;
+        this.name = 'EnvironmentImagePending';
+    }
+}
+/** D95: the image a new sandbox is created from. `pending` means the environment has a recipe and no image has built. */
+export async function imageForWake(environmentId) {
+    return await resolveEnvironmentImage(environmentId);
+}
+/** Step 3: a `running` Blaxel execution is a resident broker, possibly frozen. On the accepted release it is woken and nothing else
+ *  is written. On another release it is asked to stop for the release first, so the thawed broker exits and the next claim starts
+ *  a new attempt on the accepted one. `lost` means no broker answered: the execution is ended and the caller starts a new attempt. */
+export async function wakeResident(execution) {
+    if (execution.release !== acceptedRelease())
+        await requestExecutionStop(execution.nodeId, { afterTurn: false, reason: 'release' });
+    if (await runnerForExecution(execution.runner).wake(execution) === 'no_broker') {
+        await endExecution(execution.executionId, { reason: 'lost' });
+        return 'lost';
+    }
+    // Bill the idle period this wake ends: `recordExecutionAwake` resets `awake_since` and clears `idle_at`, which the sandbox lane reads.
+    await settleSandboxBeforeWake(execution.executionId, Date.now());
+    await recordExecutionAwake(execution.executionId);
+    return 'woken';
+}
+/** A `running` remote execution is live while its runner reports the broker process running, frozen or not. When the runner
+ *  reports it gone (or ended non-zero), that exact execution is ended; a runner that cannot tell leaves it as it is. */
+export async function endRemoteExecutionIfGone(execution) {
+    const status = await runnerForExecution(execution.runner).status(execution);
+    if (status.state !== 'gone' && status.state !== 'failed')
+        return false;
+    const ended = await endExecution(execution.executionId, { reason: 'lost' });
+    if (ended)
+        emitEvent({ level: 'warn', event: 'execution.lost', node_id: execution.nodeId });
+    return ended;
+}
+/** The recovery table for a `launching` Blaxel execution found by a claim after its starter is gone. */
+export async function recoverLaunching(nodeId, now) {
+    const execution = await liveExecutionForNode(nodeId);
+    if (execution === null || execution.state !== 'launching' || execution.runner === 'process')
+        return;
+    const runner = runnerForExecution(execution.runner);
+    const age = now - Date.parse(execution.startedAt);
+    const status = await runner.status(execution);
+    if (status.state === 'unknown')
+        return;
+    if (status.state === 'running') {
+        if (age <= LAUNCH_DEADLINE_MS)
+            return;
+        // The end is claimed first and only while the row is still `launching`: a `session.bound` that won the race made it
+        // `running`, and that broker is not stopped.
+        if (!await endLaunchingExecution(execution.executionId, { reason: 'launch_timeout' }))
+            return;
+        await runner.stop(execution, { reason: 'launch_timeout' });
+        await transition(nodeId, 'crash', { reason: 'launch_failed', outcome: { detail: { schema: 'crtr.node-outcome-detail/v1', message: 'the sandbox launch did not report within the launch deadline' } } });
+        emitEvent({ level: 'warn', event: 'execution.launch_timeout', node_id: nodeId });
+        return;
+    }
+    if (status.state === 'failed') {
+        if (!await endLaunchingExecution(execution.executionId, { reason: 'launch_failed' }))
+            return;
+        await transition(nodeId, 'crash', { reason: 'launch_failed', outcome: { detail: { schema: 'crtr.node-outcome-detail/v1', message: `the sandbox start script failed: ${status.reason}` } } });
+        return;
+    }
+    // gone: the next wake is attempt + 1, and the lane does not count it as a respawn failure.
+    if (age > LAUNCH_LOST_GRACE_MS) {
+        if (!await endLaunchingExecution(execution.executionId, { reason: 'launch_lost' }))
+            return;
+        emitEvent({ level: 'warn', event: 'execution.launch_lost', node_id: nodeId });
+    }
+}

package/dist/daemon/reconcilers/recipe-image.d.ts

@@ -0,0 +1,49 @@
+import { type Recipe } from '../../core/runtime/runner/blaxel/image-build.js';
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
+/** How long a claim holds without renewal. The holder renews at a third of it. */
+export declare const LEASE_MS: number;
+/** After a failed build the same revision is not tried again for this long. */
+export declare const RETRY_AFTER_FAILURE_MS: number;
+export type EnvironmentImage = {
+    state: 'base';
+} | {
+    state: 'image';
+    ref: string;
+    revision: number;
+} | {
+    state: 'pending';
+    error: string | null;
+};
+/** What a wake of a node in `environmentId` creates its sandbox from (D25, D95). `base`: the environment has no recipe content (never set, or cleared). `pending`: it has one and no image has ever built, so the wake must not start a runner call. */
+export declare function resolveEnvironmentImage(environmentId: string): Promise<EnvironmentImage>;
+/** Starts a supervision tick when an environment's image is built, so the nodes waiting on it wake now instead of at the next beat. Returns the unsubscribe. */
+export declare function subscribeImageBuilt(requestTick: (environmentId: string) => void): Promise<() => void>;
+export interface RecipeBuildInput {
+    environmentId: string;
+    revision: number;
+    recipe: Recipe;
+    person: string;
+}
+export interface RecipeImageDeps {
+    /** Builds and pushes the image; returns its reference. The default clones with a G-1 token and uses Blaxel's build API. */
+    build?: (input: RecipeBuildInput) => Promise<{
+        ref: string;
+    }>;
+    now?: () => number;
+}
+export declare class RecipeImageReconciler {
+    private readonly deps;
+    constructor(deps?: RecipeImageDeps);
+    /** One pass: claim every environment with an unbuilt revision and build it detached from the tick. */
+    run(nowMs: number, ctx: {
+        lifecycle: DetachedWorkLifecycle;
+    }): Promise<void>;
+    private claim;
+    private buildClaimed;
+    private recordFailure;
+}
+/** Nodes of profiles that run in `environmentId` with wake mail unseen and no execution: the ones a missing image keeps dormant. */
+export declare function nodesWaitingOnImage(environmentId: string): Promise<Array<{
+    node_id: string;
+    run_id: string;
+}>>;

package/dist/daemon/reconcilers/recipe-image.js

@@ -0,0 +1,145 @@
+// The `recipe-image` lane (C-A5, D25/D93/D94/D95): builds one sandbox image per environment recipe revision.
+//
+// An environment whose `recipe_revision` is ahead of its `image_revision` is claimed with a conditional UPDATE of `image_building_at`
+// (the lease; renewed while the build runs, fenced on its exact value), so two reconciler processes build a revision once. A build
+// that succeeds sets `image_revision`/`image_ref` (the schema's trigger sends the `environment_image` signal); one that fails writes
+// `image_error`, leaves the last good `image_ref` and `image_revision`, and appends run event `environment.image_failed` to each node
+// waiting on the environment. A running sandbox is never touched: it keeps the revision it was created from (D93).
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { hasUnseenWakeMail } from '../../core/canvas/deliveries.js';
+import { openDb, withCanvasWrite } from '../../core/canvas/db.js';
+import { emitEvent } from '../../core/events/emit.js';
+import { appendRunEvent } from '../../core/runs/events.js';
+import { blaxelConfigFromEnv } from '../../core/runtime/runner/blaxel/client.js';
+import { buildImage, environmentImageName, prepareBuildDir, removeBuildDir } from '../../core/runtime/runner/blaxel/image-build.js';
+import { mintInstallationToken } from '../api/handlers/git-credential.js';
+import { RECIPE_HAS_CONTENT_SQL, recipeOf } from '../api/environment-of.js';
+import { brokerSignalBus } from '../broker-signals/bus.js';
+/** How long a claim holds without renewal. The holder renews at a third of it. */
+export const LEASE_MS = 3 * 60_000;
+/** After a failed build the same revision is not tried again for this long. */
+export const RETRY_AFTER_FAILURE_MS = 10 * 60_000;
+/** What a wake of a node in `environmentId` creates its sandbox from (D25, D95). `base`: the environment has no recipe content (never set, or cleared). `pending`: it has one and no image has ever built, so the wake must not start a runner call. */
+export async function resolveEnvironmentImage(environmentId) {
+    const row = await (await openDb()).prepare(`SELECT ${RECIPE_HAS_CONTENT_SQL} AS has_content, image_revision, image_ref, image_error FROM environments WHERE object = ?`).get(environmentId);
+    if (row === undefined)
+        throw new Error(`no environment ${environmentId}`);
+    if (!row.has_content)
+        return { state: 'base' };
+    if (row.image_ref === null || row.image_revision === null)
+        return { state: 'pending', error: row.image_error };
+    return { state: 'image', ref: row.image_ref, revision: Number(row.image_revision) };
+}
+/** Starts a supervision tick when an environment's image is built, so the nodes waiting on it wake now instead of at the next beat. Returns the unsubscribe. */
+export async function subscribeImageBuilt(requestTick) {
+    return await brokerSignalBus.subscribe((environmentId) => { requestTick(environmentId); }, { kinds: ['environment_image'] });
+}
+async function defaultBuild(input, config) {
+    const repositories = input.recipe.repositories.map((r) => r.github_full_name);
+    const minted = repositories.length === 0 ? null : await mintInstallationToken(input.person, repositories);
+    const dir = mkdtempSync(join(tmpdir(), 'crtr-recipe-image-'));
+    try {
+        await prepareBuildDir({ recipe: input.recipe, revision: input.revision, token: minted?.token ?? null, dir });
+        return await buildImage(config, environmentImageName(input.environmentId), dir);
+    }
+    finally {
+        removeBuildDir(dir);
+    }
+}
+export class RecipeImageReconciler {
+    deps;
+    constructor(deps = {}) {
+        this.deps = deps;
+    }
+    /** One pass: claim every environment with an unbuilt revision and build it detached from the tick. */
+    async run(nowMs, ctx) {
+        if (!ctx.lifecycle.acceptsDetachedWork())
+            return;
+        try {
+            const now = new Date(nowMs).toISOString();
+            const due = await (await openDb()).prepare(`SELECT object FROM environments WHERE kind = 'blaxel' AND recipe_revision > COALESCE(image_revision, 0) AND ${RECIPE_HAS_CONTENT_SQL}
+        AND (image_building_at IS NULL OR image_building_at <= ?)`).all(now);
+            for (const { object } of due) {
+                const claim = await this.claim(object, nowMs);
+                if (claim !== null)
+                    ctx.lifecycle.registerDetached(this.buildClaimed(object, claim.revision, claim.lease).catch((error) => { emitEvent({ level: 'error', event: 'recipe_image.failed', error }); }));
+            }
+        }
+        catch (error) {
+            emitEvent({ level: 'error', event: 'recipe_image.reconcile_failed', error });
+        }
+    }
+    async claim(environmentId, nowMs) {
+        const lease = new Date(nowMs + LEASE_MS).toISOString();
+        const row = await (await openDb()).prepare(`UPDATE environments SET image_building_at = ?
+      WHERE object = ? AND kind = 'blaxel' AND recipe_revision > COALESCE(image_revision, 0) AND ${RECIPE_HAS_CONTENT_SQL} AND (image_building_at IS NULL OR image_building_at <= ?)
+      RETURNING recipe_revision`).get(lease, environmentId, new Date(nowMs).toISOString());
+        return row === undefined ? null : { revision: Number(row.recipe_revision), lease };
+    }
+    async buildClaimed(environmentId, revision, firstLease) {
+        const now = this.deps.now ?? Date.now;
+        let lease = firstLease;
+        const renew = setInterval(() => {
+            const next = new Date(now() + LEASE_MS).toISOString();
+            void (async () => {
+                const renewed = await (await openDb()).prepare('UPDATE environments SET image_building_at = ? WHERE object = ? AND image_building_at = ?').run(next, environmentId, lease);
+                if (renewed.changes === 1)
+                    lease = next;
+            })().catch(() => undefined);
+        }, LEASE_MS / 3);
+        renew.unref();
+        try {
+            const recipe = await recipeOf(environmentId);
+            // The recipe moved on between the claim and this read: the next pass claims the newer revision.
+            if (recipe === null || recipe.revision !== revision) {
+                await (await openDb()).prepare('UPDATE environments SET image_building_at = NULL WHERE object = ? AND image_building_at = ?').run(environmentId, lease);
+                return;
+            }
+            let built;
+            try {
+                const person = await (await openDb()).prepare('SELECT person FROM profiles WHERE environment_id = ? ORDER BY created_at LIMIT 1').get(environmentId);
+                if (person === undefined)
+                    throw new Error(`no profile runs in environment ${environmentId}, so there is no person to take a GitHub token for`);
+                const input = { environmentId, revision, recipe, person: person.person };
+                built = await (this.deps.build ?? ((i) => defaultBuild(i, blaxelConfigFromEnv())))(input);
+            }
+            catch (error) {
+                await this.recordFailure(environmentId, lease, error instanceof Error ? error.message : String(error), now());
+                return;
+            }
+            await (await openDb()).prepare(`UPDATE environments SET image_revision = ?, image_ref = ?, image_error = NULL, image_building_at = NULL
+        WHERE object = ? AND image_building_at = ? AND COALESCE(image_revision, 0) < ?`).run(revision, built.ref, environmentId, lease, revision);
+        }
+        finally {
+            clearInterval(renew);
+        }
+    }
+    async recordFailure(environmentId, lease, error, nowMs) {
+        const retryAt = new Date(nowMs + RETRY_AFTER_FAILURE_MS).toISOString();
+        const written = await (await openDb()).prepare('UPDATE environments SET image_error = ?, image_building_at = ? WHERE object = ? AND image_building_at = ?').run(error, retryAt, environmentId, lease);
+        if (written.changes !== 1)
+            return; // the lease was lost: the process that holds it reports the outcome
+        emitEvent({ level: 'error', event: 'recipe_image.build_failed', fields: { environment_id: environmentId, error } });
+        for (const node of await nodesWaitingOnImage(environmentId)) {
+            try {
+                await withCanvasWrite(async (db) => { await appendRunEvent(db, node.run_id, node.node_id, 'environment.image_failed', { environment_id: environmentId, error }); });
+            }
+            catch (failure) {
+                emitEvent({ level: 'error', event: 'recipe_image.failure_event_failed', node_id: node.node_id, error: failure });
+            }
+        }
+    }
+}
+/** Nodes of profiles that run in `environmentId` with wake mail unseen and no execution: the ones a missing image keeps dormant. */
+export async function nodesWaitingOnImage(environmentId) {
+    const candidates = await (await openDb()).prepare(`SELECT n.node_id, n.run_id FROM nodes n JOIN profiles p ON p.id = n.profile_id
+    WHERE p.environment_id = ? AND n.status IN ('active', 'idle') AND n.run_id IS NOT NULL
+      AND NOT EXISTS (SELECT 1 FROM executions e WHERE e.node_id = n.node_id AND e.state != 'exited')`).all(environmentId);
+    const waiting = [];
+    for (const node of candidates)
+        if (await hasUnseenWakeMail(node.node_id))
+            waiting.push(node);
+    return waiting;
+}

package/dist/daemon/reconcilers/throttled-lane.d.ts

@@ -0,0 +1,11 @@
+export interface LanePass {
+    /** What the previous pass saved, or null. */
+    state: string | null;
+    /** Fenced: false when the lane was lost to another process, in which case the pass should stop. */
+    save(state: string | null): Promise<boolean>;
+    /** Renews the lane lease and says whether this pass still holds it. Call it before each external effect: false means another
+     *  process owns the lane (this one stalled or could not renew past the lease), and the pass must stop without enacting. */
+    holds(): Promise<boolean>;
+}
+/** Runs `pass` when this process wins the lane's claim and the last pass began at least `intervalMs` before `now`. Returns whether it ran. */
+export declare function runThrottledLane(lane: string, key: string, now: number, intervalMs: number, leaseMs: number, pass: (lane: LanePass) => Promise<void>): Promise<boolean>;

package/dist/daemon/reconcilers/throttled-lane.js

@@ -0,0 +1,63 @@
+// One pass of a lane that runs at most once per interval across every reconciler process.
+//
+// The lane's `reconciler_leases` row is both the overlap guard (one holder at a time, a dead holder's claim lapses) and the throttle
+// (its cursor records when the last pass began). A pass that needs to resume where the last one stopped keeps that position in the
+// same cursor as `state`. Nothing is kept in the process, so a restart or a second process neither repeats a pass early nor runs two at once.
+import { randomUUID } from 'node:crypto';
+import { openDb } from '../../core/canvas/db.js';
+import { advanceReconcilerCursor, claimReconcilerLease, releaseReconcilerLease, renewReconcilerLease } from '../../core/canvas/reconciler-leases.js';
+function parse(cursor) {
+    if (cursor === null)
+        return null;
+    const value = JSON.parse(cursor);
+    if (typeof value.at !== 'number')
+        throw new Error(`reconciler lease cursor is not a lane cursor: ${cursor}`);
+    return value;
+}
+/** Runs `pass` when this process wins the lane's claim and the last pass began at least `intervalMs` before `now`. Returns whether it ran. */
+export async function runThrottledLane(lane, key, now, intervalMs, leaseMs, pass) {
+    // The tick calls every lane every second or so; read first so a lane that is not due costs a SELECT, not a write.
+    const row = await (await openDb()).prepare('SELECT cursor FROM reconciler_leases WHERE lane = ? AND key = ?').get(lane, key);
+    const seen = parse(row?.cursor ?? null);
+    if (seen !== null && now - seen.at < intervalMs)
+        return false;
+    const owner = randomUUID();
+    const lease = await claimReconcilerLease(lane, key, owner, leaseMs);
+    if (lease === undefined)
+        return false;
+    try {
+        const last = parse(lease.cursor);
+        if (last !== null && now - last.at < intervalMs)
+            return false;
+        const state = last?.state ?? null;
+        if (!await advanceReconcilerCursor(lane, key, owner, JSON.stringify({ at: now, state })))
+            return false;
+        // A pass may outlive leaseMs (a slow storage service, a large sweep). Renew at a third of the lease so a live holder keeps the
+        // lane and no second process starts an overlapping pass; only a holder that stops renewing (dead, or its process stalled past
+        // the lease) loses it, and then `save` reports the loss.
+        let lost = false;
+        const renewNow = async () => {
+            if (lost)
+                return false;
+            try {
+                lost = !await renewReconcilerLease(lane, key, owner, leaseMs);
+            }
+            catch {
+                lost = true;
+            }
+            return !lost;
+        };
+        const renew = setInterval(() => { void renewNow(); }, Math.max(1, Math.floor(leaseMs / 3)));
+        renew.unref();
+        try {
+            await pass({ state, save: (next) => advanceReconcilerCursor(lane, key, owner, JSON.stringify({ at: now, state: next })), holds: renewNow });
+        }
+        finally {
+            clearInterval(renew);
+        }
+        return true;
+    }
+    finally {
+        await releaseReconcilerLease(lane, key, owner);
+    }
+}

package/dist/daemon/runs/append-section.d.ts

@@ -0,0 +1,9 @@
+import type { SessionEntry } from '../../core/conversation-store/index.js';
+import type { CanvasDb } from '../../core/canvas/db.js';
+export declare const TURN_INPUT_TYPE = "crtr-turn-input";
+export declare const SYSTEM_PROMPT_TYPE = "crtr-system-prompt";
+export declare const TURN_END_TYPE = "crtr-turn-end";
+/** The one run-status input that reads files (the armed structured-output request). A caller inside a write transaction
+ *  computes it first, outside the transaction, and passes it to `appendConversationEvents`. */
+export declare function rootRepromptPending(runId: string, nodeId: string, entries: SessionEntry[]): Promise<boolean>;
+export declare function appendConversationEvents(db: CanvasDb, runId: string, nodeId: string, entries: SessionEntry[], pendingInputIds: string[], repromptPending?: boolean): Promise<string[]>;

package/dist/daemon/runs/append-section.js

@@ -0,0 +1,135 @@
+import { appendRunEvent } from '../../core/runs/events.js';
+import { openQuestionIds } from '../../core/runs/questions.js';
+import { getNode } from '../../core/canvas/canvas.js';
+import { payerOfGrant } from '../../core/grants/mirror.js';
+import { runSchemaRepromptPending } from '../../core/runtime/stop-guard.js';
+export const TURN_INPUT_TYPE = 'crtr-turn-input';
+export const SYSTEM_PROMPT_TYPE = 'crtr-system-prompt';
+export const TURN_END_TYPE = 'crtr-turn-end';
+const zero = () => ({ input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0, cost_usd: 0 });
+function text(content) {
+    if (typeof content === 'string')
+        return content;
+    return Array.isArray(content) ? content.filter((block) => typeof block?.text === 'string').map((block) => block.text).join('\n') : '';
+}
+/** The `error` a crtr-turn-end entry carries, validated; absent on a turn that did not end in error. */
+function turnEndError(value) {
+    if (value === undefined || value === null)
+        return undefined;
+    const { code, message, retry_at } = value;
+    if ((code !== 'content_refused' && code !== 'model_error' && code !== 'rate_limited') || typeof message !== 'string')
+        throw new Error('invalid crtr-turn-end error');
+    if (code === 'rate_limited') {
+        if (typeof retry_at !== 'string')
+            throw new Error('invalid crtr-turn-end error');
+        return { code, message, retry_at };
+    }
+    return { code, message };
+}
+function usageParts(usage) {
+    return {
+        input_tokens: usage.input ?? 0, output_tokens: usage.output ?? 0,
+        cache_read_tokens: usage.cacheRead ?? 0, cache_write_tokens: usage.cacheWrite ?? 0,
+        cost_usd: usage.cost?.total ?? 0,
+    };
+}
+/** The one run-status input that reads files (the armed structured-output request). A caller inside a write transaction
+ *  computes it first, outside the transaction, and passes it to `appendConversationEvents`. */
+export async function rootRepromptPending(runId, nodeId, entries) {
+    if (nodeId !== runId || !entries.some((entry) => entry.type === 'custom' && entry.customType === TURN_END_TYPE))
+        return false;
+    const root = await getNode(runId);
+    return root !== null && await runSchemaRepromptPending(root);
+}
+export async function appendConversationEvents(db, runId, nodeId, entries, pendingInputIds, repromptPending) {
+    let pending = pendingInputIds;
+    for (const entry of entries) {
+        if (entry.type === 'custom' && entry.customType === TURN_INPUT_TYPE) {
+            const ids = entry.data?.message_ids;
+            if (!Array.isArray(ids) || !ids.every((id) => typeof id === 'string'))
+                throw new Error('invalid crtr-turn-input message_ids');
+            pending = [...new Set([...pending, ...ids])];
+        }
+        if (entry.type === 'custom' && entry.customType === SYSTEM_PROMPT_TYPE) {
+            const value = entry.data?.text;
+            if (typeof value !== 'string')
+                throw new Error('invalid crtr-system-prompt text');
+            await appendRunEvent(db, runId, nodeId, 'node.system_prompt.set', { entry_id: entry.id, text: value });
+        }
+        if (entry.type === 'custom' && entry.customType === TURN_END_TYPE) {
+            const data = entry.data;
+            const reason = data?.stop_reason;
+            if (typeof reason !== 'string')
+                throw new Error('invalid crtr-turn-end stop_reason');
+            const error = turnEndError(data?.error);
+            const started = await db.prepare("SELECT body_json FROM run_events WHERE run_id = ? AND node_id = ? AND type = 'node.turn.started' ORDER BY sequence_number DESC LIMIT 1")
+                .get(runId, nodeId);
+            if (!started)
+                throw new Error('crtr-turn-end without node.turn.started');
+            const { input } = JSON.parse(started.body_json);
+            const others = await db.prepare('SELECT status, pi_pid FROM node_rows WHERE run_id = ? AND node_id != ?').all(runId, nodeId);
+            // The root's own stop that the stop guard will reprompt with its armed schema keeps the run running (D2-RES-10).
+            const reprompt = repromptPending ?? await rootRepromptPending(runId, nodeId, [entry]);
+            const run_status = others.some((row) => row.pi_pid !== null && row.status === 'active') || reprompt
+                ? 'running' : (await openQuestionIds(runId, db)).length ? 'waiting_on_user' : 'idle';
+            await appendRunEvent(db, runId, nodeId, 'node.turn.completed', { entry_id: entry.id, message_ids: input.message_ids, run_status, stop_reason: reason, ...(error ? { error } : {}) });
+            // run.get's last_turn_error: set by the root's turn that ends in error, cleared by one that does not.
+            if (nodeId === runId)
+                await db.prepare('UPDATE nodes SET last_turn_error_json = ? WHERE node_id = ?').run(error ? JSON.stringify(error) : null, runId);
+        }
+        if (entry.type === 'message') {
+            const message = entry.message;
+            if (message.role === 'user') {
+                // A user entry inside a turn that has not completed continues it: the
+                // runtime's recovery prompt after a provider failure it retries answers
+                // the same messages, so the turn keeps them until its one completion.
+                const last = await db.prepare("SELECT type, body_json FROM run_events WHERE run_id = ? AND node_id = ? AND type IN ('node.turn.started', 'node.turn.completed') ORDER BY sequence_number DESC LIMIT 1")
+                    .get(runId, nodeId);
+                const open = last?.type === 'node.turn.started' ? JSON.parse(last.body_json).input.message_ids : [];
+                await appendRunEvent(db, runId, nodeId, 'node.turn.started', { entry_id: entry.id, input: { text: text(message.content), message_ids: [...new Set([...open, ...pending])] } });
+                pending = [];
+            }
+            if (message.role === 'assistant') {
+                // Exactly one output_text.done per assistant message, even an empty one
+                // (a refused or failed call stores no text), placed before the message's
+                // tool calls. Its text joins the text blocks as the streamed deltas do.
+                let textDone = false;
+                const doneText = async () => {
+                    if (textDone)
+                        return;
+                    textDone = true;
+                    const joined = message.content.filter((block) => block.type === 'text').map((block) => block.text).join('');
+                    // A stored message has ended, so its stop reason is never pi's in-flight `pending`/`deferred`.
+                    await appendRunEvent(db, runId, nodeId, 'node.output_text.done', { entry_id: entry.id, text: joined, stop_reason: message.stopReason });
+                };
+                for (const block of message.content) {
+                    if (block.type === 'thinking')
+                        await appendRunEvent(db, runId, nodeId, 'node.thinking.done', { entry_id: entry.id, text: block.thinking });
+                    if (block.type === 'text')
+                        await doneText();
+                    if (block.type === 'toolCall') {
+                        await doneText();
+                        await appendRunEvent(db, runId, nodeId, 'node.tool_call.started', { entry_id: entry.id, tool_call_id: block.id, name: block.name, arguments: block.arguments });
+                    }
+                }
+                await doneText();
+                if (message.usage) {
+                    const delta = usageParts(message.usage);
+                    const root = await db.prepare('SELECT usage_json, grant_id FROM nodes WHERE node_id = ?').get(runId);
+                    const priorTotal = root.usage_json ? JSON.parse(root.usage_json) : zero();
+                    const total = zero();
+                    for (const key of Object.keys(total))
+                        total[key] = priorTotal[key] + delta[key];
+                    await db.prepare('UPDATE nodes SET usage_json = ? WHERE node_id = ?').run(JSON.stringify(total), runId);
+                    await appendRunEvent(db, runId, nodeId, 'usage', { entry_id: entry.id, served_model: message.model, payer: await payerOfGrant(root.grant_id), ...delta, usage: total });
+                }
+                // The stored message carries the full text in its done events, so the
+                // streamed deltas that previewed it are spent.
+                await db.prepare("DELETE FROM run_events WHERE run_id = ? AND node_id = ? AND type IN ('node.thinking.delta', 'node.output_text.delta')").run(runId, nodeId);
+            }
+            if (message.role === 'toolResult')
+                await appendRunEvent(db, runId, nodeId, 'node.tool_call.completed', { entry_id: entry.id, tool_call_id: message.toolCallId, name: message.toolName, result: message.content, status: message.isError ? 'error' : 'ok' });
+        }
+    }
+    return pending;
+}

package/dist/daemon/runs/observer.d.ts

@@ -0,0 +1,17 @@
+/** What the model is streaming now in a node's open assistant message. */
+export type OpenAssistantBlock = {
+    kind: 'thinking' | 'text';
+} | {
+    kind: 'tool_call';
+    name: string;
+};
+/** The start sequence of the node's open assistant message, from the observer's columns; null when none is open or the node has no live broker. */
+export declare function openAssistantMessageStartSequence(nodeId: string): Promise<number | null>;
+/** The newest block of the node's open assistant message; null when no message is open or nothing has streamed yet. */
+export declare function openAssistantBlock(nodeId: string): Promise<OpenAssistantBlock | null>;
+/** Test-only: stop every observer and wait out the polls in flight, so none touches a home the test is about to delete. */
+export declare function closeRunObserversForTests(): Promise<void>;
+export declare function ensureRunObserver(nodeId: string, runId: string): Promise<void>;
+/** Observes run nodes with a live broker, plus those whose current outcome
+ *  (settled within the event retention window) has no `node.settled` event yet. */
+export declare function pollRunObservers(): Promise<void>;