@north-light/api 0.3.580 → 0.3.587

package/dist/core/runs/operations.js

@@ -0,0 +1,741 @@
+import { coveredByAny } from '../../shared/env.js';
+import { createHash } from 'node:crypto';
+import { ApiError } from '../../api/errors.js';
+import { CrtrError } from '../errors.js';
+import { isCloudRole } from '../canvas/person.js';
+import { isAbsolute, relative, resolve, sep, join } from 'node:path';
+import { SessionManager } from '@earendil-works/pi-coding-agent';
+import { covers } from '@north-light/identity';
+import { openDb, withCanvasWrite } from '../canvas/db.js';
+import { deleteNodeWithOwned } from '../graph/deletion.js';
+import { getById } from '../graph/objects.js';
+import { repairRunConversations } from '../canvas/sessions.js';
+import { getNode, nodeFiles, removeNodeFiles, updateNode } from '../canvas/canvas.js';
+import { fullName } from '../canvas/labels.js';
+import { renameWindow } from '../runtime/placement.js';
+import { liveExecutionForNode } from '../canvas/executions.js';
+import { conversationStore, isConversationNotFound } from '../conversation-store/index.js';
+import { PARKED_RESIDENT_SQL } from '../canvas/canvas.js';
+import { WAKE_MAIL_SQL } from '../canvas/deliveries.js';
+import { readNodeRecap } from '../canvas/node-recap.js';
+import { enqueueNodeMail } from '../feed/messages.js';
+import { newestReportBody } from '../feed/reports.js';
+import { ensureAppProfile, loadExactProfileManifest } from '../profiles/manifest.js';
+import { stopNodeSandboxes } from '../spaces/stop.js';
+import { BrokerUnreachableError } from '../runtime/broker-request.js';
+import { interruptLive } from '../runtime/interactive-deliver.js';
+import { isBrokerLive } from '../runtime/model-swap.js';
+import { newNodeId } from '../runtime/nodes.js';
+import { reviveNode } from '../runtime/revive.js';
+import { spawnChildPrepared } from '../runtime/spawn.js';
+import { isPortableModelToken } from '../runtime/launch.js';
+import { parseDurationMs } from '../wake.js';
+import { expandModelCandidates, parseModelSpec } from '../model-routes.js';
+import { payerOfGrant } from '../grants/mirror.js';
+import { isolatedBrokerLauncherAvailable } from '../runtime/host.js';
+import { blaxelEnvironmentOf } from '../runtime/blaxel-environment.js';
+import { resolveRunKind, resolveRunModel } from '../runtime/launch-target.js';
+import { cycleAwareMessages } from '../runtime/session-cycles.js';
+import { originFor } from '../runtime/stamp/protocol.js';
+import { readOutputRequest, readOutputResult } from '../runtime/output-result.js';
+import { admitLegacyStructuredOutput } from '../../daemon/structured-requests/sidecar.js';
+import { runSchemaRepromptPending } from '../runtime/stop-guard.js';
+import { toRunOutcomeDTO } from '../../daemon/api/map.js';
+import { openAssistantMessageStartSequence } from '../../daemon/runs/observer.js';
+import { closeNode } from '../runtime/close.js';
+import { parseScope } from '@north-light/identity';
+import { appendRunEvent, lockRunRoot, subscribeRunEvents } from './events.js';
+import { runActivity } from './activity.js';
+import { cancelOpenQuestions, openNoticeIds, openQuestionIds } from './questions.js';
+import { appSpace } from '../layout.js';
+import { ensureAppSpace } from '../spaces/space.js';
+import { assertUploadsUsable, resolveStorageUrls } from '../storage/attachments.js';
+import { storageClient, storageConfig } from '../storage/client.js';
+import { deleteUploadRow, insertUploadUses, releaseRunUploads, runUsesPath } from '../storage/tables.js';
+import { abortProviderCalls } from '../providers/inflight.js';
+import { filterAsync, mapAsync } from '../async-array.js';
+const defaultResolver = (runId, grantee, text, taken) => resolveStorageUrls(grantee, runId, text, taken ? { taken } : {});
+/** Storage deletes `run.delete` waits for before it answers. */
+export const RUN_DELETE_STORAGE_BUDGET_MS = 3_000;
+export class RunError extends Error {
+    status;
+    code;
+    param;
+    details;
+    scope;
+    retryable;
+    constructor(status, code, message, param, details, scope, retryable = status === 503) {
+        super(message);
+        this.status = status;
+        this.code = code;
+        this.param = param;
+        this.details = details;
+        this.scope = scope;
+        this.retryable = retryable;
+    }
+}
+const invalid = (message, param) => new RunError(400, 'invalid_request', message, param);
+const missing = () => new RunError(404, 'not_found', 'run not found');
+const usage = () => ({ input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0, cost_usd: 0 });
+const db = async () => await openDb();
+export async function ownedRun(id, grantee, allowDeleting = false) {
+    const node = await getNode(id);
+    if (!node || node.parent !== null || node.run_id !== id || node.grantee !== grantee || (!allowDeleting && node.deleting_at))
+        throw missing();
+    return node;
+}
+/** Statuses `run.get` `wait` returns on: the run needs no more waiting from the caller's side. */
+const RESTING = ['waiting_on_user', 'idle', 'settled'];
+async function runStatus(root) {
+    if (root.status === 'canceled' || root.status === 'dead' || (root.status === 'done' && root.intent !== 'parked'))
+        return 'settled';
+    const active = await (await db()).prepare(`SELECT 1 FROM node_rows n WHERE n.run_id = ? AND n.status = 'active' AND
+    (n.last_settled_at IS NULL OR EXISTS (
+      SELECT 1 FROM run_events e WHERE e.run_id = ? AND e.node_id = n.node_id AND e.type = 'node.turn.started'
+        AND e.sequence_number > COALESCE((SELECT MAX(c.sequence_number) FROM run_events c
+          WHERE c.run_id = e.run_id AND c.node_id = e.node_id AND c.type = 'node.turn.completed'), 0)
+    )) LIMIT 1`).get(root.node_id, root.node_id);
+    // A root the stop guard is about to reprompt with its armed schema is still
+    // working: the run reads idle only once the bound lets it go dormant (D2-RES-10).
+    // A root that has not yet settled a turn is still starting, unless it is parked:
+    // parking ends its broker, so nothing is left to start (run-model row 5).
+    const starting = !root.last_settled_at && root.status !== 'done';
+    if (active || starting || await runSchemaRepromptPending(root))
+        return 'running';
+    // Row 3 ranks open questions ahead of wake mail (run-model "Node status → run status").
+    if ((await openQuestionIds(root.node_id)).length)
+        return 'waiting_on_user';
+    // Row 4: only wake-tier mail moves a run, and only mail the runtime will launch a broker for:
+    // a live node (`active`/`idle`) or a parked resident it reopens (the node-lifecycle tick). Mail to
+    // a finalized, canceled, dead or deleting node never launches one, and passive notes, such as a
+    // stopped background job's, wait for the next turn. Mail the broker has claimed (`handed_off`) still counts:
+    // the claim precedes the turn's `node.turn.started` event, and until that turn settles and acknowledges the
+    // mail, an `idle` read would let `run.get` `wait` return before the turn that mail starts.
+    const mail = await (await db()).prepare(`SELECT 1 FROM deliveries d JOIN nodes n ON n.node_id = d.destination_id
+    WHERE n.run_id = ? AND ${WAKE_MAIL_SQL('d', "'armed', 'pending', 'handed_off'")}
+      AND (n.status IN ('active', 'idle') OR ${PARKED_RESIDENT_SQL('n')}) LIMIT 1`).get(root.node_id);
+    return mail ? 'running' : 'idle';
+}
+/** Whether the run has a turn in flight or work owed to it (the `running` run status). */
+export async function runIsRunning(runId) {
+    const root = await getNode(runId);
+    return root !== null && await runStatus(root) === 'running';
+}
+export async function runObject(root) {
+    await repairRunConversations(root.node_id);
+    root = await getNode(root.node_id) ?? root;
+    const status = await runStatus(root);
+    const pending = status === 'settled' ? [] : await openQuestionIds(root.node_id);
+    const report = await newestReportBody(root.node_id) || undefined;
+    const outcome = status === 'settled' ? await toRunOutcomeDTO(root) : null;
+    const result = outcome?.kind === 'result' ? await readOutputResult(root.node_id) : undefined;
+    const recap = readNodeRecap(root.node_id);
+    return {
+        run_id: root.node_id, name: root.name, status, ...(outcome ? { outcome } : {}),
+        ...(result === undefined ? {} : { result }),
+        ...(report !== undefined ? { last_report: report } : {}), ...(recap ? { recap } : {}), pending_question_ids: pending, open_notice_ids: status === 'settled' ? [] : await openNoticeIds(root.node_id),
+        ...(root.last_turn_error_json ? { last_turn_error: JSON.parse(root.last_turn_error_json) } : {}),
+        usage: JSON.parse(root.usage_json ?? JSON.stringify(usage())), payer: await payerOfGrant(root.grant_id), scopes: root.scopes,
+        metadata: root.metadata_json ? JSON.parse(root.metadata_json) : {}, created_at: root.created,
+        ...(status === 'settled' && root.outcome_at ? { settled_at: root.outcome_at } : {}), latest_sequence: (root.event_cursor ?? 1) - 1,
+        last_activity_at: root.last_activity_at, activity: await runActivity(root.node_id, status),
+    };
+}
+/** How long a kept key replays: a record this old or older reads as absent. The SDK's conformance scenarios hold FakeCloud to it. */
+export const IDEMPOTENCY_WINDOW_MS = 86_400_000;
+const MAX_IDEMPOTENCY_KEY = 255;
+/** An `Idempotency-Key` header value: absent, or 1–255 characters. */
+export function idempotencyKeyOf(header) {
+    if (header === undefined)
+        return undefined;
+    if (typeof header !== 'string' || !header.length || header.length > MAX_IDEMPOTENCY_KEY)
+        throw invalid(`Idempotency-Key must be 1-${MAX_IDEMPOTENCY_KEY} characters`);
+    return header;
+}
+/** The person a cloud role's per-person rows and keys are cut to: undefined on a Mac (one person, nothing to cut); on a cloud role the caller's `sub`,
+ *  and a caller without one is refused, so a missing `sub` can never read as "every person". */
+export function cloudPersonScope(sub) {
+    if (!isCloudRole())
+        return undefined;
+    if (!sub)
+        throw new RunError(403, 'permission_error', 'a cloud role serves this call only for a token that names a person');
+    return sub;
+}
+/** The key as stored and locked. On a cloud role many people share one database and one app grant name, so a key belongs to its person:
+ *  one person's key never replays, or conflicts with, another's. A Mac holds one person and keeps the bare key. */
+function personKey(sub, key) {
+    const person = cloudPersonScope(sub);
+    return person === undefined ? key : `${person}:${key}`;
+}
+/** The stored result of an earlier operation under (grantee, key) within the window. A kept key
+ *  with a different request hash is `idempotency_conflict`; the hash covers the operation and its
+ *  target, so one key cannot stand for two different operations. Reads in the caller's transaction when given. */
+async function findIdempotent(grantee, key, bodyHash, db) {
+    // In a transaction, concurrent requests under one key take turns: the later waits for the earlier commit, then reads its record.
+    if (db)
+        await db.prepare('SELECT pg_advisory_xact_lock(hashtext(?))').get(`idempotency:${grantee}:${key}`);
+    db ??= await openDb();
+    const existing = await db.prepare('SELECT body_hash, run_id, response_json, created_at FROM idempotency_keys WHERE grantee = ? AND key = ?').get(grantee, key);
+    if (!existing || Date.now() - Date.parse(existing.created_at) >= IDEMPOTENCY_WINDOW_MS)
+        return undefined;
+    if (existing.body_hash !== bodyHash)
+        throw new RunError(409, 'idempotency_conflict', 'key used with a different request');
+    return { run_id: existing.run_id, response: JSON.parse(existing.response_json) };
+}
+/** Records the result in the caller's transaction, replacing an expired row for the same key. */
+async function recordIdempotent(tx, grantee, key, bodyHash, runId, response, createdAt) {
+    await tx.prepare('DELETE FROM idempotency_keys WHERE grantee = ? AND key = ? AND created_at <= ?')
+        .run(grantee, key, new Date(Date.now() - IDEMPOTENCY_WINDOW_MS).toISOString());
+    await tx.prepare('INSERT INTO idempotency_keys (grantee, key, body_hash, run_id, response_json, created_at) VALUES (?, ?, ?, ?, ?, ?)')
+        .run(grantee, key, bodyHash, runId, JSON.stringify(response), createdAt);
+}
+/** Delete keys past their window; returns how many were removed. */
+export async function pruneExpiredIdempotencyKeys(now = Date.now()) {
+    return Number((await (await openDb()).prepare('DELETE FROM idempotency_keys WHERE created_at <= ?').run(new Date(now - IDEMPOTENCY_WINDOW_MS).toISOString())).changes);
+}
+/** The one path that accepts a message to a run's root: the mail row and its `node.message.accepted`
+ *  event, plus the attachment uses, in the caller's transaction. Returns the message id and the event's sequence. */
+async function acceptRunMessage(tx, runId, grantee, attachments, startTurn, steer = false) {
+    const message = attachments.text;
+    let sequence = 0;
+    const entry = await enqueueNodeMail(runId, { kind: 'message', label: message.slice(0, 120), senderNodeId: 'human', senderName: grantee, bodyJson: startTurn ? { body: message, ...(steer ? { steer: true } : {}) } : { body: message, start_turn: false }, urgency: startTurn ? 'normal' : 'deferred', wake: startTurn ? 1 : 0,
+        withinEnqueue: async (inner, messageId) => {
+            if (attachments.uses.length) {
+                await assertUploadsUsable(grantee, attachments.uses, inner);
+                await insertUploadUses(attachments.uses.map(({ upload_id, path, blob_key }) => ({ upload_id, path, blob_key, run_id: runId })), inner);
+            }
+            sequence = (await appendRunEvent(inner, runId, runId, 'node.message.accepted', { message_id: messageId, start_turn: startTurn })).sequence_number;
+        } });
+    return { messageId: entry.messageId, sequence };
+}
+/** Thrown inside the create transaction when a concurrent start under the same key committed first: rolls the
+ *  loser's writes back and carries the winner's stored answer to `startRun`'s catch. Never leaves `startRun`. */
+class IdempotentReplay extends Error {
+    winner;
+    constructor(winner) {
+        super('idempotent replay');
+        this.winner = winner;
+    }
+}
+/** Longest `run.start` deadline an app may set: 24 hours. The grammar is `crtr node new --deadline`'s (parts of `<int>[smhd]`, summed). */
+export const RUN_DEADLINE_MAX_MS = 24 * 60 * 60 * 1000;
+/** The milliseconds of a `run.start` `deadline` duration; refuses a malformed, zero, or over-24h value as `invalid_request` on `deadline`. */
+export function parseRunDeadlineMs(raw) {
+    const ms = typeof raw === 'string' ? parseDurationMs(raw) : null;
+    if (ms === null || !Number.isSafeInteger(ms))
+        throw invalid(`deadline '${String(raw)}' is not a duration; use one or more <int>[smhd] parts such as 10m or 1h30m`, 'deadline');
+    if (ms <= 0)
+        throw invalid('deadline must be greater than zero', 'deadline');
+    if (ms > RUN_DEADLINE_MAX_MS)
+        throw invalid('deadline must be at most 24h', 'deadline');
+    return ms;
+}
+export async function startRun(principal, input, idempotencyKey, resolveAttachments = defaultResolver) {
+    // Refuse before idempotency replay, attachment fetch, or node birth: a direct
+    // broker spawn would expose the host filesystem and network to this run.
+    // A run whose profile points at a Blaxel environment is confined by its sandbox and needs no launcher.
+    if (input.isolated && !isolatedBrokerLauncherAvailable()
+        && await blaxelEnvironmentOf((input.profile === undefined ? await ensureAppProfile(principal.grantee) : loadExactProfileManifest(input.profile)).profileId) === null) {
+        throw new RunError(503, 'isolated_unavailable', 'isolated runs require the Linux root launcher', 'isolated');
+    }
+    // Validated before anything is created or replayed. The duration string (not the instant) is what the body hash covers,
+    // so a retry of the same keyed request replays instead of conflicting on a moved clock.
+    if (input.deadline !== undefined)
+        parseRunDeadlineMs(input.deadline);
+    const { causedBy: _causedBy, ...hashed } = input; // a replay returns the stored run whatever its caused-by
+    const bodyHash = createHash('sha256').update(JSON.stringify(hashed)).digest('hex');
+    if (idempotencyKey) {
+        const existing = await findIdempotent(principal.grantee, personKey(principal.sub, idempotencyKey), bodyHash);
+        if (existing) {
+            const root = await ownedRun(existing.run_id, principal.grantee);
+            if (!root.pi_pid && await runStatus(root) === 'running') {
+                // The create committed but the daemon died before arming the schema
+                // (it is written at launch, outside the create transaction): re-arm it
+                // so the revived root still owes a result rather than a plain final.
+                if (input.output_schema !== undefined && !root.outcome_at && await readOutputRequest(root.node_id) === null
+                    && await readOutputResult(root.node_id) === undefined) {
+                    await admitLegacyStructuredOutput(root.node_id, 'terminal', input.output_schema);
+                }
+                await reviveNode(root.node_id, { resume: true, capacity: 'freeze' });
+            }
+            return existing.response;
+        }
+    }
+    await ensureAppSpace(principal.grantee);
+    const profile = input.profile === undefined ? await ensureAppProfile(principal.grantee) : loadExactProfileManifest(input.profile);
+    const owner = profile.manifest.grantee ?? 'app:terminal';
+    if (owner !== principal.grantee) {
+        const permission = `crtr:memory:rw:${owner}:profile:${profile.profileId}`;
+        if (!principal.resolved.some((held) => covers(held, permission)) ||
+            (input.scopes !== null && !input.scopes.some((held) => covers(held, permission)))) {
+            throw new RunError(403, 'scope_missing', `missing scope ${permission}`, 'profile', undefined, permission);
+        }
+    }
+    // The spawn preflight only checks registration of a concrete id it can parse, and skips a logical `family/strength`.
+    // Refuse the two shapes it would let through: an id with no `provider/model` form, and a strength of a family no route declares.
+    if (input.model !== undefined) {
+        const model = input.model;
+        if (!isPortableModelToken(model)) {
+            if (parseModelSpec(model) === undefined)
+                throw invalid(`model '${model}' is not a strength (ultra, strong, medium, light), a family/strength, or a provider/model id`, 'model');
+        }
+        else if (model.includes('/') && await payerOfGrant(principal.grant) !== 'app') {
+            // An app-paid grant resolves any portable token through the app rung resolver, whatever family it names.
+            const slash = model.lastIndexOf('/');
+            const family = model.slice(0, slash);
+            const strength = model.slice(slash + 1);
+            if (['ultra', 'strong', 'medium', 'light'].includes(strength)
+                && (await expandModelCandidates({ kind: 'logical', family, strength: strength, spec: model }, appSpace(principal.grantee).files, profile.profileId)).every((candidate) => candidate.family !== family)) {
+                throw invalid(`model '${model}' names a model family that is not configured in this daemon`, 'model');
+            }
+        }
+    }
+    const runId = await newNodeId();
+    const rootFolder = join(appSpace(principal.grantee).nodes, runId);
+    if (input.isolated && input.cwd !== undefined) {
+        const beneath = relative(rootFolder, resolve(input.cwd));
+        if (!isAbsolute(input.cwd) || beneath === '..' || beneath.startsWith(`..${sep}`) || isAbsolute(beneath)) {
+            throw invalid('cwd must be inside the isolated root node folder', 'cwd');
+        }
+    }
+    // Fetched before the run exists (the idempotency body hash above is over the
+    // original request); the prompt the agent and the goal see carries runtime paths.
+    const attachments = await resolveAttachments(runId, principal.grantee, input.prompt);
+    const prompt = attachments.text;
+    // `message` is a `start_turn: false` message enqueued in the create
+    // transaction, so it precedes the broker's kickoff claim and always joins the
+    // first turn — a separate `run.message` after start can lose that race.
+    // The message's files take names the prompt's did not (no run row records the prompt's uses yet).
+    const firstMessage = input.message === undefined ? undefined : await resolveAttachments(runId, principal.grantee, input.message, attachments.uses.map((use) => use.path));
+    const payer = await payerOfGrant(principal.grant);
+    let response;
+    try {
+        const model = resolveRunModel(profile, input.model);
+        await spawnChildPrepared({ kind: resolveRunKind(profile), mode: 'base', root: true, rootLifecycle: 'resident', readiness: 'launch',
+            cwd: input.isolated ? input.cwd ?? rootFolder : appSpace(principal.grantee).files, profile: profile.profileId, prompt, nodeId: runId,
+            isolated: input.isolated ?? false,
+            ...(input.causedBy === undefined ? {} : { causedBy: input.causedBy }),
+            ...(model === undefined ? {} : { model }),
+            ...(input.deadline === undefined ? {} : { deadlineAt: new Date(Date.now() + parseRunDeadlineMs(input.deadline)).toISOString() }),
+            ...(input.scopes === null ? {} : { scopes: input.scopes }),
+            run: { grantee: principal.grantee, sub: principal.sub, grant_id: principal.grant, client_id: principal.client_id,
+                metadata_json: JSON.stringify(input.metadata), run_id: runId, event_cursor: 1, usage_json: JSON.stringify(usage()) },
+            withinCreate: async (tx, row) => {
+                // A concurrent start under this key may have committed while this one fetched attachments and awaited the
+                // spawn: re-check on the transaction before any write, so exactly one start creates the run.
+                if (idempotencyKey) {
+                    const winner = await findIdempotent(principal.grantee, personKey(principal.sub, idempotencyKey), bodyHash, tx);
+                    if (winner)
+                        throw new IdempotentReplay(winner);
+                }
+                // The first message's uses are inserted by acceptRunMessage below, so only the prompt's go here.
+                const uses = attachments.uses;
+                if (uses.length) {
+                    await assertUploadsUsable(principal.grantee, uses, tx);
+                    await insertUploadUses(uses.map(({ upload_id, path, blob_key }) => ({ upload_id, path, blob_key, run_id: runId })), tx);
+                }
+                response = { run_id: runId, node_id: runId, grantee: principal.grantee, grant: principal.grant, rev: principal.rev, scopes: input.scopes, payer, status: 'running', created_at: row.created, metadata: input.metadata, sequence_number: 0 };
+                await appendRunEvent(tx, runId, runId, 'run.started', { scopes: input.scopes, payer, usage: usage() });
+                if (firstMessage) {
+                    const accepted = await acceptRunMessage(tx, runId, principal.grantee, firstMessage, false);
+                    response.message = { run_id: runId, message_id: accepted.messageId, sequence_number: accepted.sequence };
+                }
+                if (idempotencyKey)
+                    await recordIdempotent(tx, principal.grantee, personKey(principal.sub, idempotencyKey), bodyHash, runId, response, row.created);
+            },
+        }, input.output_schema === undefined ? undefined : async (nodeId) => { await admitLegacyStructuredOutput(nodeId, 'terminal', input.output_schema); });
+    }
+    catch (error) {
+        // A concurrent start with this key won: this one rolled back. The winner, in this daemon, owns the launch
+        // (and the schema arming), so answer with its stored response and do not revive its row.
+        if (error instanceof IdempotentReplay)
+            return error.winner.response;
+        // The app listener speaks RunError envelopes; preflight may detect a
+        // concurrent ancestor change after the request's own validation.
+        // The spawn preflight refused the requested model (unknown, or not registered in this daemon) before any row existed.
+        if (input.model !== undefined && error instanceof CrtrError && error.code === 'usage')
+            throw invalid(error.message, 'model');
+        if (error instanceof ApiError && (error.code === 'isolated_run' || error.code === 'invalid_request')) {
+            const param = error.details && typeof error.details === 'object' && 'param' in error.details && typeof error.details.param === 'string'
+                ? error.details.param : undefined;
+            throw new RunError(error.status, error.code, error.message, param);
+        }
+        throw error;
+    }
+    if (!response)
+        throw new Error('run creation did not commit');
+    return response;
+}
+export async function listRuns(grantee, options) {
+    let after;
+    if (options.after) {
+        try {
+            const parsed = JSON.parse(Buffer.from(options.after, 'base64url').toString());
+            if (!parsed || typeof parsed !== 'object' || !('activity' in parsed) || typeof parsed.activity !== 'string' || !('id' in parsed) || typeof parsed.id !== 'string')
+                throw new Error('invalid cursor');
+            after = { activity: parsed.activity, id: parsed.id };
+        }
+        catch {
+            throw invalid('invalid list cursor', 'after');
+        }
+    }
+    const rows = await (await db()).prepareUncached(`SELECT node_id, COALESCE(last_activity_at, created) AS activity FROM node_rows
+    WHERE run_id = node_id AND deleting_at IS NULL ${grantee === null ? '' : 'AND grantee = ?'}${options.sub === undefined ? '' : ' AND sub = ?'}
+    ORDER BY activity DESC, node_id DESC`).all(...(grantee === null ? [] : [grantee]), ...(options.sub === undefined ? [] : [options.sub]));
+    const runs = (await filterAsync((await mapAsync((await filterAsync(rows, async ({ node_id }) => !options.visible || await options.visible(node_id))).filter(({ activity, node_id }) => !after || activity < after.activity || activity === after.activity && node_id < after.id), async (row) => ({ root: (await getNode(row.node_id)), activity: row.activity }))), async ({ root }) => !options.status || await runStatus(root) === options.status));
+    const page = runs.slice(0, options.limit);
+    const last = page.at(-1);
+    return { runs: (await mapAsync(page, async ({ root }) => await runObject(root))), next: runs.length > options.limit && last ? Buffer.from(JSON.stringify({ activity: last.activity, id: last.root.node_id })).toString('base64url') : null };
+}
+export async function waitRun(id, grantee, seconds) {
+    const initial = await ownedRun(id, grantee);
+    const read = async () => await runObject(await ownedRun(id, grantee));
+    if (!seconds || RESTING.includes(await runStatus(initial)))
+        return await read();
+    return new Promise((resolve, reject) => {
+        let unsubscribe = () => undefined;
+        const settle = () => { clearTimeout(timer); clearInterval(poll); unsubscribe(); };
+        const finish = async () => { settle(); try {
+            resolve(await read());
+        }
+        catch (err) {
+            reject(err);
+        } };
+        const check = async () => { if (RESTING.includes(String((await read()).status)))
+            await finish(); };
+        // A rejected check fails the wait; the timers and the subscription ignore their callbacks' results.
+        const run = () => { check().catch((err) => { settle(); reject(err); }); };
+        const poll = setInterval(run, 250);
+        const timer = setTimeout(() => { void finish(); }, seconds * 1000);
+        subscribeRunEvents(id, run).then((off) => { unsubscribe = off; }, (err) => { settle(); reject(err); });
+    });
+}
+/** Resolutions of one message's attachment names before `messageRun` gives up on concurrent messages taking them. */
+const MESSAGE_NAME_ATTEMPTS = 5;
+export async function messageRun(root, original, resolveAttachments = defaultResolver, startTurn = true, idempotencyKey, steer = true) {
+    const refused = root.status === 'canceled' || root.outcome_reason === 'closed';
+    const grantee = root.grantee ?? 'app:terminal';
+    // The hash covers the operation, the target run and the original text (storage URLs unexpanded).
+    const bodyHash = createHash('sha256').update(JSON.stringify(['run.message', root.node_id, original, startTurn, ...(startTurn && !steer ? ['steer:false'] : [])])).digest('hex');
+    // A replay answers before any attachment is fetched or the run is reopened.
+    const replay = async (response) => {
+        const current = await getNode(root.node_id);
+        // The accept committed but the daemon died before the revive: finish it, only while the message still waits.
+        const waiting = await (await db()).prepare("SELECT 1 FROM deliveries WHERE message_id = ? AND state = 'pending' AND canceled_at IS NULL").get(String(response['message_id']));
+        if (current && !refused && startTurn && !current.pi_pid && waiting)
+            await reviveNode(root.node_id, { resume: true, capacity: 'freeze' });
+        return response;
+    };
+    if (idempotencyKey) {
+        const existing = await findIdempotent(grantee, personKey(root.sub, idempotencyKey), bodyHash);
+        if (existing)
+            return replay(existing.response);
+    }
+    // A new send to a canceled or closed run is refused; a replay of one accepted earlier was answered above.
+    if (refused)
+        throw new RunError(409, 'run_settled', 'canceled run cannot receive a message');
+    const reopen = startTurn && root.final_report && !root.pi_pid ? (await import('../runtime/reopen.js')).commitReopenResident : undefined;
+    // Two messages to one run can each pick the same file name before either commits. The accepting transaction holds the
+    // run's lock, so it sees every name taken so far: when one of its names was taken meanwhile it rolls back and the text
+    // is resolved again (the blobs are content-keyed, so only the names change). Names are never shared, so a lost race
+    // costs a retry, not a failed request, unless it loses every time.
+    for (let attempt = 1;; attempt++) {
+        // Before the reopen: a refused URL never reopens a settled run.
+        const attachments = await resolveAttachments(root.node_id, grantee, original);
+        // One transaction: the key check, the reopen, the mail row, its event and the key record commit together,
+        // so a concurrent retry of the same key sees either all of it or none.
+        const outcome = await withCanvasWrite(async (tx) => {
+            if (idempotencyKey) {
+                const existing = await findIdempotent(grantee, personKey(root.sub, idempotencyKey), bodyHash, tx);
+                if (existing)
+                    return { replayed: existing.response };
+            }
+            if (attachments.uses.length) {
+                await lockRunRoot(tx, root.node_id);
+                for (const use of attachments.uses)
+                    if (await runUsesPath(root.node_id, use.path, tx))
+                        return 'path_taken';
+            }
+            await reopen?.(root.node_id);
+            const accepted = await acceptRunMessage(tx, root.node_id, grantee, attachments, startTurn, startTurn && steer);
+            const result = { run_id: root.node_id, message_id: accepted.messageId, sequence_number: accepted.sequence };
+            if (idempotencyKey)
+                await recordIdempotent(tx, grantee, personKey(root.sub, idempotencyKey), bodyHash, root.node_id, result, new Date().toISOString());
+            return { result };
+        });
+        if (outcome === 'path_taken') {
+            if (attempt >= MESSAGE_NAME_ATTEMPTS)
+                throw new RunError(503, 'service_unavailable', 'attachments of concurrent messages kept taking each other\'s file names; retry');
+            continue;
+        }
+        if ('replayed' in outcome)
+            return replay(outcome.replayed);
+        if (startTurn && !root.pi_pid) {
+            await reviveNode(root.node_id, { resume: true, capacity: 'freeze' });
+        }
+        return outcome.result;
+    }
+}
+async function interruptNode(id) {
+    const unavailable = () => new RunError(503, 'service_unavailable', `node ${id} broker was unavailable before the interrupt reached it`);
+    for (let attempt = 0; attempt < 2; attempt++) {
+        const current = await getNode(id);
+        if (!current)
+            throw missing();
+        if (!isBrokerLive(current)) {
+            if (current.streaming)
+                throw unavailable();
+            return;
+        }
+        const executionId = (await liveExecutionForNode(id))?.executionId;
+        try {
+            const interrupted = await interruptLive(id);
+            // A background command can call a provider while this node has no turn.
+            // Only a broker-acknowledged turn abort owns that provider request.
+            if (interrupted === 'turn')
+                abortProviderCalls({ nodes: [id] });
+            return;
+        }
+        catch (error) {
+            if (!(error instanceof BrokerUnreachableError))
+                throw error;
+            const latest = await getNode(id);
+            if (!latest)
+                throw missing();
+            if (!latest.streaming)
+                return;
+            if (attempt === 0 && isBrokerLive(latest) && (await liveExecutionForNode(id))?.executionId !== executionId)
+                continue;
+            throw unavailable();
+        }
+    }
+    throw unavailable();
+}
+export async function interruptRun(root, children = 'stop') {
+    const ids = children === 'stop'
+        ? (await (await db()).prepare('SELECT node_id FROM nodes WHERE run_id = ?').all(root.node_id)).map(({ node_id }) => node_id)
+        : [root.node_id];
+    await Promise.all(ids.map((id) => interruptNode(id)));
+    return await runObject(await getNode(root.node_id) ?? root);
+}
+export async function cancelRun(root, _options = {}) {
+    const ids = (await (await db()).prepare('SELECT node_id FROM nodes WHERE run_id = ? ORDER BY created DESC').all(root.node_id)).map((row) => row.node_id);
+    // Cancel the HTTP request before closing nodes, so a response cannot land
+    // while process teardown and run settlement are in progress.
+    abortProviderCalls({ run: root.node_id });
+    const closed = [];
+    if (await runStatus(root) !== 'settled')
+        closed.push(...(await closeNode(root.node_id, { preserveEmpty: true, stopReason: 'run_canceled' })).closed);
+    else {
+        for (const id of ids) {
+            if (id === root.node_id)
+                continue;
+            const current = await getNode(id);
+            if (current && current.status !== 'done' && current.status !== 'canceled' && current.status !== 'dead')
+                closed.push(...(await closeNode(id, { preserveEmpty: true, stopReason: 'run_canceled' })).closed);
+        }
+    }
+    // The run's open questions end with it (run.cancel: "cancels the run's open questions").
+    await cancelOpenQuestions({ run_id: root.node_id }, 'the run was canceled');
+    const teardowns = await Promise.all([...new Set([root.node_id, ...closed])].map((id) => stopNodeSandboxes(id, 'run_canceled')));
+    if (teardowns.some((result) => !result.confirmed))
+        throw new RunError(500, 'server_error', 'run process teardown could not be confirmed', undefined, undefined, undefined, true);
+    const current = await getNode(root.node_id);
+    return await runObject(current ?? root);
+}
+/** The longest run name `run.rename` accepts. */
+export const RUN_NAME_MAX = 200;
+/** `run.rename`: set the root node's name — the name `crtr` shows for the node —
+ *  through the same path as `PATCH /v1/nodes/:id/config` `name`. */
+export async function renameRun(root, name) {
+    if (typeof name !== 'string' || !name.trim())
+        throw invalid('name is required', 'name');
+    const trimmed = name.trim();
+    if (trimmed.length > RUN_NAME_MAX)
+        throw invalid(`name exceeds ${RUN_NAME_MAX} characters`, 'name');
+    const meta = await updateNode(root.node_id, { name: trimmed });
+    if (meta.window)
+        renameWindow(meta.window, fullName(meta));
+    return await runObject(meta);
+}
+/** Delete a run. Answers how many of the run's released uploads storage has
+ *  not confirmed deleted; the due-work tick retries those. */
+export async function deleteRun(root, storage = { config: storageConfig, client: storageClient }) {
+    // First, while the run still takes mail: the asking agent is told, and a
+    // canceled question stays canceled, never answerable.
+    await cancelOpenQuestions({ run_id: root.node_id }, 'the run was deleted');
+    const ids = await withCanvasWrite(async (tx) => {
+        await tx.prepare('UPDATE nodes SET deleting_at = COALESCE(deleting_at, ?) WHERE node_id = ?').run(new Date().toISOString(), root.node_id);
+        return (await tx.prepare('SELECT node_id FROM nodes WHERE run_id = ? ORDER BY created DESC').all(root.node_id)).map((r) => r.node_id);
+    });
+    const teardowns = await Promise.all(ids.map((id) => stopNodeSandboxes(id, 'run_deleted', { deleting: true })));
+    if (teardowns.some((r) => !r.confirmed))
+        throw new RunError(500, 'server_error', 'run process teardown could not be confirmed', undefined, undefined, undefined, true);
+    const files = (await mapAsync(ids, async (id) => {
+        const node = await nodeFiles(id);
+        if (node.app === null)
+            throw new Error(`run node missing during deletion: ${id}`);
+        return node;
+    }));
+    const sessions = await (await db()).prepare(`SELECT DISTINCT session_id FROM sessions WHERE session_id IS NOT NULL AND node_id IN (SELECT node_id FROM nodes WHERE run_id = ?)`).all(root.node_id);
+    for (const { session_id } of sessions) {
+        try {
+            await conversationStore().delete(session_id);
+        }
+        catch (error) {
+            if (!isConversationNotFound(error))
+                throw error;
+        }
+    }
+    const released = await withCanvasWrite(async (tx) => {
+        // Each node's object and what it owns are marked deleted, as any node deletion does
+        // (kept documents stay with their node); this drops the `nodes` row. First, so a
+        // `deleted` event delivered to another run node's inbox is removed with its mail below.
+        for (const id of ids) {
+            const node = await getById(tx, id);
+            if (node !== null && node.deleted_at === null)
+                await deleteNodeWithOwned(tx, node);
+        }
+        const orphaned = await releaseRunUploads([root.node_id], tx);
+        await tx.prepare('DELETE FROM run_events WHERE run_id = ?').run(root.node_id);
+        await tx.prepare('DELETE FROM idempotency_keys WHERE run_id = ?').run(root.node_id);
+        const list = ids.map(() => '?').join(',');
+        const mail = await tx.prepareUncached(`SELECT message_id FROM messages WHERE sender_node_id IN (${list}) UNION SELECT message_id FROM deliveries WHERE destination_id IN (${list})`).all(...ids, ...ids);
+        await tx.prepareUncached(`DELETE FROM deliveries WHERE destination_id IN (${list})`).run(...ids);
+        for (const { message_id } of mail)
+            await tx.prepare('DELETE FROM messages WHERE message_id = ? AND NOT EXISTS (SELECT 1 FROM deliveries WHERE message_id = ?)').run(message_id, message_id);
+        return orphaned;
+    });
+    for (const node of files)
+        await removeNodeFiles(node);
+    // Wake every stream; its next read observes the missing run.
+    const { notifyRunDeleted } = await import('./events.js');
+    notifyRunDeleted(root.node_id);
+    return { uploads_pending: await deleteReleasedUploads(released, storage) };
+}
+/** Try each released upload's storage delete in parallel under one budget; a
+ *  confirmed delete drops its row. Returns how many are still unconfirmed. */
+async function deleteReleasedUploads(uploadIds, storage) {
+    if (!uploadIds.length || !storage.config())
+        return uploadIds.length;
+    let client;
+    try {
+        client = storage.client();
+    }
+    catch {
+        return uploadIds.length;
+    }
+    const confirmed = new Set();
+    const attempts = uploadIds.map((id) => client.deleteUpload(id).then(async () => { await deleteUploadRow(id); confirmed.add(id); }, () => undefined));
+    let timer;
+    await Promise.race([Promise.all(attempts), new Promise((resolve) => { timer = setTimeout(resolve, storage.budgetMs ?? RUN_DELETE_STORAGE_BUDGET_MS); timer.unref?.(); })]);
+    clearTimeout(timer);
+    return uploadIds.length - confirmed.size;
+}
+export async function traceRun(id, grantee, format) {
+    await ownedRun(id, grantee);
+    await repairRunConversations(id);
+    // One short transaction captures the run's nodes, their session ids and the event cursor with its completions and
+    // deltas, so the trace is coherent with `latest_sequence`. The conversation files are read after it releases the
+    // write lock: file reads must not run inside it.
+    const snapshot = await withCanvasWrite(async (db) => {
+        const nodes = await db.prepare('SELECT node_id, parent FROM nodes WHERE run_id = ? ORDER BY CASE WHEN node_id = ? THEN 0 ELSE 1 END, created, node_id').all(id, id);
+        const sessionIds = new Map();
+        for (const node of nodes) {
+            sessionIds.set(node.node_id, (await db.prepare('SELECT session_id FROM sessions WHERE node_id = ? GROUP BY session_id ORDER BY min(created_at)').all(node.node_id)).map((row) => row.session_id));
+        }
+        const root = await db.prepare('SELECT event_cursor FROM nodes WHERE node_id = ?').get(id);
+        const latestSequence = root.event_cursor - 1;
+        const completed = await db.prepare(`SELECT node_id, sequence_number FROM run_events
+      WHERE run_id = ? AND sequence_number <= ? AND type IN ('node.thinking.done', 'node.output_text.done', 'usage')
+      ORDER BY sequence_number`).all(id, latestSequence);
+        const deltas = await db.prepare(`SELECT node_id, sequence_number, type, body_json FROM run_events
+      WHERE run_id = ? AND sequence_number <= ? AND type IN ('node.thinking.delta', 'node.output_text.delta')
+      ORDER BY sequence_number`).all(id, latestSequence);
+        return { nodes, sessionIds, latestSequence, completed, deltas };
+    });
+    const { nodes, sessionIds, latestSequence, completed, deltas } = snapshot;
+    const output = (await mapAsync(nodes, async (node) => ({ node_id: node.node_id, parent_node_id: node.parent, conversations: (await mapAsync(sessionIds.get(node.node_id) ?? [], async (session_id) => {
+            const { header, entries } = await conversationStore().read(session_id);
+            if (format === 'entries')
+                return { conversation_id: session_id, header, entries };
+            const manager = SessionManager.inMemory(header.cwd, undefined, [header, ...entries]);
+            const visible = cycleAwareMessages(manager) ?? [];
+            const positions = new Map(entries.map((entry, index) => [entry.id, index]));
+            const prompts = entries.filter((entry) => entry.type === 'custom' && entry.customType === 'crtr-system-prompt').map((entry) => ({ id: entry.id, message: { role: 'custom', customType: 'crtr-system-prompt', data: 'data' in entry ? entry.data : undefined, timestamp: entry.timestamp } }));
+            const appMessages = entries.filter((entry) => entry.type === 'custom_message' && entry.customType === 'app_message').map((entry) => ({ id: entry.id, message: { role: 'custom', customType: 'app_message', content: 'content' in entry ? entry.content : '', timestamp: entry.timestamp } }));
+            const rootPrompt = node.node_id === id ? entries.find((entry) => {
+                if (entry.type !== 'message' || entry.message.role !== 'user')
+                    return false;
+                const origin = originFor(entries, entry.message.timestamp);
+                return origin?.type === 'runtime' && origin.kind === 'kickoff';
+            })?.id : undefined;
+            const messages = [...visible.filter(({ id }) => !appMessages.some((entry) => entry.id === id)), ...prompts, ...appMessages].sort((a, b) => (positions.get(a.id) ?? Number.MAX_SAFE_INTEGER) - (positions.get(b.id) ?? Number.MAX_SAFE_INTEGER)).map(({ id: entry_id, message }) => {
+                const m = message;
+                const assistantUsage = m.usage;
+                const origin = m.role === 'user' ? originFor(entries, Number(m.timestamp)) : null;
+                const callerPrompt = origin?.type === 'human' || entry_id === rootPrompt;
+                const role = m.role === 'toolResult' ? 'tool_result' : m.role === 'bashExecution' ? 'bash_execution' : m.role === 'compactionSummary' ? 'compaction_summary' : m.role === 'branchSummary' ? 'branch_summary' : m.role === 'custom' && m.customType === 'crtr-system-prompt' ? 'system' : m.role === 'user' ? callerPrompt ? 'user' : 'custom' : m.role;
+                const customType = role === 'custom' ? m.role === 'custom' ? m.customType : origin?.type === 'runtime' ? origin.kind : 'unknown' : undefined;
+                const content = role === 'system' && m.customType === 'crtr-system-prompt' ? [{ type: 'text', text: m.data?.text ?? '' }] : typeof m.content === 'string' ? [{ type: 'text', text: m.content }] : Array.isArray(m.content) ? m.content.map((part) => part.type === 'toolCall' ? { type: 'tool_call', tool_call_id: part.id, name: part.name, arguments: part.arguments } : part.type === 'image' ? { type: 'image', mime_type: part.mimeType } : part.type === 'thinking' ? { type: 'thinking', text: part.thinking } : part) : [];
+                return { entry_id, role, ...(role === 'custom' ? { custom_type: customType } : {}), time: typeof m.timestamp === 'string' ? m.timestamp : new Date(Number(m.timestamp ?? Date.now())).toISOString(), content,
+                    ...(role === 'assistant' ? { model: m.model, provider: m.provider, stop_reason: m.stopReason,
+                        ...(m.errorMessage ? { error_message: m.errorMessage } : {}),
+                        ...(assistantUsage ? { usage: { input_tokens: assistantUsage.input ?? 0, output_tokens: assistantUsage.output ?? 0,
+                                cache_read_tokens: assistantUsage.cacheRead ?? 0, cache_write_tokens: assistantUsage.cacheWrite ?? 0,
+                                cost_usd: assistantUsage.cost?.total ?? 0 } } : {}) } : {}),
+                    ...(role === 'tool_result' ? { tool_call_id: m.toolCallId, tool_name: m.toolName, is_error: m.isError } : {}) };
+            }) ?? [];
+            return { conversation_id: session_id, messages };
+        })) })));
+    const starts = new Map(await Promise.all(output.map(async (node) => [node.node_id, await openAssistantMessageStartSequence(node.node_id)])));
+    for (const row of completed) {
+        const start = starts.get(row.node_id);
+        if (start !== null && start !== undefined && row.sequence_number >= start)
+            starts.set(row.node_id, null);
+    }
+    const progress = new Map();
+    for (const row of deltas) {
+        const start = starts.get(row.node_id);
+        if (start === null || start === undefined || row.sequence_number < start)
+            continue;
+        const event = JSON.parse(row.body_json);
+        const current = progress.get(row.node_id) ?? { thinking: '', text: '' };
+        if (row.type === 'node.thinking.delta')
+            current.thinking += event.delta;
+        else
+            current.text += event.delta;
+        progress.set(row.node_id, current);
+    }
+    const tracedNodes = output.map((node) => ({ ...node,
+        in_progress_message: starts.get(node.node_id) == null ? null : progress.get(node.node_id) ?? { thinking: '', text: '' },
+    }));
+    return { run_id: id, latest_sequence: latestSequence, nodes: tracedNodes };
+}
+export function requestedScopes(held, value, grantee) {
+    if (value === undefined) {
+        if (!held.some((scope) => covers(scope, 'crtr:llm')))
+            throw new RunError(403, 'scope_missing', 'missing scope crtr:llm', undefined, undefined, 'crtr:llm');
+        return null;
+    }
+    if (!Array.isArray(value) || value.some((scope) => typeof scope !== 'string'))
+        throw invalid('scopes must be a list of scope strings', 'scopes');
+    for (const scope of value) {
+        try {
+            parseScope(scope);
+        }
+        catch {
+            throw invalid(`invalid scope ${scope}`, 'scopes');
+        }
+        if (/^crtr:files:(?:read|rw):app:/.test(scope) && scope.split(':')[4] === grantee.slice(4)) {
+            throw invalid(`own-app files cannot narrow a run: ${scope}`, 'scopes');
+        }
+        const ownSpace = /^crtr:memory:(?:read|rw):app:/.test(scope) && scope.split(':')[4] === grantee.slice(4);
+        if (!ownSpace && !coveredByAny(held, scope)) {
+            throw new RunError(403, 'scope_missing', `missing scope ${scope}`, undefined, undefined, scope);
+        }
+    }
+    if (!value.includes('crtr:llm'))
+        throw new RunError(403, 'scope_missing', 'missing scope crtr:llm', undefined, undefined, 'crtr:llm');
+    return value;
+}