@nocobase/app-plugin-agents 0.0.0-stage → 0.1.0-beta.2

package/dist/server/core/runs/claim.js

@@ -0,0 +1,862 @@
+/**
+ * Claiming: handing queued runs to a runner that fits them, within each agent's concurrency limit, one run per
+ * (agent, subject, thread) at a time, through the runners' one long-poll (`routes/runners/runner.ts`), which hands
+ * out jobs from the same slots. A runner fits a run when it has the agent's
+ * tool enabled (on the web), installed and signed in, the agent names no runner or names it, it is shared with the
+ * team or belongs to the person who woke the agent, its features cover what the run requires (worked out from the
+ * payload: checkouts, directories, skills, secrets), its owner's local policy (`Runner.policy`) lets it take the
+ * agent's runs on the run's subject and repositories, and it has a free slot. A working directory that is a directory
+ * on one runner pins the run to that runner, one run at a time per directory. A run's variables go to the runner that
+ * takes it, personal ones included, unless one of them is for team runners only (`teamRunnersOnly`): then only a team
+ * runner takes the run, and the variables that asked for one are noted on it (`teamOnlyVariables`), so its wait says
+ * which.
+ *
+ * A slot is free for a run when the runner's total has room and so does the coding tool it runs with: the tool's limit
+ * (`Runner.toolSlots`, never above the total) over the runs of that tool the runner holds here, and what the runner
+ * said it can still take of that tool across every application (`ClaimRequest.tools`). The run takes the first of its
+ * agent's entries whose tool the runner runs and has room for, so an agent listing two tools runs with the second while
+ * the first is full; a run none of whose tools has room is passed over for the runs behind it.
+ *
+ * Variables are checked and opened in the claim's own transaction, on its connection: which ones the assembled payload
+ * gets, whether any is for team runners only, and their values, so what the runner receives is what was checked.
+ *
+ * A claim is one transaction per run. It first writes the agent's row (`lockAgentForClaim`): on databases with row
+ * locks, concurrent claims for the same agent wait there, so the concurrency count read next is current. The run is
+ * then taken with a guarded update (`status = 'queued'`), so two runners can never take the same run. The payload is
+ * assembled in the same transaction; if assembly fails the transaction rolls back, the run stays queued, and the
+ * failure is counted (`claimFailures`) until the run fails `setupFailed`.
+ *
+ * Portable by design: the database layer offers no `FOR UPDATE SKIP LOCKED` or advisory locks, and the guarded update
+ * plus the agent-row write give the same guarantees on every dialect (SQLite serializes writers anyway).
+ *
+ * Runners take only runner agents' runs. Online agents' runs are taken by the application itself (`claimServer`, for the
+ * server executor of each instance), through the same guarded update, so two instances never take the same run: the
+ * holder is the instance (`server:<instance>`), there is no per-agent or per-holder limit, and the payload is a brief,
+ * the run's token and its skills (read in the run's sandbox, `online/sandbox.ts`), without directories, variables or
+ * mounts.
+ */
+import { policyAllowsAgent, policyAllowsRepo, policyAllowsSubject, TIMINGS, } from '@nocobase/agent-protocol';
+import { runsTool, toolLimit } from '../../../shared/runners.js';
+import { createHash } from 'node:crypto';
+import { onlineEntriesOrDefault, runnerEntries, sameEntry, } from '../../../shared/agents.js';
+import { later } from '../../kernel/clock.js';
+import { notFound } from '../../kernel/errors.js';
+import { runSecretsKey } from '../../kernel/redaction.js';
+import { cleanList, covers } from '../../kernel/values.js';
+import { findAgent, lockAgentForClaim } from '../agents/index.js';
+import { consultSection, } from '../consultations/index.js';
+import { joinBrief, onlineSkillsSection, renderBrief, } from '../brief/index.js';
+import { cliPackageFor } from './cli-package.js';
+import { describeVariables, variableRefs, } from '../variables/index.js';
+import { mountsFor, prepareExtensions, repoAccessFor, sessionKeyOf, withSections, } from './extensions.js';
+import { dialectOf, } from './ports.js';
+import { MAX_CLAIM_FAILURES, toolPolicyFor } from './policy.js';
+import { mintRunToken } from './run-tokens.js';
+import { runsHeldByTool } from './runner-view.js';
+import { ACTIVE, countActive, eventsRepo, findRunRecord, markDelivered, pendingInputs, runsOfKey, runsRepo, sessionsRepo, toInput, toRun, } from './run.store.js';
+import { finishRun } from './transitions.js';
+import { consumeReset, saveBrief } from './workspace.store.js';
+/** Queued runs looked at per claim, oldest first by priority. */
+const CANDIDATES = 50;
+/** Whether a run holder id is an application instance's (an online run's), not a runner's. */
+export function isServerHolder(runnerId) {
+    return runnerId !== null && runnerId.startsWith('server:');
+}
+class AssemblyError extends Error {
+    cause;
+    constructor(message, cause) {
+        super(message);
+        this.cause = cause;
+        this.name = 'AssemblyError';
+    }
+}
+/** The run does not fit this runner after all (its payload needs another runner, or a busy directory). */
+class NotForThisRunner extends Error {
+    constructor() {
+        super('The run does not fit this runner.');
+        this.name = 'NotForThisRunner';
+    }
+}
+/** The run gets variables only team runners may receive, and this runner is a personal one. */
+class TeamRunnersOnly extends NotForThisRunner {
+    variables;
+    constructor(variables) {
+        super();
+        this.variables = variables;
+        this.name = 'TeamRunnersOnly';
+    }
+}
+function toolRoom(tools) {
+    const left = new Map(Object.entries(tools ?? {}));
+    return {
+        has: (tool) => (left.get(tool) ?? Number.POSITIVE_INFINITY) > 0,
+        take: (tool) => {
+            const count = left.get(tool);
+            if (count !== undefined)
+                left.set(tool, count - 1);
+        },
+    };
+}
+/**
+ * The entry `runner` takes a run of `agent` with: the first whose tool it has enabled, installed and signed in. Null
+ * when it has none of them, and always for an online agent, which has no tool.
+ */
+export function pickEntry(runner, agent) {
+    return (runnerEntries(agent).find((entry) => runsTool(runner, entry.tool)) ?? null);
+}
+/**
+ * The entry `runner` takes a run of `agent` with when some of its tools are full: the first whose tool it runs and
+ * `open` says has room. Null when none has.
+ */
+export function pickOpenEntry(runner, agent, open) {
+    return (runnerEntries(agent).find((entry) => runsTool(runner, entry.tool) && open(entry.tool)) ?? null);
+}
+/** Whether `runner` has one of `agent`'s tools enabled, installed and signed in; never for an online agent. */
+export function hasTool(runner, agent) {
+    return pickEntry(runner, agent) !== null;
+}
+/**
+ * The entry an online run uses: the one its subject asks for while the agent lists it, else the agent's first; for an
+ * agent that lists none, the system default chat model (`fallback`).
+ */
+export function onlineEntryOf(agent, wanted, fallback = null) {
+    const entries = onlineEntriesOrDefault(agent, fallback);
+    return ((wanted && entries.find((entry) => sameEntry(entry, wanted))) ??
+        entries[0] ??
+        null);
+}
+/**
+ * Whether `runner` may run work of `agent` as `actorUserId` that needs `requires`, before any lock is taken and before
+ * a payload is known: the agent's tool signed in there, its owner's policy, the agent's named runners, and trust.
+ */
+export function fitsActor(runner, agent, actorUserId, requires) {
+    if (agent.archivedAt || agent.type !== 'runner')
+        return false;
+    if (!covers(runner.features, requires))
+        return false;
+    if (!hasTool(runner, agent))
+        return false;
+    if (!policyAllowsAgent(runner.policy, agent))
+        return false;
+    if (agent.runnerIds.length > 0 && !agent.runnerIds.includes(runner.id))
+        return false;
+    return runner.trust === 'team' || runner.ownerUserId === actorUserId;
+}
+/** Whether `runner` may run `run` of `agent`, before any lock is taken and before its payload is known. */
+export function fits(runner, agent, run) {
+    return fitsActor(runner, agent, run.actorUserId, toRun(run).requires);
+}
+/** The runner features a payload needs. */
+export function payloadRequires(dirs, skills, variables) {
+    return [
+        ...(dirs.some((dir) => dir.kind === 'repo') ? ['checkout'] : []),
+        ...(dirs.some((dir) => dir.kind === 'directory')
+            ? ['directories']
+            : []),
+        ...(skills > 0 ? ['skills'] : []),
+        ...(variables > 0 ? ['secrets'] : []),
+    ];
+}
+/** The working directories as a runner receives them: without what only the server uses. */
+export function payloadDirs(dirs) {
+    return dirs.map((dir) => {
+        const { scopeId: _scope, runnerId: _runner, ...rest } = dir;
+        return rest;
+    });
+}
+/** Where a run's variables come from, in merge order: the subject's scopes, its directories', the agent's. */
+export function variableTargets(agent, assembly) {
+    return [
+        ...assembly.scopes,
+        ...assembly.dirs.flatMap((dir) => dir.scopeId ? [{ scope: 'workdir', scopeId: dir.scopeId }] : []),
+        { scope: 'agent', scopeId: agent.id },
+    ];
+}
+/** Where a run's skills come from, the agent's first so its own win a shared slug. */
+export function skillTargets(agent, assembly) {
+    return [
+        { scope: 'agent', scopeId: agent.id },
+        ...assembly.scopes,
+        ...assembly.dirs.flatMap((dir) => dir.scopeId ? [{ scope: 'workdir', scopeId: dir.scopeId }] : []),
+    ];
+}
+function fingerprint(agent, entry, layers, skills) {
+    return createHash('sha256')
+        .update(JSON.stringify([
+        entry.tool,
+        entry.model,
+        agent.instructions,
+        layers.system,
+        layers.agent,
+        skills.map((skill) => [skill.slug, skill.hash]),
+    ]))
+        .digest('hex');
+}
+/** The brief for `assembly`, with the skills a run would get. */
+export function briefOf(agent, assembly, skills, cli, appName) {
+    const rendered = renderBrief({
+        dialect: dialectOf(agent),
+        agentName: agent.name,
+        subject: { noun: assembly.subject.noun, key: assembly.subject.key },
+        task: assembly.task,
+        context: assembly.context,
+        instructions: agent.instructions,
+        dirs: payloadDirs(assembly.dirs),
+        skills,
+        cli,
+        appName,
+        ...(assembly.guidance ? { guidance: assembly.guidance } : {}),
+    });
+    const layers = assembly.system === undefined
+        ? rendered
+        : { ...rendered, system: assembly.system };
+    const turn = assembly.turn.previousSummary
+        ? `${assembly.turn.prompt}\n\nWhat earlier runs on this already did:\n\n${assembly.turn.previousSummary}`
+        : assembly.turn.prompt;
+    return { layers, turn, prompt: joinBrief(layers) };
+}
+const DIRECTORY_KEY_SEPARATOR = '\n';
+/** The directories a run holds on its runner, as stored in `agRuns.directoryKey`. */
+function directoryKeyOf(runnerId, dirs) {
+    const paths = dirs
+        .filter((dir) => dir.kind === 'directory')
+        .map((dir) => `${runnerId}:${dir.path}`);
+    return paths.length > 0 ? paths.join(DIRECTORY_KEY_SEPARATOR) : null;
+}
+/** Whether another run held by the runner works in any of `key`'s directories. */
+async function directoryBusy(conn, runnerId, runId, key) {
+    const wanted = new Set(key.split(DIRECTORY_KEY_SEPARATOR));
+    const held = await runsRepo(conn).findMany({
+        filter: (f) => f.and([
+            f.string('runnerId').eq(runnerId),
+            f.string('id').ne(runId),
+            f.or(ACTIVE.map((status) => f.string('status').eq(status))),
+        ]),
+    });
+    return held.some((run) => (run.directoryKey ?? '')
+        .split(DIRECTORY_KEY_SEPARATOR)
+        .some((part) => part !== '' && wanted.has(part)));
+}
+export function createClaimService(deps) {
+    const { tx, clock } = deps;
+    async function assemble({ conn }, run, agent, runner, inputs, leaseExpiresAt, prepared, entry) {
+        const binding = deps.subjects.get(run.subjectKind);
+        if (!binding)
+            throw new Error(`No context provider is registered for subjects of kind "${run.subjectKind}".`);
+        const view = toRun(run);
+        const claim = {
+            run: view,
+            agent,
+            runner,
+            inputs,
+            cli: deps.cli.name,
+            appName: deps.app.name,
+            dialect: 'cli',
+        };
+        const assembly = await binding.context.assemble(conn, claim);
+        // Its owner's local policy keeps this runner off the subject, or off one of the repositories: another runner takes
+        // the run.
+        if (!policyAllowsSubject(runner.policy, assembly.subject.key) ||
+            assembly.dirs.some((dir) => dir.kind === 'repo' && !policyAllowsRepo(runner.policy, dir.url)))
+            throw new NotForThisRunner();
+        // A directory on another runner: that runner takes the run.
+        if (assembly.dirs.some((dir) => dir.kind === 'directory' && dir.runnerId !== runner.id))
+            throw new NotForThisRunner();
+        const dirs = payloadDirs(assembly.dirs);
+        const directoryKey = directoryKeyOf(runner.id, dirs);
+        if (directoryKey !== null &&
+            (await directoryBusy(conn, runner.id, run.id, directoryKey)))
+            throw new NotForThisRunner();
+        const skills = await deps.skills.forRun(conn, skillTargets(agent, assembly), run.id);
+        const variables = variableTargets(agent, assembly);
+        const names = await deps.variables.namesOf(conn, variables);
+        const requires = cleanList([
+            ...view.requires,
+            ...payloadRequires(dirs, skills.length, names.length),
+        ]);
+        if (!covers(runner.features, requires))
+            throw new NotForThisRunner();
+        // A variable for team runners only keeps a personal runner off the run.
+        if (names.length > 0 && runner.trust !== 'team') {
+            const teamOnly = await deps.variables.teamOnly(conn, variables);
+            if (teamOnly.length > 0)
+                throw new TeamRunnersOnly(teamOnly);
+        }
+        const env = names.length > 0
+            ? await deps.variables.forRun(conn, variables, {
+                runId: run.id,
+                runnerId: runner.id,
+            })
+            : [];
+        const nowText = clock.now().toISOString();
+        const clean = await consumeReset(conn, { kind: run.subjectKind, id: run.subjectId }, run.id, nowText);
+        const session = await sessionsRepo(conn).findOne({
+            filter: {
+                agentId: run.agentId,
+                runnerId: runner.id,
+                subjectKind: run.subjectKind,
+                subjectId: run.subjectId,
+                threadScope: run.threadScope,
+            },
+        });
+        const token = await mintRunToken(conn, deps, run, runner.id);
+        const git = await repoAccessFor(conn, deps.repoAccess, {
+            claim,
+            repos: dirs.filter((dir) => dir.kind === 'repo'),
+        }, prepared);
+        deps.secrets?.remember(runSecretsKey(run.id), [
+            ...env.map((variable) => variable.value),
+            token.value,
+            ...(git?.credentials ?? []).map((credential) => credential.password),
+        ]);
+        const last = await eventsRepo(conn).findOne({
+            filter: { runId: run.id },
+            sort: (sort) => sort.field('seq').desc(),
+        });
+        const brief = briefOf(agent, await withSections(conn, deps.sections, claim, assembly, prepared), skills, deps.cli.name, deps.app.name);
+        const briefFingerprint = fingerprint(agent, entry, brief.layers, skills);
+        const resume = !clean &&
+            session &&
+            !session.poisoned &&
+            session.fingerprint === briefFingerprint;
+        // Mounts only for a runner that places them; their hashes stay out of the fingerprint, and a resumed session
+        // hears what changed in them in its turn instead.
+        const { mounts, resumeNotes } = runner.features.includes('mounts')
+            ? await mountsFor(conn, deps.mounts, {
+                claim,
+                assembly,
+                session: resume ? 'resume' : 'fresh',
+                sessionKey: sessionKeyOf(view),
+            }, prepared)
+            : { mounts: [], resumeNotes: [] };
+        const turn = [brief.turn, ...resumeNotes].join('\n\n');
+        await saveBrief(conn, () => deps.ids.next(), {
+            runId: run.id,
+            attempt: Number(run.attempt),
+            layers: brief.layers,
+            turn,
+            prompt: brief.prompt,
+            createdAt: nowText,
+        });
+        const payload = {
+            run: {
+                id: run.id,
+                attempt: Number(run.attempt),
+                maxAttempts: Number(run.maxAttempts),
+                priority: Number(run.priority),
+                createdAt: run.createdAt,
+                leaseExpiresAt,
+                requires,
+                firstSeq: (last ? Number(last.seq) : 0) + 1,
+            },
+            app: deps.app,
+            subject: {
+                key: assembly.subject.key,
+                ...(assembly.subject.title ? { title: assembly.subject.title } : {}),
+                url: assembly.subject.url,
+            },
+            tool: {
+                kind: entry.tool,
+                ...(entry.model ? { model: entry.model } : {}),
+                ...(entry.effort ? { effort: entry.effort } : {}),
+                policy: toolPolicyFor(agent.toolPolicy),
+            },
+            prompt: {
+                system: brief.prompt,
+                turn,
+                session: resume ? 'resume' : 'fresh',
+                ...(resume ? { resumeSessionId: session.sessionId } : {}),
+            },
+            inputs,
+            workspace: {
+                dirs,
+                env,
+                ...(clean ? { clean: true } : {}),
+                ...(git ? { git } : {}),
+            },
+            skills,
+            ...(mounts.length > 0 ? { mounts } : {}),
+            cli: {
+                name: deps.cli.name,
+                package: await cliPackageFor(deps.cli.name, deps.cli.package, deps.dist, runner),
+                credential: {
+                    file: deps.cli.credentialFile,
+                    content: {
+                        token: token.value,
+                        runId: run.id,
+                        manifestUrl: deps.manifestUrl,
+                        expiresAt: token.expiresAt,
+                    },
+                },
+            },
+            agent: { id: agent.id, name: agent.name },
+        };
+        return {
+            payload,
+            fingerprint: briefFingerprint,
+            requires,
+            directoryKey,
+            entry,
+        };
+    }
+    /**
+     * Notes on a queued run which of its variables kept a personal runner off it, and tells its actor and owner the first
+     * time; nothing when that is what it says already.
+     */
+    async function noteTeamOnly(candidate, variables) {
+        const same = (value) => JSON.stringify(variableRefs(value)) === JSON.stringify(variables);
+        if (same(candidate.teamOnlyVariables))
+            return;
+        await tx.run(async (unit) => {
+            const run = await findRunRecord(unit.conn, candidate.id);
+            if (!run || run.status !== 'queued' || same(run.teamOnlyVariables))
+                return;
+            await runsRepo(unit.conn).updateMany({
+                filter: (f) => f.and([f.string('id').eq(run.id), f.string('status').eq('queued')]),
+                values: { teamOnlyVariables: variables.map((each) => ({ ...each })) },
+            });
+            unit.emit({ type: 'run.changed', runId: run.id, status: 'queued' });
+            if (variableRefs(run.teamOnlyVariables).length === 0)
+                unit.emit({
+                    type: 'notice',
+                    notice: {
+                        key: `run-secrets:${run.id}`,
+                        type: 'run_secrets_not_allowed',
+                        userIds: [
+                            ...new Set([run.actorUserId, run.ownerUserId].filter((id) => Boolean(id))),
+                        ],
+                        subject: { kind: 'run', id: run.id, label: run.id },
+                        title: 'A run waits for a team runtime',
+                        body: `Some of its variables are for team runtimes only: ${describeVariables(variables)}.`,
+                        params: {
+                            runId: run.id,
+                            agentId: run.agentId,
+                            variables: describeVariables(variables),
+                        },
+                    },
+                });
+        });
+    }
+    async function attempt(runner, agent, candidate, reported) {
+        // Avoid assembling or asking authorization when a cheap claim rule already rules this work out.
+        const conn = tx.read();
+        if (!fits(runner, agent, candidate) ||
+            (await deps.slotsUsed(conn, runner.id)) >= runner.slots ||
+            (await countActive(conn, 'agentId', agent.id)) >=
+                agent.maxConcurrentRuns ||
+            (await runsOfKey(conn, candidate, ACTIVE)).length > 0)
+            return { kind: 'skipped' };
+        // Before the transaction: on SQLite it holds the only connection, so the providers read what else they need now.
+        const prepared = await prepareExtensions(toRun(candidate), deps.sections, deps.mounts, (error) => deps.onClaimFailure?.(candidate.id, error), deps.repoAccess);
+        let taken;
+        try {
+            taken = await tx.run(async (unit) => {
+                const { conn } = unit;
+                const now = clock.now();
+                const nowText = now.toISOString();
+                await lockAgentForClaim(conn, agent.id, nowText);
+                if ((await countActive(conn, 'agentId', agent.id)) >=
+                    agent.maxConcurrentRuns)
+                    return null;
+                // Slots are shared with the jobs the runner holds.
+                if ((await deps.slotsUsed(conn, runner.id)) >= runner.slots)
+                    return null;
+                // The first of the agent's tools with room, by its limit here and by what the runner said it can take.
+                const byTool = await runsHeldByTool(conn, runner.id);
+                const entry = pickOpenEntry(runner, agent, (tool) => (byTool[tool] ?? 0) < toolLimit(runner, tool) && reported.has(tool));
+                if (!entry)
+                    return null;
+                const busy = await runsOfKey(conn, candidate, ACTIVE);
+                if (busy.length > 0)
+                    return null;
+                const leaseExpiresAt = later(now, TIMINGS.leaseMs);
+                const claimed = await runsRepo(conn).updateMany({
+                    filter: (f) => f.and([
+                        f.string('id').eq(candidate.id),
+                        f.string('status').eq('queued'),
+                    ]),
+                    values: {
+                        status: 'dispatched',
+                        runnerId: runner.id,
+                        leaseExpiresAt,
+                        dispatchedAt: nowText,
+                        lastActivityAt: nowText,
+                        availableAt: null,
+                        updatedAt: nowText,
+                    },
+                });
+                if (claimed.updatedCount !== 1)
+                    return null;
+                const run = (await findRunRecord(conn, candidate.id));
+                const pending = await pendingInputs(conn, run.id);
+                let assembled;
+                try {
+                    assembled = await assemble(unit, run, agent, runner, pending.map(toInput), leaseExpiresAt, prepared, entry);
+                }
+                catch (error) {
+                    if (error instanceof NotForThisRunner)
+                        throw error;
+                    throw new AssemblyError(error instanceof Error ? error.message : String(error), error);
+                }
+                await markDelivered(conn, pending, nowText);
+                await runsRepo(conn).updateMany({
+                    filter: { id: run.id },
+                    values: {
+                        payloadFingerprint: assembled.fingerprint,
+                        requires: [...assembled.requires],
+                        directoryKey: assembled.directoryKey,
+                        tool: assembled.entry.tool,
+                        modelService: null,
+                        model: assembled.entry.model,
+                        effort: assembled.entry.effort ?? null,
+                        teamOnlyVariables: null,
+                    },
+                });
+                unit.emit({ type: 'run.changed', runId: run.id, status: 'dispatched' });
+                if (variableRefs(candidate.teamOnlyVariables).length > 0)
+                    unit.emit({
+                        type: 'notice.cleared',
+                        notice: {
+                            type: 'run_secrets_not_allowed',
+                            subject: { kind: 'run', id: run.id, label: run.id },
+                        },
+                    });
+                return assembled.payload;
+            });
+        }
+        catch (error) {
+            await discardRepoAccess(candidate, prepared);
+            if (error instanceof TeamRunnersOnly) {
+                await noteTeamOnly(candidate, error.variables);
+                return { kind: 'skipped' };
+            }
+            throw error;
+        }
+        if (!taken) {
+            await discardRepoAccess(candidate, prepared);
+            return { kind: 'skipped' };
+        }
+        return { kind: 'claimed', payload: taken };
+    }
+    async function discardRepoAccess(run, prepared) {
+        for (const provider of deps.repoAccess?.list() ?? []) {
+            try {
+                await provider.discard?.(toRun(run), prepared.get(`repo:${provider.key}`));
+            }
+            catch (error) {
+                deps.onClaimFailure?.(run.id, error);
+            }
+        }
+    }
+    /** Counts a failed assembly; the run fails once it has failed too often. */
+    async function recordFailure(runId, error) {
+        deps.onClaimFailure?.(runId, error.cause);
+        await tx.run(async (unit) => {
+            const run = await findRunRecord(unit.conn, runId);
+            if (!run || run.status !== 'queued')
+                return;
+            const failures = Number(run.claimFailures) + 1;
+            if (failures >= MAX_CLAIM_FAILURES) {
+                await finishRun(unit, deps, run, {
+                    status: 'failed',
+                    reason: 'setupFailed',
+                    detail: error.message,
+                });
+                return;
+            }
+            await runsRepo(unit.conn).updateMany({
+                filter: { id: runId },
+                values: {
+                    claimFailures: failures,
+                    failureDetail: error.message.slice(0, 10_000),
+                },
+            });
+        });
+    }
+    /** Queued runs of agents of `type` that may be claimed now, in claim order. */
+    async function candidatesOf(type) {
+        const now = clock.now();
+        return runsRepo(tx.read()).findMany({
+            filter: (f) => f.and([
+                f.string('status').eq('queued'),
+                f.string('agentType').eq(type),
+                // A consultation is taken by the holder of the run that asked it, never from the queue.
+                f.string('parentRunId').empty(),
+                f.or([
+                    f.date('availableAt').empty(),
+                    f.date('availableAt').notAfter(now),
+                ]),
+            ]),
+            sort: (sort) => [
+                sort.field('priority').asc(),
+                sort.field('createdAt').asc(),
+                sort.field('id').asc(),
+            ],
+            limit: CANDIDATES,
+        });
+    }
+    /** Takes an online run for `holder` in one transaction, or skips it when another holder or run got there first. */
+    async function attemptServer(holder, agent, candidate) {
+        const prepared = await prepareExtensions(toRun(candidate), deps.sections, undefined, (error) => deps.onClaimFailure?.(candidate.id, error));
+        return tx.run(async (unit) => {
+            const { conn } = unit;
+            const now = clock.now();
+            const nowText = now.toISOString();
+            // Serializes claims and enqueues of this agent, so one key never has two runs held.
+            await lockAgentForClaim(conn, agent.id, nowText);
+            if ((await runsOfKey(conn, candidate, ACTIVE)).length > 0)
+                return null;
+            const leaseExpiresAt = later(now, TIMINGS.leaseMs);
+            const taken = await runsRepo(conn).updateMany({
+                filter: (f) => f.and([
+                    f.string('id').eq(candidate.id),
+                    f.string('status').eq('queued'),
+                ]),
+                values: {
+                    status: 'dispatched',
+                    runnerId: holder.id,
+                    leaseExpiresAt,
+                    dispatchedAt: nowText,
+                    lastActivityAt: nowText,
+                    availableAt: null,
+                    updatedAt: nowText,
+                },
+            });
+            if (taken.updatedCount !== 1)
+                return null;
+            const run = (await findRunRecord(conn, candidate.id));
+            const pending = await pendingInputs(conn, run.id);
+            const inputs = pending.map(toInput);
+            let claimed;
+            try {
+                claimed = await assembleServer(unit, run, agent, holder, inputs, leaseExpiresAt, prepared);
+            }
+            catch (error) {
+                throw new AssemblyError(error instanceof Error ? error.message : String(error), error);
+            }
+            await markDelivered(conn, pending, nowText);
+            await runsRepo(conn).updateMany({
+                filter: { id: run.id },
+                values: {
+                    tool: null,
+                    modelService: claimed.model?.modelService ?? null,
+                    model: claimed.model?.model ?? null,
+                    effort: claimed.model?.effort ?? null,
+                },
+            });
+            unit.emit({ type: 'run.changed', runId: run.id, status: 'dispatched' });
+            return {
+                ...claimed,
+                run: {
+                    ...claimed.run,
+                    tool: null,
+                    modelService: claimed.model?.modelService ?? null,
+                    model: claimed.model?.model ?? null,
+                    effort: claimed.model?.effort ?? null,
+                },
+            };
+        });
+    }
+    async function assembleServer({ conn }, run, agent, holder, inputs, leaseExpiresAt, prepared) {
+        const binding = deps.subjects.get(run.subjectKind);
+        if (!binding)
+            throw new Error(`No context provider is registered for subjects of kind "${run.subjectKind}".`);
+        const view = toRun(run);
+        const claim = {
+            run: view,
+            agent,
+            runner: null,
+            inputs,
+            cli: deps.cli.name,
+            appName: deps.app.name,
+            dialect: 'tools',
+        };
+        const assembly = await binding.context.assemble(conn, claim);
+        if (assembly.dirs.length > 0)
+            throw new Error('An online agent has no working directory, and this work needs one.');
+        const skills = await deps.skills.forRun(conn, skillTargets(agent, assembly), run.id);
+        const own = new Set(skills.map((skill) => skill.slug));
+        const rendered = briefOf(agent, await withSections(conn, deps.sections, claim, assembly, prepared), [], deps.cli.name, deps.app.name);
+        // The skills an online run reads through its tools, and the agents it may consult, whatever system layer its
+        // subject gives it.
+        const consultable = (await deps.consultations?.targets(conn, run, agent)) ?? [];
+        const depth = run.parentRunId
+            ? ((await deps.consultations?.chain(conn, run))?.length ?? 1) - 1
+            : 0;
+        const section = [
+            ...onlineSkillsSection([
+                ...skills,
+                ...(deps.onlineSkills?.() ?? []).filter((skill) => !own.has(skill.slug)),
+            ]),
+            ...(consultable.length > 0 ? ['', ...consultSection(consultable)] : []),
+        ];
+        const layers = section.length > 0
+            ? {
+                ...rendered.layers,
+                system: [rendered.layers.system, '', ...section].join('\n'),
+            }
+            : rendered.layers;
+        const brief = { ...rendered, layers, prompt: joinBrief(layers) };
+        const token = await mintRunToken(conn, deps, run, holder.id);
+        deps.secrets?.remember(runSecretsKey(run.id), [token.value]);
+        const last = await eventsRepo(conn).findOne({
+            filter: { runId: run.id },
+            sort: (sort) => sort.field('seq').desc(),
+        });
+        await saveBrief(conn, () => deps.ids.next(), {
+            runId: run.id,
+            attempt: Number(run.attempt),
+            layers: brief.layers,
+            turn: brief.turn,
+            prompt: brief.prompt,
+            createdAt: clock.now().toISOString(),
+        });
+        return {
+            runId: run.id,
+            attempt: Number(run.attempt),
+            agent,
+            run: view,
+            model: onlineEntryOf(agent, assembly.model, (await deps.defaultModel?.(conn)) ?? null),
+            system: brief.prompt,
+            turn: brief.turn,
+            inputs,
+            token: token.value,
+            skills,
+            leaseExpiresAt,
+            firstSeq: (last ? Number(last.seq) : 0) + 1,
+            consultable,
+            depth,
+        };
+    }
+    async function claimOne(runner, seen, reported) {
+        const conn = tx.read();
+        const candidates = await candidatesOf('runner');
+        const agents = new Map();
+        for (const candidate of candidates) {
+            if (seen.has(candidate.id))
+                continue;
+            seen.add(candidate.id);
+            if (!agents.has(candidate.agentId))
+                agents.set(candidate.agentId, await findAgent(conn, candidate.agentId));
+            const agent = agents.get(candidate.agentId);
+            if (!agent || !fits(runner, agent, candidate))
+                continue;
+            // None of its tools has room on the runner's side: the runs behind it may still fit.
+            if (!pickOpenEntry(runner, agent, (tool) => reported.has(tool)))
+                continue;
+            try {
+                const result = await attempt(runner, agent, candidate, reported);
+                if (result.kind === 'claimed')
+                    return result.payload;
+            }
+            catch (error) {
+                // Rolled back: the run stays queued for a runner it fits.
+                if (error instanceof NotForThisRunner)
+                    continue;
+                if (!(error instanceof AssemblyError))
+                    throw error;
+                await recordFailure(candidate.id, error);
+            }
+        }
+        return null;
+    }
+    return {
+        async claim(runner, free, tools) {
+            if (runner.status !== 'online')
+                return [];
+            if ((await deps.slotsUsed(tx.read(), runner.id)) >= runner.slots)
+                return [];
+            const payloads = [];
+            const seen = new Set();
+            const limit = Math.min(free, runner.slots);
+            const reported = toolRoom(tools);
+            while (payloads.length < limit) {
+                const payload = await claimOne(runner, seen, reported);
+                if (!payload)
+                    break;
+                payloads.push(payload);
+                reported.take(payload.tool.kind);
+            }
+            return payloads;
+        },
+        async claimServer(holder, limit) {
+            const claimed = [];
+            const conn = tx.read();
+            const agents = new Map();
+            for (const candidate of await candidatesOf('online')) {
+                if (claimed.length >= limit)
+                    break;
+                if (!agents.has(candidate.agentId))
+                    agents.set(candidate.agentId, await findAgent(conn, candidate.agentId));
+                const agent = agents.get(candidate.agentId);
+                if (!agent || agent.archivedAt || agent.type !== 'online')
+                    continue;
+                try {
+                    const taken = await attemptServer(holder, agent, candidate);
+                    if (taken)
+                        claimed.push(taken);
+                }
+                catch (error) {
+                    if (!(error instanceof AssemblyError))
+                        throw error;
+                    await recordFailure(candidate.id, error);
+                }
+            }
+            return claimed;
+        },
+        async claimChild(holder, runId) {
+            const conn = tx.read();
+            const candidate = await findRunRecord(conn, runId);
+            if (!candidate?.parentRunId || candidate.status !== 'queued')
+                throw notFound('Run');
+            const parent = await findRunRecord(conn, candidate.parentRunId);
+            if (parent?.runnerId !== holder.id)
+                throw notFound('Run');
+            const agent = await findAgent(conn, candidate.agentId);
+            if (!agent || agent.archivedAt || agent.type !== 'online')
+                throw notFound('Agent');
+            try {
+                const taken = await attemptServer(holder, agent, candidate);
+                if (!taken)
+                    throw notFound('Run');
+                return taken;
+            }
+            catch (error) {
+                if (!(error instanceof AssemblyError))
+                    throw error;
+                // One attempt: a consultation that cannot be assembled fails at once.
+                await tx.run(async (unit) => {
+                    const run = await findRunRecord(unit.conn, runId);
+                    if (run)
+                        await finishRun(unit, deps, run, {
+                            status: 'failed',
+                            reason: 'setupFailed',
+                            detail: error.message,
+                        });
+                });
+                throw error;
+            }
+        },
+        async runSkills(runner, runId) {
+            const conn = tx.read();
+            const run = await findRunRecord(conn, runId);
+            if (!run)
+                throw notFound('Run');
+            const agent = await findAgent(conn, run.agentId);
+            const binding = deps.subjects.get(run.subjectKind);
+            if (!agent || !binding)
+                return [];
+            const assembly = await binding.context.assemble(conn, {
+                run: toRun(run),
+                agent,
+                runner,
+                inputs: [],
+                cli: deps.cli.name,
+                appName: deps.app.name,
+                dialect: dialectOf(agent),
+            });
+            return deps.skills.forRun(conn, skillTargets(agent, assembly), run.id);
+        },
+    };
+}
+//# sourceMappingURL=claim.js.map