@nocobase/app-plugin-agents 2.0.0-beta.0 → 2.0.0-beta.10

package/dist/server/core/runs/claim.js

@@ -19,6 +19,17 @@ import { ACTIVE_JOB, jobsRepo } from '../../jobs/job.store.js';
  * agent's entries whose tool the runner runs and has room for, so an agent listing two tools runs with the second while
  * the first is full; a run none of whose tools has room is passed over for the runs behind it.
  *
+ * Runners share a queue in proportion to each tool's limit, not by who asks first: before it locks anything, a claim
+ * waits until the run has been queued for the share of the tool's limit the runner already runs, times one second
+ * (`claimPace`). The runners' route hands out one run per request and the runner asks again at once, so of several
+ * runners woken for the same runs, the least busy with each tool takes the next one. A request that ends while it
+ * waits takes nothing.
+ *
+ * A run whose actor prefers one kind of runner (`runnerPreference`, `personal` or `team`, noted when it was queued)
+ * makes the other kind wait one second more (`preferenceDelay`), so an idle runner of the preferred kind takes it
+ * first, and one of the other kind takes it about a second later when none can. Work that only one runner may take
+ * (pinned, reserved, continuing where its session is, or for shared runners only) adds no wait. Nothing else is read.
+ *
  * Variables are checked and opened in the claim's own transaction, on its connection: which ones the assembled payload
  * gets, whether any is for shared runners only, and their values, so what the runner receives is what was checked. The
  * names taken from the runner go in `workspace.passthrough`, without values.
@@ -45,10 +56,10 @@ import { ACTIVE_JOB, jobsRepo } from '../../jobs/job.store.js';
  */
 import { GLOBAL_SKILL_TARGET } from '../../../shared/skills.js';
 import { ANNOUNCED_FAILURES, isProtocolSupported, policyAllowsAgent, policyAllowsRepo, policyAllowsSubject, TIMINGS, } from '@nocobase/agent-protocol';
-import { chatEffortCapabilities, runnerLowOnDisk, runsEntry, toolLimit, supportsWorkspaceKey, } from '../../../shared/runners.js';
+import { chatEffortCapabilities, quotaForAgent, runnerLowOnDisk, runsEntry, toolLimit, supportsWorkspaceKey, } from '../../../shared/runners.js';
 import { queueStart } from './selection.js';
 import { createHash } from 'node:crypto';
-import { later } from '../../kernel/clock.js';
+import { later, pause } from '../../kernel/clock.js';
 import { notFound } from '../../kernel/errors.js';
 import { runSecretsKey } from '../../kernel/redaction.js';
 import { cleanList, covers, jsonObject } from '../../kernel/values.js';
@@ -57,7 +68,7 @@ import { joinBrief, renderBrief } from '../brief/index.js';
 import { cliPackageFor } from './cli-package.js';
 import { executionHistory } from './execution.js';
 import { describeVariables, variableRefs, } from '../variables/index.js';
-import { mountsFor, prepareExtensions, repoAccessFor, responsibleOf, sessionKeyOf, withSections, } from './extensions.js';
+import { mountsFor, prepareExtensions, repoAccessFor, RepoAccessError, responsibleOf, sessionKeyOf, withSections, } from './extensions.js';
 import { MAX_CLAIM_FAILURES, toolPolicyFor } from './policy.js';
 import { mintRunToken } from './run-tokens.js';
 import { ACTIVE, countActive, eventsRepo, findRunRecord, markDelivered, pendingInputs, runsOfKey, runsRepo, inputsRepo, sessionsRepo, toInput, toExecutions, toRun, } from './run.store.js';
@@ -71,6 +82,11 @@ import { consumeReset, saveBrief } from './workspace.store.js';
 const CANDIDATES = 50;
 /** The initial page plus at most two further pages per claim request, shared by all free slots. */
 const CANDIDATE_PAGES = 3;
+/**
+ * What a runner waits for a run when it already runs its tool at the limit, and what one of the kind of runner the
+ * run's actor does not prefer waits on top (`preferenceDelay`).
+ */
+const PACE_MS = 1000;
 export function createClaimScan() {
     return { remainingPages: CANDIDATE_PAGES };
 }
@@ -109,6 +125,43 @@ function toolRoom(tools) {
         },
     };
 }
+/**
+ * How long `runner` waits before it takes `run` with `tool`, at `now`: the share of the tool's limit it already runs
+ * (`toolLimit`), of the busier of what this application counts (`byTool`) and what the runner last reported
+ * (`Runner.toolLoad`), times one second, counted from when the run entered the queue (`queueStart`). Several runners
+ * woken for one run take it in order of how busy each is with its tool, whichever asked first and however many runs
+ * each took before; an idle runner, and work queued longer than its share, wait nothing.
+ */
+export function claimPace(runner, tool, byTool, run, now, 
+/** Added to the share and counted from the same moment (`preferenceDelay`). */
+extraMs = 0) {
+    const limit = toolLimit(runner, tool);
+    if (limit <= 0)
+        return 0;
+    const load = runner.toolLoad?.[tool];
+    const used = Math.max(byTool[tool] ?? 0, load ? load.slots - load.free : 0);
+    const share = (Math.min(used, limit) / limit) * PACE_MS + extraMs;
+    return Math.min(share, Math.max(0, queueStart(run) + share - now));
+}
+/**
+ * What `runner` waits on top of its share (`claimPace`) for `run`, from the kind of runner its actor prefers: one
+ * second when the run prefers the other kind, nothing otherwise. Nothing either when only this runner may take it
+ * (`bound`: pinned, reserved or continuing where its session is), or when it prefers personal runners but has
+ * variables for shared runners only, which no personal runner may receive.
+ */
+export function preferenceDelay(runner, run, bound) {
+    const preferred = run.runnerPreference;
+    if (preferred !== 'personal' && preferred !== 'team')
+        return 0;
+    if (bound || run.runnerPinId)
+        return 0;
+    if (preferred === (runner.trust === 'team' ? 'team' : 'personal'))
+        return 0;
+    if (preferred === 'personal' &&
+        variableRefs(run.teamOnlyVariables).length > 0)
+        return 0;
+    return PACE_MS;
+}
 /**
  * The entry `runner` takes a run of `agent` with: the first whose tool it has enabled, installed and signed in, with
  * the entry's model among those it reports (`runsEntry`). Null when it has none of them.
@@ -146,9 +199,24 @@ export function fitsActor(runner, agent, actorUserId, requires, now) {
         return false;
     return runner.trust === 'team' || runner.ownerUserId === actorUserId;
 }
+/** A strict continuation keeps the tool/model that owns its original session. */
+function quotaAgent(run, agent) {
+    if (!run.runnerPinId || run.freshSession)
+        return agent;
+    const last = toExecutions(run.executionHistory).at(-1);
+    const tool = last?.tool ?? run.tool;
+    const model = last ? last.model : run.model;
+    return tool
+        ? {
+            ...agent,
+            modelEntries: agent.modelEntries.filter((entry) => entry.tool === tool && entry.model === model),
+        }
+        : agent;
+}
 /** Whether `runner` may run `run` of `agent`, before any lock is taken and before its payload is known. */
 export function fits(runner, agent, run, now) {
-    return fitsActor(runner, agent, run.actorUserId, toRun(run).requires, now);
+    return ((!run.runnerPinId || run.runnerPinId === runner.id) &&
+        fitsActor(runner, quotaAgent(run, agent), run.actorUserId, toRun(run).requires, now));
 }
 /** The runner features a payload needs. */
 export function payloadRequires(dirs, skills, variables, workspaceKey = false) {
@@ -185,7 +253,7 @@ export function skillTargets(_agent, assembly) {
         ...assembly.dirs.flatMap((dir) => dir.scopeId ? [{ scope: 'workdir', scopeId: dir.scopeId }] : []),
     ];
 }
-function fingerprint(agent, entry, layers, skills) {
+function fingerprint(agent, entry, layers, skills, readOnly) {
     return createHash('sha256')
         .update(JSON.stringify([
         entry.tool,
@@ -194,12 +262,16 @@ function fingerprint(agent, entry, layers, skills) {
         layers.system,
         layers.agent,
         skills.map((skill) => [skill.slug, skill.hash]),
+        // Keep existing writable fingerprints unchanged across upgrades.
+        ...(readOnly ? [{ readOnly: true }] : []),
     ]))
         .digest('hex');
 }
 /** The brief for `assembly`, with the skills a run would get. */
 export function briefOf(agent, assembly, skills, cli, appName) {
-    const rendered = renderBrief({
+    // Custom system prompts (such as conversations) keep their own layout.
+    const stablePrefix = assembly.briefLayout === 'stable-prefix' && assembly.system === undefined;
+    const input = {
         agentName: agent.name,
         subject: { noun: assembly.subject.noun, key: assembly.subject.key },
         task: assembly.task,
@@ -210,14 +282,20 @@ export function briefOf(agent, assembly, skills, cli, appName) {
         cli,
         appName,
         ...(assembly.guidance ? { guidance: assembly.guidance } : {}),
-    });
+    };
+    const rendered = renderBrief(input);
     const layers = assembly.system === undefined
         ? rendered
         : { ...rendered, system: assembly.system };
     const turn = assembly.turn.previousSummary
         ? `${assembly.turn.prompt}\n\nWhat earlier runs on this already did:\n\n${assembly.turn.previousSummary}`
         : assembly.turn.prompt;
-    return { layers, turn, prompt: joinBrief(layers) };
+    // Keep the logical layers (and their session fingerprint) unchanged. Only the outgoing text is reordered,
+    // so an ordinary follow-up can still resume the session created by an independent review.
+    const prompt = stablePrefix
+        ? joinBrief(renderBrief({ ...input, stablePrefix: true }), true)
+        : joinBrief(layers);
+    return { layers, turn, prompt };
 }
 /** Whether `runner` asks for repository credentials on demand rather than taking them with the claim. */
 export function asksCredentialsOnDemand(runner) {
@@ -330,8 +408,16 @@ export function createClaimService(deps) {
             const installed = connected.filter((runner) => runner.tools.some((tool) => tool.kind === entry.tool));
             const usable = installed.filter((runner) => runsEntry(runner, entry, clock.now().getTime()));
             const online = usable.filter((runner) => runner.trust === 'team' || runner.ownerUserId === run.actorUserId);
+            const quota = usable.length
+                ? []
+                : quotaForAgent(installed, single, run.actorUserId, now.getTime());
             const compatible = [];
-            const loads = [];
+            const loads = quota.map((runner) => ({
+                name: runner.runnerName,
+                ownerUserId: installed.find((item) => item.id === runner.runnerId)
+                    .ownerUserId,
+                reason: 'toolQuota',
+            }));
             let unknown = false;
             let incompatible = false;
             let ready = false;
@@ -429,11 +515,13 @@ export function createClaimService(deps) {
                         : 'restricted'
                     : usable.length
                         ? 'noSharedRunner'
-                        : installed.length
-                            ? 'toolUnavailable'
-                            : connected.length
-                                ? 'toolNotInstalled'
-                                : 'offline';
+                        : quota.length
+                            ? 'toolQuota'
+                            : installed.length
+                                ? 'toolUnavailable'
+                                : connected.length
+                                    ? 'toolNotInstalled'
+                                    : 'offline';
             if (!ready &&
                 compatible.length > 0 &&
                 loads.every((runner) => runner.reason === 'updatePending' || runner.reason === 'pruning'))
@@ -454,6 +542,13 @@ export function createClaimService(deps) {
                 tool: entry.tool,
                 model: entry.model,
                 reason,
+                ...(reason === 'toolQuota'
+                    ? {
+                        until: quota
+                            .flatMap((runner) => runner.resetsAt ? [runner.resetsAt] : [])
+                            .sort()[0] ?? null,
+                    }
+                    : {}),
                 compatibleCount: compatible.length,
                 blockers: [
                     ...(unknown ? ['unknown'] : []),
@@ -667,6 +762,7 @@ export function createClaimService(deps) {
         const token = await mintRunToken(conn, deps, run, runner.id);
         const git = await repoAccessFor(conn, deps.repoAccess, {
             claim,
+            readOnly: assembly.readOnly === true,
             repos: dirs.filter((dir) => dir.kind === 'repo'),
             onDemand: asksCredentialsOnDemand(runner),
             tools: wrapsHostTools(runner),
@@ -683,16 +779,20 @@ export function createClaimService(deps) {
         });
         const render = async () => briefOf(agent, await withSections(conn, deps.sections, effectiveClaim, assembly, prepared), skills, deps.cli.name, deps.app.name);
         let brief = await render();
-        const briefFingerprint = fingerprint(agent, entry, brief.layers, skills);
-        const resume = !run.freshSession &&
-            !clean &&
-            session &&
-            !session.poisoned &&
-            session.fingerprint === briefFingerprint;
+        const briefFingerprint = fingerprint(agent, entry, brief.layers, skills, assembly.readOnly === true);
+        const strict = Boolean(run.runnerPinId && !run.freshSession);
+        const resume = strict
+            ? Boolean(run.sessionId)
+            : !run.freshSession &&
+                binding.sessionPolicy?.(effectiveClaim) !== 'fresh' &&
+                !clean &&
+                session &&
+                !session.poisoned &&
+                session.fingerprint === briefFingerprint;
         // A kept session that cannot be resumed: the new one is told what came before, as a first one is. The history
         // is outside the fingerprint, so it does not change.
         let recoveryContext;
-        if (session) {
+        if (session && !strict) {
             const fresh = await binding.context.assemble(conn, { ...claim, freshSession: true }, prepared.get('subject'));
             if (resume) {
                 // Should the kept session be gone on the runner, the new one hears what came before from this.
@@ -726,6 +826,11 @@ export function createClaimService(deps) {
                     'The previous attempt was interrupted by a runner upgrade. Continue from the current state.',
                 ]
                 : []),
+            ...(strict
+                ? [
+                    'Quota recovery: continue the interrupted step from the existing session, progress and files.',
+                ]
+                : []),
             ...resumeNotes,
             ...(sessionNote ? [sessionNote] : []),
         ].join('\n\n');
@@ -738,6 +843,7 @@ export function createClaimService(deps) {
             createdAt: nowText,
         });
         const payload = {
+            ...(strict ? { session: { strict: true } } : {}),
             ...(run.subjectKind === 'conversation' &&
                 runner.features.includes('outputBlocks')
                 ? { outputBlocks: true }
@@ -768,8 +874,14 @@ export function createClaimService(deps) {
             prompt: {
                 system: brief.prompt,
                 turn,
-                session: resume ? 'resume' : 'fresh',
-                ...(resume ? { resumeSessionId: session.sessionId } : {}),
+                session: strict || resume ? 'resume' : 'fresh',
+                ...(strict
+                    ? run.sessionId
+                        ? { resumeSessionId: run.sessionId }
+                        : {}
+                    : resume && session
+                        ? { resumeSessionId: session.sessionId }
+                        : {}),
                 ...(resume && recoveryContext ? { recoveryContext } : {}),
             },
             inputs,
@@ -1033,7 +1145,11 @@ export function createClaimService(deps) {
         const pinned = await pin.runnerOf(conn, toRun(run));
         return pinned === null || pinned === runner.id;
     }
-    async function attempt(runner, agent, candidate, reported) {
+    async function attempt(runner, agent, candidate, reported, pass, 
+    /** Only this runner may take it: no preference wait (`preferenceDelay`). */
+    bound) {
+        if (candidate.runnerPinId && candidate.runnerPinId !== runner.id)
+            return { kind: 'skipped' };
         const conn = tx.read();
         // Rules that do not depend on who the run acts as come first, and settle the rest of the pass: an agent at its
         // limit is checked once per poll, not once per queued run.
@@ -1061,6 +1177,17 @@ export function createClaimService(deps) {
         const consent = await commentConsentFor(conn, runner, candidate);
         if (!consent.allowed)
             return { kind: 'skipped' };
+        // Wait this runner's turn for the run, after every read-only rule and before any lock or write. The transaction
+        // checks again whether the run is still queued and which tool has room.
+        const paced = pickOpenEntry(runner, agent, (tool) => (pass.byTool[tool] ?? 0) < toolLimit(runner, tool) &&
+            reported.has(tool), clock.now().getTime());
+        if (!paced)
+            return { kind: 'skipped' };
+        const wait = claimPace(runner, paced.tool, pass.byTool, candidate, clock.now().getTime(), preferenceDelay(runner, candidate, bound));
+        if (wait > 0)
+            await (deps.pause ?? pause)(wait, pass.signal);
+        if (pass.signal?.aborted)
+            return { kind: 'aborted' };
         // Read once, before the transaction, and handed to every repository access hook of this claim. A failed read
         // leaves the run queued for the next poll rather than letting a provider pick an identity without it.
         let responsibleUserId = null;
@@ -1136,6 +1263,8 @@ export function createClaimService(deps) {
                     status: 'dispatched',
                     failureReason: null,
                     failureDetail: null,
+                    failureParams: null,
+                    claimFailures: 0,
                     runnerId: runner.id,
                     // Brief providers observe this reservation inside the claim transaction.
                     // Replace a previous attempt's tool before computing the roster.
@@ -1223,7 +1352,16 @@ export function createClaimService(deps) {
                             : null,
                     },
                 });
-                unit.emit({ type: 'run.changed', runId: run.id, status: 'dispatched' });
+                if (current.runnerPinId && !current.freshSession)
+                    await deps.subjects
+                        .get(run.subjectKind)
+                        ?.sink?.onQuotaResumed?.(unit, toRun(run));
+                unit.emit({
+                    type: 'run.changed',
+                    ...(current.runnerPinId ? { quotaWait: false } : {}),
+                    runId: run.id,
+                    status: 'dispatched',
+                });
                 if (variableRefs(candidate.teamOnlyVariables).length > 0)
                     unit.emit({
                         type: 'notice.cleared',
@@ -1266,28 +1404,98 @@ export function createClaimService(deps) {
             }
         }
     }
-    /** Counts a failed assembly; the run fails once it has failed too often. */
-    async function recordFailure(runId, error) {
-        deps.onClaimFailure?.(runId, error.cause);
+    /** Repository failures have their own budget; ordinary assembly failures keep the existing three-try limit. */
+    async function recordFailure(candidate, error) {
+        deps.onClaimFailure?.(candidate.id, error.cause);
         await tx.run(async (unit) => {
-            const run = await findRunRecord(unit.conn, runId);
-            if (!run || run.status !== 'queued')
+            const run = await findRunRecord(unit.conn, candidate.id);
+            if (!run ||
+                run.status !== 'queued' ||
+                run.revision !== candidate.revision ||
+                Number(run.attempt) !== Number(candidate.attempt) ||
+                run.actorUserId !== candidate.actorUserId ||
+                !(await queuedIdentityCurrent(unit.conn, deps, run)))
+                return;
+            const previous = jsonObject(run.failureParams);
+            const cause = error.cause;
+            if (cause instanceof RepoAccessError) {
+                const now = clock.now().getTime();
+                const repoFailures = Number(previous.repoFailures ?? 0) + 1;
+                const first = typeof previous.repoFirstFailureAt === 'string'
+                    ? Date.parse(previous.repoFirstFailureAt)
+                    : now;
+                const retryAt = typeof cause.params.retryAt === 'string'
+                    ? Date.parse(cause.params.retryAt)
+                    : NaN;
+                const delays = [1, 2, 4, 8, 15];
+                const next = Math.max(now + (delays[repoFailures - 1] ?? 15) * 60_000, Number.isFinite(retryAt) ? retryAt : 0);
+                const params = {
+                    ...cause.params,
+                    repoFailures,
+                    repoFirstFailureAt: new Date(first).toISOString(),
+                };
+                const reason = cause.kind === 'denied'
+                    ? 'repoAccessDenied'
+                    : 'repoAccessUnavailable';
+                if (cause.kind === 'denied' ||
+                    repoFailures >= 6 ||
+                    next > first + 60 * 60_000) {
+                    await finishRun(unit, deps, run, {
+                        status: 'failed',
+                        reason,
+                        detail: cause.message,
+                        failureParams: params,
+                    }, true);
+                    return;
+                }
+                const availableAt = new Date(next).toISOString();
+                const changed = await runsRepo(unit.conn).updateMany({
+                    filter: {
+                        id: run.id,
+                        status: 'queued',
+                        revision: run.revision,
+                        attempt: Number(run.attempt),
+                        actorUserId: run.actorUserId,
+                    },
+                    values: {
+                        availableAt,
+                        failureReason: reason,
+                        failureDetail: cause.message.slice(0, 10_000),
+                        failureParams: params,
+                        updatedAt: clock.now().toISOString(),
+                    },
+                });
+                if (changed.updatedCount > 0) {
+                    unit.emit({ type: 'run.changed', runId: run.id, status: 'queued' });
+                    unit.emit({ type: 'run.requeued', runId: run.id, availableAt });
+                }
                 return;
+            }
             const failures = Number(run.claimFailures) + 1;
+            // Preserve only the independent repository retry budget across an unrelated preparation failure.
+            const budget = previous.repoFailures
+                ? {
+                    repoFailures: previous.repoFailures,
+                    repoFirstFailureAt: previous.repoFirstFailureAt,
+                    repoAccess: previous.repoAccess ?? previous,
+                }
+                : null;
             if (failures >= MAX_CLAIM_FAILURES) {
                 await finishRun(unit, deps, run, {
                     status: 'failed',
                     reason: 'setupFailed',
                     detail: error.message,
-                });
+                    failureParams: null,
+                }, true);
                 return;
             }
             await runsRepo(unit.conn).updateMany({
-                filter: { id: runId },
+                filter: { id: run.id, status: 'queued', revision: run.revision },
                 values: {
                     claimFailures: failures,
                     failureReason: 'setupFailed',
                     failureDetail: error.message.slice(0, 10_000),
+                    failureParams: budget,
                 },
             });
         });
@@ -1511,8 +1719,8 @@ export function createClaimService(deps) {
         }
         return true;
     }
-    /** False stops the whole request when a transactional capacity check finds the runner full. */
-    async function claimPage(runner, seen, reported, queued, full) {
+    /** False stops the whole request when a transactional capacity check finds the runner full, or the request ended. */
+    async function claimPage(runner, seen, reported, queued, full, pass) {
         const conn = tx.read();
         const affinity = await affinityBatch(queued, runner);
         const agents = await findAgents(conn, queued
@@ -1524,10 +1732,13 @@ export function createClaimService(deps) {
             ? deps.runners.all(conn)
             : Promise.resolve([runner]));
         for (const candidate of queued) {
-            if (seen.has(candidate.id) || full.has(candidate.agentId))
+            if (seen.has(candidate.id) ||
+                full.has(candidate.agentId) ||
+                (candidate.runnerPinId && candidate.runnerPinId !== runner.id))
                 continue;
             seen.add(candidate.id);
-            const agent = agents.get(candidate.agentId);
+            const configured = agents.get(candidate.agentId);
+            const agent = configured ? quotaAgent(candidate, configured) : undefined;
             if (!agent ||
                 (!deps.subjects.get(candidate.subjectKind)?.actsAs &&
                     !fits(runner, agent, candidate, clock.now().getTime())))
@@ -1537,7 +1748,8 @@ export function createClaimService(deps) {
                 continue;
             // Work bound to another runner waits for it; work bound here takes no other preference.
             const pin = deps.subjects.get(candidate.subjectKind)?.runnerPin;
-            const pinned = pin ? await pin.runnerOf(conn, toRun(candidate)) : null;
+            const pinned = candidate.runnerPinId ??
+                (pin ? await pin.runnerOf(conn, toRun(candidate)) : null);
             if (pinned !== null && pinned !== runner.id)
                 continue;
             const preferred = affinity.preferred.get(candidate.id);
@@ -1551,9 +1763,13 @@ export function createClaimService(deps) {
             if (!pickOpenEntry(runner, agent, (tool) => reported.has(tool), clock.now().getTime()))
                 continue;
             try {
-                const result = await attempt(runner, agent, candidate, reported);
+                const result = await attempt(runner, agent, candidate, reported, pass, reserved === runner.id ||
+                    pinned !== null ||
+                    preferred?.id === runner.id);
                 if (result.kind === 'claimed')
                     return result.payload;
+                if (result.kind === 'aborted')
+                    return false;
                 if (result.kind === 'full') {
                     if (result.scope === 'runner')
                         return false;
@@ -1568,7 +1784,7 @@ export function createClaimService(deps) {
                     continue;
                 if (!(error instanceof AssemblyError))
                     throw error;
-                await recordFailure(candidate.id, error);
+                await recordFailure(candidate, error);
             }
         }
         return null;
@@ -1615,7 +1831,7 @@ export function createClaimService(deps) {
             const agent = await findAgent(conn, run.agentId);
             return agent ? selectionFacts(conn, toRun(run), agent, false) : null;
         },
-        async claim(runner, free, tools, scan = createClaimScan()) {
+        async claim(runner, free, tools, scan = createClaimScan(), signal) {
             if (scan.remainingPages <= 0 ||
                 runner.status !== 'online' ||
                 runner.draining ||
@@ -1628,8 +1844,10 @@ export function createClaimService(deps) {
             const pages = candidates(runner, scan);
             let queued = [];
             while (payloads.length < limit) {
-                if ((await deps.slotsUsed(tx.read(), runner.id)) >= runner.slots)
+                const held = await deps.slotsHeld(tx.read(), runner.id);
+                if (held.runs + held.jobs >= runner.slots)
                     break;
+                const pass = { byTool: held.byTool ?? {}, signal };
                 const full = new Set();
                 let payload;
                 do {
@@ -1639,7 +1857,7 @@ export function createClaimService(deps) {
                             return payloads;
                         queued = page.value;
                     }
-                    payload = await claimPage(runner, seen, reported, queued, full);
+                    payload = await claimPage(runner, seen, reported, queued, full, pass);
                     if (payload === false)
                         return payloads;
                     if (!payload)