@nocobase/app-plugin-agents 2.0.0-beta.5 → 2.0.0-beta.7

package/dist/server/core/runs/claim.js

@@ -19,6 +19,12 @@ import { ACTIVE_JOB, jobsRepo } from '../../jobs/job.store.js';
  * agent's entries whose tool the runner runs and has room for, so an agent listing two tools runs with the second while
  * the first is full; a run none of whose tools has room is passed over for the runs behind it.
  *
+ * Runners share a queue in proportion to each tool's limit, not by who asks first: before it locks anything, a claim
+ * waits until the run has been queued for the share of the tool's limit the runner already runs, times one second
+ * (`claimPace`). The runners' route hands out one run per request and the runner asks again at once, so of several
+ * runners woken for the same runs, the least busy with each tool takes the next one. A request that ends while it
+ * waits takes nothing.
+ *
  * Variables are checked and opened in the claim's own transaction, on its connection: which ones the assembled payload
  * gets, whether any is for shared runners only, and their values, so what the runner receives is what was checked. The
  * names taken from the runner go in `workspace.passthrough`, without values.
@@ -48,7 +54,7 @@ import { ANNOUNCED_FAILURES, isProtocolSupported, policyAllowsAgent, policyAllow
 import { chatEffortCapabilities, quotaForAgent, runnerLowOnDisk, runsEntry, toolLimit, supportsWorkspaceKey, } from '../../../shared/runners.js';
 import { queueStart } from './selection.js';
 import { createHash } from 'node:crypto';
-import { later } from '../../kernel/clock.js';
+import { later, pause } from '../../kernel/clock.js';
 import { notFound } from '../../kernel/errors.js';
 import { runSecretsKey } from '../../kernel/redaction.js';
 import { cleanList, covers, jsonObject } from '../../kernel/values.js';
@@ -57,7 +63,7 @@ import { joinBrief, renderBrief } from '../brief/index.js';
 import { cliPackageFor } from './cli-package.js';
 import { executionHistory } from './execution.js';
 import { describeVariables, variableRefs, } from '../variables/index.js';
-import { mountsFor, prepareExtensions, repoAccessFor, responsibleOf, sessionKeyOf, withSections, } from './extensions.js';
+import { mountsFor, prepareExtensions, repoAccessFor, RepoAccessError, responsibleOf, sessionKeyOf, withSections, } from './extensions.js';
 import { MAX_CLAIM_FAILURES, toolPolicyFor } from './policy.js';
 import { mintRunToken } from './run-tokens.js';
 import { ACTIVE, countActive, eventsRepo, findRunRecord, markDelivered, pendingInputs, runsOfKey, runsRepo, inputsRepo, sessionsRepo, toInput, toExecutions, toRun, } from './run.store.js';
@@ -71,6 +77,8 @@ import { consumeReset, saveBrief } from './workspace.store.js';
 const CANDIDATES = 50;
 /** The initial page plus at most two further pages per claim request, shared by all free slots. */
 const CANDIDATE_PAGES = 3;
+/** The longest a claim waits its turn: what a runner waits for a run when it already runs its tool at the limit. */
+const PACE_MS = 1000;
 export function createClaimScan() {
     return { remainingPages: CANDIDATE_PAGES };
 }
@@ -109,6 +117,22 @@ function toolRoom(tools) {
         },
     };
 }
+/**
+ * How long `runner` waits before it takes `run` with `tool`, at `now`: the share of the tool's limit it already runs
+ * (`toolLimit`), of the busier of what this application counts (`byTool`) and what the runner last reported
+ * (`Runner.toolLoad`), times one second, counted from when the run entered the queue (`queueStart`). Several runners
+ * woken for one run take it in order of how busy each is with its tool, whichever asked first and however many runs
+ * each took before; an idle runner, and work queued longer than its share, wait nothing.
+ */
+export function claimPace(runner, tool, byTool, run, now) {
+    const limit = toolLimit(runner, tool);
+    if (limit <= 0)
+        return 0;
+    const load = runner.toolLoad?.[tool];
+    const used = Math.max(byTool[tool] ?? 0, load ? load.slots - load.free : 0);
+    const share = (Math.min(used, limit) / limit) * PACE_MS;
+    return Math.min(share, Math.max(0, queueStart(run) + share - now));
+}
 /**
  * The entry `runner` takes a run of `agent` with: the first whose tool it has enabled, installed and signed in, with
  * the entry's model among those it reports (`runsEntry`). Null when it has none of them.
@@ -146,9 +170,24 @@ export function fitsActor(runner, agent, actorUserId, requires, now) {
         return false;
     return runner.trust === 'team' || runner.ownerUserId === actorUserId;
 }
+/** A strict continuation keeps the tool/model that owns its original session. */
+function quotaAgent(run, agent) {
+    if (!run.runnerPinId || run.freshSession)
+        return agent;
+    const last = toExecutions(run.executionHistory).at(-1);
+    const tool = last?.tool ?? run.tool;
+    const model = last ? last.model : run.model;
+    return tool
+        ? {
+            ...agent,
+            modelEntries: agent.modelEntries.filter((entry) => entry.tool === tool && entry.model === model),
+        }
+        : agent;
+}
 /** Whether `runner` may run `run` of `agent`, before any lock is taken and before its payload is known. */
 export function fits(runner, agent, run, now) {
-    return fitsActor(runner, agent, run.actorUserId, toRun(run).requires, now);
+    return ((!run.runnerPinId || run.runnerPinId === runner.id) &&
+        fitsActor(runner, quotaAgent(run, agent), run.actorUserId, toRun(run).requires, now));
 }
 /** The runner features a payload needs. */
 export function payloadRequires(dirs, skills, variables, workspaceKey = false) {
@@ -185,7 +224,7 @@ export function skillTargets(_agent, assembly) {
         ...assembly.dirs.flatMap((dir) => dir.scopeId ? [{ scope: 'workdir', scopeId: dir.scopeId }] : []),
     ];
 }
-function fingerprint(agent, entry, layers, skills) {
+function fingerprint(agent, entry, layers, skills, readOnly) {
     return createHash('sha256')
         .update(JSON.stringify([
         entry.tool,
@@ -194,12 +233,16 @@ function fingerprint(agent, entry, layers, skills) {
         layers.system,
         layers.agent,
         skills.map((skill) => [skill.slug, skill.hash]),
+        // Keep existing writable fingerprints unchanged across upgrades.
+        ...(readOnly ? [{ readOnly: true }] : []),
     ]))
         .digest('hex');
 }
 /** The brief for `assembly`, with the skills a run would get. */
 export function briefOf(agent, assembly, skills, cli, appName) {
-    const rendered = renderBrief({
+    // Custom system prompts (such as conversations) keep their own layout.
+    const stablePrefix = assembly.briefLayout === 'stable-prefix' && assembly.system === undefined;
+    const input = {
         agentName: agent.name,
         subject: { noun: assembly.subject.noun, key: assembly.subject.key },
         task: assembly.task,
@@ -210,14 +253,20 @@ export function briefOf(agent, assembly, skills, cli, appName) {
         cli,
         appName,
         ...(assembly.guidance ? { guidance: assembly.guidance } : {}),
-    });
+    };
+    const rendered = renderBrief(input);
     const layers = assembly.system === undefined
         ? rendered
         : { ...rendered, system: assembly.system };
     const turn = assembly.turn.previousSummary
         ? `${assembly.turn.prompt}\n\nWhat earlier runs on this already did:\n\n${assembly.turn.previousSummary}`
         : assembly.turn.prompt;
-    return { layers, turn, prompt: joinBrief(layers) };
+    // Keep the logical layers (and their session fingerprint) unchanged. Only the outgoing text is reordered,
+    // so an ordinary follow-up can still resume the session created by an independent review.
+    const prompt = stablePrefix
+        ? joinBrief(renderBrief({ ...input, stablePrefix: true }), true)
+        : joinBrief(layers);
+    return { layers, turn, prompt };
 }
 /** Whether `runner` asks for repository credentials on demand rather than taking them with the claim. */
 export function asksCredentialsOnDemand(runner) {
@@ -684,6 +733,7 @@ export function createClaimService(deps) {
         const token = await mintRunToken(conn, deps, run, runner.id);
         const git = await repoAccessFor(conn, deps.repoAccess, {
             claim,
+            readOnly: assembly.readOnly === true,
             repos: dirs.filter((dir) => dir.kind === 'repo'),
             onDemand: asksCredentialsOnDemand(runner),
             tools: wrapsHostTools(runner),
@@ -700,17 +750,20 @@ export function createClaimService(deps) {
         });
         const render = async () => briefOf(agent, await withSections(conn, deps.sections, effectiveClaim, assembly, prepared), skills, deps.cli.name, deps.app.name);
         let brief = await render();
-        const briefFingerprint = fingerprint(agent, entry, brief.layers, skills);
-        const resume = !run.freshSession &&
-            binding.sessionPolicy?.(effectiveClaim) !== 'fresh' &&
-            !clean &&
-            session &&
-            !session.poisoned &&
-            session.fingerprint === briefFingerprint;
+        const briefFingerprint = fingerprint(agent, entry, brief.layers, skills, assembly.readOnly === true);
+        const strict = Boolean(run.runnerPinId && !run.freshSession);
+        const resume = strict
+            ? Boolean(run.sessionId)
+            : !run.freshSession &&
+                binding.sessionPolicy?.(effectiveClaim) !== 'fresh' &&
+                !clean &&
+                session &&
+                !session.poisoned &&
+                session.fingerprint === briefFingerprint;
         // A kept session that cannot be resumed: the new one is told what came before, as a first one is. The history
         // is outside the fingerprint, so it does not change.
         let recoveryContext;
-        if (session) {
+        if (session && !strict) {
             const fresh = await binding.context.assemble(conn, { ...claim, freshSession: true }, prepared.get('subject'));
             if (resume) {
                 // Should the kept session be gone on the runner, the new one hears what came before from this.
@@ -744,6 +797,11 @@ export function createClaimService(deps) {
                     'The previous attempt was interrupted by a runner upgrade. Continue from the current state.',
                 ]
                 : []),
+            ...(strict
+                ? [
+                    'Quota recovery: continue the interrupted step from the existing session, progress and files.',
+                ]
+                : []),
             ...resumeNotes,
             ...(sessionNote ? [sessionNote] : []),
         ].join('\n\n');
@@ -756,6 +814,7 @@ export function createClaimService(deps) {
             createdAt: nowText,
         });
         const payload = {
+            ...(strict ? { session: { strict: true } } : {}),
             ...(run.subjectKind === 'conversation' &&
                 runner.features.includes('outputBlocks')
                 ? { outputBlocks: true }
@@ -786,8 +845,14 @@ export function createClaimService(deps) {
             prompt: {
                 system: brief.prompt,
                 turn,
-                session: resume ? 'resume' : 'fresh',
-                ...(resume ? { resumeSessionId: session.sessionId } : {}),
+                session: strict || resume ? 'resume' : 'fresh',
+                ...(strict
+                    ? run.sessionId
+                        ? { resumeSessionId: run.sessionId }
+                        : {}
+                    : resume && session
+                        ? { resumeSessionId: session.sessionId }
+                        : {}),
                 ...(resume && recoveryContext ? { recoveryContext } : {}),
             },
             inputs,
@@ -1051,7 +1116,9 @@ export function createClaimService(deps) {
         const pinned = await pin.runnerOf(conn, toRun(run));
         return pinned === null || pinned === runner.id;
     }
-    async function attempt(runner, agent, candidate, reported) {
+    async function attempt(runner, agent, candidate, reported, pass) {
+        if (candidate.runnerPinId && candidate.runnerPinId !== runner.id)
+            return { kind: 'skipped' };
         const conn = tx.read();
         // Rules that do not depend on who the run acts as come first, and settle the rest of the pass: an agent at its
         // limit is checked once per poll, not once per queued run.
@@ -1079,6 +1146,17 @@ export function createClaimService(deps) {
         const consent = await commentConsentFor(conn, runner, candidate);
         if (!consent.allowed)
             return { kind: 'skipped' };
+        // Wait this runner's turn for the run, after every read-only rule and before any lock or write. The transaction
+        // checks again whether the run is still queued and which tool has room.
+        const paced = pickOpenEntry(runner, agent, (tool) => (pass.byTool[tool] ?? 0) < toolLimit(runner, tool) &&
+            reported.has(tool), clock.now().getTime());
+        if (!paced)
+            return { kind: 'skipped' };
+        const wait = claimPace(runner, paced.tool, pass.byTool, candidate, clock.now().getTime());
+        if (wait > 0)
+            await (deps.pause ?? pause)(wait, pass.signal);
+        if (pass.signal?.aborted)
+            return { kind: 'aborted' };
         // Read once, before the transaction, and handed to every repository access hook of this claim. A failed read
         // leaves the run queued for the next poll rather than letting a provider pick an identity without it.
         let responsibleUserId = null;
@@ -1154,6 +1232,8 @@ export function createClaimService(deps) {
                     status: 'dispatched',
                     failureReason: null,
                     failureDetail: null,
+                    failureParams: null,
+                    claimFailures: 0,
                     runnerId: runner.id,
                     // Brief providers observe this reservation inside the claim transaction.
                     // Replace a previous attempt's tool before computing the roster.
@@ -1241,7 +1321,16 @@ export function createClaimService(deps) {
                             : null,
                     },
                 });
-                unit.emit({ type: 'run.changed', runId: run.id, status: 'dispatched' });
+                if (current.runnerPinId && !current.freshSession)
+                    await deps.subjects
+                        .get(run.subjectKind)
+                        ?.sink?.onQuotaResumed?.(unit, toRun(run));
+                unit.emit({
+                    type: 'run.changed',
+                    ...(current.runnerPinId ? { quotaWait: false } : {}),
+                    runId: run.id,
+                    status: 'dispatched',
+                });
                 if (variableRefs(candidate.teamOnlyVariables).length > 0)
                     unit.emit({
                         type: 'notice.cleared',
@@ -1284,28 +1373,98 @@ export function createClaimService(deps) {
             }
         }
     }
-    /** Counts a failed assembly; the run fails once it has failed too often. */
-    async function recordFailure(runId, error) {
-        deps.onClaimFailure?.(runId, error.cause);
+    /** Repository failures have their own budget; ordinary assembly failures keep the existing three-try limit. */
+    async function recordFailure(candidate, error) {
+        deps.onClaimFailure?.(candidate.id, error.cause);
         await tx.run(async (unit) => {
-            const run = await findRunRecord(unit.conn, runId);
-            if (!run || run.status !== 'queued')
+            const run = await findRunRecord(unit.conn, candidate.id);
+            if (!run ||
+                run.status !== 'queued' ||
+                run.revision !== candidate.revision ||
+                Number(run.attempt) !== Number(candidate.attempt) ||
+                run.actorUserId !== candidate.actorUserId ||
+                !(await queuedIdentityCurrent(unit.conn, deps, run)))
+                return;
+            const previous = jsonObject(run.failureParams);
+            const cause = error.cause;
+            if (cause instanceof RepoAccessError) {
+                const now = clock.now().getTime();
+                const repoFailures = Number(previous.repoFailures ?? 0) + 1;
+                const first = typeof previous.repoFirstFailureAt === 'string'
+                    ? Date.parse(previous.repoFirstFailureAt)
+                    : now;
+                const retryAt = typeof cause.params.retryAt === 'string'
+                    ? Date.parse(cause.params.retryAt)
+                    : NaN;
+                const delays = [1, 2, 4, 8, 15];
+                const next = Math.max(now + (delays[repoFailures - 1] ?? 15) * 60_000, Number.isFinite(retryAt) ? retryAt : 0);
+                const params = {
+                    ...cause.params,
+                    repoFailures,
+                    repoFirstFailureAt: new Date(first).toISOString(),
+                };
+                const reason = cause.kind === 'denied'
+                    ? 'repoAccessDenied'
+                    : 'repoAccessUnavailable';
+                if (cause.kind === 'denied' ||
+                    repoFailures >= 6 ||
+                    next > first + 60 * 60_000) {
+                    await finishRun(unit, deps, run, {
+                        status: 'failed',
+                        reason,
+                        detail: cause.message,
+                        failureParams: params,
+                    }, true);
+                    return;
+                }
+                const availableAt = new Date(next).toISOString();
+                const changed = await runsRepo(unit.conn).updateMany({
+                    filter: {
+                        id: run.id,
+                        status: 'queued',
+                        revision: run.revision,
+                        attempt: Number(run.attempt),
+                        actorUserId: run.actorUserId,
+                    },
+                    values: {
+                        availableAt,
+                        failureReason: reason,
+                        failureDetail: cause.message.slice(0, 10_000),
+                        failureParams: params,
+                        updatedAt: clock.now().toISOString(),
+                    },
+                });
+                if (changed.updatedCount > 0) {
+                    unit.emit({ type: 'run.changed', runId: run.id, status: 'queued' });
+                    unit.emit({ type: 'run.requeued', runId: run.id, availableAt });
+                }
                 return;
+            }
             const failures = Number(run.claimFailures) + 1;
+            // Preserve only the independent repository retry budget across an unrelated preparation failure.
+            const budget = previous.repoFailures
+                ? {
+                    repoFailures: previous.repoFailures,
+                    repoFirstFailureAt: previous.repoFirstFailureAt,
+                    repoAccess: previous.repoAccess ?? previous,
+                }
+                : null;
             if (failures >= MAX_CLAIM_FAILURES) {
                 await finishRun(unit, deps, run, {
                     status: 'failed',
                     reason: 'setupFailed',
                     detail: error.message,
-                });
+                    failureParams: null,
+                }, true);
                 return;
             }
             await runsRepo(unit.conn).updateMany({
-                filter: { id: runId },
+                filter: { id: run.id, status: 'queued', revision: run.revision },
                 values: {
                     claimFailures: failures,
                     failureReason: 'setupFailed',
                     failureDetail: error.message.slice(0, 10_000),
+                    failureParams: budget,
                 },
             });
         });
@@ -1529,8 +1688,8 @@ export function createClaimService(deps) {
         }
         return true;
     }
-    /** False stops the whole request when a transactional capacity check finds the runner full. */
-    async function claimPage(runner, seen, reported, queued, full) {
+    /** False stops the whole request when a transactional capacity check finds the runner full, or the request ended. */
+    async function claimPage(runner, seen, reported, queued, full, pass) {
         const conn = tx.read();
         const affinity = await affinityBatch(queued, runner);
         const agents = await findAgents(conn, queued
@@ -1542,10 +1701,13 @@ export function createClaimService(deps) {
             ? deps.runners.all(conn)
             : Promise.resolve([runner]));
         for (const candidate of queued) {
-            if (seen.has(candidate.id) || full.has(candidate.agentId))
+            if (seen.has(candidate.id) ||
+                full.has(candidate.agentId) ||
+                (candidate.runnerPinId && candidate.runnerPinId !== runner.id))
                 continue;
             seen.add(candidate.id);
-            const agent = agents.get(candidate.agentId);
+            const configured = agents.get(candidate.agentId);
+            const agent = configured ? quotaAgent(candidate, configured) : undefined;
             if (!agent ||
                 (!deps.subjects.get(candidate.subjectKind)?.actsAs &&
                     !fits(runner, agent, candidate, clock.now().getTime())))
@@ -1555,7 +1717,8 @@ export function createClaimService(deps) {
                 continue;
             // Work bound to another runner waits for it; work bound here takes no other preference.
             const pin = deps.subjects.get(candidate.subjectKind)?.runnerPin;
-            const pinned = pin ? await pin.runnerOf(conn, toRun(candidate)) : null;
+            const pinned = candidate.runnerPinId ??
+                (pin ? await pin.runnerOf(conn, toRun(candidate)) : null);
             if (pinned !== null && pinned !== runner.id)
                 continue;
             const preferred = affinity.preferred.get(candidate.id);
@@ -1569,9 +1732,11 @@ export function createClaimService(deps) {
             if (!pickOpenEntry(runner, agent, (tool) => reported.has(tool), clock.now().getTime()))
                 continue;
             try {
-                const result = await attempt(runner, agent, candidate, reported);
+                const result = await attempt(runner, agent, candidate, reported, pass);
                 if (result.kind === 'claimed')
                     return result.payload;
+                if (result.kind === 'aborted')
+                    return false;
                 if (result.kind === 'full') {
                     if (result.scope === 'runner')
                         return false;
@@ -1586,7 +1751,7 @@ export function createClaimService(deps) {
                     continue;
                 if (!(error instanceof AssemblyError))
                     throw error;
-                await recordFailure(candidate.id, error);
+                await recordFailure(candidate, error);
             }
         }
         return null;
@@ -1633,7 +1798,7 @@ export function createClaimService(deps) {
             const agent = await findAgent(conn, run.agentId);
             return agent ? selectionFacts(conn, toRun(run), agent, false) : null;
         },
-        async claim(runner, free, tools, scan = createClaimScan()) {
+        async claim(runner, free, tools, scan = createClaimScan(), signal) {
             if (scan.remainingPages <= 0 ||
                 runner.status !== 'online' ||
                 runner.draining ||
@@ -1646,8 +1811,10 @@ export function createClaimService(deps) {
             const pages = candidates(runner, scan);
             let queued = [];
             while (payloads.length < limit) {
-                if ((await deps.slotsUsed(tx.read(), runner.id)) >= runner.slots)
+                const held = await deps.slotsHeld(tx.read(), runner.id);
+                if (held.runs + held.jobs >= runner.slots)
                     break;
+                const pass = { byTool: held.byTool ?? {}, signal };
                 const full = new Set();
                 let payload;
                 do {
@@ -1657,7 +1824,7 @@ export function createClaimService(deps) {
                             return payloads;
                         queued = page.value;
                     }
-                    payload = await claimPage(runner, seen, reported, queued, full);
+                    payload = await claimPage(runner, seen, reported, queued, full, pass);
                     if (payload === false)
                         return payloads;
                     if (!payload)