@nocobase/app-plugin-agents 0.0.0-stage → 0.1.0-beta.10

package/dist/server/core/runs/transitions.js

@@ -0,0 +1,169 @@
+import { later } from '../../kernel/clock.js';
+import { retryDelayMs, WORKSPACE_WAIT_LIMIT_MS, workspaceBusyDelayMs, } from './policy.js';
+import { endExecution } from './execution.js';
+import { ACTIVE, findRunRecord, runsRepo, tokensRepo, toExecutions, toRun, } from './run.store.js';
+/** Revokes every live token of the run. */
+export async function revokeTokens(tx, runId, now) {
+    await tokensRepo(tx.conn).updateMany({
+        filter: (f) => f.and([f.string('runId').eq(runId), f.date('revokedAt').empty()]),
+        values: { revokedAt: now },
+    });
+}
+/**
+ * Ends `run` if it is still in `run.status` (and, when held, still held by the same runner). Returns the ended run,
+ * or null when someone else moved it first.
+ */
+export async function finishRun(tx, deps, run, outcome) {
+    const now = deps.clock.now().toISOString();
+    const values = {
+        status: outcome.status,
+        finishedAt: now,
+        leaseExpiresAt: null,
+        availableAt: null,
+        updatedAt: now,
+    };
+    if (outcome.reason !== undefined)
+        values.failureReason = outcome.reason;
+    if (outcome.detail !== undefined)
+        values.failureDetail = outcome.detail.slice(0, 10_000);
+    if (outcome.summary !== undefined)
+        values.summary = outcome.summary;
+    if (outcome.sessionId !== undefined)
+        values.sessionId = outcome.sessionId;
+    if (outcome.cancelledById !== undefined)
+        values.cancelledById = outcome.cancelledById;
+    const result = await runsRepo(tx.conn).updateMany({
+        filter: (f) => f.and([
+            f.string('id').eq(run.id),
+            f.string('status').eq(run.status),
+            ...(run.status === 'queued'
+                ? [
+                    f.number('attempt').eq(Number(run.attempt)),
+                    run.queueEnteredAt
+                        ? f.and([
+                            f
+                                .date('queueEnteredAt')
+                                .notBefore(new Date(run.queueEnteredAt)),
+                            f
+                                .date('queueEnteredAt')
+                                .notAfter(new Date(run.queueEnteredAt)),
+                        ])
+                        : f.date('queueEnteredAt').empty(),
+                    run.availableAt
+                        ? f.and([
+                            f.date('availableAt').notBefore(new Date(run.availableAt)),
+                            f.date('availableAt').notAfter(new Date(run.availableAt)),
+                        ])
+                        : f.date('availableAt').empty(),
+                ]
+                : []),
+            run.runnerId === null
+                ? f.string('runnerId').eq(null)
+                : f.string('runnerId').eq(run.runnerId),
+        ]),
+        values,
+    });
+    if (result.updatedCount !== 1)
+        return null;
+    await endExecution(tx.conn, run, now, outcome.reason ?? null);
+    await revokeTokens(tx, run.id, now);
+    const ended = (await findRunRecord(tx.conn, run.id));
+    tx.emit({ type: 'run.changed', runId: run.id, status: ended.status });
+    await deps.subjects
+        .get(ended.subjectKind)
+        ?.sink?.onRunFinished?.(tx, toRun(ended));
+    return ended;
+}
+/**
+ * Workspace waits and runner upgrade interruptions do not consume maxAttempts. Include the open current attempt
+ * when reporting workspace contention; only consecutive workspace waits set the contention limit.
+ */
+function workspaceWaits(run, reporting) {
+    const executions = toExecutions(run.executionHistory);
+    const current = executions.find((item) => item.attempt === Number(run.attempt) && item.finishedAt === null);
+    const ended = executions.filter((item) => item !== current);
+    let total = ended.filter((item) => item.failureReason === 'workspaceBusy' ||
+        item.failureReason === 'runnerUpgrade').length;
+    let trailing = 0;
+    let since = null;
+    if (reporting) {
+        total += 1;
+        trailing = 1;
+        since = current?.dispatchedAt ?? run.dispatchedAt ?? null;
+        for (let i = ended.length - 1; i >= 0; i -= 1) {
+            const item = ended[i];
+            if (item.failureReason !== 'workspaceBusy')
+                break;
+            trailing += 1;
+            since = item.dispatchedAt;
+        }
+    }
+    return { total, trailing, since };
+}
+/**
+ * Takes a held run back from its runner after a retryable failure: it goes back to the queue while attempts remain,
+ * and fails with `reason` otherwise.
+ */
+export async function requeueRun(tx, deps, run, reason, detail) {
+    const now = deps.clock.now();
+    const waits = workspaceWaits(run, reason === 'workspaceBusy');
+    const exhausted = reason === 'runnerUpgrade'
+        ? false
+        : reason === 'workspaceBusy'
+            ? waits.since !== null &&
+                now.getTime() - Date.parse(waits.since) >= WORKSPACE_WAIT_LIMIT_MS
+            : Number(run.attempt) - waits.total >= Number(run.maxAttempts);
+    if (exhausted) {
+        const ended = await finishRun(tx, deps, run, {
+            status: 'failed',
+            reason,
+            ...(detail === undefined ? {} : { detail }),
+        });
+        return { status: ended?.status ?? run.status, run: ended };
+    }
+    const nowText = now.toISOString();
+    const delay = reason === 'runnerUpgrade'
+        ? 0
+        : reason === 'workspaceBusy'
+            ? workspaceBusyDelayMs(waits.trailing)
+            : retryDelayMs(reason, Number(run.attempt) - waits.total);
+    const availableAt = delay > 0 ? later(now, delay) : null;
+    const result = await runsRepo(tx.conn).updateMany({
+        filter: (f) => f.and([
+            f.string('id').eq(run.id),
+            f.or(ACTIVE.map((status) => f.string('status').eq(status))),
+            run.runnerId === null
+                ? f.string('runnerId').eq(null)
+                : f.string('runnerId').eq(run.runnerId),
+        ]),
+        values: {
+            status: 'queued',
+            queueEnteredAt: nowText,
+            teamOnlyVariables: null,
+            gitCredentialUrls: null,
+            runnerId: null,
+            attempt: Number(run.attempt) + 1,
+            availableAt,
+            leaseExpiresAt: null,
+            dispatchedAt: null,
+            startedAt: null,
+            acceptsInput: false,
+            claimFailures: 0,
+            failureReason: reason,
+            failureDetail: detail === undefined ? null : detail.slice(0, 10_000),
+            updatedAt: nowText,
+        },
+    });
+    if (result.updatedCount !== 1)
+        return { status: run.status, run: null };
+    await endExecution(tx.conn, run, nowText, reason);
+    await revokeTokens(tx, run.id, nowText);
+    tx.emit({ type: 'run.changed', runId: run.id, status: 'queued' });
+    tx.emit({ type: 'run.requeued', runId: run.id, availableAt });
+    return {
+        status: 'queued',
+        retryAt: availableAt ?? nowText,
+        run: (await findRunRecord(tx.conn, run.id)) ?? null,
+    };
+}
+//# sourceMappingURL=transitions.js.map

package/dist/server/core/runs/transitions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transitions.js","sourceRoot":"","sources":["../../../../server/core/runs/transitions.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,KAAK,EAAc,MAAM,uBAAuB,CAAC;AAG1D,OAAO,EACL,YAAY,EACZ,uBAAuB,EACvB,oBAAoB,GACrB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EACL,MAAM,EACN,aAAa,EACb,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,KAAK,GAEN,MAAM,gBAAgB,CAAC;AAgBxB,2CAA2C;AAC3C,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,EAAM,EACN,KAAa,EACb,GAAW;IAEX,MAAM,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC;QACnC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE;KAC3B,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,EAAM,EACN,IAAoB,EACpB,GAAc,EACd,OAAsB;IAEtB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,MAAM,GAA4B;QACtC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,GAAG;QACf,cAAc,EAAE,IAAI;QACpB,WAAW,EAAE,IAAI;QACjB,SAAS,EAAE,GAAG;KACf,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;QAAE,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;IACxE,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;QAC9B,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACzD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;QAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACpE,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS;QAAE,MAAM,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAC1E,IAAI,OAAO,CAAC,aAAa,KAAK,SAAS;QACrC,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC/C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC;QAChD,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,GAAG,CAAC;YACJ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC;YACjC,GAAG,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ;gBACzB,CAAC,CAAC;oBACE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBAC3C,GAAG,CAAC,cAAc;wBAChB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;4BACJ,CAAC;iCACE,IAAI,CAAC,gBAAgB,CAAC;iCACtB,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;4BAC1C,CAAC;iCACE,IAAI,CAAC,gBAAgB,CAAC;iCACtB,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;yBAC1C,CAAC;wBACJ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,KAAK,EAAE;oBACpC,GAAG,CAAC,WAAW;wBACb,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;4BACJ,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;4BAC1D,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;yBAC1D,CAAC;wBACJ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE;iBAClC;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,QAAQ,KAAK,IAAI;gBACnB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC;gBAC/B,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC1C,CAAC;QACJ,MAAM;KACP,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,YAAY,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;IAC9D,MAAM,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,CAAC,MAAM,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAE,CAAC;IACtD,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACtE,MAAM,IAAI,CAAC,QAAQ;SAChB,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC;QACvB,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5C,OAAO,KAAK,CAAC;AACf,CAAC;AAWD;;;GAGG;AACH,SAAS,cAAc,CAAC,GAAc,EAAE,SAAkB;IACxD,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAC7B,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,CAC3E,CAAC;IACF,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;IAC5D,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,CACtB,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,aAAa,KAAK,eAAe;QACtC,IAAI,CAAC,aAAa,KAAK,eAAe,CACzC,CAAC,MAAM,CAAC;IACT,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,KAAK,GAAkB,IAAI,CAAC;IAChC,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,IAAI,CAAC,CAAC;QACX,QAAQ,GAAG,CAAC,CAAC;QACb,KAAK,GAAG,OAAO,EAAE,YAAY,IAAI,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC;QAC1D,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,aAAa,KAAK,eAAe;gBAAE,MAAM;YAClD,QAAQ,IAAI,CAAC,CAAC;YACd,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AACpC,CAAC;AASD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,EAAM,EACN,IAAoB,EACpB,GAAc,EACd,MAAqB,EACrB,MAAe;IAEf,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,MAAM,KAAK,eAAe,CAAC,CAAC;IAC9D,MAAM,SAAS,GACb,MAAM,KAAK,eAAe;QACxB,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,MAAM,KAAK,eAAe;YAC1B,CAAC,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI;gBACpB,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,uBAAuB;YACpE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACrE,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;YAC3C,MAAM,EAAE,QAAQ;YAChB,MAAM;YACN,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;SAC5C,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;IAC7D,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IAClC,MAAM,KAAK,GACT,MAAM,KAAK,eAAe;QACxB,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,MAAM,KAAK,eAAe;YAC1B,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,QAAQ,CAAC;YACtC,CAAC,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IAChE,MAAM,WAAW,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACzD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC;QAChD,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,GAAG,CAAC;YACJ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;YAC3D,GAAG,CAAC,QAAQ,KAAK,IAAI;gBACnB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC;gBAC/B,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC1C,CAAC;QACJ,MAAM,EAAE;YACN,MAAM,EAAE,QAAQ;YAChB,cAAc,EAAE,OAAO;YACvB,iBAAiB,EAAE,IAAI;YACvB,iBAAiB,EAAE,IAAI;YACvB,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC;YAChC,WAAW;YACX,cAAc,EAAE,IAAI;YACpB,YAAY,EAAE,IAAI;YAClB,SAAS,EAAE,IAAI;YACf,YAAY,EAAE,KAAK;YACnB,aAAa,EAAE,CAAC;YAChB,aAAa,EAAE,MAAM;YACrB,aAAa,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;YACpE,SAAS,EAAE,OAAO;SACnB;KACF,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,YAAY,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;IACxE,MAAM,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAClD,MAAM,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACxC,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IAClE,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;IAC9D,OAAO;QACL,MAAM,EAAE,QAAQ;QAChB,OAAO,EAAE,WAAW,IAAI,OAAO;QAC/B,GAAG,EAAE,CAAC,MAAM,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI;KACpD,CAAC;AACJ,CAAC"}

package/dist/server/core/runs/workload.d.ts

@@ -0,0 +1,49 @@
+/**
+ * The open runs on one kind of subject as a board of work shows them (`RunService.workload`): the runs held, with the
+ * runner and the newest thing each reported, and the runs queued, with their place in the claim order and what holds
+ * each, worked out by the claim's own rules (`claim.ts`) without claiming anything.
+ *
+ * Common reads are batched: the open runs of the kind, every queued run (for the claim order), every held run
+ * (for the agents' and runners' counts and the busy subjects), the runners, the active jobs, the agents, the open
+ * runs' inputs, and the newest event of each held run of the kind (at most one per runner slot).
+ */
+import type { AgentTool, RunnerFeature } from '@nocobase/agent-protocol';
+import type { DatabaseConnection } from '@nocobase/db';
+import { type Agent } from '../../../shared/agents.js';
+import { type Runner } from '../../../shared/runners.js';
+import { type RunWait, type Workload, type WorkloadQuery } from '../../../shared/runs.js';
+import type { Clock } from '../../kernel/clock.js';
+import { type RunRecord } from './run.store.js';
+/** Claim order: the more urgent first, then the oldest, then by id. */
+export declare function claimOrder(a: Pick<RunRecord, 'priority' | 'createdAt' | 'id'>, b: Pick<RunRecord, 'priority' | 'createdAt' | 'id'>): number;
+/** What `explainWait` needs to know about everything but the run itself. */
+export interface WaitContext {
+    readonly now: Date;
+    readonly agent: Agent | null;
+    /** Online runners. */
+    readonly runners: readonly Runner[];
+    /** Runs and jobs each runner holds now, by runner id. */
+    readonly runnerUsed: ReadonlyMap<string, number>;
+    /** The runs each runner holds now by coding tool, by runner id; absent for a runner that holds none. */
+    readonly runnerToolUsed?: ReadonlyMap<string, Readonly<Partial<Record<AgentTool, number>>>>;
+    /** Runs the agent holds now. */
+    readonly agentActive: number;
+    /** Whether a run of the same (agent, subject, thread) is held now. */
+    readonly sameWorkActive: boolean;
+}
+type WaitFacts = Pick<RunRecord, 'actorUserId' | 'availableAt' | 'claimFailures' | 'failureDetail'> & Partial<Pick<RunRecord, 'teamOnlyVariables'>> & {
+    readonly requires: readonly RunnerFeature[];
+};
+/**
+ * Why a queued run has not been taken, by the claim's rules in the claim's order (`RUN_WAIT_REASONS`); the first rule
+ * that holds it answers.
+ */
+export declare function explainWait(run: WaitFacts, context: WaitContext): Omit<RunWait, 'position' | 'agentPosition'>;
+/** What a workload reads of the runners: every runner, and the jobs each holds (they take its slots). */
+export interface WorkloadRunners {
+    all(conn: DatabaseConnection): Promise<Runner[]>;
+    jobsByRunner(conn: DatabaseConnection): Promise<ReadonlyMap<string, number>>;
+}
+export declare function readWorkload(conn: DatabaseConnection, clock: Clock, runnerSource: WorkloadRunners, query: WorkloadQuery, diagnose?: import('./claim.js').ClaimService['diagnose']): Promise<Workload>;
+export {};
+//# sourceMappingURL=workload.d.ts.map

package/dist/server/core/runs/workload.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"workload.d.ts","sourceRoot":"","sources":["../../../../server/core/runs/workload.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAEvD,OAAO,EAAc,KAAK,KAAK,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAIL,KAAK,OAAO,EACZ,KAAK,QAAQ,EACb,KAAK,aAAa,EAEnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAInD,OAAO,EAML,KAAK,SAAS,EACf,MAAM,gBAAgB,CAAC;AAexB,uEAAuE;AACvE,wBAAgB,UAAU,CACxB,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,UAAU,GAAG,WAAW,GAAG,IAAI,CAAC,EACnD,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,UAAU,GAAG,WAAW,GAAG,IAAI,CAAC,GAClD,MAAM,CAMR;AAED,4EAA4E;AAC5E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC;IACnB,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAC7B,sBAAsB;IACtB,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,yDAAyD;IACzD,QAAQ,CAAC,UAAU,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjD,wGAAwG;IACxG,QAAQ,CAAC,cAAc,CAAC,EAAE,WAAW,CACnC,MAAM,EACN,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,CAC7C,CAAC;IACF,gCAAgC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,sEAAsE;IACtE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAED,KAAK,SAAS,GAAG,IAAI,CACnB,SAAS,EACT,aAAa,GAAG,aAAa,GAAG,eAAe,GAAG,eAAe,CAClE,GACC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC,GAAG;IAC9C,QAAQ,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,CAAC;CAC7C,CAAC;AAeJ;;;GAGG;AACH,wBAAgB,WAAW,CACzB,GAAG,EAAE,SAAS,EACd,OAAO,EAAE,WAAW,GACnB,IAAI,CAAC,OAAO,EAAE,UAAU,GAAG,eAAe,CAAC,CAuG7C;AAuCD,yGAAyG;AACzG,MAAM,WAAW,eAAe;IAC9B,GAAG,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACjD,YAAY,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC9E;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,kBAAkB,EACxB,KAAK,EAAE,KAAK,EACZ,YAAY,EAAE,eAAe,EAC7B,KAAK,EAAE,aAAa,EACpB,QAAQ,CAAC,EAAE,OAAO,YAAY,EAAE,YAAY,CAAC,UAAU,CAAC,GACvD,OAAO,CAAC,QAAQ,CAAC,CAyRnB"}

package/dist/server/core/runs/workload.js

@@ -0,0 +1,390 @@
+import { entryTools } from '../../../shared/agents.js';
+import { runsEntry, runsTool, toolLimit, } from '../../../shared/runners.js';
+import { RUN_ACTIVITY_TEXT_MAX, } from '../../../shared/runs.js';
+import { covers } from '../../kernel/values.js';
+import { variableRefs } from '../variables/index.js';
+import { listAgents } from '../agents/index.js';
+import { ACTIVE, eventsRepo, inputsRepo, runsRepo, toRun, } from './run.store.js';
+const DEFAULT_LIMIT = 200;
+const MAX_LIMIT = 1000;
+/** Queued runs read for the claim order, of every kind. */
+const QUEUE_SCAN = 2000;
+/** The event types a "last activity" line is made of. */
+const ACTIVITY_TYPES = ['text', 'toolUse', 'status', 'error'];
+const time = (value) => value === null ? null : new Date(value).getTime();
+const iso = (value) => value === null ? null : new Date(value).toISOString();
+/** Claim order: the more urgent first, then the oldest, then by id. */
+export function claimOrder(a, b) {
+    return (Number(a.priority) - Number(b.priority) ||
+        (time(a.createdAt) ?? 0) - (time(b.createdAt) ?? 0) ||
+        a.id.localeCompare(b.id));
+}
+const wait = (reason, extra = {}) => ({
+    reason,
+    until: null,
+    tool: null,
+    missing: [],
+    detail: null,
+    variables: [],
+    ...extra,
+});
+/**
+ * Why a queued run has not been taken, by the claim's rules in the claim's order (`RUN_WAIT_REASONS`); the first rule
+ * that holds it answers.
+ */
+export function explainWait(run, context) {
+    const { agent } = context;
+    if (!agent || agent.status !== 'enabled')
+        return wait('agentArchived');
+    const availableAt = time(run.availableAt);
+    if (availableAt !== null && availableAt > context.now.getTime()) {
+        const until = iso(run.availableAt);
+        return wait('delayed', { until, params: until ? { until } : {} });
+    }
+    const tools = entryTools(agent);
+    const tool = tools[0] ?? null;
+    const model = agent.modelEntries[0]?.model ?? null;
+    let fitting = context.runners;
+    if (fitting.length === 0)
+        return wait('noRunnerOnline');
+    fitting = fitting.filter((runner) => agent.modelEntries.some((entry) => runsEntry(runner, entry)));
+    if (fitting.length === 0)
+        return wait('toolUnavailable', {
+            tool,
+            params: {
+                ...(tool ? { tool } : {}),
+                ...(model ? { model } : {}),
+            },
+        });
+    fitting = fitting.filter((runner) => runner.trust === 'team' || runner.ownerUserId === run.actorUserId);
+    if (fitting.length === 0)
+        return wait('noSharedRunner');
+    const withFeatures = fitting.filter((runner) => covers(runner.features, run.requires));
+    if (withFeatures.length === 0) {
+        const offered = new Set(fitting.flatMap((runner) => runner.features));
+        const missing = run.requires.filter((feature) => !offered.has(feature));
+        return wait('missingFeatures', { missing, params: { features: missing } });
+    }
+    // Variables for team runners only, as the last claim by a personal runner found them (`claim.ts`).
+    const teamOnly = variableRefs(run.teamOnlyVariables);
+    const trusted = teamOnly.length > 0
+        ? withFeatures.filter((runner) => runner.trust === 'team')
+        : withFeatures;
+    if (trusted.length === 0)
+        return wait('secretsNotAllowed', {
+            variables: teamOnly,
+            params: { variables: [...new Set(teamOnly.map((each) => each.name))] },
+        });
+    if (context.sameWorkActive)
+        return wait('sameWorkActive');
+    if (context.agentActive >= agent.maxConcurrentRuns)
+        return wait('concurrencyFull', {
+            params: { active: context.agentActive, limit: agent.maxConcurrentRuns },
+        });
+    if (trusted.every((runner) => runner.draining))
+        return wait(trusted.some((runner) => runner.draining === 'updatePending')
+            ? 'updatePending'
+            : 'pruning');
+    const free = trusted.filter((runner) => !runner.draining &&
+        (runner.globalFree ?? 1) > 0 &&
+        (context.runnerUsed.get(runner.id) ?? 0) < runner.slots);
+    if (free.length === 0)
+        return wait('runnersBusy', { params: { runners: trusted.length } });
+    // A tool has room on a runner below its limit here and while the runner last said it could take more of it.
+    const hasRoom = (runner, each) => (context.runnerToolUsed?.get(runner.id)?.[each] ?? 0) <
+        toolLimit(runner, each) && (runner.toolLoad?.[each]?.free ?? 1) > 0;
+    if (free.every((runner) => !agent.modelEntries.some((entry) => runsEntry(runner, entry) && hasRoom(runner, entry.tool)))) {
+        const full = tools.find((each) => free.some((runner) => runsTool(runner, each))) ??
+            null;
+        // The runner that runs it with the fewest of its slots taken: the one closest to room.
+        const closest = full
+            ? free
+                .filter((runner) => runsTool(runner, full))
+                .map((runner) => ({
+                used: context.runnerToolUsed?.get(runner.id)?.[full] ?? 0,
+                limit: toolLimit(runner, full),
+            }))
+                .sort((a, b) => a.used - a.limit - (b.used - b.limit))[0]
+            : undefined;
+        return wait('toolSlotsFull', {
+            tool: full,
+            params: full ? { tool: full, ...(closest ? closest : {}) } : {},
+        });
+    }
+    if (Number(run.claimFailures) > 0)
+        return wait('setupRetrying', {
+            detail: run.failureDetail,
+            params: run.failureDetail ? { detail: run.failureDetail } : {},
+        });
+    return wait('next');
+}
+const keyOf = (run) => JSON.stringify([
+    run.agentId,
+    run.subjectKind,
+    run.subjectId,
+    run.threadScope,
+]);
+function triggerOf(payload) {
+    const value = typeof payload === 'string'
+        ? (() => {
+            try {
+                return JSON.parse(payload);
+            }
+            catch {
+                return null;
+            }
+        })()
+        : payload;
+    if (value && typeof value === 'object' && !Array.isArray(value)) {
+        const trigger = value.trigger;
+        if (typeof trigger === 'string' && trigger !== '')
+            return trigger;
+    }
+    return null;
+}
+function cut(text) {
+    if (!text)
+        return null;
+    const line = text.replace(/\s+/gu, ' ').trim();
+    if (line === '')
+        return null;
+    return line.length > RUN_ACTIVITY_TEXT_MAX
+        ? `${line.slice(0, RUN_ACTIVITY_TEXT_MAX - 1)}…`
+        : line;
+}
+export async function readWorkload(conn, clock, runnerSource, query, diagnose) {
+    const now = clock.now();
+    const limit = Math.min(Math.max(query.limit ?? DEFAULT_LIMIT, 1), MAX_LIMIT);
+    const open = await runsRepo(conn).findMany({
+        filter: (f) => f.and([
+            f.string('subjectKind').eq(query.subjectKind),
+            f.or(['queued', ...ACTIVE].map((status) => f.string('status').eq(status))),
+        ]),
+        sort: (sort) => [sort.field('createdAt').asc(), sort.field('id').asc()],
+        limit: limit + 1,
+    });
+    const truncated = open.length > limit;
+    const mine = open.slice(0, limit);
+    const queued = (await runsRepo(conn).findMany({
+        filter: { status: 'queued' },
+        limit: QUEUE_SCAN,
+    })).sort(claimOrder);
+    const held = await runsRepo(conn).findMany({
+        filter: (f) => f.or(ACTIVE.map((status) => f.string('status').eq(status))),
+    });
+    const runners = await runnerSource.all(conn);
+    const online = runners.filter((runner) => runner.status === 'online');
+    const jobs = await runnerSource.jobsByRunner(conn);
+    const agents = await listAgents(conn);
+    const agentById = new Map(agents.map((agent) => [agent.id, agent]));
+    const runnerUsed = new Map();
+    const runnerToolUsed = new Map();
+    const agentActive = new Map();
+    const busyKeys = new Set();
+    for (const run of held) {
+        if (run.runnerId) {
+            runnerUsed.set(run.runnerId, (runnerUsed.get(run.runnerId) ?? 0) + 1);
+            const tool = run.tool;
+            if (tool) {
+                const counts = runnerToolUsed.get(run.runnerId) ?? {};
+                counts[tool] = (counts[tool] ?? 0) + 1;
+                runnerToolUsed.set(run.runnerId, counts);
+            }
+        }
+        agentActive.set(run.agentId, (agentActive.get(run.agentId) ?? 0) + 1);
+        busyKeys.add(keyOf(run));
+    }
+    for (const [runnerId, count] of jobs)
+        runnerUsed.set(runnerId, (runnerUsed.get(runnerId) ?? 0) + count);
+    // Places in the claim order: among the runs claimable now, and among each agent's queued runs.
+    const position = new Map();
+    const agentPosition = new Map();
+    const agentQueued = new Map();
+    let claimable = 0;
+    for (const run of queued) {
+        const at = time(run.availableAt);
+        if (at === null || at <= now.getTime())
+            position.set(run.id, (claimable += 1));
+        const count = (agentQueued.get(run.agentId) ?? 0) + 1;
+        agentQueued.set(run.agentId, count);
+        agentPosition.set(run.id, count);
+    }
+    // The newest input's trigger, per open run.
+    const ids = mine.map((run) => run.id);
+    const triggers = new Map();
+    if (ids.length > 0) {
+        const inputs = await inputsRepo(conn).findMany({
+            filter: (f) => f.or(ids.map((id) => f.string('runId').eq(id))),
+            sort: (sort) => [sort.field('createdAt').desc(), sort.field('id').desc()],
+        });
+        for (const input of inputs) {
+            if (triggers.has(input.runId))
+                continue;
+            const trigger = triggerOf(input.payload);
+            if (trigger)
+                triggers.set(input.runId, trigger);
+        }
+    }
+    // The newest activity of each held run: at most one per runner slot.
+    const activity = new Map();
+    for (const run of mine) {
+        if (!ACTIVE.includes(run.status))
+            continue;
+        const event = await eventsRepo(conn).findOne({
+            filter: (f) => f.and([
+                f.string('runId').eq(run.id),
+                f.or(ACTIVITY_TYPES.map((type) => f.string('type').eq(type))),
+            ]),
+            sort: (sort) => sort.field('seq').desc(),
+        });
+        if (event)
+            activity.set(run.id, {
+                at: iso(event.at) ?? event.at,
+                type: event.type,
+                tool: event.tool,
+                text: cut(event.content ?? event.output),
+            });
+    }
+    const runnerName = new Map(runners.map((runner) => [runner.id, runner.name]));
+    const runs = await Promise.all(mine.map(async (record) => {
+        const run = toRun(record);
+        const isQueued = record.status === 'queued';
+        const candidateAgent = agentById.get(run.agentId);
+        const diagnostic = isQueued && candidateAgent?.status === 'enabled' && diagnose
+            ? await diagnose(conn, run, candidateAgent)
+            : null;
+        const previous = explainWait({ ...record, requires: run.requires }, {
+            now,
+            agent: candidateAgent ?? null,
+            runners: online,
+            runnerUsed,
+            runnerToolUsed,
+            agentActive: agentActive.get(run.agentId) ?? 0,
+            sameWorkActive: busyKeys.has(keyOf(record)),
+        });
+        // Keep the established identity and protected-variable explanation, including its precise parameters.
+        const detailed = [
+            'secretsNotAllowed',
+            'delayed',
+            'agentArchived',
+            'setupRetrying',
+        ].includes(previous.reason);
+        const entry = diagnostic?.entries.find((entry) => entry.reason === 'ready') ??
+            diagnostic?.entries[0];
+        const codes = {
+            ready: 'next',
+            busy: 'runnersBusy',
+            updatePending: 'updatePending',
+            pruning: 'pruning',
+            offline: 'noRunnerOnline',
+            incompatible: 'incompatible',
+            restricted: 'restricted',
+            concurrencyFull: 'concurrencyFull',
+            sameWorkActive: 'sameWorkActive',
+            delayed: 'delayed',
+            toolUnavailable: 'toolUnavailable',
+            toolNotInstalled: 'toolNotInstalled',
+            noSharedRunner: 'noSharedRunner',
+            unknown: 'unknown',
+        };
+        const loads = entry?.runners.filter((runner) => runner.reason === undefined ||
+            runner.reason === 'ready' ||
+            runner.reason === 'busy') ?? [];
+        const toolFull = entry?.reason === 'busy' &&
+            loads.some((runner) => (runner.used ?? 0) < (runner.limit ?? 0)) &&
+            loads.every((runner) => (runner.toolUsed ?? 0) >= (runner.toolLimit ?? 0) ||
+                runner.reportedFree === 0);
+        return {
+            ...run,
+            trigger: triggers.get(run.id) ?? null,
+            runnerName: run.runnerId
+                ? (runnerName.get(run.runnerId) ?? null)
+                : null,
+            lastActivity: activity.get(run.id) ?? null,
+            wait: isQueued
+                ? {
+                    ...previous,
+                    ...(entry && !detailed
+                        ? {
+                            reason: entry.reason === 'ready' &&
+                                Number(record.claimFailures) > 0
+                                ? 'setupRetrying'
+                                : toolFull
+                                    ? 'toolSlotsFull'
+                                    : codes[entry.reason],
+                            params: {
+                                blockers: (entry.blockers ?? [])
+                                    .filter((reason) => reason !== entry.reason)
+                                    .map((reason) => codes[reason]),
+                                tool: entry.tool,
+                                ...(entry.reason === 'concurrencyFull'
+                                    ? {
+                                        active: agentActive.get(run.agentId) ?? 0,
+                                        limit: candidateAgent.maxConcurrentRuns,
+                                    }
+                                    : entry.reason === 'busy'
+                                        ? {
+                                            used: loads.reduce((n, runner) => n +
+                                                (toolFull
+                                                    ? (runner.toolUsed ?? 0)
+                                                    : (runner.used ?? 0)), 0),
+                                            limit: loads.reduce((n, runner) => n +
+                                                (toolFull
+                                                    ? (runner.toolLimit ?? 0)
+                                                    : (runner.limit ?? 0)), 0),
+                                            runners: loads.length,
+                                            reportedFree: loads.map((runner) => String(runner.reportedTotalFree ?? '—')),
+                                            detail: loads
+                                                .map((runner) => `${entry.tool} ${runner.toolUsed}/${runner.toolLimit}${runner.reportedTotalFree === 0 ? '; global free 0' : ''}`)
+                                                .join('; '),
+                                        }
+                                        : {}),
+                            },
+                        }
+                        : {}),
+                    position: position.get(run.id) ?? null,
+                    agentPosition: agentPosition.get(run.id) ?? 1,
+                }
+                : null,
+        };
+    }));
+    // Held runs first, the longest-running first; then queued ones in claim order.
+    const order = new Map(queued.map((run, index) => [run.id, index]));
+    runs.sort((a, b) => {
+        const aQueued = a.status === 'queued';
+        const bQueued = b.status === 'queued';
+        if (aQueued !== bQueued)
+            return aQueued ? 1 : -1;
+        if (aQueued)
+            return ((order.get(a.id) ?? Number.MAX_SAFE_INTEGER) -
+                (order.get(b.id) ?? Number.MAX_SAFE_INTEGER));
+        return 0;
+    });
+    const loads = [];
+    const agentsWithRuns = new Set(mine.map((run) => run.agentId));
+    for (const agent of agents) {
+        if (agent.status !== 'enabled' && !agentsWithRuns.has(agent.id))
+            continue;
+        loads.push({
+            agentId: agent.id,
+            active: agentActive.get(agent.id) ?? 0,
+            queued: agentQueued.get(agent.id) ?? 0,
+            maxConcurrentRuns: agent.maxConcurrentRuns,
+            online: agent.status === 'enabled' &&
+                online.some((runner) => agent.modelEntries.some((entry) => runsEntry(runner, entry))),
+        });
+    }
+    return {
+        runs,
+        truncated,
+        agents: loads,
+        runners: {
+            online: online.length,
+            busy: online.filter((runner) => (runnerUsed.get(runner.id) ?? 0) > 0)
+                .length,
+            slots: online.reduce((sum, runner) => sum + runner.slots, 0),
+            used: online.reduce((sum, runner) => sum + Math.min(runnerUsed.get(runner.id) ?? 0, runner.slots), 0),
+        },
+    };
+}
+//# sourceMappingURL=workload.js.map