@clear-capabilities/agentic-security-scanner 0.161.0 → 0.163.0

package/src/posture/portfolio/scheduler.js

@@ -0,0 +1,488 @@
+// Bounded portfolio scheduler: budgets, fairness and scoped cancellation (X-706).
+//
+// work-units.js gives durable, leased units but hands out the next pending unit in id order, with no notion of cost and no notion of
+// who is waiting. This module decides WHICH unit is leased next, and whether ANY unit may be leased at all, from five limits declared
+// at two levels (the whole portfolio, and each repository): wall time, provider spend, concurrency, request count and storage.
+//
+// ADMISSION (X-706.AC01). The check happens BEFORE the lease, inside the same locked store transaction that creates the lease, so two
+// processes cannot both squeeze under a limit. A unit must carry an ESTIMATE of what it can cost (an unknown cost is never zero: a unit
+// with no estimate is not leased, and is reported `no-estimate`). Admission counts what has been used, plus what in-flight attempts have
+// RESERVED, plus this unit's estimate. A refusal is one of two kinds, and they are reported separately:
+//   at-capacity  would fit once an in-flight attempt settles (a wait)
+//   exhausted    would not fit even with nothing in flight (will not recover without a larger budget)
+// An attempt that ends without a usage report (expired, failed, canceled) is charged its whole reservation, because what it spent is
+// unknown. A reported overrun of the estimate is recorded and shrinks what remains; it is detected, not rolled back. Wall time is also
+// enforced while the unit runs: the executor's signal aborts at the unit's wall estimate (capped by what the portfolio has left).
+//
+// FAIRNESS (X-706.AC02). Priority is a WEIGHT, not an order. Among repositories that currently have an admissible unit, the next lease
+// goes to the one with the lowest `leasesGranted / weight` (weights are integers 1..8, default 1; ties break on repository name, and
+// within a repository on unit id). Consequences, each tested:
+//   - no starvation: a repository that is admissible is served within a bounded number of leases whatever the others hold, so a
+//     repository with 300 units cannot hold back one with 1;
+//   - priority shifts the SHARE, it never removes it: weight 8 against weight 1 is served about eight times as often, and the
+//     weight-1 repository is still served;
+//   - a repository that was blocked for a long time is owed service and is served first when it is admissible again (a bounded burst);
+//   - a blocked repository (declared blocked, over its own budget, or with every unit blocked) is SKIPPED, never waited on. The skip
+//     reason is returned per repository, so a stall is visible instead of silent.
+// A unit whose estimate cannot fit the remaining budget is never leased and never blocks a smaller sibling.
+//
+// CANCELLATION (X-706.AC03). A cancellation names a scope (one unit, one repository, or everything) and is recorded in the ledger, so a
+// process that did not receive the call still honours it. For attempts running in THIS process the scope's AbortController is aborted;
+// work started through `ctx.spawn` runs under sandbox/supervise.js, which terminates the whole process tree and reports survivors. A
+// lease is RELEASED only when the attempt has settled and nothing survived; otherwise it is left to EXPIRE (so no other worker can start
+// the same unit while a descendant may still be running) and the report says so. Verified units and their receipts are never touched.
+// The report is always `incomplete` with a reason when anything in scope was not verified.
+//
+// SPEND (routing). `routingBudgetFor` gives the routing policy the remaining dollars (portfolio and repository, less reservations), and
+// `chargeModelSpend` refuses a routing decision that is blocked, has no upper cost bound, or would exceed the unit's reservation.
+// No provider is called here; the executor does that through an injected transport.
+//
+// Heartbeats are recorded with the lease renewal. Nothing here reads a clock except where a function takes `clock`/`now`.
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { superviseSpawn } from '../../sandbox/supervise.js';
+import {
+  leaseUnit, startUnit, completeUnit, failUnit, cancelUnits, recoverExpired, renewLease, mutateStore, readStore, progressOf, typedError,
+} from './work-units.js';
+
+export const LEDGER_SCHEMA = 'agentic-security/portfolio-ledger';
+const LEDGER_VERSION = 1;
+export const DIMENSIONS = Object.freeze(['wallMs', 'spendUsd', 'requests', 'storageBytes']);
+export const WEIGHT_RANGE = Object.freeze({ min: 1, max: 8, default: 1 });
+export const HEARTBEAT = Object.freeze({ intervalMs: 5000, staleAfterMs: 15000 });
+export const SKIP_CODES = Object.freeze(['repository-blocked', 'no-estimate', 'at-capacity', 'exhausted', 'canceled']);
+const MAX_LEDGER_BYTES = 16 * 1024 * 1024;
+
+const zero = () => ({ wallMs: 0, spendUsd: 0, requests: 0, storageBytes: 0 });
+const isNum = (v) => typeof v === 'number' && Number.isFinite(v) && v >= 0;
+
+// ---------------------------------------------------------------- budgets
+
+/**
+ * Budgets: `{ portfolio: { concurrency, wallMs, spendUsd, requests, storageBytes }, repositories?: { default?, [name]: partial limits },
+ * priorities?: { [name]: weight } }`. Every portfolio limit is REQUIRED (an unlimited portfolio is a decision to make out loud, not a
+ * default); a repository limit that is absent is not enforced for that repository and the view says so.
+ */
+export function validateBudgets(b) {
+  const errors = [];
+  const bad = (p, message) => errors.push({ code: 'BAD_BUDGETS', path: p, message });
+  if (!b || typeof b !== 'object') return { ok: false, errors: [{ code: 'BAD_BUDGETS', path: '', message: 'not an object' }] };
+  const lim = (o, p, required) => {
+    if (!o || typeof o !== 'object') { if (required) bad(p, 'required'); return; }
+    if (o.concurrency !== undefined && !(Number.isInteger(o.concurrency) && o.concurrency >= 1 && o.concurrency <= 64)) bad(`${p}.concurrency`, 'an integer in 1..64');
+    else if (required && o.concurrency === undefined) bad(`${p}.concurrency`, 'required');
+    for (const d of DIMENSIONS) {
+      if (o[d] === undefined) { if (required) bad(`${p}.${d}`, 'required'); } else if (!isNum(o[d])) bad(`${p}.${d}`, 'a finite non-negative number');
+    }
+  };
+  lim(b.portfolio, 'portfolio', true);
+  for (const [name, o] of Object.entries(b.repositories ?? {})) lim(o, `repositories.${name}`, false);
+  for (const [name, w] of Object.entries(b.priorities ?? {})) {
+    if (!(Number.isInteger(w) && w >= WEIGHT_RANGE.min && w <= WEIGHT_RANGE.max)) bad(`priorities.${name}`, `an integer weight in ${WEIGHT_RANGE.min}..${WEIGHT_RANGE.max}`);
+  }
+  return { ok: errors.length === 0, errors };
+}
+
+const limitsFor = (budgets, repo) => ({ ...(budgets.repositories?.default ?? {}), ...(budgets.repositories?.[repo] ?? {}) });
+const weightOf = (budgets, repo) => budgets.priorities?.[repo] ?? WEIGHT_RANGE.default;
+
+// ---------------------------------------------------------------- ledger
+
+export const ledgerPathFor = (storeFile) => `${storeFile}.ledger.json`;
+
+export function newLedger() {
+  return { schema: LEDGER_SCHEMA, version: LEDGER_VERSION, usage: { portfolio: zero(), repositories: {} }, reservations: {}, grants: {}, heartbeats: {}, cancellations: [], overruns: [], settled: {} };
+}
+
+export function readLedger(storeFile) {
+  const file = ledgerPathFor(storeFile);
+  let raw;
+  try {
+    const st = fs.lstatSync(file);
+    if (st.isSymbolicLink() || !st.isFile() || st.size > MAX_LEDGER_BYTES) throw typedError('LEDGER_CORRUPT', 'ledger is not a regular file within the size limit');
+    raw = fs.readFileSync(file, 'utf8');
+  } catch (e) { if (e.code === 'ENOENT') return newLedger(); throw e; }
+  let l;
+  try { l = JSON.parse(raw); } catch { throw typedError('LEDGER_CORRUPT', 'ledger is not valid JSON'); }
+  if (!l || l.schema !== LEDGER_SCHEMA || l.version !== LEDGER_VERSION || typeof l.reservations !== 'object' || typeof l.usage !== 'object') throw typedError('LEDGER_CORRUPT', 'not a portfolio ledger of a supported version');
+  return l;
+}
+
+function writeLedger(storeFile, ledger) {
+  const file = ledgerPathFor(storeFile);
+  const tmp = `${file}.tmp-${process.pid}`;
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(tmp, JSON.stringify(ledger), { mode: 0o600 });
+  fs.renameSync(tmp, file);
+}
+
+const repoUsage = (ledger, repo) => (ledger.usage.repositories[repo] ??= zero());
+const add = (a, b) => { for (const d of DIMENSIONS) a[d] += b[d] ?? 0; };
+
+/** Charge an attempt's reservation to usage and release it. `usage` null charges the whole reservation (what it spent is unknown). Idempotent. */
+function settle(ledger, attemptId, usage, how) {
+  const r = ledger.reservations[attemptId];
+  if (!r) return null;
+  const charge = {};
+  const over = [];
+  for (const d of DIMENSIONS) {
+    charge[d] = usage && isNum(usage[d]) ? usage[d] : r[d];
+    if (usage && isNum(usage[d]) && usage[d] > r[d]) over.push(d);
+  }
+  add(ledger.usage.portfolio, charge); add(repoUsage(ledger, r.repository), charge);
+  if (over.length) ledger.overruns.push({ attemptId, unitId: r.unitId, repository: r.repository, dimensions: over });
+  delete ledger.reservations[attemptId];
+  delete ledger.heartbeats[attemptId];
+  ledger.settled[attemptId] = how;
+  return { charge, over };
+}
+
+const inScope = (unit, scope) => !!scope && (scope.all === true || (scope.unitId && scope.unitId === unit.id) || (scope.repository && scope.repository === unit.repository));
+
+/** Settle reservations whose lease ended and cancel pending units inside a recorded cancellation scope. Returns what changed. */
+function reconcile(store, ledger, now) {
+  const recovered = recoverExpired(store, now);
+  const settledNow = [];
+  for (const [attemptId, r] of Object.entries(ledger.reservations)) {
+    const u = store.units[r.unitId];
+    if (!u || !u.lease || u.lease.attemptId !== attemptId) { settle(ledger, attemptId, null, 'lease-ended'); settledNow.push(attemptId); }
+  }
+  const canceled = [];
+  for (const c of ledger.cancellations) {
+    for (const u of Object.values(store.units)) {
+      if ((u.state === 'pending' || u.state === 'blocked') && inScope(u, c.scope)) { cancelUnits(store, { unitId: u.id, reason: c.reason, now }); canceled.push(u.id); }
+    }
+  }
+  return { recovered, settled: settledNow, canceled };
+}
+
+// ---------------------------------------------------------------- admission and selection
+
+function checkLimits(limits, used, flight, flightCount, est, scope) {
+  const out = [];
+  if (Number.isInteger(limits.concurrency) && flightCount >= limits.concurrency) out.push({ code: 'at-capacity', scope, dimension: 'concurrency', limit: limits.concurrency, inFlight: flightCount });
+  for (const d of DIMENSIONS) {
+    if (!isNum(limits[d])) continue;
+    if (used[d] + est[d] > limits[d]) out.push({ code: 'exhausted', scope, dimension: d, limit: limits[d], used: used[d], needed: est[d] });
+    else if (used[d] + flight[d] + est[d] > limits[d]) out.push({ code: 'at-capacity', scope, dimension: d, limit: limits[d], used: used[d], reserved: flight[d], needed: est[d] });
+  }
+  return out;
+}
+
+function normalizeEstimate(raw) {
+  if (!raw || typeof raw !== 'object') return null;
+  const e = {};
+  for (const d of DIMENSIONS) { if (!isNum(raw[d])) return null; e[d] = raw[d]; }
+  return e.wallMs > 0 ? e : null;
+}
+
+/** What each repository and the portfolio currently have reserved by in-flight attempts. */
+function flightOf(ledger) {
+  const portfolio = { ...zero(), count: 0 };
+  const repos = {};
+  for (const r of Object.values(ledger.reservations)) {
+    const rr = (repos[r.repository] ??= { ...zero(), count: 0 });
+    for (const d of DIMENSIONS) { portfolio[d] += r[d]; rr[d] += r[d]; }
+    portfolio.count += 1; rr.count += 1;
+  }
+  return { portfolio, repos };
+}
+
+/**
+ * Pure selection over a reconciled store and ledger: the unit that would be leased next, or why none can be.
+ * @returns {{ unit: object|null, estimate: object|null, skipped: Array, idle: boolean, waiting: boolean }}
+ */
+export function selectUnit({ store, ledger, budgets, estimateOf, blockedRepositories = [] }) {
+  const flight = flightOf(ledger);
+  const pendingByRepo = new Map();
+  for (const u of Object.values(store.units).sort((a, b) => (a.id < b.id ? -1 : 1))) {
+    if (u.state === 'pending') { if (!pendingByRepo.has(u.repository)) pendingByRepo.set(u.repository, []); pendingByRepo.get(u.repository).push(u); }
+  }
+  const skipped = [];
+  const blocked = new Set(blockedRepositories);
+  const candidates = [];
+  for (const [repo, units] of [...pendingByRepo].sort(([a], [b]) => (a < b ? -1 : 1))) {
+    if (blocked.has(repo)) { skipped.push({ repository: repo, code: 'repository-blocked', detail: 'declared blocked; its units are skipped and independent repositories continue' }); continue; }
+    const repoFlight = flight.repos[repo] ?? { ...zero(), count: 0 };
+    const repoLimits = limitsFor(budgets, repo);
+    let pick = null; const unitSkips = [];
+    for (const u of units) {
+      const est = normalizeEstimate(estimateOf(u));
+      if (!est) { unitSkips.push({ repository: repo, unitId: u.id, code: 'no-estimate', detail: 'no cost estimate, and an unknown cost is never treated as zero' }); continue; }
+      const refusals = [
+        ...checkLimits(budgets.portfolio, ledger.usage.portfolio, flight.portfolio, flight.portfolio.count, est, 'portfolio'),
+        ...checkLimits(repoLimits, repoUsage(ledger, repo), repoFlight, repoFlight.count, est, 'repository'),
+      ];
+      if (refusals.length) {
+        const code = refusals.some((r) => r.code === 'exhausted') ? 'exhausted' : 'at-capacity';
+        unitSkips.push({ repository: repo, unitId: u.id, code, detail: refusals.map((r) => `${r.scope} ${r.dimension}`).join(', '), refusals });
+        continue;
+      }
+      pick = { unit: u, estimate: est };
+      break;
+    }
+    skipped.push(...unitSkips.filter((s) => !pick || s.unitId !== pick.unit.id));
+    if (pick) candidates.push({ repo, ...pick });
+  }
+  let chosen = null;
+  for (const c of candidates) {
+    const score = (ledger.grants[c.repo] ?? 0) / weightOf(budgets, c.repo);
+    if (!chosen || score < chosen.score) chosen = { ...c, score };
+  }
+  const idle = pendingByRepo.size === 0;
+  return { unit: chosen ? chosen.unit : null, estimate: chosen ? chosen.estimate : null, skipped, idle, waiting: !chosen && skipped.some((s) => s.code === 'at-capacity') };
+}
+
+/**
+ * Admit and lease the next unit in ONE locked transaction. Returns `{ lease, ... }` or `{ lease: null, idle, waiting, skipped }`.
+ * A repeat `requestId` from the same holder returns the existing lease and reserves nothing more.
+ */
+export function scheduleNext(file, { budgets, estimateOf, holder, now, ttlMs, requestId = null, blockedRepositories = [] }) {
+  const bv = validateBudgets(budgets);
+  if (!bv.ok) throw typedError('BAD_BUDGETS', `invalid budgets: ${bv.errors[0].path} ${bv.errors[0].message}`, { errors: bv.errors });
+  return mutateStore(file, (store) => {
+    const ledger = readLedger(file);
+    const changed = reconcile(store, ledger, now);
+    const finish = (out) => { writeLedger(file, ledger); return { ...out, reconciled: changed }; };
+    if (requestId) {
+      for (const u of Object.values(store.units)) {
+        if (u.lease && u.lease.requestId === requestId && u.lease.holder === holder) {
+          return finish({ lease: { unitId: u.id, attemptId: u.lease.attemptId, expiresAt: u.lease.expiresAt, repository: u.repository, duplicate: true }, idle: false, waiting: false, skipped: [] });
+        }
+      }
+    }
+    const sel = selectUnit({ store, ledger, budgets, estimateOf, blockedRepositories });
+    if (!sel.unit) return finish({ lease: null, idle: sel.idle, waiting: sel.waiting, skipped: sel.skipped });
+    const l = leaseUnit(store, { holder, now, ttlMs, unitId: sel.unit.id, requestId });
+    ledger.reservations[l.attemptId] = { unitId: sel.unit.id, repository: sel.unit.repository, holder, at: now, ...sel.estimate, used: { spendUsd: 0, requests: 0 } };
+    ledger.grants[sel.unit.repository] = (ledger.grants[sel.unit.repository] ?? 0) + 1;
+    return finish({ lease: { ...l, repository: sel.unit.repository, reservation: sel.estimate }, idle: false, waiting: false, skipped: sel.skipped });
+  });
+}
+
+/** Settle an attempt's reservation (idempotent). `usage` null charges the whole reservation. */
+export function settleAttempt(file, { attemptId, usage = null, how = 'reported' }) {
+  return mutateStore(file, () => {
+    const ledger = readLedger(file);
+    const r = settle(ledger, attemptId, usage, how);
+    writeLedger(file, ledger);
+    return r;
+  });
+}
+
+/** Renew the lease and record a heartbeat in one transaction. */
+export function heartbeat(file, { unitId, attemptId, now, ttlMs }) {
+  return mutateStore(file, (store) => {
+    const ledger = readLedger(file);
+    const r = renewLease(store, { unitId, attemptId, now, ttlMs });
+    if (r.ok) {
+      const u = store.units[unitId];
+      ledger.heartbeats[attemptId] = { holder: u.lease.holder, unitId, repository: u.repository, at: now };
+      writeLedger(file, ledger);
+    }
+    return r;
+  });
+}
+
+// ---------------------------------------------------------------- routing and spend
+
+/** What the routing policy may spend for a task in `repository` right now: the smaller of the portfolio and repository remainders, less reservations. */
+export function routingBudgetFor({ budgets, ledger, repository }) {
+  const flight = flightOf(ledger);
+  const left = (limit, used, reserved) => (isNum(limit) ? Math.max(0, limit - used - reserved) : Infinity);
+  const p = left(budgets.portfolio.spendUsd, ledger.usage.portfolio.spendUsd, flight.portfolio.spendUsd);
+  const r = left(limitsFor(budgets, repository).spendUsd, repoUsage(ledger, repository).spendUsd, (flight.repos[repository] ?? zero()).spendUsd);
+  const remainingUsd = Math.min(p, r);
+  return { remainingUsd: Number.isFinite(remainingUsd) ? remainingUsd : 0 };
+}
+
+/** The dollar upper bound of a routing decision, or null when the decision selects nothing or carries no bound. Unknown is never free. */
+export function spendBoundOf(decision) {
+  if (!decision || (decision.status !== 'routed' && decision.status !== 'fallback') || !decision.selected) return null;
+  const c = decision.expectedBounds?.costUsd;
+  const bound = Math.max(isNum(c?.upperBound) ? c.upperBound : 0, isNum(c?.measuredMedian) ? c.measuredMedian : 0);
+  return bound > 0 ? bound : null;
+}
+
+/** Charge one model request (a routing decision) against the attempt's reservation. Refuses when blocked, unbounded or over the reservation. */
+export function chargeModelSpend(file, { attemptId, decision }) {
+  return mutateStore(file, () => {
+    const ledger = readLedger(file);
+    const r = ledger.reservations[attemptId];
+    if (!r) return { ok: false, code: 'no-reservation', reason: 'this attempt holds no reservation (it ended, or was never admitted)' };
+    const bound = spendBoundOf(decision);
+    if (bound === null) return { ok: false, code: 'no-spend-bound', reason: 'the routing decision selects nothing or carries no cost bound; an unknown cost is not charged as zero and the request is not made' };
+    if (r.used.requests + 1 > r.requests) return { ok: false, code: 'request-limit', reason: `the unit reserved ${r.requests} request(s)` };
+    if (r.used.spendUsd + bound > r.spendUsd) return { ok: false, code: 'unit-spend-exceeded', reason: `needs up to ${bound} against ${r.spendUsd - r.used.spendUsd} remaining in the unit's reservation` };
+    r.used.spendUsd += bound; r.used.requests += 1;
+    writeLedger(file, ledger);
+    return { ok: true, charged: bound };
+  });
+}
+
+// ---------------------------------------------------------------- cancellation
+
+/** Tracks the attempts running in this process so a scope can be cancelled. */
+export function createCancelScope() {
+  const running = new Map();
+  return {
+    running,
+    register(attemptId, entry) { running.set(attemptId, entry); },
+    unregister(attemptId) { running.delete(attemptId); },
+    /**
+     * Cancel everything in `scope` ({ unitId } | { repository } | { all: true }).
+     * @returns {Promise<object>} an `incomplete` report with the reason, what was released and what is left to expire
+     */
+    async cancel({ file, scope, reason, now, waitMs = 5000 }) {
+      if (!scope || !(scope.all === true || scope.unitId || scope.repository)) throw typedError('BAD_SCOPE', 'a cancellation needs a scope: unitId, repository or all');
+      const why = String(reason ?? 'canceled').slice(0, 300);
+      mutateStore(file, (store) => {
+        const ledger = readLedger(file);
+        ledger.cancellations.push({ scope: { ...scope }, reason: why, at: now });
+        reconcile(store, ledger, now);
+        writeLedger(file, ledger);
+      });
+      const mine = [...running.entries()].filter(([, e]) => inScope({ id: e.unitId, repository: e.repository }, scope));
+      for (const [, e] of mine) e.controller.abort(Object.assign(new Error(why), { code: 'CANCELED' }));
+      await Promise.race([Promise.all(mine.map(([, e]) => e.settled)), new Promise((r) => setTimeout(r, waitMs))]);
+      return mutateStore(file, (store) => {
+        const ledger = readLedger(file);
+        const report = { status: 'nothing-to-cancel', incomplete: false, reason: why, scope: { ...scope }, canceled: [], leasesReleased: [], leasesLeftToExpire: [], verifiedPreserved: [], survivors: [] };
+        for (const id of Object.keys(store.units).sort()) {
+          const u = store.units[id];
+          if (!inScope(u, scope)) continue;
+          if (u.state === 'verified') { report.verifiedPreserved.push(id); continue; }
+          if (u.state === 'leased' || u.state === 'running') {
+            const attemptId = u.lease.attemptId;
+            const e = running.get(attemptId) ?? mine.find(([a]) => a === attemptId)?.[1];
+            if (e && e.finished === true && e.survivors.length === 0) {
+              cancelUnits(store, { unitId: id, reason: why, now });
+              settle(ledger, attemptId, null, 'canceled');
+              report.leasesReleased.push(id); report.canceled.push(id);
+            } else {
+              report.leasesLeftToExpire.push({ unitId: id, attemptId, expiresAt: u.lease.expiresAt, why: e ? (e.survivors.length ? 'a descendant survived termination' : 'the attempt did not settle in time') : 'the attempt runs in another process' });
+              if (e?.survivors.length) report.survivors.push({ unitId: id, pids: e.survivors });
+            }
+          } else if (u.state === 'canceled') report.canceled.push(id);
+        }
+        if (report.canceled.length || report.leasesLeftToExpire.length) { report.status = 'incomplete'; report.incomplete = true; }
+        writeLedger(file, ledger);
+        return report;
+      });
+    },
+  };
+}
+
+// ---------------------------------------------------------------- the driver
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * Drive a store with budgets. `executor(unit, ctx)` returns `{ resultDigest, dependencies, usage? }` where `usage` may report
+ * `{ spendUsd, requests, storageBytes }` (wall time is measured). `ctx` carries `attemptId`, `signal`, `heartbeat()`, `reservation`,
+ * `spawn(bin, args, opts)` (supervised: the process tree ends with the attempt) and `requestModel(decision)` (charges the routing decision).
+ *
+ * With `backend`, the backend is probed before every lease; an unavailable one stops the run with `stopped: 'blocked'` and the typed probe
+ * result in `blocked`. @returns {Promise<{ progress: object|null, stopped: string, order: string[], skipped: Array, blocked?: object }>}
+ */
+export async function runScheduled({
+  file, budgets, estimateOf, executor, holder = 'worker', clock = () => Date.now(), ttlMs, maxLeases = Infinity, cancelScope = null,
+  blockedRepositories = [], heartbeatMs = HEARTBEAT.intervalMs, pollMs = 25, maxWaitMs = 2000, workers = null, backend = null,
+}) {
+  const bv = validateBudgets(budgets);
+  if (!bv.ok) throw typedError('BAD_BUDGETS', `invalid budgets: ${bv.errors[0].path} ${bv.errors[0].message}`, { errors: bv.errors });
+  const order = [];
+  let leased = 0; let stopped = 'drained'; let lastSkipped = []; let blocked = null;
+  const limit = Math.max(1, Math.min(workers ?? budgets.portfolio.concurrency, 16));
+  let inFlight = 0;
+
+  async function worker(n) {
+    let waited = 0;
+    for (;;) {
+      if (leased >= maxLeases) { stopped = 'max-leases'; return; }
+      // a shared backend that went away stops the run: nothing is written anywhere else instead (X-708.AC03)
+      if (backend) { const p = backend.probe(); if (!p.ok) { stopped = 'blocked'; blocked = p; return; } }
+      let r;
+      try { r = scheduleNext(file, { budgets, estimateOf, holder: `${holder}-${n}`, now: clock(), ttlMs, blockedRepositories }); } catch (e) {
+        if (backend && !backend.probe().ok) { stopped = 'blocked'; blocked = backend.probe(); return; }
+        throw e;
+      }
+      lastSkipped = r.skipped;
+      if (!r.lease) {
+        if (r.idle) return;
+        if (r.waiting) {
+          if (inFlight === 0 && waited >= maxWaitMs) { stopped = 'waiting-timeout'; return; }
+          await sleep(pollMs); waited += pollMs; continue;
+        }
+        stopped = 'nothing-schedulable'; return;
+      }
+      waited = 0;
+      leased += 1; inFlight += 1;
+      order.push(r.lease.unitId);
+      try { await runOne(r.lease); } catch (e) {
+        // the attempt could not record its outcome because the backend is gone: its lease is left to expire, never re-homed
+        if (backend && !backend.probe().ok) { stopped = 'blocked'; blocked = backend.probe(); return; }
+        throw e;
+      } finally { inFlight -= 1; }
+    }
+  }
+
+  async function runOne(lease) {
+    const { unitId, attemptId } = lease;
+    const controller = new AbortController();
+    const entry = { unitId, repository: lease.repository, controller, finished: false, survivors: [], terminations: [], settled: null };
+    let resolveSettled; entry.settled = new Promise((r) => { resolveSettled = r; });
+    cancelScope?.register(attemptId, entry);
+    const startedAt = clock();
+    let timer = null; let beat = null;
+    try {
+      mutateStore(file, (s) => startUnit(s, { unitId, attemptId, now: clock() }));
+      const unit = readStore(file).units[unitId];
+      const wallCap = lease.reservation.wallMs;
+      timer = setTimeout(() => controller.abort(Object.assign(new Error('wall-time budget for the unit was reached'), { code: 'WALL_TIMEOUT' })), wallCap);
+      const doBeat = () => { try { heartbeat(file, { unitId, attemptId, now: clock(), ttlMs }); } catch { /* the lease is gone; the result will be refused */ } };
+      beat = setInterval(doBeat, heartbeatMs); doBeat();
+      const ctx = {
+        attemptId, signal: controller.signal, reservation: lease.reservation, heartbeat: doBeat,
+        async spawn(bin, args, opts = {}) {
+          const r = await superviseSpawn(bin, args, { ...opts, signal: controller.signal });
+          entry.terminations.push(r.termination);
+          if (r.termination?.survivors?.length) entry.survivors.push(...r.termination.survivors);
+          return r;
+        },
+        requestModel: (decision) => chargeModelSpend(file, { attemptId, decision }),
+      };
+      let res = null; let err = null;
+      try { res = await executor(unit, ctx); } catch (e) { err = e; }
+      clearTimeout(timer); clearInterval(beat);
+      const canceled = controller.signal.aborted && controller.signal.reason?.code === 'CANCELED';
+      if (canceled) return; // cancel() performs the store transition once this attempt has settled
+      const timedOut = controller.signal.aborted && controller.signal.reason?.code === 'WALL_TIMEOUT';
+      const spent = mutateStore(file, (s) => {
+        const ledger = readLedger(file);
+        const tracked = ledger.reservations[attemptId]?.used ?? { spendUsd: 0, requests: 0 };
+        const failed = err || !res || timedOut;
+        // a failed attempt spent an unknown amount: it is charged its reservation; a reported success is charged what it reported
+        const usage = failed ? null : { wallMs: Math.max(1, clock() - startedAt), spendUsd: tracked.spendUsd, requests: tracked.requests, ...(res.usage ?? {}) };
+        settle(ledger, attemptId, usage, failed ? 'failed' : 'reported');
+        writeLedger(file, ledger);
+        if (failed) failUnit(s, { unitId, attemptId, reason: timedOut ? 'wall-time budget reached' : err ? err.message : 'executor returned nothing', now: clock() });
+        else completeUnit(s, { unitId, attemptId, resultDigest: res.resultDigest, dependencies: res.dependencies, now: clock() });
+        return true;
+      });
+      void spent;
+    } finally {
+      clearTimeout(timer); clearInterval(beat);
+      entry.finished = true;
+      resolveSettled();
+      cancelScope?.unregister(attemptId);
+    }
+  }
+
+  await Promise.all(Array.from({ length: limit }, (_, i) => worker(i)));
+  let progress = null;
+  try { progress = progressOf(readStore(file)); } catch (e) { if (!blocked) throw e; }
+  return { progress, stopped, order, skipped: lastSkipped, ...(blocked ? { blocked } : {}) };
+}
+

package/src/posture/portfolio/signing.js

@@ -0,0 +1,155 @@
+// Sign and verify bounded assurance claims (X-703).
+//
+// WHAT IS SIGNED. Not the manifest bytes alone: a small claim that names the manifest id and digest, the portable bundle digest, the
+// subject commit, the blocking-finding policy, the coverage counts and the bounded headline, plus the signer identity, key id and the
+// assurance policy the signer applied. Verification re-derives every one of those from the bundle on disk, so a signature cannot be
+// carried over to different evidence.
+//
+// WHO MAY SIGN (X-703.AC01). Signing is a SIGNER-domain act (sandbox/trust-domains.js, default deny): a worker, target or verifier
+// domain asking to sign is refused before any key is touched. The key lives in its own directory (`assurance-signing/` beside, never
+// inside, the scan evidence keys) so the assurance domain has a separate key identity, and the signer refuses a key whose id equals the
+// evidence key it is told to stay independent of. The key is read from that directory by the signer; nothing here returns, logs or
+// embeds private key material, and `signAssuranceClaim` checks its own output for a PEM block before returning it.
+//
+// WHAT VERIFICATION DOES (X-703.AC02). Offline, with a trust-root policy the verifier is handed: unknown roots, revoked roots, a trust
+// basis this build cannot verify, a tampered claim, a tampered or missing bundle, and a claim that does not match the bundle's manifest
+// are each a typed failure. The only basis this build can verify is `self-issued-local-key`; a policy that names any other basis is
+// rejected, never displayed as if it were certification.
+//
+// WHAT IT NEVER CLAIMS (X-703.AC03). `independentlyCertified` is always false and is part of the signed bytes. A valid signature shows
+// the manifest and bundle are unmodified since the named key signed them. It is not third-party certification and not a statement that
+// no vulnerability exists.
+
+import * as crypto from 'node:crypto';
+import * as path from 'node:path';
+import { canonicalJson, ensureKeyPair, keyFingerprint, TRUST_BASES } from '../evidence-bundle.js';
+import { RESOURCES, mayDo, keyDirectory } from '../../sandbox/trust-domains.js';
+import { digestOf } from '../assurance/identity.js';
+import { validateManifest, manifestDigest } from './manifest.js';
+import { verifyBundle } from './bundle.js';
+import { assuranceStatement, HEADLINE } from './wording.js';
+
+const CLAIM_SCHEMA = 'agentic-security/assurance-claim';
+const CLAIM_VERSION = '1.0.0';
+const TRUST_POLICY_SCHEMA = 'agentic-security/assurance-trust-roots';
+const ASSURANCE_POLICY = Object.freeze({
+  id: 'bounded-claim-v1',
+  statement: `${HEADLINE}, with visible scope and gaps. Not a statement of safety, vulnerability absence or independent certification.`,
+});
+export const SIGNING_DIR_NAME = 'assurance-signing';
+
+/** The signing key directory for the assurance domain: separate from the scan evidence keys. */
+export function assuranceKeyDirectory(env = process.env) { return path.join(keyDirectory(env), SIGNING_DIR_NAME); }
+
+/** Load, or create on first use, the assurance signing key pair. Only the signer domain may call this. */
+export function ensureAssuranceKeys({ domain, dir = assuranceKeyDirectory() } = {}) {
+  if (!mayDo(domain, RESOURCES.SIGNING_KEY, 'read')) throw signerRefusal(domain);
+  const k = ensureKeyPair(dir);
+  return { privateKeyPem: k.privateKeyPem, publicKeyPem: k.publicKeyPem, keyId: keyFingerprint(k.publicKeyPem), created: k.created };
+}
+
+function signerRefusal(domain) {
+  return Object.assign(new Error(`the '${domain}' domain may not sign assurance claims: signing is a signer-domain act and the key is out of its reach`), { code: 'SIGNER_NOT_AUTHORIZED' });
+}
+
+/**
+ * Sign a bounded claim for a verified bundle. Throws a typed error (`code`) on a refused domain, an invalid manifest, a bundle that does
+ * not verify, or a key that is not independent of `independentOfKeyId`.
+ *
+ * @param {object} a
+ * @param {string} a.domain          the calling trust domain (must be 'signer')
+ * @param {string} a.bundleDir       an exported portable bundle
+ * @param {string} a.privateKeyPem   the signer's Ed25519 key
+ * @param {string} a.signerId        stable signer identity, e.g. 'release-signer@org'
+ * @param {string} [a.independentOfKeyId]  fingerprint of a key (the scan evidence key) this domain must not share
+ */
+export function signAssuranceClaim({ domain, bundleDir, privateKeyPem, signerId, independentOfKeyId = null }) {
+  if (!mayDo(domain, RESOURCES.SIGNING_KEY, 'read')) throw signerRefusal(domain);
+  if (typeof signerId !== 'string' || !signerId.trim()) throw Object.assign(new Error('a signer identity is required'), { code: 'SIGNER_ID_REQUIRED' });
+  const vb = verifyBundle(bundleDir);
+  if (!vb.ok || !vb.manifest) throw Object.assign(new Error(`refusing to sign: the bundle does not verify (${vb.errors[0]?.code ?? 'no manifest'})`), { code: 'BUNDLE_INVALID', errors: vb.errors });
+  const vm = validateManifest(vb.manifest);
+  if (!vm.ok) throw Object.assign(new Error('refusing to sign: the manifest is invalid'), { code: 'MANIFEST_INVALID' });
+  let publicKeyPem;
+  try { publicKeyPem = crypto.createPublicKey(privateKeyPem).export({ type: 'spki', format: 'pem' }); } catch {
+    throw Object.assign(new Error('the signing key could not be parsed'), { code: 'BAD_KEY' });
+  }
+  const keyId = keyFingerprint(publicKeyPem);
+  if (independentOfKeyId && keyId === independentOfKeyId) throw Object.assign(new Error('the assurance signing key must be independent of the scan evidence key'), { code: 'KEY_NOT_INDEPENDENT' });
+
+  const m = vb.manifest;
+  const body = {
+    schema: CLAIM_SCHEMA, schemaVersion: CLAIM_VERSION,
+    claim: {
+      manifestId: m.id, manifestDigest: manifestDigest(m), bundleDigest: vb.bundleDigest,
+      subject: { repository: m.subject.repository, commit: m.subject.commit },
+      blockingPolicy: { id: m.policy.id, digest: m.policy.digest, blockingSeverity: m.policy.blockingSeverity },
+      blockingFindings: m.findings.blocking, coverage: m.coverage, complete: m.complete,
+      headline: assuranceStatement(m).headline,
+      ...(m.synthetic === true ? { synthetic: true } : {}),
+    },
+    signer: { id: signerId, keyId },
+    assurancePolicy: { id: ASSURANCE_POLICY.id, digest: digestOf(ASSURANCE_POLICY), statement: ASSURANCE_POLICY.statement },
+    issuance: { trustBasis: 'self-issued-local-key', independentlyCertified: false, statement: TRUST_BASES['self-issued-local-key'].statement },
+  };
+  const value = crypto.sign(null, Buffer.from(canonicalJson(body)), privateKeyPem).toString('base64');
+  const envelope = { ...body, signature: { algorithm: 'ed25519', value } };
+  if (/PRIVATE KEY/.test(JSON.stringify(envelope))) throw Object.assign(new Error('private key material reached the output; refusing to return it'), { code: 'KEY_LEAK' });
+  return envelope;
+}
+
+/** A trust-root policy naming the keys a verifier accepts. Only `self-issued-local-key` is a basis this build can verify. */
+export function buildTrustPolicy({ roots = [], revoked = [] } = {}) {
+  return {
+    schema: TRUST_POLICY_SCHEMA, schemaVersion: '1.0.0',
+    roots: roots.map((r) => ({ keyId: keyFingerprint(r.publicKeyPem), signerId: r.signerId, basis: r.basis ?? 'self-issued-local-key', publicKeyPem: r.publicKeyPem })),
+    revoked: revoked.map((r) => ({ keyId: r.keyId, reason: r.reason ?? 'revoked' })),
+  };
+}
+
+const fail = (code, reason, extra = {}) => ({ ok: false, code, reason, independentlyCertified: false, ...extra });
+
+/**
+ * Verify a signed claim offline against a trust-root policy and the bundle it names. Never throws. Returns `{ ok, ... }`; on failure
+ * `code` is one of: NOT_A_CLAIM, UNSUPPORTED_ALGORITHM, NO_TRUST_POLICY, UNKNOWN_TRUST_ROOT, REVOKED_TRUST_ROOT, UNSUPPORTED_BASIS,
+ * BAD_TRUST_ROOT, OVER_CLAIM, SIGNATURE_INVALID, BUNDLE_INVALID, EVIDENCE_MISMATCH.
+ */
+export function verifyAssuranceClaim({ envelope, trustPolicy, bundleDir }) {
+  try {
+    if (!envelope || envelope.schema !== CLAIM_SCHEMA || !envelope.claim || !envelope.signer || !envelope.signature || !envelope.issuance) return fail('NOT_A_CLAIM', 'not a signed assurance claim');
+    if (envelope.signature.algorithm !== 'ed25519') return fail('UNSUPPORTED_ALGORITHM', `unsupported algorithm: ${envelope.signature.algorithm}`);
+    if (!trustPolicy || trustPolicy.schema !== TRUST_POLICY_SCHEMA || !Array.isArray(trustPolicy.roots)) return fail('NO_TRUST_POLICY', 'no usable trust-root policy was supplied; a signature is meaningless without one');
+    const keyId = envelope.signer.keyId;
+    const revoked = (trustPolicy.revoked ?? []).find((r) => r.keyId === keyId);
+    if (revoked) return fail('REVOKED_TRUST_ROOT', `key ${String(keyId).slice(0, 12)} is revoked: ${revoked.reason}`);
+    const root = trustPolicy.roots.find((r) => r.keyId === keyId);
+    if (!root) return fail('UNKNOWN_TRUST_ROOT', `key ${String(keyId).slice(0, 12)} is not a trust root in this policy`);
+    if (!TRUST_BASES[root.basis]) return fail('UNSUPPORTED_BASIS', `trust basis '${root.basis}' cannot be verified by this build; it is rejected, not displayed as certification`);
+    if (keyFingerprint(root.publicKeyPem) !== root.keyId) return fail('BAD_TRUST_ROOT', 'the trust root public key does not match its key id');
+    if (envelope.issuance.independentlyCertified !== false || envelope.issuance.trustBasis !== root.basis) return fail('OVER_CLAIM', 'a self-issued claim may not state independent certification, and its basis must match the trust root');
+
+    const { signature, ...body } = envelope;
+    let good = false;
+    try { good = crypto.verify(null, Buffer.from(canonicalJson(body)), root.publicKeyPem, Buffer.from(signature.value, 'base64')); } catch { good = false; }
+    if (!good) return fail('SIGNATURE_INVALID', 'the signature does not match the claim: it was modified after signing, or signed by another key');
+
+    const vb = verifyBundle(bundleDir);
+    if (!vb.ok || !vb.manifest) return fail('BUNDLE_INVALID', `the evidence bundle does not verify: ${vb.errors[0]?.code ?? 'no manifest'}`, { errors: vb.errors });
+    const m = vb.manifest;
+    const c = envelope.claim;
+    const mismatch = [];
+    if (c.bundleDigest !== vb.bundleDigest) mismatch.push('bundleDigest');
+    if (c.manifestDigest !== manifestDigest(m)) mismatch.push('manifestDigest');
+    if (c.manifestId !== m.id) mismatch.push('manifestId');
+    if (c.subject?.commit !== m.subject.commit) mismatch.push('subject.commit');
+    if (digestOf(c.coverage) !== digestOf(m.coverage)) mismatch.push('coverage');
+    if (c.blockingFindings !== m.findings.blocking) mismatch.push('blockingFindings');
+    if (c.blockingPolicy?.digest !== m.policy.digest) mismatch.push('blockingPolicy');
+    if (mismatch.length) return fail('EVIDENCE_MISMATCH', `the signed claim does not match the evidence: ${mismatch.join(', ')}`, { mismatch });
+
+    const verified = { ok: true, trustBasis: root.basis, independentlyCertified: false, signer: envelope.signer.id, keyId, assurancePolicy: envelope.assurancePolicy?.id ?? null, replay: vb.replay };
+    return { ...verified, statement: assuranceStatement(m, { signature: verified }).text, note: TRUST_BASES[root.basis].statement };
+  } catch (e) {
+    return fail('NOT_A_CLAIM', `verification could not complete: ${String(e?.message ?? e).split('\n')[0]}`);
+  }
+}