@danceiny/gotry 0.2.0-rc.31 → 0.2.0-rc.33

package/dist/src/tenant-seam/process-governor.js

@@ -0,0 +1,281 @@
+/**
+ * Resource limits for the child process trees that run model-driven code on a replica
+ * (docs/rfc/aaas-rfc.md §4.8, threat S-17).
+ *
+ * The tenant boundary of the state layer says nothing about CPU and memory: every tenant on a replica shares them,
+ * so one runaway planner child could take the replica, and every tenant on it, down with it. This module bounds
+ * what one child tree can take before the operating system's OOM killer decides for us. It is a ceiling for a
+ * runaway, not a budget and not isolation: a tree that stays under its limits still shares the machine.
+ *
+ * The governor samples the process table (`ps`), finds the tree that starts at a watched root, and kills the
+ * whole tree the moment it is over a limit. The root is identified by its parent as well as by its pid, so a pid
+ * that the operating system has reused for an unrelated process is never sampled and never killed. Sampling is
+ * read-only, one `ps` per tick for every watched tree, and a failed sample kills nothing.
+ *
+ * Linux places the worker in a user-systemd scope and Darwin in its own process group, so neither a group id nor a
+ * cgroup is assumed here: the tree is the root and its descendants by parent pid, which both platforms report.
+ * A child that deliberately re-parents itself away leaves the tree; the planner children cannot start processes
+ * (their shell, file and subprocess plugins are disabled), and the provider-owned range cleanup of
+ * `dsh-subprocess-local` remains the primary teardown. This is the second line.
+ */
+import { execFile } from 'node:child_process';
+/** The process table cannot be read or does not show this process: the sampler would govern nothing. */
+export class ProcessTableUnavailableError extends Error {
+    constructor(cause) {
+        super('the process table cannot be read, so planner process trees cannot be governed', { cause });
+        this.name = 'ProcessTableUnavailableError';
+    }
+}
+const DEFAULT_INTERVAL_MS = 1_000;
+const DEFAULT_MAX_BLIND_TICKS = 10;
+const DEFAULT_TABLE_TIMEOUT_MS = 5_000;
+const PS_TIMEOUT_MS = 3_000;
+const PS_MAX_BUFFER_BYTES = 16 * 1024 * 1024;
+/** `[DD-][HH:]MM:SS[.ss]`: Linux prints `HH:MM:SS` and `DD-HH:MM:SS`, Darwin `M:SS.ss` and `H:MM:SS.ss`. NaN for anything else. */
+export function parseCpuSeconds(text) {
+    const match = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)$/.exec(text.trim());
+    if (!match)
+        return Number.NaN;
+    const [, days, hours, minutes, seconds] = match;
+    return (Number(days ?? 0) * 24 + Number(hours ?? 0)) * 3_600 + Number(minutes) * 60 + Number(seconds);
+}
+/** Parses the output of `ps` with `PS_ARGS` (rss in KiB). Lines that do not parse are dropped, never guessed. */
+export function parsePsTable(stdout) {
+    const rows = [];
+    for (const line of stdout.split('\n')) {
+        const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s*$/.exec(line);
+        if (!match)
+            continue;
+        const cpuSeconds = parseCpuSeconds(match[4]);
+        if (Number.isNaN(cpuSeconds))
+            continue;
+        rows.push({ pid: Number(match[1]), ppid: Number(match[2]), rssBytes: Number(match[3]) * 1024, cpuSeconds });
+    }
+    return rows;
+}
+/**
+ * `-A` selects every process on procps (Linux) and on BSD (Darwin) alike; the BSD-style `-ax` that the cleanup diagnostics
+ * use on Darwin makes procps warn on stderr. `pid=` and the like suppress the header. `rss` is in KiB on both.
+ */
+export const PS_ARGS = ['-A', '-o', 'pid=,ppid=,rss=,time='];
+export function readPsTable() {
+    return new Promise((resolve, reject) => {
+        execFile('/bin/ps', [...PS_ARGS], {
+            encoding: 'utf8', timeout: PS_TIMEOUT_MS, maxBuffer: PS_MAX_BUFFER_BYTES,
+            env: { PATH: '/usr/bin:/bin', LC_ALL: 'C' },
+        }, (error, stdout) => (error ? reject(error) : resolve(parsePsTable(stdout))));
+    });
+}
+/**
+ * The root and its descendants, or `null` when the root is not there or is not a child of the expected parent.
+ * Pure: the suites feed crafted tables.
+ */
+export function treeOf(rows, target) {
+    const byPid = new Map();
+    const children = new Map();
+    for (const row of rows) {
+        byPid.set(row.pid, row);
+        const siblings = children.get(row.ppid);
+        if (siblings)
+            siblings.push(row);
+        else
+            children.set(row.ppid, [row]);
+    }
+    const root = byPid.get(target.rootPid);
+    if (!root || root.ppid !== target.parentPid)
+        return null;
+    const pids = [];
+    let rssBytes = 0;
+    let cpuSeconds = 0;
+    const seen = new Set();
+    const queue = [root];
+    for (let row = queue.shift(); row; row = queue.shift()) {
+        if (seen.has(row.pid))
+            continue;
+        seen.add(row.pid);
+        pids.push(row.pid);
+        rssBytes += row.rssBytes;
+        cpuSeconds += row.cpuSeconds;
+        for (const child of children.get(row.pid) ?? [])
+            queue.push(child);
+    }
+    return { pids, rssBytes, cpuSeconds };
+}
+function withTimeout(promise, ms) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error('the process table did not answer in time')), ms);
+        promise.then((value) => { clearTimeout(timer); resolve(value); }, (error) => { clearTimeout(timer); reject(error); });
+    });
+}
+function breachOf(sample, limits) {
+    if (limits.maxRssBytes !== undefined && sample.rssBytes > limits.maxRssBytes) {
+        return { kind: 'memory', limit: limits.maxRssBytes, observed: sample.rssBytes };
+    }
+    if (limits.maxCpuSeconds !== undefined && sample.cpuSeconds > limits.maxCpuSeconds) {
+        return { kind: 'cpu', limit: limits.maxCpuSeconds, observed: sample.cpuSeconds };
+    }
+    return null;
+}
+function assertPositive(name, value) {
+    if (value !== undefined && (!Number.isFinite(value) || value <= 0))
+        throw new RangeError(`${name} must be a positive number`);
+}
+function killEach(pids) {
+    for (const pid of pids) {
+        try {
+            process.kill(pid, 'SIGKILL');
+        }
+        catch { /* already gone */ }
+    }
+}
+export class ProcessTreeGovernor {
+    intervalMs;
+    maxBlindTicks;
+    tableTimeoutMs;
+    table;
+    kill;
+    onFault;
+    watches = new Map();
+    nextId = 1;
+    timer;
+    ticking = false;
+    sampleFailures = 0;
+    constructor(options = {}) {
+        this.intervalMs = options.intervalMs ?? DEFAULT_INTERVAL_MS;
+        assertPositive('intervalMs', this.intervalMs);
+        this.maxBlindTicks = options.maxBlindTicks ?? DEFAULT_MAX_BLIND_TICKS;
+        assertPositive('maxBlindTicks', this.maxBlindTicks);
+        this.tableTimeoutMs = options.tableTimeoutMs ?? DEFAULT_TABLE_TIMEOUT_MS;
+        assertPositive('tableTimeoutMs', this.tableTimeoutMs);
+        this.table = options.table ?? readPsTable;
+        this.kill = options.kill ?? killEach;
+        this.onFault = options.onFault;
+    }
+    /** Trees under watch now. */
+    get size() {
+        return this.watches.size;
+    }
+    /** Samples that failed since construction. A failed sample kills nothing. */
+    get failedSamples() {
+        return this.sampleFailures;
+    }
+    /**
+     * Watches one tree. `onBreach` runs once, before the tree is killed, so the owner can record why. The returned
+     * function stops the watch and is safe to call after a breach. With no limit given nothing is watched.
+     */
+    watch(target, limits, onBreach) {
+        if (!Number.isSafeInteger(target.rootPid) || target.rootPid <= 0)
+            throw new RangeError('rootPid must be a positive integer');
+        if (!Number.isSafeInteger(target.parentPid) || target.parentPid <= 0)
+            throw new RangeError('parentPid must be a positive integer');
+        assertPositive('maxRssBytes', limits.maxRssBytes);
+        assertPositive('maxCpuSeconds', limits.maxCpuSeconds);
+        if (limits.maxRssBytes === undefined && limits.maxCpuSeconds === undefined)
+            return () => undefined;
+        const id = this.nextId++;
+        this.watches.set(id, { target, limits, onBreach, blind: 0 });
+        this.startTimer();
+        return () => {
+            this.watches.delete(id);
+            if (this.watches.size === 0)
+                this.stopTimer();
+        };
+    }
+    /**
+     * Proves the sampler can see: the table reads, and it shows this very process. A service that cannot see its own
+     * process table would run its children ungoverned and silently (for example where `ps` is a stripped-down build),
+     * so a deployment checks this once before the first tree starts and refuses to run children without it.
+     */
+    async probe() {
+        let rows;
+        try {
+            rows = await withTimeout(this.table(), this.tableTimeoutMs);
+        }
+        catch (error) {
+            throw new ProcessTableUnavailableError(error);
+        }
+        const self = rows.find((row) => row.pid === process.pid);
+        if (!self || self.rssBytes <= 0)
+            throw new ProcessTableUnavailableError();
+    }
+    /** One sampling pass. The timer calls it; the suites call it directly to stay deterministic. */
+    async tick() {
+        if (this.ticking || this.watches.size === 0)
+            return;
+        this.ticking = true;
+        try {
+            let rows = null;
+            try {
+                rows = await withTimeout(this.table(), this.tableTimeoutMs);
+            }
+            catch (error) {
+                this.sampleFailures += 1;
+                this.report(error);
+            }
+            for (const [id, watch] of [...this.watches]) {
+                const sample = rows ? treeOf(rows, watch.target) : null;
+                if (!sample) {
+                    // A tree that cannot be seen is not a tree within its limits. A few such ticks happen (a worker exiting between
+                    // two samples); a run of them means the sampler governs nothing, and failing open would hide it.
+                    watch.blind += 1;
+                    if (watch.blind >= this.maxBlindTicks)
+                        this.end(id, watch, { kind: 'blind', limit: this.maxBlindTicks, observed: watch.blind }, []);
+                    continue;
+                }
+                watch.blind = 0;
+                const breach = breachOf(sample, watch.limits);
+                if (breach)
+                    this.end(id, watch, breach, sample.pids);
+            }
+        }
+        finally {
+            this.ticking = false;
+            if (this.watches.size === 0)
+                this.stopTimer();
+        }
+    }
+    close() {
+        this.watches.clear();
+        this.stopTimer();
+    }
+    /** The watch is over: the owner is told why, and whatever was seen of the tree is killed. */
+    end(id, watch, breach, pids) {
+        this.watches.delete(id);
+        try {
+            watch.onBreach(breach);
+        }
+        catch (error) {
+            this.report(error);
+        }
+        finally {
+            this.kill(pids);
+        }
+    }
+    report(error) {
+        try {
+            this.onFault?.(error);
+        }
+        catch { /* reporting must never stop the governor */ }
+    }
+    startTimer() {
+        if (this.timer)
+            return;
+        this.timer = setInterval(() => { this.tick().catch((error) => this.report(error)); }, this.intervalMs);
+        // The governor must not keep the process alive: a watched child cannot outlive its owner's own shutdown.
+        this.timer.unref();
+    }
+    stopTimer() {
+        if (!this.timer)
+            return;
+        clearInterval(this.timer);
+        this.timer = undefined;
+    }
+}
+let shared;
+/** The governor every managed port shares by default: one `ps` per tick, however many trees are watched. */
+export function sharedProcessTreeGovernor() {
+    shared ??= new ProcessTreeGovernor({
+        onFault: (error) => { console.error('[process-governor] fault:', error instanceof Error ? error.message : String(error)); },
+    });
+    return shared;
+}

package/dist/src/tenant-seam/quotas.js

@@ -0,0 +1,360 @@
+/**
+ * Quotas and metering per tenant and per organization (docs/rfc/aaas-rfc.md §4.4, threats S-10 and S-11).
+ *
+ * One tenant must not be able to move another tenant's latency budget, and one organization must not be able to
+ * move another organization's. The meter therefore counts at two scopes (the ledger tenant and its organization) and
+ * a global concurrency cap, and refuses with a typed `QuotaExceededError` (HTTP 429) before the expensive work: a
+ * refused turn is turned away after authentication and before the request body is read, the lease is acquired or a
+ * planner is started.
+ *
+ * Dimensions: concurrent turns, turn rate, LLM tokens and cost (priced by the frozen ADR-20 table, in the same code
+ * the nightly evidence uses), bridge job rate, and ledger size. Windows are rolling, driven by an injected
+ * monotonic clock. The LLM budget is checked before a turn and charged after it, so it can be overshot by the
+ * turns in flight: a soft cap, by design. A planner invocation that reports no usage (or zero tokens) is charged the
+ * configured worst case instead of nothing, so a silent or zero-answering feed cannot make the budget unreachable.
+ *
+ * State is per replica. Tenant-affine routing gives a tenant one owner, so tenant quotas hold fleet-wide; an
+ * organization's quota is per replica until a shared counter is chosen (A3).
+ */
+import { assertModelPriced, priceMeasuredUsage } from '../llm-price-table.js';
+import { TenantStateError } from '../tenant-state/errors.js';
+import { assertOrganizationId } from '../tenant-state/ids.js';
+import { assertSeamTenant, seamTenantKey } from './seam.js';
+export const QUOTA_DIMENSIONS = ['concurrent_turns', 'turn_rate', 'llm_tokens', 'llm_cost', 'bridge_jobs', 'ledger_size'];
+/** A turn that found every slot taken has no better hint than "soon". */
+const CONCURRENCY_RETRY_AFTER_MS = 1_000;
+/** A full ledger frees up only by an operator action, so the hint is long. */
+const LEDGER_RETRY_AFTER_MS = 3_600_000;
+const MICRO = 1_000_000;
+const SWEEP_EVERY = 1_000;
+/** Refused by a quota. Carries the narrowest scope that refused, the dimension and a retry hint; no personal data. */
+export class QuotaExceededError extends TenantStateError {
+    scope;
+    dimension;
+    limit;
+    retryAfterMs;
+    constructor(scope, dimension, limit, retryAfterMs) {
+        super('GOTRY_QUOTA_EXCEEDED', `${scope} quota exceeded: ${dimension}`);
+        this.scope = scope;
+        this.dimension = dimension;
+        this.limit = limit;
+        this.retryAfterMs = Math.max(1, Math.ceil(retryAfterMs));
+    }
+    /** Whole seconds for a `Retry-After` header, at least one. */
+    get retryAfterSeconds() {
+        return Math.max(1, Math.ceil(this.retryAfterMs / 1_000));
+    }
+}
+/**
+ * Starting values, not a decision: they are conservative and unreviewed, and a deployment sets its own.
+ * Each tenant cap sits below its organization's, and each organization's below the global one, so that no single
+ * tenant can fill an organization and no single organization can fill the service.
+ */
+export const DEFAULT_QUOTA_POLICY = Object.freeze({
+    tenant: {
+        concurrentTurns: 2,
+        turns: { limit: 30, windowMs: 60_000 },
+        llm: { tokens: 400_000, costUsd: 2, windowMs: 3_600_000 },
+        bridgeJobs: { limit: 12, windowMs: 60_000 },
+        ledgerBytes: 64 * 1024 * 1024,
+    },
+    organization: {
+        concurrentTurns: 16,
+        turns: { limit: 600, windowMs: 60_000 },
+        llm: { tokens: 20_000_000, costUsd: 100, windowMs: 3_600_000 },
+        bridgeJobs: { limit: 120, windowMs: 60_000 },
+        ledgerBytes: 8 * 1024 * 1024 * 1024,
+    },
+    globalConcurrentTurns: 64,
+    unprovenInvocation: { tokens: 50_000, costUsd: 0.25 },
+});
+/** A rolling window of amounts: the sum of what was added in the last `windowMs`. */
+class RollingSum {
+    entries = [];
+    head = 0;
+    total = 0;
+    windowMs;
+    constructor(windowMs) {
+        this.windowMs = windowMs;
+    }
+    prune(now) {
+        const horizon = now - this.windowMs;
+        while (this.head < this.entries.length && this.entries[this.head][0] <= horizon) {
+            this.total -= this.entries[this.head][1];
+            this.head += 1;
+        }
+        if (this.head > 1_024 && this.head * 2 > this.entries.length) {
+            this.entries.splice(0, this.head);
+            this.head = 0;
+        }
+    }
+    sum(now) {
+        this.prune(now);
+        return this.total;
+    }
+    add(now, amount) {
+        this.prune(now);
+        this.entries.push([now, amount]);
+        this.total += amount;
+    }
+    /** Milliseconds until the sum falls strictly below `limit` if nothing more is added; 0 when it already is. */
+    msUntilBelow(now, limit) {
+        this.prune(now);
+        let remaining = this.total;
+        if (remaining < limit)
+            return 0;
+        for (let index = this.head; index < this.entries.length; index += 1) {
+            remaining -= this.entries[index][1];
+            if (remaining < limit)
+                return Math.max(1, this.entries[index][0] + this.windowMs - now);
+        }
+        return this.windowMs;
+    }
+    idle(now) {
+        this.prune(now);
+        return this.head === this.entries.length;
+    }
+}
+class Account {
+    active = 0;
+    turns;
+    tokens;
+    costMicro;
+    bridge;
+    ledgerBytes = 0;
+    rejected = { concurrent_turns: 0, turn_rate: 0, llm_tokens: 0, llm_cost: 0, bridge_jobs: 0, ledger_size: 0 };
+    constructor(limits) {
+        this.turns = new RollingSum(limits.turns.windowMs);
+        this.tokens = new RollingSum(limits.llm.windowMs);
+        this.costMicro = new RollingSum(limits.llm.windowMs);
+        this.bridge = new RollingSum(limits.bridgeJobs.windowMs);
+    }
+    /** No live turn and nothing left in any window. The ledger size is not part of it: the sweep takes it out of the organization total. */
+    idle(now) {
+        return this.active === 0 && this.turns.idle(now) && this.tokens.idle(now) && this.costMicro.idle(now) && this.bridge.idle(now);
+    }
+}
+function positiveInteger(name, value) {
+    if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1)
+        throw new RangeError(`${name} must be a positive integer`);
+}
+function positiveNumber(name, value) {
+    if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0)
+        throw new RangeError(`${name} must be a positive number`);
+}
+function validateLimits(scope, limits) {
+    positiveInteger(`${scope}.concurrentTurns`, limits.concurrentTurns);
+    positiveInteger(`${scope}.turns.limit`, limits.turns.limit);
+    positiveInteger(`${scope}.turns.windowMs`, limits.turns.windowMs);
+    positiveInteger(`${scope}.llm.tokens`, limits.llm.tokens);
+    positiveNumber(`${scope}.llm.costUsd`, limits.llm.costUsd);
+    positiveInteger(`${scope}.llm.windowMs`, limits.llm.windowMs);
+    positiveInteger(`${scope}.bridgeJobs.limit`, limits.bridgeJobs.limit);
+    positiveInteger(`${scope}.bridgeJobs.windowMs`, limits.bridgeJobs.windowMs);
+    positiveInteger(`${scope}.ledgerBytes`, limits.ledgerBytes);
+}
+function validatePolicy(policy) {
+    validateLimits('tenant', policy.tenant);
+    validateLimits('organization', policy.organization);
+    positiveInteger('globalConcurrentTurns', policy.globalConcurrentTurns);
+    positiveInteger('unprovenInvocation.tokens', policy.unprovenInvocation.tokens);
+    positiveNumber('unprovenInvocation.costUsd', policy.unprovenInvocation.costUsd);
+    // A tenant cap above its organization's can never bind, and an organization above the global one lets it starve the rest.
+    if (policy.tenant.concurrentTurns > policy.organization.concurrentTurns)
+        throw new RangeError('tenant.concurrentTurns must not exceed organization.concurrentTurns');
+    if (policy.organization.concurrentTurns > policy.globalConcurrentTurns)
+        throw new RangeError('organization.concurrentTurns must not exceed globalConcurrentTurns');
+}
+export class TenantQuotas {
+    policy;
+    priceTable;
+    clock;
+    tenants = new Map();
+    organizations = new Map();
+    globalActive = 0;
+    admissions = 0;
+    constructor(options) {
+        validatePolicy(options.policy);
+        assertModelPriced(options.model, options.priceTable);
+        this.policy = options.policy;
+        this.priceTable = options.priceTable;
+        this.clock = options.clock ?? (() => performance.now());
+    }
+    /**
+     * Admits one turn or refuses it. Checks run narrowest scope first (tenant, organization, global) so a hot tenant is
+     * told it hit its own limit, and an innocent one only ever sees an organization limit that the sum really reached.
+     * Nothing is reserved unless every check passes.
+     */
+    admitTurn(tenant) {
+        const seam = assertSeamTenant(tenant);
+        const now = this.clock();
+        // Every attempt counts toward the sweep, so a flood of refused attempts cannot outrun it.
+        this.maybeSweep(now);
+        const key = seamTenantKey(seam);
+        // A tenant is registered only when a turn is admitted: a fresh tenant that is refused leaves nothing behind.
+        const known = this.tenants.get(key);
+        const tenantAccount = known ?? new Account(this.policy.tenant);
+        const organizationAccount = this.organizationAccount(seam.organization);
+        const { policy } = this;
+        this.refuseIfOver(tenantAccount, 'tenant', 'concurrent_turns', tenantAccount.active, policy.tenant.concurrentTurns, CONCURRENCY_RETRY_AFTER_MS);
+        this.refuseIfOverRolling(tenantAccount, 'tenant', 'turn_rate', tenantAccount.turns, now, policy.tenant.turns.limit);
+        this.refuseIfOverRolling(tenantAccount, 'tenant', 'llm_tokens', tenantAccount.tokens, now, policy.tenant.llm.tokens);
+        this.refuseIfOverRolling(tenantAccount, 'tenant', 'llm_cost', tenantAccount.costMicro, now, Math.round(policy.tenant.llm.costUsd * MICRO), MICRO);
+        this.refuseIfOver(organizationAccount, 'organization', 'concurrent_turns', organizationAccount.active, policy.organization.concurrentTurns, CONCURRENCY_RETRY_AFTER_MS);
+        this.refuseIfOverRolling(organizationAccount, 'organization', 'turn_rate', organizationAccount.turns, now, policy.organization.turns.limit);
+        this.refuseIfOverRolling(organizationAccount, 'organization', 'llm_tokens', organizationAccount.tokens, now, policy.organization.llm.tokens);
+        this.refuseIfOverRolling(organizationAccount, 'organization', 'llm_cost', organizationAccount.costMicro, now, Math.round(policy.organization.llm.costUsd * MICRO), MICRO);
+        this.refuseIfOver(organizationAccount, 'global', 'concurrent_turns', this.globalActive, policy.globalConcurrentTurns, CONCURRENCY_RETRY_AFTER_MS);
+        if (!known)
+            this.tenants.set(key, tenantAccount);
+        tenantAccount.active += 1;
+        organizationAccount.active += 1;
+        this.globalActive += 1;
+        tenantAccount.turns.add(now, 1);
+        organizationAccount.turns.add(now, 1);
+        let released = false;
+        return {
+            release: () => {
+                if (released)
+                    return;
+                released = true;
+                tenantAccount.active -= 1;
+                organizationAccount.active -= 1;
+                this.globalActive -= 1;
+            },
+        };
+    }
+    /**
+     * Charges measured usage, priced by the frozen table. Throws for an unpriced model, malformed counts or a report of zero
+     * tokens (no completion costs nothing, so a relay that answers zeros is not measuring); the caller then charges the worst case.
+     */
+    recordLlmUsage(tenant, model, usage) {
+        const seam = assertSeamTenant(tenant);
+        const priced = priceMeasuredUsage(model, usage, this.priceTable);
+        if (priced.tokens === 0)
+            throw new RangeError('a usage report of zero tokens proves nothing');
+        this.charge(seam, priced.tokens, Math.round(priced.costUsd * MICRO));
+    }
+    /** Charges one planner invocation that reported no usable usage: the configured worst case. */
+    chargeUnprovenInvocation(tenant) {
+        const seam = assertSeamTenant(tenant);
+        this.charge(seam, this.policy.unprovenInvocation.tokens, Math.round(this.policy.unprovenInvocation.costUsd * MICRO));
+    }
+    /** Counts one bridge job for a tenant and its organization, or refuses it. */
+    admitBridgeJob(tenant) {
+        const seam = assertSeamTenant(tenant);
+        const now = this.clock();
+        const tenantAccount = this.tenantAccount(seamTenantKey(seam));
+        const organizationAccount = this.organizationAccount(seam.organization);
+        this.refuseIfOverRolling(tenantAccount, 'tenant', 'bridge_jobs', tenantAccount.bridge, now, this.policy.tenant.bridgeJobs.limit);
+        this.refuseIfOverRolling(organizationAccount, 'organization', 'bridge_jobs', organizationAccount.bridge, now, this.policy.organization.bridgeJobs.limit);
+        tenantAccount.bridge.add(now, 1);
+        organizationAccount.bridge.add(now, 1);
+    }
+    /** The same for a caller that is known only by organization, such as the extension lane's HTTP surface. */
+    admitOrganizationBridgeJob(organization) {
+        const now = this.clock();
+        const organizationAccount = this.organizationAccount(assertOrganizationId(organization));
+        this.refuseIfOverRolling(organizationAccount, 'organization', 'bridge_jobs', organizationAccount.bridge, now, this.policy.organization.bridgeJobs.limit);
+        organizationAccount.bridge.add(now, 1);
+    }
+    /**
+     * Records the size of a tenant's ledger and refuses while it is at its limit, or while the ledgers of the
+     * organization's tenants served here add up to the organization's limit.
+     */
+    observeLedgerBytes(tenant, bytes) {
+        const seam = assertSeamTenant(tenant);
+        if (!Number.isSafeInteger(bytes) || bytes < 0)
+            throw new RangeError('bytes must be a non-negative integer');
+        const tenantAccount = this.tenantAccount(seamTenantKey(seam));
+        const organizationAccount = this.organizationAccount(seam.organization);
+        organizationAccount.ledgerBytes += bytes - tenantAccount.ledgerBytes;
+        tenantAccount.ledgerBytes = bytes;
+        this.refuseIfOver(tenantAccount, 'tenant', 'ledger_size', bytes, this.policy.tenant.ledgerBytes, LEDGER_RETRY_AFTER_MS);
+        this.refuseIfOver(organizationAccount, 'organization', 'ledger_size', organizationAccount.ledgerBytes, this.policy.organization.ledgerBytes, LEDGER_RETRY_AFTER_MS);
+    }
+    /** Forgets a tenant's counters (after a forget, or when its ledger leaves this replica). */
+    forgetTenant(tenant) {
+        const seam = assertSeamTenant(tenant);
+        const key = seamTenantKey(seam);
+        const account = this.tenants.get(key);
+        if (!account)
+            return;
+        const organizationAccount = this.organizations.get(seam.organization);
+        if (organizationAccount)
+            organizationAccount.ledgerBytes -= account.ledgerBytes;
+        this.tenants.delete(key);
+    }
+    /** Counters keyed by `<organization>/<hmac tenant id>`: nothing that names a person. */
+    snapshot() {
+        const now = this.clock();
+        const describe = (key, account) => ({
+            key,
+            activeTurns: account.active,
+            turnsInWindow: account.turns.sum(now),
+            llmTokens: account.tokens.sum(now),
+            llmCostUsd: account.costMicro.sum(now) / MICRO,
+            bridgeJobs: account.bridge.sum(now),
+            ledgerBytes: account.ledgerBytes,
+            rejected: { ...account.rejected },
+        });
+        return {
+            globalActiveTurns: this.globalActive,
+            tenants: [...this.tenants].map(([key, account]) => describe(key, account)),
+            organizations: [...this.organizations].map(([key, account]) => describe(key, account)),
+        };
+    }
+    charge(seam, tokens, costMicro) {
+        const now = this.clock();
+        for (const account of [this.tenantAccount(seamTenantKey(seam)), this.organizationAccount(seam.organization)]) {
+            account.tokens.add(now, tokens);
+            account.costMicro.add(now, costMicro);
+        }
+    }
+    tenantAccount(key) {
+        let account = this.tenants.get(key);
+        if (!account) {
+            account = new Account(this.policy.tenant);
+            this.tenants.set(key, account);
+        }
+        return account;
+    }
+    organizationAccount(organization) {
+        let account = this.organizations.get(organization);
+        if (!account) {
+            account = new Account(this.policy.organization);
+            this.organizations.set(organization, account);
+        }
+        return account;
+    }
+    /** Refuses when `used` has reached `limit`. */
+    refuseIfOver(account, scope, dimension, used, limit, retryAfterMs) {
+        if (used < limit)
+            return;
+        account.rejected[dimension] += 1;
+        throw new QuotaExceededError(scope, dimension, limit, retryAfterMs);
+    }
+    refuseIfOverRolling(account, scope, dimension, window, now, limit, unit = 1) {
+        if (window.sum(now) < limit)
+            return;
+        account.rejected[dimension] += 1;
+        throw new QuotaExceededError(scope, dimension, limit / unit, window.msUntilBelow(now, limit));
+    }
+    maybeSweep(now) {
+        this.admissions += 1;
+        if (this.admissions % SWEEP_EVERY !== 0)
+            return;
+        for (const [key, account] of this.tenants) {
+            if (!account.idle(now))
+                continue;
+            this.tenants.delete(key);
+            // The tenant's ledger leaves the organization's total with its account; its next turn observes the size again.
+            const organizationAccount = this.organizations.get(key.slice(0, key.indexOf('/')));
+            if (organizationAccount)
+                organizationAccount.ledgerBytes -= account.ledgerBytes;
+        }
+        for (const [key, account] of this.organizations)
+            if (account.idle(now) && account.ledgerBytes === 0)
+                this.organizations.delete(key);
+    }
+}