@capybara-hq/kernel 0.1.0-rc.1 → 0.1.0-rc.10

package/dist/runtime/usage-report.js

@@ -1,32 +1,34 @@
-import { DelayedError, Job, Queue, UnrecoverableError, Worker } from "bullmq";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+import { getQueueToken } from "@nestjs/bullmq";
+import { Inject, Injectable, Module, Optional } from "@nestjs/common";
+import { DelayedError, Job, UnrecoverableError } from "bullmq";
 import { sql } from "kysely";
-import { enabledExtensionsQuery } from "../extensions/enabled.js";
-import { deepFreeze, errMsg } from "../extensions/hooks.js";
-import { withJobContext } from "../log-context.js";
-import { bullConnection } from "./run-queue.js";
-/** Queue of `usage.report` deliveries (no `:` in the name). */
-export const USAGE_REPORT_QUEUE = "cap-kernel-usage-report";
-/** One job per `usage` event (no `:`: conversation ids are UUIDs, seq is a number). */
-export const usageReportJobId = (conversationId, seq) => `usage-${conversationId}-${seq}`;
-/** First attempt plus 5 retries (F11). */
-export const USAGE_REPORT_ATTEMPTS = 6;
-/** Retry backoff: 2 s, doubling (2, 4, 8, 16, 32 s). */
-export const USAGE_REPORT_BACKOFF_MS = 2000;
-/** Failed jobs (given up) are kept for inspection, at most {@link KEEP_FAILED_MAX} of them. */
-const KEEP_FAILED_S = 7 * 86_400;
-/** Upper bound on kept failed jobs (I-2): a long outage of a billing endpoint cannot grow Redis. */
-const KEEP_FAILED_MAX = 100_000;
+import { ExtensionsModule } from "../extensions/extensions.module.js";
+import { deepFreeze, errMsg, HookNeedsConfigError, } from "../extensions/hooks.js";
+import { ExtensionHost } from "../extensions/host.js";
+import { STALE_RECONCILE_PERIODS } from "../extensions/lifecycle-topics.js";
+import { KernelConfigService } from "../infra/config/kernel-config.service.js";
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { LockService } from "../infra/lock/lock.service.js";
+import { LockedCron } from "../infra/schedule/locked-cron.decorator.js";
+import { KERNEL_LOGGER } from "../infra/tokens.js";
+import { deliveryRowOfChildRun } from "./subagents/child.js";
+import { modelUsageJobId, queryUsageSubscribers, USAGE_REPORT_ATTEMPTS, USAGE_REPORT_BACKOFF_MS, USAGE_REPORT_JOB, USAGE_REPORT_QUEUE, USAGE_REPORT_RETRY, UsageReportJobsModule, usageReportJobId, usageReportJobOpts, } from "./usage-report-jobs.js";
+export { ModelUsageJobs, modelUsageJobId, queryUsageSubscribers, USAGE_REPORT_ATTEMPTS, USAGE_REPORT_BACKOFF_MS, USAGE_REPORT_QUEUE, USAGE_REPORT_RETRY, UsageReportJobsModule, usageReportJobId, } from "./usage-report-jobs.js";
 /** Warnings about a plugin that cannot take deliveries are logged at most once per this window. */
 const WARN_WINDOW_MS = 60_000;
-/**
- * Upper bound on kept finished jobs (Redis memory). At the default retention it only bites above
- * ~1.1 usage events per second sustained for a day; dedupe needs the jobs only for the sweeper overlap.
- */
-const KEEP_COMPLETED_MAX = 100_000;
-/** Maintenance job of the sweeper (no `:`), its scheduler id and period. */
+/** The sweeper's cron job name (no `:`; every minute, once across replicas). */
 export const USAGE_SWEEP_JOB = "usage-report-sweep";
-export const USAGE_SWEEP_SCHEDULER_ID = "usage-report-sweep-every";
-export const USAGE_SWEEP_EVERY_MS = 60_000;
+/** The maintenance scheduler the sweep ran on before it became a `@LockedCron` (retired). */
+export { USAGE_SWEEP_SCHEDULER_ID } from "./job-ids.js";
 /** The sweep leaves the newest events to the fast path (and to transactions still committing). */
 export const USAGE_SWEEP_LAG_MS = 30_000;
 /**
@@ -36,38 +38,6 @@ export const USAGE_SWEEP_LAG_MS = 30_000;
  */
 export const USAGE_SWEEP_OVERLAP_MS = 10 * 60_000;
 const SWEEP_PAGE = 500;
-const JOB = "usage-report";
-/**
- * Plugins that subscribe to `usage.report` according to deployment state: an approved permission
- * snapshot that declares the `usage.report` hook, for a plugin the deployment still knows: an enabled
- * install, or a config-declared plugin with a replica status row that is not `unloaded`. With `orgId`, only plugins
- * the organization enabled (org config rows and required bootstrap plugins).
- */
-export async function queryUsageSubscribers(db, orgId) {
-    let q = db
-        .selectFrom("extension_permission_snapshots as s")
-        .select("s.plugin")
-        .where(sql `s.permissions -> 'permissions' -> 'hooks' ? 'usage.report'`)
-        .where((eb) => eb.or([
-        eb.exists(eb
-            .selectFrom("extension_installs as i")
-            .select(sql `1`.as("one"))
-            .whereRef("i.name", "=", "s.plugin")
-            .where("i.desired_state", "=", "enabled")),
-        // config-declared plugins still declared (and not unloaded) on some replica; installs are
-        // covered by desired_state above (their replica rows linger as `unloaded` when disabled)
-        eb.exists(eb
-            .selectFrom("extension_replica_status as r")
-            .select(sql `1`.as("one"))
-            .whereRef("r.plugin", "=", "s.plugin")
-            .where("r.source", "=", "config")
-            .where("r.state", "<>", "unloaded")),
-    ]));
-    if (orgId !== undefined) {
-        q = q.where("s.plugin", "in", enabledExtensionsQuery(db, orgId));
-    }
-    return new Set((await q.execute()).map((r) => r.plugin));
-}
 /**
  * Of `plugins`, those that are **definitely** unservable right now (I-2, N-3): live replicas report
  * them (a `extension_replica_status` row with a heartbeat newer than `staleSeconds`), no live row is
@@ -107,6 +77,10 @@ class Undeliverable extends Error {
         this.name = "Undeliverable";
     }
 }
+/** A job's identity: its `usage` event or its `model_usage` row. */
+const keyOf = (d) => d.usageId !== undefined
+    ? { usageId: d.usageId }
+    : { conversationId: d.conversationId, seq: d.seq };
 /**
  * The next `pending` set (m-2 fence): plugins that failed now, narrowed to what the stored job data
  * still lists. A plugin another (stalled-over) execution already delivered never comes back.
@@ -138,33 +112,82 @@ export function narrowPending(stored, failed) {
  *   most 100 000. Each call runs through the instance's `inflight.run`, limited by `hooks.timeoutMs`;
  *   when this replica stops, in-flight calls are aborted and the job goes back to the queue without
  *   using up an attempt.
- * Handlers must be idempotent by `eventId` (`<conversationId>/<seq>`). Nothing is written to the event
- * log (never model-visible). Compaction calls log usage too (`purpose: "compaction"`) and are reported,
- * as are conversation title calls (`purpose: "title"`, no run: `runId` is "").
+ * Handlers must be idempotent by `eventId` (`<conversationId>/<seq>`, or a `model_usage` row id).
+ * Nothing is written to the event log (never model-visible). Compaction calls log usage too (`purpose: "compaction"`) and are reported,
+ * as are conversation title calls (`purpose: "title"`, no run: `runId` is "") and `WebFetch`
+ * extraction calls (`purpose: "web_fetch"`).
  * Single-step eval replays and judge requests log no usage: not reported (M7).
+ * - **Model usage outside conversations** (spec B §5.4): each `ctx.models.embed` call writes a
+ *   `model_usage` row; on its `model_usage.recorded` event `ModelUsageJobs` (`usage-report-jobs.ts`)
+ *   enqueues job `usage-model-<rowId>` (only while some plugin subscribes); the payload
+ *   (`purpose: "embedding"`, `eventId` = the row id, the calling `extension`, no conversation fields)
+ *   is read from the row when the job runs, and the row's `reported_at` is set once nothing is left
+ *   to deliver; a job that gives up after its last attempt sets `report_failed_at`. The sweep
+ *   re-enqueues recent rows that are neither (see {@link sweep}).
  */
-export class UsageReporter {
-    d;
+let UsageReporter = class UsageReporter {
+    kdb;
     queue;
-    queueConn;
+    host;
+    config;
+    locks;
+    log;
+    retry;
     worker;
-    workerConn;
     closed = false;
     stopping = new AbortController();
-    constructor(d) {
-        this.d = d;
+    constructor(kdb, queue, 
+    /**
+     * The host's hook registry: handlers of the current local plugin generations (through
+     * inflight.run), and `has("usage.report")`, the in-process check that lets `enqueue` skip all
+     * I/O without handlers; and whether a plugin's current instance is active here.
+     */
+    host, config, 
+    /** The warning throttle's markers (`capk:usage-warn:<plugin>:<kind>`, SET NX PX). */
+    locks, log, retry = {
+        attempts: USAGE_REPORT_ATTEMPTS,
+        backoffMs: USAGE_REPORT_BACKOFF_MS,
+    }) {
+        this.kdb = kdb;
+        this.queue = queue;
+        this.host = host;
+        this.config = config;
+        this.locks = locks;
+        this.log = log;
+        this.retry = retry;
+        queue.on("error", (err) => this.log.error("usage report queue error", { err }));
     }
-    jobOpts(conversationId, seq) {
-        return {
-            jobId: usageReportJobId(conversationId, seq),
-            attempts: this.d.retry?.attempts ?? USAGE_REPORT_ATTEMPTS,
-            backoff: { type: "exponential", delay: this.d.retry?.backoffMs ?? USAGE_REPORT_BACKOFF_MS },
-            removeOnComplete: {
-                age: this.d.config().hooks.usageReportKeepHours * 3600,
-                count: KEEP_COMPLETED_MAX,
-            },
-            removeOnFail: { age: KEEP_FAILED_S, count: KEEP_FAILED_MAX },
-        };
+    get db() {
+        return this.kdb.db;
+    }
+    get hooks() {
+        return this.host.hooks;
+    }
+    /**
+     * Whether the plugin's current instance is active on this replica (I-2): an active instance that
+     * registered no `usage.report` handler is not a subscriber, and its target is dropped.
+     */
+    isActive(plugin) {
+        return this.host.current(plugin) !== undefined;
+    }
+    /**
+     * Of these plugins, the ones no replica can serve right now (I-2): replicas report them (live
+     * `extension_replica_status` rows) and none has them loaded. Their targets are dropped at once
+     * instead of failing every attempt. See {@link queryUnservable}.
+     */
+    unservable(plugins) {
+        return queryUnservable(this.db, plugins, STALE_RECONCILE_PERIODS * this.config.get().extensionHost.reconcileIntervalSeconds);
+    }
+    /**
+     * Plugins that subscribe to `usage.report`, from deployment state (approved manifests in PG),
+     * not from this replica's loaded handlers. With `orgId`: only those the organization enabled.
+     * See {@link queryUsageSubscribers}.
+     */
+    subscribers(orgId) {
+        return queryUsageSubscribers(this.db, orgId);
+    }
+    jobOpts(jobId) {
+        return usageReportJobOpts(jobId, this.retry, this.config.get().hooks.usageReportKeepHours);
     }
     /**
      * Enqueues the delivery of one `usage` event. Returns without a job when no plugin subscribes.
@@ -175,41 +198,45 @@ export class UsageReporter {
             return;
         // M-2: no usage.report handler on this replica → no I/O at all. A plugin loaded only on other
         // replicas still gets the event: the sweeper re-offers it within about a minute and a half.
-        if (!this.d.hooks.has("usage.report"))
+        if (!this.hooks.has("usage.report"))
             return;
         try {
-            if ((await this.d.subscribers()).size === 0)
+            if ((await this.subscribers()).size === 0)
                 return;
-            await this.getQueue().add(JOB, { conversationId, seq }, this.jobOpts(conversationId, seq));
+            await this.queue.add(USAGE_REPORT_JOB, { conversationId, seq }, this.jobOpts(usageReportJobId(conversationId, seq)));
         }
         catch (err) {
-            this.d.log.warn("cannot enqueue usage report", { conversationId, seq, err });
+            this.log.warn("cannot enqueue usage report", { conversationId, seq, err });
         }
     }
     /**
      * One sweep (R13): re-enqueues the `usage` events created in `[watermark - overlap, now - lag]` in
      * pages, then advances the watermark to `now - lag`. With no subscriber it only advances. The first
-     * sweep starts at `now - lag - overlap`. Returns the number of events offered to the queue.
+     * sweep starts at `now - lag - overlap`. Also re-enqueues the `model_usage` rows of the same
+     * window that are neither reported nor given up (spec B §5.4),
+     * or, with no subscriber, marks those rows reported (nobody to deliver to). Returns the number of
+     * events and rows offered to the queue.
      */
     async sweep(opts = {}) {
         // bounds and keyset as full-precision timestamp text (microseconds), computed with the DB clock
         // unless a test fixes "now" (m-a, m-c)
         const to = opts.now !== undefined
             ? new Date(opts.now - USAGE_SWEEP_LAG_MS).toISOString()
-            : (await sql `SELECT (now() - make_interval(secs => ${USAGE_SWEEP_LAG_MS / 1000}))::text AS to`.execute(this.d.db)).rows[0]?.to;
+            : (await sql `SELECT (now() - make_interval(secs => ${USAGE_SWEEP_LAG_MS / 1000}))::text AS to`.execute(this.db)).rows[0]?.to;
         if (to === undefined)
             return 0;
-        const row = await this.d.db
+        const row = await this.db
             .selectFrom("usage_report_sweep")
             .select(sql `watermark::text`.as("watermark"))
             .where("id", "=", 1)
             .executeTakeFirst();
         const watermark = row?.watermark ?? to;
         let offered = 0;
-        if ((await this.d.subscribers()).size > 0) {
+        const subscribed = (await this.subscribers()).size > 0;
+        if (subscribed) {
             let after;
             for (;;) {
-                let q = this.d.db
+                let q = this.db
                     .selectFrom("conversation_events")
                     .select(["conversation_id", "seq", sql `created_at::text`.as("at")])
                     .where("type", "=", "usage")
@@ -225,10 +252,10 @@ export class UsageReporter {
                     .limit(SWEEP_PAGE)
                     .execute();
                 if (page.length > 0) {
-                    await this.getQueue().addBulk(page.map((r) => ({
-                        name: JOB,
+                    await this.queue.addBulk(page.map((r) => ({
+                        name: USAGE_REPORT_JOB,
                         data: { conversationId: r.conversation_id, seq: Number(r.seq) },
-                        opts: this.jobOpts(r.conversation_id, Number(r.seq)),
+                        opts: this.jobOpts(usageReportJobId(r.conversation_id, Number(r.seq))),
                     })));
                     offered += page.length;
                 }
@@ -238,7 +265,8 @@ export class UsageReporter {
                 after = { at: last.at, conv: last.conversation_id, seq: String(last.seq) };
             }
         }
-        await this.d.db
+        offered += await this.sweepModelUsage(watermark, to, subscribed);
+        await this.db
             .insertInto("usage_report_sweep")
             .values({ id: 1, watermark: sql `${to}::timestamptz` })
             .onConflict((oc) => oc.column("id").doUpdateSet({
@@ -248,80 +276,167 @@ export class UsageReporter {
             .execute();
         return offered;
     }
-    /** Starts the worker (worker role only). Job lock and stalled detection follow the run queue. */
-    startWorker() {
+    /**
+     * The `model_usage` half of {@link sweep}: rows neither reported nor given up that occurred in
+     * the same window as the events (`[watermark - overlap, to]`), oldest first. Bounded like the
+     * event sweep, so a row whose job failed for good is never offered again, even after its failed
+     * job left Redis (no repeated backlog, no second delivery to plugins already served).
+     */
+    async sweepModelUsage(watermark, to, subscribed) {
+        if (!subscribed) {
+            await this.db
+                .updateTable("model_usage")
+                .set({ reported_at: sql `now()` })
+                .where("reported_at", "is", null)
+                .where(sql `occurred_at <= ${to}::timestamptz`)
+                .execute();
+            return 0;
+        }
+        let offered = 0;
+        let after;
+        for (;;) {
+            let q = this.db
+                .selectFrom("model_usage")
+                .select(["id", sql `occurred_at::text`.as("at")])
+                .where("reported_at", "is", null)
+                .where("report_failed_at", "is", null)
+                .where(sql `occurred_at >= ${watermark}::timestamptz - make_interval(secs => ${USAGE_SWEEP_OVERLAP_MS / 1000})`)
+                .where(sql `occurred_at <= ${to}::timestamptz`);
+            if (after) {
+                q = q.where(sql `(occurred_at, id) > (${after.at}::timestamptz, ${after.id}::uuid)`);
+            }
+            const page = await q.orderBy("occurred_at").orderBy("id").limit(SWEEP_PAGE).execute();
+            if (page.length > 0) {
+                await this.queue.addBulk(page.map((r) => ({
+                    name: USAGE_REPORT_JOB,
+                    data: { usageId: r.id },
+                    opts: this.jobOpts(modelUsageJobId(r.id)),
+                })));
+                offered += page.length;
+            }
+            const last = page.at(-1);
+            if (!last || page.length < SWEEP_PAGE)
+                break;
+            after = { at: last.at, id: last.id };
+        }
+        return offered;
+    }
+    /**
+     * Starts consuming (worker role only): `worker` is the `@Processor` worker of
+     * {@link USAGE_REPORT_QUEUE} (built with the run queue's job lock and stalled detection).
+     */
+    startWorker(worker) {
         if (this.worker || this.closed)
             return;
-        const { jobLockMs, stalledIntervalMs } = this.d.config().runtime;
-        // the queue exists before any job runs: in-flight jobs fence and requeue through it, even on a
-        // replica that never enqueued (m-b)
-        this.getQueue();
-        this.workerConn = bullConnection(this.d.redisUrl, this.d.log);
-        this.worker = new Worker(USAGE_REPORT_QUEUE, (j, t) => withJobContext(USAGE_REPORT_QUEUE, j, () => this.process(j, t)), {
-            connection: this.workerConn,
-            concurrency: this.d.config().hooks.usageReportConcurrency,
-            lockDuration: jobLockMs,
-            stalledInterval: stalledIntervalMs,
-            maxStalledCount: 3,
-        });
-        this.worker.on("error", (err) => this.d.log.error("usage report worker error", { err }));
-        this.worker.on("failed", (j, err) => {
-            if (!j || err instanceof DelayedError || err.name === "DelayedError")
-                return;
-            // a definite drop (N-3) was already warned about (throttled); the failed job is the trace
-            if (err instanceof UnrecoverableError || err.name === "UnrecoverableError")
-                return;
-            const final = j.attemptsMade >= (j.opts.attempts ?? 1);
-            const fields = {
-                jobId: j.id,
-                conversationId: j.data.conversationId,
-                seq: j.data.seq,
-                pending: j.data.pending,
-                attempt: j.attemptsMade,
-                error: err.message,
-            };
-            if (final)
-                this.d.log.error("usage report given up after retries", fields);
-            else
-                this.d.log.warn("usage report attempt failed; retrying", fields);
-        });
+        this.worker = worker;
+        void worker.run().catch((err) => this.log.error("usage report worker error", { err }));
+    }
+    /** The worker's `error` event. */
+    onError(err) {
+        this.log.error("usage report worker error", { err });
     }
-    /** Stops the worker (in-flight calls are aborted, their jobs requeued) and the queue. */
+    /** A job failed (the worker's `failed` event): logs retries and the final failure. */
+    onFailed(j, err) {
+        if (!j || err instanceof DelayedError || err.name === "DelayedError")
+            return;
+        // a definite drop (N-3) was already warned about (throttled); the failed job is the trace
+        if (err instanceof UnrecoverableError || err.name === "UnrecoverableError")
+            return;
+        const final = j.attemptsMade >= (j.opts.attempts ?? 1);
+        const fields = {
+            jobId: j.id,
+            ...(j.data.usageId !== undefined
+                ? { usageId: j.data.usageId }
+                : { conversationId: j.data.conversationId, seq: j.data.seq }),
+            pending: j.data.pending,
+            attempt: j.attemptsMade,
+            error: err.message,
+        };
+        if (final) {
+            this.log.error("usage report given up after retries", fields);
+            void this.markFailed(j.data);
+        }
+        else
+            this.log.warn("usage report attempt failed; retrying", fields);
+    }
+    /**
+     * Stops: no more enqueues, in-flight calls are aborted (their jobs requeued) and the worker
+     * closes. The queue itself closes with the application.
+     */
     async stop() {
         this.closed = true;
         this.stopping.abort();
         const w = this.worker;
         this.worker = undefined;
-        // the worker first: its in-flight jobs still use the queue to fence and requeue
         await w?.close().catch(() => { });
-        const q = this.queue;
-        this.queue = undefined;
-        await q?.close().catch(() => { });
-        for (const c of [this.workerConn, this.queueConn]) {
-            if (c)
-                await c.quit().catch(() => c.disconnect());
-        }
-        this.workerConn = undefined;
-        this.queueConn = undefined;
     }
-    getQueue() {
-        if (!this.queue) {
-            if (this.closed)
-                throw new Error("usage report queue is closed");
-            this.queueConn = bullConnection(this.d.redisUrl, this.d.log);
-            this.queue = new Queue(USAGE_REPORT_QUEUE, { connection: this.queueConn });
-            this.queue.on("error", (err) => this.d.log.error("usage report queue error", { err }));
+    /** The payload of a job: a `usage` event's or a `model_usage` row's; null when it is gone. */
+    payloadOfJob(data) {
+        return data.usageId !== undefined
+            ? this.payloadOfUsage(data.usageId)
+            : this.payloadOf(data.conversationId, data.seq);
+    }
+    /**
+     * The payload of one `model_usage` row (spec B §5.4): purpose `embedding`, the calling extension,
+     * no conversation fields; `eventId` is the row id. Null when the row no longer exists.
+     */
+    async payloadOfUsage(id) {
+        const row = await this.db
+            .selectFrom("model_usage")
+            .select(["id", "org_id", "extension", "provider", "model", "input_tokens", "occurred_at"])
+            .where("id", "=", id)
+            .executeTakeFirst();
+        if (!row)
+            return null;
+        return {
+            eventId: row.id,
+            purpose: "embedding",
+            orgId: row.org_id,
+            extension: row.extension,
+            provider: row.provider,
+            model: row.model,
+            usage: {
+                inputTokens: row.input_tokens,
+                outputTokens: 0,
+                cacheReadTokens: 0,
+                cacheWriteTokens: 0,
+            },
+            occurredAt: new Date(row.occurred_at).toISOString(),
+        };
+    }
+    /** A `model_usage` job is done (delivered, or nothing left to deliver to): marks the row. */
+    async markReported(data) {
+        if (data.usageId === undefined)
+            return;
+        await this.db
+            .updateTable("model_usage")
+            .set({ reported_at: sql `now()` })
+            .where("id", "=", data.usageId)
+            .where("reported_at", "is", null)
+            .execute();
+    }
+    /** A `model_usage` job gave up after its last attempt: the sweep never offers the row again. */
+    async markFailed(data) {
+        if (data.usageId === undefined)
+            return;
+        try {
+            await this.db
+                .updateTable("model_usage")
+                .set({ report_failed_at: sql `now()` })
+                .where("id", "=", data.usageId)
+                .where("reported_at", "is", null)
+                .execute();
+        }
+        catch (err) {
+            this.log.warn("cannot mark a model usage report as given up", { usageId: data.usageId, err });
         }
-        return this.queue;
     }
     /** The payload of one usage event, read from the log; null when the event no longer exists. */
     async payloadOf(conversationId, seq) {
-        const row = await this.d.db
+        const row = await this.db
             .selectFrom("conversation_events as e")
             .innerJoin("conversations as c", "c.id", "e.conversation_id")
             .leftJoin("runs as r", "r.id", "e.run_id")
-            // a sub-agent's run: its dispatch says where it was dispatched from (spec §11)
-            .leftJoin("subagent_dispatches as sd", "sd.child_run_id", "e.run_id")
             .select([
             "e.payload",
             "e.run_id",
@@ -332,9 +447,6 @@ export class UsageReporter {
             "c.account_id",
             "c.kind",
             "r.trigger_account_id",
-            "sd.parent_conversation_id",
-            "sd.parent_run_id",
-            "sd.parent_call_id",
         ])
             .where("e.conversation_id", "=", conversationId)
             .where("e.seq", "=", String(seq))
@@ -342,6 +454,11 @@ export class UsageReporter {
         if (row?.type !== "usage")
             return null;
         const p = row.payload;
+        // a sub-agent's run: where it was dispatched from (spec §11), its own dispatch, else (a wake
+        // run nobody dispatched, agent map spec §5.3) its conversation's latest one
+        const parent = row.kind === "subagent" && row.run_id !== null
+            ? await deliveryRowOfChildRun(this.db, row.run_id)
+            : null;
         return {
             eventId: `${conversationId}/${seq}`,
             purpose: p.purpose ?? "step",
@@ -352,13 +469,11 @@ export class UsageReporter {
             // a sub-agent's calls report "chat" (spec §11: the SDK union is not widened)
             conversationKind: row.kind === "eval" ? "eval" : "chat",
             runId: row.run_id ?? "",
-            ...(row.parent_conversation_id !== null &&
-                row.parent_run_id !== null &&
-                row.parent_call_id !== null && {
+            ...(parent !== null && {
                 parent: {
-                    conversationId: row.parent_conversation_id,
-                    runId: row.parent_run_id,
-                    callId: row.parent_call_id,
+                    conversationId: parent.parentConversationId,
+                    runId: parent.parentRunId,
+                    callId: parent.parentCallId,
                 },
             }),
             provider: p.provider,
@@ -378,48 +493,50 @@ export class UsageReporter {
      * dropped can stay, so a plugin another execution delivered never comes back.
      */
     async writePending(j, next, dropped) {
-        const fresh = j.id === undefined ? undefined : await Job.fromId(this.getQueue(), j.id);
+        const fresh = j.id === undefined ? undefined : await Job.fromId(this.queue, j.id);
         const data = fresh?.data;
         const stored = data?.pending ?? j.data.pending;
         const storedDropped = data?.dropped ?? j.data.dropped ?? {};
         const allowed = stored === undefined ? undefined : [...stored, ...Object.keys(storedDropped)];
         const keep = [...dropped].filter(([p]) => allowed === undefined || allowed.includes(p));
         const out = {
-            conversationId: j.data.conversationId,
-            seq: j.data.seq,
+            ...keyOf(j.data),
             pending: narrowPending(allowed, next),
             ...(keep.length > 0 && { dropped: Object.fromEntries(keep) }),
         };
         await j.updateData(out);
         j.data = out;
     }
+    /** One delivery job (the `@Processor` of {@link USAGE_REPORT_QUEUE} calls it). */
     async process(j, token) {
-        const { conversationId, seq } = j.data;
-        const payload = await this.payloadOf(conversationId, seq);
+        const payload = await this.payloadOfJob(j.data);
+        // conversation (or org) deleted meanwhile: nothing to report
         if (payload === null)
-            return; // conversation deleted meanwhile: nothing to report
-        const subscribers = await this.d.subscribers(payload.orgId);
+            return;
+        const subscribers = await this.subscribers(payload.orgId);
         if (j.data.pending === undefined) {
             // first attempt: the targets come from deployment state, recorded so later attempts (on any
             // replica) deliver exactly the plugins that have not succeeded yet
             const targets = [...subscribers];
-            await j.updateData({ conversationId, seq, pending: targets });
+            await j.updateData({ ...keyOf(j.data), pending: targets });
             j.data.pending = targets;
         }
         // plugins the organization disabled meanwhile are dropped (not a failure)
         const targets = (j.data.pending ?? []).filter((p) => subscribers.has(p));
-        if (targets.length === 0)
+        if (targets.length === 0) {
+            await this.markReported(j.data);
             return;
+        }
         const byPlugin = new Map();
-        for (const h of this.d.hooks.handlers("usage.report")) {
+        for (const h of this.hooks.handlers("usage.report")) {
             byPlugin.set(h.plugin, [...(byPlugin.get(h.plugin) ?? []), h.handler]);
         }
         const frozen = deepFreeze(structuredClone(payload));
-        const timeoutMs = this.d.config().hooks.timeoutMs;
+        const timeoutMs = this.config.get().hooks.timeoutMs;
         const results = await Promise.allSettled(targets.map(async (plugin) => {
             const handlers = byPlugin.get(plugin);
             if (!handlers || handlers.length === 0) {
-                throw new Undeliverable(plugin, this.d.isActive?.(plugin) ? "no_handler" : "not_loaded");
+                throw new Undeliverable(plugin, this.isActive(plugin) ? "no_handler" : "not_loaded");
             }
             for (const h of handlers)
                 await this.call(h, frozen, timeoutMs);
@@ -441,9 +558,15 @@ export class UsageReporter {
                     notHere.push(plugin);
                 return;
             }
+            // layering spec A §6: the extension's effective config is not valid in this organization, so
+            // it is not called there (an observe hook skipped); dropped, not retried
+            if (r.reason instanceof HookNeedsConfigError) {
+                dropped.set(plugin, "needs configuration in this organization");
+                return;
+            }
             failed.push(plugin);
             if (!this.stopping.signal.aborted) {
-                this.d.log.warn("usage.report delivery to a plugin failed", {
+                this.log.warn("usage.report delivery to a plugin failed", {
                     plugin,
                     eventId: payload.eventId,
                     error: errMsg(r.reason),
@@ -454,7 +577,7 @@ export class UsageReporter {
         // target is dropped only when it is definitely unservable; unknown ones fail the job as before.
         const final = j.attemptsMade + 1 >= (j.opts.attempts ?? 1);
         const gone = final && notHere.length > 0 && !this.stopping.signal.aborted
-            ? await this.unservable(notHere)
+            ? await this.unservableOrNone(notHere)
             : new Set();
         for (const plugin of notHere) {
             if (gone.has(plugin))
@@ -482,6 +605,8 @@ export class UsageReporter {
         // m-1: pending = the dropped plugins only, so a manual retry reaches them and never the
         // plugins already served
         await this.writePending(j, [...allDropped.keys()], allDropped);
+        // delivered to every target still servable (a definite drop is final too): the row is done
+        await this.markReported(j.data);
         const traced = new Map([...allDropped].filter(([p]) => j.data.dropped?.[p] !== undefined));
         if (traced.size === 0)
             return;
@@ -496,15 +621,13 @@ export class UsageReporter {
         }
         throw new UnrecoverableError(`usage.report delivered to the rest;${droppedNote(traced)}`);
     }
-    /** {@link UsageReporterDeps.unservable}; a lookup failure counts as "unknown" (keep retrying). */
-    async unservable(plugins) {
-        if (!this.d.unservable)
-            return new Set();
+    /** {@link unservable}; a lookup failure counts as "unknown" (keep retrying). */
+    async unservableOrNone(plugins) {
         try {
-            return await this.d.unservable(plugins);
+            return await this.unservable(plugins);
         }
         catch (err) {
-            this.d.log.warn("cannot read plugin replica status for usage.report", { err });
+            this.log.warn("cannot read plugin replica status for usage.report", { err });
             return new Set();
         }
     }
@@ -512,15 +635,13 @@ export class UsageReporter {
     async warnOnce(plugin, kind, msg, fields) {
         let first = true;
         try {
-            this.getQueue();
-            const key = `capk:usage-warn:${plugin}:${kind}`;
-            first = (await this.queueConn?.set(key, "1", "PX", WARN_WINDOW_MS, "NX")) !== null;
+            first = await this.locks.lease(`capk:usage-warn:${plugin}:${kind}`, "1", WARN_WINDOW_MS);
         }
         catch {
             // Redis unavailable: log anyway rather than hiding it
         }
         if (first)
-            this.d.log.warn(msg, fields);
+            this.log.warn(msg, fields);
     }
     /** One handler call with its own signal (timeout or replica stop aborts it); rejects on either. */
     async call(h, payload, timeoutMs) {
@@ -548,4 +669,57 @@ export class UsageReporter {
             this.stopping.signal.removeEventListener("abort", onStop);
         }
     }
-}
+};
+UsageReporter = __decorate([
+    Injectable(),
+    __param(0, Inject(KernelDb)),
+    __param(1, Inject(getQueueToken(USAGE_REPORT_QUEUE))),
+    __param(2, Inject(ExtensionHost)),
+    __param(3, Inject(KernelConfigService)),
+    __param(4, Inject(LockService)),
+    __param(5, Inject(KERNEL_LOGGER)),
+    __param(6, Optional()),
+    __param(6, Inject(USAGE_REPORT_RETRY))
+], UsageReporter);
+export { UsageReporter };
+/**
+ * The usage sweeper (R13), every minute, once across replicas: {@link UsageReporter.sweep}.
+ * Replaces the maintenance scheduler `usage-report-sweep-every` (retired at worker start).
+ */
+let UsageSweepJob = class UsageSweepJob {
+    reporter;
+    lockService;
+    log;
+    constructor(reporter, lockService, log) {
+        this.reporter = reporter;
+        this.lockService = lockService;
+        this.log = log;
+    }
+    async run() {
+        await this.reporter.sweep();
+    }
+};
+__decorate([
+    LockedCron("* * * * *", { name: USAGE_SWEEP_JOB })
+], UsageSweepJob.prototype, "run", null);
+UsageSweepJob = __decorate([
+    Injectable(),
+    __param(0, Inject(UsageReporter)),
+    __param(1, Inject(LockService)),
+    __param(2, Inject(KERNEL_LOGGER))
+], UsageSweepJob);
+export { UsageSweepJob };
+/**
+ * `usage.report` delivery: the producer on every role; the consumer and the sweep ({@link UsageSweepJob})
+ * on the worker role (`WorkerModule`).
+ */
+let UsageReportModule = class UsageReportModule {
+};
+UsageReportModule = __decorate([
+    Module({
+        imports: [ExtensionsModule, UsageReportJobsModule],
+        providers: [UsageReporter],
+        exports: [UsageReporter, UsageReportJobsModule],
+    })
+], UsageReportModule);
+export { UsageReportModule };