@capybara-hq/kernel 0.1.0-rc.0 → 0.1.0-rc.2

package/dist/runtime/usage-report.js

@@ -1,8 +1,27 @@
-import { DelayedError, Job, Queue, UnrecoverableError, Worker } from "bullmq";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+import { getQueueToken } from "@nestjs/bullmq";
+import { Inject, Injectable, Module, Optional } from "@nestjs/common";
+import { DelayedError, Job, UnrecoverableError, } from "bullmq";
 import { sql } from "kysely";
 import { enabledExtensionsQuery } from "../extensions/enabled.js";
+import { ExtensionsModule } from "../extensions/extensions.module.js";
 import { deepFreeze, errMsg } from "../extensions/hooks.js";
-import { bullConnection } from "./run-queue.js";
+import { ExtensionHost } from "../extensions/host.js";
+import { STALE_RECONCILE_PERIODS } from "../extensions/lifecycle-topics.js";
+import { KernelConfigService } from "../infra/config/kernel-config.service.js";
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { LockService } from "../infra/lock/lock.service.js";
+import { QueueModule } from "../infra/queue/queue.module.js";
+import { LockedCron } from "../infra/schedule/locked-cron.decorator.js";
+import { KERNEL_LOGGER } from "../infra/tokens.js";
 /** Queue of `usage.report` deliveries (no `:` in the name). */
 export const USAGE_REPORT_QUEUE = "cap-kernel-usage-report";
 /** One job per `usage` event (no `:`: conversation ids are UUIDs, seq is a number). */
@@ -11,6 +30,8 @@ export const usageReportJobId = (conversationId, seq) => `usage-${conversationId
 export const USAGE_REPORT_ATTEMPTS = 6;
 /** Retry backoff: 2 s, doubling (2, 4, 8, 16, 32 s). */
 export const USAGE_REPORT_BACKOFF_MS = 2000;
+/** Optional provider: the {@link UsageReportRetry} instead of the defaults above (tests). */
+export const USAGE_REPORT_RETRY = Symbol("USAGE_REPORT_RETRY");
 /** Failed jobs (given up) are kept for inspection, at most {@link KEEP_FAILED_MAX} of them. */
 const KEEP_FAILED_S = 7 * 86_400;
 /** Upper bound on kept failed jobs (I-2): a long outage of a billing endpoint cannot grow Redis. */
@@ -22,10 +43,10 @@ const WARN_WINDOW_MS = 60_000;
  * ~1.1 usage events per second sustained for a day; dedupe needs the jobs only for the sweeper overlap.
  */
 const KEEP_COMPLETED_MAX = 100_000;
-/** Maintenance job of the sweeper (no `:`), its scheduler id and period. */
+/** The sweeper's cron job name (no `:`; every minute, once across replicas). */
 export const USAGE_SWEEP_JOB = "usage-report-sweep";
-export const USAGE_SWEEP_SCHEDULER_ID = "usage-report-sweep-every";
-export const USAGE_SWEEP_EVERY_MS = 60_000;
+/** The maintenance scheduler the sweep ran on before it became a `@LockedCron` (retired). */
+export { USAGE_SWEEP_SCHEDULER_ID } from "./job-ids.js";
 /** The sweep leaves the newest events to the fast path (and to transactions still committing). */
 export const USAGE_SWEEP_LAG_MS = 30_000;
 /**
@@ -142,24 +163,74 @@ export function narrowPending(stored, failed) {
  * as are conversation title calls (`purpose: "title"`, no run: `runId` is "").
  * Single-step eval replays and judge requests log no usage: not reported (M7).
  */
-export class UsageReporter {
-    d;
+let UsageReporter = class UsageReporter {
+    kdb;
     queue;
-    queueConn;
+    host;
+    config;
+    locks;
+    log;
+    retry;
     worker;
-    workerConn;
     closed = false;
     stopping = new AbortController();
-    constructor(d) {
-        this.d = d;
+    constructor(kdb, queue, 
+    /**
+     * The host's hook registry: handlers of the current local plugin generations (through
+     * inflight.run), and `has("usage.report")`, the in-process check that lets `enqueue` skip all
+     * I/O without handlers; and whether a plugin's current instance is active here.
+     */
+    host, config, 
+    /** The warning throttle's markers (`capk:usage-warn:<plugin>:<kind>`, SET NX PX). */
+    locks, log, retry = {
+        attempts: USAGE_REPORT_ATTEMPTS,
+        backoffMs: USAGE_REPORT_BACKOFF_MS,
+    }) {
+        this.kdb = kdb;
+        this.queue = queue;
+        this.host = host;
+        this.config = config;
+        this.locks = locks;
+        this.log = log;
+        this.retry = retry;
+        queue.on("error", (err) => this.log.error("usage report queue error", { err }));
+    }
+    get db() {
+        return this.kdb.db;
+    }
+    get hooks() {
+        return this.host.hooks;
+    }
+    /**
+     * Whether the plugin's current instance is active on this replica (I-2): an active instance that
+     * registered no `usage.report` handler is not a subscriber, and its target is dropped.
+     */
+    isActive(plugin) {
+        return this.host.current(plugin) !== undefined;
+    }
+    /**
+     * Of these plugins, the ones no replica can serve right now (I-2): replicas report them (live
+     * `extension_replica_status` rows) and none has them loaded. Their targets are dropped at once
+     * instead of failing every attempt. See {@link queryUnservable}.
+     */
+    unservable(plugins) {
+        return queryUnservable(this.db, plugins, STALE_RECONCILE_PERIODS * this.config.get().extensionHost.reconcileIntervalSeconds);
+    }
+    /**
+     * Plugins that subscribe to `usage.report`, from deployment state (approved manifests in PG),
+     * not from this replica's loaded handlers. With `orgId`: only those the organization enabled.
+     * See {@link queryUsageSubscribers}.
+     */
+    subscribers(orgId) {
+        return queryUsageSubscribers(this.db, orgId);
     }
     jobOpts(conversationId, seq) {
         return {
             jobId: usageReportJobId(conversationId, seq),
-            attempts: this.d.retry?.attempts ?? USAGE_REPORT_ATTEMPTS,
-            backoff: { type: "exponential", delay: this.d.retry?.backoffMs ?? USAGE_REPORT_BACKOFF_MS },
+            attempts: this.retry.attempts,
+            backoff: { type: "exponential", delay: this.retry.backoffMs },
             removeOnComplete: {
-                age: this.d.config().hooks.usageReportKeepHours * 3600,
+                age: this.config.get().hooks.usageReportKeepHours * 3600,
                 count: KEEP_COMPLETED_MAX,
             },
             removeOnFail: { age: KEEP_FAILED_S, count: KEEP_FAILED_MAX },
@@ -174,15 +245,15 @@ export class UsageReporter {
             return;
         // M-2: no usage.report handler on this replica → no I/O at all. A plugin loaded only on other
         // replicas still gets the event: the sweeper re-offers it within about a minute and a half.
-        if (!this.d.hooks.has("usage.report"))
+        if (!this.hooks.has("usage.report"))
             return;
         try {
-            if ((await this.d.subscribers()).size === 0)
+            if ((await this.subscribers()).size === 0)
                 return;
-            await this.getQueue().add(JOB, { conversationId, seq }, this.jobOpts(conversationId, seq));
+            await this.queue.add(JOB, { conversationId, seq }, this.jobOpts(conversationId, seq));
         }
         catch (err) {
-            this.d.log.warn("cannot enqueue usage report", { conversationId, seq, err });
+            this.log.warn("cannot enqueue usage report", { conversationId, seq, err });
         }
     }
     /**
@@ -195,20 +266,20 @@ export class UsageReporter {
         // unless a test fixes "now" (m-a, m-c)
         const to = opts.now !== undefined
             ? new Date(opts.now - USAGE_SWEEP_LAG_MS).toISOString()
-            : (await sql `SELECT (now() - make_interval(secs => ${USAGE_SWEEP_LAG_MS / 1000}))::text AS to`.execute(this.d.db)).rows[0]?.to;
+            : (await sql `SELECT (now() - make_interval(secs => ${USAGE_SWEEP_LAG_MS / 1000}))::text AS to`.execute(this.db)).rows[0]?.to;
         if (to === undefined)
             return 0;
-        const row = await this.d.db
+        const row = await this.db
             .selectFrom("usage_report_sweep")
             .select(sql `watermark::text`.as("watermark"))
             .where("id", "=", 1)
             .executeTakeFirst();
         const watermark = row?.watermark ?? to;
         let offered = 0;
-        if ((await this.d.subscribers()).size > 0) {
+        if ((await this.subscribers()).size > 0) {
             let after;
             for (;;) {
-                let q = this.d.db
+                let q = this.db
                     .selectFrom("conversation_events")
                     .select(["conversation_id", "seq", sql `created_at::text`.as("at")])
                     .where("type", "=", "usage")
@@ -224,7 +295,7 @@ export class UsageReporter {
                     .limit(SWEEP_PAGE)
                     .execute();
                 if (page.length > 0) {
-                    await this.getQueue().addBulk(page.map((r) => ({
+                    await this.queue.addBulk(page.map((r) => ({
                         name: JOB,
                         data: { conversationId: r.conversation_id, seq: Number(r.seq) },
                         opts: this.jobOpts(r.conversation_id, Number(r.seq)),
@@ -237,7 +308,7 @@ export class UsageReporter {
                 after = { at: last.at, conv: last.conversation_id, seq: String(last.seq) };
             }
         }
-        await this.d.db
+        await this.db
             .insertInto("usage_report_sweep")
             .values({ id: 1, watermark: sql `${to}::timestamptz` })
             .onConflict((oc) => oc.column("id").doUpdateSet({
@@ -247,75 +318,55 @@ export class UsageReporter {
             .execute();
         return offered;
     }
-    /** Starts the worker (worker role only). Job lock and stalled detection follow the run queue. */
-    startWorker() {
+    /**
+     * Starts consuming (worker role only): `worker` is the `@Processor` worker of
+     * {@link USAGE_REPORT_QUEUE} (built with the run queue's job lock and stalled detection).
+     */
+    startWorker(worker) {
         if (this.worker || this.closed)
             return;
-        const { jobLockMs, stalledIntervalMs } = this.d.config().runtime;
-        // the queue exists before any job runs: in-flight jobs fence and requeue through it, even on a
-        // replica that never enqueued (m-b)
-        this.getQueue();
-        this.workerConn = bullConnection(this.d.redisUrl, this.d.log);
-        this.worker = new Worker(USAGE_REPORT_QUEUE, (j, t) => this.process(j, t), {
-            connection: this.workerConn,
-            concurrency: this.d.config().hooks.usageReportConcurrency,
-            lockDuration: jobLockMs,
-            stalledInterval: stalledIntervalMs,
-            maxStalledCount: 3,
-        });
-        this.worker.on("error", (err) => this.d.log.error("usage report worker error", { err }));
-        this.worker.on("failed", (j, err) => {
-            if (!j || err instanceof DelayedError || err.name === "DelayedError")
-                return;
-            // a definite drop (N-3) was already warned about (throttled); the failed job is the trace
-            if (err instanceof UnrecoverableError || err.name === "UnrecoverableError")
-                return;
-            const final = j.attemptsMade >= (j.opts.attempts ?? 1);
-            const fields = {
-                jobId: j.id,
-                conversationId: j.data.conversationId,
-                seq: j.data.seq,
-                pending: j.data.pending,
-                attempt: j.attemptsMade,
-                error: err.message,
-            };
-            if (final)
-                this.d.log.error("usage report given up after retries", fields);
-            else
-                this.d.log.warn("usage report attempt failed; retrying", fields);
-        });
+        this.worker = worker;
+        void worker.run().catch((err) => this.log.error("usage report worker error", { err }));
+    }
+    /** The worker's `error` event. */
+    onError(err) {
+        this.log.error("usage report worker error", { err });
     }
-    /** Stops the worker (in-flight calls are aborted, their jobs requeued) and the queue. */
+    /** A job failed (the worker's `failed` event): logs retries and the final failure. */
+    onFailed(j, err) {
+        if (!j || err instanceof DelayedError || err.name === "DelayedError")
+            return;
+        // a definite drop (N-3) was already warned about (throttled); the failed job is the trace
+        if (err instanceof UnrecoverableError || err.name === "UnrecoverableError")
+            return;
+        const final = j.attemptsMade >= (j.opts.attempts ?? 1);
+        const fields = {
+            jobId: j.id,
+            conversationId: j.data.conversationId,
+            seq: j.data.seq,
+            pending: j.data.pending,
+            attempt: j.attemptsMade,
+            error: err.message,
+        };
+        if (final)
+            this.log.error("usage report given up after retries", fields);
+        else
+            this.log.warn("usage report attempt failed; retrying", fields);
+    }
+    /**
+     * Stops: no more enqueues, in-flight calls are aborted (their jobs requeued) and the worker
+     * closes. The queue itself closes with the application.
+     */
     async stop() {
         this.closed = true;
         this.stopping.abort();
         const w = this.worker;
         this.worker = undefined;
-        // the worker first: its in-flight jobs still use the queue to fence and requeue
         await w?.close().catch(() => { });
-        const q = this.queue;
-        this.queue = undefined;
-        await q?.close().catch(() => { });
-        for (const c of [this.workerConn, this.queueConn]) {
-            if (c)
-                await c.quit().catch(() => c.disconnect());
-        }
-        this.workerConn = undefined;
-        this.queueConn = undefined;
-    }
-    getQueue() {
-        if (!this.queue) {
-            if (this.closed)
-                throw new Error("usage report queue is closed");
-            this.queueConn = bullConnection(this.d.redisUrl, this.d.log);
-            this.queue = new Queue(USAGE_REPORT_QUEUE, { connection: this.queueConn });
-            this.queue.on("error", (err) => this.d.log.error("usage report queue error", { err }));
-        }
-        return this.queue;
     }
     /** The payload of one usage event, read from the log; null when the event no longer exists. */
     async payloadOf(conversationId, seq) {
-        const row = await this.d.db
+        const row = await this.db
             .selectFrom("conversation_events as e")
             .innerJoin("conversations as c", "c.id", "e.conversation_id")
             .leftJoin("runs as r", "r.id", "e.run_id")
@@ -377,7 +428,7 @@ export class UsageReporter {
      * dropped can stay, so a plugin another execution delivered never comes back.
      */
     async writePending(j, next, dropped) {
-        const fresh = j.id === undefined ? undefined : await Job.fromId(this.getQueue(), j.id);
+        const fresh = j.id === undefined ? undefined : await Job.fromId(this.queue, j.id);
         const data = fresh?.data;
         const stored = data?.pending ?? j.data.pending;
         const storedDropped = data?.dropped ?? j.data.dropped ?? {};
@@ -392,12 +443,13 @@ export class UsageReporter {
         await j.updateData(out);
         j.data = out;
     }
+    /** One delivery job (the `@Processor` of {@link USAGE_REPORT_QUEUE} calls it). */
     async process(j, token) {
         const { conversationId, seq } = j.data;
         const payload = await this.payloadOf(conversationId, seq);
         if (payload === null)
             return; // conversation deleted meanwhile: nothing to report
-        const subscribers = await this.d.subscribers(payload.orgId);
+        const subscribers = await this.subscribers(payload.orgId);
         if (j.data.pending === undefined) {
             // first attempt: the targets come from deployment state, recorded so later attempts (on any
             // replica) deliver exactly the plugins that have not succeeded yet
@@ -410,15 +462,15 @@ export class UsageReporter {
         if (targets.length === 0)
             return;
         const byPlugin = new Map();
-        for (const h of this.d.hooks.handlers("usage.report")) {
+        for (const h of this.hooks.handlers("usage.report")) {
             byPlugin.set(h.plugin, [...(byPlugin.get(h.plugin) ?? []), h.handler]);
         }
         const frozen = deepFreeze(structuredClone(payload));
-        const timeoutMs = this.d.config().hooks.timeoutMs;
+        const timeoutMs = this.config.get().hooks.timeoutMs;
         const results = await Promise.allSettled(targets.map(async (plugin) => {
             const handlers = byPlugin.get(plugin);
             if (!handlers || handlers.length === 0) {
-                throw new Undeliverable(plugin, this.d.isActive?.(plugin) ? "no_handler" : "not_loaded");
+                throw new Undeliverable(plugin, this.isActive(plugin) ? "no_handler" : "not_loaded");
             }
             for (const h of handlers)
                 await this.call(h, frozen, timeoutMs);
@@ -442,7 +494,7 @@ export class UsageReporter {
             }
             failed.push(plugin);
             if (!this.stopping.signal.aborted) {
-                this.d.log.warn("usage.report delivery to a plugin failed", {
+                this.log.warn("usage.report delivery to a plugin failed", {
                     plugin,
                     eventId: payload.eventId,
                     error: errMsg(r.reason),
@@ -453,7 +505,7 @@ export class UsageReporter {
         // target is dropped only when it is definitely unservable; unknown ones fail the job as before.
         const final = j.attemptsMade + 1 >= (j.opts.attempts ?? 1);
         const gone = final && notHere.length > 0 && !this.stopping.signal.aborted
-            ? await this.unservable(notHere)
+            ? await this.unservableOrNone(notHere)
             : new Set();
         for (const plugin of notHere) {
             if (gone.has(plugin))
@@ -495,15 +547,13 @@ export class UsageReporter {
         }
         throw new UnrecoverableError(`usage.report delivered to the rest;${droppedNote(traced)}`);
     }
-    /** {@link UsageReporterDeps.unservable}; a lookup failure counts as "unknown" (keep retrying). */
-    async unservable(plugins) {
-        if (!this.d.unservable)
-            return new Set();
+    /** {@link unservable}; a lookup failure counts as "unknown" (keep retrying). */
+    async unservableOrNone(plugins) {
         try {
-            return await this.d.unservable(plugins);
+            return await this.unservable(plugins);
         }
         catch (err) {
-            this.d.log.warn("cannot read plugin replica status for usage.report", { err });
+            this.log.warn("cannot read plugin replica status for usage.report", { err });
             return new Set();
         }
     }
@@ -511,15 +561,13 @@ export class UsageReporter {
     async warnOnce(plugin, kind, msg, fields) {
         let first = true;
         try {
-            this.getQueue();
-            const key = `capk:usage-warn:${plugin}:${kind}`;
-            first = (await this.queueConn?.set(key, "1", "PX", WARN_WINDOW_MS, "NX")) !== null;
+            first = await this.locks.lease(`capk:usage-warn:${plugin}:${kind}`, "1", WARN_WINDOW_MS);
         }
         catch {
             // Redis unavailable: log anyway rather than hiding it
         }
         if (first)
-            this.d.log.warn(msg, fields);
+            this.log.warn(msg, fields);
     }
     /** One handler call with its own signal (timeout or replica stop aborts it); rejects on either. */
     async call(h, payload, timeoutMs) {
@@ -547,4 +595,57 @@ export class UsageReporter {
             this.stopping.signal.removeEventListener("abort", onStop);
         }
     }
-}
+};
+UsageReporter = __decorate([
+    Injectable(),
+    __param(0, Inject(KernelDb)),
+    __param(1, Inject(getQueueToken(USAGE_REPORT_QUEUE))),
+    __param(2, Inject(ExtensionHost)),
+    __param(3, Inject(KernelConfigService)),
+    __param(4, Inject(LockService)),
+    __param(5, Inject(KERNEL_LOGGER)),
+    __param(6, Optional()),
+    __param(6, Inject(USAGE_REPORT_RETRY))
+], UsageReporter);
+export { UsageReporter };
+/**
+ * The usage sweeper (R13), every minute, once across replicas: {@link UsageReporter.sweep}.
+ * Replaces the maintenance scheduler `usage-report-sweep-every` (retired at worker start).
+ */
+let UsageSweepJob = class UsageSweepJob {
+    reporter;
+    lockService;
+    log;
+    constructor(reporter, lockService, log) {
+        this.reporter = reporter;
+        this.lockService = lockService;
+        this.log = log;
+    }
+    async run() {
+        await this.reporter.sweep();
+    }
+};
+__decorate([
+    LockedCron("* * * * *", { name: USAGE_SWEEP_JOB })
+], UsageSweepJob.prototype, "run", null);
+UsageSweepJob = __decorate([
+    Injectable(),
+    __param(0, Inject(UsageReporter)),
+    __param(1, Inject(LockService)),
+    __param(2, Inject(KERNEL_LOGGER))
+], UsageSweepJob);
+export { UsageSweepJob };
+/**
+ * `usage.report` delivery: the producer on every role; the consumer and the sweep ({@link UsageSweepJob})
+ * on the worker role (`WorkerModule`).
+ */
+let UsageReportModule = class UsageReportModule {
+};
+UsageReportModule = __decorate([
+    Module({
+        imports: [ExtensionsModule, QueueModule.registerQueue(USAGE_REPORT_QUEUE)],
+        providers: [UsageReporter],
+        exports: [UsageReporter],
+    })
+], UsageReportModule);
+export { UsageReportModule };