@capybara-hq/kernel 0.1.0-rc.25 → 0.1.0-rc.26

package/dist/infra/outbox/outbox.service.js

@@ -21,6 +21,29 @@ export const OUTBOX_FAST_PATH = Symbol("OUTBOX_FAST_PATH");
  * logging an error (spec §4, designer's call: "throws in tests, logs in production").
  */
 export const OUTBOX_STRICT_CONFLICTS = Symbol("OUTBOX_STRICT_CONFLICTS");
+/** Jobs per insert statement of {@link OutboxService.addMany} (6 parameters each). */
+const ADD_MANY_CHUNK = 1000;
+/** The row of `job` (throws on a job id with `:`). */
+function rowOf(job) {
+    if (job.jobId.includes(":")) {
+        throw new Error(`outbox job id "${job.jobId}" must not contain ":" (BullMQ uses it as its key separator); ` +
+            'join the id\'s parts with "-" instead');
+    }
+    return {
+        queue: job.queue,
+        job_name: job.name,
+        job_id: job.jobId,
+        data: JSON.stringify(job.data ?? null),
+        run_at: job.runAt ?? null,
+        org_id: job.orgId ?? null,
+    };
+}
+/**
+ * Only a job due now goes through the fast path (ruling R20): a delayed one (a backstop, a
+ * timeout) is claimed by the relay within its next tick and added with its remaining delay, so the
+ * commit's caller spends no claim transaction on work due in seconds or hours.
+ */
+const dueNow = (job) => job.runAt === undefined || job.runAt.getTime() <= Date.now();
 /**
  * The ids each open transaction inserted, keyed by its transaction store: lives only as long as
  * that transaction (a WeakMap entry, no state shared across requests).
@@ -51,31 +74,16 @@ let OutboxService = class OutboxService {
      * conflicts, else logs an error. Undefined data is stored as JSON null.
      */
     async add(job) {
-        if (job.jobId.includes(":")) {
-            throw new Error(`outbox job id "${job.jobId}" must not contain ":" (BullMQ uses it as its key separator); ` +
-                'join the id\'s parts with "-" instead');
-        }
         const json = JSON.stringify(job.data ?? null);
         const row = await this.db
             .insertInto("outbox")
-            .values({
-            queue: job.queue,
-            job_name: job.name,
-            job_id: job.jobId,
-            data: json,
-            run_at: job.runAt ?? null,
-            org_id: job.orgId ?? null,
-        })
+            .values(rowOf(job))
             .onConflict((oc) => oc.columns(["queue", "job_id"]).where("dead_at", "is", null).doNothing())
             .returning("id")
             .executeTakeFirst();
         if (row) {
-            // only a job due now goes through the fast path (ruling R20): a delayed one (a backstop, a
-            // timeout) is claimed by the relay within its next tick and added with its remaining delay,
-            // so the commit's caller spends no claim transaction on work due in seconds or hours
-            if (job.runAt === undefined || job.runAt.getTime() <= Date.now()) {
+            if (dueNow(job))
                 this.deliverAfterCommit(row.id);
-            }
             return true;
         }
         const stored = await this.db
@@ -100,6 +108,29 @@ let OutboxService = class OutboxService {
         }
         return false;
     }
+    /**
+     * {@link add} for many jobs in one multi-row insert per {@link ADD_MANY_CHUNK} jobs (an eval
+     * run's cases). A job whose (queue, jobId) is already owed is skipped without comparing its data.
+     * Returns how many rows it inserted.
+     */
+    async addMany(jobs) {
+        let inserted = 0;
+        for (let i = 0; i < jobs.length; i += ADD_MANY_CHUNK) {
+            const chunk = jobs.slice(i, i + ADD_MANY_CHUNK);
+            const rows = await this.db
+                .insertInto("outbox")
+                .values(chunk.map(rowOf))
+                .onConflict((oc) => oc.columns(["queue", "job_id"]).where("dead_at", "is", null).doNothing())
+                .returning(["id", "run_at"])
+                .execute();
+            for (const r of rows) {
+                if (dueNow({ runAt: r.run_at ?? undefined }))
+                    this.deliverAfterCommit(r.id);
+            }
+            inserted += rows.length;
+        }
+        return inserted;
+    }
     /** Collects `id` for its transaction; the first id registers the transaction's one hint. */
     deliverAfterCommit(id) {
         // `mandatory` guarantees a live store here
@@ -119,6 +150,9 @@ let OutboxService = class OutboxService {
 __decorate([
     Transactional({ propagation: "mandatory" })
 ], OutboxService.prototype, "add", null);
+__decorate([
+    Transactional({ propagation: "mandatory" })
+], OutboxService.prototype, "addMany", null);
 OutboxService = __decorate([
     Injectable(),
     __param(0, Inject(TransactionHost)),

package/dist/infra/outbox/registry.d.ts

@@ -145,6 +145,21 @@ export declare const OUTBOX_JOBS: {
             readonly removeOnFail: true;
         };
     };
+    readonly "channel-resume": {
+        readonly queue: "cap-kernel-channels";
+        readonly idempotent: "the catch-up re-reads which of the connection's conversations lag their log and adds their delivery jobs, and a delivery resumes from the delivered seq under a per-conversation lock, so a second catch-up sends nothing twice";
+        readonly processedIn: "packages/kernel/src/channels/channels.module.ts";
+        readonly opts: {
+            readonly attempts: 5;
+            readonly backoff: {
+                readonly type: "exponential";
+                readonly delay: 5000;
+            };
+            readonly priority: 10;
+            readonly removeOnComplete: true;
+            readonly removeOnFail: true;
+        };
+    };
     readonly "scope-item": {
         readonly queue: "cap-kernel-plugin-scope";
         readonly idempotent: "the sync converges each (item, org) from the scope, switches and installs it reads under the org's lock, and the installer re-checks under the org row lock, so a second run changes nothing";
@@ -221,6 +236,43 @@ export declare const OUTBOX_JOBS: {
             readonly removeOnFail: true;
         };
     };
+    readonly "device-op-resolve": {
+        readonly queue: "cap-kernel-maintenance";
+        readonly idempotent: "resolving an op whose entry is gone is a no-op (the entry is read under the machine row's lock, and under the network ask's lock for an op that answers one, so the ask is decided at most once); the resolver only reads the device, never sends a change";
+        readonly processedIn: "packages/kernel/src/runtime/maintenance.ts";
+        readonly opts: {
+            readonly attempts: 5;
+            readonly backoff: {
+                readonly type: "exponential";
+                readonly delay: 5000;
+            };
+            readonly removeOnComplete: true;
+            readonly removeOnFail: true;
+        };
+    };
+    readonly "bg-wake": {
+        readonly queue: "cap-kernel-maintenance";
+        readonly idempotent: "the wake only calls ensureRun, which decides from the conversation's state under its row lock, so a wake with nothing unconsumed or an active run starts nothing";
+        readonly processedIn: "packages/kernel/src/runtime/maintenance.ts";
+        readonly opts: {
+            readonly removeOnComplete: true;
+            readonly removeOnFail: true;
+        };
+    };
+    readonly "eval-case": {
+        readonly queue: "cap-kernel-evals";
+        readonly idempotent: "a case job replays only the case's unfinished results (and pending judgements) of a still-active run, each under a fresh claim, so a second job writes no result twice";
+        readonly processedIn: "packages/kernel/src/evals/evals.module.ts";
+        readonly opts: {
+            readonly attempts: 3;
+            readonly backoff: {
+                readonly type: "exponential";
+                readonly delay: 1000;
+            };
+            readonly removeOnComplete: true;
+            readonly removeOnFail: 1000;
+        };
+    };
     readonly lifecycle: {
         readonly queue: "cap-kernel-lifecycle";
         readonly idempotent: "lifecycle handlers must be idempotent (the SDK contract): they delete what the plugin holds for the removed member, agent or org, and deleting it again is a no-op";

package/dist/infra/outbox/registry.js

@@ -130,6 +130,22 @@ export const OUTBOX_JOBS = {
             removeOnFail: true,
         },
     },
+    // `channels/deliver-job.ts` RESUME_JOB: owed by the update that lets a connection deliver again
+    // (re-enabled, or credentials rotated while active), under `resumeJobId` (one per update)
+    "channel-resume": {
+        queue: "cap-kernel-channels",
+        idempotent: "the catch-up re-reads which of the connection's conversations lag their log and adds their delivery jobs, and a delivery resumes from the delivered seq under a per-conversation lock, so a second catch-up sends nothing twice",
+        processedIn: "packages/kernel/src/channels/channels.module.ts",
+        // retried while PG or Redis is briefly unavailable; behind inbound jobs like a delivery;
+        // removed once done
+        opts: {
+            attempts: 5,
+            backoff: { type: "exponential", delay: 5_000 },
+            priority: 10,
+            removeOnComplete: true,
+            removeOnFail: true,
+        },
+    },
     // `catalog/scope-sync.ts` SCOPE_ITEM_JOB: owed by every transaction that changes an item's
     // scope, disabled flag, list, owner or platform version, under `scopeItemJobId` (one per scope
     // version), and by a sync deferred on a replica that had not loaded the extension yet, under
@@ -210,6 +226,47 @@ export const OUTBOX_JOBS = {
             removeOnFail: true,
         },
     },
+    // `local-devices/policy-op-jobs.ts` DEVICE_OP_RESOLVE_JOB: owed by a device policy op's intent
+    // transaction under `deviceOpResolveJobId` with `runAt` = when the op can no longer reach the
+    // machine, and by a hello that reported the op's outcome under `deviceOpReportedJobId` (spec §7)
+    "device-op-resolve": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "resolving an op whose entry is gone is a no-op (the entry is read under the machine row's lock, and under the network ask's lock for an op that answers one, so the ask is decided at most once); the resolver only reads the device, never sends a change",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // retried while PG or the device is briefly unreachable; removed once done
+        opts: {
+            attempts: 5,
+            backoff: { type: "exponential", delay: 5_000 },
+            removeOnComplete: true,
+            removeOnFail: true,
+        },
+    },
+    // `background/wake-rules.ts` BG_WAKE_JOB: owed by the `ensureRun` transaction that finds a
+    // conversation's wake inside its wake window, under `bgWakeJobId`, with `runAt` = the window's
+    // end (background Bash spec §3)
+    "bg-wake": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "the wake only calls ensureRun, which decides from the conversation's state under its row lock, so a wake with nothing unconsumed or an active run starts nothing",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // the options the direct enqueue had (the queue's default retries): removed once done, so a
+        // later window's wake is never deduped away (a job that fired early owes its next retry id)
+        opts: { removeOnComplete: true, removeOnFail: true },
+    },
+    // `evals/jobs.ts` EVAL_CASE_JOB: owed by an eval run's start transaction (the run row and its
+    // pending results), one per case, under `evalJobId`
+    "eval-case": {
+        queue: "cap-kernel-evals",
+        idempotent: "a case job replays only the case's unfinished results (and pending judgements) of a still-active run, each under a fresh claim, so a second job writes no result twice",
+        processedIn: "packages/kernel/src/evals/evals.module.ts",
+        // the options the direct enqueue had: transient failures outside the handler (PG, Redis)
+        // retried with backoff; removed once done, failed ones kept (by count)
+        opts: {
+            attempts: 3,
+            backoff: { type: "exponential", delay: 1000 },
+            removeOnComplete: true,
+            removeOnFail: 1000,
+        },
+    },
     // `extensions/lifecycle-hooks.ts` LIFECYCLE_JOB: owed by the transaction that removes a member,
     // deletes an agent or deletes an org, once per subscribing plugin, under `lifecycleJobId`
     // (memory plugin spec §6.3)

package/dist/local-devices/device-policy.service.d.ts

@@ -1,11 +1,24 @@
-import { type DeviceFeature, type DeviceLocalPolicySummary, type DevicePolicyArgs } from "@capybara-hq/plugin-sdk";
+import { type DeviceFeature, type DeviceLocalPolicySummary, type DevicePolicyArgs, type DevicePolicyStatus } from "@capybara-hq/plugin-sdk";
+import { KernelConfigService } from "../infra/config/kernel-config.service.js";
 import { KernelDb } from "../infra/database/kernel-db.js";
 import { TransactionHost } from "../infra/database/transaction-host.js";
+import { OutboxService } from "../infra/outbox/outbox.service.js";
 import type { KernelLogger } from "../log.js";
 import { MachinesService } from "./machines.js";
 import { LocalDeviceRelay } from "./relay.js";
 /** How long the machine may take to apply a policy change. */
 export declare const POLICY_DEADLINE_MS = 30000;
+/** How long an op left `unknown` waits for the machine's hello to report it (from its intent). */
+export declare const UNKNOWN_OP_MS: number;
+/** A policy change as callers ask for it: the op id and the source are added here. */
+type PolicyChange = Omit<DevicePolicyArgs, "source" | "opId">;
+/**
+ * Test seam (`KernelOverride`): called by {@link DevicePolicyService.send} right after the `policy`
+ * op's call returned, before its outcome is recorded (by `apply` or a network ask's decision);
+ * throwing stands for a crash between the call and the resolve transaction.
+ * The kernel never provides it.
+ */
+export declare const DEVICE_POLICY_AFTER_CALL: unique symbol;
 /** What became of a policy change. */
 export type PolicyChangeOutcome = {
     applied: true;
@@ -15,55 +28,147 @@ export type PolicyChangeOutcome = {
     reason: "offline" | "refused" | "unsupported";
     message: string;
 };
+/**
+ * An op whose intent is recorded ({@link DevicePolicyService.begin}): the call sends `args` to
+ * the device's machine.
+ */
+export interface PolicyIntent {
+    orgId: string;
+    deviceId: string;
+    machineId: string;
+    opId: string;
+    args: DevicePolicyArgs;
+}
+/**
+ * What became of a sent op, as recorded by {@link DevicePolicyService.settleIn}: `applied` (with
+ * the policy the machine wrote, when it said), `refused` (the machine's message), `not_received`
+ * (the machine never got it and never will), `unknown` (no answer).
+ */
+export type PolicyOpResult = {
+    outcome: "applied";
+    policy?: DeviceLocalPolicySummary;
+} | {
+    outcome: "refused";
+    message: string;
+} | {
+    outcome: "not_received";
+} | {
+    outcome: "interrupted";
+} | {
+    outcome: "unknown";
+};
+/** What a machine says of an op id ({@link DevicePolicyService.outcomeOf}). */
+export type PolicyOpStatus = DevicePolicyStatus | {
+    outcome: "offline";
+};
 /**
  * Changes a local device's policy on its machine (network ask-to-allow spec §5.2; sandbox dev
  * tiers spec §7.3: its hosts, its `tier`, and `isolation` back to `sandbox`): the kernel→device
- * `policy` op over the {@link LocalDeviceRelay}. The machine is the authority: it
- * refuses widening unless its owner allows remote grants, validates every host and answers with
- * the policy as written, which the kernel then keeps on the active machine's row (§5.4) and
- * audits as `device.policy_changed`.
+ * `policy` op over the {@link LocalDeviceRelay}. The machine is the authority: it refuses
+ * widening unless its owner allows remote grants, validates every host and answers with the
+ * policy as written, which the kernel then keeps on the active machine's row (§5.4).
  *
- * The op is sent outside any transaction (a decision records its outcome afterwards, in its own
- * transaction); only the summary write and the audit row are transactional. Callers decide who
- * may change a device's policy (the device owner, in a signed-in session); this service checks
- * only that the device is a live device of the org.
+ * Intent first (transactional outbox spec §7): {@link begin} writes the intent in a transaction
+ * before anything is sent (the entry on the machine row's `pending_policy_ops`, the audit
+ * `device.policy_requested` and the `device-op-resolve` job owed for when the op can no longer
+ * reach the machine); the call sends the op with its id (no transaction open); {@link settleIn}
+ * records the outcome: `device.policy_changed` (applied), `device.policy_failed` (refused, never
+ * received) or `device.policy_unknown` (no answer: the entry stays, `unknown`, for the machine's
+ * next hello or the resolver). A crash between the call and the resolve transaction leaves the
+ * intent and its owed job, whose resolver asks the machine what became of the op (never sends it
+ * again). Callers decide who may change a device's policy (the device owner, in a signed-in
+ * session); this service checks only that the device is a live device of the org.
  */
 export declare class DevicePolicyService {
     private readonly db;
     readonly txHost: TransactionHost;
     private readonly relay;
     private readonly machines;
+    private readonly outbox;
+    private readonly config;
     private readonly log;
-    constructor(db: KernelDb, txHost: TransactionHost, relay: Pick<LocalDeviceRelay, "call" | "holderLive">, machines: Pick<MachinesService, "activeFor">, log: Pick<KernelLogger, "warn">);
+    private readonly afterCall;
+    constructor(db: KernelDb, txHost: TransactionHost, relay: Pick<LocalDeviceRelay, "call" | "holderLive">, machines: Pick<MachinesService, "activeFor">, outbox: Pick<OutboxService, "add">, config: Pick<KernelConfigService, "get">, log: Pick<KernelLogger, "warn">, afterCall?: (opId: string) => void);
     /**
-     * Sends the `policy` op (no transaction may be open), stores the returned summary and audits
-     * `device.policy_changed`. Never throws for an offline machine, a refusal or a program without
-     * the `policyOp` feature (nothing is sent to it); a device that is not a live device of `orgId`
-     * is `device.not_found`.
+     * Changes the policy: {@link begin}, the call, then the outcome recorded in its own transaction
+     * (no transaction may be open). Never throws for an offline machine, a refusal or a program
+     * without the `policyOpId` feature (nothing is sent to it); a device that is not a live device
+     * of `orgId` is `device.not_found`.
      */
     apply(input: {
         orgId: string;
         deviceId: string;
-        change: Omit<DevicePolicyArgs, "source">;
+        change: PolicyChange;
         source: DevicePolicyArgs["source"];
         actorTokenId?: string;
     }): Promise<PolicyChangeOutcome>;
+    /**
+     * Checks the device and its machine, then records the intent in its own transaction: the entry
+     * (`sent`) on the machine's row, the audit `device.policy_requested` and the `device-op-resolve`
+     * job owed for when the op can no longer reach the machine. No transaction may be open (the
+     * call follows). Returns the intent, or why nothing will be sent (offline, unsupported).
+     */
+    begin(input: {
+        orgId: string;
+        deviceId: string;
+        change: PolicyChange;
+        source: DevicePolicyArgs["source"];
+        actorTokenId?: string;
+    }): Promise<{
+        intent: PolicyIntent;
+    } | {
+        outcome: PolicyChangeOutcome;
+    }>;
+    /**
+     * Sends the intent's `policy` op (no transaction may be open) and classifies the answer:
+     * applied; refused (the machine's local policy refused it: its message); anything else is
+     * `unknown`, left to the resolver's `policyStatus`: no answer, and an answer the kernel cannot
+     * trust to mean "not applied" (a result it cannot read, the machine's own error). `notFound`:
+     * the device was deleted meanwhile (recorded as unknown; the caller throws `device.not_found`).
+     */
+    send(intent: PolicyIntent): Promise<{
+        result: PolicyOpResult;
+        outcome: PolicyChangeOutcome;
+        notFound?: true;
+    }>;
+    /** The `policy` op's call and its classification ({@link send}). */
+    private call;
+    /**
+     * What the machine says became of the op `opId` (a read-only `policyStatus`), or `offline` when
+     * no replica holds its connection or it does not answer. Never sends the change.
+     */
+    outcomeOf(deviceId: string, opId: string): Promise<PolicyOpStatus>;
+    /**
+     * Records what became of the op `opId` of machine `machineId`, in the caller's transaction (the
+     * machine's row locked): `applied` → the policy the machine wrote (when given) kept on the row,
+     * the entry removed, `device.policy_changed` (`late` when the op was left `unknown`); `refused`,
+     * `not_received` or `interrupted` → the entry removed, `device.policy_failed` (`reason` the
+     * outcome); `unknown` → the entry kept, `unknown` until a day after its intent,
+     * `device.policy_unknown` (once; ignored once a hello reported the op). Returns the outcome for
+     * the caller, or null when the op has no live entry (resolved already, or expired: a no-op).
+     */
+    settleIn(ref: {
+        machineId: string;
+        opId: string;
+    }, result: PolicyOpResult): Promise<PolicyChangeOutcome | null>;
     /** The active machine's features and policy summary (null when no machine is connected). */
     summary(deviceId: string): Promise<{
         features: DeviceFeature[];
         policy: DeviceLocalPolicySummary | null;
     } | null>;
-    /** Adds `op` to the machine's pending ops, dropping the expired ones. */
-    private notePending;
-    /** Removes the pending op `opId` from the machine's row. */
-    private dropPending;
     /**
-     * Keeps the policy the machine wrote on its row and audits the change (`confirmed`); with no
-     * policy (the answer never came) only audits the attempt, unconfirmed. The detail's `isolation`
-     * is the one asked for, or the one the machine now reports when it changed (a tier change turns
-     * the sandbox back on).
+     * How long after its intent an op can no longer reach the machine: the call's deadline, the
+     * reconnect grace and the request list's margin (the relay's list TTL), plus 5 s. A machine
+     * that does not know the op id by then never will.
+     */
+    resolveDelayMs(): number;
+    /** {@link settleIn} in its own transaction (the {@link apply} path). */
+    private settle;
+    /**
+     * The intent transaction: locks the machine's row (still the device's active machine), appends
+     * the entry (dropping the expired ones), audits `device.policy_requested` and owes the
+     * `device-op-resolve` job. False when the machine was revoked meanwhile (nothing written).
      */
-    private record;
-    /** The outcome of a failed op: offline, or the machine's refusal; a deleted device is thrown. */
-    private notApplied;
+    private recordIntent;
 }
+export {};