@capybara-hq/kernel 0.1.0-rc.24 → 0.1.0-rc.26

package/dist/infra/outbox/registry.js

@@ -54,6 +54,35 @@ export const OUTBOX_JOBS = {
             removeOnFail: true,
         },
     },
+    // `orgs/offboarding.ts` RUN_STOP_JOB: owed by the offboard transaction for each of the departed
+    // member's active runs in the org, under `runStopJobId` (spec §8.1)
+    "run-stop": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "stopping a finished run is a no-op",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // retried while PG or Redis is briefly unavailable; removed once done; a job that failed every
+        // attempt is kept a week (no sweep stops the run after it: the failure must stay visible)
+        opts: {
+            attempts: 5,
+            backoff: { type: "exponential", delay: 5_000 },
+            removeOnComplete: true,
+            removeOnFail: { age: 7 * 86_400, count: 1_000 },
+        },
+    },
+    // `sandbox/jobs.ts` SANDBOX_DESTROY_FOR_JOB: owed by the transaction that soft-deletes a device
+    // (its deletion, offboarding), under `deviceDestroyJobId` (spec §8.1)
+    "sandbox-destroy-for": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "destroying a destroyed device is a no-op",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // the options the direct enqueue had: retried while the provider is down
+        opts: {
+            attempts: 5,
+            backoff: { type: "exponential", delay: 30_000 },
+            removeOnComplete: true,
+            removeOnFail: 100,
+        },
+    },
     // `runtime/subagents/tree-jobs.ts` DELIVER_TO_PARENT_JOB: owed by a sub-agent run's terminal
     // transaction, under `deliverToParentJobId` (spec §6)
     "deliver-to-parent": {
@@ -101,6 +130,160 @@ export const OUTBOX_JOBS = {
             removeOnFail: true,
         },
     },
+    // `channels/deliver-job.ts` RESUME_JOB: owed by the update that lets a connection deliver again
+    // (re-enabled, or credentials rotated while active), under `resumeJobId` (one per update)
+    "channel-resume": {
+        queue: "cap-kernel-channels",
+        idempotent: "the catch-up re-reads which of the connection's conversations lag their log and adds their delivery jobs, and a delivery resumes from the delivered seq under a per-conversation lock, so a second catch-up sends nothing twice",
+        processedIn: "packages/kernel/src/channels/channels.module.ts",
+        // retried while PG or Redis is briefly unavailable; behind inbound jobs like a delivery;
+        // removed once done
+        opts: {
+            attempts: 5,
+            backoff: { type: "exponential", delay: 5_000 },
+            priority: 10,
+            removeOnComplete: true,
+            removeOnFail: true,
+        },
+    },
+    // `catalog/scope-sync.ts` SCOPE_ITEM_JOB: owed by every transaction that changes an item's
+    // scope, disabled flag, list, owner or platform version, under `scopeItemJobId` (one per scope
+    // version), and by a sync deferred on a replica that had not loaded the extension yet, under
+    // `scopeRetryJobId` with `runAt` (spec §8.1)
+    "scope-item": {
+        queue: "cap-kernel-plugin-scope",
+        idempotent: "the sync converges each (item, org) from the scope, switches and installs it reads under the org's lock, and the installer re-checks under the org row lock, so a second run changes nothing",
+        processedIn: "packages/kernel/src/catalog/scope-sync-workers.ts",
+        // one attempt (a pair's failure is recorded, not retried); removed once done, so the next
+        // add of the same id is not swallowed
+        opts: { attempts: 1, removeOnComplete: true, removeOnFail: true },
+    },
+    // `catalog/scope-jobs.ts` SCOPE_BACKSTOP_JOB: owed by a confirmation's commit (an import, a
+    // permissions confirm, an org upload's first version) with `runAt` 5 min on, under
+    // `scopeBackstopJobId`, so a crash before the post-deploy add still converges (ruling R24)
+    "scope-item-backstop": {
+        queue: "cap-kernel-plugin-scope",
+        idempotent: "the sync converges each (item, org) from what it reads, as a scope-item job does, so a run after the immediate job changes nothing",
+        processedIn: "packages/kernel/src/catalog/scope-sync-workers.ts",
+        // retried while the item's extension is not enabled on the deployment yet (the job throws):
+        // 1, 2, 4, 8 min; removed once done
+        opts: {
+            attempts: 5,
+            backoff: { type: "exponential", delay: 60_000 },
+            removeOnComplete: true,
+            removeOnFail: true,
+        },
+    },
+    // `catalog/scope-sync.ts` SCOPE_ORG_JOB: owed by an org's creation, under `scopeOrgJobId`
+    "scope-org": {
+        queue: "cap-kernel-plugin-scope",
+        idempotent: "the sync converges each (item, org) of the org from what it reads, so a second run changes nothing",
+        processedIn: "packages/kernel/src/catalog/scope-sync-workers.ts",
+        opts: { attempts: 1, removeOnComplete: true, removeOnFail: true },
+    },
+    // `catalog/job-ids.ts` OWNED_PLUGIN_CLEANUP_JOB: owed by the removal of an org-owned plugin
+    // (and by a confirmation that took out the extension of an item removed meanwhile), under
+    // `ownedPluginCleanupJobId` (spec §8.1)
+    "owned-plugin-cleanup": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "the cleanup takes the extension out only while no live item uses it, and a missing install or a released claim changes nothing",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // a few retries while PG is briefly unavailable; removed once done
+        opts: {
+            attempts: 3,
+            backoff: { type: "exponential", delay: 5_000 },
+            removeOnComplete: true,
+            removeOnFail: true,
+        },
+    },
+    // `schedules/job-ids.ts` SCHED_FIRE_JOB: owed by the transaction that writes a pending task run
+    // (the scan's tick, a person's "run now"), under `schedFireJobId` (spec §8.1)
+    "sched-fire": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "a fire acts only on a pending task run, under its row lock; a retried start reuses what the first one made",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // a few retries while PG is briefly unavailable; removed once done
+        opts: {
+            attempts: 3,
+            backoff: { type: "exponential", delay: 2_000 },
+            removeOnComplete: true,
+            removeOnFail: true,
+        },
+    },
+    // `schedules/job-ids.ts` SCHED_SETTLE_JOB: the claimed check, owed by the fire job's claim (the
+    // transaction that writes `fired_at`) with `runAt` 10 min on, under `schedSettleJobId` (a run
+    // still in flight owes the next check an hour on); the unfired check, owed with the task run
+    // (scan tick, "run now") under `schedSettleUnfiredJobId` (ruling R26)
+    "sched-settle": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "a check acts only on a task run still pending (claimed for the claimed check, unclaimed for the unfired one), under the task and run row locks, so a run claimed, started, ended or skipped since is left as it is",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // retried while PG is briefly unavailable; removed once done
+        opts: {
+            attempts: 5,
+            backoff: { type: "exponential", delay: 5_000 },
+            removeOnComplete: true,
+            removeOnFail: true,
+        },
+    },
+    // `local-devices/policy-op-jobs.ts` DEVICE_OP_RESOLVE_JOB: owed by a device policy op's intent
+    // transaction under `deviceOpResolveJobId` with `runAt` = when the op can no longer reach the
+    // machine, and by a hello that reported the op's outcome under `deviceOpReportedJobId` (spec §7)
+    "device-op-resolve": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "resolving an op whose entry is gone is a no-op (the entry is read under the machine row's lock, and under the network ask's lock for an op that answers one, so the ask is decided at most once); the resolver only reads the device, never sends a change",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // retried while PG or the device is briefly unreachable; removed once done
+        opts: {
+            attempts: 5,
+            backoff: { type: "exponential", delay: 5_000 },
+            removeOnComplete: true,
+            removeOnFail: true,
+        },
+    },
+    // `background/wake-rules.ts` BG_WAKE_JOB: owed by the `ensureRun` transaction that finds a
+    // conversation's wake inside its wake window, under `bgWakeJobId`, with `runAt` = the window's
+    // end (background Bash spec §3)
+    "bg-wake": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "the wake only calls ensureRun, which decides from the conversation's state under its row lock, so a wake with nothing unconsumed or an active run starts nothing",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // the options the direct enqueue had (the queue's default retries): removed once done, so a
+        // later window's wake is never deduped away (a job that fired early owes its next retry id)
+        opts: { removeOnComplete: true, removeOnFail: true },
+    },
+    // `evals/jobs.ts` EVAL_CASE_JOB: owed by an eval run's start transaction (the run row and its
+    // pending results), one per case, under `evalJobId`
+    "eval-case": {
+        queue: "cap-kernel-evals",
+        idempotent: "a case job replays only the case's unfinished results (and pending judgements) of a still-active run, each under a fresh claim, so a second job writes no result twice",
+        processedIn: "packages/kernel/src/evals/evals.module.ts",
+        // the options the direct enqueue had: transient failures outside the handler (PG, Redis)
+        // retried with backoff; removed once done, failed ones kept (by count)
+        opts: {
+            attempts: 3,
+            backoff: { type: "exponential", delay: 1000 },
+            removeOnComplete: true,
+            removeOnFail: 1000,
+        },
+    },
+    // `extensions/lifecycle-hooks.ts` LIFECYCLE_JOB: owed by the transaction that removes a member,
+    // deletes an agent or deletes an org, once per subscribing plugin, under `lifecycleJobId`
+    // (memory plugin spec §6.3)
+    lifecycle: {
+        queue: "cap-kernel-lifecycle",
+        idempotent: "lifecycle handlers must be idempotent (the SDK contract): they delete what the plugin holds for the removed member, agent or org, and deleting it again is a no-op",
+        processedIn: "packages/kernel/src/extensions/lifecycle-hooks.ts",
+        // retried while the plugin is not loaded on the replica that took the job (a rolling deploy)
+        // or its handler fails: 15 attempts from 2 s, doubling (about 18 h in all); removed once done;
+        // a job that failed every attempt is kept a week (the failure must stay visible)
+        opts: {
+            attempts: 15,
+            backoff: { type: "exponential", delay: 2_000 },
+            removeOnComplete: true,
+            removeOnFail: { age: 7 * 86_400, count: 1_000 },
+        },
+    },
 };
 /** The registry's BullMQ options of a job name (none for a name it does not list). */
 export function outboxJobOptions(name) {

package/dist/infra/queue/queue-names.d.ts

@@ -4,5 +4,5 @@
  * (admin/health.service.ts) lists the same names from the feature constants;
  * test/infra/queue-names.test.ts keeps the two equal.
  */
-export declare const KERNEL_QUEUES: readonly ["cap-kernel-runs", "cap-kernel-subagent-runs", "cap-kernel-channels", "cap-kernel-maintenance", "cap-kernel-plugin-scope", "cap-kernel-titles", "cap-kernel-usage-report", "cap-kernel-evals"];
+export declare const KERNEL_QUEUES: readonly ["cap-kernel-runs", "cap-kernel-subagent-runs", "cap-kernel-channels", "cap-kernel-maintenance", "cap-kernel-plugin-scope", "cap-kernel-titles", "cap-kernel-usage-report", "cap-kernel-evals", "cap-kernel-lifecycle"];
 export type KernelQueueName = (typeof KERNEL_QUEUES)[number];

package/dist/infra/queue/queue-names.js

@@ -13,4 +13,5 @@ export const KERNEL_QUEUES = [
     "cap-kernel-titles",
     "cap-kernel-usage-report",
     "cap-kernel-evals",
+    "cap-kernel-lifecycle",
 ];

package/dist/local-devices/device-policy.service.d.ts

@@ -1,11 +1,24 @@
-import { type DeviceFeature, type DeviceLocalPolicySummary, type DevicePolicyArgs } from "@capybara-hq/plugin-sdk";
+import { type DeviceFeature, type DeviceLocalPolicySummary, type DevicePolicyArgs, type DevicePolicyStatus } from "@capybara-hq/plugin-sdk";
+import { KernelConfigService } from "../infra/config/kernel-config.service.js";
 import { KernelDb } from "../infra/database/kernel-db.js";
 import { TransactionHost } from "../infra/database/transaction-host.js";
+import { OutboxService } from "../infra/outbox/outbox.service.js";
 import type { KernelLogger } from "../log.js";
 import { MachinesService } from "./machines.js";
 import { LocalDeviceRelay } from "./relay.js";
 /** How long the machine may take to apply a policy change. */
 export declare const POLICY_DEADLINE_MS = 30000;
+/** How long an op left `unknown` waits for the machine's hello to report it (from its intent). */
+export declare const UNKNOWN_OP_MS: number;
+/** A policy change as callers ask for it: the op id and the source are added here. */
+type PolicyChange = Omit<DevicePolicyArgs, "source" | "opId">;
+/**
+ * Test seam (`KernelOverride`): called by {@link DevicePolicyService.send} right after the `policy`
+ * op's call returned, before its outcome is recorded (by `apply` or a network ask's decision);
+ * throwing stands for a crash between the call and the resolve transaction.
+ * The kernel never provides it.
+ */
+export declare const DEVICE_POLICY_AFTER_CALL: unique symbol;
 /** What became of a policy change. */
 export type PolicyChangeOutcome = {
     applied: true;
@@ -15,55 +28,147 @@ export type PolicyChangeOutcome = {
     reason: "offline" | "refused" | "unsupported";
     message: string;
 };
+/**
+ * An op whose intent is recorded ({@link DevicePolicyService.begin}): the call sends `args` to
+ * the device's machine.
+ */
+export interface PolicyIntent {
+    orgId: string;
+    deviceId: string;
+    machineId: string;
+    opId: string;
+    args: DevicePolicyArgs;
+}
+/**
+ * What became of a sent op, as recorded by {@link DevicePolicyService.settleIn}: `applied` (with
+ * the policy the machine wrote, when it said), `refused` (the machine's message), `not_received`
+ * (the machine never got it and never will), `unknown` (no answer).
+ */
+export type PolicyOpResult = {
+    outcome: "applied";
+    policy?: DeviceLocalPolicySummary;
+} | {
+    outcome: "refused";
+    message: string;
+} | {
+    outcome: "not_received";
+} | {
+    outcome: "interrupted";
+} | {
+    outcome: "unknown";
+};
+/** What a machine says of an op id ({@link DevicePolicyService.outcomeOf}). */
+export type PolicyOpStatus = DevicePolicyStatus | {
+    outcome: "offline";
+};
 /**
  * Changes a local device's policy on its machine (network ask-to-allow spec §5.2; sandbox dev
  * tiers spec §7.3: its hosts, its `tier`, and `isolation` back to `sandbox`): the kernel→device
- * `policy` op over the {@link LocalDeviceRelay}. The machine is the authority: it
- * refuses widening unless its owner allows remote grants, validates every host and answers with
- * the policy as written, which the kernel then keeps on the active machine's row (§5.4) and
- * audits as `device.policy_changed`.
+ * `policy` op over the {@link LocalDeviceRelay}. The machine is the authority: it refuses
+ * widening unless its owner allows remote grants, validates every host and answers with the
+ * policy as written, which the kernel then keeps on the active machine's row (§5.4).
  *
- * The op is sent outside any transaction (a decision records its outcome afterwards, in its own
- * transaction); only the summary write and the audit row are transactional. Callers decide who
- * may change a device's policy (the device owner, in a signed-in session); this service checks
- * only that the device is a live device of the org.
+ * Intent first (transactional outbox spec §7): {@link begin} writes the intent in a transaction
+ * before anything is sent (the entry on the machine row's `pending_policy_ops`, the audit
+ * `device.policy_requested` and the `device-op-resolve` job owed for when the op can no longer
+ * reach the machine); the call sends the op with its id (no transaction open); {@link settleIn}
+ * records the outcome: `device.policy_changed` (applied), `device.policy_failed` (refused, never
+ * received) or `device.policy_unknown` (no answer: the entry stays, `unknown`, for the machine's
+ * next hello or the resolver). A crash between the call and the resolve transaction leaves the
+ * intent and its owed job, whose resolver asks the machine what became of the op (never sends it
+ * again). Callers decide who may change a device's policy (the device owner, in a signed-in
+ * session); this service checks only that the device is a live device of the org.
  */
 export declare class DevicePolicyService {
     private readonly db;
     readonly txHost: TransactionHost;
     private readonly relay;
     private readonly machines;
+    private readonly outbox;
+    private readonly config;
     private readonly log;
-    constructor(db: KernelDb, txHost: TransactionHost, relay: Pick<LocalDeviceRelay, "call" | "holderLive">, machines: Pick<MachinesService, "activeFor">, log: Pick<KernelLogger, "warn">);
+    private readonly afterCall;
+    constructor(db: KernelDb, txHost: TransactionHost, relay: Pick<LocalDeviceRelay, "call" | "holderLive">, machines: Pick<MachinesService, "activeFor">, outbox: Pick<OutboxService, "add">, config: Pick<KernelConfigService, "get">, log: Pick<KernelLogger, "warn">, afterCall?: (opId: string) => void);
     /**
-     * Sends the `policy` op (no transaction may be open), stores the returned summary and audits
-     * `device.policy_changed`. Never throws for an offline machine, a refusal or a program without
-     * the `policyOp` feature (nothing is sent to it); a device that is not a live device of `orgId`
-     * is `device.not_found`.
+     * Changes the policy: {@link begin}, the call, then the outcome recorded in its own transaction
+     * (no transaction may be open). Never throws for an offline machine, a refusal or a program
+     * without the `policyOpId` feature (nothing is sent to it); a device that is not a live device
+     * of `orgId` is `device.not_found`.
      */
     apply(input: {
         orgId: string;
         deviceId: string;
-        change: Omit<DevicePolicyArgs, "source">;
+        change: PolicyChange;
         source: DevicePolicyArgs["source"];
         actorTokenId?: string;
     }): Promise<PolicyChangeOutcome>;
+    /**
+     * Checks the device and its machine, then records the intent in its own transaction: the entry
+     * (`sent`) on the machine's row, the audit `device.policy_requested` and the `device-op-resolve`
+     * job owed for when the op can no longer reach the machine. No transaction may be open (the
+     * call follows). Returns the intent, or why nothing will be sent (offline, unsupported).
+     */
+    begin(input: {
+        orgId: string;
+        deviceId: string;
+        change: PolicyChange;
+        source: DevicePolicyArgs["source"];
+        actorTokenId?: string;
+    }): Promise<{
+        intent: PolicyIntent;
+    } | {
+        outcome: PolicyChangeOutcome;
+    }>;
+    /**
+     * Sends the intent's `policy` op (no transaction may be open) and classifies the answer:
+     * applied; refused (the machine's local policy refused it: its message); anything else is
+     * `unknown`, left to the resolver's `policyStatus`: no answer, and an answer the kernel cannot
+     * trust to mean "not applied" (a result it cannot read, the machine's own error). `notFound`:
+     * the device was deleted meanwhile (recorded as unknown; the caller throws `device.not_found`).
+     */
+    send(intent: PolicyIntent): Promise<{
+        result: PolicyOpResult;
+        outcome: PolicyChangeOutcome;
+        notFound?: true;
+    }>;
+    /** The `policy` op's call and its classification ({@link send}). */
+    private call;
+    /**
+     * What the machine says became of the op `opId` (a read-only `policyStatus`), or `offline` when
+     * no replica holds its connection or it does not answer. Never sends the change.
+     */
+    outcomeOf(deviceId: string, opId: string): Promise<PolicyOpStatus>;
+    /**
+     * Records what became of the op `opId` of machine `machineId`, in the caller's transaction (the
+     * machine's row locked): `applied` → the policy the machine wrote (when given) kept on the row,
+     * the entry removed, `device.policy_changed` (`late` when the op was left `unknown`); `refused`,
+     * `not_received` or `interrupted` → the entry removed, `device.policy_failed` (`reason` the
+     * outcome); `unknown` → the entry kept, `unknown` until a day after its intent,
+     * `device.policy_unknown` (once; ignored once a hello reported the op). Returns the outcome for
+     * the caller, or null when the op has no live entry (resolved already, or expired: a no-op).
+     */
+    settleIn(ref: {
+        machineId: string;
+        opId: string;
+    }, result: PolicyOpResult): Promise<PolicyChangeOutcome | null>;
     /** The active machine's features and policy summary (null when no machine is connected). */
     summary(deviceId: string): Promise<{
         features: DeviceFeature[];
         policy: DeviceLocalPolicySummary | null;
     } | null>;
-    /** Adds `op` to the machine's pending ops, dropping the expired ones. */
-    private notePending;
-    /** Removes the pending op `opId` from the machine's row. */
-    private dropPending;
     /**
-     * Keeps the policy the machine wrote on its row and audits the change (`confirmed`); with no
-     * policy (the answer never came) only audits the attempt, unconfirmed. The detail's `isolation`
-     * is the one asked for, or the one the machine now reports when it changed (a tier change turns
-     * the sandbox back on).
+     * How long after its intent an op can no longer reach the machine: the call's deadline, the
+     * reconnect grace and the request list's margin (the relay's list TTL), plus 5 s. A machine
+     * that does not know the op id by then never will.
+     */
+    resolveDelayMs(): number;
+    /** {@link settleIn} in its own transaction (the {@link apply} path). */
+    private settle;
+    /**
+     * The intent transaction: locks the machine's row (still the device's active machine), appends
+     * the entry (dropping the expired ones), audits `device.policy_requested` and owes the
+     * `device-op-resolve` job. False when the machine was revoked meanwhile (nothing written).
      */
-    private record;
-    /** The outcome of a failed op: offline, or the machine's refusal; a deleted device is thrown. */
-    private notApplied;
+    private recordIntent;
 }
+export {};