@capybara-hq/kernel 0.1.0-rc.24 → 0.1.0-rc.25

package/dist/infra/outbox/registry.js

@@ -54,6 +54,35 @@ export const OUTBOX_JOBS = {
             removeOnFail: true,
         },
     },
+    // `orgs/offboarding.ts` RUN_STOP_JOB: owed by the offboard transaction for each of the departed
+    // member's active runs in the org, under `runStopJobId` (spec §8.1)
+    "run-stop": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "stopping a finished run is a no-op",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // retried while PG or Redis is briefly unavailable; removed once done; a job that failed every
+        // attempt is kept a week (no sweep stops the run after it: the failure must stay visible)
+        opts: {
+            attempts: 5,
+            backoff: { type: "exponential", delay: 5_000 },
+            removeOnComplete: true,
+            removeOnFail: { age: 7 * 86_400, count: 1_000 },
+        },
+    },
+    // `sandbox/jobs.ts` SANDBOX_DESTROY_FOR_JOB: owed by the transaction that soft-deletes a device
+    // (its deletion, offboarding), under `deviceDestroyJobId` (spec §8.1)
+    "sandbox-destroy-for": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "destroying a destroyed device is a no-op",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // the options the direct enqueue had: retried while the provider is down
+        opts: {
+            attempts: 5,
+            backoff: { type: "exponential", delay: 30_000 },
+            removeOnComplete: true,
+            removeOnFail: 100,
+        },
+    },
     // `runtime/subagents/tree-jobs.ts` DELIVER_TO_PARENT_JOB: owed by a sub-agent run's terminal
     // transaction, under `deliverToParentJobId` (spec §6)
     "deliver-to-parent": {
@@ -101,6 +130,103 @@ export const OUTBOX_JOBS = {
             removeOnFail: true,
         },
     },
+    // `catalog/scope-sync.ts` SCOPE_ITEM_JOB: owed by every transaction that changes an item's
+    // scope, disabled flag, list, owner or platform version, under `scopeItemJobId` (one per scope
+    // version), and by a sync deferred on a replica that had not loaded the extension yet, under
+    // `scopeRetryJobId` with `runAt` (spec §8.1)
+    "scope-item": {
+        queue: "cap-kernel-plugin-scope",
+        idempotent: "the sync converges each (item, org) from the scope, switches and installs it reads under the org's lock, and the installer re-checks under the org row lock, so a second run changes nothing",
+        processedIn: "packages/kernel/src/catalog/scope-sync-workers.ts",
+        // one attempt (a pair's failure is recorded, not retried); removed once done, so the next
+        // add of the same id is not swallowed
+        opts: { attempts: 1, removeOnComplete: true, removeOnFail: true },
+    },
+    // `catalog/scope-jobs.ts` SCOPE_BACKSTOP_JOB: owed by a confirmation's commit (an import, a
+    // permissions confirm, an org upload's first version) with `runAt` 5 min on, under
+    // `scopeBackstopJobId`, so a crash before the post-deploy add still converges (ruling R24)
+    "scope-item-backstop": {
+        queue: "cap-kernel-plugin-scope",
+        idempotent: "the sync converges each (item, org) from what it reads, as a scope-item job does, so a run after the immediate job changes nothing",
+        processedIn: "packages/kernel/src/catalog/scope-sync-workers.ts",
+        // retried while the item's extension is not enabled on the deployment yet (the job throws):
+        // 1, 2, 4, 8 min; removed once done
+        opts: {
+            attempts: 5,
+            backoff: { type: "exponential", delay: 60_000 },
+            removeOnComplete: true,
+            removeOnFail: true,
+        },
+    },
+    // `catalog/scope-sync.ts` SCOPE_ORG_JOB: owed by an org's creation, under `scopeOrgJobId`
+    "scope-org": {
+        queue: "cap-kernel-plugin-scope",
+        idempotent: "the sync converges each (item, org) of the org from what it reads, so a second run changes nothing",
+        processedIn: "packages/kernel/src/catalog/scope-sync-workers.ts",
+        opts: { attempts: 1, removeOnComplete: true, removeOnFail: true },
+    },
+    // `catalog/job-ids.ts` OWNED_PLUGIN_CLEANUP_JOB: owed by the removal of an org-owned plugin
+    // (and by a confirmation that took out the extension of an item removed meanwhile), under
+    // `ownedPluginCleanupJobId` (spec §8.1)
+    "owned-plugin-cleanup": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "the cleanup takes the extension out only while no live item uses it, and a missing install or a released claim changes nothing",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // a few retries while PG is briefly unavailable; removed once done
+        opts: {
+            attempts: 3,
+            backoff: { type: "exponential", delay: 5_000 },
+            removeOnComplete: true,
+            removeOnFail: true,
+        },
+    },
+    // `schedules/job-ids.ts` SCHED_FIRE_JOB: owed by the transaction that writes a pending task run
+    // (the scan's tick, a person's "run now"), under `schedFireJobId` (spec §8.1)
+    "sched-fire": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "a fire acts only on a pending task run, under its row lock; a retried start reuses what the first one made",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // a few retries while PG is briefly unavailable; removed once done
+        opts: {
+            attempts: 3,
+            backoff: { type: "exponential", delay: 2_000 },
+            removeOnComplete: true,
+            removeOnFail: true,
+        },
+    },
+    // `schedules/job-ids.ts` SCHED_SETTLE_JOB: the claimed check, owed by the fire job's claim (the
+    // transaction that writes `fired_at`) with `runAt` 10 min on, under `schedSettleJobId` (a run
+    // still in flight owes the next check an hour on); the unfired check, owed with the task run
+    // (scan tick, "run now") under `schedSettleUnfiredJobId` (ruling R26)
+    "sched-settle": {
+        queue: "cap-kernel-maintenance",
+        idempotent: "a check acts only on a task run still pending (claimed for the claimed check, unclaimed for the unfired one), under the task and run row locks, so a run claimed, started, ended or skipped since is left as it is",
+        processedIn: "packages/kernel/src/runtime/maintenance.ts",
+        // retried while PG is briefly unavailable; removed once done
+        opts: {
+            attempts: 5,
+            backoff: { type: "exponential", delay: 5_000 },
+            removeOnComplete: true,
+            removeOnFail: true,
+        },
+    },
+    // `extensions/lifecycle-hooks.ts` LIFECYCLE_JOB: owed by the transaction that removes a member,
+    // deletes an agent or deletes an org, once per subscribing plugin, under `lifecycleJobId`
+    // (memory plugin spec §6.3)
+    lifecycle: {
+        queue: "cap-kernel-lifecycle",
+        idempotent: "lifecycle handlers must be idempotent (the SDK contract): they delete what the plugin holds for the removed member, agent or org, and deleting it again is a no-op",
+        processedIn: "packages/kernel/src/extensions/lifecycle-hooks.ts",
+        // retried while the plugin is not loaded on the replica that took the job (a rolling deploy)
+        // or its handler fails: 15 attempts from 2 s, doubling (about 18 h in all); removed once done;
+        // a job that failed every attempt is kept a week (the failure must stay visible)
+        opts: {
+            attempts: 15,
+            backoff: { type: "exponential", delay: 2_000 },
+            removeOnComplete: true,
+            removeOnFail: { age: 7 * 86_400, count: 1_000 },
+        },
+    },
 };
 /** The registry's BullMQ options of a job name (none for a name it does not list). */
 export function outboxJobOptions(name) {

package/dist/infra/queue/queue-names.d.ts

@@ -4,5 +4,5 @@
  * (admin/health.service.ts) lists the same names from the feature constants;
  * test/infra/queue-names.test.ts keeps the two equal.
  */
-export declare const KERNEL_QUEUES: readonly ["cap-kernel-runs", "cap-kernel-subagent-runs", "cap-kernel-channels", "cap-kernel-maintenance", "cap-kernel-plugin-scope", "cap-kernel-titles", "cap-kernel-usage-report", "cap-kernel-evals"];
+export declare const KERNEL_QUEUES: readonly ["cap-kernel-runs", "cap-kernel-subagent-runs", "cap-kernel-channels", "cap-kernel-maintenance", "cap-kernel-plugin-scope", "cap-kernel-titles", "cap-kernel-usage-report", "cap-kernel-evals", "cap-kernel-lifecycle"];
 export type KernelQueueName = (typeof KERNEL_QUEUES)[number];

package/dist/infra/queue/queue-names.js

@@ -13,4 +13,5 @@ export const KERNEL_QUEUES = [
     "cap-kernel-titles",
     "cap-kernel-usage-report",
     "cap-kernel-evals",
+    "cap-kernel-lifecycle",
 ];

package/dist/orgs/directory-sync.js

@@ -611,7 +611,7 @@ let DirectorySync = class DirectorySync {
         if (row?.source !== source)
             return undefined;
         const result = await this.offboarding.offboard(orgId, accountId, "directory");
-        // the markers, events and run stops follow the root commit (none after a rollback)
+        // the revocation markers and events follow the root commit (hints; none after a rollback)
         afterCommit(() => this.offboarding.afterOffboard(result));
         return result;
     }

package/dist/orgs/members.service.js

@@ -151,7 +151,7 @@ let MembersService = class MembersService {
         }
         const actor = { actorAccountId, ...(actorTokenId !== undefined && { actorTokenId }) };
         const res = await this.offboarding.offboard(orgId, targetId, "removed", actor);
-        // the markers, events and run stops follow the root commit (none after a rollback)
+        // the revocation markers and events follow the root commit (hints; none after a rollback)
         afterCommit(() => this.offboarding.afterOffboard(res));
         await audit(this.db.db, {
             ...actor,

package/dist/orgs/offboarding.d.ts

@@ -1,12 +1,11 @@
-import type { Redis } from "ioredis";
 import { EventLog } from "../conversations/event-log.js";
+import { LifecycleHooks } from "../extensions/lifecycle-hooks.js";
 import { ExtensionRegistry } from "../extensions/registry.js";
 import { SessionService } from "../identity/sessions.js";
 import { KernelDb } from "../infra/database/kernel-db.js";
 import { TransactionHost } from "../infra/database/transaction-host.js";
-import { LockService } from "../infra/lock/lock.service.js";
+import { OutboxService } from "../infra/outbox/outbox.service.js";
 import type { KernelLogger } from "../log.js";
-import { SandboxJobs } from "../sandbox/jobs.js";
 import { ApprovalsService } from "../tools/approvals.service.js";
 import { Credentials } from "../tools/credentials.js";
 import { QuestionsService } from "../tools/questions.service.js";
@@ -26,11 +25,11 @@ export interface OffboardResult {
     revokedSessionIds: string[];
     /** 写了 cancelled 事件的会话 → 最大 seq（提交后发布）。 */
     notify: Map<string, number>;
-    /** 此人在该组织进行中的运行（queued / running / waiting），提交后中止。 */
+    /** The person's active runs in the org (queued / running / waiting): a `run-stop` job each is owed. */
     runs: InFlightRun[];
     /** 负责的 agent 转交给的所有者；没有转交时为 null。 */
     newOwnerId: string | null;
-    /** The person's devices in the org, soft-deleted here; destroyed by a job after commit. */
+    /** The person's devices in the org, soft-deleted here; a destroy job each is owed. */
     devices: string[];
     /** Conversations whose queued messages were deleted (`changed: queue` after commit). */
     queueConversations: string[];
@@ -39,21 +38,28 @@ export interface OffboardResult {
 export interface OffboardingRunControl {
     abortWaiting(conversationId: string, runId: string, byAccountId: string | null): Promise<boolean>;
     abortIfOrphaned(conversationId: string, runId: string, byAccountId: string | null): Promise<boolean>;
+    abortQueued(conversationId: string, runId: string, byAccountId: string | null): Promise<boolean>;
     requestAbort(runId: string, byAccountId: string | null): Promise<void>;
 }
-/** The safety-net sweep: cron job name (no `:`), every 10 minutes, and the rows per page. */
-export declare const OFFBOARD_SWEEP_JOB = "offboard-sweep";
-export declare const OFFBOARD_SWEEP_BATCH = 500;
-/** 一次兜底扫描最多翻的页数（每页 {@link OFFBOARD_SWEEP_BATCH} 条），防止单次任务无限长。 */
-export declare const OFFBOARD_SWEEP_MAX_PAGES = 20;
-/** 已经广播中止的运行的标记保留时长（秒），长于扫描周期：期间扫描不再重复广播。 */
-export declare const OFFBOARD_ABORT_MARK_TTL_SECONDS = 1800;
-/** 「已请求中止」标记（Redis 键）。 */
-export declare const offboardAbortKey: (runId: string) => string;
+/**
+ * The stop of a departed member's run (transactional outbox spec §8.1), owed by the offboard
+ * transaction on the maintenance queue under {@link runStopJobId}; `MaintenanceProcessor` runs
+ * {@link Offboarding.stopRun}.
+ */
+export declare const RUN_STOP_JOB = "run-stop";
+/** The `run-stop` job's data. */
+export interface RunStopJobData {
+    conversationId: string;
+    runId: string;
+}
+/** The `run-stop` job id: a run is stopped once (no `:`). */
+export declare const runStopJobId: (runId: string) => string;
 /**
  * 成员离职（spec 3.3）：管理员移出、本人离开与通讯录同步共用这一条路径，全部写入在调用方的同一个事务内，
- * 要么都生效要么都不生效。Redis 吊销标记、事件发布与中止进行中的运行是提交后的尽力而为（{@link afterOffboard}），
- * 漏掉的运行由兜底扫描（{@link sweep}：发起人已不是该组织成员的进行中运行）中止。框架无关。
+ * 要么都生效要么都不生效。The stops of the person's runs and the destruction of their devices are owed
+ * in that transaction (outbox jobs, transactional outbox spec §8.1); the Redis revocation markers
+ * and the event publishes after the commit are hints ({@link afterOffboard}: `revoked_at` in PG is
+ * the authority).
  *
  * 已接受的窄窗口：离职提交时正在进行的 OAuth token 刷新（`Credentials` 刷新锁内）可能在提交之后把此人的
  * 个人凭据重新写回（upsert）。凭据只在此人以成员身份调用工具时使用，而其运行已被中止、成员关系已删除，
@@ -63,22 +69,25 @@ export declare class Offboarding {
     private readonly db;
     readonly txHost: TransactionHost;
     private readonly sessions;
-    private readonly redis;
     private readonly registry;
     private readonly eventLog;
-    /** The departed member's devices: one-off maintenance jobs, destroyed by a worker. */
-    private readonly sandboxJobs;
-    /** Stops the departed member's runs (`by: null`, a system action). */
+    /** Owes the run stops and device destroys in the offboard transaction. */
+    private readonly outbox;
+    /** Stops the departed member's runs (`by: null`, a system action) in the `run-stop` job. */
     private readonly runs;
     private readonly credentials;
     private readonly approvals;
     private readonly questions;
+    /** Owes `org.member_removed` to the subscribing plugins, in the offboard transaction. */
+    private readonly lifecycle;
     readonly log: KernelLogger;
-    constructor(db: KernelDb, txHost: TransactionHost, sessions: Pick<SessionService, "markRevoked">, redis: Pick<Redis, "set" | "exists">, registry: Pick<ExtensionRegistry, "list">, eventLog: Pick<EventLog, "notify" | "publishChanged">, 
-    /** The departed member's devices: one-off maintenance jobs, destroyed by a worker. */
-    sandboxJobs: Pick<SandboxJobs, "destroy">, 
-    /** Stops the departed member's runs (`by: null`, a system action). */
-    runs: OffboardingRunControl, credentials: Pick<Credentials, "clearAccount">, approvals: Pick<ApprovalsService, "cancelPendingFor">, questions: Pick<QuestionsService, "cancelPendingFor">, log: KernelLogger);
+    constructor(db: KernelDb, txHost: TransactionHost, sessions: Pick<SessionService, "markRevoked">, registry: Pick<ExtensionRegistry, "list">, eventLog: Pick<EventLog, "notify" | "publishChanged">, 
+    /** Owes the run stops and device destroys in the offboard transaction. */
+    outbox: Pick<OutboxService, "add">, 
+    /** Stops the departed member's runs (`by: null`, a system action) in the `run-stop` job. */
+    runs: OffboardingRunControl, credentials: Pick<Credentials, "clearAccount">, approvals: Pick<ApprovalsService, "pendingConversationsFor" | "cancelPendingFor">, questions: Pick<QuestionsService, "pendingConversationsFor" | "cancelPendingFor">, 
+    /** Owes `org.member_removed` to the subscribing plugins, in the offboard transaction. */
+    lifecycle: Pick<LifecycleHooks, "recordIn">, log: KernelLogger);
     /**
      * In the caller's transaction (`mandatory`: the caller holds the org row lock, order org row →
      * membership, and did the "last owner" checks)：
@@ -87,7 +96,8 @@ export declare class Offboarding {
      *    account (both are account-wide, so conservatively all); a person who leaves (`left`) keeps
      *    them, and the session's current org is cleared (the membership check ends the access);
      * 3. 删除其在该组织所有连接器上的个人凭据；
-     * 4. 取消其作为处理人的 pending 审批与提问（写 cancelled 事件），记下此人在该组织进行中的运行（提交后中止）；
+     * 4. 取消其作为处理人的 pending 审批与提问（写 cancelled 事件）; owes a `run-stop` job for each of the
+     *    person's active runs in the org;
      * 5. 其负责的该组织 agent 转给组织所有者（最早加入的 owner）并置 `needs_reassignment = true`；
      * 6. 归档其在该组织各渠道连接上的 IM 话题行（重新加入后在 IM 里开始新的对话，不接着旧会话），
      *    删除其在该组织的「以后都允许」工具授权（`tool_grants`）；
@@ -95,9 +105,10 @@ export declare class Offboarding {
      *    以及成员关系自身的 `directory:<插件>` 来源，不论此人当初是不是由通讯录加入（手动重新加入后再被移出也不会被同步加回）；
      *    同步不再把此人加回，直到有人手动重新加入（{@link MembersService.add} 删除排除）；
      * 8. 审计 `org.member.offboarded`；
-     * 9. Soft-deletes the person's devices in the org and clears their default row (the
-     *    instances and files are destroyed after the commit, best effort; the hourly sweep finishes
-     *    any left).
+     * 9. Soft-deletes the person's devices in the org and clears their default row, and owes the
+     *    destruction of each (instance and files, `device-destroy-<deviceId>`);
+     * 10. Owes the `org.member_removed` lifecycle hook deliveries (outbox jobs, one per subscribing
+     *    plugin).
      */
     offboard(orgId: string, accountId: string, reason: OffboardReason, actor?: {
         actorAccountId?: string;
@@ -107,36 +118,18 @@ export declare class Offboarding {
     private exclusionPlugins;
     /**
      * After the commit (callers register it with the infra `afterCommit` inside their transaction):
-     * writes the session revocation markers, publishes the cancelled events, stops the person's
-     * runs in the org and schedules destroying their devices. A failing step only logs a
-     * warning.
+     * writes the session revocation markers and publishes the cancelled events and queue changes,
+     * all hints (the run stops and device destroys were owed in the transaction). A failing step
+     * only logs a warning.
      */
     afterOffboard(r: OffboardResult): Promise<void>;
     /**
-     * 中止一个运行，顺序同会话中止接口：waiting → 直接取消等待并写终态；孤儿 → 直接写终态；
-     * 否则经 Redis 广播中止（运行所在副本收到后结束；尚未开始的运行开始时补查中止标记）。失败只记 warn。
-     * 不像会话中止那样随后起新一轮运行：离职者的待处理消息不再处理。
+     * The `run-stop` job (`MaintenanceProcessor`): stops a departed member's run the way the
+     * conversation's stop does (`POST .../abort`), by no account: a waiting run is aborted directly,
+     * a queued run is ended in PG (ruling R27: it never starts later, however long it was held), an
+     * orphan is finished directly, a running run is asked to stop through Redis (its replica ends it). A finished run is left alone. No
+     * new run follows, as after a person's stop: the departed member's queued messages were deleted.
+     * A failure throws (the job is retried).
      */
-    private stopRun;
-    /**
-     * 兜底扫描（维护队列每 10 分钟）：进行中、且发起人已不是该会话所属组织成员的运行 → 中止
-     * （提交后的中止没做成时，如进程崩溃）。按 `(runs.created_at, runs.id)` 键集分页翻完（每页 500 条，
-     * 至多 20 页）：经广播中止、尚未真正结束的运行仍是进行中，若只取前 500 条，超过 500 条卡住的运行会
-     * 一直占满这一页，后面的永远轮不到。已广播中止、仍带 `capk:offboard-abort:<runId>` 标记（30 分钟）的运行跳过，
-     * 不每轮重复广播；标记过期仍未结束的再处理一次。返回本次处理（未跳过）的运行数。
-     * 现任成员（含匿名访客这样的 guest 成员）发起的运行不受影响。
-     */
-    sweep(batch?: number): Promise<number>;
-    private recentlyAborted;
-}
-/**
- * The offboarding safety-net sweep ({@link Offboarding.sweep}) every 10 minutes, once per tick
- * across replicas; a worker-role job (`WorkerModule`).
- */
-export declare class OffboardSweepJob {
-    private readonly offboarding;
-    readonly lockService: LockService;
-    readonly log: KernelLogger;
-    constructor(offboarding: Pick<Offboarding, "sweep">, lockService: LockService, log: KernelLogger);
-    run(): Promise<void>;
+    stopRun(data: RunStopJobData): Promise<void>;
 }