@capybara-hq/kernel 0.1.0-rc.24 → 0.1.0-rc.26

package/dist/sandbox/jobs.js

@@ -19,10 +19,29 @@ export const SANDBOX_SUSPEND_JOB = "sandbox-suspend-idle";
 /** The inactivity destruction sweep (hourly, once per tick across replicas). */
 export const SANDBOX_DESTROY_JOB = "sandbox-destroy-inactive";
 /**
- * One-off job: destroy a batch of devices now, instance and file system (device
- * deletion, offboarding; data `{ deviceIds }`).
+ * One-off job: destroy devices now, instance and file system (data `{ deviceIds }`). Owed through
+ * the outbox by the transaction that soft-deletes the device (a device's deletion, offboarding),
+ * one device per job under {@link deviceDestroyJobId} ({@link oweDeviceDestroy}).
  */
 export const SANDBOX_DESTROY_FOR_JOB = "sandbox-destroy-for";
+/**
+ * The deterministic id of a deleted device's destroy job (no `:`): a device is deleted once, so
+ * its destruction is owed once.
+ */
+export const deviceDestroyJobId = (deviceId) => `device-destroy-${deviceId}`;
+/**
+ * Owes the destruction of a device in the caller's transaction, the one that soft-deleted it
+ * (transactional outbox spec §8.1; throws outside a transaction).
+ */
+export function oweDeviceDestroy(outbox, deviceId, orgId) {
+    return outbox.add({
+        queue: MAINTENANCE_QUEUE_NAME,
+        name: SANDBOX_DESTROY_FOR_JOB,
+        jobId: deviceDestroyJobId(deviceId),
+        data: { deviceIds: [deviceId] },
+        orgId,
+    });
+}
 /**
  * One-off job: start a device over with empty files (`resetFileSystem`; data
  * `{ deviceId }`, job id {@link resetFilesJobId}).
@@ -34,27 +53,14 @@ export const SANDBOX_RESET_FILES_JOB = "sandbox-reset-files";
  */
 export const resetFilesJobId = (deviceId) => `${SANDBOX_RESET_FILES_JOB}-${deviceId}`;
 /**
- * The producer of the sandbox maintenance jobs: {@link SANDBOX_DESTROY_FOR_JOB} (BullMQ's own job
- * ids, as before: every enqueue is a new job, also after a batch failed all its attempts) and
- * {@link SANDBOX_RESET_FILES_JOB}; `MaintenanceProcessor` consumes them.
+ * The producer of {@link SANDBOX_RESET_FILES_JOB} (destroys are owed through the outbox,
+ * {@link oweDeviceDestroy}); `MaintenanceProcessor` consumes both.
  */
 let SandboxJobs = class SandboxJobs {
     queue;
     constructor(queue) {
         this.queue = queue;
     }
-    /**
-     * Enqueues the destruction of these devices: a failed batch is retried whole (destroying
-     * one already destroyed does nothing).
-     */
-    async destroy(deviceIds) {
-        await this.queue.add(SANDBOX_DESTROY_FOR_JOB, { deviceIds }, {
-            attempts: 5,
-            backoff: { type: "exponential", delay: 30_000 },
-            removeOnComplete: true,
-            removeOnFail: 100,
-        });
-    }
     /**
      * Enqueues the reset of a device's files (see {@link resetFilesJobId}): retried on failure
      * (the exclusive lease not obtained in time, the provider failing); a job that failed all its

package/dist/schedules/fire.d.ts

@@ -7,8 +7,10 @@ import { KernelConfigService } from "../infra/config/kernel-config.service.js";
 import { KernelDb } from "../infra/database/kernel-db.js";
 import { TransactionHost } from "../infra/database/transaction-host.js";
 import { LockService } from "../infra/lock/lock.service.js";
+import { OutboxService } from "../infra/outbox/outbox.service.js";
 import type { KernelLogger } from "../log.js";
 import { OrgLock } from "../orgs/org-lock.js";
+import { type SchedSettleJobData } from "./job-ids.js";
 import { ScheduledTaskRunTracker } from "./run-tracker.js";
 import { ScheduledTaskStore } from "./store.js";
 /** A task run's message client id in a conversation's queue (never read back as "the task's"). */
@@ -21,10 +23,12 @@ export declare function localTimeOf(at: Date, timeZone: string): string;
  *
  * 1. The claim, one transaction under the org row lock (which serializes the org's rate limit
  *    count), then the task run and the task rows: the checks in order (task paused since the
- *    scan, unless a person's "run now"; org disabled, creator
- *    inactive or not a member, agent gone or not visible, conversation gone, connection inactive,
+ *    scan, unless a person's "run now"; a tick fired more than 10 minutes past the org's catch-up
+ *    window, `missed`, unless a "run now"; org disabled, creator inactive or not a member, agent
+ *    gone or not visible, conversation gone, connection inactive,
  *    overlap, rate limit, device offline, queue paused), each a `skipped` run with its reason, the
- *    lasting ones also pausing the task; then `fired_at` is written, after the rate limit count.
+ *    lasting ones also pausing the task; then `fired_at` is written, after the rate limit count,
+ *    with the run's delayed settle check owed (`sched-settle`, {@link settle}).
  *    A run whose claim committed before (a retried job) goes straight to the start.
  * 2. The start, idempotent: `new` creates the conversation with its first message keyed by the
  *    task run id (`create_request_id`); `conversation` queues the prompt in the target
@@ -51,12 +55,29 @@ export declare class ScheduledTaskFirer {
     /** Reads a local device's connection lease (online or not). */
     private readonly locks;
     private readonly log;
+    /** The claim owes the run's settle check (ruling R26). */
+    private readonly outbox;
     constructor(kdb: KernelDb, txHost: TransactionHost, config: Pick<KernelConfigService, "get">, store: ScheduledTaskStore, tracker: Pick<ScheduledTaskRunTracker, "inFlightIn" | "failStartIn">, orgLock: Pick<OrgLock, "tryLock">, agents: Pick<AgentLookup, "get">, access: Pick<ConversationAccess, "isAccessible">, conversations: Pick<ConversationsService, "createWithMessage" | "create">, queue: Pick<ConversationQueue, "add">, connections: Pick<ChannelConnections, "row" | "channel">, 
     /** Reads a local device's connection lease (online or not). */
-    locks: Pick<LockService, "holderOf">, log: Pick<KernelLogger, "warn" | "info">);
+    locks: Pick<LockService, "holderOf">, log: Pick<KernelLogger, "warn" | "info">, 
+    /** The claim owes the run's settle check (ruling R26). */
+    outbox: Pick<OutboxService, "add">);
     private get db();
     /** Fires task run `taskRunId` (see the class); a run not `pending` is left as it is. */
     fire(taskRunId: string): Promise<void>;
+    /**
+     * The `sched-settle` job (ruling R26; `job-ids.ts`). The claimed check: a task run claimed
+     * (`fired_at`) and still `pending` goes through the run tracker's check, which links it to its
+     * run, settles it, or releases a run that left no trace (`skipped`, `removed`), so a `once`
+     * task's last run never stays pending; one still in flight (its item queued, its message posted)
+     * owes the next check an hour on. The unfired check: a run still `pending` that no claim took
+     * (its fire job failed for good before the claim) is skipped, `missed` for a tick, `removed` for
+     * a "run now", so the overlap check never waits on it. A run claimed, started, ended or skipped
+     * since is left as it is (the claimed check follows a claimed run).
+     */
+    settle(data: SchedSettleJobData): Promise<void>;
+    /** The unfired check (see {@link settle}), task row first (the one lock order). */
+    private settleUnfired;
     /** Step 1 (see the class): a skip, the plan, or null when there is nothing to fire. */
     private claim;
     /** Step 2 (see the class). */

package/dist/schedules/fire.js

@@ -16,19 +16,20 @@ import { ConversationAccess, ConversationAccessModule } from "../conversations/a
 import { ConversationsModule } from "../conversations/conversations.module.js";
 import { ConversationsService } from "../conversations/conversations.service.js";
 import { ConversationQueue } from "../conversations/queue.service.js";
+import { channelSource } from "../conversations/source.js";
 import { ApiError } from "../errors.js";
 import { KernelConfigService } from "../infra/config/kernel-config.service.js";
 import { KernelDb } from "../infra/database/kernel-db.js";
 import { TransactionHost } from "../infra/database/transaction-host.js";
 import { Transactional } from "../infra/database/transactional.decorator.js";
 import { LockService } from "../infra/lock/lock.service.js";
-import { QueueModule } from "../infra/queue/queue.module.js";
+import { OutboxService } from "../infra/outbox/outbox.service.js";
 import { KERNEL_LOGGER } from "../infra/tokens.js";
 import { deviceLeaseKey } from "../local-devices/keys.js";
 import { OrgLock, OrgLockModule } from "../orgs/org-lock.js";
-import { MAINTENANCE_QUEUE_NAME } from "../runtime/maintenance-queue.js";
 import { RuntimeModule } from "../runtime/runtime.module.js";
 import { readScheduleSettings } from "./effective-settings.js";
+import { oweSchedSettle, SCHED_LATE_FIRE_SLACK_MS, SCHED_SETTLE_AFTER_MS, SCHED_SETTLE_LONG_CHECKS, SCHED_SETTLE_RECHECK_MS, } from "./job-ids.js";
 import { pauseTaskIn } from "./pause.js";
 import { lockTaskRow, ScheduledTaskRunTracker, ScheduledTaskTrackerModule } from "./run-tracker.js";
 import { ScheduledTaskScanJob } from "./scan.job.js";
@@ -69,10 +70,12 @@ const DEVICE_GONE = new Set(["device.not_found", "device.unavailable", "sandbox.
  *
  * 1. The claim, one transaction under the org row lock (which serializes the org's rate limit
  *    count), then the task run and the task rows: the checks in order (task paused since the
- *    scan, unless a person's "run now"; org disabled, creator
- *    inactive or not a member, agent gone or not visible, conversation gone, connection inactive,
+ *    scan, unless a person's "run now"; a tick fired more than 10 minutes past the org's catch-up
+ *    window, `missed`, unless a "run now"; org disabled, creator inactive or not a member, agent
+ *    gone or not visible, conversation gone, connection inactive,
  *    overlap, rate limit, device offline, queue paused), each a `skipped` run with its reason, the
- *    lasting ones also pausing the task; then `fired_at` is written, after the rate limit count.
+ *    lasting ones also pausing the task; then `fired_at` is written, after the rate limit count,
+ *    with the run's delayed settle check owed (`sched-settle`, {@link settle}).
  *    A run whose claim committed before (a retried job) goes straight to the start.
  * 2. The start, idempotent: `new` creates the conversation with its first message keyed by the
  *    task run id (`create_request_id`); `conversation` queues the prompt in the target
@@ -98,9 +101,12 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
     connections;
     locks;
     log;
+    outbox;
     constructor(kdb, txHost, config, store, tracker, orgLock, agents, access, conversations, queue, connections, 
     /** Reads a local device's connection lease (online or not). */
-    locks, log) {
+    locks, log, 
+    /** The claim owes the run's settle check (ruling R26). */
+    outbox) {
         this.kdb = kdb;
         this.txHost = txHost;
         this.config = config;
@@ -114,6 +120,7 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
         this.connections = connections;
         this.locks = locks;
         this.log = log;
+        this.outbox = outbox;
     }
     get db() {
         return this.kdb.db;
@@ -137,6 +144,61 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
             return;
         await this.start(claimed.plan);
     }
+    /**
+     * The `sched-settle` job (ruling R26; `job-ids.ts`). The claimed check: a task run claimed
+     * (`fired_at`) and still `pending` goes through the run tracker's check, which links it to its
+     * run, settles it, or releases a run that left no trace (`skipped`, `removed`), so a `once`
+     * task's last run never stays pending; one still in flight (its item queued, its message posted)
+     * owes the next check an hour on. The unfired check: a run still `pending` that no claim took
+     * (its fire job failed for good before the claim) is skipped, `missed` for a tick, `removed` for
+     * a "run now", so the overlap check never waits on it. A run claimed, started, ended or skipped
+     * since is left as it is (the claimed check follows a claimed run).
+     */
+    async settle(data) {
+        const { taskRunId } = data;
+        if ("unfired" in data) {
+            await this.settleUnfired(taskRunId);
+            return;
+        }
+        const run = await this.store.updateRun(this.db, taskRunId, {});
+        if (run?.status !== "pending" || run.firedAt === null)
+            return;
+        // inFlightIn takes the task row before the run (the one lock order)
+        if (!(await this.tracker.inFlightIn(run)))
+            return;
+        const now = await this.store.updateRun(this.db, taskRunId, {});
+        if (now?.status !== "pending" || now.firedAt === null)
+            return;
+        const next = data.check + 1;
+        if (next === SCHED_SETTLE_LONG_CHECKS) {
+            this.log.warn("scheduled task run still in flight a week after its claim", {
+                taskId: run.taskId,
+                taskRunId,
+            });
+        }
+        await oweSchedSettle(this.outbox, taskRunId, run.orgId, next, SCHED_SETTLE_RECHECK_MS);
+    }
+    /** The unfired check (see {@link settle}), task row first (the one lock order). */
+    async settleUnfired(taskRunId) {
+        const t = this.db;
+        const owner = await t
+            .selectFrom("scheduled_task_runs")
+            .select("task_id")
+            .where("id", "=", taskRunId)
+            .executeTakeFirst();
+        if (!owner)
+            return;
+        await lockTaskRow(t, owner.task_id);
+        const run = await t
+            .selectFrom("scheduled_task_runs")
+            .select(["status", "fired_at", "manual"])
+            .where("id", "=", taskRunId)
+            .forUpdate()
+            .executeTakeFirst();
+        if (run?.status !== "pending" || run.fired_at !== null)
+            return;
+        await this.skipIn(t, taskRunId, owner.task_id, run.manual ? "removed" : "missed");
+    }
     /** Step 1 (see the class): a skip, the plan, or null when there is nothing to fire. */
     async claim(taskRunId, orgId, deviceOffline) {
         const t = this.db;
@@ -179,6 +241,12 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
         if (!resumed && !run.manual && task.status === "paused")
             return skip("task_paused");
         const settings = await readScheduleSettings(t, this.config, orgId);
+        // a tick whose fire comes this late (workers down for long) is missed, as the scan records a
+        // tick it finds outside the window; a person's "run now" still runs
+        const lateAfter = settings.catchUpWindowMinutes * 60_000 + SCHED_LATE_FIRE_SLACK_MS;
+        if (!resumed && !run.manual && Date.now() - run.scheduledFor.getTime() > lateAfter) {
+            return skip("missed");
+        }
         if (!resumed && !settings.enabled)
             return skip("org_disabled");
         if (!resumed && !(await this.creatorActive(orgId, task.accountId))) {
@@ -260,6 +328,8 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
             firedAt: new Date(),
             ...(plan.mode === "conversation" && { conversationId: plan.conversationId }),
         });
+        // a start that fails for good leaves the run claimed: its settle check releases or ends it
+        await oweSchedSettle(this.outbox, run.id, orgId, 0, SCHED_SETTLE_AFTER_MS);
         return { plan: { ...plan, run: claimed ?? run } };
     }
     /** Step 2 (see the class). */
@@ -282,7 +352,7 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
                     ...(task.model !== null && { model: task.model }),
                     ...(task.effort !== null && { effort: task.effort }),
                 };
-                const out = await this.conversations.createWithMessage(task.orgId, task.accountId, task.agentId, { text: task.prompt, clientMessageId: run.id, ...pick }, task.deviceId === null ? {} : { deviceId: task.deviceId }, internal);
+                const out = await this.conversations.createWithMessage(task.orgId, task.accountId, task.agentId, { text: task.prompt, clientMessageId: run.id, ...pick }, task.deviceId === null ? {} : { deviceId: task.deviceId }, { ...internal, source: "web" });
                 // a retried start found the conversation made before: `createWithMessage` already asked
                 // for its run (and the first start's message owes the ensure-run backstop)
                 await this.setConversation(run.id, out.conversation.id);
@@ -345,7 +415,7 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
         const { conversationId, created } = await findOrBindThread(t, plan.thread, plan.target, async () => {
             const conv = await this.conversations.create(task.orgId, task.accountId, task.agentId, 
             // a chat task takes no device (new mode only): an IM conversation's default
-            { im: true });
+            { im: true }, { source: channelSource(plan.target.chatType) });
             await t
                 .updateTable("conversations")
                 .set({ scheduled_task_id: task.id })
@@ -382,15 +452,17 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
             return null;
         if (locked.conversation_id !== null)
             return locked.conversation_id;
+        const target = { chatId: plan.chatId, chatType: "group" };
         const conv = await this.conversations.create(task.orgId, task.accountId, task.agentId, 
         // a chat task takes no device (new mode only): an IM conversation's default
-        { im: true });
+        { im: true }, 
+        // stored now: the thread to derive it from binds only at the run's first post
+        { source: channelSource(target.chatType) });
         await t
             .updateTable("conversations")
             .set({ scheduled_task_id: task.id })
             .where("id", "=", conv.id)
             .execute();
-        const target = { chatId: plan.chatId, chatType: "group" };
         await t
             .insertInto("channel_deliveries")
             .values({
@@ -558,6 +630,9 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
         return (await this.locks.holderOf(deviceLeaseKey(deviceId))) === null;
     }
 };
+__decorate([
+    Transactional()
+], ScheduledTaskFirer.prototype, "settle", null);
 __decorate([
     Transactional()
 ], ScheduledTaskFirer.prototype, "claim", null);
@@ -590,7 +665,8 @@ ScheduledTaskFirer = __decorate([
     __param(9, Inject(ConversationQueue)),
     __param(10, Inject(ChannelConnections)),
     __param(11, Inject(LockService)),
-    __param(12, Inject(KERNEL_LOGGER))
+    __param(12, Inject(KERNEL_LOGGER)),
+    __param(13, Inject(OutboxService))
 ], ScheduledTaskFirer);
 export { ScheduledTaskFirer };
 /**
@@ -610,8 +686,6 @@ ScheduledTaskFireModule = __decorate([
             OrgLockModule,
             ScheduledTaskStoreModule,
             ScheduledTaskTrackerModule,
-            // the scanner enqueues the fire jobs
-            QueueModule.registerQueue(MAINTENANCE_QUEUE_NAME),
         ],
         providers: [ScheduledTaskFirer, ScheduledTaskScanJob],
         exports: [ScheduledTaskFirer],

package/dist/schedules/job-ids.d.ts

@@ -1,4 +1,5 @@
-/** The scheduled tasks' jobs (a leaf module: no imports). */
+import type { OutboxService } from "../infra/outbox/outbox.service.js";
+/** The scheduled tasks' jobs (a leaf module: type imports only). */
 /**
  * The once-a-minute `@LockedCron` that turns due tasks into task runs (spec §6); its name enqueued
  * on the maintenance queue runs one scan (`ScheduledRuns.runNow`).
@@ -13,20 +14,68 @@ export interface SchedFireJobData {
 /** The fire job's id: `sched-fire-<taskRunId>` (no `:`, task run ids are UUIDs). */
 export declare const schedFireJobId: (taskRunId: string) => string;
 /**
- * The fire job of one task run. Removed when done, so the scanner's repair can enqueue it again
- * after a lost job; the fire is idempotent on the task run's status, so retries are safe.
- */
-export declare function schedFireJob(taskRunId: string): {
-    name: string;
-    data: SchedFireJobData;
-    opts: {
-        jobId: string;
-        attempts: number;
-        backoff: {
-            type: "exponential";
-            delay: number;
-        };
-        removeOnComplete: true;
-        removeOnFail: true;
-    };
+ * Owes the fire job of task run `taskRunId` in the transaction in context (transactional outbox
+ * spec §8.1): the transaction that writes a pending task run (the scan's tick, a person's "run
+ * now") owes its fire, so a tick is never left unfired. The job's options are its outbox registry
+ * entry's; the fire is idempotent on the task run's status.
+ */
+export declare function oweSchedFire(outbox: Pick<OutboxService, "add">, taskRunId: string, orgId: string): Promise<boolean>;
+/**
+ * The delayed job that settles a task run left pending (ruling R26, fix round 1), on
+ * `cap-kernel-maintenance`. Two kinds:
+ * - the claimed check, owed by the fire job's claim: the run tracker links, settles or releases a
+ *   run claimed but never started (`sched-settle-<taskRunId>`, then `-<check>`);
+ * - the unfired check, owed with the task run itself (the scan's tick, "run now"): a run its fire
+ *   job never claimed (the job failed for good before its claim) is skipped, `missed` for a tick,
+ *   `removed` for a "run now", so it never blocks the task's later ticks as an overlap
+ *   (`sched-settle-<taskRunId>-unfired`).
+ */
+export declare const SCHED_SETTLE_JOB = "sched-settle";
+/**
+ * The data of a {@link SCHED_SETTLE_JOB}: the claimed check (`check` counts the checks before this
+ * one) or the unfired check.
+ */
+export type SchedSettleJobData = {
+    taskRunId: string;
+    check: number;
+} | {
+    taskRunId: string;
+    unfired: true;
 };
+/**
+ * The first claimed check: 10 minutes after the claim, when the start's retries (seconds) are long
+ * over and the run tracker's start grace (`START_GRACE_MS`, 5 minutes) has passed, so a run that
+ * left no trace is released at once. Also the unfired check's delay after a "run now".
+ */
+export declare const SCHED_SETTLE_AFTER_MS: number;
+/**
+ * A claimed run still in flight at a check (its item queued, its message posted and not yet taken
+ * by a run) is checked again hourly. The chain ends with the first check that finds the run
+ * started, ended or released: the queued item is consumed or removed, the message taken by a run.
+ * A chain past {@link SCHED_SETTLE_LONG_CHECKS} checks (a week) is logged as a warning, once.
+ */
+export declare const SCHED_SETTLE_RECHECK_MS: number;
+/** A week of hourly checks: the claimed check warns at this one (a run in flight that long). */
+export declare const SCHED_SETTLE_LONG_CHECKS = 168;
+/**
+ * How long past the org's catch-up window a tick's fire may come before the tick is missed (the
+ * fire job's own rule, and when a tick's unfired check runs): the scan fires a tick still inside
+ * the window, and its fire job follows within seconds.
+ */
+export declare const SCHED_LATE_FIRE_SLACK_MS: number;
+/** The claimed check's id: `sched-settle-<taskRunId>`, then `-<check>` for the later checks. */
+export declare const schedSettleJobId: (taskRunId: string, check: number) => string;
+/** The unfired check's id: `sched-settle-<taskRunId>-unfired`. */
+export declare const schedSettleUnfiredJobId: (taskRunId: string) => string;
+/**
+ * Owes claimed check number `check` of task run `taskRunId` in the transaction in context, due
+ * `after` milliseconds from now.
+ */
+export declare function oweSchedSettle(outbox: Pick<OutboxService, "add">, taskRunId: string, orgId: string, check: number, after: number): Promise<boolean>;
+/**
+ * Owes the unfired check of task run `taskRunId` in the transaction that writes the run: for a
+ * tick of the schedule (`catchUpWindowMinutes` given) when the fire job would itself call the
+ * tick missed (its tick plus the window plus {@link SCHED_LATE_FIRE_SLACK_MS}, at least
+ * {@link SCHED_SETTLE_AFTER_MS} on); for a "run now" (null) {@link SCHED_SETTLE_AFTER_MS} on.
+ */
+export declare function oweSchedSettleUnfired(outbox: Pick<OutboxService, "add">, taskRunId: string, orgId: string, scheduledFor: Date, catchUpWindowMinutes: number | null): Promise<boolean>;

package/dist/schedules/job-ids.js

@@ -1,4 +1,4 @@
-/** The scheduled tasks' jobs (a leaf module: no imports). */
+/** The scheduled tasks' jobs (a leaf module: type imports only). */
 /**
  * The once-a-minute `@LockedCron` that turns due tasks into task runs (spec §6); its name enqueued
  * on the maintenance queue runs one scan (`ScheduledRuns.runNow`).
@@ -9,19 +9,87 @@ export const SCHED_FIRE_JOB = "sched-fire";
 /** The fire job's id: `sched-fire-<taskRunId>` (no `:`, task run ids are UUIDs). */
 export const schedFireJobId = (taskRunId) => `${SCHED_FIRE_JOB}-${taskRunId}`;
 /**
- * The fire job of one task run. Removed when done, so the scanner's repair can enqueue it again
- * after a lost job; the fire is idempotent on the task run's status, so retries are safe.
+ * Owes the fire job of task run `taskRunId` in the transaction in context (transactional outbox
+ * spec §8.1): the transaction that writes a pending task run (the scan's tick, a person's "run
+ * now") owes its fire, so a tick is never left unfired. The job's options are its outbox registry
+ * entry's; the fire is idempotent on the task run's status.
  */
-export function schedFireJob(taskRunId) {
-    return {
+export function oweSchedFire(outbox, taskRunId, orgId) {
+    return outbox.add({
+        queue: "cap-kernel-maintenance",
         name: SCHED_FIRE_JOB,
+        jobId: schedFireJobId(taskRunId),
         data: { taskRunId },
-        opts: {
-            jobId: schedFireJobId(taskRunId),
-            attempts: 3,
-            backoff: { type: "exponential", delay: 2_000 },
-            removeOnComplete: true,
-            removeOnFail: true,
-        },
-    };
+        orgId,
+    });
+}
+/**
+ * The delayed job that settles a task run left pending (ruling R26, fix round 1), on
+ * `cap-kernel-maintenance`. Two kinds:
+ * - the claimed check, owed by the fire job's claim: the run tracker links, settles or releases a
+ *   run claimed but never started (`sched-settle-<taskRunId>`, then `-<check>`);
+ * - the unfired check, owed with the task run itself (the scan's tick, "run now"): a run its fire
+ *   job never claimed (the job failed for good before its claim) is skipped, `missed` for a tick,
+ *   `removed` for a "run now", so it never blocks the task's later ticks as an overlap
+ *   (`sched-settle-<taskRunId>-unfired`).
+ */
+export const SCHED_SETTLE_JOB = "sched-settle";
+/**
+ * The first claimed check: 10 minutes after the claim, when the start's retries (seconds) are long
+ * over and the run tracker's start grace (`START_GRACE_MS`, 5 minutes) has passed, so a run that
+ * left no trace is released at once. Also the unfired check's delay after a "run now".
+ */
+export const SCHED_SETTLE_AFTER_MS = 10 * 60_000;
+/**
+ * A claimed run still in flight at a check (its item queued, its message posted and not yet taken
+ * by a run) is checked again hourly. The chain ends with the first check that finds the run
+ * started, ended or released: the queued item is consumed or removed, the message taken by a run.
+ * A chain past {@link SCHED_SETTLE_LONG_CHECKS} checks (a week) is logged as a warning, once.
+ */
+export const SCHED_SETTLE_RECHECK_MS = 60 * 60_000;
+/** A week of hourly checks: the claimed check warns at this one (a run in flight that long). */
+export const SCHED_SETTLE_LONG_CHECKS = 168;
+/**
+ * How long past the org's catch-up window a tick's fire may come before the tick is missed (the
+ * fire job's own rule, and when a tick's unfired check runs): the scan fires a tick still inside
+ * the window, and its fire job follows within seconds.
+ */
+export const SCHED_LATE_FIRE_SLACK_MS = 10 * 60_000;
+/** The claimed check's id: `sched-settle-<taskRunId>`, then `-<check>` for the later checks. */
+export const schedSettleJobId = (taskRunId, check) => check === 0 ? `${SCHED_SETTLE_JOB}-${taskRunId}` : `${SCHED_SETTLE_JOB}-${taskRunId}-${check}`;
+/** The unfired check's id: `sched-settle-<taskRunId>-unfired`. */
+export const schedSettleUnfiredJobId = (taskRunId) => `${SCHED_SETTLE_JOB}-${taskRunId}-unfired`;
+/**
+ * Owes claimed check number `check` of task run `taskRunId` in the transaction in context, due
+ * `after` milliseconds from now.
+ */
+export function oweSchedSettle(outbox, taskRunId, orgId, check, after) {
+    return outbox.add({
+        queue: "cap-kernel-maintenance",
+        name: SCHED_SETTLE_JOB,
+        jobId: schedSettleJobId(taskRunId, check),
+        data: { taskRunId, check },
+        runAt: new Date(Date.now() + after),
+        orgId,
+    });
+}
+/**
+ * Owes the unfired check of task run `taskRunId` in the transaction that writes the run: for a
+ * tick of the schedule (`catchUpWindowMinutes` given) when the fire job would itself call the
+ * tick missed (its tick plus the window plus {@link SCHED_LATE_FIRE_SLACK_MS}, at least
+ * {@link SCHED_SETTLE_AFTER_MS} on); for a "run now" (null) {@link SCHED_SETTLE_AFTER_MS} on.
+ */
+export function oweSchedSettleUnfired(outbox, taskRunId, orgId, scheduledFor, catchUpWindowMinutes) {
+    const soonest = Date.now() + SCHED_SETTLE_AFTER_MS;
+    const at = catchUpWindowMinutes === null
+        ? soonest
+        : Math.max(soonest, scheduledFor.getTime() + catchUpWindowMinutes * 60_000 + SCHED_LATE_FIRE_SLACK_MS);
+    return outbox.add({
+        queue: "cap-kernel-maintenance",
+        name: SCHED_SETTLE_JOB,
+        jobId: schedSettleUnfiredJobId(taskRunId),
+        data: { taskRunId, unfired: true },
+        runAt: new Date(at),
+        orgId,
+    });
 }

package/dist/schedules/run-tracker.js

@@ -114,7 +114,8 @@ let ScheduledTaskRunTracker = class ScheduledTaskRunTracker {
             await this.settle(tr, run.status);
             return false;
         }
-        // not started yet: its fire job is still to come (the scanner repairs a lost one)
+        // not claimed yet: its fire job is still to come (the run's unfired settle check skips it
+        // when the job failed for good before its claim)
         if (tr.fired_at === null)
             return true;
         // only a run with a target conversation is ever queued (`conversation`, `chat`): no scan of

package/dist/schedules/scan.job.d.ts

@@ -1,10 +1,9 @@
-import type { Queue } from "bullmq";
 import { KernelConfigService } from "../infra/config/kernel-config.service.js";
 import { KernelDb } from "../infra/database/kernel-db.js";
 import { TransactionHost } from "../infra/database/transaction-host.js";
 import { LockService } from "../infra/lock/lock.service.js";
+import { OutboxService } from "../infra/outbox/outbox.service.js";
 import type { KernelLogger } from "../log.js";
-import { ScheduledTaskRunTracker } from "./run-tracker.js";
 import { ScheduledTaskStore } from "./store.js";
 /**
  * The scheduled task scanner (spec §6), once a minute across replicas. Each due task in its own
@@ -12,53 +11,35 @@ import { ScheduledTaskStore } from "./store.js";
  * same task): the ticks missed outside the org's `catchUpWindowMinutes` become one `skipped` run
  * (`missed`, at the first missed tick), the latest tick in the window gets its run (the unique
  * `(task_id, scheduled_for)` key makes a second insert a no-op), `next_run_at` moves to the next
- * tick plus the task's jitter (or a `once` task ends, or one past its `expires_at`), and after the
- * commit `sched-fire-<taskRunId>` is enqueued. A task whose stored trigger the schedule math cannot
- * evaluate is paused (`invalid_schedule`) instead of failing the scan; a database error fails the
- * pass (the next minute retries) and pauses nothing.
- *
- * Then the repair: a run still `pending` and unfired 10 minutes after its tick has its fire job
- * enqueued again, and is given up (`skipped`, `missed`) after an hour; a run fired an hour ago and
- * still `pending` (its start failed for good) goes through the run tracker's check, which links,
- * settles or releases it, so a `once` task's last run never stays pending.
+ * tick plus the task's jitter (or a `once` task ends, or one past its `expires_at`), and the same
+ * transaction owes `sched-fire-<taskRunId>` (transactional outbox spec §8.1), so a written tick is
+ * never left unfired and the scan needs no repair of lost fire jobs (a fire delivered too late
+ * marks its tick missed itself, `ScheduledTaskFirer`), and the run's unfired settle check (a fire
+ * job that failed for good before its claim: the tick is missed, `job-ids.ts`). A task whose stored
+ * trigger the schedule math cannot evaluate is paused (`invalid_schedule`) instead of failing the
+ * scan; a database error fails the pass (the next minute retries) and pauses nothing.
  */
 export declare class ScheduledTaskScanJob {
     private readonly kdb;
     readonly txHost: TransactionHost;
     private readonly config;
     private readonly store;
-    private readonly tracker;
-    private readonly maintenance;
+    private readonly outbox;
     readonly lockService: LockService;
     readonly log: KernelLogger;
-    constructor(kdb: KernelDb, txHost: TransactionHost, config: Pick<KernelConfigService, "get">, store: ScheduledTaskStore, tracker: Pick<ScheduledTaskRunTracker, "inFlightIn">, maintenance: Pick<Queue, "add">, lockService: LockService, log: KernelLogger);
+    constructor(kdb: KernelDb, txHost: TransactionHost, config: Pick<KernelConfigService, "get">, store: ScheduledTaskStore, outbox: Pick<OutboxService, "add">, lockService: LockService, log: KernelLogger);
     private get db();
     scan(): Promise<void>;
+    /** One scan as of `now` (tests pass their own); returns how many due tasks it handled. */
+    scanAt(now: Date): Promise<number>;
     /**
-     * One scan as of `now` (tests pass their own); returns how many due tasks it handled. The repair
-     * runs even when the scan loop failed (its error is rethrown afterwards). `repairBatch`: how many
-     * rows each repair step takes (tests pass a small one).
-     */
-    scanAt(now: Date, repairBatch?: number): Promise<number>;
-    /**
-     * One due task in its own transaction (a task whose trigger cannot be evaluated rolls back
-     * alone and is paused, so it cannot block the others): locked with `SKIP LOCKED`, so replicas take different tasks. False
-     * when no task is due.
+     * One due task in its own transaction (a task whose trigger cannot be evaluated rolls back alone
+     * and is paused, so it cannot block the others): locked with `SKIP LOCKED`, so replicas take
+     * different tasks. False when no task is due.
      */
     private scanOne;
     /** A task the scan cannot advance stops being due: paused with `invalid_schedule`. */
     private quarantine;
-    /** One due task (see the class); returns the id of the run to fire, if any. */
+    /** One due task (see the class); returns the run to fire, if any. */
     private advance;
-    /** Enqueues the fire jobs (after the commit; a failure is repaired by a later scan). */
-    private enqueue;
-    /** Lost fire jobs (see the class). */
-    private repair;
-    /** A run never fired an hour after its tick: given up, task row first (the one lock order). */
-    private giveUp;
-    /**
-     * A started run still pending long after: the tracker links, settles or releases it; one still
-     * in flight is stamped (`rechecked_at`) and goes to the back of the next rechecks.
-     */
-    private recheck;
 }