@capybara-hq/kernel 0.1.0-rc.21 → 0.1.0-rc.23

package/dist/schedules/scan.job.js

@@ -0,0 +1,303 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+import { randomUUID } from "node:crypto";
+import { getQueueToken } from "@nestjs/bullmq";
+import { Inject, Injectable } from "@nestjs/common";
+import { sql } from "kysely";
+import { KernelConfigService } from "../infra/config/kernel-config.service.js";
+import { afterCommit } from "../infra/database/after-commit.js";
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { TransactionHost } from "../infra/database/transaction-host.js";
+import { Transactional } from "../infra/database/transactional.decorator.js";
+import { LockService } from "../infra/lock/lock.service.js";
+import { LockedCron } from "../infra/schedule/locked-cron.decorator.js";
+import { KERNEL_LOGGER } from "../infra/tokens.js";
+import { MAINTENANCE_QUEUE_NAME } from "../runtime/maintenance-queue.js";
+import { readScheduleSettings } from "./effective-settings.js";
+import { SCHED_SCAN_JOB, schedFireJob } from "./job-ids.js";
+import { pauseTaskIn } from "./pause.js";
+import { lockTaskRow, ScheduledTaskRunTracker } from "./run-tracker.js";
+import { dueTicks, nextTick } from "./schedule-math.js";
+import { ScheduledTaskStore } from "./store.js";
+/** Due tasks per scan; what is left waits for the next minute. */
+const MAX_TASKS = 2000;
+/** A pending task run never fired this long after its tick: its fire job is enqueued again. */
+const REPAIR_AFTER_MS = 10 * 60_000;
+/** ... and this long after its tick it is given up (`skipped`, `missed`). */
+const GIVE_UP_AFTER_MS = 60 * 60_000;
+const REPAIR_BATCH = 100;
+/**
+ * The schedule math could not evaluate a due task's stored trigger (its transaction rolled back);
+ * the scan pauses it. Only the math raises it: a database error fails the scan pass instead (the
+ * next minute retries) and never pauses a healthy task.
+ */
+class TaskScanFailed extends Error {
+    taskId;
+    constructor(taskId, cause) {
+        super(`scheduled task ${taskId} cannot be advanced`, { cause });
+        this.taskId = taskId;
+    }
+}
+/**
+ * The scheduled task scanner (spec §6), once a minute across replicas. Each due task in its own
+ * transaction, locked with `FOR UPDATE SKIP LOCKED` (replicas share the work and never take the
+ * same task): the ticks missed outside the org's `catchUpWindowMinutes` become one `skipped` run
+ * (`missed`, at the first missed tick), the latest tick in the window gets its run (the unique
+ * `(task_id, scheduled_for)` key makes a second insert a no-op), `next_run_at` moves to the next
+ * tick plus the task's jitter (or a `once` task ends, or one past its `expires_at`), and after the
+ * commit `sched-fire-<taskRunId>` is enqueued. A task whose stored trigger the schedule math cannot
+ * evaluate is paused (`invalid_schedule`) instead of failing the scan; a database error fails the
+ * pass (the next minute retries) and pauses nothing.
+ *
+ * Then the repair: a run still `pending` and unfired 10 minutes after its tick has its fire job
+ * enqueued again, and is given up (`skipped`, `missed`) after an hour; a run fired an hour ago and
+ * still `pending` (its start failed for good) goes through the run tracker's check, which links,
+ * settles or releases it, so a `once` task's last run never stays pending.
+ */
+let ScheduledTaskScanJob = class ScheduledTaskScanJob {
+    kdb;
+    txHost;
+    config;
+    store;
+    tracker;
+    maintenance;
+    lockService;
+    log;
+    constructor(kdb, txHost, config, store, tracker, maintenance, lockService, log) {
+        this.kdb = kdb;
+        this.txHost = txHost;
+        this.config = config;
+        this.store = store;
+        this.tracker = tracker;
+        this.maintenance = maintenance;
+        this.lockService = lockService;
+        this.log = log;
+    }
+    get db() {
+        return this.kdb.db;
+    }
+    async scan() {
+        await this.scanAt(new Date());
+    }
+    /**
+     * One scan as of `now` (tests pass their own); returns how many due tasks it handled. The repair
+     * runs even when the scan loop failed (its error is rethrown afterwards). `repairBatch`: how many
+     * rows each repair step takes (tests pass a small one).
+     */
+    async scanAt(now, repairBatch = REPAIR_BATCH) {
+        const settings = new Map();
+        let total = 0;
+        try {
+            for (; total < MAX_TASKS; total++) {
+                try {
+                    if (!(await this.scanOne(now, settings)))
+                        break;
+                }
+                catch (err) {
+                    if (!(err instanceof TaskScanFailed))
+                        throw err;
+                    // one task that cannot be advanced must not hold the others back every minute
+                    this.log.error("scheduled task cannot be advanced; paused", {
+                        taskId: err.taskId,
+                        err: err.cause,
+                    });
+                    await this.quarantine(err.taskId);
+                }
+            }
+        }
+        finally {
+            await this.repair(now, repairBatch);
+        }
+        return total;
+    }
+    /**
+     * One due task in its own transaction (a task whose trigger cannot be evaluated rolls back
+     * alone and is paused, so it cannot block the others): locked with `SKIP LOCKED`, so replicas take different tasks. False
+     * when no task is due.
+     */
+    async scanOne(now, settings) {
+        const t = this.db;
+        const [task] = await this.store.lockDue(t, now, 1);
+        if (!task)
+            return false;
+        let s = settings.get(task.orgId);
+        if (s === undefined) {
+            s = await readScheduleSettings(t, this.config, task.orgId);
+            settings.set(task.orgId, s);
+        }
+        const fired = await this.advance(t, task, now, s.catchUpWindowMinutes);
+        if (fired !== null) {
+            const id = fired;
+            afterCommit(() => this.enqueue([id]));
+        }
+        return true;
+    }
+    /** A task the scan cannot advance stops being due: paused with `invalid_schedule`. */
+    async quarantine(taskId) {
+        await pauseTaskIn(this.db, taskId, "invalid_schedule", null);
+    }
+    /** One due task (see the class); returns the id of the run to fire, if any. */
+    async advance(t, task, now, catchUpWindowMinutes) {
+        const jitterMs = task.jitterSeconds * 1000;
+        // ticks are before jitter: the due tick is next_run_at - jitter, and a tick is due once
+        // tick + jitter <= now
+        const dueTick = new Date(task.nextRunAt.getTime() - jitterMs);
+        const until = new Date(now.getTime() - jitterMs);
+        // the schedule math alone may mark the task broken; database errors fail this pass only
+        let ticks;
+        let next;
+        try {
+            ticks = dueTicks(task.trigger, new Date(dueTick.getTime() - 1), until, catchUpWindowMinutes);
+            next = nextTick(task.trigger, until);
+        }
+        catch (err) {
+            throw new TaskScanFailed(task.id, err);
+        }
+        const expires = task.expiresAt?.getTime() ?? Number.POSITIVE_INFINITY;
+        const fireAt = ticks.fire !== null && ticks.fire.getTime() <= expires ? ticks.fire : null;
+        const first = ticks.firstMissed;
+        if (first !== null && first.getTime() <= expires) {
+            await this.store.insertRun(t, {
+                id: randomUUID(),
+                taskId: task.id,
+                orgId: task.orgId,
+                scheduledFor: first,
+                trial: false,
+                status: "skipped",
+                skipReason: "missed",
+            });
+        }
+        let fired = null;
+        if (fireAt !== null) {
+            const run = await this.store.insertRun(t, {
+                id: randomUUID(),
+                taskId: task.id,
+                orgId: task.orgId,
+                scheduledFor: fireAt,
+                trial: false,
+            });
+            fired = run?.id ?? null;
+        }
+        const patch = next === null
+            ? { status: "ended", nextRunAt: null }
+            : next.getTime() > expires
+                ? { status: "ended", pauseReason: "expired", nextRunAt: null }
+                : { nextRunAt: new Date(next.getTime() + jitterMs) };
+        await this.store.updateTask(t, task.id, patch);
+        return fired;
+    }
+    /** Enqueues the fire jobs (after the commit; a failure is repaired by a later scan). */
+    async enqueue(ids) {
+        for (const id of ids) {
+            const job = schedFireJob(id);
+            try {
+                await this.maintenance.add(job.name, job.data, job.opts);
+            }
+            catch (err) {
+                this.log.warn("enqueueing a scheduled task run failed", { taskRunId: id, err });
+            }
+        }
+    }
+    /** Lost fire jobs (see the class). */
+    async repair(now, batch) {
+        const stale = await this.db
+            .selectFrom("scheduled_task_runs")
+            .select(["id", "scheduled_for"])
+            .where("status", "=", "pending")
+            .where("fired_at", "is", null)
+            .where("scheduled_for", "<", new Date(now.getTime() - REPAIR_AFTER_MS))
+            .orderBy("scheduled_for")
+            .limit(batch)
+            .execute();
+        const retry = [];
+        const lost = [];
+        for (const r of stale) {
+            const at = r.scheduled_for instanceof Date ? r.scheduled_for : new Date(r.scheduled_for);
+            (now.getTime() - at.getTime() > GIVE_UP_AFTER_MS ? lost : retry).push(r.id);
+        }
+        for (const id of lost)
+            await this.giveUp(id);
+        await this.enqueue(retry);
+        // the least recently rechecked first, so runs still in flight do not starve the others
+        const started = await this.db
+            .selectFrom("scheduled_task_runs")
+            .select("id")
+            .where("status", "=", "pending")
+            .where("fired_at", "<", new Date(now.getTime() - GIVE_UP_AFTER_MS))
+            .orderBy(sql `rechecked_at NULLS FIRST`)
+            .orderBy("scheduled_for")
+            .limit(batch)
+            .execute();
+        for (const r of started)
+            await this.recheck(r.id, now);
+    }
+    /** A run never fired an hour after its tick: given up, task row first (the one lock order). */
+    async giveUp(taskRunId) {
+        const owner = await this.db
+            .selectFrom("scheduled_task_runs")
+            .select("task_id")
+            .where("id", "=", taskRunId)
+            .executeTakeFirst();
+        if (!owner)
+            return;
+        await lockTaskRow(this.db, owner.task_id);
+        await this.db
+            .updateTable("scheduled_task_runs")
+            .set({ status: "skipped", skip_reason: "missed", error: "The run could not be started" })
+            .where("id", "=", taskRunId)
+            .where("status", "=", "pending")
+            .where("fired_at", "is", null)
+            .execute();
+    }
+    /**
+     * A started run still pending long after: the tracker links, settles or releases it; one still
+     * in flight is stamped (`rechecked_at`) and goes to the back of the next rechecks.
+     */
+    async recheck(taskRunId, now) {
+        const run = await this.store.updateRun(this.db, taskRunId, {});
+        if (run?.status !== "pending")
+            return;
+        // inFlightIn takes the task row before the run (the one lock order)
+        if (await this.tracker.inFlightIn(run, now)) {
+            await this.db
+                .updateTable("scheduled_task_runs")
+                .set({ rechecked_at: now })
+                .where("id", "=", taskRunId)
+                .execute();
+        }
+    }
+};
+__decorate([
+    LockedCron("* * * * *", { name: SCHED_SCAN_JOB })
+], ScheduledTaskScanJob.prototype, "scan", null);
+__decorate([
+    Transactional()
+], ScheduledTaskScanJob.prototype, "scanOne", null);
+__decorate([
+    Transactional()
+], ScheduledTaskScanJob.prototype, "quarantine", null);
+__decorate([
+    Transactional()
+], ScheduledTaskScanJob.prototype, "giveUp", null);
+__decorate([
+    Transactional()
+], ScheduledTaskScanJob.prototype, "recheck", null);
+ScheduledTaskScanJob = __decorate([
+    Injectable(),
+    __param(0, Inject(KernelDb)),
+    __param(1, Inject(TransactionHost)),
+    __param(2, Inject(KernelConfigService)),
+    __param(3, Inject(ScheduledTaskStore)),
+    __param(4, Inject(ScheduledTaskRunTracker)),
+    __param(5, Inject(getQueueToken(MAINTENANCE_QUEUE_NAME))),
+    __param(6, Inject(LockService)),
+    __param(7, Inject(KERNEL_LOGGER))
+], ScheduledTaskScanJob);
+export { ScheduledTaskScanJob };

package/dist/schedules/schedule-math.d.ts

@@ -0,0 +1,31 @@
+import type { ScheduleTrigger } from "./types.js";
+export type TriggerError = "schedule.invalid_cron" | "schedule.invalid_time_zone" | "schedule.invalid_once_time" | "schedule.once_in_past";
+export declare function validateTrigger(t: ScheduleTrigger, now: Date): {
+    ok: true;
+} | {
+    ok: false;
+    code: TriggerError;
+};
+/** The first tick strictly after `after` (before jitter); null for a past once task. */
+export declare function nextTick(t: ScheduleTrigger, after: Date): Date | null;
+/** The smallest gap, in minutes, among the next 50 fire times after `from`; null for once. */
+export declare function minGapMinutes(t: ScheduleTrigger, from: Date): number | null;
+/** A fixed per-task offset: `fnv1a32(id) mod min(floor(10 % of the interval), 300 s)`; once is 0. */
+export declare function jitterSeconds(taskId: string, t: ScheduleTrigger, from: Date): number;
+/**
+ * {@link jitterSeconds} from a gap already sampled ({@link minGapMinutes} of the same trigger and
+ * time), so a caller that checked the interval does not walk the 50 ticks again; null gap is 0.
+ */
+export declare function jitterForGap(taskId: string, gap: number | null): number;
+/**
+ * The ticks in (lastTick, now], without walking them (spec §6 step 1): `fire` is the latest one
+ * when it lies inside the catch-up window, else null; `firstMissed` is the earliest of the others
+ * (every tick but `fire`), which the scanner records as one skipped run, or null when there are
+ * none. The latest tick is found by widening a window back from `now` (1 minute, doubling) until
+ * it holds a tick, then stepping forward inside it, so a month-long outage costs a handful of tick
+ * computations, not thousands.
+ */
+export declare function dueTicks(t: ScheduleTrigger, lastTick: Date, now: Date, catchUpWindowMinutes: number): {
+    fire: Date | null;
+    firstMissed: Date | null;
+};

package/dist/schedules/schedule-math.js

@@ -0,0 +1,223 @@
+import { CronTime } from "cron";
+import { isTimeZone } from "../time-zone.js";
+/** Spec §5: how many upcoming fire times the minimum-interval check looks at. */
+const GAP_SAMPLE = 50;
+const JITTER_CAP_SECONDS = 300;
+const LOCAL_AT = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2}))?$/;
+function cronTime(expr, timeZone) {
+    if (expr.trim().split(/\s+/).length !== 5)
+        return null;
+    try {
+        return new CronTime(expr.trim(), timeZone);
+    }
+    catch {
+        return null;
+    }
+}
+const formatters = new Map();
+/** One formatter per zone (pure, so a module-level cache is safe). */
+function zoneFormat(timeZone) {
+    let f = formatters.get(timeZone);
+    if (!f) {
+        f = new Intl.DateTimeFormat("en-US", {
+            timeZone,
+            hourCycle: "h23",
+            year: "numeric",
+            month: "numeric",
+            day: "numeric",
+            hour: "numeric",
+            minute: "numeric",
+            second: "numeric",
+        });
+        formatters.set(timeZone, f);
+    }
+    return f;
+}
+/** The wall clock of `utc` (epoch ms) in `timeZone`, read as if it were UTC (epoch ms). */
+function wallOf(utc, timeZone) {
+    const p = Object.fromEntries(zoneFormat(timeZone)
+        .formatToParts(new Date(utc))
+        .map((x) => [x.type, x.value]));
+    return Date.UTC(Number(p.year), Number(p.month) - 1, Number(p.day), Number(p.hour), Number(p.minute), Number(p.second));
+}
+/** The zone's UTC offset (ms) at `utc`. */
+function offsetAt(utc, timeZone) {
+    return wallOf(Math.floor(utc / 1000) * 1000, timeZone) - Math.floor(utc / 1000) * 1000;
+}
+/**
+ * The UTC instant of a local date-time (no offset) in `timeZone`; null when malformed or out of
+ * range. A repeated local time takes its first occurrence; a time inside a DST gap resolves to the
+ * first valid instant after the gap.
+ */
+function onceInstant(at, timeZone) {
+    const m = LOCAL_AT.exec(at);
+    if (!m)
+        return null;
+    const n = m.slice(1).map((v) => (v === undefined ? 0 : Number(v)));
+    const [y = 0, mo = 0, d = 0, h = 0, mi = 0, sec = 0] = n;
+    const wall = Date.UTC(y, mo - 1, d, h, mi, sec);
+    const check = new Date(wall);
+    if (check.getUTCFullYear() !== y ||
+        check.getUTCMonth() !== mo - 1 ||
+        check.getUTCDate() !== d ||
+        h > 23 ||
+        mi > 59 ||
+        sec > 59) {
+        return null;
+    }
+    const before = offsetAt(wall - 86400000, timeZone);
+    const after = offsetAt(wall + 86400000, timeZone);
+    const valid = [before, after]
+        .map((o) => wall - o)
+        .filter((u, i) => offsetAt(u, timeZone) === [before, after][i])
+        .sort((x, z) => x - z);
+    if (valid[0] !== undefined)
+        return new Date(valid[0]);
+    // In a gap: find the transition second between the two candidates.
+    let lo = Math.floor((wall - after) / 1000);
+    let hi = Math.floor((wall - before) / 1000);
+    while (lo < hi) {
+        const mid = Math.floor((lo + hi) / 2);
+        if (offsetAt(mid * 1000, timeZone) === before)
+            lo = mid + 1;
+        else
+            hi = mid;
+    }
+    return new Date(lo * 1000);
+}
+export function validateTrigger(t, now) {
+    if (!isTimeZone(t.timeZone))
+        return { ok: false, code: "schedule.invalid_time_zone" };
+    if (t.kind === "cron") {
+        return cronTime(t.expr, t.timeZone)
+            ? { ok: true }
+            : { ok: false, code: "schedule.invalid_cron" };
+    }
+    const at = onceInstant(t.at, t.timeZone);
+    if (!at)
+        return { ok: false, code: "schedule.invalid_once_time" };
+    if (at.getTime() <= now.getTime())
+        return { ok: false, code: "schedule.once_in_past" };
+    return { ok: true };
+}
+/** The first tick strictly after `after` (before jitter); null for a past once task. */
+export function nextTick(t, after) {
+    return ticker(t)(after);
+}
+/** How far back a DST transition can move a tick: longer than any gap (spec §5). */
+const TRANSITION_LOOKBACK_MS = 6 * 3600_000;
+/**
+ * {@link nextTick} for one trigger, with its `CronTime`s built once: callers that walk many ticks
+ * (the gap sample, the catch-up enumeration) reuse one ticker instead of rebuilding per tick.
+ */
+function ticker(t) {
+    if (t.kind === "once") {
+        const at = onceInstant(t.at, t.timeZone);
+        return (after) => (at && at.getTime() > after.getTime() ? at : null);
+    }
+    const time = cronTime(t.expr, t.timeZone);
+    const wallTime = cronTime(t.expr, "UTC");
+    if (!time || !wallTime)
+        return () => null;
+    return (after) => {
+        let cursor = after;
+        for (let i = 0; i < 10; i++) {
+            const tick = new Date(time.getNextDateFrom(cursor, t.timeZone).toMillis());
+            const ms = tick.getTime();
+            // CronTime moves a tick that falls in a DST gap to the next valid instant; the spec says a
+            // skipped local time does not fire, so keep only ticks whose wall clock the expression
+            // names. A moved tick lies just after a transition, so only then is the check needed.
+            if (offsetAt(ms, t.timeZone) === offsetAt(ms - TRANSITION_LOOKBACK_MS, t.timeZone)) {
+                return tick;
+            }
+            const wall = localWallClock(tick, t.timeZone);
+            if (wallTime.getNextDateFrom(new Date(wall - 1000), "UTC").toMillis() === wall)
+                return tick;
+            cursor = tick;
+        }
+        return null;
+    };
+}
+/** The wall clock of `at` in `timeZone`, read as if it were UTC (epoch ms, seconds dropped). */
+function localWallClock(at, timeZone) {
+    return Math.floor(wallOf(at.getTime(), timeZone) / 60000) * 60000;
+}
+/** The smallest gap, in minutes, among the next 50 fire times after `from`; null for once. */
+export function minGapMinutes(t, from) {
+    if (t.kind === "once")
+        return null;
+    const next = ticker(t);
+    let prev = next(from);
+    let min = Number.POSITIVE_INFINITY;
+    for (let i = 1; prev && i < GAP_SAMPLE; i++) {
+        const tick = next(prev);
+        if (!tick)
+            break;
+        min = Math.min(min, (tick.getTime() - prev.getTime()) / 60000);
+        prev = tick;
+    }
+    return Number.isFinite(min) ? min : null;
+}
+/** 32-bit FNV-1a over the UTF-8 bytes of `s`. */
+function fnv1a32(s) {
+    let h = 0x811c9dc5;
+    for (const b of new TextEncoder().encode(s)) {
+        h ^= b;
+        h = Math.imul(h, 0x01000193) >>> 0;
+    }
+    return h;
+}
+/** A fixed per-task offset: `fnv1a32(id) mod min(floor(10 % of the interval), 300 s)`; once is 0. */
+export function jitterSeconds(taskId, t, from) {
+    return jitterForGap(taskId, minGapMinutes(t, from));
+}
+/**
+ * {@link jitterSeconds} from a gap already sampled ({@link minGapMinutes} of the same trigger and
+ * time), so a caller that checked the interval does not walk the 50 ticks again; null gap is 0.
+ */
+export function jitterForGap(taskId, gap) {
+    if (gap === null)
+        return 0;
+    const range = Math.min(Math.floor(0.1 * gap * 60), JITTER_CAP_SECONDS);
+    return range > 0 ? fnv1a32(taskId) % range : 0;
+}
+/**
+ * The ticks in (lastTick, now], without walking them (spec §6 step 1): `fire` is the latest one
+ * when it lies inside the catch-up window, else null; `firstMissed` is the earliest of the others
+ * (every tick but `fire`), which the scanner records as one skipped run, or null when there are
+ * none. The latest tick is found by widening a window back from `now` (1 minute, doubling) until
+ * it holds a tick, then stepping forward inside it, so a month-long outage costs a handful of tick
+ * computations, not thousands.
+ */
+export function dueTicks(t, lastTick, now, catchUpWindowMinutes) {
+    const next = ticker(t);
+    const end = now.getTime();
+    const first = next(lastTick);
+    if (!first || first.getTime() > end)
+        return { fire: null, firstMissed: null };
+    const latest = latestTick(next, lastTick.getTime(), end) ?? first;
+    const fire = latest.getTime() >= end - catchUpWindowMinutes * 60000 ? latest : null;
+    const missed = fire === null || first.getTime() < fire.getTime();
+    return { fire, firstMissed: missed ? first : null };
+}
+/** The latest tick in (lastMs, endMs]; null when there is none. */
+function latestTick(next, lastMs, endMs) {
+    let from = null;
+    for (let span = 60_000;; span *= 2) {
+        const lo = Math.max(endMs - span, lastMs);
+        const tick = next(new Date(lo));
+        if (tick && tick.getTime() <= endMs) {
+            from = tick;
+            break;
+        }
+        if (lo === lastMs)
+            return null;
+    }
+    let latest = from;
+    for (;;) {
+        const n = next(latest);
+        if (!n || n.getTime() > endMs)
+            return latest;
+        latest = n;
+    }
+}