@theowlops/boombot 0.0.0-stage → 1.1.1

package/dist/node_modules/@boombot/core/scheduler/runner.js

@@ -0,0 +1,257 @@
+/**
+ * Scheduler tick loop.
+ *
+ * Each tick (non-overlapping by construction):
+ *   1. recoverExpiredLeases — crash recovery for claimed/running runs;
+ *   2. claimDueJobs — atomic claim + misfire-policy evaluation;
+ *   3. driveRuns — submit turns through the narrow `TurnSubmitter` port,
+ *      bounded by a monotonic deadline (spec 04: wall clock for eligibility,
+ *      monotonic clock for elapsed deadlines).
+ *
+ * The runner never executes schedule payloads itself: it only submits the
+ * registered prompt text as an agent turn with the run's idempotency key.
+ */
+import { randomUUID } from 'node:crypto';
+import { performance } from 'node:perf_hooks';
+import { parseCron } from './cron.js';
+import { classifyMisfire, computeNextFire } from './evaluator.js';
+import { buildClaimFires } from './registry.js';
+import {} from './store.js';
+import {} from './types.js';
+const EMPTY_STATS = {
+    recovered: 0,
+    claimed: 0,
+    skippedConcurrency: 0,
+    driven: 0,
+    completed: 0,
+    retried: 0,
+    deadLettered: 0,
+};
+function asIso(ms) {
+    return new Date(ms).toISOString();
+}
+export class SchedulerRunner {
+    store;
+    submitter;
+    clock;
+    runnerId;
+    tickIntervalMs;
+    misfireGraceMs;
+    leaseMs;
+    maxTickDurationMs;
+    claimBatchSize;
+    jitter;
+    onError;
+    timer = null;
+    tickInFlight = null;
+    stopped = false;
+    constructor(options) {
+        this.store = options.store;
+        this.submitter = options.submitter;
+        this.clock = options.clock ?? { now: () => Date.now() };
+        this.runnerId = options.runnerId ?? `runner_${randomUUID().replace(/-/g, '')}`;
+        this.tickIntervalMs = options.tickIntervalMs ?? 1000;
+        this.misfireGraceMs = options.misfireGraceMs ?? 60000;
+        this.leaseMs = options.leaseMs ?? 60000;
+        this.maxTickDurationMs = options.maxTickDurationMs ?? 5000;
+        this.claimBatchSize = options.claimBatchSize ?? 100;
+        this.jitter = options.jitter ?? Math.random;
+        this.onError = options.onError ?? (() => undefined);
+    }
+    get id() {
+        return this.runnerId;
+    }
+    /** Start the non-overlapping tick chain. */
+    start() {
+        this.stopped = false;
+        this.scheduleNext();
+    }
+    /** Stop ticking; resolves when the in-flight tick (if any) settles. */
+    async stop() {
+        this.stopped = true;
+        if (this.timer !== null) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+        if (this.tickInFlight !== null) {
+            await this.tickInFlight;
+        }
+    }
+    scheduleNext() {
+        if (this.stopped)
+            return;
+        this.timer = setTimeout(() => {
+            this.timer = null;
+            const tick = this.tickOnce().catch((error) => this.onError(error, 'drive'));
+            this.tickInFlight = tick.then(() => undefined);
+            void tick.finally(() => {
+                this.tickInFlight = null;
+                this.scheduleNext();
+            });
+        }, this.tickIntervalMs);
+        // Don't keep the process alive just for the scheduler in tests/scripts.
+        if (typeof this.timer.unref === 'function')
+            this.timer.unref();
+    }
+    /** Run one full tick. Exposed for tests; `start()` uses it internally. */
+    async tickOnce(nowMs) {
+        const now = nowMs ?? this.clock.now();
+        const stats = { ...EMPTY_STATS };
+        // 1. Crash recovery.
+        try {
+            const recovered = this.store.recoverExpiredLeases({
+                nowIso: asIso(now),
+                runnerId: this.runnerId,
+                leaseMs: this.leaseMs,
+            });
+            stats.recovered = recovered.length;
+        }
+        catch (error) {
+            this.onError(error, 'recover');
+        }
+        // 2. Claim due jobs.
+        try {
+            const due = this.store.getDueJobs(asIso(now), this.claimBatchSize);
+            for (const job of due) {
+                const result = this.claimJob(job, now);
+                if (result === 'claimed')
+                    stats.claimed += 1;
+                else if (result === 'skipped_concurrency')
+                    stats.skippedConcurrency += 1;
+            }
+        }
+        catch (error) {
+            this.onError(error, 'claim');
+        }
+        // 3. Drive runs, bounded by a monotonic deadline.
+        const deadline = performance.now() + this.maxTickDurationMs;
+        try {
+            const drivable = this.store.getDrivableRuns(asIso(now), this.claimBatchSize);
+            for (const run of drivable) {
+                if (performance.now() >= deadline)
+                    break;
+                await this.driveRun(run, now, stats);
+            }
+        }
+        catch (error) {
+            this.onError(error, 'drive');
+        }
+        return stats;
+    }
+    claimJob(job, nowMs) {
+        let schedule;
+        try {
+            schedule = parseCron(job.cronExpr);
+        }
+        catch {
+            return 'skipped'; // validated at creation; defensive
+        }
+        const nextRunAtMs = job.nextRunAt === null ? NaN : Date.parse(job.nextRunAt);
+        if (Number.isNaN(nextRunAtMs) || nextRunAtMs > nowMs)
+            return 'skipped';
+        const misfired = classifyMisfire(nextRunAtMs, nowMs, this.misfireGraceMs) === 'misfired';
+        let fireTimes;
+        let newNextRunAt;
+        if (!misfired) {
+            fireTimes = [asIso(nextRunAtMs)];
+            const next = computeNextFire(schedule, nextRunAtMs, job.timezone);
+            if (next === undefined)
+                return 'skipped';
+            newNextRunAt = asIso(next);
+        }
+        else if (job.missedRunPolicy === 'skip') {
+            fireTimes = [];
+            const next = computeNextFire(schedule, nowMs, job.timezone);
+            if (next === undefined)
+                return 'skipped';
+            newNextRunAt = asIso(next);
+        }
+        else if (job.missedRunPolicy === 'run_once') {
+            fireTimes = [asIso(nextRunAtMs)];
+            const next = computeNextFire(schedule, nowMs, job.timezone);
+            if (next === undefined)
+                return 'skipped';
+            newNextRunAt = asIso(next);
+        }
+        else {
+            // catch_up_bounded: oldest-first missed fires, capped.
+            fireTimes = [];
+            let cursor = nextRunAtMs;
+            for (let i = 0; i < job.maxCatchUp; i += 1) {
+                if (cursor > nowMs)
+                    break;
+                fireTimes.push(asIso(cursor));
+                const next = computeNextFire(schedule, cursor, job.timezone);
+                if (next === undefined)
+                    break;
+                cursor = next;
+            }
+            const resume = computeNextFire(schedule, nowMs, job.timezone);
+            if (resume === undefined)
+                return 'skipped';
+            newNextRunAt = asIso(resume);
+        }
+        const claim = this.store.claimJob({
+            jobId: job.id,
+            expectedNextRunAt: job.nextRunAt,
+            fires: buildClaimFires(job, fireTimes),
+            newNextRunAt,
+            concurrency: job.concurrencyPolicy,
+            runnerId: this.runnerId,
+            leaseExpiresAt: asIso(nowMs + this.leaseMs),
+            nowIso: asIso(nowMs),
+        });
+        if (claim.ok)
+            return 'claimed';
+        if (claim.reason === 'concurrency_forbid')
+            return 'skipped_concurrency';
+        return 'skipped';
+    }
+    async driveRun(run, nowMs, stats) {
+        const jobId = run.jobId;
+        const runId = run.id;
+        const job = this.store.getJob(jobId);
+        if (job === undefined || !job.enabled) {
+            this.store.deadLetterRun(runId, 'job_disabled', asIso(nowMs));
+            stats.deadLettered += 1;
+            return;
+        }
+        if (!this.store.markRunning(runId, this.runnerId, asIso(nowMs))) {
+            return; // lost race with another driver
+        }
+        stats.driven += 1;
+        try {
+            await this.submitter.submit({
+                principalId: job.ownerPrincipalId,
+                text: job.promptText,
+                idempotencyKey: run.idempotencyKey,
+                // Run context: hosts (gateway scheduler service) need job/run ids
+                // to record delivery outcomes (`schedule.run-now` reads them back).
+                jobId: job.id,
+                jobName: job.name,
+                runId: run.id,
+                attempt: run.attempt,
+            });
+            this.store.completeRun(runId, asIso(nowMs));
+            stats.completed += 1;
+        }
+        catch {
+            const attempt = run.attempt + 1;
+            if (attempt > job.maxAttempts) {
+                this.store.deadLetterRun(runId, 'submit_failed_exhausted', asIso(nowMs));
+                stats.deadLettered += 1;
+            }
+            else {
+                const delay = this.backoffDelay(job, attempt);
+                this.store.requeueRun(runId, attempt, asIso(nowMs + delay), asIso(nowMs + this.leaseMs), 'submit_failed');
+                stats.retried += 1;
+            }
+        }
+    }
+    /** Bounded exponential backoff with jitter. */
+    backoffDelay(job, attempt) {
+        const exp = job.baseRetryDelayMs * 2 ** (attempt - 1);
+        const capped = Math.min(job.maxRetryDelayMs, exp);
+        return capped + Math.floor(this.jitter() * job.baseRetryDelayMs);
+    }
+}

package/dist/node_modules/@boombot/core/scheduler/store.d.ts

@@ -0,0 +1,150 @@
+import { type ConcurrencyPolicy, type MissedRunPolicy, type RunStatus, type SchedulerJob } from './types.js';
+export interface SchedulerRunRecord {
+    readonly id: string;
+    readonly jobId: string;
+    readonly idempotencyKey: string;
+    readonly scheduledFireAt: string;
+    readonly status: RunStatus;
+    readonly attempt: number;
+    readonly notBeforeAt: string;
+    readonly claimedBy: string;
+    readonly leaseExpiresAt: string;
+    readonly claimedAt: string;
+    readonly completedAt: string | null;
+    readonly errorCode: string | null;
+}
+/** Outcome of one dispatch attempt for a claimed scheduler run. */
+export type DeliveryOutcome = 'fired' | 'completed' | 'failed';
+/**
+ * One row of `scheduler_deliveries`: the gateway records `fired` when a run
+ * is claimed for dispatch, then `completed` (with the turn id) or `failed`
+ * (with the error). `schedule.run-now` reads these back to find the turn id
+ * of the manual fire.
+ */
+export interface SchedulerDeliveryRecord {
+    readonly runId: string;
+    readonly jobId: string;
+    readonly attempt: number;
+    readonly recordedAt: string;
+    readonly outcome: DeliveryOutcome;
+    readonly turnId: string | null;
+    readonly error: string | null;
+}
+/** Migration SQL for repo integration (would become migration v6). */
+export declare const SCHEDULER_SCHEMA_SQL = "\nCREATE TABLE IF NOT EXISTS scheduler_schedules (\n  id TEXT PRIMARY KEY,\n  owner_principal_id TEXT NOT NULL,\n  name TEXT NOT NULL,\n  cron_expr TEXT NOT NULL,\n  timezone TEXT NOT NULL,\n  prompt_text TEXT NOT NULL,\n  enabled INTEGER NOT NULL DEFAULT 1 CHECK (enabled IN (0, 1)),\n  concurrency_policy TEXT NOT NULL DEFAULT 'forbid'\n    CHECK (concurrency_policy IN ('forbid', 'allow', 'replace')),\n  missed_run_policy TEXT NOT NULL DEFAULT 'skip'\n    CHECK (missed_run_policy IN ('skip', 'run_once', 'catch_up_bounded')),\n  max_catch_up INTEGER NOT NULL DEFAULT 3\n    CHECK (max_catch_up BETWEEN 1 AND 100),\n  max_attempts INTEGER NOT NULL DEFAULT 3\n    CHECK (max_attempts BETWEEN 1 AND 10),\n  base_retry_delay_ms INTEGER NOT NULL DEFAULT 5000\n    CHECK (base_retry_delay_ms BETWEEN 100 AND 3600000),\n  max_retry_delay_ms INTEGER NOT NULL DEFAULT 300000\n    CHECK (max_retry_delay_ms BETWEEN 1000 AND 86400000),\n  next_run_at TEXT,\n  created_at TEXT NOT NULL,\n  updated_at TEXT NOT NULL\n);\nCREATE INDEX IF NOT EXISTS scheduler_schedules_due\n  ON scheduler_schedules (enabled, next_run_at);\nCREATE INDEX IF NOT EXISTS scheduler_schedules_owner\n  ON scheduler_schedules (owner_principal_id);\n\nCREATE TABLE IF NOT EXISTS scheduler_runs (\n  id TEXT PRIMARY KEY,\n  job_id TEXT NOT NULL REFERENCES scheduler_schedules(id) ON DELETE CASCADE,\n  idempotency_key TEXT NOT NULL,\n  scheduled_fire_at TEXT NOT NULL,\n  status TEXT NOT NULL CHECK (status IN\n    ('claimed', 'running', 'completed', 'failed', 'dead_letter')),\n  attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt >= 1),\n  not_before_at TEXT NOT NULL,\n  claimed_by TEXT NOT NULL,\n  lease_expires_at TEXT NOT NULL,\n  claimed_at TEXT NOT NULL,\n  completed_at TEXT,\n  error_code TEXT,\n  UNIQUE (job_id, idempotency_key)\n);\nCREATE INDEX IF NOT EXISTS scheduler_runs_drivable\n  ON scheduler_runs (status, not_before_at, lease_expires_at);\nCREATE INDEX IF NOT EXISTS scheduler_runs_job\n  ON scheduler_runs (job_id, status);\n\nCREATE TABLE IF NOT EXISTS scheduler_deliveries (\n  run_id TEXT NOT NULL,\n  job_id TEXT NOT NULL REFERENCES scheduler_schedules(id) ON DELETE CASCADE,\n  attempt INTEGER NOT NULL CHECK (attempt >= 1),\n  recorded_at TEXT NOT NULL,\n  outcome TEXT NOT NULL CHECK (outcome IN ('fired', 'completed', 'failed')),\n  turn_id TEXT,\n  error TEXT\n);\nCREATE INDEX IF NOT EXISTS scheduler_deliveries_job\n  ON scheduler_deliveries (job_id, recorded_at DESC);\n";
+export declare function newJobId(): string;
+export declare function newRunId(): string;
+/** Deterministic idempotency key per (job, fire time). */
+export declare function idempotencyKeyFor(jobId: string, scheduledFireAtIso: string): string;
+export interface ClaimFire {
+    readonly runId: string;
+    readonly idempotencyKey: string;
+    readonly scheduledFireAt: string;
+}
+export type ClaimResult = {
+    readonly ok: true;
+    readonly runs: readonly SchedulerRunRecord[];
+} | {
+    readonly ok: false;
+    readonly reason: 'lost_race' | 'concurrency_forbid' | 'not_due' | 'disabled' | 'not_found';
+};
+export declare class ScheduleStore {
+    private readonly db;
+    private constructor();
+    /** Uniform error contract: storage failures surface as SchedulerError. */
+    private io;
+    static open(path: string): ScheduleStore;
+    static openMemory(): ScheduleStore;
+    close(): void;
+    private insertJobRow;
+    insertJob(job: SchedulerJob): void;
+    /**
+     * SECURITY FIX (quota check-then-act, low): count + insert inside one
+     * `BEGIN IMMEDIATE` transaction. The old code did `countByOwner()` then
+     * `insertJob()` as two statements — safe in a single process (no await
+     * between them), but two processes sharing one SQLite file could both pass
+     * the count check before either inserted, bypassing the per-principal
+     * quota. `BEGIN IMMEDIATE` takes the write lock up front, so exactly one
+     * contender wins the race; the loser sees the winner's row in its count.
+     */
+    insertJobWithQuotaCheck(job: SchedulerJob, maxPerPrincipal: number): void;
+    getJob(id: string): SchedulerJob | undefined;
+    updateJob(id: string, patch: {
+        readonly name?: string;
+        readonly cronExpr?: string;
+        readonly timezone?: string;
+        readonly promptText?: string;
+        readonly enabled?: boolean;
+        readonly concurrencyPolicy?: ConcurrencyPolicy;
+        readonly missedRunPolicy?: MissedRunPolicy;
+        readonly maxCatchUp?: number;
+        readonly maxAttempts?: number;
+        readonly baseRetryDelayMs?: number;
+        readonly maxRetryDelayMs?: number;
+        readonly nextRunAt?: string | null;
+        readonly updatedAt: string;
+    }): SchedulerJob | undefined;
+    deleteJob(id: string): boolean;
+    countByOwner(ownerPrincipalId: string): number;
+    listByOwner(ownerPrincipalId: string): SchedulerJob[];
+    /** Jobs eligible to fire at `nowIso`, oldest first. */
+    getDueJobs(nowIso: string, limit: number): SchedulerJob[];
+    hasActiveRun(jobId: string): boolean;
+    /**
+     * Atomically claim a job: CAS on the previously-read `next_run_at`,
+     * insert run rows (UNIQUE backstop on idempotency), advance `next_run_at`.
+     * Concurrency is enforced inside the same transaction:
+     * - `forbid`: refuse when an active run exists;
+     * - `allow`: no check, overlapping runs permitted;
+     * - `replace`: supersede in-flight runs before claiming.
+     * Single BEGIN IMMEDIATE transaction — one winner under races.
+     */
+    claimJob(input: {
+        readonly jobId: string;
+        readonly expectedNextRunAt: string | null;
+        readonly fires: readonly ClaimFire[];
+        readonly newNextRunAt: string | null;
+        readonly concurrency: ConcurrencyPolicy;
+        readonly runnerId: string;
+        readonly leaseExpiresAt: string;
+        readonly nowIso: string;
+    }): ClaimResult;
+    getRun(id: string): SchedulerRunRecord | undefined;
+    listRuns(jobId: string): SchedulerRunRecord[];
+    /** Runs this runner may drive now. */
+    getDrivableRuns(nowIso: string, limit: number): SchedulerRunRecord[];
+    markRunning(runId: string, runnerId: string, _nowIso: string): boolean;
+    completeRun(runId: string, nowIso: string): boolean;
+    /**
+     * Requeue a failed run for retry. Returns false when the run is not in a
+     * retryable state (lost race with another driver).
+     */
+    requeueRun(runId: string, attempt: number, notBeforeAt: string, leaseExpiresAt: string, errorCode: string): boolean;
+    deadLetterRun(runId: string, errorCode: string, nowIso: string): boolean;
+    /**
+     * Crash recovery: runs stuck in claimed/running past their lease.
+     * Exhausted attempts go to dead_letter; the rest are requeued with a fresh
+     * lease under the recovering runner. Returns requeued runs.
+     */
+    recoverExpiredLeases(input: {
+        readonly nowIso: string;
+        readonly runnerId: string;
+        readonly leaseMs: number;
+    }): SchedulerRunRecord[];
+    /**
+     * Record one dispatch outcome for a run (`fired` when the run is claimed
+     * for dispatch, then `completed` with the turn id or `failed` with the
+     * error). Rows are append-only; readers take the latest per run.
+     */
+    recordDelivery(input: {
+        readonly runId: string;
+        readonly jobId: string;
+        readonly attempt: number;
+        readonly recordedAt: string;
+        readonly outcome: DeliveryOutcome;
+        readonly turnId?: string | null;
+        readonly error?: string | null;
+    }): void;
+    /** Latest-first delivery rows for a job (bounded by `limit`). */
+    listDeliveries(jobId: string, limit: number): SchedulerDeliveryRecord[];
+}