cursedbelt-server 1.0.0

package/src/server/jobs/runner.spec.ts

@@ -0,0 +1,213 @@
+import { Database } from 'bun:sqlite';
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { ApiError } from '../errors';
+import { createJobRunner } from './runner';
+import { JobTimeoutError } from './timeout';
+
+const JOB_RUNS_DDL = `CREATE TABLE job_runs (
+  id TEXT PRIMARY KEY,
+  job_name TEXT NOT NULL,
+  started_at TEXT NOT NULL,
+  ended_at TEXT,
+  status TEXT NOT NULL DEFAULT 'running',
+  error TEXT
+)`;
+
+const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+const quietLogger = { error() {}, warn() {}, info() {}, debug() {} };
+const runCount = (db: Database, name: string) =>
+  (db.query('SELECT COUNT(*) AS n FROM job_runs WHERE job_name = ?').get(name) as { n: number }).n;
+
+describe('createJobRunner', () => {
+  let db: Database;
+  beforeEach(() => {
+    db = new Database(':memory:');
+    db.run(JOB_RUNS_DDL);
+  });
+  afterEach(() => db.close());
+
+  test('runOnce executes the job immediately and audits an ok run (acceptance)', async () => {
+    let ran = false;
+    const runner = createJobRunner(db, { logger: quietLogger });
+    runner.register({
+      name: 'once',
+      async run() {
+        ran = true;
+      },
+    });
+    runner.start();
+    await runner.runOnce('once');
+    expect(ran).toBe(true);
+
+    await runner.stop();
+    const last = runner.getLastRun('once');
+    expect(last?.status).toBe('ok');
+    expect(last?.endedAt).toBeTruthy();
+    expect(last?.error).toBeUndefined();
+  });
+
+  test('getLastRun returns null before any run, then the latest run', async () => {
+    const runner = createJobRunner(db, { logger: quietLogger });
+    runner.register({ name: 'j', async run() {} });
+    expect(runner.getLastRun('j')).toBeNull();
+    await runner.runOnce('j');
+    await runner.runOnce('j');
+    expect(runCount(db, 'j')).toBe(2);
+    expect(runner.getLastRun('j')?.status).toBe('ok');
+    await runner.stop();
+  });
+
+  test('a throwing job is audited as error and runOnce rejects', async () => {
+    const runner = createJobRunner(db, { logger: quietLogger });
+    runner.register({
+      name: 'boom',
+      async run() {
+        throw new Error('kaboom');
+      },
+    });
+    await expect(runner.runOnce('boom')).rejects.toThrow('kaboom');
+    const last = runner.getLastRun('boom');
+    expect(last?.status).toBe('error');
+    expect(last?.error).toContain('kaboom');
+    await runner.stop();
+  });
+
+  test('runOnce on an unknown job throws ApiError(404) (integration)', async () => {
+    const runner = createJobRunner(db, { logger: quietLogger });
+    try {
+      await runner.runOnce('nope');
+      throw new Error('should have thrown');
+    } catch (err) {
+      expect(err).toBeInstanceOf(ApiError);
+      expect((err as ApiError).status).toBe(404);
+    }
+    await runner.stop();
+  });
+
+  test('a job that exceeds its timeout is aborted and audited as a timeout error', async () => {
+    const runner = createJobRunner(db, { logger: quietLogger });
+    let sawAbort = false;
+    runner.register({
+      name: 'slow',
+      timeout: 30,
+      run({ signal }) {
+        // Resolve only when aborted, so the handler promise never leaks.
+        return new Promise<void>((resolve) => {
+          signal.addEventListener(
+            'abort',
+            () => {
+              sawAbort = true;
+              resolve();
+            },
+            { once: true },
+          );
+        });
+      },
+    });
+    await expect(runner.runOnce('slow')).rejects.toBeInstanceOf(JobTimeoutError);
+    expect(sawAbort).toBe(true);
+    const last = runner.getLastRun('slow');
+    expect(last?.status).toBe('error');
+    expect(last?.error).toContain('timed out');
+    await runner.stop();
+  });
+
+  test('a cron job fires durably via its worker and audits ok runs (integration)', async () => {
+    const runner = createJobRunner(db, { logger: quietLogger, pollIntervalMs: 30 });
+    runner.register({ name: 'tick', cron: '* * * * * *', async run() {} });
+    runner.start();
+    await sleep(160);
+    await runner.stop();
+    // The first schedule fires immediately (next_run_at = 0), then on the cron.
+    expect(runCount(db, 'tick')).toBeGreaterThanOrEqual(1);
+    expect(runner.getLastRun('tick')?.status).toBe('ok');
+    // plainjob created its own tables — no collision with the job_runs audit table.
+    const tables = db
+      .query("SELECT name FROM sqlite_master WHERE type='table'")
+      .all()
+      .map((r) => (r as { name: string }).name);
+    expect(tables).toContain('plainjob_jobs');
+    expect(tables).toContain('job_runs');
+  });
+
+  test('stop() does NOT close the shared database', async () => {
+    const runner = createJobRunner(db, { logger: quietLogger, pollIntervalMs: 30 });
+    runner.register({ name: 'tick', cron: '* * * * * *', async run() {} });
+    runner.start();
+    await sleep(60);
+    await runner.stop();
+    // Would throw if queue.close() had closed the underlying db.
+    expect(() => db.query('SELECT 1 AS x').get()).not.toThrow();
+  });
+
+  test('registering a duplicate name throws', () => {
+    const runner = createJobRunner(db, { logger: quietLogger });
+    runner.register({ name: 'dup', async run() {} });
+    expect(() => runner.register({ name: 'dup', async run() {} })).toThrow('already registered');
+  });
+
+  test('register/start after stop throws', async () => {
+    const runner = createJobRunner(db, { logger: quietLogger });
+    await runner.stop();
+    expect(() => runner.register({ name: 'late', async run() {} })).toThrow('stopped');
+    expect(() => runner.start()).toThrow('stopped');
+  });
+
+  test('registering a cron job after start spins up its worker immediately', async () => {
+    const runner = createJobRunner(db, { logger: quietLogger, pollIntervalMs: 30 });
+    runner.start();
+    runner.register({ name: 'late-cron', cron: '* * * * * *', async run() {} });
+    await sleep(120);
+    await runner.stop();
+    expect(runner.getLastRun('late-cron')?.status).toBe('ok');
+  });
+
+  const seedRun = (name: string, startedAt: string) =>
+    db
+      .query("INSERT INTO job_runs (id, job_name, started_at, ended_at, status) VALUES (?, ?, ?, ?, 'ok')")
+      .run(crypto.randomUUID(), name, startedAt, startedAt);
+
+  test('retention prunes audit rows older than auditMaxAgeMs after a run', async () => {
+    const eightDaysAgo = new Date(Date.now() - 8 * 24 * 60 * 60 * 1000).toISOString();
+    seedRun('aged', eightDaysAgo);
+    seedRun('aged', eightDaysAgo);
+    // pruneIntervalMs 0 → prune after every run (no lazy gap to wait out in a test).
+    const runner = createJobRunner(db, { logger: quietLogger, retention: { pruneIntervalMs: 0 } });
+    runner.register({ name: 'aged', async run() {} });
+    await runner.runOnce('aged');
+    await runner.stop();
+    // Both 8-day-old rows are gone; the fresh run survives (default window is 7 days).
+    expect(runCount(db, 'aged')).toBe(1);
+    expect(runner.getLastRun('aged')?.status).toBe('ok');
+  });
+
+  test('retention caps job_runs at auditMaxRowsPerJob newest rows, per job name', async () => {
+    const runner = createJobRunner(db, {
+      logger: quietLogger,
+      retention: { pruneIntervalMs: 0, auditMaxAgeMs: 0, auditMaxRowsPerJob: 3 },
+    });
+    runner.register({ name: 'chatty', async run() {} });
+    runner.register({ name: 'quiet', async run() {} });
+    await runner.runOnce('quiet');
+    for (let i = 0; i < 6; i += 1) await runner.runOnce('chatty');
+    await runner.stop();
+    // The cap applies per job: 'chatty' trimmed to its 3 newest, 'quiet' untouched.
+    expect(runCount(db, 'chatty')).toBe(3);
+    expect(runCount(db, 'quiet')).toBe(1);
+    expect(runner.getLastRun('chatty')?.status).toBe('ok');
+  });
+
+  test('retention fields set to 0 disable pruning entirely', async () => {
+    const tenDaysAgo = new Date(Date.now() - 10 * 24 * 60 * 60 * 1000).toISOString();
+    seedRun('keeper', tenDaysAgo);
+    const runner = createJobRunner(db, {
+      logger: quietLogger,
+      retention: { pruneIntervalMs: 0, auditMaxAgeMs: 0, auditMaxRowsPerJob: 0 },
+    });
+    runner.register({ name: 'keeper', async run() {} });
+    for (let i = 0; i < 4; i += 1) await runner.runOnce('keeper');
+    await runner.stop();
+    // Seeded ancient row + all 4 fresh runs all still present.
+    expect(runCount(db, 'keeper')).toBe(5);
+  });
+});

package/src/server/jobs/runner.ts

@@ -0,0 +1,213 @@
+import type { Database } from 'bun:sqlite';
+import { defineQueue, defineWorker, type Queue, type Worker } from 'plainjob';
+import { ApiError } from '../errors';
+import { defaultJobLogger } from './logger';
+import { createPlainjobConnection } from './plainjobAdapter';
+import { withTimeout } from './timeout';
+import {
+  type CreateJobRunnerOptions,
+  DEFAULT_JOB_RETENTION,
+  DEFAULT_JOB_TIMEOUT_MS,
+  type JobContext,
+  type JobDef,
+  type JobLogger,
+  type JobRunner,
+  type JobRunRecord,
+} from './types';
+
+/**
+ * Create the durable cron job runner — a thin facade over **plainjob** (MIT,
+ * SQLite-backed, no Redis). It generalizes ca's scattered `setInterval` cron-ish
+ * pollers + backup-target-hour check into one restart-aware runner whose schedules
+ * live in the *same* SQLite file as the business data (no separate store).
+ *
+ * Audit trail: every run (cron-fired or {@link JobRunner.runOnce}) writes a
+ * `job_runs` row — `running` on start, then `ok`/`error` on finish — so the admin
+ * "last run" view and {@link JobRunner.getLastRun} reflect reality. plainjob keeps
+ * its OWN internal tables (`plainjob_jobs`, `plainjob_scheduled_jobs`); the
+ * `job_runs`/`job_state` plumbing tables (task 22) are this wrapper's audit view
+ * and never collide with them.
+ *
+ * History is bounded by default ({@link DEFAULT_JOB_RETENTION}): `job_runs` is
+ * pruned lazily (age window + per-job row cap, at most once per prune interval),
+ * and plainjob's maintenance timer removes its internal done/failed rows on the
+ * configured horizons — so a sub-minute cron can run for months without the DB
+ * growing without bound.
+ *
+ * The `job_runs` table must already exist (the app's migration runner builds it).
+ */
+export function createJobRunner(db: Database, opts: CreateJobRunnerOptions = {}): JobRunner {
+  const logger: JobLogger = opts.logger ?? defaultJobLogger;
+  const pollIntervalMs = opts.pollIntervalMs ?? 1000;
+  const retention = { ...DEFAULT_JOB_RETENTION, ...opts.retention };
+
+  const defs = new Map<string, JobDef>();
+  const workers = new Map<string, Worker>();
+  let queue: Queue | null = null;
+  let started = false;
+  let stopped = false;
+  // Aborted on stop() so every in-flight handler's context signal fires.
+  const stopController = new AbortController();
+
+  const insertRun = db.prepare(
+    "INSERT INTO job_runs (id, job_name, started_at, status) VALUES (?, ?, ?, 'running')",
+  );
+  const finishRun = db.prepare(
+    'UPDATE job_runs SET ended_at = ?, status = ?, error = ? WHERE id = ?',
+  );
+  const lastRunQuery = db.query(
+    `SELECT started_at AS startedAt, ended_at AS endedAt, status, error
+       FROM job_runs WHERE job_name = ? ORDER BY started_at DESC, rowid DESC LIMIT 1`,
+  );
+
+  let lastPruneAt = 0;
+
+  /**
+   * Lazily bound the `job_runs` audit table: an age window (ISO-8601 strings
+   * compare lexicographically, so plain `<` is correct) plus a newest-N cap per
+   * job name. Runs at most once per `pruneIntervalMs`, after a run finishes, and
+   * never lets a prune failure disturb the run that triggered it.
+   */
+  function maybePruneAudit(): void {
+    const now = Date.now();
+    if (now - lastPruneAt < retention.pruneIntervalMs) return;
+    lastPruneAt = now;
+    try {
+      if (retention.auditMaxAgeMs > 0) {
+        db.query('DELETE FROM job_runs WHERE started_at < ?').run(
+          new Date(now - retention.auditMaxAgeMs).toISOString(),
+        );
+      }
+      if (retention.auditMaxRowsPerJob > 0) {
+        db.query(
+          `DELETE FROM job_runs WHERE id IN (
+             SELECT id FROM (
+               SELECT id, ROW_NUMBER() OVER (
+                 PARTITION BY job_name ORDER BY started_at DESC, rowid DESC
+               ) AS rn FROM job_runs
+             ) WHERE rn > ?
+           )`,
+        ).run(retention.auditMaxRowsPerJob);
+      }
+    } catch (err) {
+      logger.warn('[jobs] audit prune failed (non-fatal)', err);
+    }
+  }
+
+  // plainjob's queue (and its maintenance timer) is created lazily — a runner with
+  // only manual (`runOnce`) jobs never touches plainjob at all. A retention field
+  // of 0 means "never remove"; plainjob has no explicit off switch, so it maps to
+  // an effectively-infinite horizon.
+  function ensureQueue(): Queue {
+    if (!queue) {
+      queue = defineQueue({
+        connection: createPlainjobConnection(db),
+        logger,
+        removeDoneJobsOlderThan:
+          retention.doneJobsMaxAgeMs > 0 ? retention.doneJobsMaxAgeMs : Number.MAX_SAFE_INTEGER,
+        removeFailedJobsOlderThan:
+          retention.failedJobsMaxAgeMs > 0 ? retention.failedJobsMaxAgeMs : Number.MAX_SAFE_INTEGER,
+      });
+    }
+    return queue;
+  }
+
+  /**
+   * Run one job: write the audit row, build a per-run abort signal (fires on stop
+   * OR timeout), run with the timeout, then record `ok`/`error`. Re-throws only
+   * when `rethrow` is set — the cron worker path swallows (the audit row is the
+   * record of truth, and plainjob would otherwise double-log + retain the failure),
+   * while `runOnce` surfaces the error to its admin caller.
+   */
+  async function executeJob(def: JobDef, rethrow: boolean): Promise<void> {
+    const id = crypto.randomUUID();
+    insertRun.run(id, def.name, new Date().toISOString());
+
+    const runController = new AbortController();
+    const propagateStop = () => runController.abort(stopController.signal.reason);
+    if (stopController.signal.aborted) propagateStop();
+    else stopController.signal.addEventListener('abort', propagateStop, { once: true });
+
+    const context: JobContext = { db, signal: runController.signal, logger };
+    try {
+      await withTimeout(
+        Promise.resolve(def.run(context)),
+        def.timeout ?? DEFAULT_JOB_TIMEOUT_MS,
+        () => runController.abort(new Error(`job "${def.name}" timed out`)),
+      );
+      finishRun.run(new Date().toISOString(), 'ok', null, id);
+    } catch (err) {
+      finishRun.run(new Date().toISOString(), 'error', String(err), id);
+      logger.error(`[jobs] ${def.name} failed`, err);
+      if (rethrow) throw err;
+    } finally {
+      stopController.signal.removeEventListener('abort', propagateStop);
+      maybePruneAudit();
+    }
+  }
+
+  function startWorker(def: JobDef): void {
+    const worker = defineWorker(def.name, () => executeJob(def, false), {
+      queue: ensureQueue(),
+      pollIntervall: pollIntervalMs,
+      logger,
+    });
+    workers.set(def.name, worker);
+    // start() resolves only when the worker stops; run it detached.
+    void worker.start().catch((err) => logger.error(`[jobs] worker "${def.name}" crashed`, err));
+  }
+
+  return {
+    register(job) {
+      if (stopped) throw new Error('cannot register on a stopped job runner');
+      if (defs.has(job.name)) throw new Error(`job already registered: ${job.name}`);
+      defs.set(job.name, job);
+      if (job.cron) {
+        // schedule() upserts by type, so re-registering across restarts just
+        // refreshes the cron expression (no duplicate scheduled rows).
+        ensureQueue().schedule(job.name, { cron: job.cron });
+        if (started) startWorker(job);
+      }
+    },
+
+    start() {
+      if (stopped) throw new Error('cannot restart a stopped job runner');
+      if (started) return;
+      started = true;
+      for (const def of defs.values()) if (def.cron) startWorker(def);
+    },
+
+    async stop() {
+      if (stopped) return;
+      stopped = true;
+      started = false;
+      stopController.abort(new Error('job runner stopped'));
+      await Promise.all([...workers.values()].map((worker) => worker.stop()));
+      workers.clear();
+      // Closes the queue's maintenance timer; the adapter's no-op close keeps the
+      // shared DB open for the rest of shutdown.
+      queue?.close();
+      queue = null;
+    },
+
+    async runOnce(name) {
+      const def = defs.get(name);
+      if (!def) throw new ApiError(404, `Unknown job: ${name}`);
+      await executeJob(def, true);
+    },
+
+    getLastRun(name) {
+      const row = lastRunQuery.get(name) as {
+        startedAt: string;
+        endedAt: string | null;
+        status: string;
+        error: string | null;
+      } | null;
+      if (!row) return null;
+      const record: JobRunRecord = { startedAt: row.startedAt, status: row.status };
+      if (row.endedAt !== null) record.endedAt = row.endedAt;
+      if (row.error !== null) record.error = row.error;
+      return record;
+    },
+  };
+}

package/src/server/jobs/timeout.spec.ts

@@ -0,0 +1,35 @@
+import { describe, expect, test } from 'bun:test';
+import { JobTimeoutError, withTimeout } from './timeout';
+
+const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+
+describe('withTimeout', () => {
+  test('resolves with the promise value when it settles in time', async () => {
+    await expect(withTimeout(Promise.resolve('ok'), 1000)).resolves.toBe('ok');
+  });
+
+  test('rejects with JobTimeoutError when the promise is too slow', async () => {
+    const slow = sleep(500).then(() => 'late');
+    const p = withTimeout(slow, 20);
+    await expect(p).rejects.toBeInstanceOf(JobTimeoutError);
+    await expect(p).rejects.toThrow('timed out after 20ms');
+  });
+
+  test('invokes onTimeout (used to abort the handler) on timeout', async () => {
+    let aborted = false;
+    await withTimeout(sleep(500), 20, () => {
+      aborted = true;
+    }).catch(() => {});
+    expect(aborted).toBe(true);
+  });
+
+  test('a non-positive timeout means "no timeout" — the promise passes through', async () => {
+    await expect(withTimeout(Promise.resolve(42), 0)).resolves.toBe(42);
+    await expect(withTimeout(Promise.resolve(42), Number.POSITIVE_INFINITY)).resolves.toBe(42);
+  });
+
+  test('propagates a rejection from the underlying promise unchanged', async () => {
+    const boom = Promise.reject(new Error('boom'));
+    await expect(withTimeout(boom, 1000)).rejects.toThrow('boom');
+  });
+});

package/src/server/jobs/timeout.ts

@@ -0,0 +1,53 @@
+/**
+ * A promise-timeout for in-process job handlers.
+ *
+ * NB: cwip's `runWithTimeout` is a *subprocess* runner (it spawns + kills a child
+ * process), so it is the wrong tool for racing an in-process handler promise — we
+ * own this tiny helper instead. The timer is `unref()`'d so it never keeps the
+ * event loop alive, and it is always cleared once the race settles.
+ */
+
+/** Thrown when a job handler exceeds its `timeout`. */
+export class JobTimeoutError extends Error {
+  readonly timeoutMs: number;
+  constructor(timeoutMs: number) {
+    super(`Job timed out after ${timeoutMs}ms`);
+    this.name = 'JobTimeoutError';
+    this.timeoutMs = timeoutMs;
+  }
+}
+
+/**
+ * Resolve/reject with `promise`, but reject with a {@link JobTimeoutError} if it
+ * has not settled within `timeoutMs`. `onTimeout` fires on timeout — the runner
+ * uses it to abort the handler's context signal so a cooperative handler can
+ * short-circuit (a pending promise cannot be force-cancelled).
+ *
+ * Order matters: the timeout promise is rejected BEFORE `onTimeout` runs. Aborting
+ * the signal can synchronously resolve a cooperative handler; if that resolution
+ * reached `Promise.race` first the run would be mis-recorded as `ok`. Rejecting
+ * first guarantees a timed-out run is recorded as a timeout, not a success.
+ *
+ * A non-positive / non-finite `timeoutMs` means "no timeout": `promise` is
+ * returned unchanged.
+ */
+export function withTimeout<T>(
+  promise: Promise<T>,
+  timeoutMs: number,
+  onTimeout?: () => void,
+): Promise<T> {
+  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return promise;
+
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const timeout = new Promise<never>((_resolve, reject) => {
+    timer = setTimeout(() => {
+      reject(new JobTimeoutError(timeoutMs));
+      onTimeout?.();
+    }, timeoutMs);
+    timer.unref?.();
+  });
+
+  return Promise.race([promise, timeout]).finally(() => {
+    if (timer) clearTimeout(timer);
+  }) as Promise<T>;
+}