@robodev-ai/runtime 0.1.0 → 0.3.0

package/src/local-jobs.ts

@@ -0,0 +1,353 @@
+import type { JobDefinition, JobEnqueueInput, RobodevDb } from "@robodev-ai/sdk";
+import { assertFiveFieldCron, nextRunAfter } from "./cron.js";
+import { HandlerTimeoutError, clampTimeoutMs, withHandlerTimeout } from "./handler-timeout.js";
+import { id } from "./ids.js";
+import type { RouteClients } from "./invoke.js";
+import {
+  JOB_CONCURRENCY,
+  JOB_MAX_ATTEMPTS,
+  assertKnownJob,
+  nextJobFailureState,
+  serializeJobPayload,
+  type JobCronDefinition,
+} from "./job-queue.js";
+
+/** Hosted uses metadata `now()`; local worker/ticker intervals match hosted. */
+export const LOCAL_JOB_WORKER_MS = 1000;
+export const LOCAL_JOB_CRON_MS = 60_000;
+
+export type JobsQueryable = {
+  query: <T extends Record<string, unknown> = Record<string, unknown>>(
+    sql: string,
+    values?: unknown[],
+  ) => Promise<{ rows: T[] }>;
+};
+
+export type LocalJobStatus = "queued" | "running" | "succeeded" | "dead";
+
+export type LocalJobRow = {
+  id: string;
+  name: string;
+  payload_json: string;
+  status: LocalJobStatus;
+  attempts: number;
+  max_attempts: number;
+  run_at: Date;
+  last_error: string | null;
+  created_at: Date;
+  updated_at: Date;
+};
+
+export type LocalCronRow = {
+  job_name: string;
+  cron: string;
+  next_run_at: Date;
+};
+
+export type LocalJobsGeneration = {
+  db: RobodevDb;
+  jobs: readonly { name: string; file?: string; def: JobDefinition }[];
+};
+
+export type LocalJobsEngineOptions = {
+  query: JobsQueryable["query"];
+  getGeneration: () => LocalJobsGeneration | null;
+  log?: (line: string) => void;
+  clients: () => RouteClients;
+};
+
+export type LocalJobsEngine = {
+  enqueue: (input: JobEnqueueInput) => Promise<{ id: string }>;
+  reconcile: (jobs: readonly JobCronDefinition[], now?: Date) => Promise<void>;
+  tickDue: (now?: Date) => Promise<void>;
+  tickJobs: () => Promise<void>;
+  start: () => Promise<void>;
+  stop: () => Promise<void>;
+  readonly generation: LocalJobsGeneration | null;
+  readonly running: boolean;
+};
+
+function resolveRunAt(input: JobEnqueueInput): Date {
+  if (input.runAt) return input.runAt;
+  if (typeof input.delayMs === "number" && Number.isFinite(input.delayMs)) {
+    return new Date(Date.now() + Math.max(0, input.delayMs));
+  }
+  return new Date();
+}
+
+function desiredCronJobs(jobs: readonly JobCronDefinition[]): { name: string; cron: string }[] {
+  return jobs
+    .filter((job) => job.def.cron !== undefined)
+    .map((job) => ({ name: job.name, cron: assertFiveFieldCron(job.def.cron!) }));
+}
+
+function shortError(message: string): string {
+  const oneLine = message.split("\n")[0]?.trim() || "handler_failed";
+  return oneLine.length > 160 ? `${oneLine.slice(0, 157)}...` : oneLine;
+}
+
+/** Creates the reserved `robodev_jobs` schema. Idempotent, not drizzle, not public. */
+export async function ensureJobsSchema(db: JobsQueryable): Promise<void> {
+  await db.query(`CREATE SCHEMA IF NOT EXISTS robodev_jobs`);
+  await db.query(`
+    CREATE TABLE IF NOT EXISTS robodev_jobs.jobs (
+      id TEXT PRIMARY KEY,
+      name TEXT NOT NULL,
+      payload_json TEXT NOT NULL DEFAULT 'null',
+      status TEXT NOT NULL CHECK (status IN ('queued', 'running', 'succeeded', 'dead')),
+      attempts INTEGER NOT NULL DEFAULT 0,
+      max_attempts INTEGER NOT NULL DEFAULT 5,
+      run_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+      last_error TEXT,
+      created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+      updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+    )
+  `);
+  await db.query(`
+    CREATE INDEX IF NOT EXISTS robodev_jobs_queued_run_at
+      ON robodev_jobs.jobs (status, run_at)
+      WHERE status = 'queued'
+  `);
+  await db.query(`
+    CREATE TABLE IF NOT EXISTS robodev_jobs.crons (
+      job_name TEXT PRIMARY KEY,
+      cron TEXT NOT NULL,
+      next_run_at TIMESTAMPTZ NOT NULL,
+      updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+    )
+  `);
+  await db.query(`
+    CREATE INDEX IF NOT EXISTS robodev_jobs_crons_next_run_at
+      ON robodev_jobs.crons (next_run_at)
+  `);
+}
+
+export function createLocalJobsEngine(options: LocalJobsEngineOptions): LocalJobsEngine {
+  const query = options.query;
+  const log = options.log ?? (() => undefined);
+  let inFlight = 0;
+  const running = new Set<Promise<void>>();
+  let workerTimer: ReturnType<typeof setInterval> | undefined;
+  let cronTimer: ReturnType<typeof setInterval> | undefined;
+
+  async function enqueue(input: JobEnqueueInput): Promise<{ id: string }> {
+    const generation = options.getGeneration();
+    const known = generation?.jobs.map((job) => job.name) ?? [];
+    assertKnownJob(input.name, known);
+    const payloadJson = serializeJobPayload(input.payload);
+    const jobId = id("job");
+    await query(
+      `INSERT INTO robodev_jobs.jobs
+         (id, name, payload_json, status, attempts, max_attempts, run_at)
+       VALUES ($1, $2, $3, 'queued', 0, $4, $5)`,
+      [jobId, input.name, payloadJson, JOB_MAX_ATTEMPTS, resolveRunAt(input)],
+    );
+    return { id: jobId };
+  }
+
+  async function reconcile(jobs: readonly JobCronDefinition[], now = new Date()): Promise<void> {
+    const desired = desiredCronJobs(jobs);
+    const existing = await query<LocalCronRow>(
+      `SELECT job_name, cron, next_run_at
+       FROM robodev_jobs.crons`,
+    );
+    const byName = new Map(existing.rows.map((row) => [row.job_name, row]));
+
+    for (const job of desired) {
+      const row = byName.get(job.name);
+      if (!row) {
+        await query(
+          `INSERT INTO robodev_jobs.crons (job_name, cron, next_run_at)
+           VALUES ($1, $2, $3)`,
+          [job.name, job.cron, nextRunAfter(job.cron, now)],
+        );
+        continue;
+      }
+      if (row.cron !== job.cron) {
+        await query(
+          `UPDATE robodev_jobs.crons
+           SET cron = $2, next_run_at = $3, updated_at = now()
+           WHERE job_name = $1`,
+          [job.name, job.cron, nextRunAfter(job.cron, now)],
+        );
+      }
+    }
+
+    await query(`DELETE FROM robodev_jobs.crons WHERE NOT (job_name = ANY($1::text[]))`, [
+      desired.map((job) => job.name),
+    ]);
+  }
+
+  async function tickDue(now = new Date()): Promise<void> {
+    const due = await query<LocalCronRow>(
+      `SELECT job_name, cron, next_run_at
+       FROM robodev_jobs.crons
+       WHERE next_run_at <= $1
+       ORDER BY next_run_at ASC
+       LIMIT 50`,
+      [now],
+    );
+    for (const row of due.rows) {
+      const nextRunAt = nextRunAfter(row.cron, now);
+      await query(
+        `UPDATE robodev_jobs.crons
+         SET next_run_at = $2, updated_at = now()
+         WHERE job_name = $1`,
+        [row.job_name, nextRunAt],
+      );
+      try {
+        await enqueue({ name: row.job_name, payload: null });
+      } catch {
+        // Tick is skipped after next_run_at advanced; do not roll back.
+      }
+    }
+  }
+
+  async function claimNextJob(): Promise<LocalJobRow | null> {
+    const result = await query<LocalJobRow>(
+      `UPDATE robodev_jobs.jobs
+       SET status = 'running', updated_at = now()
+       WHERE id = (
+         SELECT id FROM robodev_jobs.jobs
+         WHERE status = 'queued' AND run_at <= now()
+         ORDER BY run_at ASC
+         FOR UPDATE SKIP LOCKED
+         LIMIT 1
+       )
+       RETURNING id, name, payload_json, status, attempts, max_attempts, run_at,
+                 last_error, created_at, updated_at`,
+    );
+    return result.rows[0] ?? null;
+  }
+
+  async function finishJob(row: LocalJobRow, error: string | null): Promise<void> {
+    if (!error) {
+      await query(
+        `UPDATE robodev_jobs.jobs
+         SET status = 'succeeded', last_error = NULL, updated_at = now()
+         WHERE id = $1`,
+        [row.id],
+      );
+      log(`  ${row.name} succeeded`);
+      return;
+    }
+    const attempts = row.attempts + 1;
+    const next = nextJobFailureState(attempts);
+    await query(
+      `UPDATE robodev_jobs.jobs
+       SET status = $2, attempts = $3, last_error = $4, run_at = COALESCE($5, run_at), updated_at = now()
+       WHERE id = $1`,
+      [row.id, next.status, attempts, error, next.runAt],
+    );
+    if (next.status === "dead") {
+      log(`  ${row.name} dead: ${shortError(error)}`);
+    } else {
+      log(`  ${row.name} failed attempt ${attempts}/${JOB_MAX_ATTEMPTS}: ${shortError(error)}`);
+    }
+  }
+
+  async function runClaimedJob(
+    row: LocalJobRow,
+    generation: LocalJobsGeneration | null,
+  ): Promise<void> {
+    const job = generation?.jobs.find((entry) => entry.name === row.name);
+    if (!generation || !job) {
+      await finishJob(row, "route_not_found");
+      return;
+    }
+    let payload: unknown = null;
+    try {
+      payload = JSON.parse(row.payload_json) as unknown;
+    } catch {
+      payload = null;
+    }
+    const clients = options.clients();
+    try {
+      await withHandlerTimeout(async () => {
+        await (job.def as JobDefinition).handler({
+          db: generation.db,
+          email: clients.email,
+          llm: clients.llm,
+          agent: clients.agent,
+          env: clients.env,
+          push: clients.push,
+          jobs: clients.jobs,
+          sockets: clients.sockets,
+          payload,
+        });
+      }, clampTimeoutMs(job.def.timeoutMs));
+      await finishJob(row, null);
+    } catch (error) {
+      const message =
+        error instanceof HandlerTimeoutError
+          ? "handler_timeout"
+          : error instanceof Error
+            ? error.message
+            : "handler_failed";
+      await finishJob(row, message);
+    }
+  }
+
+  async function tickJobs(): Promise<void> {
+    while (inFlight < JOB_CONCURRENCY) {
+      const generation = options.getGeneration();
+      const row = await claimNextJob().catch(() => null);
+      if (!row) return;
+      inFlight += 1;
+      const run = runClaimedJob(row, generation);
+      running.add(run);
+      void run.finally(() => {
+        inFlight -= 1;
+        running.delete(run);
+      });
+    }
+  }
+
+  async function requeueRunning(): Promise<void> {
+    await query(
+      `UPDATE robodev_jobs.jobs
+       SET status = 'queued', updated_at = now()
+       WHERE status = 'running'`,
+    );
+  }
+
+  async function start(): Promise<void> {
+    if (workerTimer || cronTimer) return;
+    await requeueRunning();
+    workerTimer = setInterval(() => {
+      void tickJobs();
+    }, LOCAL_JOB_WORKER_MS);
+    workerTimer.unref?.();
+    cronTimer = setInterval(() => {
+      void tickDue();
+    }, LOCAL_JOB_CRON_MS);
+    cronTimer.unref?.();
+  }
+
+  async function stop(): Promise<void> {
+    if (workerTimer) {
+      clearInterval(workerTimer);
+      workerTimer = undefined;
+    }
+    if (cronTimer) {
+      clearInterval(cronTimer);
+      cronTimer = undefined;
+    }
+    await Promise.all([...running]);
+  }
+
+  return {
+    enqueue,
+    reconcile,
+    tickDue,
+    tickJobs,
+    start,
+    stop,
+    get generation() {
+      return options.getGeneration();
+    },
+    get running() {
+      return Boolean(workerTimer || cronTimer);
+    },
+  };
+}