@aion0/forge 0.6.1 → 0.8.0

package/lib/jobs/scheduler.ts

@@ -0,0 +1,334 @@
+/**
+ * Jobs scheduler — ticks every 60s, runs each due Job, dispatches new items.
+ *
+ * Started by lib/init.ts on first request. Runs inside the Next.js worker
+ * alongside pipeline-scheduler. Each tick is async — the loop never waits
+ * for a job's connector call to finish (just kicks it off and advances
+ * next_run_at). The job's own run row is updated when the work completes.
+ */
+
+import {
+  ensureSchema, getDueJobs, hasInflightRun, startRun, finishRun,
+  markSeen, recordDispatch, getJob, updateJob,
+} from './store';
+import type { Job, JobRunStatus, PipelineDispatchParams, ChatDispatchParams } from './types';
+import { dispatchTool } from '@/lib/chat/tool-dispatcher';
+import { dispatchToPipeline, dispatchToChat, dispatchToChatSummary } from './dispatcher';
+
+const TICK_INTERVAL_MS = 60_000;
+
+const schedulerKey = Symbol.for('forge-jobs-scheduler');
+const g = globalThis as any;
+
+export function startScheduler(): void {
+  if (g[schedulerKey]) return;
+  g[schedulerKey] = true;
+  ensureSchema();
+  console.log('[jobs-scheduler] started');
+  void tick();
+  setInterval(() => { void tick(); }, TICK_INTERVAL_MS).unref?.();
+}
+
+async function tick(): Promise<void> {
+  let due: Job[] = [];
+  try { due = getDueJobs(); } catch (e) { console.error('[jobs-scheduler] getDueJobs failed', e); return; }
+
+  for (const job of due) {
+    if (hasInflightRun(job.id)) {
+      console.log(`[jobs] skip ${job.id} — previous run still inflight`);
+      // Still advance the schedule so we re-evaluate at the next interval.
+      advanceSchedule(job);
+      continue;
+    }
+    advanceSchedule(job);
+    // Kick off the run; don't await — long connector calls / pipeline triggers
+    // shouldn't block the scheduler loop.
+    const { runId } = prepareRun(job, 'schedule');
+    void executeRun(job, runId).catch((e) => {
+      console.error(`[jobs] runJob ${job.id} crashed`, e);
+    });
+  }
+}
+
+function advanceSchedule(job: Job): void {
+  const next = new Date(Date.now() + Math.max(1, job.schedule_interval_minutes) * 60_000);
+  // setNextRunAt also bumps last_run_at — call updateJob via raw SQL for clarity.
+  const { setNextRunAt } = require('./store') as typeof import('./store');
+  setNextRunAt(job.id, next.toISOString().replace('T', ' ').slice(0, 19));
+}
+
+/**
+ * Execute one Job tick fully. Public for use by the scheduler loop and
+ * the manual-fire API. The manual route prefers prepareRun + executeRun
+ * (split below) so it can return the run id before the work finishes.
+ */
+export async function runJob(jobOrId: Job | string, trigger: 'schedule' | 'manual'): Promise<string> {
+  const { job, runId } = prepareRun(jobOrId, trigger);
+  await executeRun(job, runId);
+  return runId;
+}
+
+/**
+ * Insert a run row and return its id immediately. Use with executeRun(job, runId)
+ * when you want fire-and-forget semantics (e.g. manual API trigger).
+ */
+export function prepareRun(jobOrId: Job | string, trigger: 'schedule' | 'manual'): { job: Job; runId: string } {
+  const job = typeof jobOrId === 'string' ? getJob(jobOrId) : jobOrId;
+  if (!job) throw new Error(`job not found: ${typeof jobOrId === 'string' ? jobOrId : '(null)'}`);
+  const run = startRun(job.id, trigger);
+  return { job, runId: run.id };
+}
+
+/**
+ * Do the actual tick work (connector call + dedup + dispatch). Updates the run
+ * row, including a verbose plain-text execution log saved to `job_runs.log`
+ * for inspection in the UI ("show log" expansion under each run row).
+ *
+ * Every meaningful step appends one timestamped line to a per-tick buffer;
+ * we also mirror the high-level lines to console for live tailing via
+ * `tail -f forge.log | grep [jobs]`.
+ */
+export async function executeRun(job: Job, runId: string): Promise<void> {
+  const t0 = Date.now();
+  let itemsSeen = 0, itemsNew = 0, itemsDispatched = 0;
+  let runError: string | null = null;
+  const callName = `${job.source_connector}.${job.source_tool}`;
+  const lines: string[] = [];
+  const logLine = (level: 'info' | 'warn' | 'error', msg: string, mirror = true) => {
+    const ts = new Date().toISOString().slice(11, 23);
+    const ms = String(Date.now() - t0).padStart(5, ' ') + 'ms';
+    lines.push(`${ts}  ${ms}  ${level.toUpperCase().padEnd(5)} ${msg}`);
+    if (mirror) {
+      const tag = `[jobs] ${job.id} ${callName}`;
+      if (level === 'error') console.error(tag, msg);
+      else if (level === 'warn') console.warn(tag, msg);
+      else console.log(tag, msg);
+    }
+  };
+  const truncate = (s: string, n: number) => (s.length > n ? s.slice(0, n) + ` …(${s.length - n} bytes truncated)` : s);
+
+  // Always persist whatever log we accumulated, even on uncaught throws.
+  const persist = (extra: { status: JobRunStatus; error?: string | null; notes?: string | null }) => {
+    finishRun(runId, {
+      status: extra.status,
+      items_seen: itemsSeen,
+      items_new: itemsNew,
+      items_dispatched: itemsDispatched,
+      error: extra.error ?? null,
+      notes: extra.notes ?? null,
+      log: lines.join('\n'),
+    });
+  };
+
+  try {
+    const { __mark_existing_as_seen, ...sourceInput } = job.source_input as any;
+    logLine('info', `tick start trigger=${(__mark_existing_as_seen ? 'backfill' : 'normal')} dispatch=${job.dispatch_type}`);
+    logLine('info', `source input: ${JSON.stringify(sourceInput)}`);
+    logLine('info', `calling connector ${callName}…`);
+
+    const toolResult = await dispatchTool({ id: `job-${runId}`, name: callName, input: sourceInput });
+
+    const respBytes = toolResult.content?.length ?? 0;
+    logLine(toolResult.is_error ? 'error' : 'info',
+      `connector returned ${toolResult.is_error ? 'is_error=true ' : ''}${respBytes} bytes`);
+    logLine('info', `response preview:\n${truncate(toolResult.content || '(empty)', 600)}`, false);
+
+    if (toolResult.is_error) {
+      throw new Error(`connector ${callName} failed: ${toolResult.content.slice(0, 500)}`);
+    }
+
+    // ── Parse + extract items ────────────────────────────────────
+    const parsed = safeParseJson(toolResult.content);
+    if (parsed === undefined) {
+      const note = `Connector returned non-JSON content (${respBytes} bytes). Preview: ${toolResult.content.slice(0, 200)}`;
+      logLine('warn', note);
+      persist({ status: 'ok', notes: note });
+      return;
+    }
+    logLine('info', `parsed JSON; type=${Array.isArray(parsed) ? 'array' : typeof parsed}`);
+
+    let items = pickPath(parsed, job.items_path) as unknown;
+    // Detail-style connectors (mantis.get_bug, gitlab.get_issue, …) return a
+    // single object, not an array. Treat that as a 1-item list so the same
+    // Job machinery (dedup, dispatch, render) works for both list and detail
+    // tools. This is what lets you test a pipeline end-to-end with one
+    // known item without writing a separate code path.
+    if (items && typeof items === 'object' && !Array.isArray(items)) {
+      logLine('info', `items_path resolved to a single object — wrapping as 1-item list`);
+      items = [items];
+    }
+    if (!Array.isArray(items)) {
+      const topKeys = parsed && typeof parsed === 'object' && !Array.isArray(parsed)
+        ? Object.keys(parsed as Record<string, unknown>).join(', ')
+        : Array.isArray(parsed) ? '(top-level is an array — leave items_path empty)' : typeof parsed;
+      const note = `items_path "${job.items_path || '(empty)'}" did not resolve to an array or object. Top-level keys: ${topKeys}`;
+      logLine('warn', note);
+      persist({ status: 'ok', notes: note });
+      return;
+    }
+    const itemsArr = items as unknown[];
+    itemsSeen = itemsArr.length;
+    logLine('info', `items_path="${job.items_path || '(root)'}" → ${itemsSeen} item(s)`);
+
+    // ── Backfill guard ───────────────────────────────────────────
+    if (__mark_existing_as_seen) {
+      let backfilledKeys = 0, missingKey = 0;
+      for (const item of itemsArr) {
+        const key = pickDedupKey(item, job.dedup_field);
+        if (key) { markSeen(job.id, key); backfilledKeys++; }
+        else missingKey++;
+      }
+      updateJob(job.id, { source_input: sourceInput });
+      logLine('info', `backfill complete: marked ${backfilledKeys} items as seen` + (missingKey ? `, ${missingKey} skipped (missing dedup_field "${job.dedup_field}")` : ''));
+      const note = `Backfill (first tick): marked ${backfilledKeys} existing items as seen — no dispatches by design. Future ticks dispatch only NEW items. To re-process everything, click "Reset dedup".`;
+      persist({ status: 'ok', notes: note });
+      return;
+    }
+
+    // ── Chat summary mode — bundle all NEW items into ONE message ──
+    // Useful when search_bugs returns N items and you want the LLM to
+    // analyze + summarize, instead of getting N separate chat messages
+    // (or just the first item if reuse_session was off).
+    if (job.dispatch_type === 'chat' &&
+        (job.dispatch_params as ChatDispatchParams).mode === 'summary') {
+      const chatParams = job.dispatch_params as ChatDispatchParams;
+      let dedupHits = 0, missingKey = 0;
+      const newItems: unknown[] = [];
+      for (const [idx, item] of itemsArr.entries()) {
+        const key = pickDedupKey(item, job.dedup_field);
+        if (!key) { missingKey++; logLine('warn', `[${idx}] item missing dedup_field "${job.dedup_field}" — skipping`); continue; }
+        if (!markSeen(job.id, key)) { dedupHits++; continue; }
+        itemsNew++;
+        newItems.push(item);
+      }
+      if (newItems.length === 0) {
+        const note = itemsSeen === 0
+          ? 'Connector returned 0 items.'
+          : `All ${itemsSeen} items already seen — nothing new to summarize.`;
+        logLine('info', `summary mode: ${note}`);
+        persist({ status: 'ok', notes: note });
+        return;
+      }
+      logLine('info', `summary mode: bundling ${newItems.length} new item(s) into one chat message`);
+      // Pull the connector's server-side total (e.g. mantis search_bugs
+      // total_matching = "151" parsed from "Viewing Bugs (1 - 50 / 151)").
+      // The {{count}} placeholder in the user's template now means "items
+      // they're actually seeing"; {{total_matching}} is "how many matched
+      // the filter on the server". Falls back to parsed.total when the
+      // connector doesn't emit total_matching.
+      const totalMatching = (parsed && typeof parsed === 'object' && !Array.isArray(parsed))
+        ? (parsed as any).total_matching ?? (parsed as any).total ?? null
+        : null;
+      try {
+        const out = await dispatchToChatSummary(chatParams, newItems, job.id, { totalMatching });
+        recordDispatch({
+          job_run_id: runId,
+          item_key: `summary-${runId}`,
+          item_preview: `[${newItems.length} items folded into one message]`,
+          dispatch_type: 'chat', dispatch_target_id: out.target_id, status: 'dispatched',
+        });
+        itemsDispatched = 1; // one logical dispatch, regardless of item count
+        logLine('info', `summary → chat:${out.target_id} (${newItems.length} items)`);
+      } catch (e) {
+        const msg = e instanceof Error ? e.message : String(e);
+        recordDispatch({
+          job_run_id: runId, item_key: `summary-${runId}`, item_preview: '',
+          dispatch_type: 'chat', dispatch_target_id: null, status: 'error', error: msg,
+        });
+        logLine('error', `summary dispatch failed: ${msg}`);
+      }
+      logLine('info', `tick done in ${Date.now() - t0}ms — ${itemsSeen} seen, ${itemsNew} new, ${itemsDispatched} dispatched (summary), ${dedupHits} dedup hits` + (missingKey ? `, ${missingKey} missing-key` : ''));
+      persist({ status: 'ok', notes: null });
+      return;
+    }
+
+    // ── Per-item dispatch ────────────────────────────────────────
+    let dedupHits = 0, missingKey = 0;
+    for (const [idx, item] of itemsArr.entries()) {
+      const key = pickDedupKey(item, job.dedup_field);
+      if (!key) {
+        missingKey++;
+        logLine('warn', `[${idx}] item missing dedup_field "${job.dedup_field}" — skipping`);
+        continue;
+      }
+      const isNew = markSeen(job.id, key);
+      if (!isNew) {
+        dedupHits++;
+        // Don't mirror to console — too chatty in the typical "0 new" case
+        logLine('info', `[${idx}] ${key} — already seen, skip`, false);
+        continue;
+      }
+      itemsNew++;
+      const preview = renderItemPreview(item);
+      logLine('info', `[${idx}] ${key} — new — dispatching ${job.dispatch_type}…`);
+      const dispatchStart = Date.now();
+      try {
+        const out = job.dispatch_type === 'pipeline'
+          ? dispatchToPipeline(job.dispatch_params as PipelineDispatchParams, item, key)
+          : await dispatchToChat(job.dispatch_params as ChatDispatchParams, item, job.id);
+        recordDispatch({
+          job_run_id: runId, item_key: key, item_preview: preview,
+          dispatch_type: job.dispatch_type, dispatch_target_id: out.target_id, status: 'dispatched',
+        });
+        itemsDispatched++;
+        logLine('info', `[${idx}] ${key} → ${job.dispatch_type}:${out.target_id} (${Date.now() - dispatchStart}ms)`);
+        const emptyKeys = (out as any).empty_keys as string[] | undefined;
+        if (emptyKeys && emptyKeys.length > 0) {
+          logLine('warn', `[${idx}] ${key} — these input_template keys rendered EMPTY (likely a {{item.X}} where X isn't on the item): ${emptyKeys.join(', ')}. Pipeline node will probably fail with 'is required' — fix by typing a literal value in the job's input_template.`);
+        }
+      } catch (e) {
+        const msg = e instanceof Error ? e.message : String(e);
+        recordDispatch({
+          job_run_id: runId, item_key: key, item_preview: preview,
+          dispatch_type: job.dispatch_type, dispatch_target_id: null, status: 'error', error: msg,
+        });
+        logLine('error', `[${idx}] ${key} dispatch failed: ${msg}`);
+      }
+    }
+
+    let note: string | null = null;
+    if (itemsDispatched === 0) {
+      if (itemsSeen === 0) note = 'Connector returned 0 items.';
+      else if (dedupHits === itemsSeen) note = `All ${itemsSeen} items were already seen (dedup hits). No new items to dispatch. Click "Reset dedup" to re-process everything.`;
+      else if (missingKey === itemsSeen) note = `All ${itemsSeen} items lacked the dedup_field "${job.dedup_field}". Check the field name vs the connector's response shape.`;
+    }
+
+    logLine('info', `tick done in ${Date.now() - t0}ms — ${itemsSeen} seen, ${itemsNew} new, ${itemsDispatched} dispatched, ${dedupHits} dedup hits` + (missingKey ? `, ${missingKey} missing-key` : ''));
+    persist({ status: 'ok', notes: note });
+  } catch (e) {
+    runError = e instanceof Error ? e.message : String(e);
+    logLine('error', `tick failed: ${runError}`);
+    persist({ status: 'error', error: runError });
+  }
+}
+
+// ─── Helpers ──────────────────────────────────────────────
+
+function safeParseJson(content: string): unknown | undefined {
+  try { return JSON.parse(content); }
+  catch { return undefined; }
+}
+
+function pickPath(obj: unknown, path: string): unknown {
+  if (!path) return obj;
+  const parts = path.split('.');
+  let cur: any = obj;
+  for (const p of parts) {
+    if (cur == null || typeof cur !== 'object') return undefined;
+    cur = cur[p];
+  }
+  return cur;
+}
+
+function pickDedupKey(item: unknown, field: string): string | null {
+  const v = pickPath(item, field);
+  if (v == null) return null;
+  return typeof v === 'string' ? v : String(v);
+}
+
+function renderItemPreview(item: unknown): string {
+  try {
+    const s = JSON.stringify(item);
+    return s.length > 200 ? s.slice(0, 200) + '…' : s;
+  } catch { return ''; }
+}

package/lib/jobs/store.ts

@@ -0,0 +1,319 @@
+/**
+ * Sqlite-backed CRUD for Jobs. Schema lives here so the table is auto-
+ * created on first use (mirrors lib/issue-scanner.ts style).
+ */
+
+import { getDb } from '@/src/core/db/database';
+import { getDbPath } from '@/src/config';
+import { randomUUID } from 'node:crypto';
+import { toIsoUTC } from '@/lib/iso-time';
+import type {
+  Job, JobRun, JobDispatch, CreateJobInput,
+  JobRunStatus, JobRunTrigger, JobDispatchStatus,
+  DispatchParams,
+} from './types';
+
+function db() { return getDb(getDbPath()); }
+
+let ensured = false;
+export function ensureSchema(): void {
+  if (ensured) return;
+  db().exec(`
+    CREATE TABLE IF NOT EXISTS jobs (
+      id                          TEXT PRIMARY KEY,
+      name                        TEXT NOT NULL,
+      enabled                     INTEGER NOT NULL DEFAULT 1,
+      schedule_interval_minutes   INTEGER NOT NULL DEFAULT 30,
+      source_connector            TEXT NOT NULL,
+      source_tool                 TEXT NOT NULL,
+      source_input                TEXT NOT NULL DEFAULT '{}',
+      items_path                  TEXT,
+      dedup_field                 TEXT NOT NULL,
+      dispatch_type               TEXT NOT NULL,
+      dispatch_params             TEXT NOT NULL DEFAULT '{}',
+      last_run_at                 TEXT,
+      next_run_at                 TEXT,
+      created_at                  TEXT NOT NULL DEFAULT (datetime('now')),
+      updated_at                  TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+    CREATE TABLE IF NOT EXISTS job_runs (
+      id                  TEXT PRIMARY KEY,
+      job_id              TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE,
+      started_at          TEXT NOT NULL DEFAULT (datetime('now')),
+      finished_at         TEXT,
+      status              TEXT NOT NULL DEFAULT 'running',
+      items_seen          INTEGER NOT NULL DEFAULT 0,
+      items_new           INTEGER NOT NULL DEFAULT 0,
+      items_dispatched    INTEGER NOT NULL DEFAULT 0,
+      error               TEXT,
+      trigger             TEXT NOT NULL DEFAULT 'schedule',
+      notes               TEXT,
+      /** Plain-text execution log for this tick — one timestamped line per step. */
+      log                 TEXT
+    );
+    CREATE TABLE IF NOT EXISTS job_seen (
+      job_id              TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE,
+      dedup_key           TEXT NOT NULL,
+      first_seen_at       TEXT NOT NULL DEFAULT (datetime('now')),
+      PRIMARY KEY (job_id, dedup_key)
+    );
+    CREATE TABLE IF NOT EXISTS job_dispatches (
+      id                  TEXT PRIMARY KEY,
+      job_run_id          TEXT NOT NULL REFERENCES job_runs(id) ON DELETE CASCADE,
+      item_key            TEXT NOT NULL,
+      item_preview        TEXT,
+      dispatch_type       TEXT NOT NULL,
+      dispatch_target_id  TEXT,
+      status              TEXT NOT NULL DEFAULT 'dispatched',
+      error               TEXT,
+      created_at          TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+    CREATE INDEX IF NOT EXISTS idx_job_runs_job ON job_runs(job_id, started_at DESC);
+    CREATE INDEX IF NOT EXISTS idx_job_dispatches_run ON job_dispatches(job_run_id, created_at DESC);
+  `);
+  // Migrations for already-existing job_runs tables.
+  try { db().exec(`ALTER TABLE job_runs ADD COLUMN notes TEXT`); } catch {}
+  try { db().exec(`ALTER TABLE job_runs ADD COLUMN log TEXT`); } catch {}
+  ensured = true;
+}
+
+// ─── Row → Job/JobRun/JobDispatch mapping ─────────────────
+
+function rowToJob(r: any): Job {
+  return {
+    id: r.id,
+    name: r.name,
+    enabled: !!r.enabled,
+    schedule_interval_minutes: r.schedule_interval_minutes,
+    source_connector: r.source_connector,
+    source_tool: r.source_tool,
+    source_input: safeParse(r.source_input, {}) as Record<string, unknown>,
+    items_path: r.items_path || '',
+    dedup_field: r.dedup_field,
+    dispatch_type: r.dispatch_type,
+    dispatch_params: safeParse(r.dispatch_params, {}) as DispatchParams,
+    last_run_at: toIsoUTC(r.last_run_at),
+    next_run_at: toIsoUTC(r.next_run_at),
+    created_at: toIsoUTC(r.created_at) || r.created_at,
+    updated_at: toIsoUTC(r.updated_at) || r.updated_at,
+  };
+}
+
+function rowToRun(r: any): JobRun {
+  return {
+    id: r.id,
+    job_id: r.job_id,
+    started_at: toIsoUTC(r.started_at) || r.started_at,
+    finished_at: toIsoUTC(r.finished_at),
+    status: r.status as JobRunStatus,
+    items_seen: r.items_seen,
+    items_new: r.items_new,
+    items_dispatched: r.items_dispatched,
+    error: r.error,
+    trigger: r.trigger as JobRunTrigger,
+    notes: r.notes || null,
+    log: r.log || null,
+  };
+}
+
+function rowToDispatch(r: any): JobDispatch {
+  return {
+    id: r.id,
+    job_run_id: r.job_run_id,
+    item_key: r.item_key,
+    item_preview: r.item_preview,
+    dispatch_type: r.dispatch_type,
+    dispatch_target_id: r.dispatch_target_id,
+    status: r.status as JobDispatchStatus,
+    error: r.error,
+    created_at: toIsoUTC(r.created_at) || r.created_at,
+  };
+}
+
+function safeParse(raw: string | null, fallback: unknown): unknown {
+  if (!raw) return fallback;
+  try { return JSON.parse(raw); } catch { return fallback; }
+}
+
+// ─── Job CRUD ─────────────────────────────────────────────
+
+export function listJobs(): Job[] {
+  ensureSchema();
+  const rows = db().prepare('SELECT * FROM jobs ORDER BY created_at DESC').all() as any[];
+  return rows.map(rowToJob);
+}
+
+export function getJob(id: string): Job | null {
+  ensureSchema();
+  const r = db().prepare('SELECT * FROM jobs WHERE id = ?').get(id) as any;
+  return r ? rowToJob(r) : null;
+}
+
+export function createJob(input: CreateJobInput): Job {
+  ensureSchema();
+  const id = randomUUID().slice(0, 12);
+  db().prepare(`
+    INSERT INTO jobs (id, name, enabled, schedule_interval_minutes,
+      source_connector, source_tool, source_input,
+      items_path, dedup_field,
+      dispatch_type, dispatch_params)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(
+    id,
+    input.name,
+    input.enabled === false ? 0 : 1,
+    input.schedule_interval_minutes ?? 30,
+    input.source_connector,
+    input.source_tool,
+    JSON.stringify(input.source_input ?? {}),
+    input.items_path ?? '',
+    input.dedup_field,
+    input.dispatch_type,
+    JSON.stringify(input.dispatch_params),
+  );
+
+  // Backfill guard: if mark_existing_as_seen is true (default), we don't pre-seed
+  // here — the first tick will do it (calling the connector once, marking
+  // everything, dispatching nothing). This keeps create() pure DB work.
+  // The flag is stashed in source_input under __mark_existing_as_seen so the
+  // first tick can read & clear it.
+  if (input.mark_existing_as_seen !== false) {
+    const inputWithFlag = { ...(input.source_input ?? {}), __mark_existing_as_seen: true };
+    db().prepare('UPDATE jobs SET source_input = ? WHERE id = ?').run(JSON.stringify(inputWithFlag), id);
+  }
+
+  return getJob(id)!;
+}
+
+export function updateJob(id: string, patch: Partial<{
+  name: string; enabled: boolean; schedule_interval_minutes: number;
+  source_connector: string; source_tool: string; source_input: Record<string, unknown>;
+  items_path: string; dedup_field: string;
+  dispatch_type: 'pipeline' | 'chat'; dispatch_params: DispatchParams;
+}>): boolean {
+  ensureSchema();
+  const sets: string[] = []; const vals: any[] = [];
+  if (patch.name !== undefined) { sets.push('name = ?'); vals.push(patch.name); }
+  if (patch.enabled !== undefined) { sets.push('enabled = ?'); vals.push(patch.enabled ? 1 : 0); }
+  if (patch.schedule_interval_minutes !== undefined) { sets.push('schedule_interval_minutes = ?'); vals.push(patch.schedule_interval_minutes); }
+  if (patch.source_connector !== undefined) { sets.push('source_connector = ?'); vals.push(patch.source_connector); }
+  if (patch.source_tool !== undefined) { sets.push('source_tool = ?'); vals.push(patch.source_tool); }
+  if (patch.source_input !== undefined) { sets.push('source_input = ?'); vals.push(JSON.stringify(patch.source_input)); }
+  if (patch.items_path !== undefined) { sets.push('items_path = ?'); vals.push(patch.items_path); }
+  if (patch.dedup_field !== undefined) { sets.push('dedup_field = ?'); vals.push(patch.dedup_field); }
+  if (patch.dispatch_type !== undefined) { sets.push('dispatch_type = ?'); vals.push(patch.dispatch_type); }
+  if (patch.dispatch_params !== undefined) { sets.push('dispatch_params = ?'); vals.push(JSON.stringify(patch.dispatch_params)); }
+  if (sets.length === 0) return false;
+  sets.push("updated_at = datetime('now')");
+  vals.push(id);
+  const r = db().prepare(`UPDATE jobs SET ${sets.join(', ')} WHERE id = ?`).run(...vals);
+  return r.changes > 0;
+}
+
+export function deleteJob(id: string): boolean {
+  ensureSchema();
+  const r = db().prepare('DELETE FROM jobs WHERE id = ?').run(id);
+  return r.changes > 0;
+}
+
+export function setNextRunAt(id: string, nextRunAt: string | null): void {
+  ensureSchema();
+  db().prepare("UPDATE jobs SET next_run_at = ?, last_run_at = datetime('now') WHERE id = ?").run(nextRunAt, id);
+}
+
+/** Jobs due to run: enabled AND (next_run_at IS NULL OR next_run_at <= now). */
+export function getDueJobs(): Job[] {
+  ensureSchema();
+  const rows = db().prepare(`
+    SELECT * FROM jobs
+    WHERE enabled = 1
+      AND (next_run_at IS NULL OR next_run_at <= datetime('now'))
+    ORDER BY (next_run_at IS NULL) DESC, next_run_at ASC
+  `).all() as any[];
+  return rows.map(rowToJob);
+}
+
+// ─── Run CRUD ─────────────────────────────────────────────
+
+export function startRun(jobId: string, trigger: JobRunTrigger): JobRun {
+  ensureSchema();
+  const id = randomUUID().slice(0, 12);
+  db().prepare(`INSERT INTO job_runs (id, job_id, trigger) VALUES (?, ?, ?)`).run(id, jobId, trigger);
+  return db().prepare('SELECT * FROM job_runs WHERE id = ?').get(id) as any as JobRun;
+}
+
+export function finishRun(runId: string, patch: { status: JobRunStatus; items_seen?: number; items_new?: number; items_dispatched?: number; error?: string | null; notes?: string | null; log?: string | null }): void {
+  ensureSchema();
+  db().prepare(`
+    UPDATE job_runs
+    SET finished_at = datetime('now'),
+        status = ?,
+        items_seen = COALESCE(?, items_seen),
+        items_new = COALESCE(?, items_new),
+        items_dispatched = COALESCE(?, items_dispatched),
+        error = ?,
+        notes = COALESCE(?, notes),
+        log = COALESCE(?, log)
+    WHERE id = ?
+  `).run(patch.status, patch.items_seen ?? null, patch.items_new ?? null, patch.items_dispatched ?? null, patch.error ?? null, patch.notes ?? null, patch.log ?? null, runId);
+}
+
+export function listRuns(jobId: string, limit = 20): JobRun[] {
+  ensureSchema();
+  const rows = db().prepare('SELECT * FROM job_runs WHERE job_id = ? ORDER BY started_at DESC LIMIT ?').all(jobId, limit) as any[];
+  return rows.map(rowToRun);
+}
+
+export function getRun(runId: string): JobRun | null {
+  ensureSchema();
+  const r = db().prepare('SELECT * FROM job_runs WHERE id = ?').get(runId) as any;
+  return r ? rowToRun(r) : null;
+}
+
+/** True if a previous tick is still running for this job. */
+export function hasInflightRun(jobId: string): boolean {
+  ensureSchema();
+  const r = db().prepare(`SELECT 1 FROM job_runs WHERE job_id = ? AND status = 'running' AND finished_at IS NULL LIMIT 1`).get(jobId);
+  return !!r;
+}
+
+// ─── Dedup ────────────────────────────────────────────────
+
+/** Returns true if the (job, key) pair was newly inserted. */
+export function markSeen(jobId: string, key: string): boolean {
+  ensureSchema();
+  const r = db().prepare(`INSERT OR IGNORE INTO job_seen (job_id, dedup_key) VALUES (?, ?)`).run(jobId, key);
+  return r.changes > 0;
+}
+
+export function resetDedup(jobId: string): number {
+  ensureSchema();
+  const r = db().prepare('DELETE FROM job_seen WHERE job_id = ?').run(jobId);
+  return r.changes;
+}
+
+// ─── Dispatches ───────────────────────────────────────────
+
+export function recordDispatch(input: {
+  job_run_id: string;
+  item_key: string;
+  item_preview?: string;
+  dispatch_type: 'pipeline' | 'chat';
+  dispatch_target_id?: string | null;
+  status: JobDispatchStatus;
+  error?: string | null;
+}): JobDispatch {
+  ensureSchema();
+  const id = randomUUID().slice(0, 12);
+  db().prepare(`
+    INSERT INTO job_dispatches (id, job_run_id, item_key, item_preview, dispatch_type, dispatch_target_id, status, error)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(id, input.job_run_id, input.item_key, input.item_preview ?? null, input.dispatch_type, input.dispatch_target_id ?? null, input.status, input.error ?? null);
+  return db().prepare('SELECT * FROM job_dispatches WHERE id = ?').get(id) as any as JobDispatch;
+}
+
+export function listDispatches(runId: string): JobDispatch[] {
+  ensureSchema();
+  const rows = db().prepare('SELECT * FROM job_dispatches WHERE job_run_id = ? ORDER BY created_at ASC').all(runId) as any[];
+  return rows.map(rowToDispatch);
+}