@soleri/core 7.0.0 → 8.0.0

package/src/planning/planner.ts

@@ -385,6 +385,17 @@ export class Planner {
     return [...this.store.plans];
   }
 
+  /**
+   * Permanently remove a plan by ID. Returns true if found and removed.
+   */
+  remove(planId: string): boolean {
+    const idx = this.store.plans.findIndex((p) => p.id === planId);
+    if (idx < 0) return false;
+    this.store.plans.splice(idx, 1);
+    this.save();
+    return true;
+  }
+
   /**
    * Transition a plan to a new status using the typed FSM.
    * Validates that the transition is allowed before applying it.

package/src/queue/job-queue.ts

@@ -0,0 +1,281 @@
+/**
+ * Job Queue — SQLite-backed FIFO queue with DAG dependencies and retries.
+ *
+ * Generic infrastructure — not curator-specific. Reusable by agency, intake, etc.
+ *
+ * Features:
+ * - Persistent jobs (survive process restarts)
+ * - DAG dependency resolution (job B waits for job A)
+ * - Pipeline grouping (group related jobs under one ID)
+ * - Configurable retries with max limit
+ * - Status tracking: pending → running → completed | failed
+ */
+
+import { randomUUID } from 'node:crypto';
+import type { PersistenceProvider } from '../persistence/types.js';
+
+// ─── Types ───────────────────────────────────────────────────────────
+
+export type JobStatus = 'pending' | 'running' | 'completed' | 'failed';
+
+export interface Job {
+  id: string;
+  type: string;
+  status: JobStatus;
+  entryId: string | null;
+  payload: Record<string, unknown>;
+  dependsOn: string[];
+  pipelineId: string | null;
+  retryCount: number;
+  maxRetries: number;
+  result: unknown | null;
+  error: string | null;
+  createdAt: string;
+  startedAt: string | null;
+  completedAt: string | null;
+}
+
+export interface JobQueueStats {
+  pending: number;
+  running: number;
+  completed: number;
+  failed: number;
+  total: number;
+}
+
+export interface EnqueueOptions {
+  entryId?: string;
+  payload?: Record<string, unknown>;
+  dependsOn?: string[];
+  pipelineId?: string;
+  maxRetries?: number;
+}
+
+// ─── Class ───────────────────────────────────────────────────────────
+
+export class JobQueue {
+  private provider: PersistenceProvider;
+
+  constructor(provider: PersistenceProvider) {
+    this.provider = provider;
+    this.initializeTable();
+  }
+
+  private initializeTable(): void {
+    this.provider.execSql(`
+      CREATE TABLE IF NOT EXISTS job_queue (
+        id TEXT PRIMARY KEY,
+        type TEXT NOT NULL,
+        status TEXT NOT NULL DEFAULT 'pending',
+        entry_id TEXT,
+        payload TEXT DEFAULT '{}',
+        depends_on TEXT DEFAULT '[]',
+        pipeline_id TEXT,
+        retry_count INTEGER DEFAULT 0,
+        max_retries INTEGER DEFAULT 3,
+        result TEXT,
+        error TEXT,
+        created_at TEXT NOT NULL DEFAULT (datetime('now')),
+        started_at TEXT,
+        completed_at TEXT
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_job_queue_status ON job_queue(status);
+      CREATE INDEX IF NOT EXISTS idx_job_queue_pipeline ON job_queue(pipeline_id);
+    `);
+  }
+
+  /**
+   * Enqueue a new job. Returns the job ID.
+   */
+  enqueue(type: string, options?: EnqueueOptions): string {
+    const id = randomUUID().slice(0, 12);
+    this.provider.run(
+      `INSERT INTO job_queue (id, type, entry_id, payload, depends_on, pipeline_id, max_retries)
+       VALUES (?, ?, ?, ?, ?, ?, ?)`,
+      [
+        id,
+        type,
+        options?.entryId ?? null,
+        JSON.stringify(options?.payload ?? {}),
+        JSON.stringify(options?.dependsOn ?? []),
+        options?.pipelineId ?? null,
+        options?.maxRetries ?? 3,
+      ],
+    );
+    return id;
+  }
+
+  /**
+   * Dequeue the oldest pending job with all dependencies completed.
+   * Marks it as running. Returns null if no ready jobs.
+   */
+  dequeue(): Job | null {
+    const ready = this.dequeueReady(1);
+    return ready.length > 0 ? ready[0] : null;
+  }
+
+  /**
+   * Dequeue up to `limit` pending jobs whose dependencies are all completed.
+   */
+  dequeueReady(limit: number = 10): Job[] {
+    const rows = this.provider.all<JobRow>(
+      "SELECT * FROM job_queue WHERE status = 'pending' ORDER BY created_at ASC LIMIT ?",
+      [limit * 3], // Over-fetch to filter by deps
+    );
+
+    const ready: Job[] = [];
+    for (const row of rows) {
+      if (ready.length >= limit) break;
+      const deps = JSON.parse(row.depends_on) as string[];
+      if (deps.length === 0 || this.allDepsCompleted(deps)) {
+        this.provider.run(
+          "UPDATE job_queue SET status = 'running', started_at = datetime('now') WHERE id = ?",
+          [row.id],
+        );
+        const job = rowToJob(row);
+        job.status = 'running';
+        ready.push(job);
+      }
+    }
+    return ready;
+  }
+
+  /**
+   * Mark a job as completed with an optional result.
+   */
+  complete(jobId: string, result?: unknown): void {
+    this.provider.run(
+      "UPDATE job_queue SET status = 'completed', completed_at = datetime('now'), result = ? WHERE id = ?",
+      [result !== undefined ? JSON.stringify(result) : null, jobId],
+    );
+  }
+
+  /**
+   * Mark a job as failed with an error message.
+   */
+  fail(jobId: string, error: string): void {
+    this.provider.run(
+      "UPDATE job_queue SET status = 'failed', completed_at = datetime('now'), error = ? WHERE id = ?",
+      [error, jobId],
+    );
+  }
+
+  /**
+   * Retry a failed job (resets to pending). Returns false if max retries exceeded.
+   */
+  retry(jobId: string): boolean {
+    const row = this.provider.get<JobRow>('SELECT * FROM job_queue WHERE id = ?', [jobId]);
+    if (!row) return false;
+    if (row.retry_count >= row.max_retries) return false;
+
+    this.provider.run(
+      "UPDATE job_queue SET status = 'pending', retry_count = retry_count + 1, error = NULL, started_at = NULL, completed_at = NULL WHERE id = ?",
+      [jobId],
+    );
+    return true;
+  }
+
+  /**
+   * Get queue statistics.
+   */
+  getStats(): JobQueueStats {
+    const rows = this.provider.all<{ status: string; count: number }>(
+      'SELECT status, COUNT(*) as count FROM job_queue GROUP BY status',
+    );
+    const byStatus: Record<string, number> = {};
+    let total = 0;
+    for (const row of rows) {
+      byStatus[row.status] = row.count;
+      total += row.count;
+    }
+    return {
+      pending: byStatus['pending'] ?? 0,
+      running: byStatus['running'] ?? 0,
+      completed: byStatus['completed'] ?? 0,
+      failed: byStatus['failed'] ?? 0,
+      total,
+    };
+  }
+
+  /**
+   * Get all jobs for a pipeline.
+   */
+  getByPipeline(pipelineId: string): Job[] {
+    const rows = this.provider.all<JobRow>(
+      'SELECT * FROM job_queue WHERE pipeline_id = ? ORDER BY created_at ASC',
+      [pipelineId],
+    );
+    return rows.map(rowToJob);
+  }
+
+  /**
+   * Get a single job by ID.
+   */
+  get(jobId: string): Job | null {
+    const row = this.provider.get<JobRow>('SELECT * FROM job_queue WHERE id = ?', [jobId]);
+    return row ? rowToJob(row) : null;
+  }
+
+  /**
+   * Purge completed/failed jobs older than N days.
+   */
+  purge(olderThanDays: number = 30): number {
+    const result = this.provider.run(
+      "DELETE FROM job_queue WHERE status IN ('completed', 'failed') AND completed_at < datetime('now', ?)",
+      [`-${olderThanDays} days`],
+    );
+    return result.changes;
+  }
+
+  // ─── Internal ──────────────────────────────────────────────────────
+
+  private allDepsCompleted(deps: string[]): boolean {
+    for (const depId of deps) {
+      const row = this.provider.get<{ status: string }>(
+        'SELECT status FROM job_queue WHERE id = ?',
+        [depId],
+      );
+      if (!row || row.status !== 'completed') return false;
+    }
+    return true;
+  }
+}
+
+// ─── Row Types ───────────────────────────────────────────────────────
+
+interface JobRow {
+  id: string;
+  type: string;
+  status: string;
+  entry_id: string | null;
+  payload: string;
+  depends_on: string;
+  pipeline_id: string | null;
+  retry_count: number;
+  max_retries: number;
+  result: string | null;
+  error: string | null;
+  created_at: string;
+  started_at: string | null;
+  completed_at: string | null;
+}
+
+function rowToJob(row: JobRow): Job {
+  return {
+    id: row.id,
+    type: row.type,
+    status: row.status as JobStatus,
+    entryId: row.entry_id,
+    payload: JSON.parse(row.payload) as Record<string, unknown>,
+    dependsOn: JSON.parse(row.depends_on) as string[],
+    pipelineId: row.pipeline_id,
+    retryCount: row.retry_count,
+    maxRetries: row.max_retries,
+    result: row.result ? JSON.parse(row.result) : null,
+    error: row.error,
+    createdAt: row.created_at,
+    startedAt: row.started_at,
+    completedAt: row.completed_at,
+  };
+}

package/src/queue/pipeline-runner.ts

@@ -0,0 +1,149 @@
+/**
+ * Pipeline Runner — background polling loop for DAG job execution.
+ *
+ * Polls the job queue for ready jobs (dependencies completed),
+ * dispatches them to registered handlers, and marks them complete/failed.
+ *
+ * Generic — handlers are registered by type, any module can add its own.
+ */
+
+import type { JobQueue, Job } from './job-queue.js';
+
+// ─── Types ───────────────────────────────────────────────────────────
+
+export type JobHandler = (job: Job) => Promise<unknown>;
+
+export interface PipelineRunnerStatus {
+  running: boolean;
+  pollIntervalMs: number;
+  tickCount: number;
+  jobsProcessed: number;
+  jobsFailed: number;
+  jobsRetried: number;
+  lastTickAt: string | null;
+}
+
+// ─── Class ───────────────────────────────────────────────────────────
+
+export class PipelineRunner {
+  private queue: JobQueue;
+  private handlers = new Map<string, JobHandler>();
+  private pollIntervalMs: number;
+  private running = false;
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private tickCount = 0;
+  private jobsProcessed = 0;
+  private jobsFailed = 0;
+  private jobsRetried = 0;
+  private lastTickAt: string | null = null;
+  private processing = false;
+
+  constructor(queue: JobQueue, pollIntervalMs: number = 5000) {
+    this.queue = queue;
+    this.pollIntervalMs = pollIntervalMs;
+  }
+
+  /**
+   * Register a handler for a job type. When a job of this type is dequeued,
+   * the handler is called. Return value is stored as the job result.
+   */
+  registerHandler(type: string, handler: JobHandler): void {
+    this.handlers.set(type, handler);
+  }
+
+  /**
+   * Start background polling. Idempotent — calling start() twice is safe.
+   */
+  start(): void {
+    if (this.running) return;
+    this.running = true;
+    this.timer = setInterval(() => this.tick(), this.pollIntervalMs);
+  }
+
+  /**
+   * Stop background polling.
+   */
+  stop(): void {
+    this.running = false;
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+  }
+
+  /**
+   * Process one batch of ready jobs immediately (without waiting for poll).
+   * Useful for testing or manual triggering.
+   */
+  async processOnce(batchSize: number = 5): Promise<number> {
+    return this.processBatch(batchSize);
+  }
+
+  /**
+   * Get runner status.
+   */
+  getStatus(): PipelineRunnerStatus {
+    return {
+      running: this.running,
+      pollIntervalMs: this.pollIntervalMs,
+      tickCount: this.tickCount,
+      jobsProcessed: this.jobsProcessed,
+      jobsFailed: this.jobsFailed,
+      jobsRetried: this.jobsRetried,
+      lastTickAt: this.lastTickAt,
+    };
+  }
+
+  /**
+   * Check if a handler is registered for a job type.
+   */
+  hasHandler(type: string): boolean {
+    return this.handlers.has(type);
+  }
+
+  // ─── Internal ──────────────────────────────────────────────────────
+
+  private tick(): void {
+    if (this.processing) return; // Skip if previous tick still running
+    this.tickCount++;
+    this.lastTickAt = new Date().toISOString();
+    this.processBatch(5).catch(() => {
+      /* best-effort */
+    });
+  }
+
+  private async processBatch(batchSize: number): Promise<number> {
+    this.processing = true;
+    let processed = 0;
+    try {
+      const jobs = this.queue.dequeueReady(batchSize);
+      for (const job of jobs) {
+        const handler = this.handlers.get(job.type);
+        if (!handler) {
+          this.queue.fail(job.id, `No handler registered for job type: ${job.type}`);
+          this.jobsFailed++;
+          continue;
+        }
+
+        try {
+          const result = await handler(job);
+          this.queue.complete(job.id, result);
+          this.jobsProcessed++;
+          processed++;
+        } catch (err) {
+          const errorMsg = err instanceof Error ? err.message : String(err);
+          const retried = this.queue.retry(job.id);
+          if (retried) {
+            this.jobsRetried++;
+          } else {
+            this.queue.fail(job.id, errorMsg);
+            this.jobsFailed++;
+          }
+        }
+      }
+    } finally {
+      this.processing = false;
+    }
+    return processed;
+  }
+}