@soleri/core 9.0.3 → 9.2.0

package/src/queue/job-queue.test.ts

@@ -0,0 +1,331 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { JobQueue } from './job-queue.js';
+import type { PersistenceProvider } from '../persistence/types.js';
+
+// ─── Mock Persistence (in-memory SQLite-like store) ──────────────────
+
+function createMockProvider(): PersistenceProvider {
+  const rows = new Map<string, Record<string, unknown>>();
+
+  return {
+    backend: 'sqlite' as const,
+    execSql: vi.fn(),
+    run: vi.fn((sql: string, params?: unknown[]) => {
+      const p = (params ?? []) as unknown[];
+      if (sql.includes('INSERT INTO job_queue')) {
+        const row: Record<string, unknown> = {
+          id: p[0],
+          type: p[1],
+          status: 'pending',
+          entry_id: p[2],
+          payload: p[3],
+          depends_on: p[4],
+          pipeline_id: p[5],
+          retry_count: 0,
+          max_retries: p[6],
+          result: null,
+          error: null,
+          created_at: new Date().toISOString(),
+          started_at: null,
+          completed_at: null,
+        };
+        rows.set(row.id as string, row);
+      }
+      if (sql.includes('UPDATE') && sql.includes("status = 'running'")) {
+        const id = p[0] as string;
+        const row = rows.get(id);
+        if (row) {
+          row.status = 'running';
+          row.started_at = new Date().toISOString();
+        }
+      }
+      if (sql.includes('UPDATE') && sql.includes("status = 'completed'")) {
+        const id = p[1] as string;
+        const row = rows.get(id);
+        if (row) {
+          row.status = 'completed';
+          row.result = p[0] as string;
+          row.completed_at = new Date().toISOString();
+        }
+      }
+      if (sql.includes('UPDATE') && sql.includes("status = 'failed'")) {
+        const id = p[1] as string;
+        const row = rows.get(id);
+        if (row) {
+          row.status = 'failed';
+          row.error = p[0] as string;
+          row.completed_at = new Date().toISOString();
+        }
+      }
+      if (sql.includes('UPDATE') && sql.includes("status = 'pending'") && sql.includes('retry_count')) {
+        const id = p[0] as string;
+        const row = rows.get(id);
+        if (row) {
+          row.status = 'pending';
+          row.retry_count = (row.retry_count as number) + 1;
+          row.error = null;
+          row.started_at = null;
+          row.completed_at = null;
+        }
+      }
+      if (sql.includes('DELETE FROM job_queue')) {
+        let deleted = 0;
+        for (const [id, row] of rows) {
+          if (row.status === 'completed' || row.status === 'failed') {
+            rows.delete(id);
+            deleted++;
+          }
+        }
+        return { changes: deleted, lastInsertRowid: 0 };
+      }
+      return { changes: 1, lastInsertRowid: 0 };
+    }),
+    get: vi.fn((sql: string, params?: unknown[]) => {
+      const p = (params ?? []) as unknown[];
+      const id = p[0] as string;
+      return rows.get(id);
+    }),
+    all: vi.fn((sql: string, params?: unknown[]) => {
+      if (sql.includes('GROUP BY status')) {
+        const counts: Record<string, number> = {};
+        for (const row of rows.values()) {
+          const s = row.status as string;
+          counts[s] = (counts[s] ?? 0) + 1;
+        }
+        return Object.entries(counts).map(([status, count]) => ({ status, count }));
+      }
+      if (sql.includes('pipeline_id')) {
+        const pid = (params as unknown[])[0];
+        return [...rows.values()].filter((r) => r.pipeline_id === pid);
+      }
+      if (sql.includes("status = 'pending'")) {
+        return [...rows.values()]
+          .filter((r) => r.status === 'pending')
+          .sort(
+            (a, b) =>
+              new Date(a.created_at as string).getTime() -
+              new Date(b.created_at as string).getTime(),
+          );
+      }
+      return [...rows.values()];
+    }),
+    transaction: vi.fn((fn) => fn()),
+    ftsSearch: vi.fn(() => []),
+    ftsRebuild: vi.fn(),
+    close: vi.fn(),
+  };
+}
+
+// ─── Tests ───────────────────────────────────────────────────────────
+
+describe('JobQueue', () => {
+  let provider: PersistenceProvider;
+  let queue: JobQueue;
+
+  beforeEach(() => {
+    provider = createMockProvider();
+    queue = new JobQueue(provider);
+  });
+
+  it('initializes the table on construction', () => {
+    expect(provider.execSql).toHaveBeenCalledTimes(1);
+    expect((provider.execSql as ReturnType<typeof vi.fn>).mock.calls[0][0]).toContain(
+      'CREATE TABLE IF NOT EXISTS job_queue',
+    );
+  });
+
+  describe('enqueue', () => {
+    it('returns a 12-char job ID', () => {
+      const id = queue.enqueue('test-type');
+      expect(id).toHaveLength(12);
+    });
+
+    it('inserts a row with default options', () => {
+      queue.enqueue('groom');
+      expect(provider.run).toHaveBeenCalledWith(
+        expect.stringContaining('INSERT INTO job_queue'),
+        expect.arrayContaining([
+          expect.any(String),
+          'groom',
+          null,
+          '{}',
+          '[]',
+          null,
+          3,
+        ]),
+      );
+    });
+
+    it('passes custom options through', () => {
+      queue.enqueue('enrich', {
+        entryId: 'e-1',
+        payload: { depth: 2 },
+        dependsOn: ['dep-1'],
+        pipelineId: 'pipe-1',
+        maxRetries: 5,
+      });
+      expect(provider.run).toHaveBeenCalledWith(
+        expect.stringContaining('INSERT INTO job_queue'),
+        expect.arrayContaining([
+          expect.any(String),
+          'enrich',
+          'e-1',
+          '{"depth":2}',
+          '["dep-1"]',
+          'pipe-1',
+          5,
+        ]),
+      );
+    });
+  });
+
+  describe('dequeue', () => {
+    it('returns null when queue is empty', () => {
+      (provider.all as ReturnType<typeof vi.fn>).mockReturnValueOnce([]);
+      const job = queue.dequeue();
+      expect(job).toBeNull();
+    });
+
+    it('returns a job with status running', () => {
+      queue.enqueue('groom');
+      const job = queue.dequeue();
+      expect(job).not.toBeNull();
+      expect(job!.status).toBe('running');
+      expect(job!.type).toBe('groom');
+    });
+
+    it('skips jobs whose dependencies are not completed', () => {
+      const depId = queue.enqueue('step-a');
+      queue.enqueue('step-b', { dependsOn: [depId] });
+      // step-a is pending, so step-b should not dequeue
+      // dequeue should return step-a (no deps)
+      const job = queue.dequeue();
+      expect(job).not.toBeNull();
+      expect(job!.type).toBe('step-a');
+    });
+  });
+
+  describe('complete', () => {
+    it('marks a job as completed with a result', () => {
+      const id = queue.enqueue('groom');
+      queue.dequeue();
+      queue.complete(id, { score: 95 });
+      const job = queue.get(id);
+      expect(job).not.toBeNull();
+      expect(job!.status).toBe('completed');
+    });
+
+    it('marks a job as completed without result', () => {
+      const id = queue.enqueue('groom');
+      queue.dequeue();
+      queue.complete(id);
+      expect(provider.run).toHaveBeenCalledWith(
+        expect.stringContaining("status = 'completed'"),
+        [null, id],
+      );
+    });
+  });
+
+  describe('fail', () => {
+    it('marks a job as failed with error message', () => {
+      const id = queue.enqueue('groom');
+      queue.dequeue();
+      queue.fail(id, 'timeout');
+      const job = queue.get(id);
+      expect(job).not.toBeNull();
+      expect(job!.status).toBe('failed');
+      expect(job!.error).toBe('timeout');
+    });
+  });
+
+  describe('retry', () => {
+    it('returns true and resets status to pending', () => {
+      const id = queue.enqueue('groom', { maxRetries: 3 });
+      queue.dequeue();
+      queue.fail(id, 'err');
+      const result = queue.retry(id);
+      expect(result).toBe(true);
+      const job = queue.get(id);
+      expect(job!.status).toBe('pending');
+      expect(job!.retryCount).toBe(1);
+    });
+
+    it('returns false when job not found', () => {
+      (provider.get as ReturnType<typeof vi.fn>).mockReturnValueOnce(undefined);
+      expect(queue.retry('nonexistent')).toBe(false);
+    });
+
+    it('returns false when max retries exceeded', () => {
+      const id = queue.enqueue('groom', { maxRetries: 1 });
+      queue.dequeue();
+      queue.fail(id, 'err');
+      queue.retry(id); // retry_count → 1, max_retries = 1
+      // Now retry_count == max_retries, should return false
+      expect(queue.retry(id)).toBe(false);
+    });
+  });
+
+  describe('getStats', () => {
+    it('returns zeroes for empty queue', () => {
+      (provider.all as ReturnType<typeof vi.fn>).mockReturnValueOnce([]);
+      const stats = queue.getStats();
+      expect(stats).toEqual({
+        pending: 0,
+        running: 0,
+        completed: 0,
+        failed: 0,
+        total: 0,
+      });
+    });
+
+    it('aggregates counts by status', () => {
+      queue.enqueue('a');
+      queue.enqueue('b');
+      const id = queue.enqueue('c');
+      queue.dequeue();
+      queue.complete(id);
+      const stats = queue.getStats();
+      expect(stats.total).toBeGreaterThan(0);
+    });
+  });
+
+  describe('getByPipeline', () => {
+    it('returns jobs filtered by pipeline ID', () => {
+      queue.enqueue('a', { pipelineId: 'pipe-1' });
+      queue.enqueue('b', { pipelineId: 'pipe-1' });
+      queue.enqueue('c', { pipelineId: 'pipe-2' });
+      const jobs = queue.getByPipeline('pipe-1');
+      expect(jobs.length).toBe(2);
+      for (const j of jobs) {
+        expect(j.pipelineId).toBe('pipe-1');
+      }
+    });
+  });
+
+  describe('get', () => {
+    it('returns a job by ID', () => {
+      const id = queue.enqueue('groom');
+      const job = queue.get(id);
+      expect(job).not.toBeNull();
+      expect(job!.id).toBe(id);
+      expect(job!.type).toBe('groom');
+    });
+
+    it('returns null for unknown ID', () => {
+      (provider.get as ReturnType<typeof vi.fn>).mockReturnValueOnce(undefined);
+      expect(queue.get('missing')).toBeNull();
+    });
+  });
+
+  describe('purge', () => {
+    it('deletes completed/failed jobs and returns count', () => {
+      const id1 = queue.enqueue('a');
+      const id2 = queue.enqueue('b');
+      queue.dequeue();
+      queue.complete(id1);
+      queue.fail(id2, 'err');
+      const deleted = queue.purge(30);
+      expect(deleted).toBeGreaterThanOrEqual(0);
+    });
+  });
+});

package/src/queue/pipeline-runner.test.ts

@@ -0,0 +1,209 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import { PipelineRunner } from './pipeline-runner.js';
+import type { JobQueue, Job } from './job-queue.js';
+
+// ─── Mock JobQueue ───────────────────────────────────────────────────
+
+function makeJob(overrides: Partial<Job> = {}): Job {
+  return {
+    id: 'job-1',
+    type: 'groom',
+    status: 'running',
+    entryId: null,
+    payload: {},
+    dependsOn: [],
+    pipelineId: null,
+    retryCount: 0,
+    maxRetries: 3,
+    result: null,
+    error: null,
+    createdAt: new Date().toISOString(),
+    startedAt: new Date().toISOString(),
+    completedAt: null,
+    ...overrides,
+  };
+}
+
+function createMockQueue(jobs: Job[] = []): JobQueue {
+  return {
+    enqueue: vi.fn(),
+    dequeue: vi.fn(),
+    dequeueReady: vi.fn(() => {
+      const batch = jobs.splice(0);
+      return batch;
+    }),
+    complete: vi.fn(),
+    fail: vi.fn(),
+    retry: vi.fn(() => true),
+    getStats: vi.fn(),
+    getByPipeline: vi.fn(),
+    get: vi.fn(),
+    purge: vi.fn(),
+  } as unknown as JobQueue;
+}
+
+// ─── Tests ───────────────────────────────────────────────────────────
+
+describe('PipelineRunner', () => {
+  let runner: PipelineRunner;
+  let mockQueue: JobQueue;
+
+  afterEach(() => {
+    runner?.stop();
+  });
+
+  describe('registerHandler', () => {
+    it('registers a handler for a job type', () => {
+      mockQueue = createMockQueue();
+      runner = new PipelineRunner(mockQueue);
+      runner.registerHandler('groom', vi.fn());
+      expect(runner.hasHandler('groom')).toBe(true);
+    });
+
+    it('returns false for unregistered type', () => {
+      mockQueue = createMockQueue();
+      runner = new PipelineRunner(mockQueue);
+      expect(runner.hasHandler('unknown')).toBe(false);
+    });
+  });
+
+  describe('processOnce', () => {
+    it('processes a batch of ready jobs', async () => {
+      const handler = vi.fn().mockResolvedValue({ enriched: true });
+      const job = makeJob({ type: 'groom' });
+      mockQueue = createMockQueue([job]);
+      runner = new PipelineRunner(mockQueue);
+      runner.registerHandler('groom', handler);
+
+      const processed = await runner.processOnce();
+      expect(processed).toBe(1);
+      expect(handler).toHaveBeenCalledWith(job);
+      expect(mockQueue.complete).toHaveBeenCalledWith(job.id, { enriched: true });
+    });
+
+    it('fails jobs with no registered handler', async () => {
+      const job = makeJob({ type: 'unregistered' });
+      mockQueue = createMockQueue([job]);
+      runner = new PipelineRunner(mockQueue);
+
+      const processed = await runner.processOnce();
+      expect(processed).toBe(0);
+      expect(mockQueue.fail).toHaveBeenCalledWith(
+        job.id,
+        expect.stringContaining('No handler registered'),
+      );
+    });
+
+    it('retries on handler error when retryable', async () => {
+      const handler = vi.fn().mockRejectedValue(new Error('transient'));
+      const job = makeJob({ type: 'groom' });
+      mockQueue = createMockQueue([job]);
+      (mockQueue.retry as ReturnType<typeof vi.fn>).mockReturnValue(true);
+      runner = new PipelineRunner(mockQueue);
+      runner.registerHandler('groom', handler);
+
+      await runner.processOnce();
+      expect(mockQueue.retry).toHaveBeenCalledWith(job.id);
+    });
+
+    it('fails job when retry returns false (exhausted)', async () => {
+      const handler = vi.fn().mockRejectedValue(new Error('permanent'));
+      const job = makeJob({ type: 'groom' });
+      mockQueue = createMockQueue([job]);
+      (mockQueue.retry as ReturnType<typeof vi.fn>).mockReturnValue(false);
+      runner = new PipelineRunner(mockQueue);
+      runner.registerHandler('groom', handler);
+
+      await runner.processOnce();
+      expect(mockQueue.fail).toHaveBeenCalledWith(job.id, 'permanent');
+    });
+
+    it('handles non-Error thrown values', async () => {
+      const handler = vi.fn().mockRejectedValue('string-error');
+      const job = makeJob({ type: 'groom' });
+      mockQueue = createMockQueue([job]);
+      (mockQueue.retry as ReturnType<typeof vi.fn>).mockReturnValue(false);
+      runner = new PipelineRunner(mockQueue);
+      runner.registerHandler('groom', handler);
+
+      await runner.processOnce();
+      expect(mockQueue.fail).toHaveBeenCalledWith(job.id, 'string-error');
+    });
+
+    it('returns 0 when no jobs are ready', async () => {
+      mockQueue = createMockQueue([]);
+      runner = new PipelineRunner(mockQueue);
+      expect(await runner.processOnce()).toBe(0);
+    });
+
+    it('processes multiple jobs in one batch', async () => {
+      const handler = vi.fn().mockResolvedValue('ok');
+      const jobs = [
+        makeJob({ id: 'j1', type: 'groom' }),
+        makeJob({ id: 'j2', type: 'groom' }),
+        makeJob({ id: 'j3', type: 'groom' }),
+      ];
+      mockQueue = createMockQueue(jobs);
+      runner = new PipelineRunner(mockQueue);
+      runner.registerHandler('groom', handler);
+
+      const processed = await runner.processOnce(10);
+      expect(processed).toBe(3);
+      expect(handler).toHaveBeenCalledTimes(3);
+    });
+  });
+
+  describe('start / stop', () => {
+    it('start is idempotent', () => {
+      mockQueue = createMockQueue();
+      runner = new PipelineRunner(mockQueue, 100_000);
+      runner.start();
+      runner.start(); // second call should be a no-op
+      const status = runner.getStatus();
+      expect(status.running).toBe(true);
+    });
+
+    it('stop clears the interval', () => {
+      mockQueue = createMockQueue();
+      runner = new PipelineRunner(mockQueue, 100_000);
+      runner.start();
+      runner.stop();
+      expect(runner.getStatus().running).toBe(false);
+    });
+  });
+
+  describe('getStatus', () => {
+    it('returns initial status', () => {
+      mockQueue = createMockQueue();
+      runner = new PipelineRunner(mockQueue, 5000);
+      const status = runner.getStatus();
+      expect(status).toEqual({
+        running: false,
+        pollIntervalMs: 5000,
+        tickCount: 0,
+        jobsProcessed: 0,
+        jobsFailed: 0,
+        jobsRetried: 0,
+        lastTickAt: null,
+      });
+    });
+
+    it('tracks processed/failed/retried counts', async () => {
+      const successHandler = vi.fn().mockResolvedValue('ok');
+      const failHandler = vi.fn().mockRejectedValue(new Error('fail'));
+      const successJob = makeJob({ id: 's1', type: 'success' });
+      const failJob = makeJob({ id: 'f1', type: 'fail' });
+
+      mockQueue = createMockQueue([successJob, failJob]);
+      (mockQueue.retry as ReturnType<typeof vi.fn>).mockReturnValue(true);
+      runner = new PipelineRunner(mockQueue);
+      runner.registerHandler('success', successHandler);
+      runner.registerHandler('fail', failHandler);
+
+      await runner.processOnce();
+      const status = runner.getStatus();
+      expect(status.jobsProcessed).toBe(1);
+      expect(status.jobsRetried).toBe(1);
+    });
+  });
+});