@dotdrelle/wiki-manager 0.7.3 → 0.9.3

package/src/runtime/store.js

@@ -3,17 +3,23 @@ import { dirname, join, resolve } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 import { applyAgentProjectionToSession, dispatchAgentEvent, reduceAgentEvents } from '../core/agentEvents.js';
 import { defaultRuntimeStateDir } from '../core/env.js';
+import { projectQueue } from '../core/jobQueue.js';
 
 export { defaultRuntimeStateDir };
 
 const NON_PERSISTED_EVENT_TYPES = new Set(['runtime_log']);
 
-const RUN_STATUS_BY_TERMINAL_EVENT = {
+const RUN_STATUS_BY_EVENT = {
   run_done: 'done',
   run_error: 'error',
   run_cancelled: 'cancelled',
+  run_pending_approval: 'pending_approval',
+  run_approved: 'running',
 };
 
+const RECOVERABLE_RUN_STATUSES = ['running', 'waiting', 'pending_approval'];
+export const RECOVERABLE_QUEUE_STATUSES = ['waiting', 'queued', 'starting', 'running', 'blocked', 'pending_approval'];
+
 export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName = 'runtime.db' } = {}) {
   const resolvedStateDir = resolve(stateDir);
   mkdirSync(resolvedStateDir, { recursive: true });
@@ -23,11 +29,13 @@ export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName
   db.exec('PRAGMA foreign_keys = ON');
   db.exec(`
     CREATE TABLE IF NOT EXISTS events (
-      id TEXT PRIMARY KEY,
+      sequence INTEGER PRIMARY KEY AUTOINCREMENT,
+      id TEXT NOT NULL UNIQUE,
       ts TEXT NOT NULL,
       type TEXT NOT NULL,
       run_id TEXT,
       turn_id TEXT,
+      workspace TEXT,
       origin TEXT,
       payload TEXT NOT NULL
     );
@@ -35,6 +43,7 @@ export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName
     CREATE INDEX IF NOT EXISTS idx_events_run_id ON events(run_id);
     CREATE TABLE IF NOT EXISTS runs (
       id TEXT PRIMARY KEY,
+      workspace TEXT,
       status TEXT NOT NULL,
       input TEXT,
       created_at TEXT NOT NULL,
@@ -58,31 +67,69 @@ export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName
       updated_at TEXT NOT NULL
     );
   `);
+  ensureColumn(db, 'events', 'sequence', 'INTEGER');
+  ensureColumn(db, 'events', 'workspace', 'TEXT');
+  ensureColumn(db, 'runs', 'workspace', 'TEXT');
+  backfillEventSequence(db);
+  db.exec('CREATE INDEX IF NOT EXISTS idx_events_workspace ON events(workspace)');
+  db.exec('CREATE INDEX IF NOT EXISTS idx_events_sequence ON events(sequence)');
 
   let lastEventId = null;
+  let lastEventSequence = null;
 
   const insertEvent = db.prepare(`
-    INSERT OR IGNORE INTO events (id, ts, type, run_id, turn_id, origin, payload)
-    VALUES (?, ?, ?, ?, ?, ?, ?)
+    INSERT OR IGNORE INTO events (sequence, id, ts, type, run_id, turn_id, workspace, origin, payload)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
   `);
+  const nextEventSequenceStatement = db.prepare('SELECT COALESCE(MAX(sequence), 0) + 1 AS next_sequence FROM events');
   const listEventsStatement = db.prepare(`
-    SELECT id, ts, type, run_id, turn_id, origin, payload
+    SELECT sequence, id, ts, type, run_id, turn_id, workspace, origin, payload
+    FROM events
+    ORDER BY sequence ASC
+  `);
+  const listEventsByWorkspaceStatement = db.prepare(`
+    SELECT sequence, id, ts, type, run_id, turn_id, workspace, origin, payload
     FROM events
-    ORDER BY ts ASC, id ASC
+    WHERE workspace = ?
+    ORDER BY sequence ASC
   `);
   const upsertRun = db.prepare(`
-    INSERT INTO runs (id, status, input, created_at, updated_at)
-    VALUES (?, ?, ?, ?, ?)
+    INSERT INTO runs (id, workspace, status, input, created_at, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?)
     ON CONFLICT(id) DO UPDATE SET
+      workspace = COALESCE(excluded.workspace, runs.workspace),
       status = excluded.status,
       input = COALESCE(excluded.input, runs.input),
       updated_at = excluded.updated_at
   `);
   const listRunsStatement = db.prepare(`
-    SELECT id, status, input, created_at, updated_at
+    SELECT id, workspace, status, input, created_at, updated_at
     FROM runs
     ORDER BY created_at DESC
   `);
+  const listRunsByWorkspaceStatement = db.prepare(`
+    SELECT id, workspace, status, input, created_at, updated_at
+    FROM runs
+    WHERE workspace = ?
+    ORDER BY created_at DESC
+  `);
+  const listRecoverableRunsStatement = db.prepare(`
+    SELECT id, workspace, status, input, created_at, updated_at
+    FROM runs
+    WHERE status IN (${RECOVERABLE_RUN_STATUSES.map(() => '?').join(', ')})
+    ORDER BY created_at ASC
+  `);
+  const listRecoverableRunsByWorkspaceStatement = db.prepare(`
+    SELECT id, workspace, status, input, created_at, updated_at
+    FROM runs
+    WHERE workspace = ? AND status IN (${RECOVERABLE_RUN_STATUSES.map(() => '?').join(', ')})
+    ORDER BY created_at ASC
+  `);
+  const interruptRunsStatement = db.prepare(`
+    UPDATE runs
+    SET status = 'interrupted', updated_at = ?
+    WHERE workspace = ? AND status IN (${RECOVERABLE_RUN_STATUSES.map(() => '?').join(', ')})
+  `);
   const upsertQueueItem = db.prepare(`
     INSERT INTO queue_items (
       id, workspace, server, tool, args, lock_key, status, reason, job_id, activity_key,
@@ -109,6 +156,11 @@ export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName
     DELETE FROM queue_items
     WHERE id NOT IN (SELECT value FROM json_each(?))
   `);
+  const deleteMissingQueueItemsForWorkspace = db.prepare(`
+    DELETE FROM queue_items
+    WHERE workspace = ? AND id NOT IN (SELECT value FROM json_each(?))
+  `);
+  const clearQueueItemsForWorkspace = db.prepare('DELETE FROM queue_items WHERE workspace = ?');
   const clearQueueItems = db.prepare('DELETE FROM queue_items');
   const listQueueStatement = db.prepare(`
     SELECT id, workspace, server, tool, args, lock_key, status, reason, job_id, activity_key,
@@ -116,33 +168,58 @@ export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName
     FROM queue_items
     ORDER BY COALESCE(created_at, updated_at) ASC, id ASC
   `);
+  const listQueueByWorkspaceStatement = db.prepare(`
+    SELECT id, workspace, server, tool, args, lock_key, status, reason, job_id, activity_key,
+      error, created_at, started_at, finished_at, updated_at
+    FROM queue_items
+    WHERE workspace = ?
+    ORDER BY COALESCE(created_at, updated_at) ASC, id ASC
+  `);
+  const listRecoverableWorkspacesStatement = db.prepare(`
+    SELECT DISTINCT workspace FROM runs
+    WHERE workspace IS NOT NULL AND status IN (${RECOVERABLE_RUN_STATUSES.map(() => '?').join(', ')})
+    UNION
+    SELECT DISTINCT workspace FROM queue_items
+    WHERE workspace IS NOT NULL AND status IN (${RECOVERABLE_QUEUE_STATUSES.map(() => '?').join(', ')})
+    ORDER BY workspace
+  `);
 
   function persistEvent(event) {
     if (NON_PERSISTED_EVENT_TYPES.has(event.type)) return event;
-    insertEvent.run(
+    const ws = event.workspace ?? event.payload?.workspace ?? null;
+    const sequence = nextEventSequenceStatement.get().next_sequence;
+    const result = insertEvent.run(
+      sequence,
       event.id,
       event.ts,
       event.type,
       event.runId ?? null,
       event.turnId ?? null,
+      ws,
       event.origin ?? null,
       JSON.stringify(event.payload ?? {}),
     );
-    lastEventId = event.id;
+    if (Number(result.changes ?? 0) > 0) {
+      event.sequence = sequence;
+      lastEventId = event.id;
+      lastEventSequence = sequence;
+    }
     if (event.type === 'run_started') {
       persistRun({
         id: event.runId ?? event.payload?.runId ?? event.id,
         status: 'running',
         input: event.payload?.input ?? null,
+        workspace: ws,
         createdAt: event.ts,
         updatedAt: event.ts,
       });
-    } else if (RUN_STATUS_BY_TERMINAL_EVENT[event.type]) {
+    } else if (RUN_STATUS_BY_EVENT[event.type]) {
       const runId = event.runId ?? event.payload?.runId ?? null;
       if (runId) {
         persistRun({
           id: runId,
-          status: RUN_STATUS_BY_TERMINAL_EVENT[event.type],
+          status: RUN_STATUS_BY_EVENT[event.type],
+          workspace: ws,
           updatedAt: event.ts,
         });
       }
@@ -150,19 +227,26 @@ export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName
     return event;
   }
 
-  function persistRun({ id, status, input = null, createdAt = null, updatedAt = null }) {
+  function persistRun({ id, status, input = null, workspace = null, createdAt = null, updatedAt = null }) {
     if (!id) return;
     const now = new Date().toISOString();
-    upsertRun.run(id, status, input, createdAt ?? now, updatedAt ?? now);
+    upsertRun.run(id, workspace, status, input, createdAt ?? now, updatedAt ?? now);
   }
 
-  function listEvents() {
-    return listEventsStatement.all().map(rowToEvent);
+  function listEvents({ workspace = null } = {}) {
+    const rows = workspace
+      ? listEventsByWorkspaceStatement.all(workspace)
+      : listEventsStatement.all();
+    return rows.map(rowToEvent);
   }
 
-  function listRuns() {
-    return listRunsStatement.all().map((row) => ({
+  function listRuns({ workspace = null } = {}) {
+    const rows = workspace
+      ? listRunsByWorkspaceStatement.all(workspace)
+      : listRunsStatement.all();
+    return rows.map((row) => ({
       id: row.id,
+      workspace: row.workspace ?? null,
       status: row.status,
       input: row.input,
       createdAt: row.created_at,
@@ -170,19 +254,48 @@ export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName
     }));
   }
 
-  function saveQueue(queue = []) {
+  function listRecoverableRuns({ workspace = null } = {}) {
+    const rows = workspace
+      ? listRecoverableRunsByWorkspaceStatement.all(workspace, ...RECOVERABLE_RUN_STATUSES)
+      : listRecoverableRunsStatement.all(...RECOVERABLE_RUN_STATUSES);
+    return rows.map((row) => ({
+      id: row.id,
+      workspace: row.workspace ?? null,
+      status: row.status,
+      input: row.input,
+      createdAt: row.created_at,
+      updatedAt: row.updated_at,
+    }));
+  }
+
+  function listRecoverableWorkspaces() {
+    return listRecoverableWorkspacesStatement
+      .all(...RECOVERABLE_RUN_STATUSES, ...RECOVERABLE_QUEUE_STATUSES)
+      .map((row) => row.workspace);
+  }
+
+  function interruptRuns({ workspace, reason = 'Runtime restart recovery failed.' } = {}) {
+    if (!workspace) return 0;
+    const now = new Date().toISOString();
+    const result = interruptRunsStatement.run(now, workspace, ...RECOVERABLE_RUN_STATUSES);
+    return Number(result.changes ?? 0);
+  }
+
+  function saveQueue(queue = [], { workspace = null } = {}) {
     const items = Array.isArray(queue) ? queue : [];
     const now = new Date().toISOString();
     db.exec('BEGIN');
     try {
       if (items.length === 0) {
-        clearQueueItems.run();
+        if (workspace) clearQueueItemsForWorkspace.run(workspace);
+        else clearQueueItems.run();
       } else {
-        deleteMissingQueueItems.run(JSON.stringify(items.map((item) => item.id)));
+        if (workspace) deleteMissingQueueItemsForWorkspace.run(workspace, JSON.stringify(items.map((item) => item.id)));
+        else deleteMissingQueueItems.run(JSON.stringify(items.map((item) => item.id)));
         for (const item of items) {
           upsertQueueItem.run(
             item.id,
-            item.workspace ?? null,
+            item.workspace ?? workspace ?? null,
             item.server ?? 'production',
             item.tool ?? 'production_start_job',
             JSON.stringify(item.args ?? {}),
@@ -206,8 +319,11 @@ export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName
     }
   }
 
-  function listQueue() {
-    return listQueueStatement.all().map((row) => ({
+  function listQueue({ workspace = null } = {}) {
+    const rows = workspace
+      ? listQueueByWorkspaceStatement.all(workspace)
+      : listQueueStatement.all();
+    return rows.map((row) => ({
       id: row.id,
       workspace: row.workspace ?? null,
       server: row.server,
@@ -226,34 +342,41 @@ export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName
     }));
   }
 
-  function replayEvents(session) {
-    const events = listEvents();
+  function replayEvents(session, { workspace = null } = {}) {
+    const events = listEvents({ workspace });
     for (const event of events) {
       dispatchAgentEvent(session, event);
     }
-    if (events.length > 0) lastEventId = events.at(-1).id;
+    if (events.length > 0) {
+      lastEventId = events.at(-1).id;
+      lastEventSequence = events.at(-1).sequence ?? null;
+    }
     return session.agentProjection ?? reduceAgentEvents([]);
   }
 
-  function getProjection() {
-    return reduceAgentEvents(listEvents());
+  function getProjection({ workspace = null } = {}) {
+    return reduceAgentEvents(listEvents({ workspace }));
   }
 
-  function getState(session = null) {
-    const projection = session?.agentProjection ?? reduceAgentEvents(listEvents());
-    const queue = session?.queueStore?.list() ?? listQueue();
+  function getState(session = null, { workspace = null } = {}) {
+    const events = session?.agentProjection ? null : listEvents({ workspace });
+    const projection = session?.agentProjection ?? reduceAgentEvents(events);
+    const rawQueue = session?.queueStore?.list() ?? listQueue({ workspace });
     return {
       ...projection,
-      runs: listRuns(),
-      queue: queue.map((item) => ({ ...item })),
-      eventsCursor: lastEventId,
+      runs: listRuns({ workspace }),
+      queue: projectQueue(projection.plan, rawQueue, { workspace }),
+      controlQueue: Array.isArray(projection.controlQueue)
+        ? projection.controlQueue.filter((item) => !workspace || item.workspace === workspace || !item.workspace).map((item) => ({ ...item }))
+        : [],
+      eventsCursor: session?.agentEvents?.at(-1)?.sequence ?? events?.at(-1)?.sequence ?? lastEventSequence,
     };
   }
 
-  function hydrateSession(session) {
-    const projection = replayEvents(session);
+  function hydrateSession(session, { workspace = null } = {}) {
+    const projection = replayEvents(session, { workspace });
     applyAgentProjectionToSession(session, projection);
-    session.jobQueue = listQueue();
+    session.jobQueue = listQueue({ workspace });
     return projection;
   }
 
@@ -269,6 +392,9 @@ export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName
     persistRun,
     listEvents,
     listRuns,
+    listRecoverableRuns,
+    listRecoverableWorkspaces,
+    interruptRuns,
     saveQueue,
     listQueue,
     replayEvents,
@@ -281,12 +407,28 @@ export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName
 
 function rowToEvent(row) {
   return {
+    sequence: row.sequence ?? null,
     id: row.id,
     ts: row.ts,
     type: row.type,
     origin: row.origin ?? 'system',
     runId: row.run_id ?? null,
     turnId: row.turn_id ?? null,
+    workspace: row.workspace ?? null,
     payload: row.payload ? JSON.parse(row.payload) : {},
   };
 }
+
+function ensureColumn(db, table, column, definition) {
+  const existing = db.prepare(`PRAGMA table_info(${table})`).all();
+  if (existing.some((row) => row.name === column)) return;
+  db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
+}
+
+function backfillEventSequence(db) {
+  db.exec(`
+    UPDATE events
+    SET sequence = rowid
+    WHERE sequence IS NULL
+  `);
+}