@dotdrelle/wiki-manager 0.6.47 → 0.7.3

package/src/runtime/store.js

@@ -0,0 +1,292 @@
+import { mkdirSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { applyAgentProjectionToSession, dispatchAgentEvent, reduceAgentEvents } from '../core/agentEvents.js';
+import { defaultRuntimeStateDir } from '../core/env.js';
+
+export { defaultRuntimeStateDir };
+
+const NON_PERSISTED_EVENT_TYPES = new Set(['runtime_log']);
+
+const RUN_STATUS_BY_TERMINAL_EVENT = {
+  run_done: 'done',
+  run_error: 'error',
+  run_cancelled: 'cancelled',
+};
+
+export function openRuntimeStore({ stateDir = defaultRuntimeStateDir(), fileName = 'runtime.db' } = {}) {
+  const resolvedStateDir = resolve(stateDir);
+  mkdirSync(resolvedStateDir, { recursive: true });
+  const dbPath = join(resolvedStateDir, fileName);
+  const db = new DatabaseSync(dbPath);
+  db.exec('PRAGMA journal_mode = WAL');
+  db.exec('PRAGMA foreign_keys = ON');
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS events (
+      id TEXT PRIMARY KEY,
+      ts TEXT NOT NULL,
+      type TEXT NOT NULL,
+      run_id TEXT,
+      turn_id TEXT,
+      origin TEXT,
+      payload TEXT NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_events_ts ON events(ts);
+    CREATE INDEX IF NOT EXISTS idx_events_run_id ON events(run_id);
+    CREATE TABLE IF NOT EXISTS runs (
+      id TEXT PRIMARY KEY,
+      status TEXT NOT NULL,
+      input TEXT,
+      created_at TEXT NOT NULL,
+      updated_at TEXT NOT NULL
+    );
+    CREATE TABLE IF NOT EXISTS queue_items (
+      id TEXT PRIMARY KEY,
+      workspace TEXT,
+      server TEXT NOT NULL,
+      tool TEXT NOT NULL,
+      args TEXT NOT NULL,
+      lock_key TEXT,
+      status TEXT NOT NULL,
+      reason TEXT,
+      job_id TEXT,
+      activity_key TEXT,
+      error TEXT,
+      created_at TEXT,
+      started_at TEXT,
+      finished_at TEXT,
+      updated_at TEXT NOT NULL
+    );
+  `);
+
+  let lastEventId = null;
+
+  const insertEvent = db.prepare(`
+    INSERT OR IGNORE INTO events (id, ts, type, run_id, turn_id, origin, payload)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+  `);
+  const listEventsStatement = db.prepare(`
+    SELECT id, ts, type, run_id, turn_id, origin, payload
+    FROM events
+    ORDER BY ts ASC, id ASC
+  `);
+  const upsertRun = db.prepare(`
+    INSERT INTO runs (id, status, input, created_at, updated_at)
+    VALUES (?, ?, ?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      status = excluded.status,
+      input = COALESCE(excluded.input, runs.input),
+      updated_at = excluded.updated_at
+  `);
+  const listRunsStatement = db.prepare(`
+    SELECT id, status, input, created_at, updated_at
+    FROM runs
+    ORDER BY created_at DESC
+  `);
+  const upsertQueueItem = db.prepare(`
+    INSERT INTO queue_items (
+      id, workspace, server, tool, args, lock_key, status, reason, job_id, activity_key,
+      error, created_at, started_at, finished_at, updated_at
+    )
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      workspace = excluded.workspace,
+      server = excluded.server,
+      tool = excluded.tool,
+      args = excluded.args,
+      lock_key = excluded.lock_key,
+      status = excluded.status,
+      reason = excluded.reason,
+      job_id = excluded.job_id,
+      activity_key = excluded.activity_key,
+      error = excluded.error,
+      created_at = excluded.created_at,
+      started_at = excluded.started_at,
+      finished_at = excluded.finished_at,
+      updated_at = excluded.updated_at
+  `);
+  const deleteMissingQueueItems = db.prepare(`
+    DELETE FROM queue_items
+    WHERE id NOT IN (SELECT value FROM json_each(?))
+  `);
+  const clearQueueItems = db.prepare('DELETE FROM queue_items');
+  const listQueueStatement = db.prepare(`
+    SELECT id, workspace, server, tool, args, lock_key, status, reason, job_id, activity_key,
+      error, created_at, started_at, finished_at, updated_at
+    FROM queue_items
+    ORDER BY COALESCE(created_at, updated_at) ASC, id ASC
+  `);
+
+  function persistEvent(event) {
+    if (NON_PERSISTED_EVENT_TYPES.has(event.type)) return event;
+    insertEvent.run(
+      event.id,
+      event.ts,
+      event.type,
+      event.runId ?? null,
+      event.turnId ?? null,
+      event.origin ?? null,
+      JSON.stringify(event.payload ?? {}),
+    );
+    lastEventId = event.id;
+    if (event.type === 'run_started') {
+      persistRun({
+        id: event.runId ?? event.payload?.runId ?? event.id,
+        status: 'running',
+        input: event.payload?.input ?? null,
+        createdAt: event.ts,
+        updatedAt: event.ts,
+      });
+    } else if (RUN_STATUS_BY_TERMINAL_EVENT[event.type]) {
+      const runId = event.runId ?? event.payload?.runId ?? null;
+      if (runId) {
+        persistRun({
+          id: runId,
+          status: RUN_STATUS_BY_TERMINAL_EVENT[event.type],
+          updatedAt: event.ts,
+        });
+      }
+    }
+    return event;
+  }
+
+  function persistRun({ id, status, input = null, createdAt = null, updatedAt = null }) {
+    if (!id) return;
+    const now = new Date().toISOString();
+    upsertRun.run(id, status, input, createdAt ?? now, updatedAt ?? now);
+  }
+
+  function listEvents() {
+    return listEventsStatement.all().map(rowToEvent);
+  }
+
+  function listRuns() {
+    return listRunsStatement.all().map((row) => ({
+      id: row.id,
+      status: row.status,
+      input: row.input,
+      createdAt: row.created_at,
+      updatedAt: row.updated_at,
+    }));
+  }
+
+  function saveQueue(queue = []) {
+    const items = Array.isArray(queue) ? queue : [];
+    const now = new Date().toISOString();
+    db.exec('BEGIN');
+    try {
+      if (items.length === 0) {
+        clearQueueItems.run();
+      } else {
+        deleteMissingQueueItems.run(JSON.stringify(items.map((item) => item.id)));
+        for (const item of items) {
+          upsertQueueItem.run(
+            item.id,
+            item.workspace ?? null,
+            item.server ?? 'production',
+            item.tool ?? 'production_start_job',
+            JSON.stringify(item.args ?? {}),
+            item.lockKey ?? null,
+            item.status ?? 'waiting',
+            item.reason ?? null,
+            item.jobId ?? null,
+            item.activityKey ?? null,
+            item.error ?? null,
+            item.createdAt ?? now,
+            item.startedAt ?? null,
+            item.finishedAt ?? null,
+            now,
+          );
+        }
+      }
+      db.exec('COMMIT');
+    } catch (err) {
+      db.exec('ROLLBACK');
+      throw err;
+    }
+  }
+
+  function listQueue() {
+    return listQueueStatement.all().map((row) => ({
+      id: row.id,
+      workspace: row.workspace ?? null,
+      server: row.server,
+      tool: row.tool,
+      args: row.args ? JSON.parse(row.args) : {},
+      lockKey: row.lock_key ?? null,
+      status: row.status,
+      reason: row.reason ?? null,
+      jobId: row.job_id ?? undefined,
+      activityKey: row.activity_key ?? undefined,
+      error: row.error ?? undefined,
+      createdAt: row.created_at ?? undefined,
+      startedAt: row.started_at ?? undefined,
+      finishedAt: row.finished_at ?? undefined,
+      updatedAt: row.updated_at,
+    }));
+  }
+
+  function replayEvents(session) {
+    const events = listEvents();
+    for (const event of events) {
+      dispatchAgentEvent(session, event);
+    }
+    if (events.length > 0) lastEventId = events.at(-1).id;
+    return session.agentProjection ?? reduceAgentEvents([]);
+  }
+
+  function getProjection() {
+    return reduceAgentEvents(listEvents());
+  }
+
+  function getState(session = null) {
+    const projection = session?.agentProjection ?? reduceAgentEvents(listEvents());
+    const queue = session?.queueStore?.list() ?? listQueue();
+    return {
+      ...projection,
+      runs: listRuns(),
+      queue: queue.map((item) => ({ ...item })),
+      eventsCursor: lastEventId,
+    };
+  }
+
+  function hydrateSession(session) {
+    const projection = replayEvents(session);
+    applyAgentProjectionToSession(session, projection);
+    session.jobQueue = listQueue();
+    return projection;
+  }
+
+  function close() {
+    db.close();
+  }
+
+  return {
+    db,
+    dbPath,
+    stateDir: resolvedStateDir,
+    persistEvent,
+    persistRun,
+    listEvents,
+    listRuns,
+    saveQueue,
+    listQueue,
+    replayEvents,
+    hydrateSession,
+    getProjection,
+    getState,
+    close,
+  };
+}
+
+function rowToEvent(row) {
+  return {
+    id: row.id,
+    ts: row.ts,
+    type: row.type,
+    origin: row.origin ?? 'system',
+    runId: row.run_id ?? null,
+    turnId: row.turn_id ?? null,
+    payload: row.payload ? JSON.parse(row.payload) : {},
+  };
+}

package/src/runtime/store.test.js

@@ -0,0 +1,103 @@
+import assert from 'node:assert/strict';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import test from 'node:test';
+import { createAgentEvent, dispatchAgentEvent } from '../core/agentEvents.js';
+import { openRuntimeStore } from './store.js';
+
+test('runtime store persists and replays agent events into a projection', () => {
+  const stateDir = mkdtempSync(join(tmpdir(), 'wiki-manager-runtime-'));
+  const store = openRuntimeStore({ stateDir });
+  const session = { activities: {}, headlessPlan: null };
+
+  const event = createAgentEvent('plan_set', {
+    origin: 'test',
+    payload: { steps: [{ description: 'Ingest', status: 'running' }] },
+  });
+  store.persistEvent(dispatchAgentEvent(session, event));
+  assert.equal(store.listEvents().length, 1);
+  store.close();
+
+  const reopened = openRuntimeStore({ stateDir });
+  const replayedSession = { activities: {}, headlessPlan: null };
+  reopened.hydrateSession(replayedSession);
+  assert.equal(replayedSession.headlessPlan.length, 1);
+  assert.equal(replayedSession.headlessPlan[0].description, 'Ingest');
+  assert.equal(reopened.getState(replayedSession).eventsCursor, event.id);
+  reopened.close();
+});
+
+test('runtime store persists duplicate events idempotently', () => {
+  const stateDir = mkdtempSync(join(tmpdir(), 'wiki-manager-runtime-'));
+  const store = openRuntimeStore({ stateDir });
+  const event = createAgentEvent('run_started', {
+    origin: 'test',
+    runId: 'run-1',
+    payload: { input: 'build wiki' },
+  });
+
+  store.persistEvent(event);
+  store.persistEvent(event);
+
+  assert.equal(store.listEvents().length, 1);
+  assert.equal(store.listRuns()[0].id, 'run-1');
+  assert.equal(store.listRuns()[0].status, 'running');
+  store.close();
+});
+
+test('runtime store persists cancelled run status', () => {
+  const stateDir = mkdtempSync(join(tmpdir(), 'wiki-manager-runtime-'));
+  const store = openRuntimeStore({ stateDir });
+  store.persistEvent(createAgentEvent('run_started', {
+    origin: 'test',
+    runId: 'run-1',
+    payload: { input: 'build wiki' },
+  }));
+  store.persistEvent(createAgentEvent('run_cancelled', {
+    origin: 'test',
+    runId: 'run-1',
+    payload: { runId: 'run-1' },
+  }));
+
+  assert.equal(store.listRuns()[0].status, 'cancelled');
+  store.close();
+});
+
+test('runtime store does not persist runtime logs', () => {
+  const stateDir = mkdtempSync(join(tmpdir(), 'wiki-manager-runtime-'));
+  const store = openRuntimeStore({ stateDir });
+  store.persistEvent(createAgentEvent('runtime_log', {
+    origin: 'runtime',
+    payload: { message: 'agentic-loop: turn 1/20' },
+  }));
+
+  assert.equal(store.listEvents().length, 0);
+  store.close();
+});
+
+test('runtime store persists and hydrates queue items', () => {
+  const stateDir = mkdtempSync(join(tmpdir(), 'wiki-manager-runtime-'));
+  const store = openRuntimeStore({ stateDir });
+  store.saveQueue([{
+    id: 'q-1',
+    workspace: 'docs',
+    server: 'production',
+    tool: 'production_start_job',
+    args: { type: 'build', steps: ['ingest'] },
+    lockKey: 'production:docs',
+    status: 'waiting',
+    reason: 'workspace_busy',
+    createdAt: '2026-01-01T00:00:00.000Z',
+  }]);
+  store.close();
+
+  const reopened = openRuntimeStore({ stateDir });
+  const session = { activities: {}, headlessPlan: null };
+  reopened.hydrateSession(session);
+  assert.equal(session.jobQueue.length, 1);
+  assert.equal(session.jobQueue[0].id, 'q-1');
+  assert.equal(session.jobQueue[0].args.type, 'build');
+  assert.equal(reopened.getState(session).queue[0].workspace, 'docs');
+  reopened.close();
+});

package/src/runtime/supervisor.js

@@ -0,0 +1,98 @@
+import { createAgentEvent, dispatchAgentEvent } from '../core/agentEvents.js';
+import { extractActivity, parseJsonText, sessionActivities } from '../core/activity.js';
+import { callMcpTool, formatMcpToolResult } from '../core/mcp.js';
+import { startNextQueuedJob, syncQueueWithActivity } from '../core/jobQueue.js';
+
+export function startActivitySupervisor(session, {
+  intervalMs = 1000,
+  queueIntervalMs = 10000,
+  callTool = callMcpTool,
+} = {}) {
+  const pollBusy = new Set();
+  let stopped = false;
+  let runSignal = null;
+
+  const pollTimer = setInterval(() => {
+    void pollActivitiesOnce(session, { pollBusy, callTool, signal: runSignal });
+  }, intervalMs);
+
+  const queueTimer = setInterval(() => {
+    if (session.jobQueue?.some((item) => item.status === 'waiting')) {
+      void startNextQueuedJob(session, {
+        addLog: (message) => emitRuntimeLog(session, message),
+      });
+    }
+  }, queueIntervalMs);
+
+  void pollActivitiesOnce(session, { pollBusy, callTool, signal: runSignal });
+
+  return {
+    pollBusy,
+    setRunSignal(signal) {
+      runSignal = signal ?? null;
+    },
+    stop() {
+      if (stopped) return;
+      stopped = true;
+      clearInterval(pollTimer);
+      clearInterval(queueTimer);
+      pollBusy.clear();
+    },
+  };
+}
+
+export async function pollActivitiesOnce(session, {
+  pollBusy = new Set(),
+  callTool = callMcpTool,
+  signal = null,
+} = {}) {
+  for (const activity of sessionActivities(session)) {
+    if (signal?.aborted) break;
+    if (activity.terminal || !activity.poll) continue;
+    const key = activity.key ?? `${activity.poll.server}:${activity.id ?? activity.label}`;
+    if (pollBusy.has(key)) continue;
+    const endpoint = session.mcp?.[activity.poll.server];
+    if (!endpoint || endpoint.status !== 'connected') {
+      emitRuntimeLog(session, `activity: MCP server '${activity.poll.server}' not connected, cannot poll ${key}`);
+      continue;
+    }
+    const intervalMs = activity.poll.intervalMs ?? 2500;
+    const lastPolledAt = Date.parse(activity.lastPolledAt ?? '0');
+    if (Date.now() - lastPolledAt < intervalMs) continue;
+
+    pollBusy.add(key);
+    activity.lastPolledAt = new Date().toISOString();
+    try {
+      const result = await callTool(session.mcp, activity.poll.server, activity.poll.tool, activity.poll.args ?? {}, signal);
+      const payload = parseJsonText(formatMcpToolResult(result));
+      const polledActivity = extractActivity(payload, {
+        server: activity.poll.server,
+        tool: activity.poll.tool,
+      });
+      if (polledActivity) {
+        dispatchAgentEvent(session, createAgentEvent('activity_upserted', {
+          origin: 'runtime_poll',
+          payload: { activity: polledActivity },
+        }));
+        syncQueueWithActivity(session, polledActivity);
+        emitRuntimeLog(session, `activity: ${polledActivity.label} -> ${polledActivity.status}`);
+        if (polledActivity.terminal) {
+          await startNextQueuedJob(session, {
+            addLog: (message) => emitRuntimeLog(session, message),
+          });
+        }
+      }
+    } catch (err) {
+      emitRuntimeLog(session, `activity poll error ${key}: ${err instanceof Error ? err.message : String(err)}`);
+    } finally {
+      pollBusy.delete(key);
+    }
+  }
+}
+
+export function emitRuntimeLog(session, message) {
+  dispatchAgentEvent(session, createAgentEvent('runtime_log', {
+    origin: 'runtime',
+    payload: { message },
+  }));
+}

package/src/runtime/supervisor.test.js

@@ -0,0 +1,99 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { createAgentEvent, dispatchAgentEvent } from '../core/agentEvents.js';
+import { pollActivitiesOnce, startActivitySupervisor } from './supervisor.js';
+
+test('pollActivitiesOnce updates activity through the event reducer', async () => {
+  const session = {
+    mcp: { production: { status: 'connected' } },
+    activities: {},
+    headlessPlan: null,
+    jobQueue: [],
+  };
+  dispatchAgentEvent(session, createAgentEvent('activity_upserted', {
+    payload: {
+      activity: {
+        id: 'job-1',
+        source: 'production',
+        status: 'running',
+        poll: { server: 'production', tool: 'production_job_status', args: { jobId: 'job-1' }, intervalMs: 1000 },
+      },
+    },
+  }));
+  const key = Object.keys(session.activities)[0];
+  session.activities[key].lastPolledAt = '1970-01-01T00:00:00.000Z';
+
+  await pollActivitiesOnce(session, {
+    callTool: async () => ({
+      content: [{ type: 'text', text: JSON.stringify({
+        _activity: {
+          id: 'job-1',
+          source: 'production',
+          status: 'done',
+          terminal: true,
+        },
+      }) }],
+    }),
+  });
+
+  assert.equal(session.activities[key].status, 'done');
+  assert.equal(session.activities[key].terminal, true);
+  assert.ok(session.agentProjection.logs.some((line) => line.includes('activity:')));
+});
+
+test('startActivitySupervisor passes the active run signal to background polls', async () => {
+  const session = {
+    mcp: { production: { status: 'connected' } },
+    activities: {},
+    headlessPlan: null,
+    jobQueue: [],
+  };
+  dispatchAgentEvent(session, createAgentEvent('activity_upserted', {
+    payload: {
+      activity: {
+        id: 'job-2',
+        source: 'production',
+        status: 'running',
+        poll: { server: 'production', tool: 'production_job_status', args: { jobId: 'job-2' }, intervalMs: 0 },
+      },
+    },
+  }));
+
+  const controller = new AbortController();
+  const signals = [];
+  const supervisor = startActivitySupervisor(session, {
+    intervalMs: 10,
+    queueIntervalMs: 1000,
+    callTool: async (_mcp, _server, _tool, _args, signal) => {
+      signals.push(signal ?? null);
+      return {
+        content: [{ type: 'text', text: JSON.stringify({
+          _activity: {
+            id: 'job-2',
+            source: 'production',
+            status: 'running',
+            terminal: false,
+            poll: { server: 'production', tool: 'production_job_status', args: { jobId: 'job-2' }, intervalMs: 0 },
+          },
+        }) }],
+      };
+    },
+  });
+
+  try {
+    supervisor.setRunSignal(controller.signal);
+    await waitFor(() => signals.includes(controller.signal));
+    assert.ok(signals.includes(controller.signal));
+  } finally {
+    supervisor.stop();
+  }
+});
+
+async function waitFor(predicate, timeoutMs = 250) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (predicate()) return;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+  assert.fail('Timed out waiting for condition.');
+}