mixdog 1.2.0 → 1.2.1

package/src/runtime/agent/orchestrator/session/catalog-db/index.mjs

@@ -0,0 +1,284 @@
+/**
+ * Session catalog DB, phase 2: in the daemon, SQLite is the read source of
+ * the ordinary session catalog.
+ *
+ * Everything runs on one worker thread (./worker.mjs): the catalog engine's
+ * change detection (summary index, sessions directory, per-file stamps, `.hb`
+ * liveness, background verification) feeds `<dataDir>/session-catalog.sqlite`
+ * and an ordinary listing is one request that answers from it. This thread
+ * does no filesystem work for it and never loads node:sqlite.
+ *
+ * JSON stays what every other process reads and writes: the store keeps
+ * writing the summary index exactly as before. When one of the daemon's own
+ * writes has changed a session file on disk, the store hints the id here
+ * (hintSessionCatalogStorage); the worker re-checks it from storage on the
+ * next listing. Hints carry no row data. Whenever the DB cannot answer — not
+ * the daemon, not Electron, SQLite gate, another process holds the DB,
+ * off-switch, backfill or rebuild in progress, any error — `list()` resolves
+ * null and the caller uses the JSON path unchanged.
+ */
+import { isMainThread } from 'node:worker_threads';
+import { createWorkerRequestClient } from '../../../../shared/worker-requests.mjs';
+import { dataDir as currentDataDir } from '../store-summary-locations.mjs';
+
+export const SESSION_CATALOG_DB_FILE = 'session-catalog.sqlite';
+/** Config: `{ "sessionStore": { "sqliteCatalog": false } }` forces the JSON
+ *  path (an off-switch); absent or true uses the DB whenever it can. */
+export const SESSION_STORE_CONFIG_SECTION = 'sessionStore';
+export const SESSION_STORE_CONFIG_DEFAULTS = Object.freeze({ sqliteCatalog: true });
+/** Stable prefix of every daemon.log line of the catalog DB. */
+export const SESSION_CATALOG_LOG_PREFIX = '[session-catalog-db]';
+
+// A worker that keeps dying is given up on for a while.
+const CRASH_WINDOW_MS = 10 * 60_000;
+const MAX_CRASHES = 3;
+const ERROR_LOG_INTERVAL_MS = 60_000;
+
+let testOptions = null;
+let requestWorker = null;
+let controller = null;
+const changeListeners = new Set();
+const noop = () => {};
+
+function log(line) {
+  const text = `${SESSION_CATALOG_LOG_PREFIX} ${line}`;
+  if (typeof testOptions?.log === 'function') {
+    testOptions.log(text);
+    return;
+  }
+  // The daemon's stderr is daemon.log (mirrored by the spawner before ready,
+  // redirected into the file after).
+  try {
+    process.stderr.write(`${text}\n`);
+  } catch {
+    /* logging is best effort */
+  }
+}
+
+function onWorkerEvent(event) {
+  if (event?.type === 'log') log(event.line);
+  else if (event?.type === 'changed') {
+    for (const listener of changeListeners) {
+      try {
+        listener();
+      } catch {
+        /* a listener's failure is its own */
+      }
+    }
+  }
+}
+
+function request(payload) {
+  requestWorker ||= createWorkerRequestClient(new URL('./worker.mjs', import.meta.url), {
+    // The worker is the only node:sqlite user; its experimental warning is noise.
+    execArgv: ['--disable-warning=ExperimentalWarning'],
+    onEvent: onWorkerEvent,
+  });
+  return requestWorker(payload);
+}
+
+/** Worker settings a test configured (functions stay on this thread). */
+function workerSettings() {
+  if (!testOptions) return undefined;
+  const settings = {};
+  for (const [key, value] of Object.entries(testOptions)) {
+    if (typeof value !== 'function' && key !== 'writer') settings[key] = value;
+  }
+  return settings;
+}
+
+/**
+ * True only in the daemon process itself, under the packaged or dev Electron
+ * runtime: daemon.mjs sets MIXDOG_DAEMON_HOST and MIXDOG_SERVER_PID (its own
+ * pid) at startup; children inherit the environment but have another pid,
+ * worker threads are not the main thread, and a daemon started by the CLI
+ * under system Node stays on JSON.
+ */
+export function isSessionCatalogDbWriter() {
+  if (testOptions && 'writer' in testOptions) return testOptions.writer === true;
+  return (
+    isMainThread &&
+    Boolean(process.versions.electron) &&
+    process.env.MIXDOG_DAEMON_HOST === '1' &&
+    process.env.MIXDOG_SERVER_PID === String(process.pid)
+  );
+}
+
+/** The catalog DB of `dataDir`, or null when this process does not keep one. */
+export function sessionCatalogDb(dataDir) {
+  if (!isSessionCatalogDbWriter()) return null;
+  if (controller?.dataDir === dataDir) return controller;
+  void controller?.close();
+  controller = new SessionCatalogDb(dataDir);
+  return controller;
+}
+
+/** The existing catalog DB of `dataDir` (never creates one). */
+export function currentSessionCatalogDb(dataDir) {
+  return controller?.dataDir === dataDir ? controller : null;
+}
+
+/** `listener()` runs when the worker's background pass finds a changed
+ *  session file (an ordinary listing would now differ). */
+export function onSessionCatalogDbChange(listener) {
+  changeListeners.add(listener);
+  return () => changeListeners.delete(listener);
+}
+
+/**
+ * Storage hint from the store (store/summary-cache.mjs): a write of this
+ * process changed the files of `ids` — a save landed, a delete removed the
+ * file. Tagged with the current data dir; the worker re-checks those ids from
+ * storage on the next listing. A lost hint only delays the change to the
+ * engine's own detection.
+ */
+export function hintSessionCatalogStorage(ids) {
+  currentSessionCatalogDb(currentDataDir())?.hint(ids);
+}
+
+/** Closes the catalog DB (releasing its file, WAL and lock) — the teardown
+ *  boundary before a data dir is removed or switched. */
+export async function closeSessionCatalogDb() {
+  const current = controller;
+  controller = null;
+  if (current) await current.close();
+}
+
+class SessionCatalogDb {
+  constructor(dataDir) {
+    this.dataDir = dataDir;
+    this.state = 'active'; // active | off (final: gate) | failed (until failedUntil) | closed
+    this.failedUntil = 0;
+    this.hints = new Set();
+    this.flushHandle = null;
+    this.started = false;
+    this.crashes = [];
+    this.errorLoggedAt = 0;
+  }
+
+  send(op, payload = {}) {
+    // No running worker: this request starts one. After the first start it
+    // is a respawn (the last one died): it reopens and backfills by itself.
+    if (!requestWorker?.running()) {
+      if (this.started) this.noteRespawn();
+      this.started = true;
+    }
+    return request({ op, dataDir: this.dataDir, settings: workerSettings(), ...payload }).catch((error) => {
+      this.noteError(error);
+      throw error;
+    });
+  }
+
+  /** A worker died; more than MAX_CRASHES deaths in the window give up. */
+  noteRespawn() {
+    const now = Date.now();
+    this.crashes = this.crashes.filter((at) => now - at < CRASH_WINDOW_MS);
+    this.crashes.push(now);
+    if (this.crashes.length > MAX_CRASHES && this.state === 'active') {
+      this.state = 'failed';
+      this.failedUntil = now + CRASH_WINDOW_MS;
+      log(`worker failed ${this.crashes.length} times in 10 min; catalog stays JSON`);
+    }
+  }
+
+  noteError(error) {
+    const now = Date.now();
+    if (now - this.errorLoggedAt >= ERROR_LOG_INTERVAL_MS) {
+      this.errorLoggedAt = now;
+      log(`request failed: ${error?.message || error}; listing answered from JSON`);
+    }
+  }
+
+  usable() {
+    if (this.state === 'failed' && Date.now() >= this.failedUntil) {
+      this.state = 'active';
+      this.crashes = [];
+    }
+    return this.state === 'active';
+  }
+
+  hint(ids) {
+    if (this.state !== 'active') return;
+    for (const id of ids) if (id) this.hints.add(String(id));
+    if (this.hints.size === 0) return;
+    this.flushHandle ||= setImmediate(() => {
+      this.flushHandle = null;
+      this.flushHints();
+    });
+  }
+
+  /** Posts the collected hints ahead of any later request. A worker that is
+   *  not running gets none: a (re)started worker rescans storage anyway. */
+  flushHints() {
+    if (this.hints.size === 0) return;
+    const ids = [...this.hints];
+    this.hints.clear();
+    if (!requestWorker?.running()) return;
+    this.send('hint', { ids }).catch(noop);
+  }
+
+  /**
+   * One ordinary listing (`page` = `{ limit, after }` or null for all):
+   * `{ rows, more }`, or null when the DB cannot answer (use JSON). Hints
+   * collected before this call are posted first.
+   */
+  async list(page = null) {
+    if (!this.usable()) return null;
+    this.flushHints();
+    let answer;
+    try {
+      answer = await this.send('list', { page });
+    } catch {
+      return null;
+    }
+    if (answer.status === 'ok') return { rows: answer.rows, more: answer.more };
+    if (answer.final && this.state === 'active') this.state = 'off';
+    return null;
+  }
+
+  verifyFiles() {
+    if (!this.usable()) return Promise.resolve();
+    return this.send('verifyFiles').catch(noop);
+  }
+
+  /** Test/benchmark seams. */
+  status() {
+    return this.send('status');
+  }
+  shadow() {
+    return this.send('shadow');
+  }
+  settled() {
+    this.flushHints();
+    return this.send('settle');
+  }
+  testExec(sql, params = []) {
+    return this.send('testExec', { sql, params });
+  }
+
+  async close() {
+    if (this.state === 'closed') return;
+    this.state = 'closed';
+    if (this.flushHandle) clearImmediate(this.flushHandle);
+    this.flushHandle = null;
+    this.hints.clear();
+    // Only a running worker can hold the DB open.
+    if (requestWorker?.running()) {
+      await request({ op: 'close', dataDir: this.dataDir }).catch(noop);
+    }
+  }
+}
+
+// ── Test seams ──────────────────────────────────────────────────────────────
+/** Test seam: `{ writer, enabled, sqliteVersion, backfillBatch,
+ *  shadowIntervalMs, retryMs, fenceTimeoutMs, crashAfterBatches, log }`;
+ *  null restores defaults. */
+export function configureSessionCatalogDbForTest(options) {
+  testOptions = options ? { ...options } : null;
+}
+
+/** Test seam: post a storage hint tagged with `dataDir` (another data dir's
+ *  hint must be ignored by the worker). */
+export function postSessionCatalogHintForTest(dataDir, ids) {
+  return request({ op: 'hint', dataDir, ids });
+}

package/src/runtime/agent/orchestrator/session/catalog-db/schema.mjs

@@ -0,0 +1,268 @@
+/**
+ * Session catalog DB: table layout, row codec and the listing comparison.
+ * Pure (no IO, no node:sqlite) so both the worker and tests can use it.
+ *
+ * One row per catalog session: every field store-summary-reader.mjs
+ * normalizedRow produces, the storage fingerprint (full stamp of the session
+ * file the row was judged at), the title normalized at write, and the
+ * bookkeeping the mirror needs (lead visibility, sync epoch, row signature).
+ */
+import { createHash } from 'node:crypto';
+
+/** Bumped whenever the table layout or a codec changes: the DB is then
+ *  dropped and backfilled again from the JSON catalog. */
+export const SCHEMA_VERSION = 3;
+/** PRAGMA application_id ('MXSC'): a file without it is not this catalog. */
+export const APPLICATION_ID = 0x4d585343;
+
+// [row key, column, codec]
+//   real/int  numbers; bool 0/1; text strings
+//   loose     whatever the source carried (string/number/null as is, anything
+//             else as a JSON BLOB), so the row round-trips exactly
+//   json      object or null, as JSON text
+//   optjson   like json, but the key is absent from the row when NULL
+const FIELDS = [
+  ['id', 'id', 'text'],
+  ['updatedAt', 'updated_at', 'real'],
+  ['lastUsedAt', 'last_used_at', 'real'],
+  ['createdAt', 'created_at', 'real'],
+  ['lastHeartbeatAt', 'last_heartbeat_at', 'real'],
+  // Always 0 in a stored row: liveness is overlaid from `.hb` sidecars at read.
+  ['heartbeatAt', 'heartbeat_at', 'real'],
+  ['closed', 'closed', 'bool'],
+  ['status', 'status', 'text'],
+  ['owner', 'owner', 'loose'],
+  ['agent', 'agent', 'loose'],
+  ['sourceType', 'source_type', 'loose'],
+  ['sourceName', 'source_name', 'loose'],
+  ['scopeKey', 'scope_key', 'loose'],
+  ['ownerSessionId', 'owner_session_id', 'loose'],
+  ['visibility', 'visibility', 'loose'],
+  ['clientHostPid', 'client_host_pid', 'loose'],
+  ['cwd', 'cwd', 'loose'],
+  ['desktopSession', 'desktop_session', 'json'],
+  ['workspace', 'workspace', 'optjson'],
+  ['provider', 'provider', 'loose'],
+  ['model', 'model', 'loose'],
+  ['agentTag', 'agent_tag', 'loose'],
+  ['task_id', 'task_id', 'loose'],
+  ['permission', 'permission', 'loose'],
+  ['toolPermission', 'tool_permission', 'loose'],
+  ['messageCount', 'message_count', 'int'],
+  ['title', 'title', 'text'],
+  ['preview', 'preview', 'text'],
+  ['generation', 'generation', 'loose'],
+  ['storageMtimeMs', 'storage_mtime_ms', 'real'],
+  ['storageSize', 'storage_size', 'real'],
+  ['detachedReason', 'detached_reason', 'loose'],
+  // importOrigin.key of an imported conversation (schema 3).
+  ['importKey', 'import_key', 'loose'],
+];
+
+const COLUMN_TYPES = { text: 'TEXT', real: 'REAL', int: 'INTEGER', bool: 'INTEGER', json: 'TEXT', optjson: 'TEXT' };
+
+const columnDefinition = ([, column, codec]) => {
+  if (column === 'id') return 'id TEXT PRIMARY KEY';
+  // No declared type = no affinity: loose values are stored exactly as bound.
+  return codec === 'loose' ? column : `${column} ${COLUMN_TYPES[codec]}`;
+};
+
+export const SCHEMA_SQL = `
+  CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
+  CREATE TABLE IF NOT EXISTS sessions (
+    ${FIELDS.map(columnDefinition).join(',\n    ')},
+    activity REAL NOT NULL,
+    title_norm TEXT NOT NULL,
+    fingerprint TEXT,
+    lead_visible INTEGER NOT NULL,
+    row_sig TEXT NOT NULL
+  );
+  CREATE INDEX IF NOT EXISTS sessions_recent ON sessions(activity DESC, id);
+  CREATE INDEX IF NOT EXISTS sessions_lead_recent ON sessions(lead_visible, activity DESC, id);
+  CREATE INDEX IF NOT EXISTS sessions_visibility_recent ON sessions(visibility, activity DESC, id);
+  CREATE INDEX IF NOT EXISTS sessions_owner_session ON sessions(owner_session_id);
+`;
+
+const WRITE_COLUMNS = [
+  ...FIELDS.map(([, column]) => column),
+  'activity',
+  'title_norm',
+  'fingerprint',
+  'lead_visible',
+  'row_sig',
+];
+
+export const UPSERT_SQL = `INSERT INTO sessions(${WRITE_COLUMNS.join(',')})
+  VALUES (${WRITE_COLUMNS.map(() => '?').join(',')})
+  ON CONFLICT(id) DO UPDATE SET ${WRITE_COLUMNS.slice(1)
+    .map((column) => `${column}=excluded.${column}`)
+    .join(',')}`;
+
+/** Recent-activity key of the JSON listing (store-summary-reader byRecentActivity). */
+export function activityOf(row) {
+  return row.lastUsedAt || row.updatedAt || 0;
+}
+
+/** Newest activity first, ties by id: the DB order and the keyset order.
+ *  Ids are ASCII ([A-Za-z0-9_-]), so code-unit order equals SQLite BINARY. */
+export function byActivityThenId(left, right) {
+  const delta = activityOf(right) - activityOf(left);
+  if (delta !== 0) return delta;
+  if (left.id === right.id) return 0;
+  return left.id < right.id ? -1 : 1;
+}
+
+/** Title normalized once at write: what a title lookup or sort compares. */
+export function normalizedTitle(row) {
+  return String(row.title || row.preview || '')
+    .normalize('NFKC')
+    .toLowerCase()
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function encodeLoose(value) {
+  if (value === null || value === undefined) return null;
+  if (typeof value === 'string' || typeof value === 'number') return value;
+  return Buffer.from(JSON.stringify(value), 'utf8');
+}
+
+function decodeLoose(value) {
+  if (value instanceof Uint8Array) return JSON.parse(Buffer.from(value).toString('utf8'));
+  return value;
+}
+
+function encodeField(codec, value) {
+  switch (codec) {
+    case 'bool':
+      return value === true ? 1 : 0;
+    case 'json':
+    case 'optjson':
+      return value == null ? null : JSON.stringify(value);
+    case 'loose':
+      return encodeLoose(value);
+    default:
+      return value;
+  }
+}
+
+function decodeField(codec, value) {
+  switch (codec) {
+    case 'bool':
+      return value === 1;
+    case 'json':
+    case 'optjson':
+      return value == null ? null : JSON.parse(value);
+    case 'loose':
+      return decodeLoose(value);
+    default:
+      return value;
+  }
+}
+
+/** Signature of what one mirror entry writes: equal signature, no write. */
+export function rowSignature({ row, fingerprint, leadVisible }) {
+  return createHash('sha1')
+    .update(JSON.stringify([fingerprint ?? null, leadVisible === true, row]))
+    .digest('base64');
+}
+
+/** Bind parameters of UPSERT_SQL for one mirror entry. */
+export function upsertParams(entry, signature) {
+  const { row } = entry;
+  return [
+    ...FIELDS.map(([key, , codec]) => encodeField(codec, row[key])),
+    activityOf(row),
+    normalizedTitle(row),
+    entry.fingerprint ?? null,
+    entry.leadVisible === true ? 1 : 0,
+    signature,
+  ];
+}
+
+/** A `sessions` row back in normalizedRow shape. */
+export function rowFromDb(record) {
+  const row = {};
+  for (const [key, column, codec] of FIELDS) {
+    const value = decodeField(codec, record[column]);
+    if (codec === 'optjson' && value === null) continue;
+    row[key] = value;
+  }
+  return row;
+}
+
+// Overlays a listing adds at read time; never part of the comparison.
+const OVERLAY_KEYS = new Set(['heartbeatAt', 'agentHeartbeatAt']);
+
+/** Comparable JSON of a row's stored fields (overlays excluded). */
+function comparable(row) {
+  const out = [];
+  for (const [key] of FIELDS) {
+    if (OVERLAY_KEYS.has(key)) continue;
+    out.push(row[key] === undefined ? null : row[key]);
+  }
+  return JSON.stringify(out);
+}
+
+const FIRST_IDS = 5;
+
+/**
+ * Shadow comparison of the DB listing against the JSON listing, both in the
+ * one catalog order (byActivityThenId) and compared exactly as given: ids
+ * present in only one, positions of the common ids, and stored row fields.
+ */
+export function compareListings(json, dbRows) {
+  const jsonById = new Map(json.map((row) => [row.id, row]));
+  const dbById = new Map(dbRows.map((row) => [row.id, row]));
+  const missing = json.filter((row) => !dbById.has(row.id)).map((row) => row.id);
+  const extra = dbRows.filter((row) => !jsonById.has(row.id)).map((row) => row.id);
+  const commonJson = json.filter((row) => dbById.has(row.id));
+  const commonDb = dbRows.filter((row) => jsonById.has(row.id));
+  const order = [];
+  for (let index = 0; index < commonJson.length; index++) {
+    if (commonJson[index].id !== commonDb[index].id) order.push(commonJson[index].id);
+  }
+  const fields = commonJson.filter((row) => comparable(row) !== comparable(dbById.get(row.id))).map((row) => row.id);
+  return {
+    json: json.length,
+    db: dbRows.length,
+    missing: missing.length,
+    extra: extra.length,
+    order: order.length,
+    fields: fields.length,
+    ids: {
+      missing: missing.slice(0, FIRST_IDS),
+      extra: extra.slice(0, FIRST_IDS),
+      order: order.slice(0, FIRST_IDS),
+      fields: fields.slice(0, FIRST_IDS),
+    },
+  };
+}
+
+// ── Keyset cursor (activity time, id) ──────────────────────────────────────
+/** Opaque cursor naming the last row of a page. */
+export function encodeCatalogCursor(row) {
+  return Buffer.from(JSON.stringify([activityOf(row), row.id]), 'utf8').toString('base64url');
+}
+
+/** `{ activity, id }` of a cursor, null for none; throws on a malformed one. */
+export function decodeCatalogCursor(cursor) {
+  if (cursor === undefined || cursor === null || cursor === '') return null;
+  let parsed;
+  try {
+    parsed = JSON.parse(Buffer.from(String(cursor), 'base64url').toString('utf8'));
+  } catch {
+    parsed = null;
+  }
+  if (!Array.isArray(parsed) || !Number.isFinite(parsed[0]) || typeof parsed[1] !== 'string') {
+    throw Object.assign(new Error('invalid session catalog cursor'), { code: 'EINVALIDCURSOR' });
+  }
+  return { activity: parsed[0], id: parsed[1] };
+}
+
+/** Whether `row` sorts strictly after the cursor position. */
+export function isAfterCursor(row, after) {
+  if (!after) return true;
+  const activity = activityOf(row);
+  return activity < after.activity || (activity === after.activity && row.id > after.id);
+}

package/src/runtime/agent/orchestrator/session/catalog-db/sqlite-gate.mjs

@@ -0,0 +1,27 @@
+/**
+ * SQLite runtime gate of the session catalog DB. SQLite 3.7.0–3.51.2 can
+ * corrupt a WAL database when the WAL is reset while another connection
+ * writes (fixed in 3.51.3, backported to 3.50.7). The catalog opens its file
+ * only on a library with that fix; any other runtime keeps the JSON catalog.
+ */
+function parts(version) {
+  const match = /^(\d+)\.(\d+)\.(\d+)/.exec(String(version || ''));
+  return match ? match.slice(1, 4).map(Number) : null;
+}
+
+function atLeast(version, minimum) {
+  for (let index = 0; index < 3; index++) {
+    if (version[index] !== minimum[index]) return version[index] > minimum[index];
+  }
+  return true;
+}
+
+/** True for SQLite ≥ 3.51.3, or 3.50.x ≥ 3.50.7. */
+export function sqliteVersionSupported(version) {
+  const value = parts(version);
+  if (!value) return false;
+  if (atLeast(value, [3, 51, 3])) return true;
+  return value[0] === 3 && value[1] === 50 && value[2] >= 7;
+}
+
+export const SQLITE_REQUIREMENT = '>= 3.51.3 or 3.50.7+';