@dotdotgod/context 0.2.25 → 0.2.27

package/src/store.mjs

@@ -1,4 +1,4 @@
-import { mkdirSync, rmSync } from 'node:fs';
+import { copyFileSync, existsSync, mkdirSync, renameSync, rmSync, statSync } from 'node:fs';
 import { dirname, join, resolve } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 import { extname } from 'node:path';
@@ -20,40 +20,174 @@ export function contextDbPath(root) {
   return join(safeRoot(root), '.dotdotgod', 'context', 'context.sqlite');
 }
 
+const SCHEMA_VERSION = 3;
+const MAX_PENDING_JOBS = 100;
+
+function inspectSchema(db) {
+  const objects = new Map(db.prepare("SELECT name, type, sql FROM sqlite_master WHERE name IN ('sources', 'chunks')").all().map((row) => [row.name, row]));
+  const sourceRows = db.prepare('PRAGMA table_info(sources)').all();
+  const chunkRows = db.prepare('PRAGMA table_info(chunks)').all();
+  const requiredSources = ['id', 'scope', 'session_id', 'label', 'kind', 'metadata', 'created_at', 'expires_at'];
+  const requiredChunks = ['source_id', 'scope', 'session_id', 'ordinal', 'body'];
+  const missingSources = requiredSources.filter((name) => !sourceRows.some((row) => row.name === name));
+  const missingChunks = requiredChunks.filter((name) => !chunkRows.some((row) => row.name === name));
+  const sourceShape = sourceRows.length === requiredSources.length && sourceRows.every((row, index) => row.name === requiredSources[index])
+    && sourceRows[0].pk === 1
+    && sourceRows.every((row) => ['scope', 'label', 'kind', 'metadata', 'created_at'].includes(row.name) ? row.notnull === 1 : true);
+  const chunkShape = chunkRows.length === requiredChunks.length && chunkRows.every((row, index) => row.name === requiredChunks[index]);
+  const chunkSql = String(objects.get('chunks')?.sql ?? '').toLowerCase();
+  const compatible = objects.get('sources')?.type === 'table'
+    && objects.get('chunks')?.type === 'table'
+    && chunkSql.includes('using fts5')
+    && chunkSql.includes("tokenize='porter unicode61'")
+    && sourceShape
+    && chunkShape
+    && missingSources.length === 0
+    && missingChunks.length === 0;
+  let version = 0;
+  if (compatible) {
+    const ledger = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'schema_migrations'").get();
+    if (ledger) version = Number(db.prepare('SELECT coalesce(max(version), 0) AS version FROM schema_migrations').get().version);
+    else version = 1;
+  }
+  return { compatible: compatible && version <= SCHEMA_VERSION, schemaProfile: compatible ? version : null, missing: [...missingSources, ...missingChunks] };
+}
+
+function migrate(db, fromVersion) {
+  if (fromVersion > SCHEMA_VERSION) throw new Error(`Incompatible context database schema version ${fromVersion}; runtime supports ${SCHEMA_VERSION}.`);
+  db.exec('BEGIN IMMEDIATE');
+  try {
+    db.exec('CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL);');
+    if (fromVersion < 2) {
+      db.exec(`CREATE TABLE IF NOT EXISTS ingestion_jobs (
+        id TEXT PRIMARY KEY, state TEXT NOT NULL CHECK(state IN ('queued','running','completed','failed','cancelled')),
+        kind TEXT NOT NULL, input TEXT NOT NULL, result TEXT, error TEXT,
+        created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL
+      ); CREATE INDEX IF NOT EXISTS ingestion_jobs_state_created ON ingestion_jobs(state, created_at);`);
+      db.prepare('INSERT OR IGNORE INTO schema_migrations(version, applied_at) VALUES (1, ?)').run(Date.now());
+      db.prepare('INSERT OR IGNORE INTO schema_migrations(version, applied_at) VALUES (2, ?)').run(Date.now());
+    }
+    if (fromVersion < 3) {
+      const columns = db.prepare('PRAGMA table_info(ingestion_jobs)').all();
+      if (!columns.some((row) => row.name === 'session_id')) db.exec('ALTER TABLE ingestion_jobs ADD COLUMN session_id TEXT;');
+      db.prepare('INSERT OR IGNORE INTO schema_migrations(version, applied_at) VALUES (3, ?)').run(Date.now());
+    }
+    db.exec('COMMIT');
+  } catch (error) { db.exec('ROLLBACK'); throw error; }
+}
+
+function trigrams(value) {
+  const normalized = `  ${String(value).toLocaleLowerCase('en').replace(/[^\p{L}\p{N}]+/gu, ' ').trim()}  `;
+  const out = new Set();
+  for (let index = 0; index + 3 <= normalized.length; index += 1) out.add(normalized.slice(index, index + 3));
+  return out;
+}
+function trigramScore(left, right) {
+  const a = trigrams(left); const b = trigrams(right);
+  if (!a.size || !b.size) return 0;
+  let overlap = 0; for (const item of a) if (b.has(item)) overlap += 1;
+  return (2 * overlap) / (a.size + b.size);
+}
+function editDistance(left, right, ceiling = 2) {
+  if (Math.abs(left.length - right.length) > ceiling) return ceiling + 1;
+  let previous = Array.from({ length: right.length + 1 }, (_, index) => index);
+  for (let i = 1; i <= left.length; i += 1) {
+    const current = [i]; let rowMin = i;
+    for (let j = 1; j <= right.length; j += 1) { current[j] = Math.min(current[j - 1] + 1, previous[j] + 1, previous[j - 1] + (left[i - 1] === right[j - 1] ? 0 : 1)); rowMin = Math.min(rowMin, current[j]); }
+    if (rowMin > ceiling) return ceiling + 1; previous = current;
+  }
+  return previous[right.length];
+}
+function typoScore(query, value) {
+  const queryTerms = normalizeSearchTerms(query);
+  const valueTerms = normalizeSearchTerms(value);
+  let best = 0;
+  for (const needle of queryTerms) for (const token of valueTerms) {
+    if (needle.length >= 5 && editDistance(needle, token) <= 2) best = Math.max(best, trigramScore(needle, token));
+  }
+  return best;
+}
+
+export function healContextDatabase(root = process.cwd()) {
+  const path = contextDbPath(root);
+  if (!existsSync(path)) throw new Error('No context database exists to heal.');
+  const stamp = Date.now();
+  const backupPath = `${path}.backup-${stamp}`;
+  const rebuiltPath = `${path}.healed-${stamp}`;
+  let inspection;
+  let replaced = false;
+  const db = new DatabaseSync(path);
+  try {
+    db.exec('PRAGMA busy_timeout=1000; PRAGMA wal_checkpoint(FULL);');
+    inspection = inspectSchema(db);
+    if (!inspection.compatible || ![1, 2, 3].includes(inspection.schemaProfile)) {
+      throw new Error('Schema is not a recognized recoverable context database; no rebuild performed.');
+    }
+    copyFileSync(path, backupPath, 0);
+    migrate(db, inspection.schemaProfile);
+    db.exec('PRAGMA wal_checkpoint(TRUNCATE);');
+    db.exec(`VACUUM INTO '${rebuiltPath.replaceAll("'", "''")}';`);
+  } catch (error) {
+    rmSync(rebuiltPath, { force: true });
+    throw error;
+  } finally { db.close(); }
+  try {
+    renameSync(rebuiltPath, path);
+    replaced = true;
+    rmSync(`${path}-wal`, { force: true });
+    rmSync(`${path}-shm`, { force: true });
+  } finally {
+    if (!replaced) rmSync(rebuiltPath, { force: true });
+  }
+  return { ok: true, backupPath, fromVersion: inspection.schemaProfile, toVersion: SCHEMA_VERSION, rebuilt: true };
+}
+
 export class ContextStore {
   constructor(root = process.cwd()) {
     this.root = safeRoot(root);
     this.path = contextDbPath(this.root);
+    const existing = existsSync(this.path);
+    let schemaVersion = 1;
+    if (existing) {
+      if (statSync(this.path).size === 0) throw new Error('Incompatible context database schema; empty database cannot be repaired automatically.');
+      const inspection = new DatabaseSync(this.path, { readOnly: true });
+      try {
+        const result = inspectSchema(inspection);
+        if (!result.compatible) throw new Error(`Incompatible context database schema; run doctor before explicit healing${result.missing.length ? ` (missing: ${result.missing.join(', ')})` : ''}.`);
+        schemaVersion = result.schemaProfile;
+      } finally {
+        inspection.close();
+      }
+    }
     mkdirSync(dirname(this.path), { recursive: true });
     this.db = new DatabaseSync(this.path);
-    this.db.exec(`
-      PRAGMA journal_mode=WAL;
-      CREATE TABLE IF NOT EXISTS sources (
-        id TEXT PRIMARY KEY,
-        scope TEXT NOT NULL,
-        session_id TEXT,
-        label TEXT NOT NULL,
-        kind TEXT NOT NULL,
-        metadata TEXT NOT NULL,
-        created_at INTEGER NOT NULL,
-        expires_at INTEGER
-      );
-      CREATE VIRTUAL TABLE IF NOT EXISTS chunks USING fts5(
-        source_id UNINDEXED,
-        scope UNINDEXED,
-        session_id UNINDEXED,
-        ordinal UNINDEXED,
-        body,
-        tokenize='porter unicode61'
-      );
-    `);
-    const sourceColumns = new Set(this.db.prepare('PRAGMA table_info(sources)').all().map((row) => row.name));
-    const chunkColumns = new Set(this.db.prepare('PRAGMA table_info(chunks)').all().map((row) => row.name));
-    const missingSources = ['id', 'scope', 'session_id', 'label', 'kind', 'metadata', 'created_at', 'expires_at'].filter((name) => !sourceColumns.has(name));
-    const missingChunks = ['source_id', 'scope', 'session_id', 'ordinal', 'body'].filter((name) => !chunkColumns.has(name));
-    if (missingSources.length || missingChunks.length) {
+    try {
+      this.db.exec('PRAGMA busy_timeout=1000; PRAGMA journal_mode=WAL;');
+      if (!existing) this.db.exec(`
+        CREATE TABLE sources (
+          id TEXT PRIMARY KEY,
+          scope TEXT NOT NULL,
+          session_id TEXT,
+          label TEXT NOT NULL,
+          kind TEXT NOT NULL,
+          metadata TEXT NOT NULL,
+          created_at INTEGER NOT NULL,
+          expires_at INTEGER
+        );
+        CREATE VIRTUAL TABLE chunks USING fts5(
+          source_id UNINDEXED,
+          scope UNINDEXED,
+          session_id UNINDEXED,
+          ordinal UNINDEXED,
+          body,
+          tokenize='porter unicode61'
+        );
+      `);
+      migrate(this.db, schemaVersion);
+      this.db.prepare("UPDATE ingestion_jobs SET state = 'queued', updated_at = ? WHERE state = 'running'").run(Date.now());
+    } catch (error) {
       this.db.close();
-      throw new Error(`Incompatible context database schema; run doctor before repair (missing: ${[...missingSources, ...missingChunks].join(', ')}).`);
+      throw error;
     }
   }
 
@@ -67,7 +201,7 @@ export class ContextStore {
     const contentType = String(provenance.contentType ?? metadata.contentType ?? '').toLowerCase();
     const extension = extname(String(metadata.path ?? '')).toLowerCase();
     const format = provenance.format ?? (contentType.includes('json') || extension === '.json' ? 'json' : contentType.includes('markdown') || ['.md', '.mdx'].includes(extension) ? 'markdown' : 'text');
-    const extractor = format === 'json' ? 'json-v1' : format === 'markdown' ? 'markdown-v1' : 'plain';
+    const extractor = provenance.extractor ?? (format === 'json' ? 'json-v1' : format === 'markdown' ? 'markdown-v1' : 'plain');
     const normalizedMetadata = createProvenanceMetadata(metadata, { sourceType, origin, content: text, extractor, indexedAt: new Date(createdAt).toISOString() });
     this.db.exec('BEGIN IMMEDIATE');
     try {
@@ -87,10 +221,18 @@ export class ContextStore {
 
   expire() {
     const ids = this.db.prepare('SELECT id FROM sources WHERE expires_at IS NOT NULL AND expires_at <= ?').all(Date.now()).map((row) => row.id);
+    if (ids.length === 0) return 0;
     const removeChunks = this.db.prepare('DELETE FROM chunks WHERE source_id = ?');
     const removeSource = this.db.prepare('DELETE FROM sources WHERE id = ?');
-    for (const id of ids) { removeChunks.run(id); removeSource.run(id); }
-    return ids.length;
+    this.db.exec('BEGIN IMMEDIATE');
+    try {
+      for (const id of ids) { removeChunks.run(id); removeSource.run(id); }
+      this.db.exec('COMMIT');
+      return ids.length;
+    } catch (error) {
+      this.db.exec('ROLLBACK');
+      throw error;
+    }
   }
 
   search({ query, scope, sessionId, source, limit = 5 }) {
@@ -121,6 +263,27 @@ export class ContextStore {
         AND (${labelClauses.join(' OR ')})${filterSql}
       ORDER BY lower(s.label), c.ordinal LIMIT ?
     `).all(...labelParams, ...filterParams, candidateLimit);
+    const typoWindow = Math.min(500, candidateLimit * 5);
+    const typoHalf = Math.max(1, Math.floor(typoWindow / 2));
+    const trigramRows = this.db.prepare(`
+      SELECT * FROM (
+        SELECT ${select.replace('SELECT ', '')}, 0 AS rank
+        FROM chunks c JOIN sources s ON s.id = c.source_id
+        WHERE 1 = 1${filterSql}
+        ORDER BY c.rowid ASC LIMIT ?
+      )
+      UNION ALL
+      SELECT * FROM (
+        SELECT ${select.replace('SELECT ', '')}, 0 AS rank
+        FROM chunks c JOIN sources s ON s.id = c.source_id
+        WHERE 1 = 1${filterSql}
+        ORDER BY c.rowid DESC LIMIT ?
+      )
+    `).all(...filterParams, typoHalf, ...filterParams, typoWindow - typoHalf)
+      .map((row) => ({ ...row, trigramScore: Math.max(typoScore(query, row.body), Number(row.ordinal) === 0 ? typoScore(query, row.label) : 0) }))
+      .filter((row) => row.trigramScore >= 0.5)
+      .sort((left, right) => right.trigramScore - left.trigramScore || String(left.source_id).localeCompare(String(right.source_id), 'en') || Number(left.ordinal) - Number(right.ordinal))
+      .slice(0, candidateLimit);
     const toCandidate = (row) => {
       let storedMetadata = {};
       try { storedMetadata = JSON.parse(row.metadata); } catch { /* legacy invalid metadata */ }
@@ -129,6 +292,7 @@ export class ContextStore {
     const fused = reciprocalRankFusion([
       { name: 'porter-bm25', candidates: porterRows.map(toCandidate) },
       { name: 'label-path', candidates: labelRows.map(toCandidate) },
+      { name: 'trigram-v1', candidates: trigramRows.map(toCandidate) },
     ], { limit: candidateLimit });
     return rerankCandidates(fused, query, { limit: resultLimit }).map((entry) => {
       const row = entry.candidate;
@@ -150,6 +314,53 @@ export class ContextStore {
     });
   }
 
+  createJob({ id = crypto.randomUUID(), kind, input, sessionId = null }) {
+    const now = Date.now();
+    const serializedInput = JSON.stringify(input);
+    if (Buffer.byteLength(serializedInput) > 64 * 1024) throw new Error('Background ingestion job input exceeds 64 KiB.');
+    this.db.exec('BEGIN IMMEDIATE');
+    try {
+      const pending = Number(this.db.prepare("SELECT count(*) AS count FROM ingestion_jobs WHERE state IN ('queued','running')").get().count);
+      if (pending >= MAX_PENDING_JOBS) throw new Error('Background ingestion queue is full.');
+      this.db.prepare('INSERT INTO ingestion_jobs(id, state, kind, input, session_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)')
+        .run(id, 'queued', kind, serializedInput, sessionId, now, now);
+      this.db.exec('COMMIT');
+    } catch (error) { this.db.exec('ROLLBACK'); throw error; }
+    return this.getJob(id);
+  }
+
+  getJob(id) {
+    const row = this.db.prepare('SELECT * FROM ingestion_jobs WHERE id = ?').get(id);
+    if (!row) return null;
+    return { id: row.id, state: row.state, kind: row.kind, input: JSON.parse(row.input), sessionId: row.session_id ?? null, result: row.result ? JSON.parse(row.result) : null, error: row.error, createdAt: row.created_at, updatedAt: row.updated_at };
+  }
+
+  claimNextJob() {
+    const row = this.db.prepare("SELECT id FROM ingestion_jobs WHERE state = 'queued' ORDER BY created_at, id LIMIT 1").get();
+    if (!row) return null;
+    const changed = this.db.prepare("UPDATE ingestion_jobs SET state = 'running', updated_at = ? WHERE id = ? AND state = 'queued'").run(Date.now(), row.id).changes;
+    return changed ? this.getJob(row.id) : null;
+  }
+
+  finishJob(id, state, value = null) {
+    if (!['completed', 'failed', 'cancelled'].includes(state)) throw new Error('Invalid terminal job state.');
+    let result = state === 'completed' ? JSON.stringify(value) : null;
+    if (result && Buffer.byteLength(result) > 256 * 1024) result = JSON.stringify({ truncated: true, message: 'Completed result exceeded the durable status limit.' });
+    const error = state === 'failed' ? String(value ?? 'Job failed').slice(0, 4096) : null;
+    this.db.prepare("UPDATE ingestion_jobs SET state = ?, result = ?, error = ?, updated_at = ? WHERE id = ? AND state IN ('queued','running')")
+      .run(state, result, error, Date.now(), id);
+    return this.getJob(id);
+  }
+
+  cancelJob(id) {
+    const changed = this.db.prepare("UPDATE ingestion_jobs SET state = 'cancelled', updated_at = ? WHERE id = ? AND state IN ('queued','running')").run(Date.now(), id).changes;
+    return { changed: Number(changed), job: this.getJob(id) };
+  }
+
+  listPendingJobs(limit = 20) {
+    return this.db.prepare("SELECT id FROM ingestion_jobs WHERE state IN ('queued','running') ORDER BY created_at, id LIMIT ?").all(Math.min(100, Math.max(1, limit))).map((row) => this.getJob(row.id));
+  }
+
   stats() {
     this.expire();
     const sources = this.db.prepare('SELECT count(*) AS count FROM sources').get().count;
@@ -168,11 +379,18 @@ export class ContextStore {
     const removeChunks = this.db.prepare('DELETE FROM chunks WHERE source_id = ?');
     const removeSource = this.db.prepare('DELETE FROM sources WHERE id = ?');
     let removed = 0;
-    for (const id of ids) {
-      removeChunks.run(id);
-      removed += Number(removeSource.run(id).changes ?? 0);
+    this.db.exec('BEGIN IMMEDIATE');
+    try {
+      for (const id of ids) {
+        removeChunks.run(id);
+        removed += Number(removeSource.run(id).changes ?? 0);
+      }
+      this.db.exec('COMMIT');
+      return { removed };
+    } catch (error) {
+      this.db.exec('ROLLBACK');
+      throw error;
     }
-    return { removed };
   }
 
   destroy() {

package/src/tool-contracts.mjs

@@ -0,0 +1,13 @@
+const object = (properties, required = []) => Object.freeze({ type: 'object', properties, required, additionalProperties: false, $schema: 'http://json-schema.org/draft-07/schema#' });
+const string = (options = {}) => Object.freeze({ type: 'string', ...options });
+
+export const PHASE3_TOOL_INPUT_SCHEMAS = Object.freeze({
+  session_resume: object({ sessionId: string({ minLength: 1, maxLength: 128, pattern: '^[A-Za-z0-9][A-Za-z0-9._~-]{0,127}$' }) }, ['sessionId']),
+  ingestion_job_start: object({
+    kind: Object.freeze({ type: 'string', enum: ['index', 'fetch'] }),
+    input: Object.freeze({ type: 'object', additionalProperties: Object.freeze({}) }),
+  }, ['kind', 'input']),
+  ingestion_job_status: object({ id: string({ format: 'uuid' }) }, ['id']),
+  ingestion_job_cancel: object({ id: string({ format: 'uuid' }) }, ['id']),
+  context_heal: object({ confirm: Object.freeze({ const: true, type: 'boolean' }) }, ['confirm']),
+});