opencode-lcm 0.11.0

package/src/store-retention.ts

@@ -0,0 +1,152 @@
+import type { SqlDatabaseLike } from './store-types.js';
+
+/**
+ * Retention policy operations.
+ * Handles stale/deleted session pruning and orphan blob cleanup.
+ */
+
+export type RetentionSessionCandidate = {
+  session_id: string;
+  title: string | null;
+  session_directory: string | null;
+  root_session_id: string | null;
+  pinned: number;
+  deleted: number;
+  updated_at: number;
+  event_count: number;
+  message_count: number;
+  artifact_count: number;
+};
+
+export type RetentionBlobCandidate = {
+  content_hash: string;
+  char_count: number;
+  created_at: number;
+};
+
+export type ResolvedRetentionPolicy = {
+  staleSessionDays?: number;
+  deletedSessionDays?: number;
+  orphanBlobDays?: number;
+};
+
+export type RetentionPruneResult = {
+  deletedSessions: number;
+  deletedBlobs: number;
+  deletedBlobChars: number;
+};
+
+export function retentionCutoff(days: number): number {
+  return Date.now() - days * 24 * 60 * 60 * 1000;
+}
+
+export function readSessionRetentionCandidates(
+  db: SqlDatabaseLike,
+  deleted: boolean,
+  days: number,
+  limit?: number,
+): RetentionSessionCandidate[] {
+  const params: Array<number | string> = [retentionCutoff(days), deleted ? 1 : 0];
+  const sql = `
+    SELECT
+      s.session_id,
+      s.title,
+      s.session_directory,
+      s.root_session_id,
+      s.pinned,
+      s.deleted,
+      s.updated_at,
+      s.event_count,
+      (SELECT COUNT(*) FROM messages m WHERE m.session_id = s.session_id) AS message_count,
+      (SELECT COUNT(*) FROM artifacts a WHERE a.session_id = s.session_id) AS artifact_count
+    FROM sessions s
+    WHERE s.updated_at <= ?
+      AND s.deleted = ?
+      AND s.pinned = 0
+      AND NOT EXISTS (
+        SELECT 1 FROM sessions child WHERE child.parent_session_id = s.session_id
+      )
+    ORDER BY s.updated_at ASC
+    ${limit ? 'LIMIT ?' : ''}`;
+
+  if (limit) params.push(limit);
+  return db.prepare(sql).all(...params) as RetentionSessionCandidate[];
+}
+
+export function countSessionRetentionCandidates(
+  db: SqlDatabaseLike,
+  deleted: boolean,
+  days: number,
+): number {
+  const row = db
+    .prepare(
+      `SELECT COUNT(*) AS count
+       FROM sessions s
+       WHERE s.updated_at <= ?
+         AND s.deleted = ?
+         AND s.pinned = 0
+         AND NOT EXISTS (
+           SELECT 1 FROM sessions child WHERE child.parent_session_id = s.session_id
+         )`,
+    )
+    .get(retentionCutoff(days), deleted ? 1 : 0) as { count: number };
+  return row.count;
+}
+
+export function readOrphanBlobRetentionCandidates(
+  db: SqlDatabaseLike,
+  days: number,
+  limit?: number,
+): RetentionBlobCandidate[] {
+  const params: Array<number> = [retentionCutoff(days)];
+  const sql = `
+    SELECT content_hash, char_count, created_at
+    FROM artifact_blobs b
+    WHERE b.created_at <= ?
+      AND NOT EXISTS (
+        SELECT 1 FROM artifacts a WHERE a.content_hash = b.content_hash
+      )
+    ORDER BY char_count DESC, created_at ASC
+    ${limit ? 'LIMIT ?' : ''}`;
+  if (limit) params.push(limit);
+  return db.prepare(sql).all(...params) as RetentionBlobCandidate[];
+}
+
+export function countOrphanBlobRetentionCandidates(db: SqlDatabaseLike, days: number): number {
+  const row = db
+    .prepare(
+      `SELECT COUNT(*) AS count
+       FROM artifact_blobs b
+       WHERE b.created_at <= ?
+         AND NOT EXISTS (
+           SELECT 1 FROM artifacts a WHERE a.content_hash = b.content_hash
+         )`,
+    )
+    .get(retentionCutoff(days)) as { count: number };
+  return row.count;
+}
+
+export function sumOrphanBlobRetentionChars(db: SqlDatabaseLike, days: number): number {
+  const row = db
+    .prepare(
+      `SELECT COALESCE(SUM(char_count), 0) AS chars
+       FROM artifact_blobs b
+       WHERE b.created_at <= ?
+         AND NOT EXISTS (
+           SELECT 1 FROM artifacts a WHERE a.content_hash = b.content_hash
+         )`,
+    )
+    .get(retentionCutoff(days)) as { chars: number };
+  return row.chars;
+}
+
+export function clearSessionData(db: SqlDatabaseLike, sessionID: string): void {
+  db.prepare('DELETE FROM parts WHERE session_id = ?').run(sessionID);
+  db.prepare('DELETE FROM messages WHERE session_id = ?').run(sessionID);
+  db.prepare('DELETE FROM artifacts WHERE session_id = ?').run(sessionID);
+  db.prepare('DELETE FROM sessions WHERE session_id = ?').run(sessionID);
+  db.prepare('DELETE FROM resumes WHERE session_id = ?').run(sessionID);
+  db.prepare('DELETE FROM summary_nodes WHERE session_id = ?').run(sessionID);
+  db.prepare('DELETE FROM summary_edges WHERE session_id = ?').run(sessionID);
+  db.prepare('DELETE FROM summary_state WHERE session_id = ?').run(sessionID);
+}

package/src/store-search.ts

@@ -0,0 +1,440 @@
+import { getLogger } from './logging.js';
+import { rankSearchCandidates, type SearchCandidate } from './search-ranking.js';
+import type { ArtifactRow, SummaryNodeRow } from './store-snapshot.js';
+import type { SqlDatabaseLike } from './store-types.js';
+import type { NormalizedSession, SearchResult } from './types.js';
+import { buildSnippet, sanitizeFtsTokens, tokenizeQuery } from './utils.js';
+
+type FtsDeps = {
+  getDb(): SqlDatabaseLike;
+  readScopedSessionsSync(sessionIDs?: string[]): NormalizedSession[];
+  readScopedSummaryRowsSync(sessionIDs?: string[]): SummaryNodeRow[];
+  readScopedArtifactRowsSync(sessionIDs?: string[]): ArtifactRow[];
+  buildArtifactSearchContent(row: ArtifactRow): string;
+  ignoreToolPrefixes: string[];
+  guessMessageText(
+    message: NormalizedSession['messages'][number],
+    ignorePrefixes: string[],
+  ): string;
+};
+
+function deleteScopedFtsRows(
+  db: SqlDatabaseLike,
+  table: 'message_fts' | 'summary_fts' | 'artifact_fts',
+  sessionIDs?: string[],
+): void {
+  if (!sessionIDs) {
+    db.prepare(`DELETE FROM ${table}`).run();
+    return;
+  }
+  if (sessionIDs.length === 0) return;
+
+  db.prepare(
+    `DELETE FROM ${table} WHERE session_id IN (${sessionIDs.map(() => '?').join(', ')})`,
+  ).run(...sessionIDs);
+}
+
+export function buildFtsQuery(query: string): string | undefined {
+  const phrases = [...query.matchAll(/"([^"]+)"/g)]
+    .map((match) => sanitizeFtsTokens(tokenizeQuery(match[1])).join(' '))
+    .filter(Boolean)
+    .map((phrase) => `"${phrase}"`);
+  const remainder = query.replace(/"[^"]+"/g, ' ');
+  const tokens = sanitizeFtsTokens(tokenizeQuery(remainder)).map((token) => `${token}*`);
+  const parts = [...phrases, ...tokens];
+  if (parts.length === 0) return undefined;
+  return parts.join(' AND ');
+}
+
+/**
+ * Compute TF-IDF weights for candidate query tokens against the FTS5 corpus.
+ * Returns tokens sorted by descending IDF score (most informative first).
+ * Tokens that appear in >80% of documents are dropped as corpus-common noise.
+ *
+ * Uses a single FTS5 query per token to get document frequency, which is
+ * acceptable since automatic retrieval works with ≤10 candidate tokens.
+ */
+function getTotalDocCount(db: SqlDatabaseLike): number {
+  const messageCount = (db.prepare('SELECT COUNT(*) AS count FROM message_fts').get() as {
+    count: number;
+  }) ?? { count: 0 };
+  const summaryCount = (db.prepare('SELECT COUNT(*) AS count FROM summary_fts').get() as {
+    count: number;
+  }) ?? { count: 0 };
+  const artifactCount = (db.prepare('SELECT COUNT(*) AS count FROM artifact_fts').get() as {
+    count: number;
+  }) ?? { count: 0 };
+  return Math.max(1, messageCount.count + summaryCount.count + artifactCount.count);
+}
+
+export function computeTfidfWeights(
+  db: SqlDatabaseLike,
+  candidateTokens: string[],
+): Array<{ token: string; idf: number; docFreq: number }> {
+  if (candidateTokens.length === 0) return [];
+
+  const totalDocs = getTotalDocCount(db);
+
+  const results: Array<{ token: string; idf: number; docFreq: number }> = [];
+
+  for (const token of candidateTokens) {
+    // Query document frequency across all FTS tables
+    // FTS5 MATCH 'token*' finds all documents containing terms with this prefix
+    const query = `${token}*`;
+    let docFreq = 0;
+
+    try {
+      const msgFreq = db
+        .prepare('SELECT COUNT(*) AS count FROM message_fts WHERE message_fts MATCH ?')
+        .get(query) as { count: number } | undefined;
+      docFreq += msgFreq?.count ?? 0;
+    } catch (error) {
+      getLogger().debug('TF-IDF message_fts query failed for token', { token, error });
+    }
+
+    try {
+      const sumFreq = db
+        .prepare('SELECT COUNT(*) AS count FROM summary_fts WHERE summary_fts MATCH ?')
+        .get(query) as { count: number } | undefined;
+      docFreq += sumFreq?.count ?? 0;
+    } catch (error) {
+      getLogger().debug('TF-IDF summary_fts query failed for token', { token, error });
+    }
+
+    try {
+      const artFreq = db
+        .prepare('SELECT COUNT(*) AS count FROM artifact_fts WHERE artifact_fts MATCH ?')
+        .get(query) as { count: number } | undefined;
+      docFreq += artFreq?.count ?? 0;
+    } catch (error) {
+      getLogger().debug('TF-IDF artifact_fts query failed for token', { token, error });
+    }
+
+    // Smoothed IDF: log(N / (df + 1)) + 1
+    // Smoothing prevents division by zero and ensures non-zero weights
+    const idf = Math.log(totalDocs / (docFreq + 1)) + 1;
+    results.push({ token, idf, docFreq });
+  }
+
+  // Sort by descending IDF — most informative tokens first
+  results.sort((a, b) => b.idf - a.idf);
+
+  return results;
+}
+
+/**
+ * Filter candidate tokens using TF-IDF weights.
+ * Drops tokens whose IDF is below the median (corpus-common terms)
+ * and tokens that appear in >80% of documents.
+ * Returns tokens sorted by descending IDF.
+ */
+export function filterTokensByTfidf(
+  db: SqlDatabaseLike,
+  candidateTokens: string[],
+  options?: { maxCommonRatio?: number; minTokens?: number },
+): string[] {
+  const { maxCommonRatio = 0.8, minTokens = 1 } = options ?? {};
+
+  const weights = computeTfidfWeights(db, candidateTokens);
+  if (weights.length === 0) return candidateTokens;
+
+  // Get total docs for common-ratio threshold (already computed inside computeTfidfWeights, but needed here for ratio)
+  const totalDocs = getTotalDocCount(db);
+
+  // Compute median IDF
+  const sortedIdfs = weights.map((w) => w.idf).sort((a, b) => a - b);
+  const medianIdf =
+    sortedIdfs.length % 2 === 0
+      ? (sortedIdfs[sortedIdfs.length / 2 - 1] + sortedIdfs[sortedIdfs.length / 2]) / 2
+      : sortedIdfs[Math.floor(sortedIdfs.length / 2)];
+
+  // Filter: keep tokens with IDF >= median AND below common-ratio threshold
+  // Always keep at least minTokens tokens (the highest-IDF ones)
+  const filtered = weights.filter((w) => {
+    const docRatio = w.docFreq / totalDocs;
+    return w.idf >= medianIdf && docRatio <= maxCommonRatio;
+  });
+
+  // Ensure minimum token count
+  if (filtered.length < minTokens) {
+    return weights.slice(0, minTokens).map((w) => w.token);
+  }
+
+  return filtered.map((w) => w.token);
+}
+
+export function searchWithFts(
+  deps: FtsDeps,
+  query: string,
+  sessionIDs?: string[],
+  limit = 5,
+): SearchResult[] {
+  const ftsQuery = buildFtsQuery(query);
+  if (!ftsQuery) return [];
+  if (sessionIDs && sessionIDs.length === 0) return [];
+
+  try {
+    const db = deps.getDb();
+    const fetchLimit = Math.max(limit * 8, 12);
+    const buildScopeClause = (ids: string[] | undefined) => {
+      if (!ids) return { clause: '', params: [] as string[] };
+      return {
+        clause: `session_id IN (${ids.map(() => '?').join(', ')}) AND `,
+        params: ids,
+      };
+    };
+    const scope = buildScopeClause(sessionIDs);
+
+    const messageRows = db
+      .prepare(
+        `SELECT message_id, session_id, role, created_at, content, snippet(message_fts, 4, '[', ']', '...', 12) AS snippet, bm25(message_fts) AS rank
+         FROM message_fts
+         WHERE ${scope.clause}message_fts MATCH ?
+         ORDER BY rank, created_at DESC
+         LIMIT ?`,
+      )
+      .all(...scope.params, ftsQuery, fetchLimit) as Array<{
+      message_id: string;
+      session_id: string;
+      role: string;
+      created_at: string | number;
+      content: string;
+      snippet: string;
+      rank: number;
+    }>;
+
+    const summaryRows = db
+      .prepare(
+        `SELECT node_id, session_id, created_at, content, snippet(summary_fts, 4, '[', ']', '...', 14) AS snippet, bm25(summary_fts) AS rank
+         FROM summary_fts
+         WHERE ${scope.clause}summary_fts MATCH ?
+         ORDER BY rank, created_at DESC
+         LIMIT ?`,
+      )
+      .all(...scope.params, ftsQuery, fetchLimit) as Array<{
+      node_id: string;
+      session_id: string;
+      created_at: string | number;
+      content: string;
+      snippet: string;
+      rank: number;
+    }>;
+
+    const artifactRows = db
+      .prepare(
+        `SELECT artifact_id, session_id, artifact_kind, created_at, content, snippet(artifact_fts, 6, '[', ']', '...', 14) AS snippet, bm25(artifact_fts) AS rank
+         FROM artifact_fts
+         WHERE ${scope.clause}artifact_fts MATCH ?
+         ORDER BY rank, created_at DESC
+         LIMIT ?`,
+      )
+      .all(...scope.params, ftsQuery, fetchLimit) as Array<{
+      artifact_id: string;
+      session_id: string;
+      artifact_kind: string;
+      created_at: string | number;
+      content: string;
+      snippet: string;
+      rank: number;
+    }>;
+
+    const candidates: SearchCandidate[] = [
+      ...messageRows.map((row, index) => ({
+        id: row.message_id,
+        type: row.role,
+        sessionID: row.session_id,
+        timestamp: Number(row.created_at),
+        snippet: row.snippet || buildSnippet(row.content, query),
+        content: row.content,
+        sourceKind: 'message' as const,
+        sourceOrder: index,
+      })),
+      ...summaryRows.map((row, index) => ({
+        id: row.node_id,
+        type: 'summary',
+        sessionID: row.session_id,
+        timestamp: Number(row.created_at),
+        snippet: row.snippet || buildSnippet(row.content, query),
+        content: row.content,
+        sourceKind: 'summary' as const,
+        sourceOrder: index,
+      })),
+      ...artifactRows.map((row, index) => ({
+        id: row.artifact_id,
+        type: `artifact:${row.artifact_kind}`,
+        sessionID: row.session_id,
+        timestamp: Number(row.created_at),
+        snippet: row.snippet || buildSnippet(row.content, query),
+        content: row.content,
+        sourceKind: 'artifact' as const,
+        sourceOrder: index,
+      })),
+    ];
+
+    return rankSearchCandidates(candidates, query, limit);
+  } catch (error) {
+    getLogger().debug('FTS search failed, returning empty results', { query, error });
+    return [];
+  }
+}
+
+export function searchByScan(
+  deps: FtsDeps,
+  query: string,
+  sessionIDs?: string[],
+  limit = 5,
+): SearchResult[] {
+  const sessions = deps.readScopedSessionsSync(sessionIDs);
+  const candidates: SearchCandidate[] = [];
+
+  for (const session of sessions) {
+    for (const [index, message] of session.messages.entries()) {
+      const blob = deps.guessMessageText(message, deps.ignoreToolPrefixes);
+      if (!blob.toLowerCase().includes(query)) continue;
+
+      candidates.push({
+        id: message.info.id,
+        type: message.info.role,
+        sessionID: session.sessionID,
+        timestamp: message.info.time.created,
+        snippet: buildSnippet(blob, query),
+        content: blob,
+        sourceKind: 'message',
+        sourceOrder: index,
+      });
+    }
+  }
+
+  const summaryRows = deps.readScopedSummaryRowsSync(sessionIDs);
+
+  summaryRows.forEach((row, index) => {
+    if (!row.summary_text.toLowerCase().includes(query)) return;
+    candidates.push({
+      id: row.node_id,
+      type: 'summary',
+      sessionID: row.session_id,
+      timestamp: row.created_at,
+      snippet: buildSnippet(row.summary_text, query),
+      content: row.summary_text,
+      sourceKind: 'summary',
+      sourceOrder: index,
+    });
+  });
+
+  const artifactRows = deps.readScopedArtifactRowsSync(sessionIDs);
+
+  for (const [index, row] of artifactRows.entries()) {
+    const haystack = `${row.preview_text}\n${row.content_text}`.toLowerCase();
+    if (!haystack.includes(query)) continue;
+
+    candidates.push({
+      id: row.artifact_id,
+      type: `artifact:${row.artifact_kind}`,
+      sessionID: row.session_id,
+      timestamp: row.created_at,
+      snippet: buildSnippet(`${row.preview_text}\n${row.content_text}`, query),
+      content: row.content_text,
+      sourceKind: 'artifact',
+      sourceOrder: index,
+    });
+  }
+
+  return rankSearchCandidates(candidates, query, limit);
+}
+
+export function replaceMessageSearchRowsSync(deps: FtsDeps, session: NormalizedSession): void {
+  const db = deps.getDb();
+  db.prepare('DELETE FROM message_fts WHERE session_id = ?').run(session.sessionID);
+  insertMessageSearchRowsSync(deps, session);
+}
+
+function insertMessageSearchRowsSync(deps: FtsDeps, session: NormalizedSession): void {
+  const db = deps.getDb();
+  const insert = db.prepare(
+    'INSERT INTO message_fts (session_id, message_id, role, created_at, content) VALUES (?, ?, ?, ?, ?)',
+  );
+
+  for (const message of session.messages) {
+    const content = deps.guessMessageText(message, deps.ignoreToolPrefixes);
+    if (!content) continue;
+    insert.run(
+      session.sessionID,
+      message.info.id,
+      message.info.role,
+      String(message.info.time.created),
+      content,
+    );
+  }
+}
+
+export function replaceMessageSearchRowSync(
+  deps: FtsDeps,
+  sessionID: string,
+  message: NormalizedSession['messages'][number],
+): void {
+  const db = deps.getDb();
+  db.prepare('DELETE FROM message_fts WHERE message_id = ?').run(message.info.id);
+
+  const content = deps.guessMessageText(message, deps.ignoreToolPrefixes);
+  if (!content) return;
+
+  db.prepare(
+    'INSERT INTO message_fts (session_id, message_id, role, created_at, content) VALUES (?, ?, ?, ?, ?)',
+  ).run(sessionID, message.info.id, message.info.role, String(message.info.time.created), content);
+}
+
+export function replaceSummarySearchRowsSync(deps: FtsDeps, sessionIDs?: string[]): void {
+  const db = deps.getDb();
+  deleteScopedFtsRows(db, 'summary_fts', sessionIDs);
+
+  const summaryRows = deps.readScopedSummaryRowsSync(sessionIDs);
+  const insert = db.prepare(
+    'INSERT INTO summary_fts (session_id, node_id, level, created_at, content) VALUES (?, ?, ?, ?, ?)',
+  );
+  for (const row of summaryRows) {
+    insert.run(
+      row.session_id,
+      row.node_id,
+      String(row.level),
+      String(row.created_at),
+      row.summary_text,
+    );
+  }
+}
+
+export function replaceArtifactSearchRowsSync(deps: FtsDeps, sessionIDs?: string[]): void {
+  const db = deps.getDb();
+  deleteScopedFtsRows(db, 'artifact_fts', sessionIDs);
+
+  const artifactRows = deps.readScopedArtifactRowsSync(sessionIDs);
+  const insert = db.prepare(
+    'INSERT INTO artifact_fts (session_id, artifact_id, message_id, part_id, artifact_kind, created_at, content) VALUES (?, ?, ?, ?, ?, ?, ?)',
+  );
+  for (const row of artifactRows) {
+    insert.run(
+      row.session_id,
+      row.artifact_id,
+      row.message_id,
+      row.part_id,
+      row.artifact_kind,
+      String(row.created_at),
+      deps.buildArtifactSearchContent(row),
+    );
+  }
+}
+
+export function refreshSearchIndexesSync(deps: FtsDeps, sessionIDs?: string[]): void {
+  const db = deps.getDb();
+  deleteScopedFtsRows(db, 'message_fts', sessionIDs);
+
+  for (const session of deps.readScopedSessionsSync(sessionIDs)) {
+    insertMessageSearchRowsSync(deps, session);
+  }
+
+  replaceSummarySearchRowsSync(deps, sessionIDs);
+  replaceArtifactSearchRowsSync(deps, sessionIDs);
+}
+
+export function rebuildSearchIndexesSync(deps: FtsDeps): void {
+  refreshSearchIndexesSync(deps);
+}