opencode-lcm 0.11.0

package/dist/store-search.js

@@ -0,0 +1,298 @@
+import { getLogger } from './logging.js';
+import { rankSearchCandidates } from './search-ranking.js';
+import { buildSnippet, sanitizeFtsTokens, tokenizeQuery } from './utils.js';
+function deleteScopedFtsRows(db, table, sessionIDs) {
+    if (!sessionIDs) {
+        db.prepare(`DELETE FROM ${table}`).run();
+        return;
+    }
+    if (sessionIDs.length === 0)
+        return;
+    db.prepare(`DELETE FROM ${table} WHERE session_id IN (${sessionIDs.map(() => '?').join(', ')})`).run(...sessionIDs);
+}
+export function buildFtsQuery(query) {
+    const phrases = [...query.matchAll(/"([^"]+)"/g)]
+        .map((match) => sanitizeFtsTokens(tokenizeQuery(match[1])).join(' '))
+        .filter(Boolean)
+        .map((phrase) => `"${phrase}"`);
+    const remainder = query.replace(/"[^"]+"/g, ' ');
+    const tokens = sanitizeFtsTokens(tokenizeQuery(remainder)).map((token) => `${token}*`);
+    const parts = [...phrases, ...tokens];
+    if (parts.length === 0)
+        return undefined;
+    return parts.join(' AND ');
+}
+/**
+ * Compute TF-IDF weights for candidate query tokens against the FTS5 corpus.
+ * Returns tokens sorted by descending IDF score (most informative first).
+ * Tokens that appear in >80% of documents are dropped as corpus-common noise.
+ *
+ * Uses a single FTS5 query per token to get document frequency, which is
+ * acceptable since automatic retrieval works with ≤10 candidate tokens.
+ */
+function getTotalDocCount(db) {
+    const messageCount = db.prepare('SELECT COUNT(*) AS count FROM message_fts').get() ?? { count: 0 };
+    const summaryCount = db.prepare('SELECT COUNT(*) AS count FROM summary_fts').get() ?? { count: 0 };
+    const artifactCount = db.prepare('SELECT COUNT(*) AS count FROM artifact_fts').get() ?? { count: 0 };
+    return Math.max(1, messageCount.count + summaryCount.count + artifactCount.count);
+}
+export function computeTfidfWeights(db, candidateTokens) {
+    if (candidateTokens.length === 0)
+        return [];
+    const totalDocs = getTotalDocCount(db);
+    const results = [];
+    for (const token of candidateTokens) {
+        // Query document frequency across all FTS tables
+        // FTS5 MATCH 'token*' finds all documents containing terms with this prefix
+        const query = `${token}*`;
+        let docFreq = 0;
+        try {
+            const msgFreq = db
+                .prepare('SELECT COUNT(*) AS count FROM message_fts WHERE message_fts MATCH ?')
+                .get(query);
+            docFreq += msgFreq?.count ?? 0;
+        }
+        catch (error) {
+            getLogger().debug('TF-IDF message_fts query failed for token', { token, error });
+        }
+        try {
+            const sumFreq = db
+                .prepare('SELECT COUNT(*) AS count FROM summary_fts WHERE summary_fts MATCH ?')
+                .get(query);
+            docFreq += sumFreq?.count ?? 0;
+        }
+        catch (error) {
+            getLogger().debug('TF-IDF summary_fts query failed for token', { token, error });
+        }
+        try {
+            const artFreq = db
+                .prepare('SELECT COUNT(*) AS count FROM artifact_fts WHERE artifact_fts MATCH ?')
+                .get(query);
+            docFreq += artFreq?.count ?? 0;
+        }
+        catch (error) {
+            getLogger().debug('TF-IDF artifact_fts query failed for token', { token, error });
+        }
+        // Smoothed IDF: log(N / (df + 1)) + 1
+        // Smoothing prevents division by zero and ensures non-zero weights
+        const idf = Math.log(totalDocs / (docFreq + 1)) + 1;
+        results.push({ token, idf, docFreq });
+    }
+    // Sort by descending IDF — most informative tokens first
+    results.sort((a, b) => b.idf - a.idf);
+    return results;
+}
+/**
+ * Filter candidate tokens using TF-IDF weights.
+ * Drops tokens whose IDF is below the median (corpus-common terms)
+ * and tokens that appear in >80% of documents.
+ * Returns tokens sorted by descending IDF.
+ */
+export function filterTokensByTfidf(db, candidateTokens, options) {
+    const { maxCommonRatio = 0.8, minTokens = 1 } = options ?? {};
+    const weights = computeTfidfWeights(db, candidateTokens);
+    if (weights.length === 0)
+        return candidateTokens;
+    // Get total docs for common-ratio threshold (already computed inside computeTfidfWeights, but needed here for ratio)
+    const totalDocs = getTotalDocCount(db);
+    // Compute median IDF
+    const sortedIdfs = weights.map((w) => w.idf).sort((a, b) => a - b);
+    const medianIdf = sortedIdfs.length % 2 === 0
+        ? (sortedIdfs[sortedIdfs.length / 2 - 1] + sortedIdfs[sortedIdfs.length / 2]) / 2
+        : sortedIdfs[Math.floor(sortedIdfs.length / 2)];
+    // Filter: keep tokens with IDF >= median AND below common-ratio threshold
+    // Always keep at least minTokens tokens (the highest-IDF ones)
+    const filtered = weights.filter((w) => {
+        const docRatio = w.docFreq / totalDocs;
+        return w.idf >= medianIdf && docRatio <= maxCommonRatio;
+    });
+    // Ensure minimum token count
+    if (filtered.length < minTokens) {
+        return weights.slice(0, minTokens).map((w) => w.token);
+    }
+    return filtered.map((w) => w.token);
+}
+export function searchWithFts(deps, query, sessionIDs, limit = 5) {
+    const ftsQuery = buildFtsQuery(query);
+    if (!ftsQuery)
+        return [];
+    if (sessionIDs && sessionIDs.length === 0)
+        return [];
+    try {
+        const db = deps.getDb();
+        const fetchLimit = Math.max(limit * 8, 12);
+        const buildScopeClause = (ids) => {
+            if (!ids)
+                return { clause: '', params: [] };
+            return {
+                clause: `session_id IN (${ids.map(() => '?').join(', ')}) AND `,
+                params: ids,
+            };
+        };
+        const scope = buildScopeClause(sessionIDs);
+        const messageRows = db
+            .prepare(`SELECT message_id, session_id, role, created_at, content, snippet(message_fts, 4, '[', ']', '...', 12) AS snippet, bm25(message_fts) AS rank
+         FROM message_fts
+         WHERE ${scope.clause}message_fts MATCH ?
+         ORDER BY rank, created_at DESC
+         LIMIT ?`)
+            .all(...scope.params, ftsQuery, fetchLimit);
+        const summaryRows = db
+            .prepare(`SELECT node_id, session_id, created_at, content, snippet(summary_fts, 4, '[', ']', '...', 14) AS snippet, bm25(summary_fts) AS rank
+         FROM summary_fts
+         WHERE ${scope.clause}summary_fts MATCH ?
+         ORDER BY rank, created_at DESC
+         LIMIT ?`)
+            .all(...scope.params, ftsQuery, fetchLimit);
+        const artifactRows = db
+            .prepare(`SELECT artifact_id, session_id, artifact_kind, created_at, content, snippet(artifact_fts, 6, '[', ']', '...', 14) AS snippet, bm25(artifact_fts) AS rank
+         FROM artifact_fts
+         WHERE ${scope.clause}artifact_fts MATCH ?
+         ORDER BY rank, created_at DESC
+         LIMIT ?`)
+            .all(...scope.params, ftsQuery, fetchLimit);
+        const candidates = [
+            ...messageRows.map((row, index) => ({
+                id: row.message_id,
+                type: row.role,
+                sessionID: row.session_id,
+                timestamp: Number(row.created_at),
+                snippet: row.snippet || buildSnippet(row.content, query),
+                content: row.content,
+                sourceKind: 'message',
+                sourceOrder: index,
+            })),
+            ...summaryRows.map((row, index) => ({
+                id: row.node_id,
+                type: 'summary',
+                sessionID: row.session_id,
+                timestamp: Number(row.created_at),
+                snippet: row.snippet || buildSnippet(row.content, query),
+                content: row.content,
+                sourceKind: 'summary',
+                sourceOrder: index,
+            })),
+            ...artifactRows.map((row, index) => ({
+                id: row.artifact_id,
+                type: `artifact:${row.artifact_kind}`,
+                sessionID: row.session_id,
+                timestamp: Number(row.created_at),
+                snippet: row.snippet || buildSnippet(row.content, query),
+                content: row.content,
+                sourceKind: 'artifact',
+                sourceOrder: index,
+            })),
+        ];
+        return rankSearchCandidates(candidates, query, limit);
+    }
+    catch (error) {
+        getLogger().debug('FTS search failed, returning empty results', { query, error });
+        return [];
+    }
+}
+export function searchByScan(deps, query, sessionIDs, limit = 5) {
+    const sessions = deps.readScopedSessionsSync(sessionIDs);
+    const candidates = [];
+    for (const session of sessions) {
+        for (const [index, message] of session.messages.entries()) {
+            const blob = deps.guessMessageText(message, deps.ignoreToolPrefixes);
+            if (!blob.toLowerCase().includes(query))
+                continue;
+            candidates.push({
+                id: message.info.id,
+                type: message.info.role,
+                sessionID: session.sessionID,
+                timestamp: message.info.time.created,
+                snippet: buildSnippet(blob, query),
+                content: blob,
+                sourceKind: 'message',
+                sourceOrder: index,
+            });
+        }
+    }
+    const summaryRows = deps.readScopedSummaryRowsSync(sessionIDs);
+    summaryRows.forEach((row, index) => {
+        if (!row.summary_text.toLowerCase().includes(query))
+            return;
+        candidates.push({
+            id: row.node_id,
+            type: 'summary',
+            sessionID: row.session_id,
+            timestamp: row.created_at,
+            snippet: buildSnippet(row.summary_text, query),
+            content: row.summary_text,
+            sourceKind: 'summary',
+            sourceOrder: index,
+        });
+    });
+    const artifactRows = deps.readScopedArtifactRowsSync(sessionIDs);
+    for (const [index, row] of artifactRows.entries()) {
+        const haystack = `${row.preview_text}\n${row.content_text}`.toLowerCase();
+        if (!haystack.includes(query))
+            continue;
+        candidates.push({
+            id: row.artifact_id,
+            type: `artifact:${row.artifact_kind}`,
+            sessionID: row.session_id,
+            timestamp: row.created_at,
+            snippet: buildSnippet(`${row.preview_text}\n${row.content_text}`, query),
+            content: row.content_text,
+            sourceKind: 'artifact',
+            sourceOrder: index,
+        });
+    }
+    return rankSearchCandidates(candidates, query, limit);
+}
+export function replaceMessageSearchRowsSync(deps, session) {
+    const db = deps.getDb();
+    db.prepare('DELETE FROM message_fts WHERE session_id = ?').run(session.sessionID);
+    insertMessageSearchRowsSync(deps, session);
+}
+function insertMessageSearchRowsSync(deps, session) {
+    const db = deps.getDb();
+    const insert = db.prepare('INSERT INTO message_fts (session_id, message_id, role, created_at, content) VALUES (?, ?, ?, ?, ?)');
+    for (const message of session.messages) {
+        const content = deps.guessMessageText(message, deps.ignoreToolPrefixes);
+        if (!content)
+            continue;
+        insert.run(session.sessionID, message.info.id, message.info.role, String(message.info.time.created), content);
+    }
+}
+export function replaceMessageSearchRowSync(deps, sessionID, message) {
+    const db = deps.getDb();
+    db.prepare('DELETE FROM message_fts WHERE message_id = ?').run(message.info.id);
+    const content = deps.guessMessageText(message, deps.ignoreToolPrefixes);
+    if (!content)
+        return;
+    db.prepare('INSERT INTO message_fts (session_id, message_id, role, created_at, content) VALUES (?, ?, ?, ?, ?)').run(sessionID, message.info.id, message.info.role, String(message.info.time.created), content);
+}
+export function replaceSummarySearchRowsSync(deps, sessionIDs) {
+    const db = deps.getDb();
+    deleteScopedFtsRows(db, 'summary_fts', sessionIDs);
+    const summaryRows = deps.readScopedSummaryRowsSync(sessionIDs);
+    const insert = db.prepare('INSERT INTO summary_fts (session_id, node_id, level, created_at, content) VALUES (?, ?, ?, ?, ?)');
+    for (const row of summaryRows) {
+        insert.run(row.session_id, row.node_id, String(row.level), String(row.created_at), row.summary_text);
+    }
+}
+export function replaceArtifactSearchRowsSync(deps, sessionIDs) {
+    const db = deps.getDb();
+    deleteScopedFtsRows(db, 'artifact_fts', sessionIDs);
+    const artifactRows = deps.readScopedArtifactRowsSync(sessionIDs);
+    const insert = db.prepare('INSERT INTO artifact_fts (session_id, artifact_id, message_id, part_id, artifact_kind, created_at, content) VALUES (?, ?, ?, ?, ?, ?, ?)');
+    for (const row of artifactRows) {
+        insert.run(row.session_id, row.artifact_id, row.message_id, row.part_id, row.artifact_kind, String(row.created_at), deps.buildArtifactSearchContent(row));
+    }
+}
+export function refreshSearchIndexesSync(deps, sessionIDs) {
+    const db = deps.getDb();
+    deleteScopedFtsRows(db, 'message_fts', sessionIDs);
+    for (const session of deps.readScopedSessionsSync(sessionIDs)) {
+        insertMessageSearchRowsSync(deps, session);
+    }
+    replaceSummarySearchRowsSync(deps, sessionIDs);
+    replaceArtifactSearchRowsSync(deps, sessionIDs);
+}
+export function rebuildSearchIndexesSync(deps) {
+    refreshSearchIndexesSync(deps);
+}

package/dist/store-snapshot.d.ts

@@ -0,0 +1,133 @@
+import type { SqlDatabaseLike } from './store-types.js';
+export type SnapshotScope = 'session' | 'root' | 'worktree' | 'all';
+export type SnapshotWorktreeMode = 'auto' | 'preserve' | 'current';
+export type SessionRow = {
+    session_id: string;
+    title: string | null;
+    session_directory: string | null;
+    worktree_key: string | null;
+    parent_session_id: string | null;
+    root_session_id: string | null;
+    lineage_depth: number | null;
+    pinned: number | null;
+    pin_reason: string | null;
+    updated_at: number;
+    compacted_at: number | null;
+    deleted: number;
+    event_count: number;
+};
+export type MessageRow = {
+    message_id: string;
+    session_id: string;
+    created_at: number;
+    info_json: string;
+};
+export type PartRow = {
+    part_id: string;
+    session_id: string;
+    message_id: string;
+    sort_key: number;
+    part_json: string;
+};
+export type SummaryNodeRow = {
+    node_id: string;
+    session_id: string;
+    level: number;
+    node_kind: string;
+    start_index: number;
+    end_index: number;
+    message_ids_json: string;
+    summary_text: string;
+    created_at: number;
+};
+export type SummaryEdgeRow = {
+    session_id: string;
+    parent_id: string;
+    child_id: string;
+    child_position: number;
+};
+export type SummaryStateRow = {
+    session_id: string;
+    archived_count: number;
+    latest_message_created: number;
+    archived_signature: string | null;
+    root_node_ids_json: string;
+    updated_at: number;
+};
+export type ArtifactRow = {
+    artifact_id: string;
+    session_id: string;
+    message_id: string;
+    part_id: string;
+    artifact_kind: string;
+    field_name: string;
+    preview_text: string;
+    content_text: string;
+    content_hash: string | null;
+    metadata_json: string;
+    char_count: number;
+    created_at: number;
+};
+export type ArtifactBlobRow = {
+    content_hash: string;
+    content_text: string;
+    char_count: number;
+    created_at: number;
+};
+export type SnapshotPayload = {
+    version: 1;
+    exportedAt: number;
+    scope: string;
+    sessions: SessionRow[];
+    messages: MessageRow[];
+    parts: PartRow[];
+    resumes: Array<{
+        session_id: string;
+        note: string;
+        updated_at: number;
+    }>;
+    artifacts: ArtifactRow[];
+    artifact_blobs: ArtifactBlobRow[];
+    summary_nodes: SummaryNodeRow[];
+    summary_edges: SummaryEdgeRow[];
+    summary_state: SummaryStateRow[];
+};
+export type ExportSnapshotInput = {
+    filePath: string;
+    sessionID?: string;
+    scope?: string;
+};
+export type ImportSnapshotInput = {
+    filePath: string;
+    mode?: 'replace' | 'merge';
+    worktreeMode?: SnapshotWorktreeMode;
+};
+export type SnapshotExportBindings = {
+    workspaceDirectory: string;
+    normalizeScope(scope?: string): SnapshotScope | undefined;
+    resolveScopeSessionIDs(scope?: string, sessionID?: string): string[] | undefined;
+    readScopedSessionRowsSync(sessionIDs?: string[]): SessionRow[];
+    readScopedMessageRowsSync(sessionIDs?: string[]): MessageRow[];
+    readScopedPartRowsSync(sessionIDs?: string[]): PartRow[];
+    readScopedResumeRowsSync(sessionIDs?: string[]): Array<{
+        session_id: string;
+        note: string;
+        updated_at: number;
+    }>;
+    readScopedArtifactRowsSync(sessionIDs?: string[]): ArtifactRow[];
+    readScopedArtifactBlobRowsSync(sessionIDs?: string[]): ArtifactBlobRow[];
+    readScopedSummaryRowsSync(sessionIDs?: string[]): SummaryNodeRow[];
+    readScopedSummaryEdgeRowsSync(sessionIDs?: string[]): SummaryEdgeRow[];
+    readScopedSummaryStateRowsSync(sessionIDs?: string[]): SummaryStateRow[];
+};
+export type SnapshotImportBindings = {
+    workspaceDirectory: string;
+    getDb(): SqlDatabaseLike;
+    clearSessionDataSync(sessionID: string): void;
+    backfillArtifactBlobsSync(): void;
+    refreshAllLineageSync(): void;
+    syncAllDerivedSessionStateSync(force: boolean): void;
+    refreshSearchIndexesSync(sessionIDs?: string[]): void;
+};
+export declare function exportStoreSnapshot(bindings: SnapshotExportBindings, input: ExportSnapshotInput): Promise<string>;
+export declare function importStoreSnapshot(bindings: SnapshotImportBindings, input: ImportSnapshotInput): Promise<string>;