@winci/local-rag 0.2.1

package/src/db/checkpoints.ts

@@ -0,0 +1,166 @@
+import { Database } from "bun:sqlite";
+import { type CheckpointRow } from "./types";
+
+export function createCheckpoint(
+  db: Database,
+  sessionId: string,
+  turnIndex: number,
+  timestamp: string,
+  type: string,
+  title: string,
+  summary: string,
+  filesInvolved: string[],
+  tags: string[],
+  embedding: Float32Array
+): number {
+  let checkpointId = 0;
+
+  const tx = db.transaction(() => {
+    db.run(
+      `INSERT INTO conversation_checkpoints
+       (session_id, turn_index, timestamp, type, title, summary, files_involved, tags, embedding)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      [
+        sessionId,
+        turnIndex,
+        timestamp,
+        type,
+        title,
+        summary,
+        JSON.stringify(filesInvolved),
+        JSON.stringify(tags),
+        null,
+      ]
+    );
+
+    checkpointId = Number(
+      db.query<{ id: number }, []>("SELECT last_insert_rowid() as id").get()!.id
+    );
+
+    db.run(
+      "INSERT INTO vec_checkpoints (checkpoint_id, embedding) VALUES (?, ?)",
+      [checkpointId, new Uint8Array(embedding.buffer)]
+    );
+  });
+
+  tx();
+  return checkpointId;
+}
+
+export function listCheckpoints(
+  db: Database,
+  sessionId?: string,
+  type?: string,
+  limit: number = 20
+): CheckpointRow[] {
+  let sql = "SELECT * FROM conversation_checkpoints WHERE 1=1";
+  const params: (string | number)[] = [];
+
+  if (sessionId) {
+    sql += " AND session_id = ?";
+    params.push(sessionId);
+  }
+  if (type) {
+    sql += " AND type = ?";
+    params.push(type);
+  }
+
+  sql += " ORDER BY timestamp DESC LIMIT ?";
+  params.push(limit);
+
+  return db
+    .query<
+      { id: number; session_id: string; turn_index: number; timestamp: string; type: string; title: string; summary: string; files_involved: string; tags: string },
+      (string | number)[]
+    >(sql)
+    .all(...params)
+    .map((r) => ({
+      id: r.id,
+      sessionId: r.session_id,
+      turnIndex: r.turn_index,
+      timestamp: r.timestamp,
+      type: r.type,
+      title: r.title,
+      summary: r.summary,
+      filesInvolved: JSON.parse(r.files_involved || "[]"),
+      tags: JSON.parse(r.tags || "[]"),
+    }));
+}
+
+export function searchCheckpoints(
+  db: Database,
+  queryEmbedding: Float32Array,
+  topK: number = 5,
+  type?: string
+): (CheckpointRow & { score: number })[] {
+  const rows = db
+    .query<
+      {
+        checkpoint_id: number;
+        distance: number;
+        id: number;
+        session_id: string;
+        turn_index: number;
+        timestamp: string;
+        type: string;
+        title: string;
+        summary: string;
+        files_involved: string;
+        tags: string;
+      },
+      [Uint8Array, number]
+    >(
+      `SELECT v.checkpoint_id, v.distance,
+              cp.id, cp.session_id, cp.turn_index, cp.timestamp, cp.type,
+              cp.title, cp.summary, cp.files_involved, cp.tags
+       FROM (SELECT checkpoint_id, distance FROM vec_checkpoints WHERE embedding MATCH ? ORDER BY distance LIMIT ?) v
+       JOIN conversation_checkpoints cp ON cp.id = v.checkpoint_id`
+    )
+    .all(new Uint8Array(queryEmbedding.buffer), topK * 2);
+
+  const results: (CheckpointRow & { score: number })[] = [];
+
+  for (const row of rows) {
+    if (type && row.type !== type) continue;
+
+    results.push({
+      id: row.id,
+      sessionId: row.session_id,
+      turnIndex: row.turn_index,
+      timestamp: row.timestamp,
+      type: row.type,
+      title: row.title,
+      summary: row.summary,
+      filesInvolved: JSON.parse(row.files_involved || "[]"),
+      tags: JSON.parse(row.tags || "[]"),
+      score: 1 / (1 + row.distance),
+    });
+
+    if (results.length >= topK) break;
+  }
+
+  return results;
+}
+
+export function getCheckpoint(db: Database, id: number): CheckpointRow | null {
+  const r = db
+    .query<
+      { id: number; session_id: string; turn_index: number; timestamp: string; type: string; title: string; summary: string; files_involved: string; tags: string },
+      [number]
+    >(
+      "SELECT id, session_id, turn_index, timestamp, type, title, summary, files_involved, tags FROM conversation_checkpoints WHERE id = ?"
+    )
+    .get(id);
+  if (!r) return null;
+  return {
+    id: r.id,
+    sessionId: r.session_id,
+    turnIndex: r.turn_index,
+    timestamp: r.timestamp,
+    type: r.type,
+    title: r.title,
+    summary: r.summary,
+    filesInvolved: JSON.parse(r.files_involved || "[]"),
+    tags: JSON.parse(r.tags || "[]"),
+  };
+}

package/src/db/conversation.ts

@@ -0,0 +1,241 @@
+import { Database } from "bun:sqlite";
+import { type ConversationSearchResult } from "./types";
+import { sanitizeFTS } from "../search/usages";
+
+export function upsertSession(
+  db: Database,
+  sessionId: string,
+  jsonlPath: string,
+  startedAt: string,
+  mtime: number,
+  readOffset: number
+) {
+  db.run(
+    `INSERT INTO conversation_sessions (session_id, jsonl_path, started_at, indexed_at, file_mtime, read_offset)
+     VALUES (?, ?, ?, ?, ?, ?)
+     ON CONFLICT(session_id) DO UPDATE SET
+       file_mtime = excluded.file_mtime,
+       indexed_at = excluded.indexed_at,
+       read_offset = excluded.read_offset`,
+    [sessionId, jsonlPath, startedAt, new Date().toISOString(), mtime, readOffset]
+  );
+}
+
+export function getSession(db: Database, sessionId: string): {
+  id: number;
+  sessionId: string;
+  jsonlPath: string;
+  mtime: number;
+  readOffset: number;
+  turnCount: number;
+} | null {
+  const row = db
+    .query<
+      { id: number; session_id: string; jsonl_path: string; file_mtime: number; read_offset: number; turn_count: number },
+      [string]
+    >("SELECT id, session_id, jsonl_path, file_mtime, read_offset, turn_count FROM conversation_sessions WHERE session_id = ?")
+    .get(sessionId);
+  if (!row) return null;
+  return {
+    id: row.id,
+    sessionId: row.session_id,
+    jsonlPath: row.jsonl_path,
+    mtime: row.file_mtime,
+    readOffset: row.read_offset,
+    turnCount: row.turn_count,
+  };
+}
+
+export function updateSessionStats(db: Database, sessionId: string, turnCount: number, totalTokens: number, readOffset: number) {
+  db.run(
+    `UPDATE conversation_sessions SET turn_count = ?, total_tokens = ?, read_offset = ?, indexed_at = ? WHERE session_id = ?`,
+    [turnCount, totalTokens, readOffset, new Date().toISOString(), sessionId]
+  );
+}
+
+export function insertTurn(
+  db: Database,
+  sessionId: string,
+  turnIndex: number,
+  timestamp: string,
+  userText: string,
+  assistantText: string,
+  toolsUsed: string[],
+  filesReferenced: string[],
+  tokenCost: number,
+  summary: string,
+  chunks: { snippet: string; embedding: Float32Array }[]
+): number {
+  let turnId = 0;
+
+  const tx = db.transaction(() => {
+    db.run(
+      `INSERT OR IGNORE INTO conversation_turns
+       (session_id, turn_index, timestamp, user_text, assistant_text, tools_used, files_referenced, token_cost, summary)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      [
+        sessionId,
+        turnIndex,
+        timestamp,
+        userText,
+        assistantText,
+        JSON.stringify(toolsUsed),
+        JSON.stringify(filesReferenced),
+        tokenCost,
+        summary,
+      ]
+    );
+
+    // If the INSERT was ignored (duplicate), changes() returns 0
+    const inserted = db.query<{ c: number }, []>("SELECT changes() as c").get()!.c;
+    if (inserted === 0) return;
+
+    turnId = Number(
+      db.query<{ id: number }, []>("SELECT last_insert_rowid() as id").get()!.id
+    );
+
+    for (let i = 0; i < chunks.length; i++) {
+      const { snippet, embedding } = chunks[i];
+      db.run(
+        "INSERT INTO conversation_chunks (turn_id, chunk_index, snippet) VALUES (?, ?, ?)",
+        [turnId, i, snippet]
+      );
+      const chunkId = Number(
+        db.query<{ id: number }, []>("SELECT last_insert_rowid() as id").get()!.id
+      );
+      db.run(
+        "INSERT INTO vec_conversation (chunk_id, embedding) VALUES (?, ?)",
+        [chunkId, new Uint8Array(embedding.buffer)]
+      );
+    }
+  });
+
+  tx();
+  return turnId;
+}
+
+export function getTurnCount(db: Database, sessionId: string): number {
+  const row = db
+    .query<{ count: number }, [string]>(
+      "SELECT COUNT(*) as count FROM conversation_turns WHERE session_id = ?"
+    )
+    .get(sessionId)!;
+  return row.count;
+}
+
+export function searchConversation(
+  db: Database,
+  queryEmbedding: Float32Array,
+  topK: number = 5,
+  sessionId?: string
+): ConversationSearchResult[] {
+  // Use subquery for vector search, then JOIN for turn data
+  const rows = db
+    .query<
+      {
+        chunk_id: number;
+        distance: number;
+        snippet: string;
+        turn_id: number;
+        turn_index: number;
+        session_id: string;
+        timestamp: string;
+        summary: string;
+        tools_used: string;
+        files_referenced: string;
+      },
+      [Uint8Array, number]
+    >(
+      `SELECT v.chunk_id, v.distance, cc.snippet, cc.turn_id,
+              ct.turn_index, ct.session_id, ct.timestamp, ct.summary, ct.tools_used, ct.files_referenced
+       FROM (SELECT chunk_id, distance FROM vec_conversation WHERE embedding MATCH ? ORDER BY distance LIMIT ?) v
+       JOIN conversation_chunks cc ON cc.id = v.chunk_id
+       JOIN conversation_turns ct ON ct.id = cc.turn_id`
+    )
+    .all(new Uint8Array(queryEmbedding.buffer), sessionId ? topK * 10 : topK * 3);
+
+  const results: ConversationSearchResult[] = [];
+  const seenTurns = new Set<number>();
+
+  for (const row of rows) {
+    if (seenTurns.has(row.turn_id)) continue;
+    seenTurns.add(row.turn_id);
+
+    if (sessionId && row.session_id !== sessionId) continue;
+
+    results.push({
+      turnId: row.turn_id,
+      turnIndex: row.turn_index,
+      sessionId: row.session_id,
+      timestamp: row.timestamp,
+      summary: row.summary || "",
+      snippet: row.snippet,
+      toolsUsed: JSON.parse(row.tools_used || "[]"),
+      filesReferenced: JSON.parse(row.files_referenced || "[]"),
+      score: 1 / (1 + row.distance),
+    });
+
+    if (results.length >= topK) break;
+  }
+
+  return results;
+}
+
+export function textSearchConversation(
+  db: Database,
+  query: string,
+  topK: number = 5,
+  sessionId?: string
+): ConversationSearchResult[] {
+  const rows = db
+    .query<
+      {
+        snippet: string;
+        turn_id: number;
+        rank: number;
+        turn_index: number;
+        session_id: string;
+        timestamp: string;
+        summary: string;
+        tools_used: string;
+        files_referenced: string;
+      },
+      [string, number]
+    >(
+      `SELECT cc.snippet, cc.turn_id, rank,
+              ct.turn_index, ct.session_id, ct.timestamp, ct.summary, ct.tools_used, ct.files_referenced
+       FROM fts_conversation fts
+       JOIN conversation_chunks cc ON cc.id = fts.rowid
+       JOIN conversation_turns ct ON ct.id = cc.turn_id
+       WHERE fts_conversation MATCH ?
+       ORDER BY rank
+       LIMIT ?`
+    )
+    .all(sanitizeFTS(query), sessionId ? topK * 10 : topK * 3);
+
+  const results: ConversationSearchResult[] = [];
+  const seenTurns = new Set<number>();
+
+  for (const row of rows) {
+    if (seenTurns.has(row.turn_id)) continue;
+    seenTurns.add(row.turn_id);
+
+    if (sessionId && row.session_id !== sessionId) continue;
+
+    results.push({
+      turnId: row.turn_id,
+      turnIndex: row.turn_index,
+      sessionId: row.session_id,
+      timestamp: row.timestamp,
+      summary: row.summary || "",
+      snippet: row.snippet,
+      toolsUsed: JSON.parse(row.tools_used || "[]"),
+      filesReferenced: JSON.parse(row.files_referenced || "[]"),
+      score: 1 / (1 + Math.abs(row.rank)),
+    });
+
+    if (results.length >= topK) break;
+  }
+
+  return results;
+}

package/src/db/files.ts

@@ -0,0 +1,146 @@
+import { Database } from "bun:sqlite";
+import { type EmbeddedChunk } from "../types";
+import { type StoredFile } from "./types";
+
+export function getFileByPath(db: Database, path: string): StoredFile | null {
+  return db
+    .query<StoredFile, [string]>(
+      "SELECT id, path, hash, indexed_at as indexedAt FROM files WHERE path = ?"
+    )
+    .get(path);
+}
+
+export function upsertFileStart(db: Database, path: string, hash: string): number {
+  const existing = getFileByPath(db, path);
+  if (existing) {
+    // UPDATE instead of DELETE+INSERT to preserve files.id — this keeps
+    // file_imports.resolved_file_id FKs pointing at this file intact.
+    const tx = db.transaction(() => {
+      const oldChunks = db
+        .query<{ id: number }, [number]>("SELECT id FROM chunks WHERE file_id = ?")
+        .all(existing.id);
+      for (const c of oldChunks) {
+        db.run("DELETE FROM vec_chunks WHERE chunk_id = ?", [c.id]);
+      }
+      db.run("DELETE FROM chunks WHERE file_id = ?", [existing.id]);
+      db.run(
+        "UPDATE files SET hash = ?, indexed_at = ? WHERE id = ?",
+        [hash, new Date().toISOString(), existing.id]
+      );
+    });
+    tx();
+    return existing.id;
+  }
+
+  db.run(
+    "INSERT INTO files (path, hash, indexed_at) VALUES (?, ?, ?)",
+    [path, hash, new Date().toISOString()]
+  );
+  return Number(
+    db.query<{ id: number }, []>("SELECT last_insert_rowid() as id").get()!.id
+  );
+}
+
+export function insertChunkBatch(
+  db: Database,
+  fileId: number,
+  chunks: EmbeddedChunk[],
+  startIndex: number
+) {
+  const tx = db.transaction(() => {
+    for (let i = 0; i < chunks.length; i++) {
+      const { snippet, embedding, entityName, chunkType, startLine, endLine } = chunks[i];
+      db.run(
+        "INSERT INTO chunks (file_id, chunk_index, snippet, entity_name, chunk_type, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?)",
+        [fileId, startIndex + i, snippet, entityName ?? null, chunkType ?? null, startLine ?? null, endLine ?? null]
+      );
+      const chunkId = Number(
+        db.query<{ id: number }, []>("SELECT last_insert_rowid() as id").get()!.id
+      );
+      db.run(
+        "INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?, ?)",
+        [chunkId, new Uint8Array(embedding.buffer)]
+      );
+    }
+  });
+  tx();
+}
+
+export function upsertFile(
+  db: Database,
+  path: string,
+  hash: string,
+  chunks: EmbeddedChunk[]
+) {
+  const fileId = upsertFileStart(db, path, hash);
+  insertChunkBatch(db, fileId, chunks, 0);
+}
+
+export function removeFile(db: Database, path: string): boolean {
+  const existing = getFileByPath(db, path);
+  if (!existing) return false;
+
+  const tx = db.transaction(() => {
+    const oldChunks = db
+      .query<{ id: number }, [number]>("SELECT id FROM chunks WHERE file_id = ?")
+      .all(existing.id);
+    for (const c of oldChunks) {
+      db.run("DELETE FROM vec_chunks WHERE chunk_id = ?", [c.id]);
+    }
+    db.run("DELETE FROM chunks WHERE file_id = ?", [existing.id]);
+    db.run("DELETE FROM files WHERE id = ?", [existing.id]);
+  });
+
+  tx();
+  return true;
+}
+
+export function pruneDeleted(db: Database, existingPaths: Set<string>): number {
+  const allFiles = db
+    .query<{ id: number; path: string }, []>("SELECT id, path FROM files")
+    .all();
+
+  const toRemove = allFiles.filter(f => !existingPaths.has(f.path));
+  if (toRemove.length === 0) return 0;
+
+  const tx = db.transaction(() => {
+    for (const file of toRemove) {
+      const oldChunks = db
+        .query<{ id: number }, [number]>("SELECT id FROM chunks WHERE file_id = ?")
+        .all(file.id);
+      for (const c of oldChunks) {
+        db.run("DELETE FROM vec_chunks WHERE chunk_id = ?", [c.id]);
+      }
+      db.run("DELETE FROM chunks WHERE file_id = ?", [file.id]);
+      db.run("DELETE FROM files WHERE id = ?", [file.id]);
+    }
+  });
+  tx();
+  return toRemove.length;
+}
+
+export function getAllFilePaths(db: Database): { id: number; path: string }[] {
+  return db
+    .query<{ id: number; path: string }, []>("SELECT id, path FROM files")
+    .all();
+}
+
+export function getStatus(db: Database): { totalFiles: number; totalChunks: number; lastIndexed: string | null } {
+  const files = db
+    .query<{ count: number }, []>("SELECT COUNT(*) as count FROM files")
+    .get()!;
+  const chunks = db
+    .query<{ count: number }, []>("SELECT COUNT(*) as count FROM chunks")
+    .get()!;
+  const last = db
+    .query<{ indexed_at: string }, []>(
+      "SELECT indexed_at FROM files ORDER BY indexed_at DESC LIMIT 1"
+    )
+    .get();
+
+  return {
+    totalFiles: files.count,
+    totalChunks: chunks.count,
+    lastIndexed: last?.indexed_at ?? null,
+  };
+}