@chendpoc/pi-memory 0.1.13 → 0.2.1

package/src/fallback/sessionIndex.ts

@@ -1,290 +0,0 @@
-import { createHash } from "node:crypto";
-import fs from "node:fs/promises";
-import { createRequire } from "node:module";
-import path from "node:path";
-import type { SessionSearchHit } from "./sessionSearch.js";
-import { messageText, parseJsonlSession as parseActiveBranchJsonl } from "../session/activeBranch.js";
-
-export interface SqliteDatabase {
-  pragma(sql: string): unknown;
-  exec(sql: string): void;
-  prepare(sql: string): {
-    run(...args: unknown[]): unknown;
-    get(...args: unknown[]): unknown;
-    all(...args: unknown[]): unknown[];
-  };
-  transaction<T>(fn: () => T): () => T;
-  close(): void;
-}
-
-type SqliteConstructor = new (path: string) => SqliteDatabase;
-
-/**
- * Lazy-loaded better-sqlite3 via createRequire (ESM-compatible).
- * Returns null if the package is not installed (graceful degradation).
- */
-function loadSqlite(): SqliteConstructor | null {
-  try {
-    const require = createRequire(import.meta.url);
-    const mod = require("better-sqlite3") as SqliteConstructor;
-    return mod;
-  } catch {
-    return null;
-  }
-}
-
-interface PiSessionMessage {
-  role?: string;
-  content?: unknown;
-}
-
-interface PiSessionFile {
-  id?: string;
-  title?: string;
-  created_at?: string;
-  messages?: PiSessionMessage[];
-}
-
-export interface SessionIndex {
-  /** Full rebuild — drop and re-populate from all session files. */
-  rebuildIndex(sessionsDir: string): Promise<{ indexed: number }>;
-  /** Incremental — only index sessions modified after lastIndexedTs. */
-  incrementalIndex(sessionsDir: string, lastIndexedTs?: Date | null): Promise<{ indexed: number }>;
-  /** FTS5 search returning the same shape as sessionKeywordSearch. */
-  search(query: string, limit: number): SessionSearchHit[];
-  /** Retrieve the last-indexed timestamp from DB metadata. */
-  getLastIndexedTs(): Date | null;
-  /** Close the database connection. */
-  close(): void;
-}
-
-const SCHEMA_SQL = `
-CREATE TABLE IF NOT EXISTS meta (
-  key TEXT PRIMARY KEY,
-  value TEXT
-);
-CREATE VIRTUAL TABLE IF NOT EXISTS session_fts USING fts5(
-  session_id,
-  turn_idx,
-  role,
-  content,
-  session_title,
-  created_at,
-  tokenize='unicode61'
-);
-`;
-
-/**
- * Open (or create) the SQLite FTS5 index.
- * Pass ":memory:" as dbPath for tests.
- * Optionally pass a pre-built SqliteDatabase for testing without native deps.
- */
-export function openSessionIndex(dbPath: string, injectedDb?: SqliteDatabase): SessionIndex | null {
-  let db: SqliteDatabase;
-  if (injectedDb) {
-    db = injectedDb;
-  } else {
-    const Sqlite = loadSqlite();
-    if (!Sqlite) return null;
-    db = new Sqlite(dbPath);
-  }
-  db.pragma("journal_mode = WAL");
-  db.exec(SCHEMA_SQL);
-
-  function setMeta(key: string, value: string): void {
-    db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)").run(key, value);
-  }
-
-  function getMeta(key: string): string | null {
-    const row = db.prepare("SELECT value FROM meta WHERE key = ?").get(key) as
-      | { value: string }
-      | undefined;
-    return row?.value ?? null;
-  }
-
-  function insertTurns(sessions: Array<{
-    id: string;
-    title: string;
-    createdAt: string;
-    messages: Array<{ role: string; content: string; index: number }>;
-  }>): number {
-    const insert = db.prepare(
-      "INSERT INTO session_fts (session_id, turn_idx, role, content, session_title, created_at) VALUES (?, ?, ?, ?, ?, ?)",
-    );
-    let count = 0;
-    const seenContent = new Set<string>();
-    const tx = db.transaction(() => {
-      for (const s of sessions) {
-        for (const m of s.messages) {
-          if (!m.content.trim()) continue;
-          const fingerprint = createHash("sha256")
-            .update(m.content)
-            .digest("hex")
-            .slice(0, 16);
-          if (seenContent.has(fingerprint)) continue;
-          seenContent.add(fingerprint);
-          insert.run(s.id, String(m.index), m.role, m.content, s.title, s.createdAt);
-          count++;
-        }
-      }
-    });
-    tx();
-    return count;
-  }
-
-  async function collectFiles(dir: string): Promise<string[]> {
-    let names: string[];
-    try {
-      names = await fs.readdir(dir);
-    } catch {
-      return [];
-    }
-    const files: string[] = [];
-    for (const name of names) {
-      const full = path.join(dir, name);
-      let st;
-      try { st = await fs.stat(full); } catch { continue; }
-      if (st.isDirectory()) {
-        files.push(...await collectFiles(full));
-      } else if (st.isFile() && (name.endsWith(".json") || name.endsWith(".jsonl"))) {
-        files.push(full);
-      }
-    }
-    return files;
-  }
-
-  async function loadSessionFiles(sessionsDir: string, modifiedAfter?: Date | null): Promise<Array<{
-    id: string;
-    title: string;
-    createdAt: string;
-    messages: Array<{ role: string; content: string; index: number }>;
-  }>> {
-    const filePaths = await collectFiles(sessionsDir);
-    const results: Array<{
-      id: string; title: string; createdAt: string;
-      messages: Array<{ role: string; content: string; index: number }>;
-    }> = [];
-
-    for (const filePath of filePaths) {
-      let st;
-      try { st = await fs.stat(filePath); } catch { continue; }
-      if (!st.isFile()) continue;
-      if (modifiedAfter && st.mtime <= modifiedAfter) continue;
-
-      let raw: string;
-      try { raw = await fs.readFile(filePath, "utf8"); } catch { continue; }
-
-      if (filePath.endsWith(".jsonl")) {
-        const parsed = parseActiveBranchJsonl(raw, filePath);
-        if (!parsed) continue;
-        const messages = parsed.turns.map((turn) => ({
-          role: turn.role,
-          content: turn.content,
-          index: turn.turnIndex,
-        }));
-        results.push({
-          id: parsed.id,
-          title: parsed.title,
-          createdAt: parsed.createdAt,
-          messages,
-        });
-      } else {
-        let session: PiSessionFile;
-        try { session = JSON.parse(raw) as PiSessionFile; } catch { continue; }
-        const sessionId = session.id ?? path.basename(filePath, ".json");
-        const messages: Array<{ role: string; content: string; index: number }> = [];
-        for (let i = 0; i < (session.messages?.length ?? 0); i++) {
-          const msg = session.messages![i]!;
-          const text = messageText(msg.content);
-          if (text.trim()) {
-            messages.push({ role: msg.role ?? "unknown", content: text, index: i });
-          }
-        }
-        if (messages.length > 0) {
-          results.push({ id: sessionId, title: session.title ?? "", createdAt: session.created_at ?? "", messages });
-        }
-      }
-    }
-    return results;
-  }
-
-  return {
-    async rebuildIndex(sessionsDir: string): Promise<{ indexed: number }> {
-      db.exec("DELETE FROM session_fts");
-      const sessions = await loadSessionFiles(sessionsDir);
-      const indexed = insertTurns(sessions);
-      setMeta("last_indexed_ts", new Date().toISOString());
-      return { indexed };
-    },
-
-    async incrementalIndex(sessionsDir: string, lastIndexedTs?: Date | null): Promise<{ indexed: number }> {
-      const effectiveTs = lastIndexedTs ??
-        (() => {
-          const raw = getMeta("last_indexed_ts");
-          return raw ? new Date(raw) : null;
-        })();
-      const sessions = await loadSessionFiles(sessionsDir, effectiveTs);
-      if (sessions.length === 0) return { indexed: 0 };
-
-      const sessionIds = sessions.map((s) => s.id);
-      const deletePlaceholders = sessionIds.map(() => "?").join(",");
-      db.prepare(`DELETE FROM session_fts WHERE session_id IN (${deletePlaceholders})`).run(...sessionIds);
-
-      const indexed = insertTurns(sessions);
-      setMeta("last_indexed_ts", new Date().toISOString());
-      return { indexed };
-    },
-
-    search(query: string, limit: number): SessionSearchHit[] {
-      const q = query.trim();
-      if (!q || limit <= 0) return [];
-
-      const ftsQuery = q
-        .split(/\s+/)
-        .filter(Boolean)
-        .map((term) => `"${term.replace(/"/g, '""')}"`)
-        .join(" AND ");
-
-      if (!ftsQuery) return [];
-
-      try {
-        const rows = db.prepare(
-          `SELECT session_id, turn_idx, role, snippet(session_fts, 3, '>>>', '<<<', '...', 40) as snip, session_title, created_at
-           FROM session_fts
-           WHERE session_fts MATCH ?
-           ORDER BY rank
-           LIMIT ?`,
-        ).all(ftsQuery, limit) as Array<{
-          session_id: string;
-          turn_idx: string;
-          role: string;
-          snip: string;
-          session_title: string;
-          created_at: string;
-        }>;
-
-        return rows.map((r) => ({
-          session_id: r.session_id,
-          session_title: r.session_title,
-          role: r.role,
-          snippet: r.snip.replace(/>>>/g, "").replace(/<<</g, ""),
-          msg_index: parseInt(r.turn_idx, 10),
-          created_at: r.created_at,
-        }));
-      } catch {
-        return [];
-      }
-    },
-
-    getLastIndexedTs(): Date | null {
-      const raw = getMeta("last_indexed_ts");
-      if (!raw) return null;
-      const d = new Date(raw);
-      return isNaN(d.getTime()) ? null : d;
-    },
-
-    close(): void {
-      db.close();
-    },
-  };
-}

package/src/fallback/sessionSearch.ts

@@ -1,261 +0,0 @@
-import fsSync from "node:fs";
-import fs from "node:fs/promises";
-import path from "node:path";
-import { openSessionIndex, type SessionIndex } from "./sessionIndex.js";
-
-/** Mirrors Kocoro session.SearchResult for fallback hits. */
-export interface SessionSearchHit {
-  session_id: string;
-  session_title: string;
-  role: string;
-  snippet: string;
-  msg_index: number;
-  created_at: string;
-}
-
-let cachedIndex: SessionIndex | null = null;
-let cachedDbPath: string | null = null;
-
-function getSessionIndex(dbPath: string): SessionIndex | null {
-  if (cachedIndex && cachedDbPath === dbPath) return cachedIndex;
-  if (!fsSync.existsSync(dbPath)) return null;
-  cachedIndex = openSessionIndex(dbPath);
-  cachedDbPath = dbPath;
-  return cachedIndex;
-}
-
-/** Default session DB path. */
-export function defaultSessionDbPath(sessionsDir: string): string {
-  return path.join(path.dirname(sessionsDir), "memory", "sessions.db");
-}
-
-interface PiSessionMessage {
-  role?: string;
-  content?: unknown;
-}
-
-interface PiSessionFile {
-  id?: string;
-  title?: string;
-  created_at?: string;
-  messages?: PiSessionMessage[];
-}
-
-const SNIPPET_MAX = 240;
-
-async function collectFiles(dir: string): Promise<string[]> {
-  let names: string[];
-  try {
-    names = await fs.readdir(dir);
-  } catch {
-    return [];
-  }
-  const files: string[] = [];
-  for (const name of names) {
-    const full = path.join(dir, name);
-    let st;
-    try { st = await fs.stat(full); } catch { continue; }
-    if (st.isDirectory()) {
-      files.push(...await collectFiles(full));
-    } else if (st.isFile() && (name.endsWith(".json") || name.endsWith(".jsonl"))) {
-      files.push(full);
-    }
-  }
-  return files;
-}
-
-/**
- * Keyword search over Pi-style session files (JSON + JSONL, recursive subdirectories).
- * Uses FTS5 index when available, falls back to file scan.
- * All whitespace-separated terms must match (case-insensitive AND).
- */
-export async function sessionKeywordSearch(
-  sessionsDir: string,
-  query: string,
-  limit: number,
-): Promise<SessionSearchHit[]> {
-  if (!sessionsDir.trim()) return [];
-  const q = query.trim();
-  if (!q) return [];
-  if (limit <= 0) limit = 20;
-
-  const dbPath = defaultSessionDbPath(sessionsDir);
-  const idx = getSessionIndex(dbPath);
-  if (idx) {
-    const results = idx.search(q, limit);
-    if (results.length > 0) return results;
-  }
-
-  const terms = splitTerms(q);
-  if (terms.length === 0) return [];
-
-  const filePaths = await collectFiles(sessionsDir);
-  const hits: SessionSearchHit[] = [];
-
-  for (const filePath of filePaths) {
-    let st;
-    try {
-      st = await fs.stat(filePath);
-    } catch {
-      continue;
-    }
-    if (!st.isFile()) continue;
-
-    let raw: string;
-    try {
-      raw = await fs.readFile(filePath, "utf8");
-    } catch {
-      continue;
-    }
-
-    if (filePath.endsWith(".jsonl")) {
-      scanJsonlFile(raw, filePath, terms, hits, limit);
-    } else {
-      scanJsonFile(raw, filePath, terms, hits, limit);
-    }
-
-    if (hits.length >= limit) return hits;
-  }
-  return hits;
-}
-
-function scanJsonFile(
-  raw: string,
-  filePath: string,
-  terms: string[],
-  hits: SessionSearchHit[],
-  limit: number,
-): void {
-  let session: PiSessionFile;
-  try {
-    session = JSON.parse(raw) as PiSessionFile;
-  } catch {
-    return;
-  }
-
-  const sessionId = session.id ?? path.basename(filePath, ".json");
-  const title = session.title ?? "";
-  const createdAt = session.created_at ?? "";
-
-  for (let i = 0; i < (session.messages?.length ?? 0); i++) {
-    const msg = session.messages![i]!;
-    const text = messageText(msg.content);
-    if (!text || !allTermsMatch(text, terms)) continue;
-    hits.push({
-      session_id: sessionId,
-      session_title: title,
-      role: msg.role ?? "unknown",
-      snippet: makeSnippet(text, terms[0]!),
-      msg_index: i,
-      created_at: createdAt,
-    });
-    if (hits.length >= limit) return;
-  }
-}
-
-function scanJsonlFile(
-  raw: string,
-  filePath: string,
-  terms: string[],
-  hits: SessionSearchHit[],
-  limit: number,
-): void {
-  const lines = raw.split("\n").filter((l) => l.trim());
-  let sessionId = path.basename(filePath, ".jsonl");
-  let title = "";
-  let createdAt = "";
-  let msgIndex = 0;
-
-  for (const line of lines) {
-    let obj: Record<string, unknown>;
-    try {
-      obj = JSON.parse(line) as Record<string, unknown>;
-    } catch {
-      continue;
-    }
-
-    if (obj.type === "session") {
-      sessionId = (obj.id as string) ?? sessionId;
-      title = (obj.title as string) ?? "";
-      createdAt = (obj.timestamp as string) ?? "";
-      continue;
-    }
-
-    if (obj.type === "message") {
-      const msg = (obj as { message?: PiSessionMessage }).message;
-      if (!msg?.role || !msg.content) continue;
-      if (msg.role !== "user" && msg.role !== "assistant") continue;
-      const text = messageText(msg.content);
-      if (!text || !allTermsMatch(text, terms)) continue;
-      hits.push({
-        session_id: sessionId,
-        session_title: title,
-        role: msg.role,
-        snippet: makeSnippet(text, terms[0]!),
-        msg_index: msgIndex,
-        created_at: createdAt,
-      });
-      msgIndex++;
-      if (hits.length >= limit) return;
-    }
-  }
-}
-
-function splitTerms(query: string): string[] {
-  const out: string[] = [];
-  let cur = "";
-  let inQuote = false;
-  for (const ch of query) {
-    if (ch === '"') {
-      inQuote = !inQuote;
-      continue;
-    }
-    if (!inQuote && /\s/.test(ch)) {
-      if (cur) {
-        out.push(cur.toLowerCase());
-        cur = "";
-      }
-      continue;
-    }
-    cur += ch;
-  }
-  if (cur) out.push(cur.toLowerCase());
-  return out;
-}
-
-function messageText(content: unknown): string {
-  if (typeof content === "string") return content;
-  if (!Array.isArray(content)) return "";
-  const parts: string[] = [];
-  for (const block of content) {
-    if (typeof block === "string") {
-      parts.push(block);
-      continue;
-    }
-    if (block && typeof block === "object") {
-      const b = block as Record<string, unknown>;
-      if (typeof b.text === "string") parts.push(b.text);
-      else if (typeof b.content === "string") parts.push(b.content);
-    }
-  }
-  return parts.join("\n");
-}
-
-function allTermsMatch(text: string, terms: string[]): boolean {
-  const lower = text.toLowerCase();
-  return terms.every((t) => lower.includes(t));
-}
-
-function makeSnippet(text: string, firstTerm: string): string {
-  const lower = text.toLowerCase();
-  const idx = lower.indexOf(firstTerm.toLowerCase());
-  if (idx < 0) {
-    return text.length <= SNIPPET_MAX ? text : text.slice(0, SNIPPET_MAX) + "...";
-  }
-  const start = Math.max(0, idx - 60);
-  const end = Math.min(text.length, idx + firstTerm.length + 120);
-  let snip = text.slice(start, end);
-  if (start > 0) snip = "..." + snip;
-  if (end < text.length) snip += "...";
-  return snip;
-}