conare 0.5.10 → 0.5.12

package/dist/index.js

@@ -693,7 +693,7 @@ var init_api = __esm(() => {
   };
 });
 
-// ../node_modules/sisteransi/src/index.js
+// node_modules/sisteransi/src/index.js
 var require_src = __commonJS((exports, module) => {
   var ESC = "\x1B";
   var CSI = `${ESC}[`;
@@ -751,7 +751,7 @@ var require_src = __commonJS((exports, module) => {
   module.exports = { cursor, scroll, erase, beep };
 });
 
-// ../node_modules/picocolors/picocolors.js
+// node_modules/picocolors/picocolors.js
 var require_picocolors = __commonJS((exports, module) => {
   var p = process || {};
   var argv = p.argv || [];
@@ -821,7 +821,7 @@ var require_picocolors = __commonJS((exports, module) => {
   module.exports.createColors = createColors;
 });
 
-// ../node_modules/@clack/core/dist/index.mjs
+// node_modules/@clack/core/dist/index.mjs
 import { stdin as j, stdout as M } from "node:process";
 import * as g from "node:readline";
 import O from "node:readline";
@@ -1357,7 +1357,7 @@ var init_dist = __esm(() => {
   };
 });
 
-// ../node_modules/@clack/prompts/dist/index.mjs
+// node_modules/@clack/prompts/dist/index.mjs
 var exports_dist = {};
 __export(exports_dist, {
   updateSettings: () => cD,
@@ -1789,7 +1789,9 @@ var exports_interactive = {};
 __export(exports_interactive, {
   startSetup: () => startSetup,
   spinner: () => Y2,
+  showLocalChatsCounted: () => showLocalChatsCounted,
   showDetectedApps: () => showDetectedApps,
+  showCountingLocalChats: () => showCountingLocalChats,
   selectMcpTargets: () => selectMcpTargets,
   selectChatSources: () => selectChatSources,
   promptAuth: () => promptAuth,
@@ -1819,6 +1821,12 @@ function showDetectedApps(targets) {
   Me(targets.map((target) => `• ${target.label}: ${target.available === false ? "not detected" : formatDetectedCount(target.detectedCount, target.detectedCountApproximate)}`).join(`
 `), "Detected apps");
 }
+function showCountingLocalChats() {
+  M2.step("Counting local chats...");
+}
+function showLocalChatsCounted() {
+  M2.success("Local chats counted");
+}
 async function promptApiKey(options) {
   if (options.providedApiKey) {
     return options.providedApiKey;
@@ -1875,7 +1883,7 @@ async function promptAuth(options) {
 }
 async function selectChatSources(targets) {
   return ensureValue(await fe({
-    message: "Select chat sources",
+    message: "Select chat sources to index",
     required: false,
     initialValues: targets.filter((target) => target.available !== false && (target.detectedCount ?? 0) > 0).map((target) => target.id),
     options: targets.map((target) => ({
@@ -2337,12 +2345,13 @@ function walkCodexSessions(dir, memories, sessionIds, stats) {
 
 // src/ingest/cursor.ts
 init_shared();
+import { Buffer } from "node:buffer";
 import { existsSync as existsSync4, readFileSync as readFileSync4, statSync } from "node:fs";
 import { dirname, join as join4 } from "node:path";
 import { createRequire as createRequire2 } from "node:module";
 import { homedir as homedir4 } from "node:os";
-var MAX_DB_SIZE = 2 * 1024 * 1024 * 1024;
-var WARN_DB_SIZE = 500 * 1024 * 1024;
+var MAX_SQLJS_DB_SIZE = 2 * 1024 * 1024 * 1024;
+var LARGE_SQLJS_DB_SIZE = 500 * 1024 * 1024;
 var MIN_TURN_LEN2 = 50;
 function parseCursorTimestamp(value) {
   const parsed = parseTimestampMs(value);
@@ -2375,45 +2384,217 @@ function resolveSqlJsWasmDir(wasmDir) {
     return existsSync4(join4(installedBundleDir, "sql.js", "package.json")) ? installedBundleDir : undefined;
   }
 }
-function openDb(initSqlJs, dbPath, wasmDir) {
+function toText(value) {
+  if (typeof value === "string")
+    return value;
+  if (value instanceof Uint8Array)
+    return Buffer.from(value).toString("utf-8");
+  if (value === null || value === undefined)
+    return "";
+  return String(value);
+}
+function normalizeKvRow(row) {
+  const key = toText(row.key);
+  const value = toText(row.value);
+  return key ? { key, value } : null;
+}
+function normalizeRows(rows) {
+  return rows.map(normalizeKvRow).filter((row) => row !== null);
+}
+async function openBunSqliteDb(dbPath) {
+  let raw;
+  try {
+    const mod = await import("bun:sqlite");
+    const Database = mod.Database;
+    if (!Database)
+      return null;
+    raw = new Database(dbPath, { readonly: true });
+    const composerStmt = raw.query("SELECT key, value FROM cursorDiskKV WHERE key LIKE ?");
+    const valueStmt = raw.query("SELECT value FROM cursorDiskKV WHERE key = ? LIMIT 1");
+    return {
+      kind: "bun:sqlite",
+      selectComposerData() {
+        return normalizeRows(composerStmt.all("composerData:%"));
+      },
+      selectValue(key) {
+        const row = valueStmt.get(key);
+        if (!row)
+          return null;
+        const value = toText(row.value);
+        return value || null;
+      },
+      close() {
+        raw.close();
+      }
+    };
+  } catch {
+    try {
+      raw?.close();
+    } catch {}
+    return null;
+  }
+}
+function openBetterSqliteDb(dbPath) {
+  let raw;
+  try {
+    const require2 = createRequire2(import.meta.url);
+    const BetterSqlite3 = require2("better-sqlite3");
+    raw = new BetterSqlite3(dbPath, { readonly: true, fileMustExist: true });
+    raw.pragma("query_only = ON");
+    const composerStmt = raw.prepare("SELECT key, value FROM cursorDiskKV WHERE key LIKE ?");
+    const valueStmt = raw.prepare("SELECT value FROM cursorDiskKV WHERE key = ? LIMIT 1");
+    return {
+      kind: "better-sqlite3",
+      selectComposerData() {
+        return normalizeRows(composerStmt.all("composerData:%"));
+      },
+      selectValue(key) {
+        const row = valueStmt.get(key);
+        if (!row)
+          return null;
+        const value = toText(row.value);
+        return value || null;
+      },
+      close() {
+        raw.close();
+      }
+    };
+  } catch {
+    try {
+      raw?.close();
+    } catch {}
+    return null;
+  }
+}
+async function importNodeSqlite() {
+  const originalEmitWarning = process.emitWarning;
+  try {
+    process.emitWarning = function(warning, ...args) {
+      const message = typeof warning === "string" ? warning : warning && typeof warning === "object" && ("message" in warning) ? String(warning.message) : "";
+      const type = typeof args[0] === "string" ? args[0] : args[0]?.type;
+      if (type === "ExperimentalWarning" && message.includes("SQLite"))
+        return;
+      return originalEmitWarning.call(process, warning, ...args);
+    };
+    return await import("node:sqlite");
+  } catch {
+    return null;
+  } finally {
+    process.emitWarning = originalEmitWarning;
+  }
+}
+async function openNodeSqliteDb(dbPath) {
+  let raw;
+  try {
+    const mod = await importNodeSqlite();
+    const DatabaseSync = mod?.DatabaseSync;
+    if (!DatabaseSync)
+      return null;
+    raw = new DatabaseSync(dbPath, { readOnly: true });
+    const composerStmt = raw.prepare("SELECT key, value FROM cursorDiskKV WHERE key LIKE ?");
+    const valueStmt = raw.prepare("SELECT value FROM cursorDiskKV WHERE key = ? LIMIT 1");
+    return {
+      kind: "node:sqlite",
+      selectComposerData() {
+        return normalizeRows(composerStmt.all("composerData:%"));
+      },
+      selectValue(key) {
+        const row = valueStmt.get(key);
+        if (!row)
+          return null;
+        const value = toText(row.value);
+        return value || null;
+      },
+      close() {
+        raw.close();
+      }
+    };
+  } catch {
+    try {
+      raw?.close();
+    } catch {}
+    return null;
+  }
+}
+function allowLargeSqlJsLoad() {
+  const value = process.env.CONARE_CURSOR_ALLOW_SQLJS_LARGE;
+  return value === "1" || value === "true";
+}
+async function openSqlJsDb(initSqlJs, dbPath, wasmDir) {
   const locateOpts = {};
   if (wasmDir) {
     locateOpts.locateFile = (file) => join4(wasmDir, "sql.js", "dist", file);
   }
-  return initSqlJs(locateOpts).then((SQL) => {
-    const buffer = readFileSync4(dbPath);
-    return new SQL.Database(buffer);
-  });
+  const SQL = await initSqlJs(locateOpts);
+  const buffer = readFileSync4(dbPath);
+  const raw = new SQL.Database(buffer);
+  function select(sql, params) {
+    const stmt = raw.prepare(sql);
+    try {
+      stmt.bind(params);
+      const rows = [];
+      while (stmt.step()) {
+        rows.push(stmt.getAsObject());
+      }
+      return rows;
+    } finally {
+      stmt.free();
+    }
+  }
+  return {
+    kind: "sql.js",
+    selectComposerData() {
+      return normalizeRows(select("SELECT key, value FROM cursorDiskKV WHERE key LIKE ?", ["composerData:%"]));
+    },
+    selectValue(key) {
+      const row = select("SELECT value FROM cursorDiskKV WHERE key = ? LIMIT 1", [key])[0];
+      if (!row)
+        return null;
+      const value = toText(row.value);
+      return value || null;
+    },
+    close() {
+      raw.close();
+    }
+  };
 }
-async function countImportableCursorSessions(dbPath, wasmDir) {
-  try {
-    const fileSize = statSync(dbPath).size;
-    if (fileSize > MAX_DB_SIZE)
-      return 0;
-  } catch {
-    return 0;
+async function openCursorKvDb(dbPath, fileSize, wasmDir) {
+  const nativeDb = await openBunSqliteDb(dbPath) || openBetterSqliteDb(dbPath) || await openNodeSqliteDb(dbPath);
+  if (nativeDb)
+    return { db: nativeDb };
+  if (fileSize > MAX_SQLJS_DB_SIZE)
+    return { db: null, reason: "sqljs-too-large" };
+  if (fileSize > LARGE_SQLJS_DB_SIZE && !allowLargeSqlJsLoad()) {
+    return { db: null, reason: "sqljs-large" };
   }
   const effectiveWasmDir = resolveSqlJsWasmDir(wasmDir);
   const initSqlJs = loadSqlJs(effectiveWasmDir);
   if (!initSqlJs)
-    return 0;
-  let db;
+    return { db: null, reason: "no-driver" };
+  try {
+    return { db: await openSqlJsDb(initSqlJs, dbPath, effectiveWasmDir) };
+  } catch (e) {
+    return { db: null, reason: "open-failed", message: e?.message || String(e) };
+  }
+}
+function formatMiB(bytes) {
+  return (bytes / 1024 / 1024).toFixed(0);
+}
+async function countImportableCursorSessions(dbPath, wasmDir) {
+  let fileSize;
   try {
-    db = await openDb(initSqlJs, dbPath, effectiveWasmDir);
+    fileSize = statSync(dbPath).size;
   } catch {
     return 0;
   }
+  const openResult = await openCursorKvDb(dbPath, fileSize, wasmDir);
+  if (!openResult.db)
+    return 0;
+  const db = openResult.db;
   try {
-    let rows = [];
-    try {
-      const results = db.exec("SELECT key, value FROM cursorDiskKV WHERE key LIKE 'composerData:%'");
-      if (results.length > 0)
-        rows = results[0].values;
-    } catch {
-      return 0;
-    }
+    const rows = db.selectComposerData();
     let count = 0;
-    for (const [key, value] of rows) {
+    for (const { key, value } of rows) {
       const composerId = key.replace("composerData:", "");
       let parsed;
       try {
@@ -2441,11 +2622,11 @@ function extractTurns(db, composerId, bubbleHeaders) {
   for (const header of bubbleHeaders) {
     let text = "";
     try {
-      const result = db.exec(`SELECT value FROM cursorDiskKV WHERE key = 'bubbleId:${composerId}:${header.bubbleId}'`);
-      if (result.length > 0) {
-        const bubble = JSON.parse(result[0].values[0][0]);
-        text = cleanText(bubble.text || "");
-      }
+      const rawBubble = db.selectValue(`bubbleId:${composerId}:${header.bubbleId}`);
+      if (!rawBubble)
+        continue;
+      const bubble = JSON.parse(rawBubble);
+      text = cleanText(bubble.text || "");
     } catch {
       continue;
     }
@@ -2472,34 +2653,33 @@ async function ingestCursor(dbPath, wasmDir) {
     console.log("  Skipping Cursor: database not accessible");
     return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
-  if (fileSize > MAX_DB_SIZE) {
-    console.log(`  Skipping Cursor: database too large (${(fileSize / 1024 / 1024 / 1024).toFixed(1)}GB)`);
+  const openResult = await openCursorKvDb(dbPath, fileSize, wasmDir);
+  if (!openResult.db) {
+    if (openResult.reason === "sqljs-too-large") {
+      console.log(`  Skipping Cursor: database too large (${(fileSize / 1024 / 1024 / 1024).toFixed(1)}GB)`);
+    } else if (openResult.reason === "sqljs-large") {
+      console.log(`  Skipping Cursor: large database (${formatMiB(fileSize)}MB) requires streaming SQLite support; not loading it into memory`);
+    } else if (openResult.reason === "no-driver") {
+      console.log("  Skipping Cursor: SQLite support not available");
+    } else {
+      console.log(`  Skipping Cursor: cannot open database (${openResult.message || "unknown error"})`);
+    }
     return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
-  if (fileSize > WARN_DB_SIZE) {
-    console.log(`  Warning: large database (${(fileSize / 1024 / 1024).toFixed(0)}MB), loading into memory...`);
+  if (openResult.db.kind === "sql.js" && fileSize > LARGE_SQLJS_DB_SIZE) {
+    console.log(`  Warning: large database (${formatMiB(fileSize)}MB), loading into memory via sql.js fallback...`);
   }
-  const effectiveWasmDir = resolveSqlJsWasmDir(wasmDir);
-  const initSqlJs = loadSqlJs(effectiveWasmDir);
-  if (!initSqlJs) {
-    console.log("  Skipping Cursor: sql.js not available");
-    return { memories, sessionIds, skipped: 0, filtered, deduped };
-  }
-  let db;
-  try {
-    db = await openDb(initSqlJs, dbPath, effectiveWasmDir);
-  } catch (e) {
-    console.log(`  Skipping Cursor: cannot open database (${e.message})`);
+  const db = openResult.db;
+  if (fileSize > MAX_SQLJS_DB_SIZE && db.kind === "sql.js") {
+    console.log(`  Skipping Cursor: database too large (${(fileSize / 1024 / 1024 / 1024).toFixed(1)}GB)`);
     return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
   try {
     let rows = [];
     try {
-      const results = db.exec("SELECT key, value FROM cursorDiskKV WHERE key LIKE 'composerData:%'");
-      if (results.length > 0)
-        rows = results[0].values;
+      rows = db.selectComposerData();
     } catch {}
-    for (const [key, value] of rows) {
+    for (const { key, value } of rows) {
       const composerId = key.replace("composerData:", "");
       let parsed;
       try {
@@ -3190,6 +3370,7 @@ import { existsSync as existsSync9, mkdirSync as mkdirSync4, writeFileSync as wr
 import { join as join9, dirname as dirname3 } from "node:path";
 import { homedir as homedir8, platform as platform6 } from "node:os";
 import { execSync as execSync3 } from "node:child_process";
+import { createRequire as createRequire3 } from "node:module";
 var CONARE_DIR = join9(homedir8(), ".conare");
 var BIN_DIR = join9(CONARE_DIR, "bin");
 var CONFIG_PATH2 = join9(CONARE_DIR, "config.json");
@@ -3342,12 +3523,9 @@ function persistBinary(apiKey) {
   const dest = join9(BIN_DIR, "conare-ingest.mjs");
   const content = readFileSync8(cliEntry, "utf-8");
   writeFileSync4(dest, content);
-  const sqlJsDir = findSqlJs();
-  if (sqlJsDir) {
-    const targetDir = join9(BIN_DIR, "node_modules", "sql.js");
-    mkdirSync4(targetDir, { recursive: true });
+  for (const moduleName of ["sql.js", "better-sqlite3", "bindings", "file-uri-to-path"]) {
     try {
-      cpSync(sqlJsDir, targetDir, { recursive: true });
+      copyNodeModule(moduleName);
     } catch {}
   }
   const runShPath = join9(BIN_DIR, "run.sh");
@@ -3387,11 +3565,15 @@ function findCliBundle() {
     return entry;
   return null;
 }
-function findSqlJs() {
+function findNodeModule(name) {
+  try {
+    const require2 = createRequire3(import.meta.url);
+    return dirname3(require2.resolve(`${name}/package.json`));
+  } catch {}
   const candidates = [
-    join9(process.cwd(), "node_modules", "sql.js"),
-    join9(dirname3(new URL(import.meta.url).pathname), "..", "node_modules", "sql.js"),
-    join9(dirname3(new URL(import.meta.url).pathname), "..", "..", "node_modules", "sql.js")
+    join9(process.cwd(), "node_modules", name),
+    join9(dirname3(new URL(import.meta.url).pathname), "..", "node_modules", name),
+    join9(dirname3(new URL(import.meta.url).pathname), "..", "..", "node_modules", name)
   ];
   for (const c of candidates) {
     if (existsSync9(c))
@@ -3399,6 +3581,15 @@ function findSqlJs() {
   }
   return null;
 }
+function copyNodeModule(name) {
+  const sourceDir = findNodeModule(name);
+  if (!sourceDir)
+    return;
+  const targetDir = join9(BIN_DIR, "node_modules", name);
+  rmSync2(targetDir, { recursive: true, force: true });
+  mkdirSync4(dirname3(targetDir), { recursive: true });
+  cpSync(sourceDir, targetDir, { recursive: true });
+}
 function cleanupOldLaunchAgent() {
   try {
     execSync3(`launchctl bootout gui/${uid()} "${PLIST_PATH}" 2>/dev/null`, { stdio: "ignore" });
@@ -3969,10 +4160,9 @@ async function main() {
         apiKey = authResult || apiKey;
       }
     }
-    const detectSpinner = Y2();
-    detectSpinner.start("Detecting apps...");
+    showCountingLocalChats();
     const detectedTools = await detect();
-    detectSpinner.stop("Apps detected");
+    showLocalChatsCounted();
     interactiveTargets = MCP_TARGETS.map((target) => {
       const detected = detectedTools.find((tool) => tool.id === target.id);
       return {

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "conare",
-  "version": "0.5.10",
+  "version": "0.5.12",
   "description": "Conare CLI for indexing AI chat history and configuring memory at conare.ai",
   "type": "module",
   "bin": {
     "conare": "./dist/index.js"
   },
   "scripts": {
-    "build": "bun build src/index.ts --outdir dist --target node --format esm --external sql.js",
+    "build": "bun build src/index.ts --outdir dist --target node --format esm --external sql.js --external better-sqlite3",
     "prepublishOnly": "bun run build"
   },
   "files": [
@@ -28,5 +28,8 @@
   "dependencies": {
     "@clack/prompts": "^0.11.0",
     "sql.js": "^1.12.0"
+  },
+  "optionalDependencies": {
+    "better-sqlite3": "^12.10.0"
   }
 }