conare 0.1.1 → 0.1.2

package/dist/index.js

@@ -376,12 +376,13 @@ function ingestClaude() {
   const projectsDir = join3(homedir3(), ".claude", "projects");
   const memories = [];
   const sessionIds = [];
-  let skipped = 0;
+  let filtered = 0;
+  let deduped = 0;
   let projectDirs;
   try {
     projectDirs = readdirSync2(projectsDir);
   } catch {
-    return { memories, sessionIds, skipped };
+    return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
   for (const projDir of projectDirs) {
     const projPath = join3(projectsDir, projDir);
@@ -398,7 +399,7 @@ function ingestClaude() {
       const { turns, date } = parseSession(raw.split(`
 `));
       if (turns.length === 0) {
-        skipped++;
+        filtered++;
         continue;
       }
       const header = `# Chat: ${project}${date ? ` | ${date}` : ""}`;
@@ -423,7 +424,7 @@ ${body}`;
       const dedupKey = `claude:${sessionId}`;
       const fingerprint = `${dedupKey}:${contentHash}`;
       if (isIngested("claude", fingerprint)) {
-        skipped++;
+        deduped++;
         continue;
       }
       memories.push({
@@ -441,7 +442,7 @@ ${body}`;
       sessionIds.push(sessionId);
     }
   }
-  return { memories, sessionIds, skipped };
+  return { memories, sessionIds, skipped: filtered + deduped, filtered, deduped };
 }
 
 // src/ingest/codex.ts
@@ -452,7 +453,8 @@ var MAX_CONTENT2 = 48000;
 function ingestCodex() {
   const memories = [];
   const sessionIds = [];
-  let skipped = 0;
+  let filtered = 0;
+  let deduped = 0;
   const historyPath = join4(homedir4(), ".codex", "history.jsonl");
   if (existsSync3(historyPath)) {
     try {
@@ -490,14 +492,14 @@ ${body}`;
 
 [truncated]`;
         if (content.length < 100) {
-          skipped++;
+          filtered++;
           continue;
         }
         const contentHash = createContentHash(content);
         const dedupKey = `codex:${sessionId}`;
         const fingerprint = `${dedupKey}:${contentHash}`;
         if (isIngested("codex", fingerprint)) {
-          skipped++;
+          deduped++;
           continue;
         }
         memories.push({
@@ -512,16 +514,19 @@ ${body}`;
   const sessionsDir = join4(homedir4(), ".codex", "sessions");
   if (existsSync3(sessionsDir)) {
     try {
-      walkCodexSessions(sessionsDir, memories, sessionIds, skipped);
+      const stats = { filtered: 0, deduped: 0 };
+      walkCodexSessions(sessionsDir, memories, sessionIds, stats);
+      filtered += stats.filtered;
+      deduped += stats.deduped;
     } catch {}
   }
-  return { memories, sessionIds, skipped };
+  return { memories, sessionIds, skipped: filtered + deduped, filtered, deduped };
 }
-function walkCodexSessions(dir, memories, sessionIds, skipped) {
+function walkCodexSessions(dir, memories, sessionIds, stats) {
   try {
     for (const entry of readdirSync3(dir, { withFileTypes: true })) {
       if (entry.isDirectory()) {
-        walkCodexSessions(join4(dir, entry.name), memories, sessionIds, skipped);
+        walkCodexSessions(join4(dir, entry.name), memories, sessionIds, stats);
       } else if (entry.name.endsWith(".jsonl")) {
         const sessionId = basename2(entry.name, ".jsonl");
         if (sessionIds.includes(sessionId))
@@ -552,8 +557,10 @@ function walkCodexSessions(dir, memories, sessionIds, skipped) {
               continue;
             }
           }
-          if (turns.length === 0)
+          if (turns.length === 0) {
+            stats.filtered++;
             continue;
+          }
           const body = turns.map((t) => t.length > 500 ? t.slice(0, 500) + "..." : t).join(`
 
 ---
@@ -570,6 +577,7 @@ ${body}`;
           const dedupKey = `codex:${sessionId}`;
           const fingerprint = `${dedupKey}:${contentHash}`;
           if (isIngested("codex", fingerprint)) {
+            stats.deduped++;
             continue;
           }
           memories.push({
@@ -649,17 +657,18 @@ function extractTurns(db, composerId, bubbleHeaders) {
 async function ingestCursor(dbPath, wasmDir) {
   const memories = [];
   const sessionIds = [];
-  let skipped = 0;
+  let filtered = 0;
+  let deduped = 0;
   let fileSize;
   try {
     fileSize = statSync(dbPath).size;
   } catch {
     console.log("  Skipping Cursor: database not accessible");
-    return { memories, sessionIds, skipped };
+    return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
   if (fileSize > MAX_DB_SIZE) {
     console.log(`  Skipping Cursor: database too large (${(fileSize / 1024 / 1024 / 1024).toFixed(1)}GB)`);
-    return { memories, sessionIds, skipped };
+    return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
   if (fileSize > WARN_DB_SIZE) {
     console.log(`  Warning: large database (${(fileSize / 1024 / 1024).toFixed(0)}MB), loading into memory...`);
@@ -667,14 +676,14 @@ async function ingestCursor(dbPath, wasmDir) {
   const initSqlJs = loadSqlJs(wasmDir);
   if (!initSqlJs) {
     console.log("  Skipping Cursor: sql.js not available");
-    return { memories, sessionIds, skipped };
+    return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
   let db;
   try {
     db = await openDb(initSqlJs, dbPath, wasmDir);
   } catch (e) {
     console.log(`  Skipping Cursor: cannot open database (${e.message})`);
-    return { memories, sessionIds, skipped };
+    return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
   try {
     let rows = [];
@@ -689,21 +698,21 @@ async function ingestCursor(dbPath, wasmDir) {
       try {
         parsed = JSON.parse(value);
         if (!parsed || typeof parsed !== "object") {
-          skipped++;
+          filtered++;
           continue;
         }
       } catch {
-        skipped++;
+        filtered++;
         continue;
       }
       const bubbleHeaders = parsed.fullConversationHeadersOnly;
       if (!Array.isArray(bubbleHeaders) || bubbleHeaders.length === 0) {
-        skipped++;
+        filtered++;
         continue;
       }
       const turns = extractTurns(db, composerId, bubbleHeaders);
       if (turns.length === 0) {
-        skipped++;
+        filtered++;
         continue;
       }
       const sessionName = parsed.name || "Cursor Chat";
@@ -730,7 +739,7 @@ ${body}`;
       const dedupKey = `cursor:${composerId}`;
       const fingerprint = `${dedupKey}:${contentHash}`;
       if (isIngested("cursor", fingerprint)) {
-        skipped++;
+        deduped++;
         continue;
       }
       memories.push({
@@ -752,7 +761,7 @@ ${body}`;
   } finally {
     db.close();
   }
-  return { memories, sessionIds, skipped };
+  return { memories, sessionIds, skipped: filtered + deduped, filtered, deduped };
 }
 
 // src/ingest/codebase.ts
@@ -1000,7 +1009,7 @@ function indexCodebase(rootPath) {
 
 // src/api.ts
 var API_URL = "https://mcp.conare.ai";
-function createUploadBatches(memories, maxItems = 5, maxChars = 120000) {
+function createUploadBatches(memories, maxItems = 20, maxChars = 400000) {
   const batches = [];
   let current = [];
   let currentChars = 0;
@@ -1926,6 +1935,14 @@ function renderProgressSummary(success, failed, noun) {
   return `\r  \x1B[32m✓\x1B[0m [\x1B[36m${bar}\x1B[0m] ${success} ${noun}, ${failed} failed (${pct}%)${" ".repeat(12)}
 `;
 }
+function renderDiscoverySummary(discovered, filtered, deduped) {
+  const parts = [`${discovered} ingestible`];
+  if (filtered > 0)
+    parts.push(`${filtered} filtered`);
+  if (deduped > 0)
+    parts.push(`${deduped} already imported`);
+  return parts.join(", ");
+}
 function parseArgs() {
   const args = process.argv.slice(2);
   let key = "";
@@ -2160,22 +2177,22 @@ Nothing new to index.`);
   const shouldIngest = (name) => selectedSources.includes(name);
   if (shouldIngest("claude") && tools.find((t) => t.name === "Claude Code")?.available) {
     process.stdout.write("Ingesting Claude Code... ");
-    const { memories, skipped } = ingestClaude();
+    const { memories, filtered, deduped } = ingestClaude();
     allMemories.push(...memories);
-    console.log(`${memories.length} new, ${skipped} skipped`);
+    console.log(renderDiscoverySummary(memories.length, filtered, deduped));
   }
   if (shouldIngest("codex") && tools.find((t) => t.name === "Codex")?.available) {
     process.stdout.write("Ingesting Codex... ");
-    const { memories, skipped } = ingestCodex();
+    const { memories, filtered, deduped } = ingestCodex();
     allMemories.push(...memories);
-    console.log(`${memories.length} new, ${skipped} skipped`);
+    console.log(renderDiscoverySummary(memories.length, filtered, deduped));
   }
   if (shouldIngest("cursor") && tools.find((t) => t.name === "Cursor")?.available) {
     process.stdout.write("Ingesting Cursor... ");
     const cursorTool = tools.find((t) => t.name === "Cursor");
-    const { memories, skipped } = await ingestCursor(cursorTool.path, opts.wasmDir);
+    const { memories, filtered, deduped } = await ingestCursor(cursorTool.path, opts.wasmDir);
     allMemories.push(...memories);
-    console.log(`${memories.length} new, ${skipped} skipped`);
+    console.log(renderDiscoverySummary(memories.length, filtered, deduped));
   }
   console.log();
   if (allMemories.length === 0) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "conare",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Conare CLI for ingesting AI chat history and configuring memory at conare.ai",
   "type": "module",
   "bin": {