conare 0.1.0 → 0.1.2

package/dist/index.js

@@ -281,6 +281,7 @@ import { homedir as homedir3 } from "node:os";
 
 // src/ingest/shared.ts
 import { existsSync as existsSync2, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { createHash } from "node:crypto";
 import { join as join2 } from "node:path";
 import { homedir as homedir2 } from "node:os";
 var MANIFEST_PATH = join2(homedir2(), ".conare", "ingested.json");
@@ -290,6 +291,9 @@ function cleanText(raw) {
   text = text.replace(/<attached-context[\s\S]*?<\/attached-context>/g, "");
   return text.trim();
 }
+function createContentHash(content) {
+  return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
 function getIngested() {
   try {
     if (existsSync2(MANIFEST_PATH)) {
@@ -372,12 +376,13 @@ function ingestClaude() {
   const projectsDir = join3(homedir3(), ".claude", "projects");
   const memories = [];
   const sessionIds = [];
-  let skipped = 0;
+  let filtered = 0;
+  let deduped = 0;
   let projectDirs;
   try {
     projectDirs = readdirSync2(projectsDir);
   } catch {
-    return { memories, sessionIds, skipped };
+    return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
   for (const projDir of projectDirs) {
     const projPath = join3(projectsDir, projDir);
@@ -390,15 +395,11 @@ function ingestClaude() {
     }
     for (const file of files) {
       const sessionId = basename(file, ".jsonl");
-      if (isIngested("claude", sessionId)) {
-        skipped++;
-        continue;
-      }
       const raw = readFileSync2(join3(projPath, file), "utf-8");
       const { turns, date } = parseSession(raw.split(`
 `));
       if (turns.length === 0) {
-        skipped++;
+        filtered++;
         continue;
       }
       const header = `# Chat: ${project}${date ? ` | ${date}` : ""}`;
@@ -419,15 +420,29 @@ ${body}`;
         content = content.slice(0, MAX_CONTENT) + `
 
 [truncated]`;
+      const contentHash = createContentHash(content);
+      const dedupKey = `claude:${sessionId}`;
+      const fingerprint = `${dedupKey}:${contentHash}`;
+      if (isIngested("claude", fingerprint)) {
+        deduped++;
+        continue;
+      }
       memories.push({
         content,
         containerTag: "claude-chats",
-        metadata: { source: "claude-code", sessionId, project, date: date || "unknown" }
+        metadata: {
+          dedupKey,
+          contentHash,
+          source: "claude-code",
+          sessionId,
+          project,
+          date: date || "unknown"
+        }
       });
       sessionIds.push(sessionId);
     }
   }
-  return { memories, sessionIds, skipped };
+  return { memories, sessionIds, skipped: filtered + deduped, filtered, deduped };
 }
 
 // src/ingest/codex.ts
@@ -438,7 +453,8 @@ var MAX_CONTENT2 = 48000;
 function ingestCodex() {
   const memories = [];
   const sessionIds = [];
-  let skipped = 0;
+  let filtered = 0;
+  let deduped = 0;
   const historyPath = join4(homedir4(), ".codex", "history.jsonl");
   if (existsSync3(historyPath)) {
     try {
@@ -458,10 +474,6 @@ function ingestCodex() {
         }
       }
       for (const [sessionId, entries] of sessions) {
-        if (isIngested("codex", sessionId)) {
-          skipped++;
-          continue;
-        }
         entries.sort((a, b) => a.ts - b.ts);
         const date = new Date(entries[0].ts * 1000).toISOString().slice(0, 10);
         const body = entries.map((e) => {
@@ -480,13 +492,20 @@ ${body}`;
 
 [truncated]`;
         if (content.length < 100) {
-          skipped++;
+          filtered++;
+          continue;
+        }
+        const contentHash = createContentHash(content);
+        const dedupKey = `codex:${sessionId}`;
+        const fingerprint = `${dedupKey}:${contentHash}`;
+        if (isIngested("codex", fingerprint)) {
+          deduped++;
           continue;
         }
         memories.push({
           content,
           containerTag: "codex-chats",
-          metadata: { source: "codex", sessionId, date }
+          metadata: { dedupKey, contentHash, source: "codex", sessionId, date }
         });
         sessionIds.push(sessionId);
       }
@@ -495,19 +514,22 @@ ${body}`;
   const sessionsDir = join4(homedir4(), ".codex", "sessions");
   if (existsSync3(sessionsDir)) {
     try {
-      walkCodexSessions(sessionsDir, memories, sessionIds, skipped);
+      const stats = { filtered: 0, deduped: 0 };
+      walkCodexSessions(sessionsDir, memories, sessionIds, stats);
+      filtered += stats.filtered;
+      deduped += stats.deduped;
     } catch {}
   }
-  return { memories, sessionIds, skipped };
+  return { memories, sessionIds, skipped: filtered + deduped, filtered, deduped };
 }
-function walkCodexSessions(dir, memories, sessionIds, skipped) {
+function walkCodexSessions(dir, memories, sessionIds, stats) {
   try {
     for (const entry of readdirSync3(dir, { withFileTypes: true })) {
       if (entry.isDirectory()) {
-        walkCodexSessions(join4(dir, entry.name), memories, sessionIds, skipped);
+        walkCodexSessions(join4(dir, entry.name), memories, sessionIds, stats);
       } else if (entry.name.endsWith(".jsonl")) {
         const sessionId = basename2(entry.name, ".jsonl");
-        if (isIngested("codex", sessionId) || sessionIds.includes(sessionId))
+        if (sessionIds.includes(sessionId))
           continue;
         try {
           const lines = readFileSync3(join4(dir, entry.name), "utf-8").split(`
@@ -535,8 +557,10 @@ function walkCodexSessions(dir, memories, sessionIds, skipped) {
               continue;
             }
           }
-          if (turns.length === 0)
+          if (turns.length === 0) {
+            stats.filtered++;
             continue;
+          }
           const body = turns.map((t) => t.length > 500 ? t.slice(0, 500) + "..." : t).join(`
 
 ---
@@ -549,10 +573,23 @@ ${body}`;
             content = content.slice(0, MAX_CONTENT2) + `
 
 [truncated]`;
+          const contentHash = createContentHash(content);
+          const dedupKey = `codex:${sessionId}`;
+          const fingerprint = `${dedupKey}:${contentHash}`;
+          if (isIngested("codex", fingerprint)) {
+            stats.deduped++;
+            continue;
+          }
           memories.push({
             content,
             containerTag: "codex-chats",
-            metadata: { source: "codex-session", sessionId, date: date || "unknown" }
+            metadata: {
+              dedupKey,
+              contentHash,
+              source: "codex-session",
+              sessionId,
+              date: date || "unknown"
+            }
           });
           sessionIds.push(sessionId);
         } catch {}
@@ -620,17 +657,18 @@ function extractTurns(db, composerId, bubbleHeaders) {
 async function ingestCursor(dbPath, wasmDir) {
   const memories = [];
   const sessionIds = [];
-  let skipped = 0;
+  let filtered = 0;
+  let deduped = 0;
   let fileSize;
   try {
     fileSize = statSync(dbPath).size;
   } catch {
     console.log("  Skipping Cursor: database not accessible");
-    return { memories, sessionIds, skipped };
+    return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
   if (fileSize > MAX_DB_SIZE) {
     console.log(`  Skipping Cursor: database too large (${(fileSize / 1024 / 1024 / 1024).toFixed(1)}GB)`);
-    return { memories, sessionIds, skipped };
+    return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
   if (fileSize > WARN_DB_SIZE) {
     console.log(`  Warning: large database (${(fileSize / 1024 / 1024).toFixed(0)}MB), loading into memory...`);
@@ -638,14 +676,14 @@ async function ingestCursor(dbPath, wasmDir) {
   const initSqlJs = loadSqlJs(wasmDir);
   if (!initSqlJs) {
     console.log("  Skipping Cursor: sql.js not available");
-    return { memories, sessionIds, skipped };
+    return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
   let db;
   try {
     db = await openDb(initSqlJs, dbPath, wasmDir);
   } catch (e) {
     console.log(`  Skipping Cursor: cannot open database (${e.message})`);
-    return { memories, sessionIds, skipped };
+    return { memories, sessionIds, skipped: 0, filtered, deduped };
   }
   try {
     let rows = [];
@@ -656,29 +694,25 @@ async function ingestCursor(dbPath, wasmDir) {
     } catch {}
     for (const [key, value] of rows) {
       const composerId = key.replace("composerData:", "");
-      if (isIngested("cursor", composerId)) {
-        skipped++;
-        continue;
-      }
       let parsed;
       try {
         parsed = JSON.parse(value);
         if (!parsed || typeof parsed !== "object") {
-          skipped++;
+          filtered++;
           continue;
         }
       } catch {
-        skipped++;
+        filtered++;
         continue;
       }
       const bubbleHeaders = parsed.fullConversationHeadersOnly;
       if (!Array.isArray(bubbleHeaders) || bubbleHeaders.length === 0) {
-        skipped++;
+        filtered++;
         continue;
       }
       const turns = extractTurns(db, composerId, bubbleHeaders);
       if (turns.length === 0) {
-        skipped++;
+        filtered++;
         continue;
       }
       const sessionName = parsed.name || "Cursor Chat";
@@ -701,10 +735,24 @@ ${body}`;
         content = content.slice(0, MAX_CONTENT3) + `
 
 [truncated]`;
+      const contentHash = createContentHash(content);
+      const dedupKey = `cursor:${composerId}`;
+      const fingerprint = `${dedupKey}:${contentHash}`;
+      if (isIngested("cursor", fingerprint)) {
+        deduped++;
+        continue;
+      }
       memories.push({
         content,
         containerTag: "cursor-chats",
-        metadata: { source: "cursor", sessionId: composerId, name: sessionName, date }
+        metadata: {
+          dedupKey,
+          contentHash,
+          source: "cursor",
+          sessionId: composerId,
+          name: sessionName,
+          date
+        }
       });
       sessionIds.push(composerId);
     }
@@ -713,11 +761,11 @@ ${body}`;
   } finally {
     db.close();
   }
-  return { memories, sessionIds, skipped };
+  return { memories, sessionIds, skipped: filtered + deduped, filtered, deduped };
 }
 
 // src/ingest/codebase.ts
-import { createHash } from "node:crypto";
+import { createHash as createHash2 } from "node:crypto";
 import { readdirSync as readdirSync4, readFileSync as readFileSync5, statSync as statSync2, existsSync as existsSync4 } from "node:fs";
 import { join as join6, relative, extname, resolve } from "node:path";
 var DEFAULT_IGNORE = new Set([
@@ -884,6 +932,7 @@ ${content}
 }
 function indexCodebase(rootPath) {
   const absRoot = resolve(rootPath);
+  const repoHash = createHash2("sha256").update(absRoot).digest("hex").slice(0, 12);
   const gitignorePatterns = parseGitignore(absRoot);
   const memories = [];
   let fileCount = 0;
@@ -930,9 +979,10 @@ function indexCodebase(rootPath) {
         continue;
       }
       const relPath = relative(absRoot, fullPath);
-      const contentHash = createHash("sha256").update(raw).digest("hex").slice(0, 16);
-      const fileHash = `${relPath}:${contentHash}`;
-      if (isIngested("codebase", fileHash)) {
+      const contentHash = createContentHash(raw);
+      const dedupKey = `codebase:${repoHash}:${relPath}`;
+      const fingerprint = `${dedupKey}:${contentHash}`;
+      if (isIngested("codebase", fingerprint)) {
         skipped++;
         continue;
       }
@@ -941,9 +991,12 @@ function indexCodebase(rootPath) {
         content,
         containerTag: "codebase",
         metadata: {
+          dedupKey,
+          contentHash,
           source: "codebase-index",
+          repoHash,
           filePath: relPath,
-          fileHash,
+          fileHash: `${relPath}:${contentHash}`,
           language: langFromExt(ext)
         }
       });
@@ -956,7 +1009,7 @@ function indexCodebase(rootPath) {
 
 // src/api.ts
 var API_URL = "https://mcp.conare.ai";
-function createUploadBatches(memories, maxItems = 5, maxChars = 120000) {
+function createUploadBatches(memories, maxItems = 20, maxChars = 400000) {
   const batches = [];
   let current = [];
   let currentChars = 0;
@@ -992,6 +1045,21 @@ async function validateKey(apiKey) {
     return { valid: false };
   }
 }
+async function getRemoteMemoryCount(apiKey) {
+  try {
+    const res = await fetch(`${API_URL}/api/containers`, {
+      headers: { Authorization: `Bearer ${apiKey}` }
+    });
+    if (!res.ok)
+      return null;
+    const data = await res.json();
+    if (!Array.isArray(data.containers))
+      return 0;
+    return data.containers.reduce((sum, container) => sum + (container.count || 0), 0);
+  } catch {
+    return null;
+  }
+}
 async function uploadItems(apiKey, items) {
   let retries = 4;
   while (retries > 0) {
@@ -1822,9 +1890,12 @@ async function confirmIndexCodebase() {
 }
 
 // src/index.ts
-function getDedupKey(memory) {
+function getManifestFingerprint(memory) {
   const metadata = memory.metadata;
-  return metadata?.sessionId || metadata?.fileHash || null;
+  if (metadata?.dedupKey && metadata?.contentHash) {
+    return `${metadata.dedupKey}:${metadata.contentHash}`;
+  }
+  return metadata?.dedupKey || metadata?.sessionId || metadata?.fileHash || null;
 }
 function printMissingKeyError() {
   console.error("Error: no API key configured.");
@@ -1864,6 +1935,14 @@ function renderProgressSummary(success, failed, noun) {
   return `\r  \x1B[32m✓\x1B[0m [\x1B[36m${bar}\x1B[0m] ${success} ${noun}, ${failed} failed (${pct}%)${" ".repeat(12)}
 `;
 }
+function renderDiscoverySummary(discovered, filtered, deduped) {
+  const parts = [`${discovered} ingestible`];
+  if (filtered > 0)
+    parts.push(`${filtered} filtered`);
+  if (deduped > 0)
+    parts.push(`${deduped} already imported`);
+  return parts.join(", ");
+}
 function parseArgs() {
   const args = process.argv.slice(2);
   let key = "";
@@ -1987,6 +2066,16 @@ async function main() {
   console.log(auth.email ? `OK (${auth.email})` : "OK");
   saveApiKey(apiKey);
   console.log();
+  if (!opts.force && !opts.dryRun) {
+    const remoteMemoryCount = await getRemoteMemoryCount(apiKey);
+    const localManifest = getIngested();
+    const localManifestCount = Object.values(localManifest).reduce((sum, entries) => sum + entries.length, 0);
+    if (remoteMemoryCount === 0 && localManifestCount > 0) {
+      clearIngested();
+      console.log("Remote account is empty; cleared stale local ingestion history.");
+      console.log("");
+    }
+  }
   if (!opts.wasmDir && existsSync7(join9(process.cwd(), "node_modules", "sql.js"))) {
     opts.wasmDir = join9(process.cwd(), "node_modules");
   }
@@ -2046,7 +2135,7 @@ Nothing new to index.`);
         process.stdout.write(`\r  Uploading [\x1B[36m${bar}\x1B[0m] ${uploaded}/${total} (${pct}%)`);
       });
       process.stdout.write(renderProgressSummary(success, failed, "indexed"));
-      const fileHashes = results.filter((result) => result.success).map((result) => getDedupKey(memories[result.index])).filter((key) => !!key);
+      const fileHashes = results.filter((result) => result.success).map((result) => getManifestFingerprint(memories[result.index])).filter((key) => !!key);
       markIngested("codebase", fileHashes);
       printFailureSummary(results, memories);
     }
@@ -2088,22 +2177,22 @@ Nothing new to index.`);
   const shouldIngest = (name) => selectedSources.includes(name);
   if (shouldIngest("claude") && tools.find((t) => t.name === "Claude Code")?.available) {
     process.stdout.write("Ingesting Claude Code... ");
-    const { memories, skipped } = ingestClaude();
+    const { memories, filtered, deduped } = ingestClaude();
     allMemories.push(...memories);
-    console.log(`${memories.length} new, ${skipped} skipped`);
+    console.log(renderDiscoverySummary(memories.length, filtered, deduped));
   }
   if (shouldIngest("codex") && tools.find((t) => t.name === "Codex")?.available) {
     process.stdout.write("Ingesting Codex... ");
-    const { memories, skipped } = ingestCodex();
+    const { memories, filtered, deduped } = ingestCodex();
     allMemories.push(...memories);
-    console.log(`${memories.length} new, ${skipped} skipped`);
+    console.log(renderDiscoverySummary(memories.length, filtered, deduped));
   }
   if (shouldIngest("cursor") && tools.find((t) => t.name === "Cursor")?.available) {
     process.stdout.write("Ingesting Cursor... ");
     const cursorTool = tools.find((t) => t.name === "Cursor");
-    const { memories, skipped } = await ingestCursor(cursorTool.path, opts.wasmDir);
+    const { memories, filtered, deduped } = await ingestCursor(cursorTool.path, opts.wasmDir);
     allMemories.push(...memories);
-    console.log(`${memories.length} new, ${skipped} skipped`);
+    console.log(renderDiscoverySummary(memories.length, filtered, deduped));
   }
   console.log();
   if (allMemories.length === 0) {
@@ -2137,7 +2226,7 @@ Nothing new to index.`);
       if (!result.success)
         continue;
       const memory = allMemories[result.index];
-      const key = getDedupKey(memory);
+      const key = getManifestFingerprint(memory);
       if (!key)
         continue;
       switch (memory.containerTag) {
@@ -2200,7 +2289,7 @@ Nothing new to index.`);
         process.stdout.write(`\r  Uploading [\x1B[36m${bar}\x1B[0m] ${uploaded}/${total} (${pct}%)`);
       });
       process.stdout.write(renderProgressSummary(success, failed, "indexed"));
-      const fileHashes = results.filter((result) => result.success).map((result) => getDedupKey(memories[result.index])).filter((key) => !!key);
+      const fileHashes = results.filter((result) => result.success).map((result) => getManifestFingerprint(memories[result.index])).filter((key) => !!key);
       markIngested("codebase", fileHashes);
       printFailureSummary(results, memories);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "conare",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Conare CLI for ingesting AI chat history and configuring memory at conare.ai",
   "type": "module",
   "bin": {