conare 0.0.9 → 0.1.1

package/dist/index.js

@@ -281,6 +281,7 @@ import { homedir as homedir3 } from "node:os";
 
 // src/ingest/shared.ts
 import { existsSync as existsSync2, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { createHash } from "node:crypto";
 import { join as join2 } from "node:path";
 import { homedir as homedir2 } from "node:os";
 var MANIFEST_PATH = join2(homedir2(), ".conare", "ingested.json");
@@ -290,6 +291,9 @@ function cleanText(raw) {
   text = text.replace(/<attached-context[\s\S]*?<\/attached-context>/g, "");
   return text.trim();
 }
+function createContentHash(content) {
+  return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
 function getIngested() {
   try {
     if (existsSync2(MANIFEST_PATH)) {
@@ -390,10 +394,6 @@ function ingestClaude() {
     }
     for (const file of files) {
       const sessionId = basename(file, ".jsonl");
-      if (isIngested("claude", sessionId)) {
-        skipped++;
-        continue;
-      }
       const raw = readFileSync2(join3(projPath, file), "utf-8");
       const { turns, date } = parseSession(raw.split(`
 `));
@@ -419,10 +419,24 @@ ${body}`;
         content = content.slice(0, MAX_CONTENT) + `
 
 [truncated]`;
+      const contentHash = createContentHash(content);
+      const dedupKey = `claude:${sessionId}`;
+      const fingerprint = `${dedupKey}:${contentHash}`;
+      if (isIngested("claude", fingerprint)) {
+        skipped++;
+        continue;
+      }
       memories.push({
         content,
         containerTag: "claude-chats",
-        metadata: { source: "claude-code", sessionId, project, date: date || "unknown" }
+        metadata: {
+          dedupKey,
+          contentHash,
+          source: "claude-code",
+          sessionId,
+          project,
+          date: date || "unknown"
+        }
       });
       sessionIds.push(sessionId);
     }
@@ -458,10 +472,6 @@ function ingestCodex() {
         }
       }
       for (const [sessionId, entries] of sessions) {
-        if (isIngested("codex", sessionId)) {
-          skipped++;
-          continue;
-        }
         entries.sort((a, b) => a.ts - b.ts);
         const date = new Date(entries[0].ts * 1000).toISOString().slice(0, 10);
         const body = entries.map((e) => {
@@ -483,10 +493,17 @@ ${body}`;
           skipped++;
           continue;
         }
+        const contentHash = createContentHash(content);
+        const dedupKey = `codex:${sessionId}`;
+        const fingerprint = `${dedupKey}:${contentHash}`;
+        if (isIngested("codex", fingerprint)) {
+          skipped++;
+          continue;
+        }
         memories.push({
           content,
           containerTag: "codex-chats",
-          metadata: { source: "codex", sessionId, date }
+          metadata: { dedupKey, contentHash, source: "codex", sessionId, date }
         });
         sessionIds.push(sessionId);
       }
@@ -507,7 +524,7 @@ function walkCodexSessions(dir, memories, sessionIds, skipped) {
         walkCodexSessions(join4(dir, entry.name), memories, sessionIds, skipped);
       } else if (entry.name.endsWith(".jsonl")) {
         const sessionId = basename2(entry.name, ".jsonl");
-        if (isIngested("codex", sessionId) || sessionIds.includes(sessionId))
+        if (sessionIds.includes(sessionId))
           continue;
         try {
           const lines = readFileSync3(join4(dir, entry.name), "utf-8").split(`
@@ -549,10 +566,22 @@ ${body}`;
             content = content.slice(0, MAX_CONTENT2) + `
 
 [truncated]`;
+          const contentHash = createContentHash(content);
+          const dedupKey = `codex:${sessionId}`;
+          const fingerprint = `${dedupKey}:${contentHash}`;
+          if (isIngested("codex", fingerprint)) {
+            continue;
+          }
           memories.push({
             content,
             containerTag: "codex-chats",
-            metadata: { source: "codex-session", sessionId, date: date || "unknown" }
+            metadata: {
+              dedupKey,
+              contentHash,
+              source: "codex-session",
+              sessionId,
+              date: date || "unknown"
+            }
           });
           sessionIds.push(sessionId);
         } catch {}
@@ -656,10 +685,6 @@ async function ingestCursor(dbPath, wasmDir) {
     } catch {}
     for (const [key, value] of rows) {
       const composerId = key.replace("composerData:", "");
-      if (isIngested("cursor", composerId)) {
-        skipped++;
-        continue;
-      }
       let parsed;
       try {
         parsed = JSON.parse(value);
@@ -701,10 +726,24 @@ ${body}`;
         content = content.slice(0, MAX_CONTENT3) + `
 
 [truncated]`;
+      const contentHash = createContentHash(content);
+      const dedupKey = `cursor:${composerId}`;
+      const fingerprint = `${dedupKey}:${contentHash}`;
+      if (isIngested("cursor", fingerprint)) {
+        skipped++;
+        continue;
+      }
       memories.push({
         content,
         containerTag: "cursor-chats",
-        metadata: { source: "cursor", sessionId: composerId, name: sessionName, date }
+        metadata: {
+          dedupKey,
+          contentHash,
+          source: "cursor",
+          sessionId: composerId,
+          name: sessionName,
+          date
+        }
       });
       sessionIds.push(composerId);
     }
@@ -717,7 +756,7 @@ ${body}`;
 }
 
 // src/ingest/codebase.ts
-import { createHash } from "node:crypto";
+import { createHash as createHash2 } from "node:crypto";
 import { readdirSync as readdirSync4, readFileSync as readFileSync5, statSync as statSync2, existsSync as existsSync4 } from "node:fs";
 import { join as join6, relative, extname, resolve } from "node:path";
 var DEFAULT_IGNORE = new Set([
@@ -884,6 +923,7 @@ ${content}
 }
 function indexCodebase(rootPath) {
   const absRoot = resolve(rootPath);
+  const repoHash = createHash2("sha256").update(absRoot).digest("hex").slice(0, 12);
   const gitignorePatterns = parseGitignore(absRoot);
   const memories = [];
   let fileCount = 0;
@@ -930,9 +970,10 @@ function indexCodebase(rootPath) {
         continue;
       }
       const relPath = relative(absRoot, fullPath);
-      const contentHash = createHash("sha256").update(raw).digest("hex").slice(0, 16);
-      const fileHash = `${relPath}:${contentHash}`;
-      if (isIngested("codebase", fileHash)) {
+      const contentHash = createContentHash(raw);
+      const dedupKey = `codebase:${repoHash}:${relPath}`;
+      const fingerprint = `${dedupKey}:${contentHash}`;
+      if (isIngested("codebase", fingerprint)) {
         skipped++;
         continue;
       }
@@ -941,9 +982,12 @@ function indexCodebase(rootPath) {
         content,
         containerTag: "codebase",
         metadata: {
+          dedupKey,
+          contentHash,
           source: "codebase-index",
+          repoHash,
           filePath: relPath,
-          fileHash,
+          fileHash: `${relPath}:${contentHash}`,
           language: langFromExt(ext)
         }
       });
@@ -992,6 +1036,21 @@ async function validateKey(apiKey) {
     return { valid: false };
   }
 }
+async function getRemoteMemoryCount(apiKey) {
+  try {
+    const res = await fetch(`${API_URL}/api/containers`, {
+      headers: { Authorization: `Bearer ${apiKey}` }
+    });
+    if (!res.ok)
+      return null;
+    const data = await res.json();
+    if (!Array.isArray(data.containers))
+      return 0;
+    return data.containers.reduce((sum, container) => sum + (container.count || 0), 0);
+  } catch {
+    return null;
+  }
+}
 async function uploadItems(apiKey, items) {
   let retries = 4;
   while (retries > 0) {
@@ -1822,9 +1881,12 @@ async function confirmIndexCodebase() {
 }
 
 // src/index.ts
-function getDedupKey(memory) {
+function getManifestFingerprint(memory) {
   const metadata = memory.metadata;
-  return metadata?.sessionId || metadata?.fileHash || null;
+  if (metadata?.dedupKey && metadata?.contentHash) {
+    return `${metadata.dedupKey}:${metadata.contentHash}`;
+  }
+  return metadata?.dedupKey || metadata?.sessionId || metadata?.fileHash || null;
 }
 function printMissingKeyError() {
   console.error("Error: no API key configured.");
@@ -1854,6 +1916,16 @@ function printFailureSummary(results, memories) {
     console.log(`  - ... and ${failures.length - 3} more`);
   }
 }
+function renderProgressSummary(success, failed, noun) {
+  const total = success + failed;
+  const barWidth = 20;
+  const ratio = total === 0 ? 1 : success / total;
+  const pct = (ratio * 100).toFixed(1);
+  const filled = Math.round(ratio * barWidth);
+  const bar = "█".repeat(filled) + "░".repeat(barWidth - filled);
+  return `\r  \x1B[32m✓\x1B[0m [\x1B[36m${bar}\x1B[0m] ${success} ${noun}, ${failed} failed (${pct}%)${" ".repeat(12)}
+`;
+}
 function parseArgs() {
   const args = process.argv.slice(2);
   let key = "";
@@ -1977,6 +2049,16 @@ async function main() {
   console.log(auth.email ? `OK (${auth.email})` : "OK");
   saveApiKey(apiKey);
   console.log();
+  if (!opts.force && !opts.dryRun) {
+    const remoteMemoryCount = await getRemoteMemoryCount(apiKey);
+    const localManifest = getIngested();
+    const localManifestCount = Object.values(localManifest).reduce((sum, entries) => sum + entries.length, 0);
+    if (remoteMemoryCount === 0 && localManifestCount > 0) {
+      clearIngested();
+      console.log("Remote account is empty; cleared stale local ingestion history.");
+      console.log("");
+    }
+  }
   if (!opts.wasmDir && existsSync7(join9(process.cwd(), "node_modules", "sql.js"))) {
     opts.wasmDir = join9(process.cwd(), "node_modules");
   }
@@ -2035,9 +2117,8 @@ Nothing new to index.`);
         const bar = "█".repeat(filled) + "░".repeat(barWidth - filled);
         process.stdout.write(`\r  Uploading [\x1B[36m${bar}\x1B[0m] ${uploaded}/${total} (${pct}%)`);
       });
-      process.stdout.write(`\r  \x1B[32m✓\x1B[0m ${success} files indexed, ${failed} failed${" ".repeat(20)}
-`);
-      const fileHashes = results.filter((result) => result.success).map((result) => getDedupKey(memories[result.index])).filter((key) => !!key);
+      process.stdout.write(renderProgressSummary(success, failed, "indexed"));
+      const fileHashes = results.filter((result) => result.success).map((result) => getManifestFingerprint(memories[result.index])).filter((key) => !!key);
       markIngested("codebase", fileHashes);
       printFailureSummary(results, memories);
     }
@@ -2114,8 +2195,7 @@ Nothing new to index.`);
       const bar = "█".repeat(filled) + "░".repeat(barWidth - filled);
       process.stdout.write(`\r  Uploading [\x1B[33m${bar}\x1B[0m] ${uploaded}/${total} (${pct}%)`);
     });
-    process.stdout.write(`\r  \x1B[32m✓\x1B[0m ${success} uploaded, ${failed} failed${" ".repeat(20)}
-`);
+    process.stdout.write(renderProgressSummary(success, failed, "uploaded"));
     if (failed > 0) {
       console.log(`  Re-run with --force to retry failed memories.`);
     }
@@ -2129,7 +2209,7 @@ Nothing new to index.`);
       if (!result.success)
         continue;
       const memory = allMemories[result.index];
-      const key = getDedupKey(memory);
+      const key = getManifestFingerprint(memory);
       if (!key)
         continue;
       switch (memory.containerTag) {
@@ -2191,9 +2271,8 @@ Nothing new to index.`);
         const bar = "█".repeat(filled) + "░".repeat(barWidth - filled);
         process.stdout.write(`\r  Uploading [\x1B[36m${bar}\x1B[0m] ${uploaded}/${total} (${pct}%)`);
       });
-      process.stdout.write(`\r  \x1B[32m✓\x1B[0m ${success} files indexed, ${failed} failed${" ".repeat(20)}
-`);
-      const fileHashes = results.filter((result) => result.success).map((result) => getDedupKey(memories[result.index])).filter((key) => !!key);
+      process.stdout.write(renderProgressSummary(success, failed, "indexed"));
+      const fileHashes = results.filter((result) => result.success).map((result) => getManifestFingerprint(memories[result.index])).filter((key) => !!key);
       markIngested("codebase", fileHashes);
       printFailureSummary(results, memories);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "conare",
-  "version": "0.0.9",
+  "version": "0.1.1",
   "description": "Conare CLI for ingesting AI chat history and configuring memory at conare.ai",
   "type": "module",
   "bin": {