@c3-oss/prosa 0.1.0 → 0.2.0

package/dist/index.js

@@ -579,6 +579,12 @@ function objectStoragePath(hashHex, compression) {
 }
 
 // src/core/cas/index.ts
+var ensuredDirs = /* @__PURE__ */ new Set();
+async function ensureDir(absoluteDir) {
+  if (ensuredDirs.has(absoluteDir)) return;
+  await mkdir2(absoluteDir, { recursive: true });
+  ensuredDirs.add(absoluteDir);
+}
 async function putBytes(bundle, bytes, options = {}) {
   const hash = blake3Hex(bytes);
   const objectId = objectIdFromHash(hash);
@@ -592,7 +598,7 @@ async function putBytes(bundle, bytes, options = {}) {
   const { bytes: stored, compression } = compressBytes(bytes);
   const storagePath = objectStoragePath(hash, compression);
   const absolutePath = path2.join(bundle.path, storagePath);
-  await mkdir2(path2.dirname(absolutePath), { recursive: true });
+  await ensureDir(path2.dirname(absolutePath));
   await writeFile2(absolutePath, stored);
   prepare(
     bundle.db,
@@ -656,6 +662,112 @@ function getObjectMeta(bundle, objectId) {
          FROM objects WHERE object_id = ?`
   ).get(objectId) ?? null;
 }
+function createPendingObjects() {
+  return { byId: /* @__PURE__ */ new Map() };
+}
+function stageBytes(pending, bytes, options = {}) {
+  const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
+  const hash = blake3Hex(buf);
+  const objectId = objectIdFromHash(hash);
+  if (!pending.byId.has(objectId)) {
+    pending.byId.set(objectId, {
+      objectId,
+      hash,
+      bytes: buf,
+      mimeType: options.mimeType ?? null,
+      encoding: options.encoding ?? null
+    });
+  }
+  return objectId;
+}
+function stageText(pending, text, options = {}) {
+  return stageBytes(pending, Buffer.from(text, "utf8"), {
+    mimeType: options.mimeType ?? "text/plain; charset=utf-8",
+    encoding: "utf-8"
+  });
+}
+function stageJson(pending, value) {
+  return stageBytes(pending, Buffer.from(JSON.stringify(value), "utf8"), {
+    mimeType: "application/json",
+    encoding: "utf-8"
+  });
+}
+async function flushPendingObjects(bundle, pending) {
+  if (pending.byId.size === 0) return;
+  const ids = [...pending.byId.keys()];
+  const existingIds = queryExistingObjectIds(bundle, ids);
+  const toWrite = [];
+  for (const obj of pending.byId.values()) {
+    if (existingIds.has(obj.objectId)) continue;
+    const { bytes: compressedBytes, compression } = compressBytes(obj.bytes);
+    const storagePath = objectStoragePath(obj.hash, compression);
+    toWrite.push({
+      staged: obj,
+      compression,
+      compressedBytes,
+      storagePath,
+      absolutePath: path2.join(bundle.path, storagePath)
+    });
+  }
+  if (toWrite.length > 0) {
+    await writeFilesParallel(toWrite);
+  }
+  const insertObject = prepare(
+    bundle.db,
+    `INSERT OR IGNORE INTO objects (
+       object_id, hash_alg, hash, size_bytes, compressed_size_bytes,
+       compression, mime_type, encoding, storage_path, created_at
+     ) VALUES (?, 'blake3', ?, ?, ?, ?, ?, ?, ?, ?)`
+  );
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  for (const p of toWrite) {
+    insertObject.run(
+      p.staged.objectId,
+      p.staged.hash,
+      p.staged.bytes.byteLength,
+      p.compression === "zstd" ? p.compressedBytes.byteLength : null,
+      p.compression,
+      p.staged.mimeType,
+      p.staged.encoding,
+      p.storagePath,
+      now
+    );
+  }
+}
+function queryExistingObjectIds(bundle, ids) {
+  const found = /* @__PURE__ */ new Set();
+  if (ids.length === 0) return found;
+  const CHUNK = 500;
+  for (let start = 0; start < ids.length; start += CHUNK) {
+    const slice = ids.slice(start, start + CHUNK);
+    const placeholders = slice.map(() => "?").join(",");
+    const rows = bundle.db.prepare(
+      `SELECT object_id FROM objects WHERE object_id IN (${placeholders})`
+    ).all(...slice);
+    for (const row of rows) found.add(row.object_id);
+  }
+  return found;
+}
+var FS_WRITE_CONCURRENCY = 16;
+async function writeFilesParallel(tasks) {
+  let cursor = 0;
+  const workers = [];
+  const limit = Math.min(FS_WRITE_CONCURRENCY, tasks.length);
+  for (let w = 0; w < limit; w++) {
+    workers.push(
+      (async () => {
+        while (true) {
+          const i = cursor++;
+          if (i >= tasks.length) return;
+          const task = tasks[i];
+          await ensureDir(path2.dirname(task.absolutePath));
+          await writeFile2(task.absolutePath, task.compressedBytes);
+        }
+      })()
+    );
+  }
+  await Promise.all(workers);
+}
 
 // src/core/domain/ids.ts
 var ID_PREFIX_BYTES = 16;
@@ -766,7 +878,7 @@ async function recordError(bundle, batchId, args) {
 }
 
 // src/core/ingest/idempotency.ts
-import { access as access2, mkdir as mkdir3, readFile as readFile3, stat as stat2, writeFile as writeFile3 } from "fs/promises";
+import { access as access2, readFile as readFile3, stat as stat2, writeFile as writeFile3 } from "fs/promises";
 import path3 from "path";
 async function registerSourceFile(bundle, args) {
   const st = await stat2(args.absolutePath);
@@ -852,7 +964,7 @@ async function preserveRawSourceBytes(bundle, bytes) {
   const { bytes: stored, compression } = compressBytes(bytes);
   const storagePath = rawSourceStoragePath(hash, compression);
   const absolutePath = path3.join(bundle.path, storagePath);
-  await mkdir3(path3.dirname(absolutePath), { recursive: true });
+  await ensureDir(path3.dirname(absolutePath));
   if (!await fileExists(absolutePath)) {
     await writeFile3(absolutePath, stored);
   }
@@ -994,7 +1106,7 @@ import { existsSync } from "fs";
 import { createRequire } from "module";
 
 // src/services/indexing.ts
-import { mkdir as mkdir4, rm, writeFile as writeFile4 } from "fs/promises";
+import { mkdir as mkdir3, rm, writeFile as writeFile4 } from "fs/promises";
 import path4 from "path";
 var FTS5_TRIGGER_SQL = `
 CREATE TRIGGER IF NOT EXISTS search_docs_ai AFTER INSERT ON search_docs BEGIN
@@ -1109,7 +1221,7 @@ async function rebuildTantivyIndex(bundle) {
     const tantivy = await import("@oxdev03/node-tantivy-binding");
     const schema = new tantivy.SchemaBuilder().addTextField("doc_id", { stored: true, tokenizerName: "raw" }).addTextField("entity_type", { stored: true, tokenizerName: "raw" }).addTextField("entity_id", { stored: true, tokenizerName: "raw" }).addTextField("session_id", { stored: true, tokenizerName: "raw" }).addTextField("project_id", { stored: true, tokenizerName: "raw" }).addTextField("timestamp", { stored: true, tokenizerName: "raw" }).addTextField("role", { stored: true, tokenizerName: "raw" }).addTextField("tool_name", { stored: true, tokenizerName: "raw" }).addTextField("canonical_tool_type", { stored: true, tokenizerName: "raw" }).addTextField("field_kind", { stored: true, tokenizerName: "raw" }).addTextField("text", { stored: true }).build();
     await rm(bundle.paths.tantivy, { recursive: true, force: true });
-    await mkdir4(bundle.paths.tantivy, { recursive: true });
+    await mkdir3(bundle.paths.tantivy, { recursive: true });
     const index = new tantivy.Index(schema, bundle.paths.tantivy, false);
     const writer = index.writer(5e7, 1);
     let indexedDocCount = 0;
@@ -1421,7 +1533,7 @@ function renderToolCall(c) {
 }
 
 // src/services/export/parquet.ts
-import { mkdir as mkdir5, rm as rm2, writeFile as writeFile5 } from "fs/promises";
+import { mkdir as mkdir4, rm as rm2, writeFile as writeFile5 } from "fs/promises";
 import path5 from "path";
 import { DuckDBConnection } from "@duckdb/node-api";
 var PARQUET_TABLES = [
@@ -1446,7 +1558,7 @@ var PARQUET_TABLES = [
 async function exportBundleParquet(options) {
   const snapshot = await openBundleSnapshot(options.bundlePath);
   const outDir = path5.resolve(options.outDir ?? snapshot.defaultOutDir);
-  await mkdir5(outDir, { recursive: true });
+  await mkdir4(outDir, { recursive: true });
   const files = Object.fromEntries(
     PARQUET_TABLES.map((table) => [table, path5.join(outDir, `${table}.parquet`)])
   );
@@ -1585,17 +1697,27 @@ async function* walk(dir) {
 
 // src/importers/codex/index.ts
 var PREVIEW_MAX = 4e3;
-async function compileCodex(bundle, root) {
+async function compileCodex(bundle, root, options = {}) {
+  const logger = options.logger;
   const batch = startBatch(bundle, "codex", [root]);
   const counts = emptyCounts();
+  logger?.info({ batch_id: batch.batch_id, root }, "codex batch started");
   try {
     for await (const filePath of discoverCodexSessions(root)) {
       counts.source_files_seen++;
+      logger?.debug({ path: filePath }, "codex source file discovered");
       try {
-        const fileCounts = await compileCodexFile(bundle, batch, filePath);
+        const fileCounts = await compileCodexFile(bundle, batch, filePath, logger);
         addCounts(counts, fileCounts);
       } catch (error) {
         counts.errors++;
+        logger?.warn(
+          {
+            err: error,
+            path: filePath
+          },
+          "codex source file failed"
+        );
         await recordError(bundle, batch.batch_id, {
           kind: "codex_file_failed",
           message: error instanceof Error ? error.message : String(error),
@@ -1604,9 +1726,12 @@ async function compileCodex(bundle, root) {
       }
     }
     linkSubagentParents(bundle);
+    logger?.debug({ batch_id: batch.batch_id }, "codex subagent parent links refreshed");
     finishBatch(bundle, batch, counts, "completed");
+    logger?.info({ batch_id: batch.batch_id, counts }, "codex batch completed");
   } catch (error) {
     finishBatch(bundle, batch, counts, "failed");
+    logger?.error({ err: error, batch_id: batch.batch_id, counts }, "codex batch failed");
     throw error;
   }
   return { batch, counts };
@@ -1659,7 +1784,7 @@ function addCounts(target, source) {
   target.edges += source.edges;
   target.errors += source.errors;
 }
-async function compileCodexFile(bundle, batch, filePath) {
+async function compileCodexFile(bundle, batch, filePath, logger) {
   const counts = emptyFileCounts();
   const { row: sourceFileRow, alreadyKnown } = await registerSourceFile(bundle, {
     sourceTool: "codex",
@@ -1668,9 +1793,17 @@ async function compileCodexFile(bundle, batch, filePath) {
   });
   if (alreadyKnown) {
     counts.source_files_skipped = 1;
+    logger?.debug(
+      { path: filePath, source_file_id: sourceFileRow.source_file_id },
+      "codex source file skipped"
+    );
     return counts;
   }
   counts.source_files_imported = 1;
+  logger?.debug(
+    { path: filePath, source_file_id: sourceFileRow.source_file_id },
+    "codex source file registered"
+  );
   const text = await readFile4(filePath, "utf8");
   const rawLines = text.split("\n");
   const lines = rawLines[rawLines.length - 1] === "" ? rawLines.slice(0, -1) : rawLines;
@@ -1687,7 +1820,8 @@ async function compileCodexFile(bundle, batch, filePath) {
     toolResults: [],
     artifacts: [],
     edges: [],
-    searchDocs: []
+    searchDocs: [],
+    objects: createPendingObjects()
   };
   let sessionStartTs = null;
   let sessionEndTs = null;
@@ -1701,7 +1835,7 @@ async function compileCodexFile(bundle, batch, filePath) {
     const lineNo = i + 1;
     const ordinal = i;
     const lineBytes = Buffer.from(line, "utf8");
-    const rawObjectId = await putBytes(bundle, lineBytes, {
+    const rawObjectId = stageBytes(pending.objects, lineBytes, {
       mimeType: "application/jsonl-line",
       encoding: "utf-8"
     });
@@ -1712,7 +1846,7 @@ async function compileCodexFile(bundle, batch, filePath) {
     } catch {
       parserStatus = "failed";
     }
-    const decodedObjectId = parsed != null && parserStatus === "ok" ? await putJson(bundle, parsed) : null;
+    const decodedObjectId = null;
     const nativeId = parsed ? extractNativeId(parsed) : null;
     const rawRecordId2 = rawRecordId(sourceFileRow.source_file_id, ordinal, rawObjectId);
     pending.rawRecords.push({
@@ -1881,6 +2015,7 @@ async function compileCodexFile(bundle, batch, filePath) {
     pending.session.start_ts ??= sessionStartTs;
   }
   buildSearchDocs(pending);
+  await flushPendingObjects(bundle, pending.objects);
   transactional(bundle.db, () => {
     flushPending(bundle, pending, {
       sessionEndTs,
@@ -1899,6 +2034,10 @@ async function compileCodexFile(bundle, batch, filePath) {
   counts.tool_results = pending.toolResults.length;
   counts.artifacts = pending.artifacts.length;
   counts.edges = pending.edges.length;
+  logger?.debug(
+    { path: filePath, source_file_id: sourceFileRow.source_file_id, counts },
+    "codex source file imported"
+  );
   return counts;
 }
 function handleResponseItem(_bundle, sessionId2, currentTurnId, rawRecordId2, ordinal, ts, ri, payloadObjectId, nextMsgOrdinal, currentModel, pending) {
@@ -2058,8 +2197,8 @@ async function handleEventMsg(bundle, sessionId2, currentTurnId, rawRecordId2, o
   const subtype = em.type ?? "unknown";
   if (subtype === "exec_command_end") {
     const sourceCallId = em.call_id ?? null;
-    const stdoutId = em.stdout ? await putText(bundle, em.stdout, { mimeType: "text/plain" }) : null;
-    const stderrId = em.stderr ? await putText(bundle, em.stderr, { mimeType: "text/plain" }) : null;
+    const stdoutId = em.stdout ? stageText(pending.objects, em.stdout, { mimeType: "text/plain" }) : null;
+    const stderrId = em.stderr ? stageText(pending.objects, em.stderr, { mimeType: "text/plain" }) : null;
     const preview = (em.formatted_output ?? em.aggregated_output ?? em.stdout ?? "").slice(
       0,
       PREVIEW_MAX
@@ -2729,17 +2868,34 @@ async function readdirSafe(dir) {
 
 // src/importers/claude/index.ts
 var PREVIEW_MAX2 = 4e3;
-async function compileClaude(bundle, root) {
+async function compileClaude(bundle, root, options = {}) {
+  const logger = options.logger;
   const batch = startBatch(bundle, "claude", [root]);
   const counts = emptyCounts();
+  logger?.info({ batch_id: batch.batch_id, root }, "claude batch started");
   try {
     for await (const file of discoverClaudeFiles(root)) {
       counts.source_files_seen++;
+      logger?.debug(
+        {
+          path: file.filePath,
+          project_slug: file.projectSlug,
+          is_subagent: file.isSubagent
+        },
+        "claude source file discovered"
+      );
       try {
-        const fc = await compileClaudeFile(bundle, batch, file);
+        const fc = await compileClaudeFile(bundle, batch, file, logger);
         addCounts2(counts, fc);
       } catch (error) {
         counts.errors++;
+        logger?.warn(
+          {
+            err: error,
+            path: file.filePath
+          },
+          "claude source file failed"
+        );
         await recordError(bundle, batch.batch_id, {
           kind: "claude_file_failed",
           message: error instanceof Error ? error.message : String(error),
@@ -2748,9 +2904,12 @@ async function compileClaude(bundle, root) {
       }
     }
     linkSubagentParents2(bundle);
+    logger?.debug({ batch_id: batch.batch_id }, "claude subagent parent links refreshed");
     finishBatch(bundle, batch, counts, "completed");
+    logger?.info({ batch_id: batch.batch_id, counts }, "claude batch completed");
   } catch (error) {
     finishBatch(bundle, batch, counts, "failed");
+    logger?.error({ err: error, batch_id: batch.batch_id, counts }, "claude batch failed");
     throw error;
   }
   return { batch, counts };
@@ -2804,7 +2963,7 @@ function addCounts2(target, source) {
   target.edges += source.edges;
   target.errors += source.errors;
 }
-async function compileClaudeFile(bundle, batch, file) {
+async function compileClaudeFile(bundle, batch, file, logger) {
   const counts = emptyFileCounts2();
   const { row: sourceFile, alreadyKnown } = await registerSourceFile(bundle, {
     sourceTool: "claude",
@@ -2814,9 +2973,17 @@ async function compileClaudeFile(bundle, batch, file) {
   });
   if (alreadyKnown) {
     counts.source_files_skipped = 1;
+    logger?.debug(
+      { path: file.filePath, source_file_id: sourceFile.source_file_id },
+      "claude source file skipped"
+    );
     return counts;
   }
   counts.source_files_imported = 1;
+  logger?.debug(
+    { path: file.filePath, source_file_id: sourceFile.source_file_id },
+    "claude source file registered"
+  );
   const text = await readFile5(file.filePath, "utf8");
   const rawLines = text.split("\n");
   const lines = rawLines[rawLines.length - 1] === "" ? rawLines.slice(0, -1) : rawLines;
@@ -2833,7 +3000,8 @@ async function compileClaudeFile(bundle, batch, file) {
     artifacts: [],
     edges: [],
     searchDocs: [],
-    uuidToMessageId: /* @__PURE__ */ new Map()
+    uuidToMessageId: /* @__PURE__ */ new Map(),
+    objects: createPendingObjects()
   };
   let modelFirst = null;
   let modelLast = null;
@@ -2848,7 +3016,7 @@ async function compileClaudeFile(bundle, batch, file) {
     const lineNo = i + 1;
     const ordinal = i;
     const lineBytes = Buffer.from(line, "utf8");
-    const rawObjectId = await putBytes(bundle, lineBytes, {
+    const rawObjectId = stageBytes(pending.objects, lineBytes, {
       mimeType: "application/jsonl-line",
       encoding: "utf-8"
     });
@@ -2859,7 +3027,7 @@ async function compileClaudeFile(bundle, batch, file) {
     } catch {
       parserStatus = "failed";
     }
-    const decodedObjectId = parsed != null && parserStatus === "ok" ? await putJson(bundle, parsed) : null;
+    const decodedObjectId = null;
     const nativeId = parsed?.uuid ?? null;
     const rawRecordId2 = rawRecordId(sourceFile.source_file_id, ordinal, rawObjectId);
     pending.rawRecords.push({
@@ -2957,7 +3125,7 @@ async function compileClaudeFile(bundle, batch, file) {
           raw_record_id: rawRecordId2
         });
         if (content.length > PREVIEW_MAX2) {
-          const fullId = await putText(bundle, content);
+          const fullId = stageText(pending.objects, content);
           const last = pending.blocks[pending.blocks.length - 1];
           if (last) last.text_object_id = fullId;
         }
@@ -3097,6 +3265,7 @@ async function compileClaudeFile(bundle, batch, file) {
     pending.session.git_branch_initial ??= branchInitial;
   }
   buildSearchDocs2(pending);
+  await flushPendingObjects(bundle, pending.objects);
   transactional(bundle.db, () => {
     flushPending2(bundle, pending, { modelFirst, modelLast });
   });
@@ -3109,6 +3278,10 @@ async function compileClaudeFile(bundle, batch, file) {
   counts.tool_results = pending.toolResults.length;
   counts.artifacts = pending.artifacts.length;
   counts.edges = pending.edges.length;
+  logger?.debug(
+    { path: file.filePath, source_file_id: sourceFile.source_file_id, counts },
+    "claude source file imported"
+  );
   return counts;
 }
 function createSessionFromFirstRecord(file, parsed, meta, ts, rawRecordId2) {
@@ -3156,7 +3329,7 @@ async function processContentBlock(bundle, sessionId2, messageId2, eventId2, blo
       event_id: null,
       ordinal: blockOrdinal,
       block_type: "text",
-      text_object_id: text.length > PREVIEW_MAX2 ? await putText(bundle, text) : null,
+      text_object_id: text.length > PREVIEW_MAX2 ? stageText(pending.objects, text) : null,
       text_inline: text.slice(0, PREVIEW_MAX2),
       is_error: 0,
       visibility: "default",
@@ -3172,7 +3345,7 @@ async function processContentBlock(bundle, sessionId2, messageId2, eventId2, blo
       event_id: null,
       ordinal: blockOrdinal,
       block_type: "thinking",
-      text_object_id: text.length > PREVIEW_MAX2 ? await putText(bundle, text) : null,
+      text_object_id: text.length > PREVIEW_MAX2 ? stageText(pending.objects, text) : null,
       text_inline: text.slice(0, PREVIEW_MAX2),
       is_error: 0,
       visibility: "hidden_by_default",
@@ -3184,7 +3357,7 @@ async function processContentBlock(bundle, sessionId2, messageId2, eventId2, blo
     const tu = block;
     const sourceCallId = tu.id ?? `${blockOrdinal}`;
     const toolName = tu.name ?? "unknown";
-    const argsId = tu.input != null ? await putJson(bundle, tu.input) : null;
+    const argsId = tu.input != null ? stageJson(pending.objects, tu.input) : null;
     const command = inferCommandFromArgs2(toolName, tu.input);
     const filePath = inferPathFromArgs2(tu.input);
     const tcId = toolCallId(sessionId2, sourceCallId);
@@ -3225,13 +3398,14 @@ async function processContentBlock(bundle, sessionId2, messageId2, eventId2, blo
     const sourceCallId = tr.tool_use_id ?? null;
     const isError = tr.is_error === true ? 1 : 0;
     const text = stringifyOrNull2(tr.content) ?? "";
+    const overflowId = text.length > PREVIEW_MAX2 ? stageText(pending.objects, text) : null;
     pending.blocks.push({
       block_id: blkId,
       message_id: messageId2,
       event_id: null,
       ordinal: blockOrdinal,
       block_type: "tool_result",
-      text_object_id: text.length > PREVIEW_MAX2 ? await putText(bundle, text) : null,
+      text_object_id: overflowId,
       text_inline: text.slice(0, PREVIEW_MAX2),
       is_error: isError,
       visibility: "default",
@@ -3250,7 +3424,7 @@ async function processContentBlock(bundle, sessionId2, messageId2, eventId2, blo
       duration_ms: null,
       stdout_object_id: null,
       stderr_object_id: null,
-      output_object_id: text.length > PREVIEW_MAX2 ? await putText(bundle, text) : null,
+      output_object_id: overflowId,
       preview: text.slice(0, PREVIEW_MAX2),
       raw_record_id: rawRecordId2
     });
@@ -3678,17 +3852,34 @@ async function readdirSafe2(dir) {
 
 // src/importers/gemini/index.ts
 var PREVIEW_MAX3 = 4e3;
-async function compileGemini(bundle, root) {
+async function compileGemini(bundle, root, options = {}) {
+  const logger = options.logger;
   const batch = startBatch(bundle, "gemini", [root]);
   const counts = emptyCounts();
+  logger?.info({ batch_id: batch.batch_id, root }, "gemini batch started");
   try {
     for await (const file of discoverGeminiChats(root)) {
       counts.source_files_seen++;
+      logger?.debug(
+        {
+          path: file.filePath,
+          project_dir: file.projectDir,
+          project_root: file.projectRoot
+        },
+        "gemini source file discovered"
+      );
       try {
-        const fc = await compileGeminiFile(bundle, batch, file);
+        const fc = await compileGeminiFile(bundle, batch, file, logger);
         addCounts3(counts, fc);
       } catch (error) {
         counts.errors++;
+        logger?.warn(
+          {
+            err: error,
+            path: file.filePath
+          },
+          "gemini source file failed"
+        );
         await recordError(bundle, batch.batch_id, {
           kind: "gemini_file_failed",
           message: error instanceof Error ? error.message : String(error),
@@ -3697,8 +3888,10 @@ async function compileGemini(bundle, root) {
       }
     }
     finishBatch(bundle, batch, counts, "completed");
+    logger?.info({ batch_id: batch.batch_id, counts }, "gemini batch completed");
   } catch (error) {
     finishBatch(bundle, batch, counts, "failed");
+    logger?.error({ err: error, batch_id: batch.batch_id, counts }, "gemini batch failed");
     throw error;
   }
   return { batch, counts };
@@ -3733,7 +3926,7 @@ function addCounts3(target, source) {
   target.edges += source.edges;
   target.errors += source.errors;
 }
-async function compileGeminiFile(bundle, batch, file) {
+async function compileGeminiFile(bundle, batch, file, logger) {
   const counts = emptyFileCounts3();
   const { row: sourceFile, alreadyKnown } = await registerSourceFile(bundle, {
     sourceTool: "gemini",
@@ -3743,12 +3936,21 @@ async function compileGeminiFile(bundle, batch, file) {
   });
   if (alreadyKnown) {
     counts.source_files_skipped = 1;
+    logger?.debug(
+      { path: file.filePath, source_file_id: sourceFile.source_file_id },
+      "gemini source file skipped"
+    );
     return counts;
   }
   counts.source_files_imported = 1;
+  logger?.debug(
+    { path: file.filePath, source_file_id: sourceFile.source_file_id },
+    "gemini source file registered"
+  );
   const text = await readFile7(file.filePath, "utf8");
   const parsed = JSON.parse(text);
-  const fileObjectId = await putBytes(bundle, Buffer.from(text, "utf8"), {
+  const objects = createPendingObjects();
+  const fileObjectId = stageBytes(objects, Buffer.from(text, "utf8"), {
     mimeType: "application/json",
     encoding: "utf-8"
   });
@@ -3778,7 +3980,8 @@ async function compileGeminiFile(bundle, batch, file) {
     toolResults: [],
     artifacts: [],
     searchDocs: [],
-    project: null
+    project: null,
+    objects
   };
   const sourceSid = parsed.sessionId ?? path11.basename(file.filePath, ".json");
   const sessionPk = sessionId("gemini", sourceSid);
@@ -3815,6 +4018,7 @@ async function compileGeminiFile(bundle, batch, file) {
     );
   }
   buildSearchDocs3(pending);
+  await flushPendingObjects(bundle, pending.objects);
   transactional(bundle.db, () => {
     flushPending3(bundle, pending);
   });
@@ -3826,12 +4030,16 @@ async function compileGeminiFile(bundle, batch, file) {
   counts.tool_calls = pending.toolCallsList.length;
   counts.tool_results = pending.toolResults.length;
   counts.artifacts = pending.artifacts.length;
+  logger?.debug(
+    { path: file.filePath, source_file_id: sourceFile.source_file_id, counts },
+    "gemini source file imported"
+  );
   return counts;
 }
 async function processMessage(bundle, sessionId2, sourceFileId2, index, msg, batchId, pending) {
   const ordinal = index + 1;
   const ts = msg.timestamp ?? null;
-  const payloadId = await putJson(bundle, msg);
+  const payloadId = stageJson(pending.objects, msg);
   const pointer = `/messages/${index}`;
   const rawObjectIdInput = sha256Hex(`${pointer}
 ${JSON.stringify(msg)}`);
@@ -3948,7 +4156,7 @@ ${JSON.stringify(msg)}`);
 }
 async function pushTextBlock(bundle, pending, messageId2, blockOrdinal, blockType, text, rawRecordId2, visibility = "default") {
   if (!text) return;
-  const overflowId = text.length > PREVIEW_MAX3 ? await putText(bundle, text) : null;
+  const overflowId = text.length > PREVIEW_MAX3 ? stageText(pending.objects, text) : null;
   pending.blocks.push({
     block_id: blockId(messageId2, blockOrdinal),
     message_id: messageId2,
@@ -3965,7 +4173,7 @@ async function processToolCall(bundle, sessionId2, messageId2, eventId2, index,
   const sourceCallId = tc.id ?? `${messageId2}:${index}`;
   const toolName = tc.name ?? "unknown";
   const toolCallId2 = toolCallId(sessionId2, sourceCallId);
-  const argsObjectId = tc.args ? await putJson(bundle, tc.args) : null;
+  const argsObjectId = tc.args ? stageJson(pending.objects, tc.args) : null;
   pending.toolCallsList.push({
     tool_call_id: toolCallId2,
     message_id: messageId2,
@@ -3984,7 +4192,7 @@ async function processToolCall(bundle, sessionId2, messageId2, eventId2, index,
   });
   const isError = tc.status === "error" ? 1 : 0;
   const resultText = renderToolResultText(tc.result);
-  const overflowId = resultText.length > PREVIEW_MAX3 ? await putText(bundle, resultText) : null;
+  const overflowId = resultText.length > PREVIEW_MAX3 ? stageText(pending.objects, resultText) : null;
   pending.toolResults.push({
     tool_result_id: toolResultId(sessionId2, sourceCallId),
     tool_call_id: toolCallId2,
@@ -4001,7 +4209,7 @@ async function processToolCall(bundle, sessionId2, messageId2, eventId2, index,
     const rd = tc.resultDisplay;
     if (rd.fileDiff || rd.filePath) {
       const diffText = rd.fileDiff ?? "";
-      const diffId = diffText ? await putText(bundle, diffText, { mimeType: "text/x-diff" }) : null;
+      const diffId = diffText ? stageText(pending.objects, diffText, { mimeType: "text/x-diff" }) : null;
       pending.artifacts.push({
         artifact_id: artifactId(sessionId2, "gemini", `${toolCallId2}:diff`),
         kind: "diff",
@@ -4393,17 +4601,34 @@ async function readdirSafe3(dir) {
 
 // src/importers/cursor/index.ts
 var PREVIEW_MAX4 = 4e3;
-async function compileCursor(bundle, root) {
+async function compileCursor(bundle, root, options = {}) {
+  const logger = options.logger;
   const batch = startBatch(bundle, "cursor", [root]);
   const counts = emptyCounts();
+  logger?.info({ batch_id: batch.batch_id, root }, "cursor batch started");
   try {
     for await (const store of discoverCursorStores(root)) {
       counts.source_files_seen++;
+      logger?.debug(
+        {
+          path: store.filePath,
+          workspace_id: store.workspaceId,
+          agent_id: store.agentId
+        },
+        "cursor store discovered"
+      );
       try {
-        const fc = await compileCursorStore(bundle, batch, store);
+        const fc = await compileCursorStore(bundle, batch, store, logger);
         addCounts4(counts, fc);
       } catch (error) {
         counts.errors++;
+        logger?.warn(
+          {
+            err: error,
+            path: store.filePath
+          },
+          "cursor store failed"
+        );
         await recordError(bundle, batch.batch_id, {
           kind: "cursor_store_failed",
           message: error instanceof Error ? error.message : String(error),
@@ -4412,8 +4637,10 @@ async function compileCursor(bundle, root) {
       }
     }
     finishBatch(bundle, batch, counts, "completed");
+    logger?.info({ batch_id: batch.batch_id, counts }, "cursor batch completed");
   } catch (error) {
     finishBatch(bundle, batch, counts, "failed");
+    logger?.error({ err: error, batch_id: batch.batch_id, counts }, "cursor batch failed");
     throw error;
   }
   return { batch, counts };
@@ -4448,7 +4675,7 @@ function addCounts4(target, source) {
   target.edges += source.edges;
   target.errors += source.errors;
 }
-async function compileCursorStore(bundle, batch, store) {
+async function compileCursorStore(bundle, batch, store, logger) {
   const counts = emptyFileCounts4();
   const { row: sourceFile, alreadyKnown } = await registerSourceFile(bundle, {
     sourceTool: "cursor",
@@ -4458,9 +4685,17 @@ async function compileCursorStore(bundle, batch, store) {
   });
   if (alreadyKnown) {
     counts.source_files_skipped = 1;
+    logger?.debug(
+      { path: store.filePath, source_file_id: sourceFile.source_file_id },
+      "cursor store skipped"
+    );
     return counts;
   }
   counts.source_files_imported = 1;
+  logger?.debug(
+    { path: store.filePath, source_file_id: sourceFile.source_file_id },
+    "cursor store registered"
+  );
   const cdb = new Database2(store.filePath, { readonly: true, fileMustExist: true });
   try {
     const pending = {
@@ -4473,7 +4708,8 @@ async function compileCursorStore(bundle, batch, store) {
       toolCallsList: [],
       toolResults: [],
       artifacts: [],
-      searchDocs: []
+      searchDocs: [],
+      objects: createPendingObjects()
     };
     const metaRow = cdb.prepare(`SELECT value FROM meta WHERE key='0'`).get();
     let meta = {};
@@ -4485,7 +4721,7 @@ async function compileCursorStore(bundle, batch, store) {
       } catch {
         meta = {};
       }
-      const metaObjId = await putBytes(bundle, Buffer.from(metaText, "utf8"), {
+      const metaObjId = stageBytes(pending.objects, Buffer.from(metaText, "utf8"), {
         mimeType: "application/json",
         encoding: "utf-8"
       });
@@ -4524,7 +4760,7 @@ async function compileCursorStore(bundle, batch, store) {
       const blob = blobs[i];
       if (!blob) continue;
       const ordinal = i + 1;
-      const blobObjectId = await putBytes(bundle, blob.data);
+      const blobObjectId = stageBytes(pending.objects, blob.data);
       const blobRawId = rawRecordId(sourceFile.source_file_id, ordinal, blobObjectId);
       let parsed = null;
       const firstByte = blob.data[0];
@@ -4544,7 +4780,7 @@ async function compileCursorStore(bundle, batch, store) {
         json_pointer: `blobs/${blob.id}`,
         native_id: blob.id,
         raw_object_id: blobObjectId,
-        decoded_json_object_id: parsed != null ? await putJson(bundle, parsed) : null,
+        decoded_json_object_id: parsed != null ? stageJson(pending.objects, parsed) : null,
         parser_status: parsed != null ? "ok" : looksJson ? "failed" : "partial",
         confidence: "low",
         // timeline order from blob list isn't canonical
@@ -4599,6 +4835,7 @@ async function compileCursorStore(bundle, batch, store) {
       }
     }
     buildSearchDocs4(pending);
+    await flushPendingObjects(bundle, pending.objects);
     transactional(bundle.db, () => {
       flushPending4(bundle, pending);
     });
@@ -4610,6 +4847,10 @@ async function compileCursorStore(bundle, batch, store) {
     counts.tool_calls = pending.toolCallsList.length;
     counts.tool_results = pending.toolResults.length;
     counts.artifacts = pending.artifacts.length;
+    logger?.debug(
+      { path: store.filePath, source_file_id: sourceFile.source_file_id, counts },
+      "cursor store imported"
+    );
     return counts;
   } finally {
     cdb.close();
@@ -4634,7 +4875,7 @@ function mapRole(role) {
 }
 async function pushTextBlock2(bundle, pending, messageId2, ordinal, blockType, text, rawRecordId2, visibility = "default") {
   if (!text) return;
-  const overflow = text.length > PREVIEW_MAX4 ? await putText(bundle, text) : null;
+  const overflow = text.length > PREVIEW_MAX4 ? stageText(pending.objects, text) : null;
   pending.blocks.push({
     block_id: blockId(messageId2, ordinal),
     message_id: messageId2,
@@ -4685,7 +4926,7 @@ async function processContentItem(bundle, sessionId2, messageId2, eventId2, ordi
   if (t === "tool-call") {
     const sourceCallId = item.toolCallId ?? `${ordinal}`;
     const toolName = item.toolName ?? "unknown";
-    const argsObjectId = item.args != null ? await putJson(bundle, item.args) : null;
+    const argsObjectId = item.args != null ? stageJson(pending.objects, item.args) : null;
     const tcId = toolCallId(sessionId2, sourceCallId);
     pending.blocks.push({
       block_id: blockId(messageId2, ordinal),
@@ -4722,7 +4963,7 @@ async function processContentItem(bundle, sessionId2, messageId2, eventId2, ordi
   if (t === "tool-result") {
     const sourceCallId = item.toolCallId ?? `${ordinal}`;
     const text = stringifyOrNull3(item.result) ?? "";
-    const overflow = text.length > PREVIEW_MAX4 ? await putText(bundle, text) : null;
+    const overflow = text.length > PREVIEW_MAX4 ? stageText(pending.objects, text) : null;
     const isError = readIsError(item) ? 1 : 0;
     pending.blocks.push({
       block_id: blockId(messageId2, ordinal),