@nocoo/pika 0.5.4 → 0.5.6

package/dist/bin.js

@@ -95,7 +95,7 @@ var package_default;
 var init_package = __esm(() => {
   package_default = {
     name: "pika",
-    version: "0.5.4",
+    version: "0.5.6",
     private: true,
     workspaces: [
       "packages/*"
@@ -2327,7 +2327,7 @@ function buildRawSourceFiles(session, sessionRow, messageRawStrings, messageIds,
 function createOpenCodeSqliteDriver(openDb, dbPath) {
   return {
     source: "opencode",
-    async run(prevCursor, ctx) {
+    async run(prevCursor, ctx, onResult) {
       let dbStat;
       try {
         dbStat = await stat10(dbPath);
@@ -2391,7 +2391,11 @@ function createOpenCodeSqliteDriver(openDb, dbPath) {
           if (shouldSkipForJson(sessionKey, result.canonical.lastMessageAt, result.canonical.messages.length, ctx.openCodeSessionState)) {
             continue;
           }
-          results.push(result);
+          if (onResult) {
+            await onResult(result);
+          } else {
+            results.push(result);
+          }
           if (!ctx.openCodeSessionState) {
             ctx.openCodeSessionState = new Map;
           }
@@ -2874,14 +2878,121 @@ async function runSyncPipeline(input, opts) {
     ...input.cursorState,
     files: { ...input.cursorState.files }
   };
-  const allResults = [];
-  const parseErrors = [];
+  let totalParsed = 0;
+  let totalEmpty = 0;
   let totalFiles = 0;
   let totalSkipped = 0;
-  const sessionKeyToFile = new Map;
-  const prevCursors = new Map;
+  const parseErrors = [];
+  const currentBatch = [];
+  const batchSessionToFile = new Map;
+  const batchPrevCursors = new Map;
   const dbSourcedSessionKeys = new Set;
   let prevDbCursor;
+  let uploadResult;
+  let contentResult;
+  let uploadOpts;
+  let contentOpts;
+  let contentProgressDone = 0;
+  async function flushBatch() {
+    if (currentBatch.length === 0)
+      return;
+    const batchEmpty = currentBatch.filter((r) => r.canonical.messages.length === 0).length;
+    const uploadable = currentBatch.filter((r) => r.canonical.messages.length > 0);
+    totalEmpty += batchEmpty;
+    if (uploadable.length === 0 || !opts.upload) {
+      totalParsed += uploadable.length;
+      currentBatch.length = 0;
+      batchSessionToFile.clear();
+      batchPrevCursors.clear();
+      return;
+    }
+    if (!uploadOpts) {
+      uploadOpts = {
+        apiUrl: opts.apiUrl,
+        apiKey: opts.apiKey,
+        userId: opts.userId,
+        fetch: opts.fetch,
+        sleep: opts.sleep
+      };
+      contentOpts = {
+        apiUrl: opts.apiUrl,
+        apiKey: opts.apiKey,
+        fetch: opts.fetch,
+        sleep: opts.sleep
+      };
+      uploadResult = {
+        totalIngested: 0,
+        totalConflicts: 0,
+        totalBatches: 0,
+        errors: []
+      };
+      contentResult = { uploaded: 0, skipped: 0, errors: [] };
+    }
+    const transformed = uploadable.map((r) => toSessionSnapshot(r.canonical, r.raw));
+    const batchSnapshots = transformed.map((t) => t.snapshot);
+    const batchUploadResult = await uploadMetadataBatches(batchSnapshots, uploadOpts);
+    uploadResult.totalIngested += batchUploadResult.totalIngested;
+    uploadResult.totalConflicts += batchUploadResult.totalConflicts;
+    uploadResult.totalBatches += batchUploadResult.totalBatches;
+    uploadResult.errors.push(...batchUploadResult.errors);
+    const effectiveContentOpts = { ...contentOpts };
+    if (log) {
+      const completedSessions = new Set;
+      const originalFetch = contentOpts.fetch ?? globalThis.fetch;
+      effectiveContentOpts.fetch = async (input2, init) => {
+        const response = await originalFetch(input2, init);
+        const url = typeof input2 === "string" ? input2 : input2.url;
+        if (url.includes("/api/ingest/content/") && url.endsWith("/canonical")) {
+          const parts = url.split("/");
+          const sessionKey = decodeURIComponent(parts[parts.length - 2]);
+          if (!completedSessions.has(sessionKey)) {
+            completedSessions.add(sessionKey);
+            contentProgressDone++;
+            log.uploadContentProgress(contentProgressDone, totalParsed + uploadable.length);
+          }
+        }
+        return response;
+      };
+    }
+    const batchContentResult = await uploadContentBatch(uploadable.map((r, i) => ({
+      canonical: r.canonical,
+      raw: r.raw,
+      precomputed: transformed[i].precomputed
+    })), log ? effectiveContentOpts : contentOpts, opts.contentConcurrency);
+    contentResult.uploaded += batchContentResult.uploaded;
+    contentResult.skipped += batchContentResult.skipped;
+    contentResult.errors.push(...batchContentResult.errors);
+    if (batchContentResult.errors.length > 0) {
+      const rolledBackFiles = new Set;
+      let rollbackDbCursor = false;
+      for (const { sessionKey } of batchContentResult.errors) {
+        const filePath = batchSessionToFile.get(sessionKey);
+        if (filePath && !rolledBackFiles.has(filePath)) {
+          rolledBackFiles.add(filePath);
+          const prev = batchPrevCursors.get(filePath);
+          if (prev === undefined) {
+            delete cursorState.files[filePath];
+          } else {
+            cursorState.files[filePath] = prev;
+          }
+        }
+        if (dbSourcedSessionKeys.has(sessionKey)) {
+          rollbackDbCursor = true;
+        }
+      }
+      if (rollbackDbCursor) {
+        cursorState.openCodeSqlite = prevDbCursor;
+      }
+    }
+    totalParsed += uploadable.length;
+    for (const r of currentBatch) {
+      r.raw = undefined;
+      r.canonical.messages = [];
+    }
+    currentBatch.length = 0;
+    batchSessionToFile.clear();
+    batchPrevCursors.clear();
+  }
   for (const driver of fileDrivers) {
     log?.discoverStart(driver.source);
     const files = await driver.discover(discoverOpts);
@@ -2903,14 +3014,17 @@ async function runSyncPipeline(input, opts) {
       try {
         const results = await driver.parse(filePath, resume);
         if (results.length > 0) {
-          allResults.push(...results);
+          currentBatch.push(...results);
           log?.parseDone(driver.source, filePath, results.length);
-          prevCursors.set(filePath, cursorState.files[filePath]);
+          batchPrevCursors.set(filePath, cursorState.files[filePath]);
           for (const r of results) {
-            sessionKeyToFile.set(r.canonical.sessionKey, filePath);
+            batchSessionToFile.set(r.canonical.sessionKey, filePath);
           }
           const newCursor = driver.buildCursor(fingerprint, results);
           cursorState.files[filePath] = newCursor;
+          if (currentBatch.length >= METADATA_BATCH_SIZE) {
+            await flushBatch();
+          }
         }
       } catch (err) {
         parseErrors.push({
@@ -2926,13 +3040,22 @@ async function runSyncPipeline(input, opts) {
     try {
       log?.dbDriverStart(dbDriver.source);
       prevDbCursor = cursorState.openCodeSqlite;
-      const dbResult = await dbDriver.run(prevDbCursor, syncCtx);
-      allResults.push(...dbResult.results);
+      const dbResult = await dbDriver.run(prevDbCursor, syncCtx, async (r) => {
+        currentBatch.push(r);
+        dbSourcedSessionKeys.add(r.canonical.sessionKey);
+        if (currentBatch.length >= METADATA_BATCH_SIZE) {
+          await flushBatch();
+        }
+      });
+      cursorState.openCodeSqlite = dbResult.cursor;
+      log?.dbDriverDone(dbDriver.source, dbResult.rowCount);
       for (const r of dbResult.results) {
+        currentBatch.push(r);
         dbSourcedSessionKeys.add(r.canonical.sessionKey);
+        if (currentBatch.length >= METADATA_BATCH_SIZE) {
+          await flushBatch();
+        }
       }
-      cursorState.openCodeSqlite = dbResult.cursor;
-      log?.dbDriverDone(dbDriver.source, dbResult.results.length);
     } catch (err) {
       parseErrors.push({
         timestamp: new Date().toISOString(),
@@ -2942,112 +3065,19 @@ async function runSyncPipeline(input, opts) {
       });
     }
   }
-  cursorState.updatedAt = new Date().toISOString();
-  const emptyCount = allResults.filter((r) => r.canonical.messages.length === 0).length;
-  const uploadableResults = allResults.filter((r) => r.canonical.messages.length > 0);
-  let uploadResult;
-  let contentResult;
-  if (opts.upload && uploadableResults.length > 0) {
-    const uploadOpts = {
-      apiUrl: opts.apiUrl,
-      apiKey: opts.apiKey,
-      userId: opts.userId,
-      fetch: opts.fetch,
-      sleep: opts.sleep
-    };
-    const contentOpts = {
-      apiUrl: opts.apiUrl,
-      apiKey: opts.apiKey,
-      fetch: opts.fetch,
-      sleep: opts.sleep
-    };
-    const totalSessions = uploadableResults.length;
-    const pipelineBatches = splitBatches(uploadableResults, METADATA_BATCH_SIZE);
-    uploadResult = {
-      totalIngested: 0,
-      totalConflicts: 0,
-      totalBatches: 0,
-      errors: []
-    };
-    contentResult = {
-      uploaded: 0,
-      skipped: 0,
-      errors: []
-    };
-    log?.uploadMetadataStart(totalSessions);
-    for (const batch of pipelineBatches) {
-      const transformed = batch.map((r) => toSessionSnapshot(r.canonical, r.raw));
-      const batchSnapshots = transformed.map((t) => t.snapshot);
-      const batchUploadResult = await uploadMetadataBatches(batchSnapshots, uploadOpts);
-      uploadResult.totalIngested += batchUploadResult.totalIngested;
-      uploadResult.totalConflicts += batchUploadResult.totalConflicts;
-      uploadResult.totalBatches += batchUploadResult.totalBatches;
-      uploadResult.errors.push(...batchUploadResult.errors);
-    }
+  await flushBatch();
+  if (uploadResult) {
+    log?.uploadMetadataStart(totalParsed);
     log?.uploadMetadataDone(uploadResult.totalIngested, uploadResult.totalConflicts);
-    let contentDone = 0;
-    log?.uploadContentStart(totalSessions);
-    for (const batch of pipelineBatches) {
-      const batchPrecomputed = new Map;
-      for (const r of batch) {
-        const t = toSessionSnapshot(r.canonical, r.raw);
-        batchPrecomputed.set(t.snapshot.sessionKey, t.precomputed);
-      }
-      const wrappedContentOpts = { ...contentOpts };
-      const originalFetch = contentOpts.fetch ?? globalThis.fetch;
-      if (log) {
-        const completedSessions = new Set;
-        wrappedContentOpts.fetch = async (input2, init) => {
-          const response = await originalFetch(input2, init);
-          const url = typeof input2 === "string" ? input2 : input2.url;
-          if (url.includes("/api/ingest/content/") && url.endsWith("/canonical")) {
-            const parts = url.split("/");
-            const sessionKey = decodeURIComponent(parts[parts.length - 2]);
-            if (!completedSessions.has(sessionKey)) {
-              completedSessions.add(sessionKey);
-              contentDone++;
-              log.uploadContentProgress(contentDone, totalSessions);
-            }
-          }
-          return response;
-        };
-      }
-      const batchContentResult = await uploadContentBatch(batch.map((r) => ({
-        canonical: r.canonical,
-        raw: r.raw,
-        precomputed: batchPrecomputed.get(r.canonical.sessionKey)
-      })), log ? wrappedContentOpts : contentOpts, opts.contentConcurrency);
-      contentResult.uploaded += batchContentResult.uploaded;
-      contentResult.skipped += batchContentResult.skipped;
-      contentResult.errors.push(...batchContentResult.errors);
-      if (batchContentResult.errors.length > 0) {
-        const rolledBackFiles = new Set;
-        let rollbackDbCursor = false;
-        for (const { sessionKey } of batchContentResult.errors) {
-          const filePath = sessionKeyToFile.get(sessionKey);
-          if (filePath && !rolledBackFiles.has(filePath)) {
-            rolledBackFiles.add(filePath);
-            const prev = prevCursors.get(filePath);
-            if (prev === undefined) {
-              delete cursorState.files[filePath];
-            } else {
-              cursorState.files[filePath] = prev;
-            }
-          }
-          if (dbSourcedSessionKeys.has(sessionKey)) {
-            rollbackDbCursor = true;
-          }
-        }
-        if (rollbackDbCursor) {
-          cursorState.openCodeSqlite = prevDbCursor;
-        }
-      }
-    }
+  }
+  if (contentResult) {
+    log?.uploadContentStart(totalParsed);
     log?.uploadContentDone(contentResult.uploaded, contentResult.skipped, contentResult.errors.length);
   }
+  cursorState.updatedAt = new Date().toISOString();
   return {
-    totalParsed: uploadableResults.length,
-    totalEmpty: emptyCount,
+    totalParsed,
+    totalEmpty,
     totalFiles,
     totalSkipped,
     parseErrors,
@@ -3071,6 +3101,20 @@ __export(exports_sync, {
 import { basename as basename3, join as join11 } from "path";
 import { defineCommand } from "citty";
 import consola from "consola";
+function spinner() {
+  return SPINNER_FRAMES[spinnerIdx++ % SPINNER_FRAMES.length];
+}
+function rewriteLine(msg) {
+  process.stdout.write(`\r\x1B[2K${msg}`);
+}
+function finalizeLine(consolaFn, msg) {
+  process.stdout.write("\r\x1B[2K");
+  consolaFn(msg);
+}
+function progressBar(pct, width) {
+  const filled = Math.round(pct / 100 * width);
+  return "\u2588".repeat(filled) + "\u2591".repeat(width - filled);
+}
 async function buildDbDriver(driverSet, openDbOverride) {
   if (!driverSet.dbDriversAvailable || !driverSet.discoverOpts.openCodeDbPath) {
     return;
@@ -3085,7 +3129,7 @@ async function buildDbDriver(driverSet, openDbOverride) {
   }
   return createOpenCodeSqliteDriver(openDb, driverSet.discoverOpts.openCodeDbPath);
 }
-var sync_default;
+var SPINNER_FRAMES, spinnerIdx = 0, sync_default;
 var init_sync = __esm(() => {
   init_src();
   init_manager();
@@ -3093,6 +3137,7 @@ var init_sync = __esm(() => {
   init_opencode_sqlite();
   init_cursor_store();
   init_sync_pipeline();
+  SPINNER_FRAMES = ["\u280B", "\u2819", "\u2839", "\u2838", "\u283C", "\u2834", "\u2826", "\u2827", "\u2807", "\u280F"];
   sync_default = defineCommand({
     meta: {
       name: "sync",
@@ -3149,13 +3194,13 @@ var init_sync = __esm(() => {
       consola.start(`Syncing ${sourceCount} source(s)...`);
       const logger = {
         discoverStart(source) {
-          consola.info(`  [${source}] Scanning...`);
+          rewriteLine(`${spinner()} [${source}] Scanning...`);
         },
         discoverDone(source, fileCount) {
-          consola.info(`  [${source}] Found ${fileCount} file(s)`);
+          finalizeLine(consola.info, `  [${source}] Found ${fileCount} file(s)`);
         },
         parseDone(source, filePath, sessionCount) {
-          consola.info(`  [${source}] Parsed ${sessionCount} session(s) from ${basename3(filePath)}`);
+          rewriteLine(`${spinner()} [${source}] Parsed ${sessionCount} session(s) from ${basename3(filePath)}`);
         },
         uploadMetadataStart(sessionCount) {
           consola.info(`Uploading metadata for ${sessionCount} session(s)...`);
@@ -3170,19 +3215,21 @@ var init_sync = __esm(() => {
           consola.info(`Uploading content for ${sessionCount} session(s)...`);
         },
         uploadContentProgress(done, total) {
-          consola.info(`  Content: ${done}/${total}`);
+          const pct = total > 0 ? Math.round(done / total * 100) : 0;
+          const bar = progressBar(pct, 20);
+          rewriteLine(`${spinner()} Content: ${bar} ${done}/${total} (${pct}%)`);
         },
         uploadContentDone(uploaded, skipped, errors) {
           const parts = [`${uploaded} uploaded`, `${skipped} skipped`];
           if (errors > 0)
             parts.push(`${errors} errors`);
-          consola.info(`Content upload done: ${parts.join(", ")}`);
+          finalizeLine(consola.info, `Content upload done: ${parts.join(", ")}`);
         },
         dbDriverStart(source) {
-          consola.info(`  [${source}] Querying database...`);
+          rewriteLine(`${spinner()} [${source}] Querying database...`);
         },
         dbDriverDone(source, sessionCount) {
-          consola.info(`  [${source}] Found ${sessionCount} session(s) from DB`);
+          finalizeLine(consola.info, `  [${source}] Found ${sessionCount} session(s) from DB`);
         }
       };
       const result = await runSyncPipeline({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nocoo/pika",
-  "version": "0.5.4",
+  "version": "0.5.6",
   "description": "Replay and search coding agent sessions",
   "type": "module",
   "bin": {