chatlab-cli 0.21.0 → 0.21.1

package/dist/{chunk-EWCSGYVT.mjs → chunk-AIR7EAF4.mjs}

@@ -13,7 +13,7 @@ import {
   incrementalImport,
   openBetterSqliteDatabase,
   streamingImport
-} from "./chunk-EIRLEPXP.mjs";
+} from "./chunk-7HQWGLX6.mjs";
 
 // src/import/stream-import.ts
 import * as fs from "fs";
@@ -141,4 +141,4 @@ export {
   analyzeIncrementalImport2 as analyzeIncrementalImport,
   analyzeNewImport2 as analyzeNewImport
 };
-//# sourceMappingURL=chunk-EWCSGYVT.mjs.map
+//# sourceMappingURL=chunk-AIR7EAF4.mjs.map

package/dist/{chunk-FBTHZU55.mjs → chunk-LI4X52E7.mjs}

@@ -7,17 +7,17 @@ const __dirname = __pathDirname(__filename);
 import {
   importData,
   parseFile
-} from "./chunk-XUHPCRVT.mjs";
+} from "./chunk-OQVKZTVD.mjs";
 import {
   analyzeIncrementalImport,
   analyzeNewImport,
   incrementalImport,
   streamImport
-} from "./chunk-EWCSGYVT.mjs";
+} from "./chunk-AIR7EAF4.mjs";
 import {
   AGENT_TOOL_REGISTRY,
   CoreDataProvider
-} from "./chunk-3DIWXYMJ.mjs";
+} from "./chunk-4GI2UPCS.mjs";
 import {
   getVersion
 } from "./chunk-2AIOU3PW.mjs";
@@ -114,7 +114,7 @@ import {
   verifyCliDataPath,
   writeAuthProfile,
   writeConfigField
-} from "./chunk-EIRLEPXP.mjs";
+} from "./chunk-7HQWGLX6.mjs";
 
 // src/http/index.ts
 import * as fs14 from "fs";
@@ -849,6 +849,11 @@ function registerSessionIndexRoutes(server2, adapter) {
     const sessionCount = session_index_service_exports.generateIndex(adapter, request.params.id, gapThreshold);
     return { sessionCount };
   });
+  server2.post("/_web/sessions/:id/generate-incremental-index", async (request) => {
+    const gapThreshold = request.body?.gapThreshold ?? 1800;
+    const sessionCount = session_index_service_exports.generateIncrementalIndex(adapter, request.params.id, gapThreshold);
+    return { sessionCount };
+  });
   server2.post("/_web/sessions/:id/clear-index", async (request) => {
     session_index_service_exports.clearIndex(adapter, request.params.id);
     return { success: true };
@@ -1472,7 +1477,7 @@ function registerMergeRoutes(server2, dbManager2, mergeCache) {
       const merged = buildMergedOutput(dataSources, outputName);
       let sessionId;
       if (andImport) {
-        const { streamImport: serverStreamImport } = await import("./stream-import-URGOCXUH.mjs");
+        const { streamImport: serverStreamImport } = await import("./stream-import-VG7RP6KJ.mjs");
         const jsonData = JSON.stringify(merged.chatLabData);
         const tmpDir = fs5.mkdtempSync(path5.join(os2.tmpdir(), "chatlab-merged-"));
         const tmpPath = path5.join(tmpDir, `${outputName}.json`);
@@ -1525,7 +1530,7 @@ function registerMergeRoutes(server2, dbManager2, mergeCache) {
       const db = ensureDb2(dbManager2, sid);
       const exported = exportSessionToJson(db);
       const { db: tempDb, tempDbPath } = mergeCache.createTempDatabase(exported.meta.name);
-      const { TempDbWriter } = await import("./src-KOIGBZJX.mjs");
+      const { TempDbWriter } = await import("./src-QBT5SYW5.mjs");
       const writer = new TempDbWriter(tempDb);
       writer.writeMeta({
         name: exported.meta.name,
@@ -2713,12 +2718,12 @@ var DataSourceManager = class {
   removeSession(sourceId, sessionId) {
     const sources = this.loadAll();
     const ds = sources.find((s) => s.id === sourceId);
-    if (!ds) return false;
-    const before = ds.sessions.length;
+    if (!ds) return null;
+    const removed = ds.sessions.find((s) => s.id === sessionId);
+    if (!removed) return null;
     ds.sessions = ds.sessions.filter((s) => s.id !== sessionId);
-    if (ds.sessions.length === before) return false;
     this.saveAll(sources);
-    return true;
+    return removed;
   }
   updateSession(sourceId, sessionId, updates) {
     const sources = this.loadAll();
@@ -2766,14 +2771,13 @@ function parseRemoteSessionsResponse(body) {
 // ../../packages/sync/src/pull-engine.ts
 import * as fs12 from "fs";
 import * as crypto3 from "crypto";
-var MAX_PAGES_PER_PULL = 50;
+var MAX_PAGES_PER_PULL = 5e3;
 var PULL_OVERLAP_SECONDS = 60;
 function buildPullUrl(baseUrl, remoteSessionId, params) {
   const base = `${baseUrl}/sessions/${remoteSessionId}/messages`;
   const qs = ["format=chatlab"];
   if (params.since !== void 0 && params.since > 0) qs.push(`since=${params.since}`);
   if (params.offset !== void 0 && params.offset > 0) qs.push(`offset=${params.offset}`);
-  if (params.end !== void 0 && params.end > 0) qs.push(`end=${params.end}`);
   if (params.limit !== void 0 && params.limit > 0) qs.push(`limit=${params.limit}`);
   return base + "?" + qs.join("&");
 }
@@ -2815,6 +2819,29 @@ function parseSyncFromFile(filePath) {
     return null;
   }
 }
+function fileContainsMessages(filePath) {
+  try {
+    if (filePath.endsWith(".jsonl")) {
+      const content = fs12.readFileSync(filePath, "utf-8");
+      for (const line of content.split("\n")) {
+        const trimmed = line.trim();
+        if (!trimmed) continue;
+        try {
+          const obj = JSON.parse(trimmed);
+          if (obj._type === "message") return true;
+        } catch {
+          continue;
+        }
+      }
+      return false;
+    }
+    const raw = fs12.readFileSync(filePath, "utf-8");
+    const parsed = JSON.parse(raw);
+    return Array.isArray(parsed.messages) && parsed.messages.length > 0;
+  } catch {
+    return false;
+  }
+}
 function cleanupTempFile(filePath) {
   try {
     if (fs12.existsSync(filePath)) fs12.unlinkSync(filePath);
@@ -2829,6 +2856,7 @@ var PullEngine = class {
   logger;
   isImporting;
   pullingSourceIds = /* @__PURE__ */ new Set();
+  progressMap = /* @__PURE__ */ new Map();
   constructor(options) {
     this.fetcher = options.fetcher;
     this.importer = options.importer;
@@ -2837,6 +2865,9 @@ var PullEngine = class {
     this.logger = options.logger ?? NOOP_LOGGER;
     this.isImporting = options.isImporting ?? (() => false);
   }
+  getProgress() {
+    return Array.from(this.progressMap.values());
+  }
   async importTempFile(baseUrl, sess, tempFile) {
     let targetId = sess.targetSessionId;
     if (!targetId) {
@@ -2850,6 +2881,11 @@ var PullEngine = class {
     return this.importer.importFile(tempFile, targetId || void 0, externalId);
   }
   async executePullSession(sourceId, ds, sess) {
+    const currentDs = this.dsManager.get(sourceId);
+    if (!currentDs || !currentDs.sessions.some((s) => s.id === sess.id)) {
+      this.logger.info(`[Pull] Skipping "${sess.name}": session no longer exists`);
+      return { success: true, newMessageCount: 0 };
+    }
     if (this.isImporting()) {
       this.logger.info(`[Pull] Skipping "${sess.name}": import in progress`);
       return { success: false, newMessageCount: 0, error: "Import in progress" };
@@ -2857,34 +2893,114 @@ var PullEngine = class {
     this.logger.info(`[Pull] Pulling "${sess.name}" from ${ds.baseUrl}`);
     let totalNewMessages = 0;
     let since = sess.lastPullAt;
-    let offset = 0;
-    let end;
     let pageCount = 0;
     let resyncAttempted = false;
+    this.progressMap.set(sess.id, { sessionId: sess.id, sessionName: sess.name, current: 0, pages: 0, done: false });
     try {
       while (pageCount < MAX_PAGES_PER_PULL) {
         pageCount++;
         const tempFile = await this.fetcher.fetchToTempFile(ds.baseUrl, sess.remoteSessionId, ds.token, {
           since,
-          offset,
-          end,
           limit: ds.pullLimit
         });
         try {
           const stat = fs12.statSync(tempFile);
           this.logger.info(`[Pull] "${sess.name}" page ${pageCount}: fetched ${stat.size} bytes`);
+          const sync0 = parseSyncFromFile(tempFile);
+          if (stat.size < 1024) {
+            if (!fileContainsMessages(tempFile) && sync0?.hasMore === false) {
+              cleanupTempFile(tempFile);
+              const retryDelays = [2e3, 3e3, 5e3];
+              let retrySuccess = false;
+              let retryHasMore = false;
+              for (let ri = 0; ri < retryDelays.length; ri++) {
+                this.logger.info(
+                  `[Pull] "${sess.name}" page ${pageCount} got empty response, retry ${ri + 1}/${retryDelays.length} after ${retryDelays[ri]}ms`
+                );
+                await new Promise((r) => setTimeout(r, retryDelays[ri]));
+                const retryFile = await this.fetcher.fetchToTempFile(ds.baseUrl, sess.remoteSessionId, ds.token, {
+                  since,
+                  limit: ds.pullLimit
+                });
+                const retryStat = fs12.statSync(retryFile);
+                this.logger.info(`[Pull] "${sess.name}" retry ${ri + 1}: fetched ${retryStat.size} bytes`);
+                const retrySync = parseSyncFromFile(retryFile);
+                if (retryStat.size < 1024 && !fileContainsMessages(retryFile) && retrySync?.hasMore === false) {
+                  cleanupTempFile(retryFile);
+                  continue;
+                }
+                const retryResult = await this.importTempFile(ds.baseUrl, sess, retryFile);
+                cleanupTempFile(retryFile);
+                if (retryResult.needFullResync && !resyncAttempted) {
+                  resyncAttempted = true;
+                  this.logger.info(`[Pull] Resetting since=0 for "${sess.name}" full resync`);
+                  since = 0;
+                  pageCount = 0;
+                  sess.targetSessionId = "";
+                  sess.lastPullAt = 0;
+                  this.dsManager.updateSession(sourceId, sess.id, { targetSessionId: "", lastPullAt: 0 });
+                  retrySuccess = true;
+                  retryHasMore = true;
+                  break;
+                }
+                if (retryResult.needFullResync) {
+                  const errMsg = "Full resync failed";
+                  this.logger.error(`[Pull] Full resync already attempted for "${sess.name}", aborting`);
+                  this.dsManager.updateSession(sourceId, sess.id, {
+                    lastPullAt: Math.floor(Date.now() / 1e3),
+                    lastStatus: "error",
+                    lastError: errMsg
+                  });
+                  this.notifier.onPullResult(sourceId, sess.id, "error", errMsg);
+                  this.markProgressDone(sess.id);
+                  return { success: false, newMessageCount: 0, error: errMsg };
+                }
+                if (!retryResult.success) {
+                  const errMsg = retryResult.error || "Import failed";
+                  this.dsManager.updateSession(sourceId, sess.id, {
+                    lastPullAt: Math.floor(Date.now() / 1e3),
+                    lastStatus: "error",
+                    lastError: errMsg
+                  });
+                  this.notifier.onPullResult(sourceId, sess.id, "error", errMsg);
+                  this.markProgressDone(sess.id);
+                  return { success: false, newMessageCount: 0, error: errMsg };
+                }
+                if (retryResult.success && retryResult.sessionId && !sess.targetSessionId) {
+                  sess.targetSessionId = retryResult.sessionId;
+                  this.dsManager.updateSession(sourceId, sess.id, { targetSessionId: retryResult.sessionId });
+                }
+                totalNewMessages += retryResult.newMessageCount;
+                this.progressMap.set(sess.id, {
+                  sessionId: sess.id,
+                  sessionName: sess.name,
+                  current: totalNewMessages,
+                  pages: pageCount,
+                  done: false
+                });
+                if (retrySync?.hasMore && retrySync.nextSince !== void 0) {
+                  since = retrySync.nextSince;
+                }
+                retryHasMore = !!retrySync?.hasMore;
+                retrySuccess = true;
+                break;
+              }
+              if (!retrySuccess) break;
+              if (!retryHasMore) break;
+              continue;
+            }
+          }
           if (stat.size === 0) {
             cleanupTempFile(tempFile);
             break;
           }
-          const sync = parseSyncFromFile(tempFile);
+          const sync = sync0;
           const result = await this.importTempFile(ds.baseUrl, sess, tempFile);
           cleanupTempFile(tempFile);
           if (result.needFullResync && !resyncAttempted) {
             resyncAttempted = true;
             this.logger.info(`[Pull] Resetting since=0 for "${sess.name}" full resync`);
             since = 0;
-            offset = 0;
             pageCount = 0;
             sess.targetSessionId = "";
             sess.lastPullAt = 0;
@@ -2900,6 +3016,7 @@ var PullEngine = class {
               lastError: errMsg
             });
             this.notifier.onPullResult(sourceId, sess.id, "error", errMsg);
+            this.markProgressDone(sess.id);
             return { success: false, newMessageCount: 0, error: errMsg };
           }
           if (!result.success) {
@@ -2910,6 +3027,7 @@ var PullEngine = class {
               lastError: errMsg
             });
             this.notifier.onPullResult(sourceId, sess.id, "error", errMsg);
+            this.markProgressDone(sess.id);
             return { success: false, newMessageCount: 0, error: errMsg };
           }
           if (!sess.targetSessionId && result.sessionId) {
@@ -2917,11 +3035,15 @@ var PullEngine = class {
             this.dsManager.updateSession(sourceId, sess.id, { targetSessionId: result.sessionId });
           }
           totalNewMessages += result.newMessageCount;
+          this.progressMap.set(sess.id, {
+            sessionId: sess.id,
+            sessionName: sess.name,
+            current: totalNewMessages,
+            pages: pageCount,
+            done: false
+          });
           if (!sync || !sync.hasMore) break;
           if (sync.nextSince !== void 0) since = sync.nextSince;
-          if (sync.nextOffset !== void 0) offset = sync.nextOffset;
-          else offset = 0;
-          if (sync.watermark !== void 0 && !end) end = sync.watermark;
         } catch (importErr) {
           cleanupTempFile(tempFile);
           throw importErr;
@@ -2938,6 +3060,7 @@ var PullEngine = class {
       });
       if (totalNewMessages > 0) this.notifier.onSessionListChanged();
       this.notifier.onPullResult(sourceId, sess.id, "success", `+${totalNewMessages} messages`);
+      this.markProgressDone(sess.id);
       return { success: true, newMessageCount: totalNewMessages };
     } catch (error) {
       const errMsg = error.message || "Pull failed";
@@ -2948,9 +3071,17 @@ var PullEngine = class {
         lastError: errMsg
       });
       this.notifier.onPullResult(sourceId, sess.id, "error", errMsg);
+      this.markProgressDone(sess.id);
       return { success: false, newMessageCount: 0, error: errMsg };
     }
   }
+  markProgressDone(sessionId) {
+    const p = this.progressMap.get(sessionId);
+    if (p) {
+      p.done = true;
+      setTimeout(() => this.progressMap.delete(sessionId), 5e3);
+    }
+  }
   async pullAllSessions(ds) {
     if (this.pullingSourceIds.has(ds.id)) {
       this.logger.info(`[Pull] Skipping pullAllSessions for "${ds.baseUrl}": pull already in progress`);
@@ -3077,7 +3208,7 @@ var NodeFetcher = class {
   async fetchToTempFile(baseUrl, remoteSessionId, token, params) {
     const url = buildPullUrl(baseUrl, remoteSessionId, params);
     const headers = {
-      Accept: "application/json, application/x-ndjson"
+      Accept: "application/json"
     };
     if (token) headers["Authorization"] = `Bearer ${token}`;
     const response = await fetch(url, { headers, signal: AbortSignal.timeout(12e4) });
@@ -3109,7 +3240,28 @@ var DirectImporter = class {
   }
   sessionExists(sessionId) {
     const dbPath = this.dbManager.getDbPath(sessionId);
-    return fs13.existsSync(dbPath);
+    if (!fs13.existsSync(dbPath)) return false;
+    try {
+      const db = openBetterSqliteDatabase(dbPath, { readonly: true, nativeBinding: this.nativeBinding });
+      const row = db.prepare("SELECT COUNT(*) as cnt FROM sqlite_master WHERE type='table' AND name='message'").get();
+      db.close();
+      if (row.cnt === 0) {
+        this.logger.warn(`[DirectImporter] DB file exists but has no message table: ${sessionId}, removing`);
+        try {
+          fs13.unlinkSync(dbPath);
+        } catch {
+        }
+        return false;
+      }
+      return true;
+    } catch {
+      this.logger.warn(`[DirectImporter] Cannot validate DB file: ${sessionId}, removing`);
+      try {
+        fs13.unlinkSync(dbPath);
+      } catch {
+      }
+      return false;
+    }
   }
   async importFile(tempFile, targetSessionId, externalId) {
     if (targetSessionId && this.sessionExists(targetSessionId)) {
@@ -3139,7 +3291,7 @@ var DirectImporter = class {
           sessionId
         };
       }
-      if (result.error?.includes("not found") || result.error?.includes("session_not_found")) {
+      if (result.error?.includes("not found") || result.error?.includes("session_not_found") || result.error?.includes("no such table")) {
         return { success: false, newMessageCount: 0, sessionId, needFullResync: true };
       }
       return { success: false, newMessageCount: 0, sessionId, error: result.error };
@@ -3177,7 +3329,7 @@ var NoopNotifier = class {
 
 // src/sync/routes.ts
 function registerAutomationRoutes(server2, ctx) {
-  const { dsManager: dsManager2, pullEngine: pullEngine2, serverInfo } = ctx;
+  const { dsManager: dsManager2, pullEngine: pullEngine2, dbManager: dbManager2, serverInfo } = ctx;
   server2.get("/_web/automation/config", async () => {
     return {
       enabled: true,
@@ -3219,9 +3371,15 @@ function registerAutomationRoutes(server2, ctx) {
     return added;
   });
   server2.delete("/_web/automation/data-sources/:id/sessions/:sessId", async (request, reply) => {
-    const ok = dsManager2.removeSession(request.params.id, request.params.sessId);
-    if (!ok) return reply.code(404).send({ error: "Session not found" });
+    const removed = dsManager2.removeSession(request.params.id, request.params.sessId);
+    if (!removed) return reply.code(404).send({ error: "Session not found" });
     reloadTimer(request.params.id);
+    if (request.query.deleteData === "true" && removed.targetSessionId) {
+      dbManager2.close(removed.targetSessionId);
+      const dbPath = dbManager2.getDbPath(removed.targetSessionId);
+      const fs15 = await import("fs");
+      if (fs15.existsSync(dbPath)) fs15.unlinkSync(dbPath);
+    }
     return { success: true };
   });
   server2.post(
@@ -3233,6 +3391,9 @@ function registerAutomationRoutes(server2, ctx) {
   server2.post("/_web/automation/data-sources/:id/pull-all", async (request) => {
     return pullEngine2.triggerPullAll(request.params.id);
   });
+  server2.get("/_web/automation/sync-progress", async () => {
+    return pullEngine2.getProgress();
+  });
   server2.get("/_web/automation/remote-sessions", async (request, reply) => {
     const { baseUrl, token, keyword, limit, cursor } = request.query;
     if (!baseUrl) return reply.code(400).send({ error: "baseUrl is required" });
@@ -3276,7 +3437,7 @@ function initSync(server2, dbManager2, pathProvider, serverInfo) {
     dsManager,
     logger: syncLogger
   });
-  registerAutomationRoutes(server2, { dsManager, pullEngine, serverInfo });
+  registerAutomationRoutes(server2, { dsManager, pullEngine, dbManager: dbManager2, serverInfo });
   initScheduler({
     dsManager,
     pullEngine,
@@ -3397,4 +3558,4 @@ export {
   startHttpServer,
   stopHttpServer
 };
-//# sourceMappingURL=chunk-FBTHZU55.mjs.map
+//# sourceMappingURL=chunk-LI4X52E7.mjs.map