chatlab-cli 0.1.0 → 0.21.0

package/dist/chunk-DL3BFEAH.mjs.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/import/stream-import.ts"],"sourcesContent":["/**\n * Server/CLI streaming import — adapter for @openchatlab/node-runtime StreamingImporter.\n *\n * Replaces the old buffered-in-memory approach with the same high-performance\n * streaming pipeline used by Electron (batched transactions, deferred indexes,\n * nickname history, FTS, format fallback).\n */\n\nimport * as fs from 'fs'\nimport * as path from 'path'\nimport type { DatabaseManager } from '@openchatlab/node-runtime'\nimport {\n  streamingImport,\n  openBetterSqliteDatabase,\n  analyzeNewImport as sharedAnalyzeNewImport,\n  analyzeIncrementalImport as sharedAnalyzeIncremental,\n  incrementalImport as sharedIncrementalImport,\n} from '@openchatlab/node-runtime'\nimport { generateSessionIndex, generateIncrementalSessionIndex } from '@openchatlab/core'\nimport type {\n  StreamImportResult,\n  StreamImportDeps,\n  ImportProgressCallback,\n  IncrementalImportResult,\n  IncrementalAnalyzeResult,\n  IncrementalImportDeps,\n  ImportOptions,\n  AnalyzeNewImportResult,\n} from '@openchatlab/node-runtime'\nimport { CHAT_DB_TABLES } from '@openchatlab/core'\nimport {\n  detectFormat as parserDetectFormat,\n  detectAllFormats,\n  getFormatFeatureById,\n  getSupportedFormats as parserGetSupportedFormats,\n  scanMultiChatFile as parserScanMultiChatFile,\n  type FormatFeature,\n  type MultiChatInfo,\n  type ParseProgress,\n} from '@openchatlab/parser'\nimport * as crypto from 'crypto'\n\n// ==================== Legacy progress interface (for SSE routes) ====================\n\nexport interface StreamImportProgress {\n  stage: 'detecting' | 'parsing' | 'saving' | 'indexing' | 'done' | 'error'\n  progress: number\n  message: string\n  bytesRead?: number\n  totalBytes?: number\n  messagesProcessed?: number\n}\n\nexport interface StreamImportOptions {\n  formatId?: string\n  chatIndex?: number\n  nativeBinding?: string\n  onProgress?: (progress: StreamImportProgress) => void\n}\n\nfunction generateSessionId(): string {\n  const ts = Date.now()\n  const rand = crypto.randomBytes(4).toString('hex')\n  return `chat_${ts}_${rand}`\n}\n\nfunction resolveNativeBinding(dbManager: DatabaseManager): string | undefined {\n  return (dbManager as any).nativeBinding\n}\n\nfunction buildStreamImportDeps(dbManager: DatabaseManager, onProgress?: ImportProgressCallback): StreamImportDeps {\n  const nativeBinding = resolveNativeBinding(dbManager)\n  return {\n    openDatabase(sessionId: string) {\n      const dbPath = dbManager.getDbPath(sessionId)\n      const dir = path.dirname(dbPath)\n      if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true })\n      const db = openBetterSqliteDatabase(dbPath, { nativeBinding })\n      db.exec(CHAT_DB_TABLES)\n      return db\n    },\n    deleteDatabase(sessionId: string) {\n      const dbPath = dbManager.getDbPath(sessionId)\n      for (const suffix of ['', '-wal', '-shm']) {\n        try {\n          const p = dbPath + suffix\n          if (fs.existsSync(p)) fs.unlinkSync(p)\n        } catch {\n          /* ignore */\n        }\n      }\n    },\n    onProgress: onProgress ?? (() => {}),\n    postImportHook(db) {\n      try {\n        generateSessionIndex(db)\n      } catch {\n        /* non-fatal: frontend can regenerate via generate-index route */\n      }\n    },\n    generateSessionId,\n  }\n}\n\n/**\n * High-performance streaming import: parse a file and write to DB\n * with batched transactions, deferred indexes, and FTS.\n */\nexport async function streamImport(\n  dbManager: DatabaseManager,\n  filePath: string,\n  options?: StreamImportOptions\n): Promise<StreamImportResult> {\n  const { formatId, chatIndex, onProgress } = options || {}\n\n  const formatOptions: Record<string, unknown> = {}\n  if (formatId) formatOptions.formatId = formatId\n  if (chatIndex !== undefined) formatOptions.chatIndex = chatIndex\n\n  const progressAdapter: ImportProgressCallback = onProgress\n    ? (progress) => {\n        let stage: StreamImportProgress['stage'] = 'parsing'\n        let pct = 0\n        switch (progress.stage) {\n          case 'detecting':\n            stage = 'detecting'\n            pct = 5\n            break\n          case 'parsing':\n            stage = 'parsing'\n            pct = Math.min(Math.round(progress.percentage * 0.7), 70)\n            break\n          case 'importing':\n          case 'saving':\n            stage = 'saving'\n            pct = 80\n            break\n          case 'done':\n            stage = 'done'\n            pct = 100\n            break\n          case 'error':\n            stage = 'error'\n            pct = 0\n            break\n        }\n        onProgress({\n          stage,\n          progress: pct,\n          message: progress.message || '',\n          bytesRead: progress.bytesRead,\n          totalBytes: progress.totalBytes,\n          messagesProcessed: progress.messagesProcessed,\n        })\n      }\n    : () => {}\n\n  const deps = buildStreamImportDeps(dbManager, progressAdapter)\n  return streamingImport(filePath, deps, formatOptions)\n}\n\n// ==================== Incremental import ====================\n\nfunction buildIncrementalDeps(dbManager: DatabaseManager, onProgress?: ImportProgressCallback): IncrementalImportDeps {\n  const nativeBinding = resolveNativeBinding(dbManager)\n  return {\n    openDatabase(sessionId: string, readonly?: boolean) {\n      const dbPath = dbManager.getDbPath(sessionId)\n      if (!fs.existsSync(dbPath)) {\n        throw new Error(`Session database not found: ${sessionId}`)\n      }\n      return openBetterSqliteDatabase(dbPath, { readonly: readonly ?? false, nativeBinding })\n    },\n    onProgress: onProgress ?? (() => {}),\n    postImportHook(db) {\n      try {\n        generateIncrementalSessionIndex(db)\n      } catch {\n        /* non-fatal */\n      }\n    },\n  }\n}\n\nexport async function incrementalImport(\n  dbManager: DatabaseManager,\n  sessionId: string,\n  filePath: string,\n  options?: ImportOptions & { onProgress?: ImportProgressCallback }\n): Promise<IncrementalImportResult> {\n  const { onProgress, ...importOpts } = options || {}\n  return sharedIncrementalImport(sessionId, filePath, buildIncrementalDeps(dbManager, onProgress), importOpts)\n}\n\nexport async function analyzeIncrementalImport(\n  dbManager: DatabaseManager,\n  sessionId: string,\n  filePath: string,\n  onProgress?: ImportProgressCallback\n): Promise<IncrementalAnalyzeResult> {\n  return sharedAnalyzeIncremental(sessionId, filePath, buildIncrementalDeps(dbManager, onProgress))\n}\n\nexport async function analyzeNewImport(\n  filePath: string,\n  onProgress?: ImportProgressCallback\n): Promise<AnalyzeNewImportResult> {\n  return sharedAnalyzeNewImport(filePath, onProgress ?? (() => {}))\n}\n\n// ==================== Re-exports from parser ====================\n\nexport {\n  parserDetectFormat as detectFormat,\n  detectAllFormats,\n  getFormatFeatureById,\n  parserGetSupportedFormats as getSupportedFormats,\n  parserScanMultiChatFile as scanMultiChatFile,\n}\nexport type { FormatFeature, MultiChatInfo, ParseProgress }\nexport type {\n  StreamImportResult,\n  IncrementalImportResult,\n  IncrementalAnalyzeResult,\n  AnalyzeNewImportResult,\n  ImportOptions,\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAQA,YAAY,QAAQ;AACpB,YAAY,UAAU;AA+BtB,YAAY,YAAY;AAoBxB,SAAS,oBAA4B;AACnC,QAAM,KAAK,KAAK,IAAI;AACpB,QAAM,OAAc,mBAAY,CAAC,EAAE,SAAS,KAAK;AACjD,SAAO,QAAQ,EAAE,IAAI,IAAI;AAC3B;AAEA,SAAS,qBAAqB,WAAgD;AAC5E,SAAQ,UAAkB;AAC5B;AAEA,SAAS,sBAAsB,WAA4B,YAAuD;AAChH,QAAM,gBAAgB,qBAAqB,SAAS;AACpD,SAAO;AAAA,IACL,aAAa,WAAmB;AAC9B,YAAM,SAAS,UAAU,UAAU,SAAS;AAC5C,YAAM,MAAW,aAAQ,MAAM;AAC/B,UAAI,CAAI,cAAW,GAAG,EAAG,CAAG,aAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAC9D,YAAM,KAAK,yBAAyB,QAAQ,EAAE,cAAc,CAAC;AAC7D,SAAG,KAAK,cAAc;AACtB,aAAO;AAAA,IACT;AAAA,IACA,eAAe,WAAmB;AAChC,YAAM,SAAS,UAAU,UAAU,SAAS;AAC5C,iBAAW,UAAU,CAAC,IAAI,QAAQ,MAAM,GAAG;AACzC,YAAI;AACF,gBAAM,IAAI,SAAS;AACnB,cAAO,cAAW,CAAC,EAAG,CAAG,cAAW,CAAC;AAAA,QACvC,QAAQ;AAAA,QAER;AAAA,MACF;AAAA,IACF;AAAA,IACA,YAAY,eAAe,MAAM;AAAA,IAAC;AAAA,IAClC,eAAe,IAAI;AACjB,UAAI;AACF,6BAAqB,EAAE;AAAA,MACzB,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAMA,eAAsB,aACpB,WACA,UACA,SAC6B;AAC7B,QAAM,EAAE,UAAU,WAAW,WAAW,IAAI,WAAW,CAAC;AAExD,QAAM,gBAAyC,CAAC;AAChD,MAAI,SAAU,eAAc,WAAW;AACvC,MAAI,cAAc,OAAW,eAAc,YAAY;AAEvD,QAAM,kBAA0C,aAC5C,CAAC,aAAa;AACZ,QAAI,QAAuC;AAC3C,QAAI,MAAM;AACV,YAAQ,SAAS,OAAO;AAAA,MACtB,KAAK;AACH,gBAAQ;AACR,cAAM;AACN;AAAA,MACF,KAAK;AACH,gBAAQ;AACR,cAAM,KAAK,IAAI,KAAK,MAAM,SAAS,aAAa,GAAG,GAAG,EAAE;AACxD;AAAA,MACF,KAAK;AAAA,MACL,KAAK;AACH,gBAAQ;AACR,cAAM;AACN;AAAA,MACF,KAAK;AACH,gBAAQ;AACR,cAAM;AACN;AAAA,MACF,KAAK;AACH,gBAAQ;AACR,cAAM;AACN;AAAA,IACJ;AACA,eAAW;AAAA,MACT;AAAA,MACA,UAAU;AAAA,MACV,SAAS,SAAS,WAAW;AAAA,MAC7B,WAAW,SAAS;AAAA,MACpB,YAAY,SAAS;AAAA,MACrB,mBAAmB,SAAS;AAAA,IAC9B,CAAC;AAAA,EACH,IACA,MAAM;AAAA,EAAC;AAEX,QAAM,OAAO,sBAAsB,WAAW,eAAe;AAC7D,SAAO,gBAAgB,UAAU,MAAM,aAAa;AACtD;AAIA,SAAS,qBAAqB,WAA4B,YAA4D;AACpH,QAAM,gBAAgB,qBAAqB,SAAS;AACpD,SAAO;AAAA,IACL,aAAa,WAAmB,UAAoB;AAClD,YAAM,SAAS,UAAU,UAAU,SAAS;AAC5C,UAAI,CAAI,cAAW,MAAM,GAAG;AAC1B,cAAM,IAAI,MAAM,+BAA+B,SAAS,EAAE;AAAA,MAC5D;AACA,aAAO,yBAAyB,QAAQ,EAAE,UAAU,YAAY,OAAO,cAAc,CAAC;AAAA,IACxF;AAAA,IACA,YAAY,eAAe,MAAM;AAAA,IAAC;AAAA,IAClC,eAAe,IAAI;AACjB,UAAI;AACF,wCAAgC,EAAE;AAAA,MACpC,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAsBA,mBACpB,WACA,WACA,UACA,SACkC;AAClC,QAAM,EAAE,YAAY,GAAG,WAAW,IAAI,WAAW,CAAC;AAClD,SAAO,kBAAwB,WAAW,UAAU,qBAAqB,WAAW,UAAU,GAAG,UAAU;AAC7G;AAEA,eAAsBC,0BACpB,WACA,WACA,UACA,YACmC;AACnC,SAAO,yBAAyB,WAAW,UAAU,qBAAqB,WAAW,UAAU,CAAC;AAClG;AAEA,eAAsBC,kBACpB,UACA,YACiC;AACjC,SAAO,iBAAuB,UAAU,eAAe,MAAM;AAAA,EAAC,EAAE;AAClE;","names":["incrementalImport","analyzeIncrementalImport","analyzeNewImport"]}