chatlab-cli 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/LICENSE +661 -0
- package/README.md +115 -3
- package/bin/chatlab.mjs +2 -0
- package/dist/chunk-2AIOU3PW.mjs +26 -0
- package/dist/chunk-2AIOU3PW.mjs.map +1 -0
- package/dist/chunk-DL3BFEAH.mjs +144 -0
- package/dist/chunk-DL3BFEAH.mjs.map +1 -0
- package/dist/chunk-JMRDNHIR.mjs +18601 -0
- package/dist/chunk-JMRDNHIR.mjs.map +1 -0
- package/dist/chunk-OC73KSMV.mjs +395 -0
- package/dist/chunk-OC73KSMV.mjs.map +1 -0
- package/dist/chunk-P4N5GEFF.mjs +411 -0
- package/dist/chunk-P4N5GEFF.mjs.map +1 -0
- package/dist/chunk-PF4QFHC6.mjs +3375 -0
- package/dist/chunk-PF4QFHC6.mjs.map +1 -0
- package/dist/chunk-YAOFJ73U.mjs +1393 -0
- package/dist/chunk-YAOFJ73U.mjs.map +1 -0
- package/dist/cli.mjs +15 -0
- package/dist/cli.mjs.map +1 -0
- package/dist/http-PQEOFGJK.mjs +32 -0
- package/dist/http-PQEOFGJK.mjs.map +1 -0
- package/dist/import-UR2IE2O7.mjs +39 -0
- package/dist/import-UR2IE2O7.mjs.map +1 -0
- package/dist/index.mjs +24 -0
- package/dist/index.mjs.map +1 -0
- package/dist/mcp-4XCDZ5PU.mjs +406 -0
- package/dist/mcp-4XCDZ5PU.mjs.map +1 -0
- package/dist/src-ZELOFE7P.mjs +327 -0
- package/dist/src-ZELOFE7P.mjs.map +1 -0
- package/dist/stream-import-RP3XB6NM.mjs +31 -0
- package/dist/stream-import-RP3XB6NM.mjs.map +1 -0
- package/dist-web/assets/DebugToolsPanel-dUV6pVqD.css +1 -0
- package/dist-web/assets/DebugToolsPanel.vue_vue_type_script_setup_true_lang-BuOhHrGp.js +155 -0
- package/dist-web/assets/EChartWordcloud-DcC12j9S.js +1 -0
- package/dist-web/assets/MessageExportModal-CUS9nnwi.js +1 -0
- package/dist-web/assets/electron-8qi0d92j.js +1 -0
- package/dist-web/assets/electron-BHjOx6B6.js +1 -0
- package/dist-web/assets/electron-CTK4NJag.js +1 -0
- package/dist-web/assets/electron-DPsrYwTt.js +1 -0
- package/dist-web/assets/electron-DfpXsXsS.js +1 -0
- package/dist-web/assets/electron-DvcSSc-D.js +1 -0
- package/dist-web/assets/electron-HoVzpIVc.js +1 -0
- package/dist-web/assets/fetch-BS4SlQjA.js +1 -0
- package/dist-web/assets/fetch-CJ8Q4qM2.js +1 -0
- package/dist-web/assets/fetch-CclRWOHf.js +1 -0
- package/dist-web/assets/fetch-DgQkWW73.js +3 -0
- package/dist-web/assets/fetch-DvfIZxuz.js +17 -0
- package/dist-web/assets/fetch-W43zfemj.js +1 -0
- package/dist-web/assets/http-CeXMt-Fv.js +1 -0
- package/dist-web/assets/index-68UnLyFT.js +173 -0
- package/dist-web/assets/index-BKFGuwJu.css +1 -0
- package/dist-web/assets/index-CCE1zef2.js +2 -0
- package/dist-web/assets/index-CLGv8Den.css +1 -0
- package/dist-web/assets/index-CUTe0Sc3.js +16 -0
- package/dist-web/assets/index-CmU6x-uJ.css +1 -0
- package/dist-web/assets/index-CzR1sElx.css +1 -0
- package/dist-web/assets/index-D3siJhmm.js +163 -0
- package/dist-web/assets/index-n2XUUgti.js +7 -0
- package/dist-web/assets/sse-CACcGGWv.js +2 -0
- package/dist-web/assets/vendor-echarts-BXkS9r5W.js +11 -0
- package/dist-web/assets/vendor-echarts-wordcloud-CmDVLUv0.js +1 -0
- package/dist-web/assets/vendor-nuxt-ui-D1aReMtR.js +15 -0
- package/dist-web/assets/vendor-reka-ui-CW2cIgMu.js +11 -0
- package/dist-web/assets/vendor-snapdom-XnBoh1gd.js +8 -0
- package/dist-web/assets/vendor-zrender-B6SpLIkk.js +8 -0
- package/dist-web/assets/web-DOZKbBMQ.js +1 -0
- package/dist-web/index.html +18 -0
- package/package.json +71 -15
- package/cli.js +0 -3
package/README.md
CHANGED
|
@@ -1,5 +1,117 @@
|
|
|
1
|
-
|
|
1
|
+
<div align="center">
|
|
2
|
+
<picture>
|
|
3
|
+
<source media="(prefers-color-scheme: light)" srcset="https://github.com/ChatLab/ChatLab/raw/main/public/images/banner-light.png">
|
|
4
|
+
<img src="https://github.com/ChatLab/ChatLab/raw/main/public/images/banner.png" alt="ChatLab" title="ChatLab" width="500" />
|
|
5
|
+
</picture>
|
|
2
6
|
|
|
3
|
-
|
|
7
|
+
Your chat history, finally yours.
|
|
4
8
|
|
|
5
|
-
|
|
9
|
+
English | [简体中文](./docs/README.zh-CN.md)
|
|
10
|
+
|
|
11
|
+
[Official Website](https://chatlab.fun/) · [Documentation](https://chatlab.fun/usage/) · [Roadmap](https://chatlab.fun/cn/roadmap/tasks)
|
|
12
|
+
|
|
13
|
+
</div>
|
|
14
|
+
|
|
15
|
+
ChatLab is an open-source desktop app for understanding your social conversations. It combines a flexible SQL engine with AI agents so you can explore patterns, ask better questions, and extract insights from chat data, all on your own machine.
|
|
16
|
+
|
|
17
|
+
Currently supported: **WhatsApp, LINE, WeChat, QQ, Discord, Instagram, and Telegram**. Coming next: **iMessage, Messenger, and KakaoTalk**.
|
|
18
|
+
|
|
19
|
+
## Core Features
|
|
20
|
+
|
|
21
|
+
- 🚀 **Built for large histories**: Stream parsing and multi-worker processing keep imports and analysis responsive, even at million-message scale.
|
|
22
|
+
- 🔒 **Private by default**: Your chat data and settings stay local. No mandatory cloud upload of raw conversations.
|
|
23
|
+
- 🤖 **AI that can actually operate on data**: Agent + Function Calling workflows can search, summarize, and analyze chat records with context.
|
|
24
|
+
- 📊 **Insight-rich visual views**: See trends, time patterns, interaction frequency, rankings, and more in one place.
|
|
25
|
+
- 🧩 **Cross-platform normalization**: Different export formats are mapped into a unified model so you can analyze them consistently.
|
|
26
|
+
|
|
27
|
+
## Usage Guides
|
|
28
|
+
|
|
29
|
+
- [Download Guide](https://chatlab.fun/?type=download)
|
|
30
|
+
- [Chat Record Export Guide](https://chatlab.fun/usage/how-to-export)
|
|
31
|
+
- [Standardized Format Specification](https://chatlab.fun/standard/chatlab-format)
|
|
32
|
+
- [Troubleshooting Guide](https://chatlab.fun/usage/troubleshooting)
|
|
33
|
+
|
|
34
|
+
## Preview
|
|
35
|
+
|
|
36
|
+
For more previews, please visit the official website: [chatlab.fun](https://chatlab.fun/)
|
|
37
|
+
|
|
38
|
+

|
|
39
|
+
|
|
40
|
+
## System Architecture
|
|
41
|
+
|
|
42
|
+
### Architecture Principles
|
|
43
|
+
|
|
44
|
+
- **Local-first by default**: Raw chat data, indexes, and settings remain on-device unless you explicitly choose otherwise.
|
|
45
|
+
- **Streaming over buffering**: Stream-first parsing and incremental processing keep large imports stable and memory-efficient.
|
|
46
|
+
- **Composable intelligence**: AI features are assembled through Agent + Tool Calling, not hard-coded into one model path.
|
|
47
|
+
- **Schema-first evolution**: Import, query, analysis, and visualization share a consistent data model that scales with new features.
|
|
48
|
+
|
|
49
|
+
### Runtime Architecture
|
|
50
|
+
|
|
51
|
+
- **Main Process (control plane)**: `apps/desktop/main/index.ts` handles lifecycle and windows. `apps/desktop/main/ipc/` defines domain-scoped IPC. Core AI, query, NLP, import, and merge logic lives in shared packages (`packages/core`, `packages/node-runtime`, `packages/tools`); `apps/desktop/main/ai/` and `apps/desktop/main/i18n/` are thin Electron adaptation layers.
|
|
52
|
+
- **Worker Layer (compute plane)**: `apps/desktop/main/worker/` runs import, indexing, and query tasks via `workerManager`, delegating core logic to `@openchatlab/core` and `@openchatlab/node-runtime`.
|
|
53
|
+
- **Renderer Layer (interaction plane)**: Vue 3 + Nuxt UI + Tailwind CSS drive management, private chat, group chat, and analysis interfaces. `apps/desktop/preload/index.ts` exposes tightly scoped APIs for secure process boundaries.
|
|
54
|
+
|
|
55
|
+
### Data Pipeline
|
|
56
|
+
|
|
57
|
+
1. **Ingestion**: `parser/` detects file format and dispatches to the matching parser module.
|
|
58
|
+
2. **Persistence**: Stream-based writes populate core local entities: sessions, members, and messages.
|
|
59
|
+
3. **Indexing**: Session- and time-oriented indexes are built for timeline navigation and retrieval.
|
|
60
|
+
4. **Query & Analysis**: `worker/query/*` powers activity metrics, interaction analysis, SQL Lab, and AI-assisted exploration.
|
|
61
|
+
5. **Presentation**: The renderer turns query output into charts, rankings, timelines, and conversational analysis flows.
|
|
62
|
+
|
|
63
|
+
### Extensibility & Reliability
|
|
64
|
+
|
|
65
|
+
- **Pluggable parser architecture**: Adding a new import source is mostly an extension in `parser/formats/*`, without reworking downstream query logic.
|
|
66
|
+
- **Full + incremental import paths**: `streamImport.ts` and `incrementalImport.ts` support both first-time onboarding and ongoing updates.
|
|
67
|
+
- **Modular IPC boundaries**: Domain-based IPC segmentation reduces cross-layer coupling and limits permission spread.
|
|
68
|
+
- **Unified i18n evolution**: Main and renderer processes share an i18n system that can evolve with product scope.
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## Local Development
|
|
73
|
+
|
|
74
|
+
### Requirements
|
|
75
|
+
|
|
76
|
+
- Node.js >= 20
|
|
77
|
+
- pnpm
|
|
78
|
+
|
|
79
|
+
### Setup
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
# install dependencies
|
|
83
|
+
pnpm install
|
|
84
|
+
|
|
85
|
+
# run electron app in dev mode
|
|
86
|
+
pnpm dev
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
If Electron encounters exceptions during startup, you can try using `electron-fix`:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
npm install electron-fix -g
|
|
93
|
+
electron-fix start
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Privacy Policy & User Agreement
|
|
98
|
+
|
|
99
|
+
Before using this software, please read the [Privacy Policy & User Agreement](./src/assets/docs/agreement_en.md).
|
|
100
|
+
|
|
101
|
+
## Community
|
|
102
|
+
|
|
103
|
+
Please follow these principles before submitting a Pull Request:
|
|
104
|
+
|
|
105
|
+
- Obvious bug fixes can be submitted directly.
|
|
106
|
+
- For new features, please submit an Issue for discussion first; **PRs submitted without prior discussion will be closed**.
|
|
107
|
+
- Keep one PR focused on one task; if changes are extensive, consider splitting them into multiple independent PRs.
|
|
108
|
+
|
|
109
|
+
Thanks to all contributors:
|
|
110
|
+
|
|
111
|
+
<a href="https://github.com/ChatLab/ChatLab/graphs/contributors">
|
|
112
|
+
<img src="https://contrib.rocks/image?repo=ChatLab/ChatLab" />
|
|
113
|
+
</a>
|
|
114
|
+
|
|
115
|
+
## License
|
|
116
|
+
|
|
117
|
+
AGPL-3.0 License
|
package/bin/chatlab.mjs
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { createRequire as __createRequire } from 'module';
|
|
2
|
+
import { dirname as __pathDirname } from 'path';
|
|
3
|
+
import { fileURLToPath as __fileURLToPath } from 'url';
|
|
4
|
+
const require = __createRequire(import.meta.url);
|
|
5
|
+
const __filename = __fileURLToPath(import.meta.url);
|
|
6
|
+
const __dirname = __pathDirname(__filename);
|
|
7
|
+
|
|
8
|
+
// src/version.ts
|
|
9
|
+
import { readFileSync } from "fs";
|
|
10
|
+
import { resolve } from "path";
|
|
11
|
+
var cached;
|
|
12
|
+
function getVersion() {
|
|
13
|
+
if (cached) return cached;
|
|
14
|
+
try {
|
|
15
|
+
const pkgPath = resolve(__dirname, "../package.json");
|
|
16
|
+
cached = JSON.parse(readFileSync(pkgPath, "utf-8")).version;
|
|
17
|
+
} catch {
|
|
18
|
+
cached = "0.0.0-dev";
|
|
19
|
+
}
|
|
20
|
+
return cached;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export {
|
|
24
|
+
getVersion
|
|
25
|
+
};
|
|
26
|
+
//# sourceMappingURL=chunk-2AIOU3PW.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/version.ts"],"sourcesContent":["import { readFileSync } from 'fs'\nimport { resolve } from 'path'\n\nlet cached: string | undefined\n\n/**\n * Read version from apps/cli/package.json at runtime.\n * Works for both tsx dev (src/) and bundled (dist/).\n */\nexport function getVersion(): string {\n if (cached) return cached\n try {\n const pkgPath = resolve(__dirname, '../package.json')\n cached = JSON.parse(readFileSync(pkgPath, 'utf-8')).version\n } catch {\n cached = '0.0.0-dev'\n }\n return cached!\n}\n"],"mappings":";;;;;;;;AAAA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AAExB,IAAI;AAMG,SAAS,aAAqB;AACnC,MAAI,OAAQ,QAAO;AACnB,MAAI;AACF,UAAM,UAAU,QAAQ,WAAW,iBAAiB;AACpD,aAAS,KAAK,MAAM,aAAa,SAAS,OAAO,CAAC,EAAE;AAAA,EACtD,QAAQ;AACN,aAAS;AAAA,EACX;AACA,SAAO;AACT;","names":[]}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { createRequire as __createRequire } from 'module';
|
|
2
|
+
import { dirname as __pathDirname } from 'path';
|
|
3
|
+
import { fileURLToPath as __fileURLToPath } from 'url';
|
|
4
|
+
const require = __createRequire(import.meta.url);
|
|
5
|
+
const __filename = __fileURLToPath(import.meta.url);
|
|
6
|
+
const __dirname = __pathDirname(__filename);
|
|
7
|
+
import {
|
|
8
|
+
CHAT_DB_TABLES,
|
|
9
|
+
analyzeIncrementalImport,
|
|
10
|
+
analyzeNewImport,
|
|
11
|
+
generateIncrementalSessionIndex,
|
|
12
|
+
generateSessionIndex,
|
|
13
|
+
incrementalImport,
|
|
14
|
+
openBetterSqliteDatabase,
|
|
15
|
+
streamingImport
|
|
16
|
+
} from "./chunk-JMRDNHIR.mjs";
|
|
17
|
+
|
|
18
|
+
// src/import/stream-import.ts
|
|
19
|
+
import * as fs from "fs";
|
|
20
|
+
import * as path from "path";
|
|
21
|
+
import * as crypto from "crypto";
|
|
22
|
+
function generateSessionId() {
|
|
23
|
+
const ts = Date.now();
|
|
24
|
+
const rand = crypto.randomBytes(4).toString("hex");
|
|
25
|
+
return `chat_${ts}_${rand}`;
|
|
26
|
+
}
|
|
27
|
+
function resolveNativeBinding(dbManager) {
|
|
28
|
+
return dbManager.nativeBinding;
|
|
29
|
+
}
|
|
30
|
+
function buildStreamImportDeps(dbManager, onProgress) {
|
|
31
|
+
const nativeBinding = resolveNativeBinding(dbManager);
|
|
32
|
+
return {
|
|
33
|
+
openDatabase(sessionId) {
|
|
34
|
+
const dbPath = dbManager.getDbPath(sessionId);
|
|
35
|
+
const dir = path.dirname(dbPath);
|
|
36
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
37
|
+
const db = openBetterSqliteDatabase(dbPath, { nativeBinding });
|
|
38
|
+
db.exec(CHAT_DB_TABLES);
|
|
39
|
+
return db;
|
|
40
|
+
},
|
|
41
|
+
deleteDatabase(sessionId) {
|
|
42
|
+
const dbPath = dbManager.getDbPath(sessionId);
|
|
43
|
+
for (const suffix of ["", "-wal", "-shm"]) {
|
|
44
|
+
try {
|
|
45
|
+
const p = dbPath + suffix;
|
|
46
|
+
if (fs.existsSync(p)) fs.unlinkSync(p);
|
|
47
|
+
} catch {
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
onProgress: onProgress ?? (() => {
|
|
52
|
+
}),
|
|
53
|
+
postImportHook(db) {
|
|
54
|
+
try {
|
|
55
|
+
generateSessionIndex(db);
|
|
56
|
+
} catch {
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
generateSessionId
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
async function streamImport(dbManager, filePath, options) {
|
|
63
|
+
const { formatId, chatIndex, onProgress } = options || {};
|
|
64
|
+
const formatOptions = {};
|
|
65
|
+
if (formatId) formatOptions.formatId = formatId;
|
|
66
|
+
if (chatIndex !== void 0) formatOptions.chatIndex = chatIndex;
|
|
67
|
+
const progressAdapter = onProgress ? (progress) => {
|
|
68
|
+
let stage = "parsing";
|
|
69
|
+
let pct = 0;
|
|
70
|
+
switch (progress.stage) {
|
|
71
|
+
case "detecting":
|
|
72
|
+
stage = "detecting";
|
|
73
|
+
pct = 5;
|
|
74
|
+
break;
|
|
75
|
+
case "parsing":
|
|
76
|
+
stage = "parsing";
|
|
77
|
+
pct = Math.min(Math.round(progress.percentage * 0.7), 70);
|
|
78
|
+
break;
|
|
79
|
+
case "importing":
|
|
80
|
+
case "saving":
|
|
81
|
+
stage = "saving";
|
|
82
|
+
pct = 80;
|
|
83
|
+
break;
|
|
84
|
+
case "done":
|
|
85
|
+
stage = "done";
|
|
86
|
+
pct = 100;
|
|
87
|
+
break;
|
|
88
|
+
case "error":
|
|
89
|
+
stage = "error";
|
|
90
|
+
pct = 0;
|
|
91
|
+
break;
|
|
92
|
+
}
|
|
93
|
+
onProgress({
|
|
94
|
+
stage,
|
|
95
|
+
progress: pct,
|
|
96
|
+
message: progress.message || "",
|
|
97
|
+
bytesRead: progress.bytesRead,
|
|
98
|
+
totalBytes: progress.totalBytes,
|
|
99
|
+
messagesProcessed: progress.messagesProcessed
|
|
100
|
+
});
|
|
101
|
+
} : () => {
|
|
102
|
+
};
|
|
103
|
+
const deps = buildStreamImportDeps(dbManager, progressAdapter);
|
|
104
|
+
return streamingImport(filePath, deps, formatOptions);
|
|
105
|
+
}
|
|
106
|
+
function buildIncrementalDeps(dbManager, onProgress) {
|
|
107
|
+
const nativeBinding = resolveNativeBinding(dbManager);
|
|
108
|
+
return {
|
|
109
|
+
openDatabase(sessionId, readonly) {
|
|
110
|
+
const dbPath = dbManager.getDbPath(sessionId);
|
|
111
|
+
if (!fs.existsSync(dbPath)) {
|
|
112
|
+
throw new Error(`Session database not found: ${sessionId}`);
|
|
113
|
+
}
|
|
114
|
+
return openBetterSqliteDatabase(dbPath, { readonly: readonly ?? false, nativeBinding });
|
|
115
|
+
},
|
|
116
|
+
onProgress: onProgress ?? (() => {
|
|
117
|
+
}),
|
|
118
|
+
postImportHook(db) {
|
|
119
|
+
try {
|
|
120
|
+
generateIncrementalSessionIndex(db);
|
|
121
|
+
} catch {
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
async function incrementalImport2(dbManager, sessionId, filePath, options) {
|
|
127
|
+
const { onProgress, ...importOpts } = options || {};
|
|
128
|
+
return incrementalImport(sessionId, filePath, buildIncrementalDeps(dbManager, onProgress), importOpts);
|
|
129
|
+
}
|
|
130
|
+
async function analyzeIncrementalImport2(dbManager, sessionId, filePath, onProgress) {
|
|
131
|
+
return analyzeIncrementalImport(sessionId, filePath, buildIncrementalDeps(dbManager, onProgress));
|
|
132
|
+
}
|
|
133
|
+
async function analyzeNewImport2(filePath, onProgress) {
|
|
134
|
+
return analyzeNewImport(filePath, onProgress ?? (() => {
|
|
135
|
+
}));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export {
|
|
139
|
+
streamImport,
|
|
140
|
+
incrementalImport2 as incrementalImport,
|
|
141
|
+
analyzeIncrementalImport2 as analyzeIncrementalImport,
|
|
142
|
+
analyzeNewImport2 as analyzeNewImport
|
|
143
|
+
};
|
|
144
|
+
//# sourceMappingURL=chunk-DL3BFEAH.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/import/stream-import.ts"],"sourcesContent":["/**\n * Server/CLI streaming import — adapter for @openchatlab/node-runtime StreamingImporter.\n *\n * Replaces the old buffered-in-memory approach with the same high-performance\n * streaming pipeline used by Electron (batched transactions, deferred indexes,\n * nickname history, FTS, format fallback).\n */\n\nimport * as fs from 'fs'\nimport * as path from 'path'\nimport type { DatabaseManager } from '@openchatlab/node-runtime'\nimport {\n streamingImport,\n openBetterSqliteDatabase,\n analyzeNewImport as sharedAnalyzeNewImport,\n analyzeIncrementalImport as sharedAnalyzeIncremental,\n incrementalImport as sharedIncrementalImport,\n} from '@openchatlab/node-runtime'\nimport { generateSessionIndex, generateIncrementalSessionIndex } from '@openchatlab/core'\nimport type {\n StreamImportResult,\n StreamImportDeps,\n ImportProgressCallback,\n IncrementalImportResult,\n IncrementalAnalyzeResult,\n IncrementalImportDeps,\n ImportOptions,\n AnalyzeNewImportResult,\n} from '@openchatlab/node-runtime'\nimport { CHAT_DB_TABLES } from '@openchatlab/core'\nimport {\n detectFormat as parserDetectFormat,\n detectAllFormats,\n getFormatFeatureById,\n getSupportedFormats as parserGetSupportedFormats,\n scanMultiChatFile as parserScanMultiChatFile,\n type FormatFeature,\n type MultiChatInfo,\n type ParseProgress,\n} from '@openchatlab/parser'\nimport * as crypto from 'crypto'\n\n// ==================== Legacy progress interface (for SSE routes) ====================\n\nexport interface StreamImportProgress {\n stage: 'detecting' | 'parsing' | 'saving' | 'indexing' | 'done' | 'error'\n progress: number\n message: string\n bytesRead?: number\n totalBytes?: number\n messagesProcessed?: number\n}\n\nexport interface StreamImportOptions {\n formatId?: string\n chatIndex?: number\n nativeBinding?: string\n onProgress?: (progress: StreamImportProgress) => void\n}\n\nfunction generateSessionId(): string {\n const ts = Date.now()\n const rand = crypto.randomBytes(4).toString('hex')\n return `chat_${ts}_${rand}`\n}\n\nfunction resolveNativeBinding(dbManager: DatabaseManager): string | undefined {\n return (dbManager as any).nativeBinding\n}\n\nfunction buildStreamImportDeps(dbManager: DatabaseManager, onProgress?: ImportProgressCallback): StreamImportDeps {\n const nativeBinding = resolveNativeBinding(dbManager)\n return {\n openDatabase(sessionId: string) {\n const dbPath = dbManager.getDbPath(sessionId)\n const dir = path.dirname(dbPath)\n if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true })\n const db = openBetterSqliteDatabase(dbPath, { nativeBinding })\n db.exec(CHAT_DB_TABLES)\n return db\n },\n deleteDatabase(sessionId: string) {\n const dbPath = dbManager.getDbPath(sessionId)\n for (const suffix of ['', '-wal', '-shm']) {\n try {\n const p = dbPath + suffix\n if (fs.existsSync(p)) fs.unlinkSync(p)\n } catch {\n /* ignore */\n }\n }\n },\n onProgress: onProgress ?? (() => {}),\n postImportHook(db) {\n try {\n generateSessionIndex(db)\n } catch {\n /* non-fatal: frontend can regenerate via generate-index route */\n }\n },\n generateSessionId,\n }\n}\n\n/**\n * High-performance streaming import: parse a file and write to DB\n * with batched transactions, deferred indexes, and FTS.\n */\nexport async function streamImport(\n dbManager: DatabaseManager,\n filePath: string,\n options?: StreamImportOptions\n): Promise<StreamImportResult> {\n const { formatId, chatIndex, onProgress } = options || {}\n\n const formatOptions: Record<string, unknown> = {}\n if (formatId) formatOptions.formatId = formatId\n if (chatIndex !== undefined) formatOptions.chatIndex = chatIndex\n\n const progressAdapter: ImportProgressCallback = onProgress\n ? (progress) => {\n let stage: StreamImportProgress['stage'] = 'parsing'\n let pct = 0\n switch (progress.stage) {\n case 'detecting':\n stage = 'detecting'\n pct = 5\n break\n case 'parsing':\n stage = 'parsing'\n pct = Math.min(Math.round(progress.percentage * 0.7), 70)\n break\n case 'importing':\n case 'saving':\n stage = 'saving'\n pct = 80\n break\n case 'done':\n stage = 'done'\n pct = 100\n break\n case 'error':\n stage = 'error'\n pct = 0\n break\n }\n onProgress({\n stage,\n progress: pct,\n message: progress.message || '',\n bytesRead: progress.bytesRead,\n totalBytes: progress.totalBytes,\n messagesProcessed: progress.messagesProcessed,\n })\n }\n : () => {}\n\n const deps = buildStreamImportDeps(dbManager, progressAdapter)\n return streamingImport(filePath, deps, formatOptions)\n}\n\n// ==================== Incremental import ====================\n\nfunction buildIncrementalDeps(dbManager: DatabaseManager, onProgress?: ImportProgressCallback): IncrementalImportDeps {\n const nativeBinding = resolveNativeBinding(dbManager)\n return {\n openDatabase(sessionId: string, readonly?: boolean) {\n const dbPath = dbManager.getDbPath(sessionId)\n if (!fs.existsSync(dbPath)) {\n throw new Error(`Session database not found: ${sessionId}`)\n }\n return openBetterSqliteDatabase(dbPath, { readonly: readonly ?? false, nativeBinding })\n },\n onProgress: onProgress ?? (() => {}),\n postImportHook(db) {\n try {\n generateIncrementalSessionIndex(db)\n } catch {\n /* non-fatal */\n }\n },\n }\n}\n\nexport async function incrementalImport(\n dbManager: DatabaseManager,\n sessionId: string,\n filePath: string,\n options?: ImportOptions & { onProgress?: ImportProgressCallback }\n): Promise<IncrementalImportResult> {\n const { onProgress, ...importOpts } = options || {}\n return sharedIncrementalImport(sessionId, filePath, buildIncrementalDeps(dbManager, onProgress), importOpts)\n}\n\nexport async function analyzeIncrementalImport(\n dbManager: DatabaseManager,\n sessionId: string,\n filePath: string,\n onProgress?: ImportProgressCallback\n): Promise<IncrementalAnalyzeResult> {\n return sharedAnalyzeIncremental(sessionId, filePath, buildIncrementalDeps(dbManager, onProgress))\n}\n\nexport async function analyzeNewImport(\n filePath: string,\n onProgress?: ImportProgressCallback\n): Promise<AnalyzeNewImportResult> {\n return sharedAnalyzeNewImport(filePath, onProgress ?? (() => {}))\n}\n\n// ==================== Re-exports from parser ====================\n\nexport {\n parserDetectFormat as detectFormat,\n detectAllFormats,\n getFormatFeatureById,\n parserGetSupportedFormats as getSupportedFormats,\n parserScanMultiChatFile as scanMultiChatFile,\n}\nexport type { FormatFeature, MultiChatInfo, ParseProgress }\nexport type {\n StreamImportResult,\n IncrementalImportResult,\n IncrementalAnalyzeResult,\n AnalyzeNewImportResult,\n ImportOptions,\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAQA,YAAY,QAAQ;AACpB,YAAY,UAAU;AA+BtB,YAAY,YAAY;AAoBxB,SAAS,oBAA4B;AACnC,QAAM,KAAK,KAAK,IAAI;AACpB,QAAM,OAAc,mBAAY,CAAC,EAAE,SAAS,KAAK;AACjD,SAAO,QAAQ,EAAE,IAAI,IAAI;AAC3B;AAEA,SAAS,qBAAqB,WAAgD;AAC5E,SAAQ,UAAkB;AAC5B;AAEA,SAAS,sBAAsB,WAA4B,YAAuD;AAChH,QAAM,gBAAgB,qBAAqB,SAAS;AACpD,SAAO;AAAA,IACL,aAAa,WAAmB;AAC9B,YAAM,SAAS,UAAU,UAAU,SAAS;AAC5C,YAAM,MAAW,aAAQ,MAAM;AAC/B,UAAI,CAAI,cAAW,GAAG,EAAG,CAAG,aAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAC9D,YAAM,KAAK,yBAAyB,QAAQ,EAAE,cAAc,CAAC;AAC7D,SAAG,KAAK,cAAc;AACtB,aAAO;AAAA,IACT;AAAA,IACA,eAAe,WAAmB;AAChC,YAAM,SAAS,UAAU,UAAU,SAAS;AAC5C,iBAAW,UAAU,CAAC,IAAI,QAAQ,MAAM,GAAG;AACzC,YAAI;AACF,gBAAM,IAAI,SAAS;AACnB,cAAO,cAAW,CAAC,EAAG,CAAG,cAAW,CAAC;AAAA,QACvC,QAAQ;AAAA,QAER;AAAA,MACF;AAAA,IACF;AAAA,IACA,YAAY,eAAe,MAAM;AAAA,IAAC;AAAA,IAClC,eAAe,IAAI;AACjB,UAAI;AACF,6BAAqB,EAAE;AAAA,MACzB,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAMA,eAAsB,aACpB,WACA,UACA,SAC6B;AAC7B,QAAM,EAAE,UAAU,WAAW,WAAW,IAAI,WAAW,CAAC;AAExD,QAAM,gBAAyC,CAAC;AAChD,MAAI,SAAU,eAAc,WAAW;AACvC,MAAI,cAAc,OAAW,eAAc,YAAY;AAEvD,QAAM,kBAA0C,aAC5C,CAAC,aAAa;AACZ,QAAI,QAAuC;AAC3C,QAAI,MAAM;AACV,YAAQ,SAAS,OAAO;AAAA,MACtB,KAAK;AACH,gBAAQ;AACR,cAAM;AACN;AAAA,MACF,KAAK;AACH,gBAAQ;AACR,cAAM,KAAK,IAAI,KAAK,MAAM,SAAS,aAAa,GAAG,GAAG,EAAE;AACxD;AAAA,MACF,KAAK;AAAA,MACL,KAAK;AACH,gBAAQ;AACR,cAAM;AACN;AAAA,MACF,KAAK;AACH,gBAAQ;AACR,cAAM;AACN;AAAA,MACF,KAAK;AACH,gBAAQ;AACR,cAAM;AACN;AAAA,IACJ;AACA,eAAW;AAAA,MACT;AAAA,MACA,UAAU;AAAA,MACV,SAAS,SAAS,WAAW;AAAA,MAC7B,WAAW,SAAS;AAAA,MACpB,YAAY,SAAS;AAAA,MACrB,mBAAmB,SAAS;AAAA,IAC9B,CAAC;AAAA,EACH,IACA,MAAM;AAAA,EAAC;AAEX,QAAM,OAAO,sBAAsB,WAAW,eAAe;AAC7D,SAAO,gBAAgB,UAAU,MAAM,aAAa;AACtD;AAIA,SAAS,qBAAqB,WAA4B,YAA4D;AACpH,QAAM,gBAAgB,qBAAqB,SAAS;AACpD,SAAO;AAAA,IACL,aAAa,WAAmB,UAAoB;AAClD,YAAM,SAAS,UAAU,UAAU,SAAS;AAC5C,UAAI,CAAI,cAAW,MAAM,GAAG;AAC1B,cAAM,IAAI,MAAM,+BAA+B,SAAS,EAAE;AAAA,MAC5D;AACA,aAAO,yBAAyB,QAAQ,EAAE,UAAU,YAAY,OAAO,cAAc,CAAC;AAAA,IACxF;AAAA,IACA,YAAY,eAAe,MAAM;AAAA,IAAC;AAAA,IAClC,eAAe,IAAI;AACjB,UAAI;AACF,wCAAgC,EAAE;AAAA,MACpC,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAsBA,mBACpB,WACA,WACA,UACA,SACkC;AAClC,QAAM,EAAE,YAAY,GAAG,WAAW,IAAI,WAAW,CAAC;AAClD,SAAO,kBAAwB,WAAW,UAAU,qBAAqB,WAAW,UAAU,GAAG,UAAU;AAC7G;AAEA,eAAsBC,0BACpB,WACA,WACA,UACA,YACmC;AACnC,SAAO,yBAAyB,WAAW,UAAU,qBAAqB,WAAW,UAAU,CAAC;AAClG;AAEA,eAAsBC,kBACpB,UACA,YACiC;AACjC,SAAO,iBAAuB,UAAU,eAAe,MAAM;AAAA,EAAC,EAAE;AAClE;","names":["incrementalImport","analyzeIncrementalImport","analyzeNewImport"]}
|