chatlab-cli 0.34.1 → 0.34.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chat-command-OWKJJ736.mjs → chat-command-6IOH2QRF.mjs} +9 -9
- package/dist/{chunk-ZM5DSHFF.mjs → chunk-2XCNTG4O.mjs} +313 -76
- package/dist/chunk-2XCNTG4O.mjs.map +1 -0
- package/dist/{chunk-RRC6U56Y.mjs → chunk-4LW6443F.mjs} +1570 -687
- package/dist/chunk-4LW6443F.mjs.map +1 -0
- package/dist/{chunk-OHFVX2HH.mjs → chunk-4RLZFGO5.mjs} +2 -2
- package/dist/{chunk-GHZFNAMF.mjs → chunk-52P5ZRBD.mjs} +2 -2
- package/dist/{chunk-XEGEOU2D.mjs → chunk-652KEAX3.mjs} +2 -2
- package/dist/{chunk-J7HAIT3T.mjs → chunk-72RWABYJ.mjs} +2 -2
- package/dist/{chunk-MPHFZYB7.mjs → chunk-H6D4DZOC.mjs} +4 -4
- package/dist/{chunk-3RWYALUP.mjs → chunk-INBJ7SND.mjs} +50 -25
- package/dist/chunk-INBJ7SND.mjs.map +1 -0
- package/dist/{chunk-VTCZM2JS.mjs → chunk-JX6MFS6F.mjs} +43 -23
- package/dist/chunk-JX6MFS6F.mjs.map +1 -0
- package/dist/{chunk-RMW3524R.mjs → chunk-MENKFEPC.mjs} +149 -44
- package/dist/chunk-MENKFEPC.mjs.map +1 -0
- package/dist/{chunk-YRCFDZ5Q.mjs → chunk-U6MVJSXX.mjs} +170 -66
- package/dist/chunk-U6MVJSXX.mjs.map +1 -0
- package/dist/chunk-UMBEHBIZ.mjs +41 -0
- package/dist/chunk-UMBEHBIZ.mjs.map +1 -0
- package/dist/{chunk-GFHSSCCR.mjs → chunk-VSOWNKPA.mjs} +1 -11
- package/dist/cli.mjs +10 -9
- package/dist/contacts-worker.mjs +3 -3
- package/dist/global-insight-worker.mjs +2 -2
- package/dist/{http-X55S6MTO.mjs → http-HSK46XEU.mjs} +13 -12
- package/dist/{import-4AX6IX57.mjs → import-DF2WD2HE.mjs} +10 -8
- package/dist/index.mjs +13 -12
- package/dist/{mcp-GWC52AWE.mjs → mcp-A2RND7KN.mjs} +12 -12
- package/dist/people-relationships-worker.mjs +3 -3
- package/dist/semantic-index-worker.mjs +1 -1
- package/dist/{service-KGL3RAIA.mjs → service-KZTHBDQ5.mjs} +2 -2
- package/dist/{src-DLLM4SEO.mjs → src-SHFHW2QB.mjs} +27 -7
- package/dist/{stream-import-K24HWAAZ.mjs → stream-import-GZ2QDE3Z.mjs} +10 -8
- package/dist/{undici-ZSXBT7EA.mjs → undici-MU5I3L6Q.mjs} +2 -2
- package/dist/{update-checker-LGMTEKMT.mjs → update-checker-3N7HQEP2.mjs} +2 -2
- package/dist/{worker-runtime-6KKR55TD.mjs → worker-runtime-OWN56GG5.mjs} +14 -8
- package/dist/worker-runtime-OWN56GG5.mjs.map +1 -0
- package/dist-cli-web/assets/{ActionToolsPanel.vue_vue_type_script_setup_true_lang-D1BiZuQL.js → ActionToolsPanel.vue_vue_type_script_setup_true_lang-B06LYYS9.js} +14 -14
- package/dist-cli-web/assets/{BufferResource-IPkSmPao.js → BufferResource-BdKXeRZS.js} +1 -1
- package/dist-cli-web/assets/{CanvasRenderer-OAQy65em.js → CanvasRenderer-DDRdIyZ8.js} +1 -1
- package/dist-cli-web/assets/{DataDirCleanupNotice-jEI-YjFp.js → DataDirCleanupNotice-CP8P1lsL.js} +1 -1
- package/dist-cli-web/assets/{EChartWordcloud-Pi48JdN2.js → EChartWordcloud-CqknPGui.js} +1 -1
- package/dist-cli-web/assets/{Filter-FQD30cRE.js → Filter-B2eEAPxI.js} +1 -1
- package/dist-cli-web/assets/{LoadingState.vue_vue_type_script_setup_true_lang-CCK5yTju.js → LoadingState.vue_vue_type_script_setup_true_lang-BFaXTz1c.js} +1 -1
- package/dist-cli-web/assets/{MessageExportModal-CZn3LbeY.js → MessageExportModal-CZge6qub.js} +1 -1
- package/dist-cli-web/assets/{PageHeader-B9B9K9by.js → PageHeader-CxKbcLDc.js} +1 -1
- package/dist-cli-web/assets/{RelationshipGalaxyCanvas-CbTmpnZ6.js → RelationshipGalaxyCanvas-CwQ63TPi.js} +7 -7
- package/dist-cli-web/assets/RelationshipGalaxyThreeCanvas-BLqISQe5.js +4108 -0
- package/dist-cli-web/assets/{RelationshipGalaxyThreeCanvas-CeYJABxI.css → RelationshipGalaxyThreeCanvas-CMuibfiA.css} +1 -1
- package/dist-cli-web/assets/{RenderTargetSystem-DnptJCQ0.js → RenderTargetSystem-eC_VrDAr.js} +1 -1
- package/dist-cli-web/assets/{ThemeCard.vue_vue_type_script_setup_true_lang-JXMFY6aD.js → ThemeCard.vue_vue_type_script_setup_true_lang-DVmpDz05.js} +1 -1
- package/dist-cli-web/assets/{TimeSelect.vue_vue_type_script_setup_true_lang-QAZFwx5S.js → TimeSelect.vue_vue_type_script_setup_true_lang-CCH1UQGc.js} +1 -1
- package/dist-cli-web/assets/{WebGLRenderer-BmjIF5se.js → WebGLRenderer-CRp_HC84.js} +1 -1
- package/dist-cli-web/assets/{WebGPURenderer-DkNcKkqj.js → WebGPURenderer-WIUnEhpi.js} +1 -1
- package/dist-cli-web/assets/{annual-summary-time-range-DJ0HWI_t.js → annual-summary-time-range-R3pLPEuN.js} +1 -1
- package/dist-cli-web/assets/{base-CnuE0ss0.js → base-CLwWn3Rs.js} +1 -1
- package/dist-cli-web/assets/{browserAll-C_NnyNBz.js → browserAll-B29Ox2in.js} +1 -1
- package/dist-cli-web/assets/cli-web-CVLb5Pkq.js +1 -0
- package/dist-cli-web/assets/{dateFormat-p6nibkgt.js → dateFormat-T7kMS5IP.js} +1 -1
- package/dist-cli-web/assets/{fetch-C4KJ7tGz.js → fetch-BBgZQob_.js} +1 -1
- package/dist-cli-web/assets/{fetch-BfYranz7.js → fetch-BKBlpBl4.js} +1 -1
- package/dist-cli-web/assets/{fetch-Ch4IMKin.js → fetch-BLi6Jsjg.js} +1 -1
- package/dist-cli-web/assets/{fetch-WElo3Vix.js → fetch-CYaGyohV.js} +1 -1
- package/dist-cli-web/assets/{fetch-CMhV3TYQ.js → fetch-Cm2WR2Cv.js} +1 -1
- package/dist-cli-web/assets/{fetch-DajjPQtq.js → fetch-D8hJZa8o.js} +1 -1
- package/dist-cli-web/assets/fetch-D9sn99zX.js +4 -0
- package/dist-cli-web/assets/{fetch-WIOgRPd9.js → fetch-DsHf-61a.js} +1 -1
- package/dist-cli-web/assets/{fetch-Bsm5YQ_7.js → fetch-DvrVGAzR.js} +1 -1
- package/dist-cli-web/assets/{fetch-BBUvWvOM.js → fetch-L6PKD8-N.js} +1 -1
- package/dist-cli-web/assets/{index-dObiAHg6.js → index-0qcENyvH.js} +1 -1
- package/dist-cli-web/assets/{index-DLnlRLCx.js → index-AkF_KqVI.js} +1 -1
- package/dist-cli-web/assets/index-BVOFyB9E.js +137 -0
- package/dist-cli-web/assets/{index-CjoTcSEc.js → index-BkdOH41D.js} +1 -1
- package/dist-cli-web/assets/index-C4TuoOLM.js +2 -0
- package/dist-cli-web/assets/{index-BcWaPFz7.js → index-C6Mk-Xqt.js} +1 -1
- package/dist-cli-web/assets/{index-Bb6EtLMB.js → index-CkgUsaeX.js} +1 -1
- package/dist-cli-web/assets/{index-CKkIHbie.js → index-CuAy1Xmt.js} +3 -3
- package/dist-cli-web/assets/index-CxGtGxLW.css +1 -0
- package/dist-cli-web/assets/index-D_0qeKKc.js +53 -0
- package/dist-cli-web/assets/{index-CsyURmWU.js → index-DeR0Vukh.js} +1 -1
- package/dist-cli-web/assets/{index-CbacXZN3.js → index-Dx-XvfIK.js} +2 -2
- package/dist-cli-web/assets/{index-BE4nd1aX.js → index-_DrviTnP.js} +1 -1
- package/dist-cli-web/assets/{index-x74lBtei.js → index-xN0sjOLK.js} +1 -1
- package/dist-cli-web/assets/{people-page-header-BW5cNPl7.js → people-page-header-CCNJOrmK.js} +1 -1
- package/dist-cli-web/assets/telemetry-DPC69b73.js +1 -0
- package/dist-cli-web/assets/{webworkerAll-8AFsQkH1.js → webworkerAll-B2qvP2qR.js} +1 -1
- package/dist-cli-web/index.html +1 -1
- package/package.json +3 -2
- package/dist/chunk-3RWYALUP.mjs.map +0 -1
- package/dist/chunk-RMW3524R.mjs.map +0 -1
- package/dist/chunk-RRC6U56Y.mjs.map +0 -1
- package/dist/chunk-VTCZM2JS.mjs.map +0 -1
- package/dist/chunk-YRCFDZ5Q.mjs.map +0 -1
- package/dist/chunk-ZM5DSHFF.mjs.map +0 -1
- package/dist/onnxruntime_binding-6Q6HXASN.node +0 -0
- package/dist/onnxruntime_binding-EKZT2NRK.node +0 -0
- package/dist/onnxruntime_binding-P6S7V3CI.node +0 -0
- package/dist/onnxruntime_binding-PJNNIIUO.node +0 -0
- package/dist/onnxruntime_binding-UN6SPTQK.node +0 -0
- package/dist/transformers.node-PUMHAODN.mjs +0 -42889
- package/dist/transformers.node-PUMHAODN.mjs.map +0 -1
- package/dist/worker-runtime-6KKR55TD.mjs.map +0 -1
- package/dist-cli-web/assets/RelationshipGalaxyThreeCanvas-Cu6NmKOq.js +0 -4108
- package/dist-cli-web/assets/cli-web-t1gabACJ.js +0 -1
- package/dist-cli-web/assets/fetch-_Hb-KRSr.js +0 -3
- package/dist-cli-web/assets/index-B8I5Pax_.js +0 -2
- package/dist-cli-web/assets/index-CsX1Wt8l.js +0 -173
- package/dist-cli-web/assets/index-DmrrdYXH.css +0 -1
- package/dist-cli-web/assets/index-YzfjPohq.js +0 -53
- /package/dist/{chat-command-OWKJJ736.mjs.map → chat-command-6IOH2QRF.mjs.map} +0 -0
- /package/dist/{chunk-OHFVX2HH.mjs.map → chunk-4RLZFGO5.mjs.map} +0 -0
- /package/dist/{chunk-GHZFNAMF.mjs.map → chunk-52P5ZRBD.mjs.map} +0 -0
- /package/dist/{chunk-XEGEOU2D.mjs.map → chunk-652KEAX3.mjs.map} +0 -0
- /package/dist/{chunk-J7HAIT3T.mjs.map → chunk-72RWABYJ.mjs.map} +0 -0
- /package/dist/{chunk-MPHFZYB7.mjs.map → chunk-H6D4DZOC.mjs.map} +0 -0
- /package/dist/{chunk-GFHSSCCR.mjs.map → chunk-VSOWNKPA.mjs.map} +0 -0
- /package/dist/{http-X55S6MTO.mjs.map → http-HSK46XEU.mjs.map} +0 -0
- /package/dist/{import-4AX6IX57.mjs.map → import-DF2WD2HE.mjs.map} +0 -0
- /package/dist/{mcp-GWC52AWE.mjs.map → mcp-A2RND7KN.mjs.map} +0 -0
- /package/dist/{service-KGL3RAIA.mjs.map → service-KZTHBDQ5.mjs.map} +0 -0
- /package/dist/{src-DLLM4SEO.mjs.map → src-SHFHW2QB.mjs.map} +0 -0
- /package/dist/{stream-import-K24HWAAZ.mjs.map → stream-import-GZ2QDE3Z.mjs.map} +0 -0
- /package/dist/{undici-ZSXBT7EA.mjs.map → undici-MU5I3L6Q.mjs.map} +0 -0
- /package/dist/{update-checker-LGMTEKMT.mjs.map → update-checker-3N7HQEP2.mjs.map} +0 -0
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
createEmptyContactsDiagnostics,
|
|
10
10
|
normalizeContactsTimeRangePreset,
|
|
11
11
|
resolveContactsTimeRange
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-52P5ZRBD.mjs";
|
|
13
13
|
import {
|
|
14
14
|
PEOPLE_RELATIONSHIPS_ALGORITHM_VERSION,
|
|
15
15
|
buildPeopleRelationshipsNeighborhoodGraph,
|
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
computePeopleRelationshipsSnapshot,
|
|
19
19
|
normalizePeopleRelationshipsTimeRangePreset,
|
|
20
20
|
resolvePeopleRelationshipsTimeRange
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-652KEAX3.mjs";
|
|
22
22
|
import {
|
|
23
23
|
ANNUAL_SUMMARY_ALGORITHM_VERSION
|
|
24
24
|
} from "./chunk-QZCNVAH6.mjs";
|
|
@@ -90,6 +90,7 @@ import {
|
|
|
90
90
|
needsStreamingThinkParsing,
|
|
91
91
|
normalizeAnnualSummaryRange,
|
|
92
92
|
normalizeBuiltinToolNames,
|
|
93
|
+
normalizeSessionGapThreshold,
|
|
93
94
|
openBetterSqliteDatabase,
|
|
94
95
|
readAnnualSummarySnapshot,
|
|
95
96
|
readJsonFile,
|
|
@@ -110,13 +111,13 @@ import {
|
|
|
110
111
|
writeAuthProfile,
|
|
111
112
|
writeJsonFileAtomically,
|
|
112
113
|
writeParseResultToDb
|
|
113
|
-
} from "./chunk-
|
|
114
|
+
} from "./chunk-INBJ7SND.mjs";
|
|
114
115
|
import {
|
|
115
116
|
__commonJS,
|
|
116
117
|
__export,
|
|
117
118
|
__require,
|
|
118
119
|
__toESM
|
|
119
|
-
} from "./chunk-
|
|
120
|
+
} from "./chunk-VSOWNKPA.mjs";
|
|
120
121
|
|
|
121
122
|
// ../../node_modules/stream-json/utils/Utf8Stream.js
|
|
122
123
|
var require_Utf8Stream = __commonJS({
|
|
@@ -726,8 +727,8 @@ var require_FilterBase = __commonJS({
|
|
|
726
727
|
var FilterBase = class _FilterBase extends Transform {
|
|
727
728
|
static stringFilter(string, separator) {
|
|
728
729
|
return (stack) => {
|
|
729
|
-
const
|
|
730
|
-
return
|
|
730
|
+
const path31 = stack.join(separator);
|
|
731
|
+
return path31.length === string.length && path31 === string || path31.length > string.length && path31.substr(0, string.length) === string && path31.substr(string.length, separator.length) === separator;
|
|
731
732
|
};
|
|
732
733
|
}
|
|
733
734
|
static regExpFilter(regExp, separator) {
|
|
@@ -1181,12 +1182,12 @@ var require_Assembler = __commonJS({
|
|
|
1181
1182
|
return (this.stack.length >> 1) + (this.done ? 0 : 1);
|
|
1182
1183
|
}
|
|
1183
1184
|
get path() {
|
|
1184
|
-
const
|
|
1185
|
+
const path31 = [];
|
|
1185
1186
|
for (let i = 0; i < this.stack.length; i += 2) {
|
|
1186
1187
|
const key = this.stack[i + 1];
|
|
1187
|
-
|
|
1188
|
+
path31.push(key === null ? this.stack[i].length : key);
|
|
1188
1189
|
}
|
|
1189
|
-
return
|
|
1190
|
+
return path31;
|
|
1190
1191
|
}
|
|
1191
1192
|
dropToLevel(level) {
|
|
1192
1193
|
if (level < this.depth) {
|
|
@@ -1468,7 +1469,7 @@ var require_pend = __commonJS({
|
|
|
1468
1469
|
var require_fd_slicer = __commonJS({
|
|
1469
1470
|
"../../packages/node-runtime/node_modules/yauzl/fd-slicer.js"(exports) {
|
|
1470
1471
|
"use strict";
|
|
1471
|
-
var
|
|
1472
|
+
var fs37 = __require("fs");
|
|
1472
1473
|
var util = __require("util");
|
|
1473
1474
|
var stream = __require("stream");
|
|
1474
1475
|
var Readable = stream.Readable;
|
|
@@ -1488,7 +1489,7 @@ var require_fd_slicer = __commonJS({
|
|
|
1488
1489
|
FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
|
|
1489
1490
|
var self = this;
|
|
1490
1491
|
self.pend.go(function(cb) {
|
|
1491
|
-
|
|
1492
|
+
fs37.read(self.fd, buffer, offset, length, position, function(err, bytesRead, buffer2) {
|
|
1492
1493
|
cb();
|
|
1493
1494
|
callback(err, bytesRead, buffer2);
|
|
1494
1495
|
});
|
|
@@ -1505,7 +1506,7 @@ var require_fd_slicer = __commonJS({
|
|
|
1505
1506
|
self.refCount -= 1;
|
|
1506
1507
|
if (self.refCount < 0) throw new Error("invalid unref");
|
|
1507
1508
|
if (self.refCount > 0) return;
|
|
1508
|
-
|
|
1509
|
+
fs37.close(self.fd, onCloseDone);
|
|
1509
1510
|
function onCloseDone(err) {
|
|
1510
1511
|
if (err) {
|
|
1511
1512
|
self.emit("error", err);
|
|
@@ -1537,7 +1538,7 @@ var require_fd_slicer = __commonJS({
|
|
|
1537
1538
|
}
|
|
1538
1539
|
self.context.pend.go(function(cb) {
|
|
1539
1540
|
var buffer = Buffer.allocUnsafe(toRead);
|
|
1540
|
-
|
|
1541
|
+
fs37.read(self.context.fd, buffer, 0, toRead, self.pos, function(err, bytesRead) {
|
|
1541
1542
|
if (err) {
|
|
1542
1543
|
self.destroy(err);
|
|
1543
1544
|
} else if (bytesRead === 0) {
|
|
@@ -1899,7 +1900,7 @@ var require_crc32 = __commonJS({
|
|
|
1899
1900
|
var require_yauzl = __commonJS({
|
|
1900
1901
|
"../../packages/node-runtime/node_modules/yauzl/index.js"(exports) {
|
|
1901
1902
|
"use strict";
|
|
1902
|
-
var
|
|
1903
|
+
var fs37 = __require("fs");
|
|
1903
1904
|
var zlib = __require("zlib");
|
|
1904
1905
|
var fd_slicer = require_fd_slicer();
|
|
1905
1906
|
var util = __require("util");
|
|
@@ -1924,9 +1925,9 @@ var require_yauzl = __commonJS({
|
|
|
1924
1925
|
exports.Entry = Entry;
|
|
1925
1926
|
exports.LocalFileHeader = LocalFileHeader;
|
|
1926
1927
|
exports.RandomAccessReader = RandomAccessReader;
|
|
1927
|
-
function openPromise2(
|
|
1928
|
+
function openPromise2(path31, options2) {
|
|
1928
1929
|
return new Promise((resolve4, reject) => {
|
|
1929
|
-
open(
|
|
1930
|
+
open(path31, { ...options2, lazyEntries: true }, function(err, zipfile) {
|
|
1930
1931
|
if (err) return reject(err);
|
|
1931
1932
|
resolve4(zipfile);
|
|
1932
1933
|
});
|
|
@@ -1956,7 +1957,7 @@ var require_yauzl = __commonJS({
|
|
|
1956
1957
|
});
|
|
1957
1958
|
});
|
|
1958
1959
|
}
|
|
1959
|
-
function open(
|
|
1960
|
+
function open(path31, options2, callback) {
|
|
1960
1961
|
if (typeof options2 === "function") {
|
|
1961
1962
|
callback = options2;
|
|
1962
1963
|
options2 = null;
|
|
@@ -1968,10 +1969,10 @@ var require_yauzl = __commonJS({
|
|
|
1968
1969
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
1969
1970
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
1970
1971
|
if (callback == null) callback = defaultCallback;
|
|
1971
|
-
|
|
1972
|
+
fs37.open(path31, "r", function(err, fd) {
|
|
1972
1973
|
if (err) return callback(err);
|
|
1973
1974
|
fromFd(fd, options2, function(err2, zipfile) {
|
|
1974
|
-
if (err2)
|
|
1975
|
+
if (err2) fs37.close(fd, defaultCallback);
|
|
1975
1976
|
callback(err2, zipfile);
|
|
1976
1977
|
});
|
|
1977
1978
|
});
|
|
@@ -1988,7 +1989,7 @@ var require_yauzl = __commonJS({
|
|
|
1988
1989
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
1989
1990
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
1990
1991
|
if (callback == null) callback = defaultCallback;
|
|
1991
|
-
|
|
1992
|
+
fs37.fstat(fd, function(err, stats) {
|
|
1992
1993
|
if (err) return callback(err);
|
|
1993
1994
|
var reader = new fd_slicer.FdSlicer(fd);
|
|
1994
1995
|
fromRandomAccessReader(reader, stats.size, options2, callback);
|
|
@@ -2767,9 +2768,9 @@ function optionsEnv(options2) {
|
|
|
2767
2768
|
function resolveChatLabTempRoot(options2 = {}) {
|
|
2768
2769
|
const envRoot = optionsEnv(options2)[CHATLAB_TEMP_ROOT_ENV]?.trim();
|
|
2769
2770
|
if (envRoot) return path.resolve(envRoot);
|
|
2770
|
-
const
|
|
2771
|
+
const platform = options2.platform ?? process.platform;
|
|
2771
2772
|
const systemTempDir = options2.systemTempDir ?? os.tmpdir();
|
|
2772
|
-
return path.resolve(
|
|
2773
|
+
return path.resolve(platform === "darwin" ? "/private/tmp/chatlab" : path.join(systemTempDir, "chatlab"));
|
|
2773
2774
|
}
|
|
2774
2775
|
function ensurePrivateOwnedDirectory(dirPath, label) {
|
|
2775
2776
|
fs.mkdirSync(dirPath, { recursive: true, mode: 448 });
|
|
@@ -3082,10 +3083,20 @@ function getNativeParserStatus() {
|
|
|
3082
3083
|
const module = loadNativeParser();
|
|
3083
3084
|
return { available: module !== null, disabled: false, error: module ? void 0 : cachedLoadError };
|
|
3084
3085
|
}
|
|
3086
|
+
function isNativeFormatAvailable(formatId) {
|
|
3087
|
+
const module = loadNativeParser();
|
|
3088
|
+
if (!module) return false;
|
|
3089
|
+
try {
|
|
3090
|
+
new module.NativeParser(formatId, "");
|
|
3091
|
+
return true;
|
|
3092
|
+
} catch {
|
|
3093
|
+
return false;
|
|
3094
|
+
}
|
|
3095
|
+
}
|
|
3085
3096
|
|
|
3086
3097
|
// ../../packages/parser/src/native/create-native-parser.ts
|
|
3087
3098
|
var PROGRESS_POLL_INTERVAL_MS = 200;
|
|
3088
|
-
async function* pumpNativeParser(adapter, options2, parser11,
|
|
3099
|
+
async function* pumpNativeParser(adapter, options2, parser11, fallbackAfterNativeFailure2) {
|
|
3089
3100
|
const { filePath, batchSize = 5e3, onProgress, onLog } = options2;
|
|
3090
3101
|
const totalBytes = getFileSize(filePath);
|
|
3091
3102
|
const initialProgress = createProgress("parsing", 0, totalBytes, 0, "");
|
|
@@ -3114,7 +3125,7 @@ async function* pumpNativeParser(adapter, options2, parser11, fallback) {
|
|
|
3114
3125
|
"warn",
|
|
3115
3126
|
`[NativeParser] Rust parse failed, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}`
|
|
3116
3127
|
);
|
|
3117
|
-
yield*
|
|
3128
|
+
yield* fallbackAfterNativeFailure2(options2);
|
|
3118
3129
|
return;
|
|
3119
3130
|
} finally {
|
|
3120
3131
|
clearInterval(progressTimer);
|
|
@@ -3135,9 +3146,12 @@ async function* pumpNativeParser(adapter, options2, parser11, fallback) {
|
|
|
3135
3146
|
yield { type: "progress", data: doneProgress };
|
|
3136
3147
|
onProgress?.(doneProgress);
|
|
3137
3148
|
onLog?.("info", `\u89E3\u6790\u5B8C\u6210: ${messagesProcessed} \u6761\u6D88\u606F, ${members.length} \u4E2A\u6210\u5458`);
|
|
3149
|
+
for (const message of adapter.completionLogs?.(metaJson) ?? []) {
|
|
3150
|
+
onLog?.("info", message);
|
|
3151
|
+
}
|
|
3138
3152
|
yield { type: "done", data: { messageCount: messagesProcessed, memberCount: members.length } };
|
|
3139
3153
|
}
|
|
3140
|
-
function createNativeFirstParser(adapter, fallback) {
|
|
3154
|
+
function createNativeFirstParser(adapter, fallback, fallbackAfterNativeFailure2 = fallback) {
|
|
3141
3155
|
return async function* (options2) {
|
|
3142
3156
|
const native = loadNativeParser();
|
|
3143
3157
|
if (!native) {
|
|
@@ -3159,7 +3173,7 @@ function createNativeFirstParser(adapter, fallback) {
|
|
|
3159
3173
|
yield* fallback(options2);
|
|
3160
3174
|
return;
|
|
3161
3175
|
}
|
|
3162
|
-
yield* pumpNativeParser(adapter, options2, parser11,
|
|
3176
|
+
yield* pumpNativeParser(adapter, options2, parser11, fallbackAfterNativeFailure2);
|
|
3163
3177
|
};
|
|
3164
3178
|
}
|
|
3165
3179
|
|
|
@@ -3675,6 +3689,12 @@ function slimMessage(msg) {
|
|
|
3675
3689
|
type: e.type
|
|
3676
3690
|
}));
|
|
3677
3691
|
}
|
|
3692
|
+
if (content?.reply && typeof content.reply === "object") {
|
|
3693
|
+
const reply = content.reply;
|
|
3694
|
+
if (reply.referencedMessageId !== void 0) {
|
|
3695
|
+
slimContent.reply = { referencedMessageId: reply.referencedMessageId };
|
|
3696
|
+
}
|
|
3697
|
+
}
|
|
3678
3698
|
const slimMsg = {
|
|
3679
3699
|
timestamp: msg.timestamp,
|
|
3680
3700
|
sender: { name: sender?.name || "" },
|
|
@@ -3682,12 +3702,12 @@ function slimMessage(msg) {
|
|
|
3682
3702
|
};
|
|
3683
3703
|
if (msg.id) slimMsg.id = msg.id;
|
|
3684
3704
|
if (msg.type) slimMsg.type = msg.type;
|
|
3685
|
-
if (msg.recalled) slimMsg.recalled = msg.recalled;
|
|
3686
|
-
if (msg.system) slimMsg.system = msg.system;
|
|
3687
|
-
if (msg.messageId) slimMsg.messageId = msg.messageId;
|
|
3705
|
+
if (msg.recalled !== void 0) slimMsg.recalled = msg.recalled;
|
|
3706
|
+
if (msg.system !== void 0) slimMsg.system = msg.system;
|
|
3707
|
+
if (msg.messageId !== void 0) slimMsg.messageId = msg.messageId;
|
|
3688
3708
|
if (msg.messageType !== void 0) slimMsg.messageType = msg.messageType;
|
|
3689
|
-
if (msg.isRecalled) slimMsg.isRecalled = msg.isRecalled;
|
|
3690
|
-
if (msg.isSystemMessage) slimMsg.isSystemMessage = msg.isSystemMessage;
|
|
3709
|
+
if (msg.isRecalled !== void 0) slimMsg.isRecalled = msg.isRecalled;
|
|
3710
|
+
if (msg.isSystemMessage !== void 0) slimMsg.isSystemMessage = msg.isSystemMessage;
|
|
3691
3711
|
if (sender?.uin) slimMsg.sender.uin = sender.uin;
|
|
3692
3712
|
if (sender?.uid) slimMsg.sender.uid = sender.uid;
|
|
3693
3713
|
if (rawMessage) {
|
|
@@ -3699,7 +3719,7 @@ function slimMessage(msg) {
|
|
|
3699
3719
|
}
|
|
3700
3720
|
return slimMsg;
|
|
3701
3721
|
}
|
|
3702
|
-
async function
|
|
3722
|
+
async function preprocessShuakamiQqJson(inputPath, onProgress) {
|
|
3703
3723
|
const totalBytes = getFileSize(inputPath);
|
|
3704
3724
|
let bytesRead = 0;
|
|
3705
3725
|
let messagesProcessed = 0;
|
|
@@ -3709,101 +3729,82 @@ async function preprocessQQJson(inputPath, onProgress) {
|
|
|
3709
3729
|
const outputPath = path5.join(tempDir, outputFilename);
|
|
3710
3730
|
onProgress?.(createProgress("parsing", 0, totalBytes, 0, ""));
|
|
3711
3731
|
const avatarsStr = readAvatarsFromFile(inputPath);
|
|
3732
|
+
const headContent = readFileHeadBytes(inputPath, 5e5);
|
|
3733
|
+
let chatInfo = { name: "\u672A\u77E5\u7FA4\u804A", type: "group" };
|
|
3734
|
+
let metadata;
|
|
3735
|
+
let statistics;
|
|
3736
|
+
try {
|
|
3737
|
+
const chatInfoJson = extractJsonObject(headContent, "chatInfo");
|
|
3738
|
+
if (chatInfoJson) chatInfo = JSON.parse(chatInfoJson);
|
|
3739
|
+
} catch {
|
|
3740
|
+
}
|
|
3741
|
+
if (!chatInfo.name) {
|
|
3742
|
+
chatInfo.name = path5.basename(inputPath).replace(/\.json$/i, "") || "\u672A\u77E5\u7FA4\u804A";
|
|
3743
|
+
}
|
|
3744
|
+
try {
|
|
3745
|
+
const metadataMatch = headContent.match(/"metadata"\s*:\s*(\{[^}]+\})/);
|
|
3746
|
+
if (metadataMatch) metadata = JSON.parse(metadataMatch[1]);
|
|
3747
|
+
} catch {
|
|
3748
|
+
}
|
|
3749
|
+
try {
|
|
3750
|
+
const statisticsMatch = headContent.match(/"statistics"\s*:\s*(\{[\s\S]*?\})\s*,\s*"messages"/);
|
|
3751
|
+
if (statisticsMatch) statistics = JSON.parse(statisticsMatch[1]);
|
|
3752
|
+
} catch {
|
|
3753
|
+
}
|
|
3712
3754
|
return new Promise((resolve4, reject) => {
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
const
|
|
3716
|
-
|
|
3717
|
-
|
|
3718
|
-
let metadata;
|
|
3719
|
-
let statistics;
|
|
3720
|
-
headStream.on("data", (chunk) => {
|
|
3721
|
-
const str = typeof chunk === "string" ? chunk : chunk.toString("utf-8");
|
|
3722
|
-
if (headSize < maxHeadSize) {
|
|
3723
|
-
headChunks.push(str);
|
|
3724
|
-
headSize += str.length;
|
|
3725
|
-
} else {
|
|
3726
|
-
headStream.destroy();
|
|
3727
|
-
}
|
|
3755
|
+
onProgress?.(createProgress("parsing", 0, totalBytes, 0, ""));
|
|
3756
|
+
const readStream = fs6.createReadStream(inputPath, { encoding: "utf-8" });
|
|
3757
|
+
const writeStream = fs6.createWriteStream(outputPath, { encoding: "utf-8" });
|
|
3758
|
+
readStream.on("data", (chunk) => {
|
|
3759
|
+
bytesRead += typeof chunk === "string" ? Buffer.byteLength(chunk) : chunk.length;
|
|
3728
3760
|
});
|
|
3729
|
-
|
|
3730
|
-
|
|
3731
|
-
|
|
3732
|
-
|
|
3733
|
-
|
|
3734
|
-
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
|
|
3738
|
-
|
|
3739
|
-
|
|
3740
|
-
|
|
3741
|
-
|
|
3742
|
-
}
|
|
3743
|
-
} catch {
|
|
3761
|
+
const header = { metadata, chatInfo, statistics, messages: [] };
|
|
3762
|
+
const headerJson = JSON.stringify(header);
|
|
3763
|
+
writeStream.write(headerJson.slice(0, -2) + "\n");
|
|
3764
|
+
let isFirstMessage = true;
|
|
3765
|
+
const pipeline2 = chain2([readStream, parser3(), pick2({ filter: /^messages\.\d+$/ }), streamValues2()]);
|
|
3766
|
+
pipeline2.on("data", ({ value }) => {
|
|
3767
|
+
const slimMsg = slimMessage(value);
|
|
3768
|
+
const msgJson = JSON.stringify(slimMsg);
|
|
3769
|
+
if (isFirstMessage) {
|
|
3770
|
+
writeStream.write(msgJson);
|
|
3771
|
+
isFirstMessage = false;
|
|
3772
|
+
} else {
|
|
3773
|
+
writeStream.write(",\n" + msgJson);
|
|
3744
3774
|
}
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
|
|
3748
|
-
|
|
3749
|
-
|
|
3750
|
-
|
|
3775
|
+
messagesProcessed++;
|
|
3776
|
+
if (messagesProcessed % 1e4 === 0) {
|
|
3777
|
+
onProgress?.(
|
|
3778
|
+
createProgress(
|
|
3779
|
+
"parsing",
|
|
3780
|
+
bytesRead,
|
|
3781
|
+
totalBytes,
|
|
3782
|
+
messagesProcessed,
|
|
3783
|
+
`\u9884\u5904\u7406\uFF1A\u5DF2\u7CBE\u7B80 ${messagesProcessed} \u6761\u6D88\u606F...`
|
|
3784
|
+
)
|
|
3785
|
+
);
|
|
3751
3786
|
}
|
|
3752
|
-
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
|
|
3756
|
-
|
|
3757
|
-
}
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
writeStream.
|
|
3761
|
-
|
|
3762
|
-
|
|
3763
|
-
pipeline2.on("data", ({ value }) => {
|
|
3764
|
-
const slimMsg = slimMessage(value);
|
|
3765
|
-
const msgJson = JSON.stringify(slimMsg);
|
|
3766
|
-
if (isFirstMessage) {
|
|
3767
|
-
writeStream.write(msgJson);
|
|
3768
|
-
isFirstMessage = false;
|
|
3769
|
-
} else {
|
|
3770
|
-
writeStream.write(",\n" + msgJson);
|
|
3771
|
-
}
|
|
3772
|
-
messagesProcessed++;
|
|
3773
|
-
if (messagesProcessed % 1e4 === 0) {
|
|
3774
|
-
onProgress?.(
|
|
3775
|
-
createProgress(
|
|
3776
|
-
"parsing",
|
|
3777
|
-
bytesRead,
|
|
3778
|
-
totalBytes,
|
|
3779
|
-
messagesProcessed,
|
|
3780
|
-
`\u9884\u5904\u7406\uFF1A\u5DF2\u7CBE\u7B80 ${messagesProcessed} \u6761\u6D88\u606F...`
|
|
3781
|
-
)
|
|
3782
|
-
);
|
|
3783
|
-
}
|
|
3784
|
-
});
|
|
3785
|
-
pipeline2.on("end", () => {
|
|
3786
|
-
writeStream.write("\n]");
|
|
3787
|
-
if (avatarsStr) {
|
|
3788
|
-
writeStream.write(',"avatars":' + avatarsStr);
|
|
3789
|
-
}
|
|
3790
|
-
writeStream.write("}");
|
|
3791
|
-
writeStream.end();
|
|
3792
|
-
writeStream.on("finish", () => {
|
|
3793
|
-
onProgress?.(createProgress("done", totalBytes, totalBytes, messagesProcessed, ""));
|
|
3794
|
-
resolve4(outputPath);
|
|
3795
|
-
});
|
|
3796
|
-
});
|
|
3797
|
-
pipeline2.on("error", (err) => {
|
|
3798
|
-
writeStream.destroy();
|
|
3799
|
-
if (fs6.existsSync(outputPath)) {
|
|
3800
|
-
fs6.unlinkSync(outputPath);
|
|
3801
|
-
}
|
|
3802
|
-
onProgress?.(createProgress("error", bytesRead, totalBytes, messagesProcessed, err.message));
|
|
3803
|
-
reject(err);
|
|
3787
|
+
});
|
|
3788
|
+
pipeline2.on("end", () => {
|
|
3789
|
+
writeStream.write("\n]");
|
|
3790
|
+
if (avatarsStr) {
|
|
3791
|
+
writeStream.write(',"avatars":' + avatarsStr);
|
|
3792
|
+
}
|
|
3793
|
+
writeStream.write("}");
|
|
3794
|
+
writeStream.end();
|
|
3795
|
+
writeStream.on("finish", () => {
|
|
3796
|
+
onProgress?.(createProgress("done", totalBytes, totalBytes, messagesProcessed, ""));
|
|
3797
|
+
resolve4(outputPath);
|
|
3804
3798
|
});
|
|
3805
3799
|
});
|
|
3806
|
-
|
|
3800
|
+
pipeline2.on("error", (err) => {
|
|
3801
|
+
writeStream.destroy();
|
|
3802
|
+
if (fs6.existsSync(outputPath)) {
|
|
3803
|
+
fs6.unlinkSync(outputPath);
|
|
3804
|
+
}
|
|
3805
|
+
onProgress?.(createProgress("error", bytesRead, totalBytes, messagesProcessed, err.message));
|
|
3806
|
+
reject(err);
|
|
3807
|
+
});
|
|
3807
3808
|
});
|
|
3808
3809
|
}
|
|
3809
3810
|
function cleanupTempFile(filePath) {
|
|
@@ -3814,14 +3815,85 @@ function cleanupTempFile(filePath) {
|
|
|
3814
3815
|
} catch {
|
|
3815
3816
|
}
|
|
3816
3817
|
}
|
|
3817
|
-
var
|
|
3818
|
+
var shuakamiQqPreprocessor = {
|
|
3818
3819
|
needsPreprocess(_filePath, fileSize) {
|
|
3819
3820
|
return fileSize > PREPROCESS_THRESHOLD;
|
|
3820
3821
|
},
|
|
3821
|
-
preprocess:
|
|
3822
|
+
preprocess: preprocessShuakamiQqJson,
|
|
3822
3823
|
cleanup: cleanupTempFile
|
|
3823
3824
|
};
|
|
3824
3825
|
|
|
3826
|
+
// ../../packages/parser/src/native/shuakami-qq-native.ts
|
|
3827
|
+
var shuakamiQqAdapter = {
|
|
3828
|
+
formatId: "shuakami-qq-exporter",
|
|
3829
|
+
label: "shuakami/qq-chat-exporter",
|
|
3830
|
+
mapMeta(metaJson) {
|
|
3831
|
+
const meta = metaJson;
|
|
3832
|
+
return {
|
|
3833
|
+
name: meta.name,
|
|
3834
|
+
platform: KNOWN_PLATFORMS.QQ,
|
|
3835
|
+
type: meta.chatType === "private" ? "private" /* PRIVATE */ : "group" /* GROUP */,
|
|
3836
|
+
groupAvatar: meta.groupAvatar ?? void 0
|
|
3837
|
+
};
|
|
3838
|
+
},
|
|
3839
|
+
mapMembers(members) {
|
|
3840
|
+
return members.map((member) => ({
|
|
3841
|
+
platformId: member.platformId,
|
|
3842
|
+
accountName: member.accountName,
|
|
3843
|
+
groupNickname: member.groupNickname,
|
|
3844
|
+
avatar: member.avatar
|
|
3845
|
+
}));
|
|
3846
|
+
},
|
|
3847
|
+
mapMessage(message) {
|
|
3848
|
+
return {
|
|
3849
|
+
platformMessageId: message.platformMessageId,
|
|
3850
|
+
senderPlatformId: message.senderPlatformId,
|
|
3851
|
+
senderAccountName: message.senderAccountName,
|
|
3852
|
+
senderGroupNickname: message.senderGroupNickname,
|
|
3853
|
+
timestamp: message.timestamp,
|
|
3854
|
+
type: message.messageType,
|
|
3855
|
+
content: message.content ?? null,
|
|
3856
|
+
replyToMessageId: message.replyToMessageId
|
|
3857
|
+
};
|
|
3858
|
+
},
|
|
3859
|
+
completionLogs(metaJson) {
|
|
3860
|
+
const skipped = metaJson.skippedMessages;
|
|
3861
|
+
return skipped > 0 ? [
|
|
3862
|
+
`[NativeParser] Skipped ${skipped} shuakami/qq-chat-exporter messages with a missing sender ID or invalid timestamp`
|
|
3863
|
+
] : [];
|
|
3864
|
+
}
|
|
3865
|
+
};
|
|
3866
|
+
async function* fallbackAfterNativeFailure(fallback, options2) {
|
|
3867
|
+
if (!shuakamiQqPreprocessor.needsPreprocess(options2.filePath, getFileSize(options2.filePath))) {
|
|
3868
|
+
yield* fallback(options2);
|
|
3869
|
+
return;
|
|
3870
|
+
}
|
|
3871
|
+
let slimFilePath = null;
|
|
3872
|
+
try {
|
|
3873
|
+
options2.onLog?.("info", "[NativeParser] Preprocessing large export before TypeScript fallback");
|
|
3874
|
+
slimFilePath = await shuakamiQqPreprocessor.preprocess(options2.filePath, options2.onProgress);
|
|
3875
|
+
} catch (error) {
|
|
3876
|
+
options2.onLog?.(
|
|
3877
|
+
"warn",
|
|
3878
|
+
`[NativeParser] Fallback preprocessing failed; retrying the original export: ${error instanceof Error ? error.message : String(error)}`
|
|
3879
|
+
);
|
|
3880
|
+
yield* fallback(options2);
|
|
3881
|
+
return;
|
|
3882
|
+
}
|
|
3883
|
+
try {
|
|
3884
|
+
yield* fallback({ ...options2, filePath: slimFilePath });
|
|
3885
|
+
} finally {
|
|
3886
|
+
if (slimFilePath) shuakamiQqPreprocessor.cleanup(slimFilePath);
|
|
3887
|
+
}
|
|
3888
|
+
}
|
|
3889
|
+
function withShuakamiQqNative(fallback) {
|
|
3890
|
+
return createNativeFirstParser(
|
|
3891
|
+
shuakamiQqAdapter,
|
|
3892
|
+
fallback,
|
|
3893
|
+
(options2) => fallbackAfterNativeFailure(fallback, options2)
|
|
3894
|
+
);
|
|
3895
|
+
}
|
|
3896
|
+
|
|
3825
3897
|
// ../../packages/parser/src/formats/shuakami-qq-exporter.ts
|
|
3826
3898
|
var { parser: parser4 } = import_stream_json3.default;
|
|
3827
3899
|
var { pick: pick3 } = import_Pick3.default;
|
|
@@ -3943,7 +4015,7 @@ function convertMessageType(messageType, content, isRecalled) {
|
|
|
3943
4015
|
if (messageType === 9) return 25 /* REPLY */;
|
|
3944
4016
|
return 0 /* TEXT */;
|
|
3945
4017
|
}
|
|
3946
|
-
async function*
|
|
4018
|
+
async function* parseShuakamiQqV4(options2) {
|
|
3947
4019
|
const { filePath, batchSize = 5e3, onProgress, onLog } = options2;
|
|
3948
4020
|
const totalBytes = getFileSize(filePath);
|
|
3949
4021
|
let bytesRead = 0;
|
|
@@ -3952,7 +4024,7 @@ async function* parseV4(options2) {
|
|
|
3952
4024
|
const initialProgress = createProgress("parsing", 0, totalBytes, 0, "");
|
|
3953
4025
|
yield { type: "progress", data: initialProgress };
|
|
3954
4026
|
onProgress?.(initialProgress);
|
|
3955
|
-
onLog?.("info", `\u5F00\u59CB\u89E3\u6790
|
|
4027
|
+
onLog?.("info", `\u5F00\u59CB\u89E3\u6790 shuakami/qq-chat-exporter \u6587\u4EF6\uFF0C\u5927\u5C0F: ${(totalBytes / 1024 / 1024).toFixed(2)} MB`);
|
|
3956
4028
|
const headContent = readFileHeadBytes(filePath, 5e5);
|
|
3957
4029
|
let chatInfo = { name: "\u672A\u77E5\u7FA4\u804A", type: "group" };
|
|
3958
4030
|
try {
|
|
@@ -4079,14 +4151,15 @@ async function* parseV4(options2) {
|
|
|
4079
4151
|
data: { messageCount: messagesProcessed, memberCount: memberMap.size }
|
|
4080
4152
|
};
|
|
4081
4153
|
}
|
|
4154
|
+
var parseShuakamiQqV4Accelerated = withShuakamiQqNative(parseShuakamiQqV4);
|
|
4082
4155
|
var parser_2 = {
|
|
4083
4156
|
feature: feature3,
|
|
4084
|
-
parse:
|
|
4157
|
+
parse: parseShuakamiQqV4Accelerated
|
|
4085
4158
|
};
|
|
4086
4159
|
var module_3 = {
|
|
4087
4160
|
feature: feature3,
|
|
4088
4161
|
parser: parser_2,
|
|
4089
|
-
preprocessor:
|
|
4162
|
+
preprocessor: shuakamiQqPreprocessor
|
|
4090
4163
|
};
|
|
4091
4164
|
var shuakami_qq_exporter_default = module_3;
|
|
4092
4165
|
|
|
@@ -5595,6 +5668,12 @@ var SYSTEM_MESSAGE_PATTERNS = [
|
|
|
5595
5668
|
/This message was deleted/i,
|
|
5596
5669
|
/disappearing messages/i
|
|
5597
5670
|
];
|
|
5671
|
+
var SYSTEM_MESSAGE_REGEX = new RegExp(SYSTEM_MESSAGE_PATTERNS.map((pattern) => pattern.source).join("|"), "i");
|
|
5672
|
+
var TIMESTAMP_SPACING = String.raw`(?:\s|\u2009|\u202F|\uFEFF|\u200E|\u200F|\u200B|\u200C|\u200D|\u2060)`;
|
|
5673
|
+
var COMMON_TIMESTAMP_REGEX = new RegExp(
|
|
5674
|
+
String.raw`^${TIMESTAMP_SPACING}*(\d{1,4})\/(\d{1,2})\/(\d{1,4})(?:,|${TIMESTAMP_SPACING})+(\d{1,2}):(\d{2})(?::(\d{2}))?(?:${TIMESTAMP_SPACING}+(AM|PM|A\.M\.|P\.M\.|上午|下午|午前|午後|오전|오후))?${TIMESTAMP_SPACING}*$`,
|
|
5675
|
+
"i"
|
|
5676
|
+
);
|
|
5598
5677
|
var AMPM_MARKERS = [
|
|
5599
5678
|
[/\bPM\b/i, true],
|
|
5600
5679
|
[/\bP\.M\.(?!\w)/i, true],
|
|
@@ -5667,7 +5746,7 @@ var WhatsAppTextAccumulator = class {
|
|
|
5667
5746
|
senderPlatformId,
|
|
5668
5747
|
senderAccountName: senderName,
|
|
5669
5748
|
timestamp: this.currentMessage.timestamp,
|
|
5670
|
-
type: detectMessageType2(content),
|
|
5749
|
+
type: detectMessageType2(content, this.currentMessage.isSystem),
|
|
5671
5750
|
content: content || null
|
|
5672
5751
|
});
|
|
5673
5752
|
if (this.currentMessage.sender) {
|
|
@@ -5678,10 +5757,11 @@ var WhatsAppTextAccumulator = class {
|
|
|
5678
5757
|
};
|
|
5679
5758
|
function createPendingMessage(timestamp, restContent) {
|
|
5680
5759
|
const senderMatch = restContent.match(SENDER_CONTENT_REGEX);
|
|
5681
|
-
|
|
5682
|
-
|
|
5760
|
+
const isSystem = isSystemMessage(restContent);
|
|
5761
|
+
if (senderMatch && !isSystem) {
|
|
5762
|
+
return { timestamp, sender: senderMatch[1].trim(), contentLines: [senderMatch[2]], isSystem };
|
|
5683
5763
|
}
|
|
5684
|
-
return { timestamp, sender: null, contentLines: [restContent] };
|
|
5764
|
+
return { timestamp, sender: null, contentLines: [restContent], isSystem };
|
|
5685
5765
|
}
|
|
5686
5766
|
function extractNameFromFileName(fileName) {
|
|
5687
5767
|
const basename10 = getBasename(fileName);
|
|
@@ -5705,9 +5785,9 @@ function cleanLine(line) {
|
|
|
5705
5785
|
return line.replace(/^(?:\uFEFF|\u200E|\u200F|\u200B|\u200C|\u200D|\u2060)+/, "").trim();
|
|
5706
5786
|
}
|
|
5707
5787
|
function isSystemMessage(content) {
|
|
5708
|
-
return
|
|
5788
|
+
return SYSTEM_MESSAGE_REGEX.test(content);
|
|
5709
5789
|
}
|
|
5710
|
-
function detectMessageType2(content) {
|
|
5790
|
+
function detectMessageType2(content, knownSystem = false) {
|
|
5711
5791
|
const trimmed = content.trim();
|
|
5712
5792
|
if (trimmed === "<\u7701\u7565\u5F71\u97F3\u5185\u5BB9>" || trimmed === "<\u5DF2\u7701\u7565\u591A\u5A92\u9AD4\u6A94\u6848>" || trimmed === "\u5716\u7247\u5DF2\u7565\u53BB" || trimmed === "\u5F71\u7247\u5DF2\u7565\u53BB" || trimmed === "\u97F3\u8A0A\u5DF2\u7565\u53BB" || trimmed === "image omitted" || trimmed === "video omitted" || trimmed === "audio omitted") {
|
|
5713
5793
|
return 1 /* IMAGE */;
|
|
@@ -5719,10 +5799,22 @@ function detectMessageType2(content) {
|
|
|
5719
5799
|
if (trimmed === "\u8FD9\u6761\u6D88\u606F\u5DF2\u5220\u9664" || trimmed.startsWith("\u6B64\u8A0A\u606F\u5DF2\u522A\u9664") || trimmed.startsWith("\u4F60\u5DF2\u522A\u9664\u6B64\u8A0A\u606F")) {
|
|
5720
5800
|
return 81 /* RECALL */;
|
|
5721
5801
|
}
|
|
5722
|
-
if (isSystemMessage(trimmed)) return 80 /* SYSTEM */;
|
|
5802
|
+
if (knownSystem || isSystemMessage(trimmed)) return 80 /* SYSTEM */;
|
|
5723
5803
|
return 0 /* TEXT */;
|
|
5724
5804
|
}
|
|
5725
5805
|
function parseFlexibleTimestamp(raw) {
|
|
5806
|
+
const commonMatch = raw.match(COMMON_TIMESTAMP_REGEX);
|
|
5807
|
+
if (commonMatch) {
|
|
5808
|
+
return buildTimestamp(
|
|
5809
|
+
Number.parseInt(commonMatch[1], 10),
|
|
5810
|
+
Number.parseInt(commonMatch[2], 10),
|
|
5811
|
+
Number.parseInt(commonMatch[3], 10),
|
|
5812
|
+
Number.parseInt(commonMatch[4], 10),
|
|
5813
|
+
Number.parseInt(commonMatch[5], 10),
|
|
5814
|
+
Number.parseInt(commonMatch[6] ?? "0", 10),
|
|
5815
|
+
parseAmPm(commonMatch[7])
|
|
5816
|
+
);
|
|
5817
|
+
}
|
|
5726
5818
|
let value = raw.replace(/(?:\u2009|\u202F|\uFEFF|\u200E|\u200F|\u200B|\u200C|\u200D|\u2060)/g, " ").trim();
|
|
5727
5819
|
let isPm = null;
|
|
5728
5820
|
for (const [pattern, pm] of AMPM_MARKERS) {
|
|
@@ -5736,23 +5828,30 @@ function parseFlexibleTimestamp(raw) {
|
|
|
5736
5828
|
if (!match) return null;
|
|
5737
5829
|
const dateParts = match[1].split("/").map((part) => Number.parseInt(part, 10));
|
|
5738
5830
|
const timeParts = match[2].split(":").map((part) => Number.parseInt(part, 10));
|
|
5831
|
+
return buildTimestamp(dateParts[0], dateParts[1], dateParts[2], timeParts[0], timeParts[1], timeParts[2] ?? 0, isPm);
|
|
5832
|
+
}
|
|
5833
|
+
function parseAmPm(marker) {
|
|
5834
|
+
if (!marker) return null;
|
|
5835
|
+
return /^(?:PM|P\.M\.|下午|午後|오후)$/i.test(marker);
|
|
5836
|
+
}
|
|
5837
|
+
function buildTimestamp(firstDatePart, secondDatePart, thirdDatePart, rawHour, minute, second, isPm) {
|
|
5739
5838
|
let year;
|
|
5740
5839
|
let month;
|
|
5741
5840
|
let day;
|
|
5742
|
-
if (
|
|
5743
|
-
;
|
|
5744
|
-
|
|
5745
|
-
|
|
5746
|
-
|
|
5747
|
-
|
|
5841
|
+
if (firstDatePart > 31) {
|
|
5842
|
+
year = firstDatePart;
|
|
5843
|
+
month = secondDatePart;
|
|
5844
|
+
day = thirdDatePart;
|
|
5845
|
+
} else if (thirdDatePart > 31) {
|
|
5846
|
+
day = firstDatePart;
|
|
5847
|
+
month = secondDatePart;
|
|
5848
|
+
year = thirdDatePart;
|
|
5748
5849
|
} else {
|
|
5749
|
-
;
|
|
5750
|
-
|
|
5751
|
-
year = 2e3 +
|
|
5850
|
+
month = firstDatePart;
|
|
5851
|
+
day = secondDatePart;
|
|
5852
|
+
year = 2e3 + thirdDatePart;
|
|
5752
5853
|
}
|
|
5753
|
-
let hour =
|
|
5754
|
-
const minute = timeParts[1];
|
|
5755
|
-
const second = timeParts[2] ?? 0;
|
|
5854
|
+
let hour = rawHour;
|
|
5756
5855
|
if (isPm === true && hour !== 12) hour += 12;
|
|
5757
5856
|
if (isPm === false && hour === 12) hour = 0;
|
|
5758
5857
|
const date = new Date(year, month - 1, day, hour, minute, second);
|
|
@@ -6036,13 +6135,14 @@ var feature12 = {
|
|
|
6036
6135
|
head: [/"is_geoblocked_for_viewer"\s*:/]
|
|
6037
6136
|
}
|
|
6038
6137
|
};
|
|
6138
|
+
var UTF8_DECODER = new TextDecoder("utf-8");
|
|
6039
6139
|
function decodeInstagramText(text) {
|
|
6040
6140
|
try {
|
|
6041
6141
|
const bytes = new Uint8Array(text.length);
|
|
6042
6142
|
for (let i = 0; i < text.length; i++) {
|
|
6043
6143
|
bytes[i] = text.charCodeAt(i);
|
|
6044
6144
|
}
|
|
6045
|
-
return
|
|
6145
|
+
return UTF8_DECODER.decode(bytes);
|
|
6046
6146
|
} catch {
|
|
6047
6147
|
return text;
|
|
6048
6148
|
}
|
|
@@ -6051,23 +6151,23 @@ function extractNameFromFilePath5(filePath) {
|
|
|
6051
6151
|
const basename10 = path9.basename(filePath);
|
|
6052
6152
|
return basename10.replace(/\.json$/i, "") || "\u672A\u77E5\u5BF9\u8BDD";
|
|
6053
6153
|
}
|
|
6154
|
+
var SYSTEM_MESSAGE_PATTERNS2 = [
|
|
6155
|
+
"You created the group",
|
|
6156
|
+
"created the group",
|
|
6157
|
+
"added",
|
|
6158
|
+
"to the group",
|
|
6159
|
+
"left the group",
|
|
6160
|
+
"removed",
|
|
6161
|
+
"named the group",
|
|
6162
|
+
"changed the group photo",
|
|
6163
|
+
"Reacted",
|
|
6164
|
+
"sent an attachment",
|
|
6165
|
+
"liked a message",
|
|
6166
|
+
"changed the theme",
|
|
6167
|
+
"set the nickname"
|
|
6168
|
+
];
|
|
6054
6169
|
function isSystemMessage2(content) {
|
|
6055
|
-
|
|
6056
|
-
"You created the group",
|
|
6057
|
-
"created the group",
|
|
6058
|
-
"added",
|
|
6059
|
-
"to the group",
|
|
6060
|
-
"left the group",
|
|
6061
|
-
"removed",
|
|
6062
|
-
"named the group",
|
|
6063
|
-
"changed the group photo",
|
|
6064
|
-
"Reacted",
|
|
6065
|
-
"sent an attachment",
|
|
6066
|
-
"liked a message",
|
|
6067
|
-
"changed the theme",
|
|
6068
|
-
"set the nickname"
|
|
6069
|
-
];
|
|
6070
|
-
return systemPatterns.some((p) => content.includes(p));
|
|
6170
|
+
return SYSTEM_MESSAGE_PATTERNS2.some((pattern) => content.includes(pattern));
|
|
6071
6171
|
}
|
|
6072
6172
|
function detectMessageType4(msg) {
|
|
6073
6173
|
const content = msg.content || "";
|
|
@@ -6151,9 +6251,9 @@ async function* parseInstagram(options2) {
|
|
|
6151
6251
|
}
|
|
6152
6252
|
const members = Array.from(memberMap.values());
|
|
6153
6253
|
yield { type: "members", data: members };
|
|
6154
|
-
const reversedMessages = [...data.messages].reverse();
|
|
6155
6254
|
const messageBatch = [];
|
|
6156
|
-
for (
|
|
6255
|
+
for (let index = data.messages.length - 1; index >= 0; index -= 1) {
|
|
6256
|
+
const msg = data.messages[index];
|
|
6157
6257
|
const senderName = decodeInstagramText(msg.sender_name);
|
|
6158
6258
|
const timestamp = Math.floor(msg.timestamp_ms / 1e3);
|
|
6159
6259
|
const type = detectMessageType4(msg);
|
|
@@ -6177,7 +6277,7 @@ async function* parseInstagram(options2) {
|
|
|
6177
6277
|
messageBatch.length = 0;
|
|
6178
6278
|
const progress = createProgress(
|
|
6179
6279
|
"parsing",
|
|
6180
|
-
Math.floor(messagesProcessed /
|
|
6280
|
+
Math.floor(messagesProcessed / data.messages.length * totalBytes),
|
|
6181
6281
|
totalBytes,
|
|
6182
6282
|
messagesProcessed,
|
|
6183
6283
|
`\u5DF2\u5904\u7406 ${messagesProcessed} \u6761\u6D88\u606F...`
|
|
@@ -6498,20 +6598,22 @@ var SPECIAL_MESSAGE_TYPES = {
|
|
|
6498
6598
|
"[\u8A18\u4E8B\u672C]": 0 /* TEXT */,
|
|
6499
6599
|
"[\u30CE\u30FC\u30C8]": 0 /* TEXT */
|
|
6500
6600
|
};
|
|
6601
|
+
var SPECIAL_MESSAGE_TYPE_ENTRIES = Object.entries(SPECIAL_MESSAGE_TYPES);
|
|
6501
6602
|
async function parseLineText(content, fileName, options2 = {}) {
|
|
6502
|
-
const lines = content.split("\n")
|
|
6603
|
+
const lines = content.split("\n");
|
|
6503
6604
|
const memberMap = /* @__PURE__ */ new Map();
|
|
6504
6605
|
const messages = [];
|
|
6505
|
-
const header = lines.length > 0 ? extractNameFromHeader(lines[0].trim()) : null;
|
|
6606
|
+
const header = lines.length > 0 ? extractNameFromHeader(lines[0].replace(/\r$/, "").trim()) : null;
|
|
6506
6607
|
const chatName = header?.name ?? extractNameFromFileName3(fileName);
|
|
6507
6608
|
let currentDate = null;
|
|
6508
6609
|
let lastMessage = null;
|
|
6509
6610
|
let quotedMultiline = false;
|
|
6510
6611
|
let lineIndex = header ? 3 : 0;
|
|
6511
6612
|
let useTabSeparator = Boolean(header);
|
|
6512
|
-
const yieldEvery = Math.max(1, options2.yieldEvery ??
|
|
6613
|
+
const yieldEvery = Math.max(1, options2.yieldEvery ?? 5e3);
|
|
6513
6614
|
if (!header) {
|
|
6514
|
-
for (const
|
|
6615
|
+
for (const rawLine of lines) {
|
|
6616
|
+
const line = rawLine.replace(/\r$/, "");
|
|
6515
6617
|
if (TAB_MESSAGE_PATTERN.test(line)) {
|
|
6516
6618
|
useTabSeparator = true;
|
|
6517
6619
|
break;
|
|
@@ -6524,7 +6626,7 @@ async function parseLineText(content, fileName, options2 = {}) {
|
|
|
6524
6626
|
}
|
|
6525
6627
|
for (; lineIndex < lines.length; lineIndex += 1) {
|
|
6526
6628
|
options2.checkCancelled?.();
|
|
6527
|
-
const line = lines[lineIndex];
|
|
6629
|
+
const line = lines[lineIndex].replace(/\r$/, "");
|
|
6528
6630
|
const date = parseDateLine(line);
|
|
6529
6631
|
if (date) {
|
|
6530
6632
|
currentDate = date;
|
|
@@ -6546,7 +6648,7 @@ async function parseLineText(content, fileName, options2 = {}) {
|
|
|
6546
6648
|
lastMessage = {
|
|
6547
6649
|
senderPlatformId: sender,
|
|
6548
6650
|
senderAccountName: sender,
|
|
6549
|
-
timestamp:
|
|
6651
|
+
timestamp: buildTimestamp2(currentDate, timeText, options2.now),
|
|
6550
6652
|
type: detectMessageType6(messageContent),
|
|
6551
6653
|
content: messageContent || null
|
|
6552
6654
|
};
|
|
@@ -6560,7 +6662,7 @@ async function parseLineText(content, fileName, options2 = {}) {
|
|
|
6560
6662
|
lastMessage = {
|
|
6561
6663
|
senderPlatformId: "system",
|
|
6562
6664
|
senderAccountName: "\u7CFB\u7D71",
|
|
6563
|
-
timestamp:
|
|
6665
|
+
timestamp: buildTimestamp2(currentDate, timeText, options2.now),
|
|
6564
6666
|
type: 80 /* SYSTEM */,
|
|
6565
6667
|
content: messageContent || null
|
|
6566
6668
|
};
|
|
@@ -6629,7 +6731,7 @@ function parseDateLine(line) {
|
|
|
6629
6731
|
}
|
|
6630
6732
|
return null;
|
|
6631
6733
|
}
|
|
6632
|
-
function
|
|
6734
|
+
function buildTimestamp2(currentDate, timeText, now) {
|
|
6633
6735
|
const { hours, minutes } = parseTime(timeText);
|
|
6634
6736
|
const value = currentDate ? new Date(currentDate) : now?.() ?? /* @__PURE__ */ new Date();
|
|
6635
6737
|
value.setHours(hours, minutes, 0, 0);
|
|
@@ -6653,7 +6755,7 @@ function parseTime(timeText) {
|
|
|
6653
6755
|
return { hours, minutes };
|
|
6654
6756
|
}
|
|
6655
6757
|
function detectMessageType6(content) {
|
|
6656
|
-
for (const [pattern, type] of
|
|
6758
|
+
for (const [pattern, type] of SPECIAL_MESSAGE_TYPE_ENTRIES) {
|
|
6657
6759
|
if (content === pattern || content.startsWith(pattern)) return type;
|
|
6658
6760
|
}
|
|
6659
6761
|
if (content.startsWith("[null]") && content.includes("maps.google.com")) return 8 /* LOCATION */;
|
|
@@ -6796,21 +6898,21 @@ function isRecord(value) {
|
|
|
6796
6898
|
function hasOwn(value, key) {
|
|
6797
6899
|
return Object.prototype.hasOwnProperty.call(value, key);
|
|
6798
6900
|
}
|
|
6799
|
-
function childLocation(location,
|
|
6800
|
-
return { ...location, path:
|
|
6901
|
+
function childLocation(location, path31) {
|
|
6902
|
+
return { ...location, path: path31 };
|
|
6801
6903
|
}
|
|
6802
|
-
function validateRequiredString(state, value,
|
|
6904
|
+
function validateRequiredString(state, value, path31, location) {
|
|
6803
6905
|
if (typeof value === "string" && value.trim().length > 0) return true;
|
|
6804
|
-
addIssue(state, "error", "INVALID_REQUIRED_STRING", "Expected a non-empty string.", childLocation(location,
|
|
6906
|
+
addIssue(state, "error", "INVALID_REQUIRED_STRING", "Expected a non-empty string.", childLocation(location, path31));
|
|
6805
6907
|
return false;
|
|
6806
6908
|
}
|
|
6807
|
-
function validateOptionalString(state, object, key,
|
|
6909
|
+
function validateOptionalString(state, object, key, path31, location) {
|
|
6808
6910
|
if (!hasOwn(object, key)) return;
|
|
6809
6911
|
if (typeof object[key] !== "string") {
|
|
6810
|
-
addIssue(state, "error", "INVALID_OPTIONAL_STRING", "Expected a string.", childLocation(location,
|
|
6912
|
+
addIssue(state, "error", "INVALID_OPTIONAL_STRING", "Expected a string.", childLocation(location, path31));
|
|
6811
6913
|
}
|
|
6812
6914
|
}
|
|
6813
|
-
function validateUnixSeconds(state, value,
|
|
6915
|
+
function validateUnixSeconds(state, value, path31, location) {
|
|
6814
6916
|
if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value < MAX_UNIX_SECONDS) {
|
|
6815
6917
|
return true;
|
|
6816
6918
|
}
|
|
@@ -6819,7 +6921,7 @@ function validateUnixSeconds(state, value, path30, location) {
|
|
|
6819
6921
|
"error",
|
|
6820
6922
|
"INVALID_UNIX_SECONDS",
|
|
6821
6923
|
"Expected a non-negative integer Unix timestamp in seconds, not milliseconds.",
|
|
6822
|
-
childLocation(location,
|
|
6924
|
+
childLocation(location, path31)
|
|
6823
6925
|
);
|
|
6824
6926
|
return false;
|
|
6825
6927
|
}
|
|
@@ -6880,30 +6982,30 @@ function validateHeader(state, chatlab, meta, location, basePath = "") {
|
|
|
6880
6982
|
}
|
|
6881
6983
|
validateOptionalString(state, meta, "sourceSessionId", `${basePath}meta.sourceSessionId`, location);
|
|
6882
6984
|
}
|
|
6883
|
-
function validateMember(state, value,
|
|
6985
|
+
function validateMember(state, value, path31, location) {
|
|
6884
6986
|
state.stats.memberCount++;
|
|
6885
6987
|
if (!isRecord(value)) {
|
|
6886
|
-
addIssue(state, "error", "INVALID_MEMBER", "Expected a member object.", childLocation(location,
|
|
6988
|
+
addIssue(state, "error", "INVALID_MEMBER", "Expected a member object.", childLocation(location, path31));
|
|
6887
6989
|
return;
|
|
6888
6990
|
}
|
|
6889
|
-
if (validateRequiredString(state, value.platformId, `${
|
|
6991
|
+
if (validateRequiredString(state, value.platformId, `${path31}.platformId`, location)) {
|
|
6890
6992
|
if (state.memberIds.has(value.platformId)) {
|
|
6891
6993
|
addIssue(state, "error", "DUPLICATE_MEMBER_ID", "Member platformId must be unique.", {
|
|
6892
6994
|
...location,
|
|
6893
|
-
path: `${
|
|
6995
|
+
path: `${path31}.platformId`
|
|
6894
6996
|
});
|
|
6895
6997
|
} else {
|
|
6896
6998
|
state.memberIds.add(value.platformId);
|
|
6897
6999
|
}
|
|
6898
7000
|
}
|
|
6899
|
-
validateRequiredString(state, value.accountName, `${
|
|
6900
|
-
validateOptionalString(state, value, "groupNickname", `${
|
|
6901
|
-
validateOptionalString(state, value, "avatar", `${
|
|
7001
|
+
validateRequiredString(state, value.accountName, `${path31}.accountName`, location);
|
|
7002
|
+
validateOptionalString(state, value, "groupNickname", `${path31}.groupNickname`, location);
|
|
7003
|
+
validateOptionalString(state, value, "avatar", `${path31}.avatar`, location);
|
|
6902
7004
|
if (hasOwn(value, "aliases")) {
|
|
6903
7005
|
if (!Array.isArray(value.aliases) || value.aliases.some((alias) => typeof alias !== "string")) {
|
|
6904
7006
|
addIssue(state, "error", "INVALID_ALIASES", "Expected aliases to be an array of strings.", {
|
|
6905
7007
|
...location,
|
|
6906
|
-
path: `${
|
|
7008
|
+
path: `${path31}.aliases`
|
|
6907
7009
|
});
|
|
6908
7010
|
}
|
|
6909
7011
|
}
|
|
@@ -6911,11 +7013,11 @@ function validateMember(state, value, path30, location) {
|
|
|
6911
7013
|
if (!Array.isArray(value.roles)) {
|
|
6912
7014
|
addIssue(state, "error", "INVALID_ROLES", "Expected roles to be an array.", {
|
|
6913
7015
|
...location,
|
|
6914
|
-
path: `${
|
|
7016
|
+
path: `${path31}.roles`
|
|
6915
7017
|
});
|
|
6916
7018
|
} else {
|
|
6917
7019
|
value.roles.forEach((role, index) => {
|
|
6918
|
-
const rolePath = `${
|
|
7020
|
+
const rolePath = `${path31}.roles[${index}]`;
|
|
6919
7021
|
if (!isRecord(role)) {
|
|
6920
7022
|
addIssue(state, "error", "INVALID_ROLE", "Expected a role object.", {
|
|
6921
7023
|
...location,
|
|
@@ -6929,25 +7031,25 @@ function validateMember(state, value, path30, location) {
|
|
|
6929
7031
|
}
|
|
6930
7032
|
}
|
|
6931
7033
|
}
|
|
6932
|
-
function validateMessage(state, value,
|
|
7034
|
+
function validateMessage(state, value, path31, location) {
|
|
6933
7035
|
state.stats.messageCount++;
|
|
6934
7036
|
if (!isRecord(value)) {
|
|
6935
|
-
addIssue(state, "error", "INVALID_MESSAGE", "Expected a message object.", childLocation(location,
|
|
7037
|
+
addIssue(state, "error", "INVALID_MESSAGE", "Expected a message object.", childLocation(location, path31));
|
|
6936
7038
|
return;
|
|
6937
7039
|
}
|
|
6938
|
-
const sender = validateRequiredString(state, value.sender, `${
|
|
7040
|
+
const sender = validateRequiredString(state, value.sender, `${path31}.sender`, location) ? value.sender : void 0;
|
|
6939
7041
|
if (sender !== void 0) {
|
|
6940
7042
|
state.senderIds.add(sender);
|
|
6941
7043
|
}
|
|
6942
|
-
validateRequiredString(state, value.accountName, `${
|
|
6943
|
-
if (validateUnixSeconds(state, value.timestamp, `${
|
|
7044
|
+
validateRequiredString(state, value.accountName, `${path31}.accountName`, location);
|
|
7045
|
+
if (validateUnixSeconds(state, value.timestamp, `${path31}.timestamp`, location)) {
|
|
6944
7046
|
if (state.lastTimestamp !== void 0 && value.timestamp < state.lastTimestamp) {
|
|
6945
7047
|
addIssue(
|
|
6946
7048
|
state,
|
|
6947
7049
|
"warning",
|
|
6948
7050
|
"OUT_OF_ORDER_TIMESTAMP",
|
|
6949
7051
|
"Messages should be ordered by timestamp ascending.",
|
|
6950
|
-
childLocation(location, `${
|
|
7052
|
+
childLocation(location, `${path31}.timestamp`)
|
|
6951
7053
|
);
|
|
6952
7054
|
}
|
|
6953
7055
|
state.lastTimestamp = value.timestamp;
|
|
@@ -6956,7 +7058,7 @@ function validateMessage(state, value, path30, location) {
|
|
|
6956
7058
|
if (messageType === void 0) {
|
|
6957
7059
|
addIssue(state, "error", "INVALID_MESSAGE_TYPE", "Expected a supported numeric ChatLab message type.", {
|
|
6958
7060
|
...location,
|
|
6959
|
-
path: `${
|
|
7061
|
+
path: `${path31}.type`
|
|
6960
7062
|
});
|
|
6961
7063
|
} else if (sender === SYSTEM_SENDER_ID && !SYSTEM_SENDER_MESSAGE_TYPES.has(messageType)) {
|
|
6962
7064
|
addIssue(
|
|
@@ -6964,22 +7066,22 @@ function validateMessage(state, value, path30, location) {
|
|
|
6964
7066
|
"error",
|
|
6965
7067
|
"INVALID_SYSTEM_SENDER_TYPE",
|
|
6966
7068
|
"The reserved SYSTEM sender is only valid for system and recall messages.",
|
|
6967
|
-
childLocation(location, `${
|
|
7069
|
+
childLocation(location, `${path31}.sender`)
|
|
6968
7070
|
);
|
|
6969
7071
|
}
|
|
6970
7072
|
if (!hasOwn(value, "content") || typeof value.content !== "string" && value.content !== null) {
|
|
6971
7073
|
addIssue(state, "error", "INVALID_MESSAGE_CONTENT", "Expected content to be a string or null.", {
|
|
6972
7074
|
...location,
|
|
6973
|
-
path: `${
|
|
7075
|
+
path: `${path31}.content`
|
|
6974
7076
|
});
|
|
6975
7077
|
}
|
|
6976
|
-
validateOptionalString(state, value, "groupNickname", `${
|
|
7078
|
+
validateOptionalString(state, value, "groupNickname", `${path31}.groupNickname`, location);
|
|
6977
7079
|
if (hasOwn(value, "platformMessageId")) {
|
|
6978
|
-
if (validateRequiredString(state, value.platformMessageId, `${
|
|
7080
|
+
if (validateRequiredString(state, value.platformMessageId, `${path31}.platformMessageId`, location)) {
|
|
6979
7081
|
if (state.messageIds.has(value.platformMessageId)) {
|
|
6980
7082
|
addIssue(state, "error", "DUPLICATE_MESSAGE_ID", "platformMessageId must be unique.", {
|
|
6981
7083
|
...location,
|
|
6982
|
-
path: `${
|
|
7084
|
+
path: `${path31}.platformMessageId`
|
|
6983
7085
|
});
|
|
6984
7086
|
} else {
|
|
6985
7087
|
state.messageIds.add(value.platformMessageId);
|
|
@@ -6987,7 +7089,7 @@ function validateMessage(state, value, path30, location) {
|
|
|
6987
7089
|
}
|
|
6988
7090
|
}
|
|
6989
7091
|
if (hasOwn(value, "replyToMessageId")) {
|
|
6990
|
-
if (validateRequiredString(state, value.replyToMessageId, `${
|
|
7092
|
+
if (validateRequiredString(state, value.replyToMessageId, `${path31}.replyToMessageId`, location)) {
|
|
6991
7093
|
state.replyTargetIds.add(value.replyToMessageId);
|
|
6992
7094
|
}
|
|
6993
7095
|
}
|
|
@@ -7438,95 +7540,113 @@ var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
|
7438
7540
|
LogLevel2["INFO"] = "INFO";
|
|
7439
7541
|
return LogLevel2;
|
|
7440
7542
|
})(LogLevel || {});
|
|
7441
|
-
|
|
7442
|
-
|
|
7443
|
-
|
|
7444
|
-
|
|
7445
|
-
|
|
7446
|
-
|
|
7447
|
-
if (!
|
|
7448
|
-
|
|
7543
|
+
function createImportPerfLogger(logDir) {
|
|
7544
|
+
let lastLogTime = Date.now();
|
|
7545
|
+
let lastMessageCount = 0;
|
|
7546
|
+
let currentLogFile = null;
|
|
7547
|
+
let errorCount = 0;
|
|
7548
|
+
const append = (line) => {
|
|
7549
|
+
if (!currentLogFile) return;
|
|
7550
|
+
try {
|
|
7551
|
+
fs19.appendFileSync(currentLogFile, line, "utf-8");
|
|
7552
|
+
} catch {
|
|
7449
7553
|
}
|
|
7450
|
-
|
|
7451
|
-
|
|
7554
|
+
};
|
|
7555
|
+
const writeLogLine = (level, message) => {
|
|
7556
|
+
append(`[${(/* @__PURE__ */ new Date()).toISOString()}] [${level}] ${message}
|
|
7557
|
+
`);
|
|
7558
|
+
};
|
|
7559
|
+
return {
|
|
7560
|
+
init(sessionId) {
|
|
7561
|
+
try {
|
|
7562
|
+
if (!fs19.existsSync(logDir)) fs19.mkdirSync(logDir, { recursive: true });
|
|
7563
|
+
currentLogFile = path12.join(logDir, `import_${sessionId}_${Date.now()}.log`);
|
|
7564
|
+
fs19.writeFileSync(currentLogFile, `=== Import Log ===
|
|
7452
7565
|
Start time: ${(/* @__PURE__ */ new Date()).toISOString()}
|
|
7453
7566
|
|
|
7454
7567
|
`, "utf-8");
|
|
7455
|
-
|
|
7456
|
-
|
|
7568
|
+
} catch {
|
|
7569
|
+
currentLogFile = null;
|
|
7570
|
+
}
|
|
7571
|
+
},
|
|
7572
|
+
perf(event, messagesProcessed, batchSize) {
|
|
7573
|
+
const now = Date.now();
|
|
7574
|
+
const duration = now - lastLogTime;
|
|
7575
|
+
const messagesDelta = messagesProcessed - lastMessageCount;
|
|
7576
|
+
const speed = duration > 0 ? Math.round(messagesDelta / duration * 1e3) : 0;
|
|
7577
|
+
let memory = 0;
|
|
7578
|
+
try {
|
|
7579
|
+
memory = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
|
|
7580
|
+
} catch {
|
|
7581
|
+
}
|
|
7582
|
+
append(
|
|
7583
|
+
`[${(/* @__PURE__ */ new Date()).toISOString()}] ${event} | messages: ${messagesProcessed.toLocaleString()} | elapsed: ${duration}ms | speed: ${speed.toLocaleString()}/s | memory: ${memory}MB` + (batchSize ? ` | batch: ${batchSize}` : "") + "\n"
|
|
7584
|
+
);
|
|
7585
|
+
lastLogTime = now;
|
|
7586
|
+
lastMessageCount = messagesProcessed;
|
|
7587
|
+
},
|
|
7588
|
+
perfDetail(detail) {
|
|
7589
|
+
append(` ${detail}
|
|
7590
|
+
`);
|
|
7591
|
+
},
|
|
7592
|
+
reset() {
|
|
7593
|
+
lastLogTime = Date.now();
|
|
7594
|
+
lastMessageCount = 0;
|
|
7595
|
+
currentLogFile = null;
|
|
7596
|
+
errorCount = 0;
|
|
7597
|
+
},
|
|
7598
|
+
getCurrentLogFile() {
|
|
7599
|
+
return currentLogFile;
|
|
7600
|
+
},
|
|
7601
|
+
error(message, error) {
|
|
7602
|
+
errorCount++;
|
|
7603
|
+
writeLogLine("ERROR" /* ERROR */, `${message}${error ? `: ${error.message}` : ""}`);
|
|
7604
|
+
},
|
|
7605
|
+
info(message) {
|
|
7606
|
+
writeLogLine("INFO" /* INFO */, message);
|
|
7607
|
+
},
|
|
7608
|
+
getErrorCount() {
|
|
7609
|
+
return errorCount;
|
|
7610
|
+
},
|
|
7611
|
+
summary(totalMessages, totalMembers) {
|
|
7612
|
+
append(`
|
|
7613
|
+
=== Import Summary ===
|
|
7614
|
+
End time: ${(/* @__PURE__ */ new Date()).toISOString()}
|
|
7615
|
+
Total messages: ${totalMessages.toLocaleString()}
|
|
7616
|
+
Total members: ${totalMembers.toLocaleString()}
|
|
7617
|
+
Errors: ${errorCount}
|
|
7618
|
+
`);
|
|
7619
|
+
}
|
|
7620
|
+
};
|
|
7621
|
+
}
|
|
7622
|
+
var defaultLogger = createImportPerfLogger("");
|
|
7623
|
+
function initPerfLog(sessionId, logDir) {
|
|
7624
|
+
defaultLogger = createImportPerfLogger(logDir);
|
|
7625
|
+
defaultLogger.init(sessionId);
|
|
7457
7626
|
}
|
|
7458
7627
|
function logPerf(event, messagesProcessed, batchSize) {
|
|
7459
|
-
|
|
7460
|
-
const duration = now - lastLogTime;
|
|
7461
|
-
const messagesDelta = messagesProcessed - lastMessageCount;
|
|
7462
|
-
const speed = duration > 0 ? Math.round(messagesDelta / duration * 1e3) : 0;
|
|
7463
|
-
let memory = 0;
|
|
7464
|
-
try {
|
|
7465
|
-
const used = process.memoryUsage();
|
|
7466
|
-
memory = Math.round(used.heapUsed / 1024 / 1024);
|
|
7467
|
-
} catch {
|
|
7468
|
-
}
|
|
7469
|
-
const logLine = `[${(/* @__PURE__ */ new Date()).toISOString()}] ${event} | messages: ${messagesProcessed.toLocaleString()} | elapsed: ${duration}ms | speed: ${speed.toLocaleString()}/s | memory: ${memory}MB` + (batchSize ? ` | batch: ${batchSize}` : "") + "\n";
|
|
7470
|
-
if (currentLogFile) {
|
|
7471
|
-
try {
|
|
7472
|
-
fs19.appendFileSync(currentLogFile, logLine, "utf-8");
|
|
7473
|
-
} catch {
|
|
7474
|
-
}
|
|
7475
|
-
}
|
|
7476
|
-
lastLogTime = now;
|
|
7477
|
-
lastMessageCount = messagesProcessed;
|
|
7628
|
+
defaultLogger.perf(event, messagesProcessed, batchSize);
|
|
7478
7629
|
}
|
|
7479
7630
|
function logPerfDetail(detail) {
|
|
7480
|
-
|
|
7481
|
-
try {
|
|
7482
|
-
fs19.appendFileSync(currentLogFile, ` ${detail}
|
|
7483
|
-
`, "utf-8");
|
|
7484
|
-
} catch {
|
|
7485
|
-
}
|
|
7486
|
-
}
|
|
7631
|
+
defaultLogger.perfDetail(detail);
|
|
7487
7632
|
}
|
|
7488
7633
|
function resetPerfLog() {
|
|
7489
|
-
|
|
7490
|
-
lastMessageCount = 0;
|
|
7491
|
-
currentLogFile = null;
|
|
7492
|
-
errorCount = 0;
|
|
7634
|
+
defaultLogger.reset();
|
|
7493
7635
|
}
|
|
7494
7636
|
function getCurrentLogFile() {
|
|
7495
|
-
return
|
|
7496
|
-
}
|
|
7497
|
-
function writeLogLine(level, message) {
|
|
7498
|
-
if (!currentLogFile) return;
|
|
7499
|
-
const logLine = `[${(/* @__PURE__ */ new Date()).toISOString()}] [${level}] ${message}
|
|
7500
|
-
`;
|
|
7501
|
-
try {
|
|
7502
|
-
fs19.appendFileSync(currentLogFile, logLine, "utf-8");
|
|
7503
|
-
} catch {
|
|
7504
|
-
}
|
|
7637
|
+
return defaultLogger.getCurrentLogFile();
|
|
7505
7638
|
}
|
|
7506
7639
|
function logError(message, error) {
|
|
7507
|
-
|
|
7508
|
-
const errorDetail = error ? `: ${error.message}` : "";
|
|
7509
|
-
writeLogLine("ERROR" /* ERROR */, `${message}${errorDetail}`);
|
|
7640
|
+
defaultLogger.error(message, error);
|
|
7510
7641
|
}
|
|
7511
7642
|
function logInfo(message) {
|
|
7512
|
-
|
|
7643
|
+
defaultLogger.info(message);
|
|
7513
7644
|
}
|
|
7514
7645
|
function getErrorCount() {
|
|
7515
|
-
return
|
|
7646
|
+
return defaultLogger.getErrorCount();
|
|
7516
7647
|
}
|
|
7517
7648
|
function logSummary(totalMessages, totalMembers) {
|
|
7518
|
-
|
|
7519
|
-
const summary = `
|
|
7520
|
-
=== Import Summary ===
|
|
7521
|
-
End time: ${(/* @__PURE__ */ new Date()).toISOString()}
|
|
7522
|
-
Total messages: ${totalMessages.toLocaleString()}
|
|
7523
|
-
Total members: ${totalMembers.toLocaleString()}
|
|
7524
|
-
Errors: ${errorCount}
|
|
7525
|
-
`;
|
|
7526
|
-
try {
|
|
7527
|
-
fs19.appendFileSync(currentLogFile, summary, "utf-8");
|
|
7528
|
-
} catch {
|
|
7529
|
-
}
|
|
7649
|
+
defaultLogger.summary(totalMessages, totalMembers);
|
|
7530
7650
|
}
|
|
7531
7651
|
|
|
7532
7652
|
// ../../packages/node-runtime/src/nlp/word-frequency.ts
|
|
@@ -7769,16 +7889,27 @@ function createFtsTable(db) {
|
|
|
7769
7889
|
function buildFtsIndex(db) {
|
|
7770
7890
|
createFtsTable(db);
|
|
7771
7891
|
const insertFts = db.prepare("INSERT INTO message_fts(rowid, content) VALUES (?, ?)");
|
|
7772
|
-
const countRow = db.prepare("SELECT COUNT(*) as total FROM message WHERE type = 0 AND content IS NOT NULL AND content != ''").get();
|
|
7773
|
-
const total = countRow?.total ?? 0;
|
|
7774
7892
|
let indexed = 0;
|
|
7775
|
-
let
|
|
7776
|
-
|
|
7777
|
-
|
|
7778
|
-
|
|
7779
|
-
|
|
7780
|
-
|
|
7781
|
-
|
|
7893
|
+
let lastId = null;
|
|
7894
|
+
const selectFirstBatch = db.prepare(
|
|
7895
|
+
`SELECT id, content FROM message
|
|
7896
|
+
WHERE type = 0
|
|
7897
|
+
AND content IS NOT NULL
|
|
7898
|
+
AND content != ''
|
|
7899
|
+
ORDER BY id ASC
|
|
7900
|
+
LIMIT ?`
|
|
7901
|
+
);
|
|
7902
|
+
const selectNextBatch = db.prepare(
|
|
7903
|
+
`SELECT id, content FROM message
|
|
7904
|
+
WHERE type = 0
|
|
7905
|
+
AND content IS NOT NULL
|
|
7906
|
+
AND content != ''
|
|
7907
|
+
AND id > ?
|
|
7908
|
+
ORDER BY id ASC
|
|
7909
|
+
LIMIT ?`
|
|
7910
|
+
);
|
|
7911
|
+
while (true) {
|
|
7912
|
+
const rows = lastId === null ? selectFirstBatch.all(BATCH_SIZE) : selectNextBatch.all(lastId, BATCH_SIZE);
|
|
7782
7913
|
if (rows.length === 0) break;
|
|
7783
7914
|
db.transaction(() => {
|
|
7784
7915
|
for (const row of rows) {
|
|
@@ -7789,7 +7920,7 @@ function buildFtsIndex(db) {
|
|
|
7789
7920
|
}
|
|
7790
7921
|
});
|
|
7791
7922
|
indexed += rows.length;
|
|
7792
|
-
|
|
7923
|
+
lastId = rows[rows.length - 1].id;
|
|
7793
7924
|
}
|
|
7794
7925
|
return { indexed };
|
|
7795
7926
|
}
|
|
@@ -7831,8 +7962,62 @@ function searchByFts(db, keywords, limit = 1e3, offset = 0) {
|
|
|
7831
7962
|
}
|
|
7832
7963
|
}
|
|
7833
7964
|
|
|
7965
|
+
// ../../packages/node-runtime/src/import/message-batch-inserter.ts
|
|
7966
|
+
var MESSAGE_COLUMN_COUNT = 8;
|
|
7967
|
+
var SQLITE_LEGACY_VARIABLE_LIMIT = 999;
|
|
7968
|
+
var MESSAGE_INSERT_MAX_ROWS = Math.floor(SQLITE_LEGACY_VARIABLE_LIMIT / MESSAGE_COLUMN_COUNT);
|
|
7969
|
+
var MessageBatchInserter = class {
|
|
7970
|
+
constructor(db) {
|
|
7971
|
+
this.db = db;
|
|
7972
|
+
}
|
|
7973
|
+
db;
|
|
7974
|
+
statementCache = /* @__PURE__ */ new Map();
|
|
7975
|
+
insert(rows) {
|
|
7976
|
+
let statementCount = 0;
|
|
7977
|
+
for (let offset = 0; offset < rows.length; offset += MESSAGE_INSERT_MAX_ROWS) {
|
|
7978
|
+
const batch = rows.slice(offset, offset + MESSAGE_INSERT_MAX_ROWS);
|
|
7979
|
+
const statement = this.getStatement(batch.length);
|
|
7980
|
+
statement.run(...batch.flatMap(toParams));
|
|
7981
|
+
statementCount++;
|
|
7982
|
+
}
|
|
7983
|
+
return statementCount;
|
|
7984
|
+
}
|
|
7985
|
+
getStatement(rowCount) {
|
|
7986
|
+
const cached = this.statementCache.get(rowCount);
|
|
7987
|
+
if (cached) return cached;
|
|
7988
|
+
const values = Array.from({ length: rowCount }, () => "(?, ?, ?, ?, ?, ?, ?, ?)").join(", ");
|
|
7989
|
+
const statement = this.db.prepare(
|
|
7990
|
+
`INSERT INTO message (
|
|
7991
|
+
sender_id,
|
|
7992
|
+
sender_account_name,
|
|
7993
|
+
sender_group_nickname,
|
|
7994
|
+
ts,
|
|
7995
|
+
type,
|
|
7996
|
+
content,
|
|
7997
|
+
reply_to_message_id,
|
|
7998
|
+
platform_message_id
|
|
7999
|
+
) VALUES ${values}`
|
|
8000
|
+
);
|
|
8001
|
+
this.statementCache.set(rowCount, statement);
|
|
8002
|
+
return statement;
|
|
8003
|
+
}
|
|
8004
|
+
};
|
|
8005
|
+
function toParams(row) {
|
|
8006
|
+
return [
|
|
8007
|
+
row.senderId,
|
|
8008
|
+
row.senderAccountName,
|
|
8009
|
+
row.senderGroupNickname,
|
|
8010
|
+
row.timestamp,
|
|
8011
|
+
row.type,
|
|
8012
|
+
row.content,
|
|
8013
|
+
row.replyToMessageId,
|
|
8014
|
+
row.platformMessageId
|
|
8015
|
+
];
|
|
8016
|
+
}
|
|
8017
|
+
|
|
7834
8018
|
// ../../packages/node-runtime/src/import/streaming-importer.ts
|
|
7835
8019
|
import * as fs21 from "fs";
|
|
8020
|
+
import { performance } from "perf_hooks";
|
|
7836
8021
|
|
|
7837
8022
|
// ../../packages/node-runtime/src/import/message-deduplicator.ts
|
|
7838
8023
|
function generateFallbackMessageKey(message) {
|
|
@@ -7867,6 +8052,10 @@ var CHECKPOINT_INTERVAL = 2e5;
|
|
|
7867
8052
|
var SYSTEM_SENDER_ID2 = "SYSTEM";
|
|
7868
8053
|
var SYSTEM_MEMBER_NAME = "\u7CFB\u7EDF\u6D88\u606F";
|
|
7869
8054
|
var RESERVED_SYSTEM_SENDER_FORMATS = /* @__PURE__ */ new Set(["chatlab", "chatlab-jsonl"]);
|
|
8055
|
+
var defaultNow = () => performance.now();
|
|
8056
|
+
function elapsedMs(startedAt, now) {
|
|
8057
|
+
return now() - startedAt;
|
|
8058
|
+
}
|
|
7870
8059
|
var yieldToEventLoop = () => new Promise((resolve4) => setImmediate(resolve4));
|
|
7871
8060
|
function defaultGenerateSessionId() {
|
|
7872
8061
|
const ts = Date.now();
|
|
@@ -7899,28 +8088,38 @@ function prepareMessageForCreate(message, senderAccountName = message.senderAcco
|
|
|
7899
8088
|
};
|
|
7900
8089
|
}
|
|
7901
8090
|
async function streamingImport(filePath, deps, formatOptions, externalSessionId) {
|
|
8091
|
+
const now = deps.now ?? defaultNow;
|
|
8092
|
+
const totalStartedAt = now();
|
|
8093
|
+
const detectionStartedAt = now();
|
|
7902
8094
|
if (formatOptions?.formatId) {
|
|
7903
8095
|
const formatId = formatOptions.formatId;
|
|
7904
8096
|
const feature15 = getFormatFeatureById(formatId);
|
|
7905
8097
|
if (!feature15) {
|
|
7906
8098
|
return { success: false, error: "error.unknown_format_id" };
|
|
7907
8099
|
}
|
|
7908
|
-
return streamImportSingle(filePath, deps, feature15, formatOptions, externalSessionId
|
|
8100
|
+
return streamImportSingle(filePath, deps, feature15, formatOptions, externalSessionId, {
|
|
8101
|
+
totalStartedAt,
|
|
8102
|
+
detectionMs: elapsedMs(detectionStartedAt, now)
|
|
8103
|
+
});
|
|
7909
8104
|
}
|
|
7910
8105
|
const candidates = detectAllFormats(filePath);
|
|
7911
8106
|
if (candidates.length === 0) {
|
|
7912
8107
|
return { success: false, error: "error.unrecognized_format" };
|
|
7913
8108
|
}
|
|
8109
|
+
const timingContext = {
|
|
8110
|
+
totalStartedAt,
|
|
8111
|
+
detectionMs: elapsedMs(detectionStartedAt, now)
|
|
8112
|
+
};
|
|
7914
8113
|
if (candidates.length > 1) {
|
|
7915
|
-
return streamImportWithFallback(filePath, deps, candidates, formatOptions, externalSessionId);
|
|
8114
|
+
return streamImportWithFallback(filePath, deps, candidates, formatOptions, externalSessionId, timingContext);
|
|
7916
8115
|
}
|
|
7917
|
-
return streamImportSingle(filePath, deps, candidates[0], formatOptions, externalSessionId);
|
|
8116
|
+
return streamImportSingle(filePath, deps, candidates[0], formatOptions, externalSessionId, timingContext);
|
|
7918
8117
|
}
|
|
7919
|
-
async function streamImportWithFallback(filePath, deps, candidates, formatOptions, externalSessionId) {
|
|
8118
|
+
async function streamImportWithFallback(filePath, deps, candidates, formatOptions, externalSessionId, timingContext) {
|
|
7920
8119
|
for (let i = 0; i < candidates.length; i++) {
|
|
7921
8120
|
const candidate = candidates[i];
|
|
7922
8121
|
deps.logger?.info(`[StreamImport] Trying format ${i + 1}/${candidates.length}: ${candidate.name} (${candidate.id})`);
|
|
7923
|
-
const result = await streamImportSingle(filePath, deps, candidate, formatOptions, externalSessionId);
|
|
8122
|
+
const result = await streamImportSingle(filePath, deps, candidate, formatOptions, externalSessionId, timingContext);
|
|
7924
8123
|
if (result.success) {
|
|
7925
8124
|
if (i > 0) {
|
|
7926
8125
|
deps.logger?.info(
|
|
@@ -7934,9 +8133,32 @@ async function streamImportWithFallback(filePath, deps, candidates, formatOption
|
|
|
7934
8133
|
}
|
|
7935
8134
|
return { success: false, error: "error.no_messages" };
|
|
7936
8135
|
}
|
|
7937
|
-
async function streamImportSingle(filePath, deps, formatFeature, formatOptions, externalSessionId) {
|
|
8136
|
+
async function streamImportSingle(filePath, deps, formatFeature, formatOptions, externalSessionId, timingContext) {
|
|
7938
8137
|
const { onProgress, logger } = deps;
|
|
7939
8138
|
const genId = deps.generateSessionId ?? defaultGenerateSessionId;
|
|
8139
|
+
const now = deps.now ?? defaultNow;
|
|
8140
|
+
const totalStartedAt = timingContext?.totalStartedAt ?? now();
|
|
8141
|
+
const timings = {
|
|
8142
|
+
detectionMs: timingContext?.detectionMs ?? 0,
|
|
8143
|
+
preprocessingMs: 0,
|
|
8144
|
+
databaseSetupMs: 0,
|
|
8145
|
+
parserMs: 0,
|
|
8146
|
+
metaWriteMs: 0,
|
|
8147
|
+
memberWriteMs: 0,
|
|
8148
|
+
messageWriteMs: 0,
|
|
8149
|
+
nicknameHistoryMs: 0,
|
|
8150
|
+
indexCreationMs: 0,
|
|
8151
|
+
ftsMs: 0,
|
|
8152
|
+
checkpointMs: 0,
|
|
8153
|
+
sessionIndexMs: 0,
|
|
8154
|
+
postImportHookMs: 0,
|
|
8155
|
+
totalMs: 0
|
|
8156
|
+
};
|
|
8157
|
+
const rssStartBytes = process.memoryUsage().rss;
|
|
8158
|
+
let rssPeakBytes = rssStartBytes;
|
|
8159
|
+
const sampleRss = () => {
|
|
8160
|
+
rssPeakBytes = Math.max(rssPeakBytes, process.memoryUsage().rss);
|
|
8161
|
+
};
|
|
7940
8162
|
logger?.reset();
|
|
7941
8163
|
const sessionId = externalSessionId || genId();
|
|
7942
8164
|
logger?.init(sessionId);
|
|
@@ -7946,8 +8168,18 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
7946
8168
|
logger?.perf("Import started", 0);
|
|
7947
8169
|
let actualFilePath = filePath;
|
|
7948
8170
|
let tempFilePath = null;
|
|
7949
|
-
const
|
|
7950
|
-
|
|
8171
|
+
const preprocessing = deps.preprocessing ?? { getPreprocessor, needsPreprocess, isNativeFormatAvailable };
|
|
8172
|
+
const getImportPreprocessor = preprocessing.getPreprocessor;
|
|
8173
|
+
const shouldPreprocess = preprocessing.needsPreprocess;
|
|
8174
|
+
const nativeFormatAvailable = preprocessing.isNativeFormatAvailable;
|
|
8175
|
+
const preprocessor = getImportPreprocessor(filePath);
|
|
8176
|
+
const needsLargeFilePreprocess = preprocessor && shouldPreprocess(filePath);
|
|
8177
|
+
const nativeCanParseOriginal = needsLargeFilePreprocess && nativeFormatAvailable(formatFeature.id);
|
|
8178
|
+
if (nativeCanParseOriginal) {
|
|
8179
|
+
logger?.info(
|
|
8180
|
+
`[NativeParser] Kernel ${formatFeature.id} is available; skipping large-file preprocessing and parsing the original export`
|
|
8181
|
+
);
|
|
8182
|
+
} else if (needsLargeFilePreprocess) {
|
|
7951
8183
|
logger?.info("File needs preprocessing, simplifying large file...");
|
|
7952
8184
|
onProgress({
|
|
7953
8185
|
stage: "parsing",
|
|
@@ -7957,6 +8189,7 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
7957
8189
|
percentage: 0,
|
|
7958
8190
|
message: ""
|
|
7959
8191
|
});
|
|
8192
|
+
const preprocessingStartedAt = now();
|
|
7960
8193
|
try {
|
|
7961
8194
|
tempFilePath = await preprocessor.preprocess(filePath, (progress) => {
|
|
7962
8195
|
onProgress({ ...progress, message: "" });
|
|
@@ -7967,45 +8200,100 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
7967
8200
|
const errorMsg = `Preprocessing failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
7968
8201
|
logger?.error(errorMsg, err instanceof Error ? err : void 0);
|
|
7969
8202
|
return { success: false, error: errorMsg };
|
|
8203
|
+
} finally {
|
|
8204
|
+
timings.preprocessingMs = elapsedMs(preprocessingStartedAt, now);
|
|
8205
|
+
sampleRss();
|
|
7970
8206
|
}
|
|
7971
8207
|
}
|
|
7972
|
-
const
|
|
7973
|
-
const
|
|
7974
|
-
|
|
7975
|
-
|
|
7976
|
-
|
|
7977
|
-
|
|
7978
|
-
|
|
7979
|
-
|
|
7980
|
-
|
|
7981
|
-
|
|
7982
|
-
|
|
7983
|
-
|
|
7984
|
-
|
|
7985
|
-
|
|
7986
|
-
|
|
7987
|
-
|
|
7988
|
-
|
|
7989
|
-
|
|
7990
|
-
|
|
7991
|
-
|
|
7992
|
-
|
|
7993
|
-
|
|
7994
|
-
|
|
7995
|
-
|
|
8208
|
+
const databaseSetupStartedAt = now();
|
|
8209
|
+
const databaseSetup = (() => {
|
|
8210
|
+
let db2;
|
|
8211
|
+
try {
|
|
8212
|
+
db2 = deps.openDatabase(sessionId);
|
|
8213
|
+
return {
|
|
8214
|
+
ok: true,
|
|
8215
|
+
db: db2,
|
|
8216
|
+
insertMeta: db2.prepare(
|
|
8217
|
+
`INSERT INTO meta (name, platform, type, imported_at, group_id, group_avatar, owner_id) VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
8218
|
+
),
|
|
8219
|
+
insertMember: db2.prepare(
|
|
8220
|
+
`INSERT INTO member (platform_id, account_name, group_nickname, aliases, avatar, roles)
|
|
8221
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
8222
|
+
ON CONFLICT(platform_id) DO UPDATE SET
|
|
8223
|
+
account_name = COALESCE(NULLIF(excluded.account_name, ''), account_name),
|
|
8224
|
+
group_nickname = COALESCE(NULLIF(excluded.group_nickname, ''), group_nickname),
|
|
8225
|
+
aliases = CASE WHEN excluded.aliases != '[]' THEN excluded.aliases ELSE aliases END,
|
|
8226
|
+
avatar = COALESCE(NULLIF(excluded.avatar, ''), avatar),
|
|
8227
|
+
roles = CASE WHEN excluded.roles != '[]' THEN excluded.roles ELSE roles END`
|
|
8228
|
+
),
|
|
8229
|
+
getMemberId: db2.prepare(`SELECT id FROM member WHERE platform_id = ?`),
|
|
8230
|
+
insertNameHistory: db2.prepare(
|
|
8231
|
+
`INSERT INTO member_name_history (member_id, name_type, name, start_ts, end_ts) VALUES (?, ?, ?, ?, ?)`
|
|
8232
|
+
),
|
|
8233
|
+
updateMemberAccountName: db2.prepare(`UPDATE member SET account_name = ? WHERE platform_id = ?`),
|
|
8234
|
+
updateMemberGroupNickname: db2.prepare(`UPDATE member SET group_nickname = ? WHERE platform_id = ?`)
|
|
8235
|
+
};
|
|
8236
|
+
} catch (error) {
|
|
8237
|
+
try {
|
|
8238
|
+
db2?.close();
|
|
8239
|
+
} catch {
|
|
8240
|
+
}
|
|
8241
|
+
return { ok: false, error, databaseOpened: db2 !== void 0 };
|
|
8242
|
+
}
|
|
8243
|
+
})();
|
|
8244
|
+
timings.databaseSetupMs = elapsedMs(databaseSetupStartedAt, now);
|
|
8245
|
+
sampleRss();
|
|
8246
|
+
if (!databaseSetup.ok) {
|
|
8247
|
+
if (tempFilePath && preprocessor) preprocessor.cleanup(tempFilePath);
|
|
8248
|
+
if (databaseSetup.databaseOpened) deps.deleteDatabase(sessionId);
|
|
8249
|
+
const error = databaseSetup.error;
|
|
8250
|
+
logger?.error("Import failed during database setup", error instanceof Error ? error : void 0);
|
|
8251
|
+
throw error;
|
|
8252
|
+
}
|
|
8253
|
+
const {
|
|
8254
|
+
db,
|
|
8255
|
+
insertMeta,
|
|
8256
|
+
insertMember,
|
|
8257
|
+
getMemberId,
|
|
8258
|
+
insertNameHistory,
|
|
8259
|
+
updateMemberAccountName,
|
|
8260
|
+
updateMemberGroupNickname
|
|
8261
|
+
} = databaseSetup;
|
|
7996
8262
|
const memberIdMap = /* @__PURE__ */ new Map();
|
|
7997
8263
|
const accountNameTracker = /* @__PURE__ */ new Map();
|
|
7998
8264
|
const groupNicknameTracker = /* @__PURE__ */ new Map();
|
|
7999
8265
|
let metaInserted = false;
|
|
8266
|
+
let importedPlatform = formatFeature.platform;
|
|
8000
8267
|
let messageCountInBatch = 0;
|
|
8001
8268
|
let totalMessageCount = 0;
|
|
8002
8269
|
let duplicateCount = 0;
|
|
8003
8270
|
let lastCheckpointCount = 0;
|
|
8004
8271
|
let inTransaction = false;
|
|
8272
|
+
let messageTransactionCount = 0;
|
|
8273
|
+
let messageTransactionMs = 0;
|
|
8274
|
+
let messageInsertStatementCount = 0;
|
|
8275
|
+
const messageBatchInserter = new MessageBatchInserter(db);
|
|
8005
8276
|
const beginTransaction = () => {
|
|
8006
8277
|
if (!inTransaction) {
|
|
8007
|
-
|
|
8008
|
-
|
|
8278
|
+
const startedAt = now();
|
|
8279
|
+
try {
|
|
8280
|
+
db.exec("BEGIN TRANSACTION");
|
|
8281
|
+
inTransaction = true;
|
|
8282
|
+
messageTransactionCount++;
|
|
8283
|
+
} finally {
|
|
8284
|
+
messageTransactionMs += elapsedMs(startedAt, now);
|
|
8285
|
+
}
|
|
8286
|
+
}
|
|
8287
|
+
};
|
|
8288
|
+
const commitTransaction = () => {
|
|
8289
|
+
if (inTransaction) {
|
|
8290
|
+
const startedAt = now();
|
|
8291
|
+
try {
|
|
8292
|
+
db.exec("COMMIT");
|
|
8293
|
+
inTransaction = false;
|
|
8294
|
+
} finally {
|
|
8295
|
+
messageTransactionMs += elapsedMs(startedAt, now);
|
|
8296
|
+
}
|
|
8009
8297
|
}
|
|
8010
8298
|
};
|
|
8011
8299
|
const doCheckpoint = () => {
|
|
@@ -8014,13 +8302,20 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8014
8302
|
} catch {
|
|
8015
8303
|
}
|
|
8016
8304
|
};
|
|
8305
|
+
const measureCheckpoint = () => {
|
|
8306
|
+
const startedAt = now();
|
|
8307
|
+
try {
|
|
8308
|
+
doCheckpoint();
|
|
8309
|
+
} finally {
|
|
8310
|
+
timings.checkpointMs += elapsedMs(startedAt, now);
|
|
8311
|
+
}
|
|
8312
|
+
};
|
|
8017
8313
|
const commitAndBeginNew = () => {
|
|
8018
8314
|
if (inTransaction) {
|
|
8019
|
-
|
|
8020
|
-
inTransaction = false;
|
|
8315
|
+
commitTransaction();
|
|
8021
8316
|
logger?.perf("Commit transaction", totalMessageCount, BATCH_COMMIT_SIZE);
|
|
8022
8317
|
if (totalMessageCount - lastCheckpointCount >= CHECKPOINT_INTERVAL) {
|
|
8023
|
-
|
|
8318
|
+
measureCheckpoint();
|
|
8024
8319
|
logger?.perf("WAL checkpoint", totalMessageCount);
|
|
8025
8320
|
lastCheckpointCount = totalMessageCount;
|
|
8026
8321
|
}
|
|
@@ -8035,7 +8330,6 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8035
8330
|
}
|
|
8036
8331
|
beginTransaction();
|
|
8037
8332
|
};
|
|
8038
|
-
beginTransaction();
|
|
8039
8333
|
let shouldDeleteDb = false;
|
|
8040
8334
|
let importError = null;
|
|
8041
8335
|
const callbackStats = {
|
|
@@ -8053,7 +8347,18 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8053
8347
|
};
|
|
8054
8348
|
const dedupState = createMessageDedupState();
|
|
8055
8349
|
logger?.info("Starting streamParseFile...");
|
|
8350
|
+
const parsePipelineStartedAt = now();
|
|
8351
|
+
let parserTimingFinished = false;
|
|
8352
|
+
const finishParserTiming = () => {
|
|
8353
|
+
if (parserTimingFinished) return;
|
|
8354
|
+
const parsePipelineMs = elapsedMs(parsePipelineStartedAt, now);
|
|
8355
|
+
const writeCallbackMs = timings.metaWriteMs + timings.memberWriteMs + timings.messageWriteMs + messageTransactionMs + timings.checkpointMs;
|
|
8356
|
+
timings.parserMs = Math.max(0, parsePipelineMs - writeCallbackMs);
|
|
8357
|
+
parserTimingFinished = true;
|
|
8358
|
+
sampleRss();
|
|
8359
|
+
};
|
|
8056
8360
|
try {
|
|
8361
|
+
beginTransaction();
|
|
8057
8362
|
await streamParseFile(
|
|
8058
8363
|
actualFilePath,
|
|
8059
8364
|
{
|
|
@@ -8072,135 +8377,139 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8072
8377
|
}
|
|
8073
8378
|
},
|
|
8074
8379
|
onMeta: (meta) => {
|
|
8075
|
-
|
|
8076
|
-
|
|
8077
|
-
|
|
8078
|
-
|
|
8079
|
-
|
|
8080
|
-
meta.platform
|
|
8081
|
-
|
|
8082
|
-
|
|
8083
|
-
|
|
8084
|
-
|
|
8085
|
-
|
|
8086
|
-
|
|
8087
|
-
|
|
8380
|
+
const startedAt = now();
|
|
8381
|
+
try {
|
|
8382
|
+
callbackStats.onMetaCalls++;
|
|
8383
|
+
importedPlatform = meta.platform || importedPlatform;
|
|
8384
|
+
if (!metaInserted) {
|
|
8385
|
+
logger?.info(`Writing meta: name=${meta.name}, type=${meta.type}, platform=${meta.platform}`);
|
|
8386
|
+
insertMeta.run(
|
|
8387
|
+
meta.name,
|
|
8388
|
+
meta.platform,
|
|
8389
|
+
meta.type,
|
|
8390
|
+
Math.floor(Date.now() / 1e3),
|
|
8391
|
+
meta.groupId || null,
|
|
8392
|
+
meta.groupAvatar || null,
|
|
8393
|
+
meta.ownerId || null
|
|
8394
|
+
);
|
|
8395
|
+
metaInserted = true;
|
|
8396
|
+
}
|
|
8397
|
+
} finally {
|
|
8398
|
+
timings.metaWriteMs += elapsedMs(startedAt, now);
|
|
8399
|
+
sampleRss();
|
|
8088
8400
|
}
|
|
8089
8401
|
},
|
|
8090
8402
|
onMembers: (members) => {
|
|
8091
|
-
|
|
8092
|
-
|
|
8093
|
-
|
|
8094
|
-
|
|
8095
|
-
|
|
8096
|
-
|
|
8097
|
-
|
|
8098
|
-
member.platformId,
|
|
8099
|
-
|
|
8100
|
-
|
|
8101
|
-
|
|
8102
|
-
|
|
8103
|
-
|
|
8104
|
-
|
|
8105
|
-
|
|
8106
|
-
|
|
8403
|
+
const startedAt = now();
|
|
8404
|
+
try {
|
|
8405
|
+
callbackStats.onMembersCalls++;
|
|
8406
|
+
callbackStats.totalMembersReceived += members.length;
|
|
8407
|
+
logger?.info(`Received member batch: ${members.length} members`);
|
|
8408
|
+
for (const member of members) {
|
|
8409
|
+
const accountName = normalizeSystemMemberName(formatFeature.id, member.platformId, member.accountName);
|
|
8410
|
+
const groupNickname = normalizeSystemMemberName(formatFeature.id, member.platformId, member.groupNickname);
|
|
8411
|
+
insertMember.run(
|
|
8412
|
+
member.platformId,
|
|
8413
|
+
accountName || null,
|
|
8414
|
+
groupNickname || null,
|
|
8415
|
+
member.aliases ? JSON.stringify(member.aliases) : "[]",
|
|
8416
|
+
member.avatar || null,
|
|
8417
|
+
member.roles ? JSON.stringify(member.roles) : "[]"
|
|
8418
|
+
);
|
|
8419
|
+
const row = getMemberId.get(member.platformId);
|
|
8420
|
+
if (row) memberIdMap.set(member.platformId, row.id);
|
|
8421
|
+
}
|
|
8422
|
+
} finally {
|
|
8423
|
+
timings.memberWriteMs += elapsedMs(startedAt, now);
|
|
8424
|
+
sampleRss();
|
|
8107
8425
|
}
|
|
8108
8426
|
},
|
|
8109
8427
|
onMessageBatch: (messages) => {
|
|
8110
|
-
|
|
8111
|
-
|
|
8112
|
-
|
|
8113
|
-
|
|
8114
|
-
|
|
8115
|
-
|
|
8116
|
-
|
|
8117
|
-
|
|
8118
|
-
|
|
8119
|
-
|
|
8120
|
-
|
|
8121
|
-
|
|
8122
|
-
|
|
8123
|
-
|
|
8124
|
-
formatFeature.id,
|
|
8125
|
-
msg.senderPlatformId,
|
|
8126
|
-
msg.senderAccountName
|
|
8127
|
-
);
|
|
8128
|
-
const senderGroupNickname = normalizeSystemMemberName(
|
|
8129
|
-
formatFeature.id,
|
|
8130
|
-
msg.senderPlatformId,
|
|
8131
|
-
msg.senderGroupNickname
|
|
8132
|
-
);
|
|
8133
|
-
const prepared = prepareMessageForCreate(msg, senderAccountName);
|
|
8134
|
-
if ("skipCounter" in prepared) {
|
|
8135
|
-
callbackStats[prepared.skipCounter]++;
|
|
8136
|
-
continue;
|
|
8137
|
-
}
|
|
8138
|
-
const dedupMessage = prepared.message;
|
|
8139
|
-
if (registerMessageAndCheckDuplicate(dedupMessage, dedupState)) {
|
|
8140
|
-
duplicateCount++;
|
|
8141
|
-
continue;
|
|
8428
|
+
const startedAt = now();
|
|
8429
|
+
const transactionMsBefore = messageTransactionMs;
|
|
8430
|
+
const checkpointMsBefore = timings.checkpointMs;
|
|
8431
|
+
try {
|
|
8432
|
+
const pendingRows = [];
|
|
8433
|
+
const flushPendingRows = () => {
|
|
8434
|
+
if (pendingRows.length === 0) return;
|
|
8435
|
+
messageInsertStatementCount += messageBatchInserter.insert(pendingRows);
|
|
8436
|
+
pendingRows.length = 0;
|
|
8437
|
+
};
|
|
8438
|
+
callbackStats.onMessageBatchCalls++;
|
|
8439
|
+
callbackStats.totalMessagesReceived += messages.length;
|
|
8440
|
+
if (callbackStats.onMessageBatchCalls <= 3 || callbackStats.onMessageBatchCalls % 10 === 0) {
|
|
8441
|
+
logger?.info(`Received message batch #${callbackStats.onMessageBatchCalls}: ${messages.length} messages`);
|
|
8142
8442
|
}
|
|
8143
|
-
|
|
8144
|
-
|
|
8145
|
-
|
|
8443
|
+
for (const msg of messages) {
|
|
8444
|
+
const senderAccountName = normalizeSystemMemberName(
|
|
8445
|
+
formatFeature.id,
|
|
8146
8446
|
msg.senderPlatformId,
|
|
8147
|
-
senderAccountName
|
|
8148
|
-
senderGroupNickname || null,
|
|
8149
|
-
"[]",
|
|
8150
|
-
null,
|
|
8151
|
-
"[]"
|
|
8447
|
+
msg.senderAccountName
|
|
8152
8448
|
);
|
|
8153
|
-
const
|
|
8154
|
-
|
|
8155
|
-
|
|
8156
|
-
|
|
8157
|
-
|
|
8158
|
-
|
|
8159
|
-
|
|
8160
|
-
|
|
8161
|
-
|
|
8162
|
-
|
|
8163
|
-
|
|
8164
|
-
|
|
8165
|
-
|
|
8166
|
-
|
|
8167
|
-
|
|
8168
|
-
|
|
8169
|
-
|
|
8170
|
-
|
|
8171
|
-
|
|
8172
|
-
|
|
8173
|
-
|
|
8174
|
-
|
|
8175
|
-
|
|
8176
|
-
|
|
8177
|
-
|
|
8178
|
-
|
|
8179
|
-
|
|
8180
|
-
|
|
8181
|
-
|
|
8182
|
-
|
|
8183
|
-
|
|
8184
|
-
|
|
8185
|
-
|
|
8186
|
-
|
|
8187
|
-
|
|
8188
|
-
|
|
8189
|
-
|
|
8190
|
-
|
|
8191
|
-
|
|
8192
|
-
|
|
8193
|
-
|
|
8449
|
+
const senderGroupNickname = normalizeSystemMemberName(
|
|
8450
|
+
formatFeature.id,
|
|
8451
|
+
msg.senderPlatformId,
|
|
8452
|
+
msg.senderGroupNickname
|
|
8453
|
+
);
|
|
8454
|
+
const prepared = prepareMessageForCreate(msg, senderAccountName);
|
|
8455
|
+
if ("skipCounter" in prepared) {
|
|
8456
|
+
callbackStats[prepared.skipCounter]++;
|
|
8457
|
+
continue;
|
|
8458
|
+
}
|
|
8459
|
+
const dedupMessage = prepared.message;
|
|
8460
|
+
if (registerMessageAndCheckDuplicate(dedupMessage, dedupState)) {
|
|
8461
|
+
duplicateCount++;
|
|
8462
|
+
continue;
|
|
8463
|
+
}
|
|
8464
|
+
if (!memberIdMap.has(msg.senderPlatformId)) {
|
|
8465
|
+
insertMember.run(
|
|
8466
|
+
msg.senderPlatformId,
|
|
8467
|
+
senderAccountName || null,
|
|
8468
|
+
senderGroupNickname || null,
|
|
8469
|
+
"[]",
|
|
8470
|
+
null,
|
|
8471
|
+
"[]"
|
|
8472
|
+
);
|
|
8473
|
+
const row = getMemberId.get(msg.senderPlatformId);
|
|
8474
|
+
if (row) memberIdMap.set(msg.senderPlatformId, row.id);
|
|
8475
|
+
}
|
|
8476
|
+
const senderId = memberIdMap.get(msg.senderPlatformId);
|
|
8477
|
+
if (senderId === void 0) continue;
|
|
8478
|
+
pendingRows.push({
|
|
8479
|
+
senderId,
|
|
8480
|
+
senderAccountName: senderAccountName || null,
|
|
8481
|
+
senderGroupNickname: senderGroupNickname || null,
|
|
8482
|
+
timestamp: dedupMessage.timestamp,
|
|
8483
|
+
type: dedupMessage.type,
|
|
8484
|
+
content: dedupMessage.content,
|
|
8485
|
+
replyToMessageId: msg.replyToMessageId || null,
|
|
8486
|
+
platformMessageId: msg.platformMessageId || null
|
|
8487
|
+
});
|
|
8488
|
+
messageCountInBatch++;
|
|
8489
|
+
totalMessageCount++;
|
|
8490
|
+
trackNickname(accountNameTracker, msg.senderPlatformId, senderAccountName, msg.timestamp);
|
|
8491
|
+
trackNickname(groupNicknameTracker, msg.senderPlatformId, senderGroupNickname, msg.timestamp);
|
|
8492
|
+
if (messageCountInBatch >= BATCH_COMMIT_SIZE) {
|
|
8493
|
+
flushPendingRows();
|
|
8494
|
+
commitAndBeginNew();
|
|
8495
|
+
messageCountInBatch = 0;
|
|
8496
|
+
}
|
|
8194
8497
|
}
|
|
8498
|
+
flushPendingRows();
|
|
8499
|
+
} finally {
|
|
8500
|
+
const nestedStageMs = messageTransactionMs - transactionMsBefore + (timings.checkpointMs - checkpointMsBefore);
|
|
8501
|
+
timings.messageWriteMs += Math.max(0, elapsedMs(startedAt, now) - nestedStageMs);
|
|
8502
|
+
sampleRss();
|
|
8195
8503
|
}
|
|
8196
8504
|
}
|
|
8197
8505
|
},
|
|
8198
8506
|
formatFeature.id
|
|
8199
8507
|
);
|
|
8508
|
+
finishParserTiming();
|
|
8200
8509
|
if (inTransaction) {
|
|
8201
|
-
|
|
8202
|
-
inTransaction = false;
|
|
8510
|
+
commitTransaction();
|
|
8203
8511
|
}
|
|
8512
|
+
timings.messageWriteMs += messageTransactionMs;
|
|
8204
8513
|
onProgress({
|
|
8205
8514
|
stage: "saving",
|
|
8206
8515
|
bytesRead: 0,
|
|
@@ -8211,6 +8520,7 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8211
8520
|
});
|
|
8212
8521
|
await yieldToEventLoop();
|
|
8213
8522
|
logger?.perf("Writing nickname history", totalMessageCount);
|
|
8523
|
+
const nicknameHistoryStartedAt = now();
|
|
8214
8524
|
db.exec("BEGIN TRANSACTION");
|
|
8215
8525
|
let historyCount = 0;
|
|
8216
8526
|
flushNicknameHistory(accountNameTracker, "account_name", memberIdMap, insertNameHistory, updateMemberAccountName);
|
|
@@ -8223,6 +8533,8 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8223
8533
|
);
|
|
8224
8534
|
historyCount = countHistory(accountNameTracker) + countHistory(groupNicknameTracker);
|
|
8225
8535
|
db.exec("COMMIT");
|
|
8536
|
+
timings.nicknameHistoryMs = elapsedMs(nicknameHistoryStartedAt, now);
|
|
8537
|
+
sampleRss();
|
|
8226
8538
|
logger?.perf(`Nickname history written (${historyCount} entries)`, totalMessageCount);
|
|
8227
8539
|
onProgress({
|
|
8228
8540
|
stage: "indexing",
|
|
@@ -8234,13 +8546,20 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8234
8546
|
});
|
|
8235
8547
|
await yieldToEventLoop();
|
|
8236
8548
|
logger?.perf("Creating indexes", totalMessageCount);
|
|
8549
|
+
const indexCreationStartedAt = now();
|
|
8237
8550
|
db.exec(CHAT_DB_INDEXES);
|
|
8551
|
+
timings.indexCreationMs = elapsedMs(indexCreationStartedAt, now);
|
|
8552
|
+
sampleRss();
|
|
8238
8553
|
logger?.perf("Indexes created", totalMessageCount);
|
|
8554
|
+
const ftsStartedAt = now();
|
|
8239
8555
|
try {
|
|
8240
8556
|
buildFtsIndex(db);
|
|
8241
8557
|
logger?.perf("FTS index built", totalMessageCount);
|
|
8242
8558
|
} catch (ftsError) {
|
|
8243
8559
|
logger?.error("FTS index build failed (non-fatal)", ftsError instanceof Error ? ftsError : void 0);
|
|
8560
|
+
} finally {
|
|
8561
|
+
timings.ftsMs = elapsedMs(ftsStartedAt, now);
|
|
8562
|
+
sampleRss();
|
|
8244
8563
|
}
|
|
8245
8564
|
onProgress({
|
|
8246
8565
|
stage: "indexing",
|
|
@@ -8251,18 +8570,36 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8251
8570
|
message: ""
|
|
8252
8571
|
});
|
|
8253
8572
|
await yieldToEventLoop();
|
|
8254
|
-
|
|
8573
|
+
measureCheckpoint();
|
|
8574
|
+
sampleRss();
|
|
8255
8575
|
logger?.perf("WAL checkpoint done", totalMessageCount);
|
|
8576
|
+
const sessionIndexStartedAt = now();
|
|
8577
|
+
const sessionGapThreshold = normalizeSessionGapThreshold(deps.sessionGapThreshold);
|
|
8256
8578
|
try {
|
|
8257
|
-
generateSessionIndex
|
|
8579
|
+
const generateSessionIndex2 = deps.generateSessionIndex ?? generateSessionIndex;
|
|
8580
|
+
generateSessionIndex2(db, sessionGapThreshold);
|
|
8581
|
+
logger?.perfDetail(`[SessionIndex] Built with gap threshold ${sessionGapThreshold}s`);
|
|
8258
8582
|
logger?.perf("Session index built", totalMessageCount);
|
|
8259
|
-
} catch {
|
|
8583
|
+
} catch (error) {
|
|
8584
|
+
logger?.error("Session index build failed (non-fatal)", error instanceof Error ? error : void 0);
|
|
8585
|
+
} finally {
|
|
8586
|
+
timings.sessionIndexMs = elapsedMs(sessionIndexStartedAt, now);
|
|
8587
|
+
sampleRss();
|
|
8260
8588
|
}
|
|
8261
|
-
|
|
8262
|
-
|
|
8263
|
-
|
|
8264
|
-
|
|
8589
|
+
if (deps.postImportHook) {
|
|
8590
|
+
const postImportHookStartedAt = now();
|
|
8591
|
+
try {
|
|
8592
|
+
await deps.postImportHook(db, sessionId);
|
|
8593
|
+
logger?.perf("Post-import hook done", totalMessageCount);
|
|
8594
|
+
} catch {
|
|
8595
|
+
} finally {
|
|
8596
|
+
timings.postImportHookMs = elapsedMs(postImportHookStartedAt, now);
|
|
8597
|
+
sampleRss();
|
|
8598
|
+
}
|
|
8265
8599
|
}
|
|
8600
|
+
logger?.perfDetail(
|
|
8601
|
+
`[Stages] parser=${timings.parserMs.toFixed(1)}ms | message-write=${timings.messageWriteMs.toFixed(1)}ms | indexes=${timings.indexCreationMs.toFixed(1)}ms | fts=${timings.ftsMs.toFixed(1)}ms | session-index=${timings.sessionIndexMs.toFixed(1)}ms | hook=${timings.postImportHookMs.toFixed(1)}ms`
|
|
8602
|
+
);
|
|
8266
8603
|
logger?.perf("Import completed", totalMessageCount);
|
|
8267
8604
|
logger?.info(`=== Parser Callback Stats ===`);
|
|
8268
8605
|
logger?.info(`onProgress calls: ${callbackStats.onProgressCalls}`);
|
|
@@ -8294,6 +8631,7 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8294
8631
|
importError = "error.no_messages";
|
|
8295
8632
|
}
|
|
8296
8633
|
} catch (error) {
|
|
8634
|
+
finishParserTiming();
|
|
8297
8635
|
logger?.error("Import failed", error instanceof Error ? error : void 0);
|
|
8298
8636
|
if (inTransaction) {
|
|
8299
8637
|
try {
|
|
@@ -8311,7 +8649,10 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8311
8649
|
if (shouldDeleteDb) {
|
|
8312
8650
|
deps.deleteDatabase(sessionId);
|
|
8313
8651
|
}
|
|
8652
|
+
sampleRss();
|
|
8314
8653
|
}
|
|
8654
|
+
timings.totalMs = elapsedMs(totalStartedAt, now);
|
|
8655
|
+
const bytesPerMegabyte = 1024 * 1024;
|
|
8315
8656
|
const diagnostics = {
|
|
8316
8657
|
logFile: logger?.getCurrentLogFile() ?? null,
|
|
8317
8658
|
detectedFormat: formatFeature ? `${formatFeature.name} (${formatFeature.id})` : null,
|
|
@@ -8324,12 +8665,21 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8324
8665
|
noAccountName: callbackStats.skippedNoAccountName,
|
|
8325
8666
|
invalidTimestamp: callbackStats.skippedInvalidTimestamp,
|
|
8326
8667
|
noType: callbackStats.skippedNoType
|
|
8668
|
+
},
|
|
8669
|
+
performance: {
|
|
8670
|
+
timings,
|
|
8671
|
+
messageBatchCount: callbackStats.onMessageBatchCalls,
|
|
8672
|
+
messageTransactionCount,
|
|
8673
|
+
messageInsertStatementCount,
|
|
8674
|
+
rssStartMb: rssStartBytes / bytesPerMegabyte,
|
|
8675
|
+
rssSampledPeakMb: rssPeakBytes / bytesPerMegabyte,
|
|
8676
|
+
rssSampledDeltaMb: (rssPeakBytes - rssStartBytes) / bytesPerMegabyte
|
|
8327
8677
|
}
|
|
8328
8678
|
};
|
|
8329
8679
|
if (importError) {
|
|
8330
|
-
return { success: false, error: importError, diagnostics };
|
|
8680
|
+
return { success: false, platform: importedPlatform, error: importError, diagnostics };
|
|
8331
8681
|
}
|
|
8332
|
-
return { success: true, sessionId, diagnostics };
|
|
8682
|
+
return { success: true, sessionId, platform: importedPlatform, diagnostics };
|
|
8333
8683
|
}
|
|
8334
8684
|
async function analyzeNewImport(filePath, onProgress, options2) {
|
|
8335
8685
|
const formatFeature = options2?.formatId ? getFormatFeatureById(options2.formatId) : detectFormat(filePath);
|
|
@@ -8590,11 +8940,13 @@ async function analyzeIncrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8590
8940
|
let totalInFile = 0;
|
|
8591
8941
|
let newMessageCount = 0;
|
|
8592
8942
|
let duplicateCount = 0;
|
|
8943
|
+
let platform = formatFeature.platform;
|
|
8593
8944
|
await streamParseFile(
|
|
8594
8945
|
filePath,
|
|
8595
8946
|
{
|
|
8596
8947
|
formatOptions: options2?.chatIndex === void 0 ? void 0 : { chatIndex: options2.chatIndex },
|
|
8597
|
-
onMeta: () => {
|
|
8948
|
+
onMeta: (meta) => {
|
|
8949
|
+
platform = meta.platform;
|
|
8598
8950
|
},
|
|
8599
8951
|
onMembers: () => {
|
|
8600
8952
|
},
|
|
@@ -8616,7 +8968,7 @@ async function analyzeIncrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8616
8968
|
},
|
|
8617
8969
|
options2?.formatId
|
|
8618
8970
|
);
|
|
8619
|
-
return { newMessageCount, duplicateCount, totalInFile };
|
|
8971
|
+
return { newMessageCount, duplicateCount, totalInFile, platform };
|
|
8620
8972
|
}
|
|
8621
8973
|
async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
8622
8974
|
const formatFeature = options2?.formatId ? getFormatFeatureById(options2.formatId) : detectFormat(filePath);
|
|
@@ -8633,7 +8985,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8633
8985
|
const memberUpdateMode = options2?.memberUpdateMode ?? "upsert";
|
|
8634
8986
|
try {
|
|
8635
8987
|
let trackError2 = function(index, reason, detail) {
|
|
8636
|
-
|
|
8988
|
+
errorCount++;
|
|
8637
8989
|
errorReasonCounts[reason] = (errorReasonCounts[reason] || 0) + 1;
|
|
8638
8990
|
if (errorSamples.length < MAX_ERROR_SAMPLES) {
|
|
8639
8991
|
errorSamples.push({ index, reason, detail });
|
|
@@ -8682,7 +9034,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8682
9034
|
let metaUpdated = false;
|
|
8683
9035
|
let membersAdded = 0;
|
|
8684
9036
|
let membersUpdated = 0;
|
|
8685
|
-
let
|
|
9037
|
+
let errorCount = 0;
|
|
8686
9038
|
const errorReasonCounts = {};
|
|
8687
9039
|
const errorSamples = [];
|
|
8688
9040
|
const MAX_ERROR_SAMPLES = 5;
|
|
@@ -8818,10 +9170,11 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8818
9170
|
}
|
|
8819
9171
|
if (newMessageCount > 0) {
|
|
8820
9172
|
try {
|
|
9173
|
+
const sessionGapThreshold = getSessionIndexStats(db).gapThreshold;
|
|
8821
9174
|
if (minWrittenTs < preWriteMaxTs) {
|
|
8822
|
-
generateSessionIndex(db);
|
|
9175
|
+
generateSessionIndex(db, sessionGapThreshold);
|
|
8823
9176
|
} else {
|
|
8824
|
-
generateIncrementalSessionIndex(db);
|
|
9177
|
+
generateIncrementalSessionIndex(db, sessionGapThreshold);
|
|
8825
9178
|
}
|
|
8826
9179
|
} catch {
|
|
8827
9180
|
}
|
|
@@ -8854,7 +9207,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8854
9207
|
receivedCount: processedCount,
|
|
8855
9208
|
writtenCount: newMessageCount,
|
|
8856
9209
|
duplicateCount,
|
|
8857
|
-
errorCount
|
|
9210
|
+
errorCount,
|
|
8858
9211
|
errorReasonCounts,
|
|
8859
9212
|
errorSample: errorSamples
|
|
8860
9213
|
},
|
|
@@ -8888,6 +9241,7 @@ function isValidImportSessionId(sessionId) {
|
|
|
8888
9241
|
}
|
|
8889
9242
|
|
|
8890
9243
|
// ../../packages/node-runtime/src/import/auto-import-matcher.ts
|
|
9244
|
+
import { createHash as createHash2 } from "crypto";
|
|
8891
9245
|
var MATCH_WINDOW_SIZE = 5;
|
|
8892
9246
|
function buildPrivateIdentity(ownerId, memberIds) {
|
|
8893
9247
|
if (!ownerId) return null;
|
|
@@ -8906,6 +9260,9 @@ function addMessageWindow(recentKeys, windows, message) {
|
|
|
8906
9260
|
if (recentKeys.length === MATCH_WINDOW_SIZE) windows.add(recentKeys.join(""));
|
|
8907
9261
|
}
|
|
8908
9262
|
async function resolveAutoImportTarget(filePath, deps, formatOptions) {
|
|
9263
|
+
return (await resolveAutoImportTargetPlan(filePath, deps, formatOptions)).decision;
|
|
9264
|
+
}
|
|
9265
|
+
async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
8909
9266
|
let sourceMeta = null;
|
|
8910
9267
|
const sourceMemberIds = /* @__PURE__ */ new Set();
|
|
8911
9268
|
const sourceWindows = /* @__PURE__ */ new Set();
|
|
@@ -8936,6 +9293,8 @@ async function resolveAutoImportTarget(filePath, deps, formatOptions) {
|
|
|
8936
9293
|
const meta = sourceMeta;
|
|
8937
9294
|
const privateIdentity = meta.type === "private" ? buildPrivateIdentity(meta.ownerId, sourceMemberIds) : null;
|
|
8938
9295
|
const hasStableIdentity = Boolean(meta.groupId || privateIdentity);
|
|
9296
|
+
const stableIdentity = meta.groupId ? `group\0${meta.groupId}` : privateIdentity ? `private\0${privateIdentity}` : null;
|
|
9297
|
+
const stableConcurrencyKey = stableIdentity ? `source:${createHash2("sha256").update(`${meta.platform}\0${meta.type}\0${stableIdentity}`).digest("hex")}` : null;
|
|
8939
9298
|
const stableMatches = [];
|
|
8940
9299
|
const trailingMatches = [];
|
|
8941
9300
|
for (const sessionId of deps.listSessionIds()) {
|
|
@@ -8980,24 +9339,62 @@ async function resolveAutoImportTarget(filePath, deps, formatOptions) {
|
|
|
8980
9339
|
}
|
|
8981
9340
|
if (hasStableIdentity) {
|
|
8982
9341
|
if (meta.sourceSessionId && stableMatches.includes(meta.sourceSessionId)) {
|
|
8983
|
-
return {
|
|
9342
|
+
return {
|
|
9343
|
+
decision: { action: "incremental", sessionId: meta.sourceSessionId, matchedBy: "source-session-id" },
|
|
9344
|
+
concurrencyKey: `session:${meta.sourceSessionId}`,
|
|
9345
|
+
exclusive: false,
|
|
9346
|
+
coalesceCreate: false
|
|
9347
|
+
};
|
|
8984
9348
|
}
|
|
8985
9349
|
if (stableMatches.length === 1) {
|
|
8986
|
-
return {
|
|
9350
|
+
return {
|
|
9351
|
+
decision: { action: "incremental", sessionId: stableMatches[0], matchedBy: "stable-id" },
|
|
9352
|
+
concurrencyKey: `session:${stableMatches[0]}`,
|
|
9353
|
+
exclusive: false,
|
|
9354
|
+
coalesceCreate: false
|
|
9355
|
+
};
|
|
8987
9356
|
}
|
|
8988
9357
|
}
|
|
8989
9358
|
if (meta.sourceSessionId && trailingMatches.includes(meta.sourceSessionId)) {
|
|
8990
|
-
return {
|
|
9359
|
+
return {
|
|
9360
|
+
decision: { action: "incremental", sessionId: meta.sourceSessionId, matchedBy: "source-session-id" },
|
|
9361
|
+
concurrencyKey: `session:${meta.sourceSessionId}`,
|
|
9362
|
+
exclusive: false,
|
|
9363
|
+
coalesceCreate: false
|
|
9364
|
+
};
|
|
8991
9365
|
}
|
|
8992
9366
|
if (trailingMatches.length === 1) {
|
|
8993
|
-
return {
|
|
9367
|
+
return {
|
|
9368
|
+
decision: { action: "incremental", sessionId: trailingMatches[0], matchedBy: "trailing-messages" },
|
|
9369
|
+
concurrencyKey: `session:${trailingMatches[0]}`,
|
|
9370
|
+
exclusive: false,
|
|
9371
|
+
coalesceCreate: false
|
|
9372
|
+
};
|
|
8994
9373
|
}
|
|
8995
9374
|
const ambiguous = stableMatches.length > 1 || trailingMatches.length > 1;
|
|
8996
|
-
|
|
9375
|
+
const decision = { action: "create", reason: ambiguous ? "ambiguous" : "no-match" };
|
|
9376
|
+
if (!ambiguous && stableConcurrencyKey) {
|
|
9377
|
+
return {
|
|
9378
|
+
decision,
|
|
9379
|
+
concurrencyKey: stableConcurrencyKey,
|
|
9380
|
+
exclusive: false,
|
|
9381
|
+
coalesceCreate: true
|
|
9382
|
+
};
|
|
9383
|
+
}
|
|
9384
|
+
return {
|
|
9385
|
+
decision,
|
|
9386
|
+
concurrencyKey: "unresolved",
|
|
9387
|
+
exclusive: true,
|
|
9388
|
+
coalesceCreate: false
|
|
9389
|
+
};
|
|
8997
9390
|
}
|
|
8998
9391
|
|
|
8999
9392
|
// ../../packages/node-runtime/src/import/auto-importer.ts
|
|
9000
9393
|
async function planAutoImport(filePath, deps, options2) {
|
|
9394
|
+
if (options2.resolvedDecision) {
|
|
9395
|
+
const decision2 = options2.resolvedDecision;
|
|
9396
|
+
return decision2.action === "incremental" ? { action: "incremental", sessionId: decision2.sessionId, matchedBy: decision2.matchedBy } : { action: "create", reason: decision2.reason };
|
|
9397
|
+
}
|
|
9001
9398
|
if (options2.explicitSessionId) {
|
|
9002
9399
|
if (!isValidImportSessionId(options2.explicitSessionId)) {
|
|
9003
9400
|
throw new Error("sessionId contains invalid characters");
|
|
@@ -9013,11 +9410,17 @@ async function planAutoImport(filePath, deps, options2) {
|
|
|
9013
9410
|
}
|
|
9014
9411
|
function mapCreateResult(result, createReason) {
|
|
9015
9412
|
if (!result.success || !result.sessionId) {
|
|
9016
|
-
return {
|
|
9413
|
+
return {
|
|
9414
|
+
success: false,
|
|
9415
|
+
...result.platform !== void 0 ? { platform: result.platform } : {},
|
|
9416
|
+
error: result.error,
|
|
9417
|
+
diagnostics: result.diagnostics
|
|
9418
|
+
};
|
|
9017
9419
|
}
|
|
9018
9420
|
return {
|
|
9019
9421
|
success: true,
|
|
9020
9422
|
sessionId: result.sessionId,
|
|
9423
|
+
platform: result.platform,
|
|
9021
9424
|
importMode: "created",
|
|
9022
9425
|
...createReason ? { createReason } : {},
|
|
9023
9426
|
newMessageCount: result.diagnostics?.messagesWritten ?? 0,
|
|
@@ -9031,12 +9434,15 @@ function mapCreateResult(result, createReason) {
|
|
|
9031
9434
|
};
|
|
9032
9435
|
}
|
|
9033
9436
|
function mapIncrementalResult(sessionId, result, matchedBy) {
|
|
9034
|
-
|
|
9035
|
-
return {
|
|
9036
|
-
success: true,
|
|
9437
|
+
const incrementalResult = {
|
|
9037
9438
|
sessionId,
|
|
9038
9439
|
importMode: "incremental",
|
|
9039
|
-
...matchedBy ? { matchedBy } : {}
|
|
9440
|
+
...matchedBy ? { matchedBy } : {}
|
|
9441
|
+
};
|
|
9442
|
+
if (!result.success) return { success: false, ...incrementalResult, error: result.error };
|
|
9443
|
+
return {
|
|
9444
|
+
success: true,
|
|
9445
|
+
...incrementalResult,
|
|
9040
9446
|
newMessageCount: result.newMessageCount,
|
|
9041
9447
|
duplicateCount: result.batch?.duplicateCount ?? 0,
|
|
9042
9448
|
batch: result.batch,
|
|
@@ -9045,17 +9451,18 @@ function mapIncrementalResult(sessionId, result, matchedBy) {
|
|
|
9045
9451
|
};
|
|
9046
9452
|
}
|
|
9047
9453
|
async function autoImportFile(filePath, deps, options2 = {}) {
|
|
9454
|
+
let plan;
|
|
9048
9455
|
try {
|
|
9049
9456
|
if (!options2.explicitSessionId) {
|
|
9050
9457
|
appLogger.info("import", "Automatic session matching started", {
|
|
9051
9458
|
candidateCount: deps.listSessionIds().length
|
|
9052
9459
|
});
|
|
9053
9460
|
}
|
|
9054
|
-
|
|
9461
|
+
plan = await planAutoImport(filePath, deps, options2);
|
|
9055
9462
|
if (plan.action === "incremental") {
|
|
9056
9463
|
const result2 = mapIncrementalResult(
|
|
9057
9464
|
plan.sessionId,
|
|
9058
|
-
await deps.appendSession(plan.sessionId, filePath, options2.formatOptions),
|
|
9465
|
+
await deps.appendSession(plan.sessionId, filePath, options2.formatOptions, deps.onProgress),
|
|
9059
9466
|
plan.matchedBy
|
|
9060
9467
|
);
|
|
9061
9468
|
appLogger.info("import", "Incremental import completed", {
|
|
@@ -9068,7 +9475,7 @@ async function autoImportFile(filePath, deps, options2 = {}) {
|
|
|
9068
9475
|
return result2;
|
|
9069
9476
|
}
|
|
9070
9477
|
const result = mapCreateResult(
|
|
9071
|
-
await deps.createSession(filePath, options2.formatOptions, plan.sessionId),
|
|
9478
|
+
await deps.createSession(filePath, options2.formatOptions, plan.sessionId, deps.onProgress),
|
|
9072
9479
|
plan.reason
|
|
9073
9480
|
);
|
|
9074
9481
|
appLogger.info("import", "Import created a new session", {
|
|
@@ -9079,7 +9486,15 @@ async function autoImportFile(filePath, deps, options2 = {}) {
|
|
|
9079
9486
|
return result;
|
|
9080
9487
|
} catch (error) {
|
|
9081
9488
|
appLogger.error("import", "Automatic import failed", error);
|
|
9082
|
-
return {
|
|
9489
|
+
return {
|
|
9490
|
+
success: false,
|
|
9491
|
+
...plan?.action === "incremental" ? {
|
|
9492
|
+
sessionId: plan.sessionId,
|
|
9493
|
+
importMode: "incremental",
|
|
9494
|
+
...plan.matchedBy ? { matchedBy: plan.matchedBy } : {}
|
|
9495
|
+
} : {},
|
|
9496
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9497
|
+
};
|
|
9083
9498
|
}
|
|
9084
9499
|
}
|
|
9085
9500
|
async function analyzeAutoImportFile(filePath, deps, options2 = {}) {
|
|
@@ -9130,6 +9545,221 @@ async function analyzeAutoImportFile(filePath, deps, options2 = {}) {
|
|
|
9130
9545
|
}
|
|
9131
9546
|
}
|
|
9132
9547
|
|
|
9548
|
+
// ../../packages/node-runtime/src/import/batch-coordinator.ts
|
|
9549
|
+
import { cpus, totalmem } from "os";
|
|
9550
|
+
function normalizeBatchConcurrency(value, taskCount) {
|
|
9551
|
+
if (taskCount <= 0) return 0;
|
|
9552
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 1) return 1;
|
|
9553
|
+
return Math.min(value, taskCount);
|
|
9554
|
+
}
|
|
9555
|
+
function resolveDefaultBatchConcurrency(taskCount, resources = {}) {
|
|
9556
|
+
if (taskCount <= 0) return 0;
|
|
9557
|
+
const cpuCount = resources.cpuCount ?? cpus().length;
|
|
9558
|
+
const totalMemoryBytes = resources.totalMemoryBytes ?? totalmem();
|
|
9559
|
+
const lowResourceRuntime = cpuCount < 4 || totalMemoryBytes < 4 * 1024 * 1024 * 1024;
|
|
9560
|
+
return Math.min(taskCount, lowResourceRuntime ? 1 : 2);
|
|
9561
|
+
}
|
|
9562
|
+
async function runKeyedBatch(tasks, options2) {
|
|
9563
|
+
if (tasks.length === 0) return [];
|
|
9564
|
+
const concurrency = normalizeBatchConcurrency(options2.concurrency, tasks.length);
|
|
9565
|
+
const results = Array(tasks.length);
|
|
9566
|
+
const pending = new Set(tasks.map((_, index) => index));
|
|
9567
|
+
const activeKeys = /* @__PURE__ */ new Set();
|
|
9568
|
+
let activeCount = 0;
|
|
9569
|
+
let exclusiveActive = false;
|
|
9570
|
+
return new Promise((resolve4) => {
|
|
9571
|
+
const finishIfDone = () => {
|
|
9572
|
+
if (activeCount !== 0 || pending.size !== 0) return false;
|
|
9573
|
+
options2.signal?.removeEventListener("abort", schedule);
|
|
9574
|
+
resolve4(results);
|
|
9575
|
+
return true;
|
|
9576
|
+
};
|
|
9577
|
+
const cancelPending = () => {
|
|
9578
|
+
if (!options2.signal?.aborted) return;
|
|
9579
|
+
for (const index of pending) {
|
|
9580
|
+
pending.delete(index);
|
|
9581
|
+
const result = { status: "cancelled" };
|
|
9582
|
+
results[index] = result;
|
|
9583
|
+
options2.onTaskComplete?.(tasks[index].value, index, result);
|
|
9584
|
+
}
|
|
9585
|
+
};
|
|
9586
|
+
const findRunnableIndex = () => {
|
|
9587
|
+
if (exclusiveActive) return void 0;
|
|
9588
|
+
for (const index of pending) {
|
|
9589
|
+
const task = tasks[index];
|
|
9590
|
+
if (task.exclusive) return activeCount === 0 ? index : void 0;
|
|
9591
|
+
if (!activeKeys.has(task.key)) return index;
|
|
9592
|
+
}
|
|
9593
|
+
return void 0;
|
|
9594
|
+
};
|
|
9595
|
+
function schedule() {
|
|
9596
|
+
cancelPending();
|
|
9597
|
+
if (finishIfDone()) return;
|
|
9598
|
+
while (activeCount < concurrency && !options2.signal?.aborted) {
|
|
9599
|
+
const index = findRunnableIndex();
|
|
9600
|
+
if (index === void 0) break;
|
|
9601
|
+
const task = tasks[index];
|
|
9602
|
+
pending.delete(index);
|
|
9603
|
+
activeCount++;
|
|
9604
|
+
activeKeys.add(task.key);
|
|
9605
|
+
if (task.exclusive) exclusiveActive = true;
|
|
9606
|
+
options2.onTaskStart?.(task.value, index);
|
|
9607
|
+
void Promise.resolve().then(() => options2.run(task.value, index)).then(
|
|
9608
|
+
(value) => ({ status: "success", value }),
|
|
9609
|
+
(error) => ({
|
|
9610
|
+
status: "failed",
|
|
9611
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9612
|
+
})
|
|
9613
|
+
).then((result) => {
|
|
9614
|
+
results[index] = result;
|
|
9615
|
+
options2.onTaskComplete?.(task.value, index, result);
|
|
9616
|
+
}).finally(() => {
|
|
9617
|
+
activeCount--;
|
|
9618
|
+
activeKeys.delete(task.key);
|
|
9619
|
+
if (task.exclusive) exclusiveActive = false;
|
|
9620
|
+
schedule();
|
|
9621
|
+
});
|
|
9622
|
+
}
|
|
9623
|
+
}
|
|
9624
|
+
options2.signal?.addEventListener("abort", schedule, { once: true });
|
|
9625
|
+
schedule();
|
|
9626
|
+
});
|
|
9627
|
+
}
|
|
9628
|
+
|
|
9629
|
+
// ../../packages/node-runtime/src/import/auto-import-batch.ts
|
|
9630
|
+
function mapTaskResult(item, result) {
|
|
9631
|
+
if (result.status === "cancelled") return { id: item.id, status: "cancelled" };
|
|
9632
|
+
if (result.status === "failed") return { id: item.id, status: "failed", error: result.error };
|
|
9633
|
+
if (!result.value.success) {
|
|
9634
|
+
return {
|
|
9635
|
+
id: item.id,
|
|
9636
|
+
status: "failed",
|
|
9637
|
+
error: result.value.error ?? "Import failed",
|
|
9638
|
+
result: result.value
|
|
9639
|
+
};
|
|
9640
|
+
}
|
|
9641
|
+
return { id: item.id, status: "success", result: result.value };
|
|
9642
|
+
}
|
|
9643
|
+
function preserveSerialTargetMatching(planned) {
|
|
9644
|
+
let priorPlannedWrite = false;
|
|
9645
|
+
for (const plannedItem of planned) {
|
|
9646
|
+
let plan = plannedItem.plan;
|
|
9647
|
+
if (priorPlannedWrite && !plannedItem.item.options?.explicitSessionId && !plan.exclusive) {
|
|
9648
|
+
plan = {
|
|
9649
|
+
...plan,
|
|
9650
|
+
concurrencyKey: `unresolved:${plannedItem.itemIndex}`,
|
|
9651
|
+
exclusive: true,
|
|
9652
|
+
coalesceCreate: false
|
|
9653
|
+
};
|
|
9654
|
+
}
|
|
9655
|
+
plannedItem.plan = plan;
|
|
9656
|
+
priorPlannedWrite = true;
|
|
9657
|
+
}
|
|
9658
|
+
}
|
|
9659
|
+
async function autoImportBatch(items, deps, options2 = {}) {
|
|
9660
|
+
if (items.length === 0) return [];
|
|
9661
|
+
const resolvePlan = options2.resolveTargetPlan ?? resolveAutoImportTargetPlan;
|
|
9662
|
+
const concurrency = normalizeBatchConcurrency(options2.concurrency, items.length);
|
|
9663
|
+
appLogger.info("import-batch", "Batch import started", {
|
|
9664
|
+
itemCount: items.length,
|
|
9665
|
+
concurrency
|
|
9666
|
+
});
|
|
9667
|
+
const preflight = await runKeyedBatch(
|
|
9668
|
+
items.map((item, index) => ({ value: { item, itemIndex: index }, key: `preflight:${index}` })),
|
|
9669
|
+
{
|
|
9670
|
+
concurrency,
|
|
9671
|
+
signal: options2.signal,
|
|
9672
|
+
onTaskStart: ({ item, itemIndex }) => options2.onItemStart?.(item, itemIndex),
|
|
9673
|
+
run: async ({ item, itemIndex }) => {
|
|
9674
|
+
const itemOptions = item.options ?? {};
|
|
9675
|
+
if (itemOptions.explicitSessionId) {
|
|
9676
|
+
const sessionId = itemOptions.explicitSessionId;
|
|
9677
|
+
const decision = deps.sessionExists(sessionId) ? { action: "incremental", sessionId, matchedBy: "source-session-id" } : { action: "create", reason: "no-match" };
|
|
9678
|
+
return {
|
|
9679
|
+
item,
|
|
9680
|
+
itemIndex,
|
|
9681
|
+
plan: {
|
|
9682
|
+
decision,
|
|
9683
|
+
concurrencyKey: `session:${sessionId}`,
|
|
9684
|
+
exclusive: false,
|
|
9685
|
+
coalesceCreate: decision.action === "create"
|
|
9686
|
+
}
|
|
9687
|
+
};
|
|
9688
|
+
}
|
|
9689
|
+
return {
|
|
9690
|
+
item,
|
|
9691
|
+
itemIndex,
|
|
9692
|
+
plan: await resolvePlan(
|
|
9693
|
+
item.filePath,
|
|
9694
|
+
{
|
|
9695
|
+
...deps,
|
|
9696
|
+
onProgress: (progress) => options2.onItemProgress?.(item, itemIndex, progress)
|
|
9697
|
+
},
|
|
9698
|
+
itemOptions.formatOptions
|
|
9699
|
+
)
|
|
9700
|
+
};
|
|
9701
|
+
}
|
|
9702
|
+
}
|
|
9703
|
+
);
|
|
9704
|
+
const finalResults = Array(items.length);
|
|
9705
|
+
const planned = [];
|
|
9706
|
+
for (let index = 0; index < preflight.length; index++) {
|
|
9707
|
+
const result = preflight[index];
|
|
9708
|
+
if (result.status === "success") {
|
|
9709
|
+
planned.push(result.value);
|
|
9710
|
+
continue;
|
|
9711
|
+
}
|
|
9712
|
+
const mapped = result.status === "cancelled" ? { id: items[index].id, status: "cancelled" } : { id: items[index].id, status: "failed", error: result.error };
|
|
9713
|
+
finalResults[index] = mapped;
|
|
9714
|
+
options2.onItemComplete?.(items[index], index, mapped);
|
|
9715
|
+
}
|
|
9716
|
+
preserveSerialTargetMatching(planned);
|
|
9717
|
+
const targetSessionByKey = /* @__PURE__ */ new Map();
|
|
9718
|
+
await runKeyedBatch(
|
|
9719
|
+
planned.map((plannedItem) => ({
|
|
9720
|
+
value: plannedItem,
|
|
9721
|
+
key: plannedItem.plan.concurrencyKey,
|
|
9722
|
+
exclusive: plannedItem.plan.exclusive
|
|
9723
|
+
})),
|
|
9724
|
+
{
|
|
9725
|
+
concurrency,
|
|
9726
|
+
signal: options2.signal,
|
|
9727
|
+
onTaskComplete: ({ item, itemIndex }, _plannedIndex, taskResult) => {
|
|
9728
|
+
const mapped = mapTaskResult(item, taskResult);
|
|
9729
|
+
finalResults[itemIndex] = mapped;
|
|
9730
|
+
options2.onItemComplete?.(item, itemIndex, mapped);
|
|
9731
|
+
},
|
|
9732
|
+
run: async ({ item, itemIndex, plan }) => {
|
|
9733
|
+
const targetSessionId = plan.coalesceCreate ? targetSessionByKey.get(plan.concurrencyKey) : void 0;
|
|
9734
|
+
const decision = targetSessionId ? { action: "incremental", sessionId: targetSessionId, matchedBy: "trailing-messages" } : plan.decision;
|
|
9735
|
+
const result = await autoImportFile(
|
|
9736
|
+
item.filePath,
|
|
9737
|
+
{
|
|
9738
|
+
...deps,
|
|
9739
|
+
onProgress: (progress) => options2.onItemProgress?.(item, itemIndex, progress)
|
|
9740
|
+
},
|
|
9741
|
+
{
|
|
9742
|
+
...item.options,
|
|
9743
|
+
resolvedDecision: plan.exclusive || item.options?.explicitSessionId && !targetSessionId ? void 0 : decision
|
|
9744
|
+
}
|
|
9745
|
+
);
|
|
9746
|
+
if (!plan.exclusive && result.success && result.sessionId) {
|
|
9747
|
+
targetSessionByKey.set(plan.concurrencyKey, result.sessionId);
|
|
9748
|
+
}
|
|
9749
|
+
return result;
|
|
9750
|
+
}
|
|
9751
|
+
}
|
|
9752
|
+
);
|
|
9753
|
+
const completedResults = finalResults;
|
|
9754
|
+
appLogger.info("import-batch", "Batch import completed", {
|
|
9755
|
+
itemCount: items.length,
|
|
9756
|
+
successCount: completedResults.filter((result) => result.status === "success").length,
|
|
9757
|
+
failedCount: completedResults.filter((result) => result.status === "failed").length,
|
|
9758
|
+
cancelledCount: completedResults.filter((result) => result.status === "cancelled").length
|
|
9759
|
+
});
|
|
9760
|
+
return completedResults;
|
|
9761
|
+
}
|
|
9762
|
+
|
|
9133
9763
|
// ../../packages/node-runtime/src/import/import-lock.ts
|
|
9134
9764
|
import * as fs22 from "fs";
|
|
9135
9765
|
import * as path14 from "path";
|
|
@@ -9964,8 +10594,8 @@ var AssistantManager = class {
|
|
|
9964
10594
|
}
|
|
9965
10595
|
// ==================== Init ====================
|
|
9966
10596
|
init() {
|
|
9967
|
-
const { fs:
|
|
9968
|
-
|
|
10597
|
+
const { fs: fs37, assistantsDir } = this.deps;
|
|
10598
|
+
fs37.ensureDir(assistantsDir);
|
|
9969
10599
|
const { generalCreated, generalUpdated } = this.syncGeneralAssistants();
|
|
9970
10600
|
this.loadAll();
|
|
9971
10601
|
this.initialized = true;
|
|
@@ -9977,25 +10607,25 @@ var AssistantManager = class {
|
|
|
9977
10607
|
return { total: this.cache.size, generalCreated, generalUpdated };
|
|
9978
10608
|
}
|
|
9979
10609
|
syncGeneralAssistants() {
|
|
9980
|
-
const { fs:
|
|
10610
|
+
const { fs: fs37, assistantsDir } = this.deps;
|
|
9981
10611
|
let generalCreated = false;
|
|
9982
10612
|
let generalUpdated = false;
|
|
9983
10613
|
for (const id of this.generalIds) {
|
|
9984
10614
|
const config = this.getBuiltinConfig(id);
|
|
9985
10615
|
if (!config) continue;
|
|
9986
|
-
const filePath =
|
|
9987
|
-
if (!
|
|
9988
|
-
|
|
10616
|
+
const filePath = fs37.joinPath(assistantsDir, `${id}.md`);
|
|
10617
|
+
if (!fs37.fileExists(filePath)) {
|
|
10618
|
+
fs37.writeFile(filePath, serializeAssistant(this.withBuiltinTracking(config)));
|
|
9989
10619
|
generalCreated = true;
|
|
9990
10620
|
continue;
|
|
9991
10621
|
}
|
|
9992
10622
|
try {
|
|
9993
|
-
const existing = parseAssistantFile(
|
|
10623
|
+
const existing = parseAssistantFile(fs37.readFile(filePath), filePath);
|
|
9994
10624
|
if (!existing) continue;
|
|
9995
10625
|
if (existing.builtinId !== id) {
|
|
9996
10626
|
try {
|
|
9997
10627
|
const backupId = this.backupAssistantOccupyingReservedId(existing);
|
|
9998
|
-
|
|
10628
|
+
fs37.writeFile(filePath, serializeAssistant(this.withBuiltinTracking(config)));
|
|
9999
10629
|
generalCreated = true;
|
|
10000
10630
|
this.deps.logger?.info("AssistantManager", "Backed up assistant occupying reserved default ID", {
|
|
10001
10631
|
reservedId: id,
|
|
@@ -10011,7 +10641,7 @@ var AssistantManager = class {
|
|
|
10011
10641
|
continue;
|
|
10012
10642
|
}
|
|
10013
10643
|
if (!this.shouldUpgradeBuiltin(existing, config)) continue;
|
|
10014
|
-
|
|
10644
|
+
fs37.writeFile(filePath, serializeAssistant(this.withBuiltinTracking(config, existing.id)));
|
|
10015
10645
|
generalUpdated = true;
|
|
10016
10646
|
this.deps.logger?.info("AssistantManager", "Updated unmodified default assistant template", {
|
|
10017
10647
|
id,
|
|
@@ -10027,14 +10657,14 @@ var AssistantManager = class {
|
|
|
10027
10657
|
return { generalCreated, generalUpdated };
|
|
10028
10658
|
}
|
|
10029
10659
|
backupAssistantOccupyingReservedId(config) {
|
|
10030
|
-
const { fs:
|
|
10660
|
+
const { fs: fs37, assistantsDir } = this.deps;
|
|
10031
10661
|
const baseId = this.deps.generateId?.() || `custom_${Date.now().toString(36)}`;
|
|
10032
10662
|
let backupId = baseId;
|
|
10033
10663
|
let suffix = 2;
|
|
10034
|
-
while (this.generalIds.includes(backupId) ||
|
|
10664
|
+
while (this.generalIds.includes(backupId) || fs37.fileExists(fs37.joinPath(assistantsDir, `${backupId}.md`))) {
|
|
10035
10665
|
backupId = `${baseId}_${suffix++}`;
|
|
10036
10666
|
}
|
|
10037
|
-
|
|
10667
|
+
fs37.writeFile(fs37.joinPath(assistantsDir, `${backupId}.md`), serializeAssistant({ ...config, id: backupId }));
|
|
10038
10668
|
return backupId;
|
|
10039
10669
|
}
|
|
10040
10670
|
getConfigDigest(config) {
|
|
@@ -10071,13 +10701,13 @@ var AssistantManager = class {
|
|
|
10071
10701
|
return { isUnmodified, isOutdated };
|
|
10072
10702
|
}
|
|
10073
10703
|
loadAll() {
|
|
10074
|
-
const { fs:
|
|
10704
|
+
const { fs: fs37, assistantsDir } = this.deps;
|
|
10075
10705
|
this.cache.clear();
|
|
10076
|
-
const files =
|
|
10706
|
+
const files = fs37.listFiles(assistantsDir, ".md");
|
|
10077
10707
|
for (const file of files) {
|
|
10078
10708
|
try {
|
|
10079
|
-
const filePath =
|
|
10080
|
-
const content =
|
|
10709
|
+
const filePath = fs37.joinPath(assistantsDir, file);
|
|
10710
|
+
const content = fs37.readFile(filePath);
|
|
10081
10711
|
const config = parseAssistantFile(content, filePath);
|
|
10082
10712
|
if (config) {
|
|
10083
10713
|
this.cache.set(config.id, config);
|
|
@@ -10300,7 +10930,7 @@ import Database from "better-sqlite3";
|
|
|
10300
10930
|
import * as fs26 from "fs";
|
|
10301
10931
|
import * as path20 from "path";
|
|
10302
10932
|
var DEFAULT_GENERAL_ID = DEFAULT_GENERAL_ASSISTANT_ID;
|
|
10303
|
-
var
|
|
10933
|
+
var defaultLogger2 = {
|
|
10304
10934
|
warn(_category, message, extra) {
|
|
10305
10935
|
console.warn(`[AI Chats] ${message}`, extra ?? "");
|
|
10306
10936
|
}
|
|
@@ -10313,7 +10943,7 @@ var AIChatManager = class {
|
|
|
10313
10943
|
pendingDebugContextMap = /* @__PURE__ */ new Map();
|
|
10314
10944
|
constructor(aiDataDir, options2) {
|
|
10315
10945
|
this.aiDataDir = aiDataDir;
|
|
10316
|
-
this.logger = options2?.logger ??
|
|
10946
|
+
this.logger = options2?.logger ?? defaultLogger2;
|
|
10317
10947
|
this.nativeBinding = options2?.nativeBinding;
|
|
10318
10948
|
}
|
|
10319
10949
|
ensureDir(dir) {
|
|
@@ -10567,16 +11197,16 @@ var AIChatManager = class {
|
|
|
10567
11197
|
if (allRows.length === 0) return [];
|
|
10568
11198
|
const rowMap = new Map(allRows.map((row) => [row.id, row]));
|
|
10569
11199
|
let currentId = leafMessageId ?? this.getActiveMessageId(aiChatId);
|
|
10570
|
-
const
|
|
11200
|
+
const path31 = [];
|
|
10571
11201
|
const seen = /* @__PURE__ */ new Set();
|
|
10572
11202
|
while (currentId && !seen.has(currentId)) {
|
|
10573
11203
|
seen.add(currentId);
|
|
10574
11204
|
const row = rowMap.get(currentId);
|
|
10575
11205
|
if (!row) break;
|
|
10576
|
-
|
|
11206
|
+
path31.push(row);
|
|
10577
11207
|
currentId = row.parentId;
|
|
10578
11208
|
}
|
|
10579
|
-
return
|
|
11209
|
+
return path31.length > 0 ? path31.reverse() : allRows;
|
|
10580
11210
|
}
|
|
10581
11211
|
// ==================== 生命周期 ====================
|
|
10582
11212
|
close() {
|
|
@@ -11146,21 +11776,21 @@ var SkillManagerCore = class {
|
|
|
11146
11776
|
}
|
|
11147
11777
|
// ==================== Init ====================
|
|
11148
11778
|
init() {
|
|
11149
|
-
const { fs:
|
|
11150
|
-
|
|
11779
|
+
const { fs: fs37, skillsDir } = this.deps;
|
|
11780
|
+
fs37.ensureDir(skillsDir);
|
|
11151
11781
|
this.loadAll();
|
|
11152
11782
|
this.initialized = true;
|
|
11153
11783
|
this.deps.logger?.info("SkillManager", "Initialized", { total: this.cache.size });
|
|
11154
11784
|
return { total: this.cache.size };
|
|
11155
11785
|
}
|
|
11156
11786
|
loadAll() {
|
|
11157
|
-
const { fs:
|
|
11787
|
+
const { fs: fs37, skillsDir } = this.deps;
|
|
11158
11788
|
this.cache.clear();
|
|
11159
|
-
const files =
|
|
11789
|
+
const files = fs37.listFiles(skillsDir, ".md");
|
|
11160
11790
|
for (const file of files) {
|
|
11161
11791
|
try {
|
|
11162
|
-
const filePath =
|
|
11163
|
-
const content =
|
|
11792
|
+
const filePath = fs37.joinPath(skillsDir, file);
|
|
11793
|
+
const content = fs37.readFile(filePath);
|
|
11164
11794
|
const def = parseSkillFile(content, filePath);
|
|
11165
11795
|
if (def) {
|
|
11166
11796
|
this.cache.set(def.id, def);
|
|
@@ -11315,7 +11945,7 @@ var SkillManagerCore = class {
|
|
|
11315
11945
|
import * as fs27 from "fs";
|
|
11316
11946
|
import * as path21 from "path";
|
|
11317
11947
|
var SKILLS_DIR_NAME = "skills";
|
|
11318
|
-
var
|
|
11948
|
+
var defaultLogger3 = {
|
|
11319
11949
|
info: () => {
|
|
11320
11950
|
},
|
|
11321
11951
|
warn: () => {
|
|
@@ -11349,7 +11979,7 @@ var SkillManager = class {
|
|
|
11349
11979
|
core;
|
|
11350
11980
|
logger;
|
|
11351
11981
|
constructor(aiDataDir, logger) {
|
|
11352
|
-
this.logger = logger ??
|
|
11982
|
+
this.logger = logger ?? defaultLogger3;
|
|
11353
11983
|
this.core = new SkillManagerCore({
|
|
11354
11984
|
fs: nodeFs,
|
|
11355
11985
|
skillsDir: path21.join(aiDataDir, SKILLS_DIR_NAME),
|
|
@@ -11772,7 +12402,7 @@ STRICT RULES:
|
|
|
11772
12402
|
- Omit: pleasantries, filler, redundant back-and-forth, detailed tables (summarize their conclusions instead).
|
|
11773
12403
|
|
|
11774
12404
|
{messages}`;
|
|
11775
|
-
var
|
|
12405
|
+
var defaultLogger4 = {
|
|
11776
12406
|
info: () => {
|
|
11777
12407
|
},
|
|
11778
12408
|
warn: () => {
|
|
@@ -11780,7 +12410,7 @@ var defaultLogger3 = {
|
|
|
11780
12410
|
error: () => {
|
|
11781
12411
|
}
|
|
11782
12412
|
};
|
|
11783
|
-
async function checkAndCompress(aiChatId, config, systemPrompt, llmAdapter, convManager, logger =
|
|
12413
|
+
async function checkAndCompress(aiChatId, config, systemPrompt, llmAdapter, convManager, logger = defaultLogger4) {
|
|
11784
12414
|
if (!config.enabled) {
|
|
11785
12415
|
return { compressed: false, reason: "skipped_disabled" };
|
|
11786
12416
|
}
|
|
@@ -11974,7 +12604,7 @@ function createCompressionLlmAdapter(options2) {
|
|
|
11974
12604
|
|
|
11975
12605
|
// ../../packages/node-runtime/src/ai/preprocessor/pipeline.ts
|
|
11976
12606
|
var MERGE_WINDOW_DEFAULT = 180;
|
|
11977
|
-
var
|
|
12607
|
+
var defaultLogger5 = {
|
|
11978
12608
|
info: () => {
|
|
11979
12609
|
},
|
|
11980
12610
|
warn: () => {
|
|
@@ -11991,10 +12621,10 @@ function emptyStats(count) {
|
|
|
11991
12621
|
desensitizeRulesApplied: 0
|
|
11992
12622
|
};
|
|
11993
12623
|
}
|
|
11994
|
-
function preprocessMessages(messages, config, logger =
|
|
12624
|
+
function preprocessMessages(messages, config, logger = defaultLogger5) {
|
|
11995
12625
|
return preprocessMessagesWithStats(messages, config, logger).messages;
|
|
11996
12626
|
}
|
|
11997
|
-
function preprocessMessagesWithStats(messages, config, logger =
|
|
12627
|
+
function preprocessMessagesWithStats(messages, config, logger = defaultLogger5) {
|
|
11998
12628
|
if (!config || !hasAnyEnabled(config)) return { messages, stats: emptyStats(messages.length) };
|
|
11999
12629
|
if (messages.length === 0) return { messages, stats: emptyStats(0) };
|
|
12000
12630
|
const stats = emptyStats(messages.length);
|
|
@@ -12176,7 +12806,7 @@ function applyCompiledRules(text, compiledRules) {
|
|
|
12176
12806
|
}
|
|
12177
12807
|
return result;
|
|
12178
12808
|
}
|
|
12179
|
-
function desensitizeText(text, rules, logger =
|
|
12809
|
+
function desensitizeText(text, rules, logger = defaultLogger5) {
|
|
12180
12810
|
const enabledRules = rules.filter((r) => r.enabled);
|
|
12181
12811
|
if (enabledRules.length === 0 || !text) return text;
|
|
12182
12812
|
return applyCompiledRules(text, compileRules(enabledRules, logger));
|
|
@@ -15218,9 +15848,9 @@ var localeMap = {
|
|
|
15218
15848
|
"ja-JP": ja_JP_default,
|
|
15219
15849
|
"zh-TW": zh_TW_default
|
|
15220
15850
|
};
|
|
15221
|
-
function getNestedValue(obj,
|
|
15851
|
+
function getNestedValue(obj, path31) {
|
|
15222
15852
|
let current = obj;
|
|
15223
|
-
for (const segment2 of
|
|
15853
|
+
for (const segment2 of path31.split(".")) {
|
|
15224
15854
|
if (current == null || typeof current !== "object") return void 0;
|
|
15225
15855
|
current = current[segment2];
|
|
15226
15856
|
}
|
|
@@ -15748,6 +16378,10 @@ var LLMConfigStore = class {
|
|
|
15748
16378
|
hasActiveConfig() {
|
|
15749
16379
|
return this.getDefaultAssistantConfig() !== null;
|
|
15750
16380
|
}
|
|
16381
|
+
hasConfiguredModel() {
|
|
16382
|
+
const config = this.getDefaultAssistantConfig();
|
|
16383
|
+
return typeof config?.model === "string" && config.model.trim().length > 0;
|
|
16384
|
+
}
|
|
15751
16385
|
};
|
|
15752
16386
|
|
|
15753
16387
|
// ../../packages/node-runtime/src/ai/file-config-storage.ts
|
|
@@ -17322,7 +17956,7 @@ function setOwnerAndApplyProfile(adapter, preferences, sessionId, ownerPlatformI
|
|
|
17322
17956
|
});
|
|
17323
17957
|
return { sessionId, platform: meta.platform, ownerId: ownerPlatformId, updatedSessionIds, updatedSessionOwnerIds };
|
|
17324
17958
|
}
|
|
17325
|
-
function applyProfileToOtherSessions(adapter,
|
|
17959
|
+
function applyProfileToOtherSessions(adapter, platform, profile, excludeSessionId) {
|
|
17326
17960
|
const updated = [];
|
|
17327
17961
|
for (const id of adapter.listSessionIds()) {
|
|
17328
17962
|
if (id === excludeSessionId) continue;
|
|
@@ -17330,8 +17964,8 @@ function applyProfileToOtherSessions(adapter, platform2, profile, excludeSession
|
|
|
17330
17964
|
const db = adapter.openReadonly(id);
|
|
17331
17965
|
if (!db || !isChatSessionDb(db)) continue;
|
|
17332
17966
|
const meta = getSessionMeta(db);
|
|
17333
|
-
if (!meta || meta.platform !==
|
|
17334
|
-
const result = matchOwnerProfile(
|
|
17967
|
+
if (!meta || meta.platform !== platform || meta.ownerId) continue;
|
|
17968
|
+
const result = matchOwnerProfile(platform, profile, getMembersWithAliases(db));
|
|
17335
17969
|
if (result.type === "exact" || result.type === "name") {
|
|
17336
17970
|
updateSessionOwnerId(adapter.ensureWritable(id), result.platformId);
|
|
17337
17971
|
updated.push({ id, ownerId: result.platformId });
|
|
@@ -18933,11 +19567,130 @@ var SemanticIndexConfigStore = class {
|
|
|
18933
19567
|
}
|
|
18934
19568
|
};
|
|
18935
19569
|
|
|
19570
|
+
// ../../packages/node-runtime/src/semantic-index/embedding/text.ts
|
|
19571
|
+
function applyQueryInstruction(instruction, query) {
|
|
19572
|
+
const trimmed = instruction.trim();
|
|
19573
|
+
if (!trimmed) return query;
|
|
19574
|
+
if (trimmed.endsWith(":") || trimmed.endsWith("\uFF1A")) return `${trimmed}${query}`;
|
|
19575
|
+
return `Instruct: ${trimmed}
|
|
19576
|
+
Query: ${query}`;
|
|
19577
|
+
}
|
|
19578
|
+
function clampTextChars(text, maxChars) {
|
|
19579
|
+
if (maxChars === void 0 || text.length <= maxChars) return text;
|
|
19580
|
+
return text.slice(0, maxChars);
|
|
19581
|
+
}
|
|
19582
|
+
|
|
19583
|
+
// ../../packages/node-runtime/src/semantic-index/embedding/local.ts
|
|
19584
|
+
var SOCKS_PROXY_PROTOCOLS = /* @__PURE__ */ new Set(["socks:", "socks4:", "socks5:"]);
|
|
19585
|
+
var LOCAL_ONNX_SESSION_OPTIONS = { intraOpNumThreads: 1, interOpNumThreads: 1 };
|
|
19586
|
+
var HUGGING_FACE_MODEL_HOST = "https://huggingface.co/";
|
|
19587
|
+
var HF_MIRROR_MODEL_HOST = "https://hf-mirror.com/";
|
|
19588
|
+
function resolveModelDownloadRemoteHost(source) {
|
|
19589
|
+
return source === "hf-mirror" ? HF_MIRROR_MODEL_HOST : HUGGING_FACE_MODEL_HOST;
|
|
19590
|
+
}
|
|
19591
|
+
async function createProxyFetch(proxyUrl) {
|
|
19592
|
+
const parsed = new URL(proxyUrl);
|
|
19593
|
+
if (SOCKS_PROXY_PROTOCOLS.has(parsed.protocol)) {
|
|
19594
|
+
throw new Error(
|
|
19595
|
+
`SOCKS proxy is not supported for local embedding model downloads: ${parsed.protocol}//${parsed.host}`
|
|
19596
|
+
);
|
|
19597
|
+
}
|
|
19598
|
+
const { fetch: undiciFetch, ProxyAgent } = await import("./undici-MU5I3L6Q.mjs");
|
|
19599
|
+
const dispatcher = new ProxyAgent(proxyUrl);
|
|
19600
|
+
return ((input, init) => {
|
|
19601
|
+
const nextInit = { ...init, dispatcher };
|
|
19602
|
+
return undiciFetch(input, nextInit);
|
|
19603
|
+
});
|
|
19604
|
+
}
|
|
19605
|
+
function createTransformersPipelineFactory(loadTransformers) {
|
|
19606
|
+
return async ({ modelId, dtype, cacheDir, modelDownloadProxyUrl, modelDownloadSource }) => {
|
|
19607
|
+
const transformers = await loadTransformers();
|
|
19608
|
+
transformers.env.remoteHost = resolveModelDownloadRemoteHost(modelDownloadSource);
|
|
19609
|
+
if (cacheDir) {
|
|
19610
|
+
transformers.env.cacheDir = cacheDir;
|
|
19611
|
+
transformers.env.allowRemoteModels = true;
|
|
19612
|
+
}
|
|
19613
|
+
if (modelDownloadProxyUrl) {
|
|
19614
|
+
transformers.env.fetch = await createProxyFetch(modelDownloadProxyUrl);
|
|
19615
|
+
}
|
|
19616
|
+
const extractor = await transformers.pipeline("feature-extraction", modelId, {
|
|
19617
|
+
...dtype ? { dtype } : {},
|
|
19618
|
+
session_options: LOCAL_ONNX_SESSION_OPTIONS
|
|
19619
|
+
});
|
|
19620
|
+
return async (texts, options2) => {
|
|
19621
|
+
const out = await extractor(texts, { pooling: options2.pooling, normalize: options2.normalize });
|
|
19622
|
+
return out.tolist();
|
|
19623
|
+
};
|
|
19624
|
+
};
|
|
19625
|
+
}
|
|
19626
|
+
var defaultPipelineFactory = createTransformersPipelineFactory(() => import("@huggingface/transformers"));
|
|
19627
|
+
var LocalEmbeddingProvider = class {
|
|
19628
|
+
modelId;
|
|
19629
|
+
dim;
|
|
19630
|
+
maxTokens;
|
|
19631
|
+
documentBatchSize;
|
|
19632
|
+
profile;
|
|
19633
|
+
options;
|
|
19634
|
+
extractorPromise = null;
|
|
19635
|
+
constructor(profile, options2 = {}) {
|
|
19636
|
+
this.profile = profile;
|
|
19637
|
+
this.options = options2;
|
|
19638
|
+
this.modelId = profile.modelId;
|
|
19639
|
+
this.dim = profile.dim;
|
|
19640
|
+
this.maxTokens = profile.maxTokens;
|
|
19641
|
+
this.documentBatchSize = profile.maxBatchSize ?? 1;
|
|
19642
|
+
}
|
|
19643
|
+
getExtractor() {
|
|
19644
|
+
if (!this.extractorPromise) {
|
|
19645
|
+
const factory = this.options.pipelineFactory ?? defaultPipelineFactory;
|
|
19646
|
+
this.extractorPromise = factory({
|
|
19647
|
+
modelId: this.profile.modelId,
|
|
19648
|
+
dtype: this.profile.dtype,
|
|
19649
|
+
cacheDir: this.options.cacheDir,
|
|
19650
|
+
modelDownloadProxyUrl: this.options.modelDownloadProxyUrl,
|
|
19651
|
+
modelDownloadSource: this.options.modelDownloadSource,
|
|
19652
|
+
sessionOptions: LOCAL_ONNX_SESSION_OPTIONS
|
|
19653
|
+
}).catch((error) => {
|
|
19654
|
+
this.extractorPromise = null;
|
|
19655
|
+
throw error;
|
|
19656
|
+
});
|
|
19657
|
+
}
|
|
19658
|
+
return this.extractorPromise;
|
|
19659
|
+
}
|
|
19660
|
+
toFloat32(vector) {
|
|
19661
|
+
if (vector.length !== this.profile.dim) {
|
|
19662
|
+
throw new Error(`model ${this.profile.modelId} returned dim ${vector.length}, expected ${this.profile.dim}`);
|
|
19663
|
+
}
|
|
19664
|
+
return Float32Array.from(vector);
|
|
19665
|
+
}
|
|
19666
|
+
async embedDocuments(texts) {
|
|
19667
|
+
if (texts.length === 0) return [];
|
|
19668
|
+
const extractor = await this.getExtractor();
|
|
19669
|
+
const batchSize = this.profile.maxBatchSize ?? texts.length;
|
|
19670
|
+
const clamped = texts.map((t2) => clampTextChars(t2, this.profile.maxTextChars));
|
|
19671
|
+
const result = [];
|
|
19672
|
+
for (let i = 0; i < clamped.length; i += batchSize) {
|
|
19673
|
+
const batch = clamped.slice(i, i + batchSize);
|
|
19674
|
+
const vectors = await extractor(batch, { pooling: this.profile.pooling, normalize: this.profile.normalize });
|
|
19675
|
+
for (const v of vectors) result.push(this.toFloat32(v));
|
|
19676
|
+
}
|
|
19677
|
+
return result;
|
|
19678
|
+
}
|
|
19679
|
+
async embedQuery(text) {
|
|
19680
|
+
const withInstruction = applyQueryInstruction(this.profile.queryInstruction, text);
|
|
19681
|
+
const [vector] = await this.embedDocuments([withInstruction]);
|
|
19682
|
+
return vector;
|
|
19683
|
+
}
|
|
19684
|
+
async preload() {
|
|
19685
|
+
await this.getExtractor();
|
|
19686
|
+
}
|
|
19687
|
+
};
|
|
19688
|
+
|
|
18936
19689
|
// ../../packages/node-runtime/src/semantic-index/service.ts
|
|
18937
19690
|
import path28 from "path";
|
|
18938
19691
|
|
|
18939
19692
|
// ../../packages/node-runtime/src/semantic-index/chunker-config.ts
|
|
18940
|
-
import { createHash as
|
|
19693
|
+
import { createHash as createHash3 } from "crypto";
|
|
18941
19694
|
var CHUNKER_VERSION = "v1.3";
|
|
18942
19695
|
var STRATEGY_ID = "balanced";
|
|
18943
19696
|
var DEFAULT_CHUNKER_CONFIG = {
|
|
@@ -18952,7 +19705,7 @@ var DEFAULT_CHUNKER_CONFIG = {
|
|
|
18952
19705
|
semanticVoidSkipThreshold: 20
|
|
18953
19706
|
};
|
|
18954
19707
|
function sha256Hex(input) {
|
|
18955
|
-
return
|
|
19708
|
+
return createHash3("sha256").update(input, "utf-8").digest("hex");
|
|
18956
19709
|
}
|
|
18957
19710
|
function computeChunkerConfigHash(config) {
|
|
18958
19711
|
const canonical = JSON.stringify({
|
|
@@ -19091,128 +19844,6 @@ var OpenAICompatibleEmbeddingProvider = class {
|
|
|
19091
19844
|
}
|
|
19092
19845
|
};
|
|
19093
19846
|
|
|
19094
|
-
// ../../packages/node-runtime/src/semantic-index/embedding/text.ts
|
|
19095
|
-
function applyQueryInstruction(instruction, query) {
|
|
19096
|
-
const trimmed = instruction.trim();
|
|
19097
|
-
if (!trimmed) return query;
|
|
19098
|
-
if (trimmed.endsWith(":") || trimmed.endsWith("\uFF1A")) return `${trimmed}${query}`;
|
|
19099
|
-
return `Instruct: ${trimmed}
|
|
19100
|
-
Query: ${query}`;
|
|
19101
|
-
}
|
|
19102
|
-
function clampTextChars(text, maxChars) {
|
|
19103
|
-
if (maxChars === void 0 || text.length <= maxChars) return text;
|
|
19104
|
-
return text.slice(0, maxChars);
|
|
19105
|
-
}
|
|
19106
|
-
|
|
19107
|
-
// ../../packages/node-runtime/src/semantic-index/embedding/local.ts
|
|
19108
|
-
var SOCKS_PROXY_PROTOCOLS = /* @__PURE__ */ new Set(["socks:", "socks4:", "socks5:"]);
|
|
19109
|
-
var LOCAL_ONNX_SESSION_OPTIONS = { intraOpNumThreads: 1, interOpNumThreads: 1 };
|
|
19110
|
-
var HUGGING_FACE_MODEL_HOST = "https://huggingface.co/";
|
|
19111
|
-
var HF_MIRROR_MODEL_HOST = "https://hf-mirror.com/";
|
|
19112
|
-
function resolveModelDownloadRemoteHost(source) {
|
|
19113
|
-
return source === "hf-mirror" ? HF_MIRROR_MODEL_HOST : HUGGING_FACE_MODEL_HOST;
|
|
19114
|
-
}
|
|
19115
|
-
async function createProxyFetch(proxyUrl) {
|
|
19116
|
-
const parsed = new URL(proxyUrl);
|
|
19117
|
-
if (SOCKS_PROXY_PROTOCOLS.has(parsed.protocol)) {
|
|
19118
|
-
throw new Error(
|
|
19119
|
-
`SOCKS proxy is not supported for local embedding model downloads: ${parsed.protocol}//${parsed.host}`
|
|
19120
|
-
);
|
|
19121
|
-
}
|
|
19122
|
-
const { fetch: undiciFetch, ProxyAgent } = await import("./undici-ZSXBT7EA.mjs");
|
|
19123
|
-
const dispatcher = new ProxyAgent(proxyUrl);
|
|
19124
|
-
return ((input, init) => {
|
|
19125
|
-
const nextInit = { ...init, dispatcher };
|
|
19126
|
-
return undiciFetch(input, nextInit);
|
|
19127
|
-
});
|
|
19128
|
-
}
|
|
19129
|
-
var defaultPipelineFactory = async ({
|
|
19130
|
-
modelId,
|
|
19131
|
-
dtype,
|
|
19132
|
-
cacheDir,
|
|
19133
|
-
modelDownloadProxyUrl,
|
|
19134
|
-
modelDownloadSource
|
|
19135
|
-
}) => {
|
|
19136
|
-
const transformers = await import("./transformers.node-PUMHAODN.mjs");
|
|
19137
|
-
transformers.env.remoteHost = resolveModelDownloadRemoteHost(modelDownloadSource);
|
|
19138
|
-
if (cacheDir) {
|
|
19139
|
-
transformers.env.cacheDir = cacheDir;
|
|
19140
|
-
transformers.env.allowRemoteModels = true;
|
|
19141
|
-
}
|
|
19142
|
-
if (modelDownloadProxyUrl) {
|
|
19143
|
-
transformers.env.fetch = await createProxyFetch(modelDownloadProxyUrl);
|
|
19144
|
-
}
|
|
19145
|
-
const extractor = await transformers.pipeline("feature-extraction", modelId, {
|
|
19146
|
-
...dtype ? { dtype } : {},
|
|
19147
|
-
session_options: LOCAL_ONNX_SESSION_OPTIONS
|
|
19148
|
-
});
|
|
19149
|
-
return async (texts, options2) => {
|
|
19150
|
-
const out = await extractor(texts, { pooling: options2.pooling, normalize: options2.normalize });
|
|
19151
|
-
return out.tolist();
|
|
19152
|
-
};
|
|
19153
|
-
};
|
|
19154
|
-
var LocalEmbeddingProvider = class {
|
|
19155
|
-
modelId;
|
|
19156
|
-
dim;
|
|
19157
|
-
maxTokens;
|
|
19158
|
-
documentBatchSize;
|
|
19159
|
-
profile;
|
|
19160
|
-
options;
|
|
19161
|
-
extractorPromise = null;
|
|
19162
|
-
constructor(profile, options2 = {}) {
|
|
19163
|
-
this.profile = profile;
|
|
19164
|
-
this.options = options2;
|
|
19165
|
-
this.modelId = profile.modelId;
|
|
19166
|
-
this.dim = profile.dim;
|
|
19167
|
-
this.maxTokens = profile.maxTokens;
|
|
19168
|
-
this.documentBatchSize = profile.maxBatchSize ?? 1;
|
|
19169
|
-
}
|
|
19170
|
-
getExtractor() {
|
|
19171
|
-
if (!this.extractorPromise) {
|
|
19172
|
-
const factory = this.options.pipelineFactory ?? defaultPipelineFactory;
|
|
19173
|
-
this.extractorPromise = factory({
|
|
19174
|
-
modelId: this.profile.modelId,
|
|
19175
|
-
dtype: this.profile.dtype,
|
|
19176
|
-
cacheDir: this.options.cacheDir,
|
|
19177
|
-
modelDownloadProxyUrl: this.options.modelDownloadProxyUrl,
|
|
19178
|
-
modelDownloadSource: this.options.modelDownloadSource,
|
|
19179
|
-
sessionOptions: LOCAL_ONNX_SESSION_OPTIONS
|
|
19180
|
-
}).catch((error) => {
|
|
19181
|
-
this.extractorPromise = null;
|
|
19182
|
-
throw error;
|
|
19183
|
-
});
|
|
19184
|
-
}
|
|
19185
|
-
return this.extractorPromise;
|
|
19186
|
-
}
|
|
19187
|
-
toFloat32(vector) {
|
|
19188
|
-
if (vector.length !== this.profile.dim) {
|
|
19189
|
-
throw new Error(`model ${this.profile.modelId} returned dim ${vector.length}, expected ${this.profile.dim}`);
|
|
19190
|
-
}
|
|
19191
|
-
return Float32Array.from(vector);
|
|
19192
|
-
}
|
|
19193
|
-
async embedDocuments(texts) {
|
|
19194
|
-
if (texts.length === 0) return [];
|
|
19195
|
-
const extractor = await this.getExtractor();
|
|
19196
|
-
const batchSize = this.profile.maxBatchSize ?? texts.length;
|
|
19197
|
-
const clamped = texts.map((t2) => clampTextChars(t2, this.profile.maxTextChars));
|
|
19198
|
-
const result = [];
|
|
19199
|
-
for (let i = 0; i < clamped.length; i += batchSize) {
|
|
19200
|
-
const batch = clamped.slice(i, i + batchSize);
|
|
19201
|
-
const vectors = await extractor(batch, { pooling: this.profile.pooling, normalize: this.profile.normalize });
|
|
19202
|
-
for (const v of vectors) result.push(this.toFloat32(v));
|
|
19203
|
-
}
|
|
19204
|
-
return result;
|
|
19205
|
-
}
|
|
19206
|
-
async embedQuery(text) {
|
|
19207
|
-
const withInstruction = applyQueryInstruction(this.profile.queryInstruction, text);
|
|
19208
|
-
const [vector] = await this.embedDocuments([withInstruction]);
|
|
19209
|
-
return vector;
|
|
19210
|
-
}
|
|
19211
|
-
async preload() {
|
|
19212
|
-
await this.getExtractor();
|
|
19213
|
-
}
|
|
19214
|
-
};
|
|
19215
|
-
|
|
19216
19847
|
// ../../packages/node-runtime/src/semantic-index/embedding/profiles.ts
|
|
19217
19848
|
var QWEN3_PROFILE = {
|
|
19218
19849
|
displayName: "Qwen3 Embedding 0.6B",
|
|
@@ -19276,39 +19907,7 @@ function createEmbedder(config, deps = {}) {
|
|
|
19276
19907
|
|
|
19277
19908
|
// ../../packages/node-runtime/src/semantic-index/store.ts
|
|
19278
19909
|
import Database2 from "better-sqlite3";
|
|
19279
|
-
|
|
19280
|
-
// ../../node_modules/sqlite-vec/index.mjs
|
|
19281
|
-
import { fileURLToPath } from "url";
|
|
19282
|
-
import { arch, platform } from "process";
|
|
19283
|
-
var BASE_PACKAGE_NAME = "sqlite-vec";
|
|
19284
|
-
var ENTRYPOINT_BASE_NAME = "vec0";
|
|
19285
|
-
var supportedPlatforms = [["darwin", "x64"], ["linux", "x64"], ["darwin", "arm64"], ["win32", "x64"], ["linux", "arm64"]];
|
|
19286
|
-
var invalidPlatformErrorMessage = `Unsupported platform for ${BASE_PACKAGE_NAME}, on a ${platform}-${arch} machine. Supported platforms are (${supportedPlatforms.map(([p, a]) => `${p}-${a}`).join(",")}). Consult the ${BASE_PACKAGE_NAME} NPM package README for details.`;
|
|
19287
|
-
function validPlatform(platform2, arch2) {
|
|
19288
|
-
return supportedPlatforms.find(([p, a]) => platform2 === p && arch2 === a) !== void 0;
|
|
19289
|
-
}
|
|
19290
|
-
function extensionSuffix(platform2) {
|
|
19291
|
-
if (platform2 === "win32") return "dll";
|
|
19292
|
-
if (platform2 === "darwin") return "dylib";
|
|
19293
|
-
return "so";
|
|
19294
|
-
}
|
|
19295
|
-
function platformPackageName(platform2, arch2) {
|
|
19296
|
-
const os3 = platform2 === "win32" ? "windows" : platform2;
|
|
19297
|
-
return `${BASE_PACKAGE_NAME}-${os3}-${arch2}`;
|
|
19298
|
-
}
|
|
19299
|
-
function getLoadablePath() {
|
|
19300
|
-
if (!validPlatform(platform, arch)) {
|
|
19301
|
-
throw new Error(
|
|
19302
|
-
invalidPlatformErrorMessage
|
|
19303
|
-
);
|
|
19304
|
-
}
|
|
19305
|
-
const packageName = platformPackageName(platform, arch);
|
|
19306
|
-
const loadablePath = fileURLToPath(import.meta.resolve(packageName + "/" + ENTRYPOINT_BASE_NAME + "." + extensionSuffix(platform)));
|
|
19307
|
-
return loadablePath;
|
|
19308
|
-
}
|
|
19309
|
-
function load(db) {
|
|
19310
|
-
db.loadExtension(getLoadablePath());
|
|
19311
|
-
}
|
|
19910
|
+
import * as sqliteVec from "sqlite-vec";
|
|
19312
19911
|
|
|
19313
19912
|
// ../../packages/node-runtime/src/semantic-index/schema.ts
|
|
19314
19913
|
var CHUNK_VECTOR_INDEX_TABLE = `
|
|
@@ -19374,7 +19973,7 @@ function vecTableSchema(dim) {
|
|
|
19374
19973
|
}
|
|
19375
19974
|
|
|
19376
19975
|
// ../../packages/node-runtime/src/semantic-index/store.ts
|
|
19377
|
-
var defaultLoadSqliteVec = (db) => load(db);
|
|
19976
|
+
var defaultLoadSqliteVec = (db) => sqliteVec.load(db);
|
|
19378
19977
|
function rowToRecord(row) {
|
|
19379
19978
|
return {
|
|
19380
19979
|
chunkId: row.chunk_id,
|
|
@@ -19820,7 +20419,7 @@ var SemanticIndexJobQueue = class {
|
|
|
19820
20419
|
};
|
|
19821
20420
|
|
|
19822
20421
|
// ../../packages/node-runtime/src/semantic-index/chunker.ts
|
|
19823
|
-
import { createHash as
|
|
20422
|
+
import { createHash as createHash4 } from "crypto";
|
|
19824
20423
|
var VOID_MESSAGE_TYPES = /* @__PURE__ */ new Set([
|
|
19825
20424
|
1 /* IMAGE */,
|
|
19826
20425
|
2 /* VOICE */,
|
|
@@ -19903,7 +20502,7 @@ function sumEffectiveChars(messages) {
|
|
|
19903
20502
|
return total;
|
|
19904
20503
|
}
|
|
19905
20504
|
function sha256Hex2(input) {
|
|
19906
|
-
return
|
|
20505
|
+
return createHash4("sha256").update(input, "utf-8").digest("hex");
|
|
19907
20506
|
}
|
|
19908
20507
|
var EMBEDDING_HEADER_BUDGET_RATIO = 0.25;
|
|
19909
20508
|
var SOURCE_TITLE_HEADER_SHARE = 1 / 3;
|
|
@@ -20479,7 +21078,8 @@ function createSemanticIndexService(params) {
|
|
|
20479
21078
|
sessionAdapter: params.sessionAdapter,
|
|
20480
21079
|
nativeBinding: params.nativeBinding,
|
|
20481
21080
|
loadSqliteVec: params.loadSqliteVec,
|
|
20482
|
-
embedderFactoryDeps: params.embedderFactoryDeps
|
|
21081
|
+
embedderFactoryDeps: params.embedderFactoryDeps,
|
|
21082
|
+
getLocalEmbeddingRuntimeStatus: params.getLocalEmbeddingRuntimeStatus
|
|
20483
21083
|
});
|
|
20484
21084
|
}
|
|
20485
21085
|
var SemanticIndexService = class {
|
|
@@ -20534,7 +21134,8 @@ var SemanticIndexService = class {
|
|
|
20534
21134
|
* 传入 apiKey 时写入固定 auth profile,config 内只保存引用,绝不持久化明文。
|
|
20535
21135
|
*/
|
|
20536
21136
|
getModelStatus() {
|
|
20537
|
-
return this.modelPreloadStatus;
|
|
21137
|
+
if (this.modelPreloadStatus !== "downloading") return this.modelPreloadStatus;
|
|
21138
|
+
return this.options.getLocalEmbeddingRuntimeStatus?.() === "installing" ? "installing-runtime" : "downloading-model";
|
|
20538
21139
|
}
|
|
20539
21140
|
setConfig(config, options2) {
|
|
20540
21141
|
const saved = persistSemanticIndexConfig(this.configStore, config, {
|
|
@@ -20548,11 +21149,17 @@ var SemanticIndexService = class {
|
|
|
20548
21149
|
if (!saved.enabled) {
|
|
20549
21150
|
for (const state of this.stateStore.listEnabled()) this.queue.pause(state.dbPathHash);
|
|
20550
21151
|
}
|
|
20551
|
-
|
|
20552
|
-
|
|
20553
|
-
|
|
20554
|
-
|
|
20555
|
-
|
|
21152
|
+
this.startLocalModelPreload(saved, preloadGeneration);
|
|
21153
|
+
return saved;
|
|
21154
|
+
}
|
|
21155
|
+
startLocalModelPreload(config, preloadGeneration, options2 = {}) {
|
|
21156
|
+
if (!config.enabled || !isSemanticIndexConfigured(config) || config.mode !== "local") return;
|
|
21157
|
+
const modelId = config.local.modelId;
|
|
21158
|
+
const downloadSource = config.local.downloadSource ?? DEFAULT_SEMANTIC_INDEX_MODEL_DOWNLOAD_SOURCE;
|
|
21159
|
+
const cacheDir = this.options.modelsCacheDir;
|
|
21160
|
+
this.modelPreloadStatus = "downloading";
|
|
21161
|
+
const run = () => {
|
|
21162
|
+
if (preloadGeneration !== this.modelPreloadGeneration) return;
|
|
20556
21163
|
appLogger.info("semantic-index", "local embedding model preload started", { modelId, downloadSource, cacheDir });
|
|
20557
21164
|
void this.getEmbedder().preload?.().then(() => {
|
|
20558
21165
|
if (preloadGeneration !== this.modelPreloadGeneration) return;
|
|
@@ -20571,8 +21178,9 @@ var SemanticIndexService = class {
|
|
|
20571
21178
|
error
|
|
20572
21179
|
);
|
|
20573
21180
|
});
|
|
20574
|
-
}
|
|
20575
|
-
|
|
21181
|
+
};
|
|
21182
|
+
if (options2.defer) queueMicrotask(run);
|
|
21183
|
+
else run();
|
|
20576
21184
|
}
|
|
20577
21185
|
currentModelId() {
|
|
20578
21186
|
return this.configStore.resolveModelId();
|
|
@@ -20948,6 +21556,8 @@ ${text}`;
|
|
|
20948
21556
|
this.stateStore.setIndexStatus(state.dbPathHash, "paused");
|
|
20949
21557
|
}
|
|
20950
21558
|
}
|
|
21559
|
+
const config = this.configStore.get();
|
|
21560
|
+
this.startLocalModelPreload(config, ++this.modelPreloadGeneration, { defer: true });
|
|
20951
21561
|
}
|
|
20952
21562
|
// ---------- 队列执行 ----------
|
|
20953
21563
|
resolveSource(db) {
|
|
@@ -21023,9 +21633,269 @@ ${text}`;
|
|
|
21023
21633
|
}
|
|
21024
21634
|
};
|
|
21025
21635
|
|
|
21636
|
+
// ../../packages/node-runtime/src/semantic-index/embedding/local-runtime.ts
|
|
21637
|
+
import { execFile } from "child_process";
|
|
21638
|
+
import fs34 from "fs";
|
|
21639
|
+
import { createRequire as createRequire2 } from "module";
|
|
21640
|
+
import path29 from "path";
|
|
21641
|
+
import { pathToFileURL } from "url";
|
|
21642
|
+
var LOCAL_EMBEDDING_RUNTIME_PACKAGE = "@huggingface/transformers";
|
|
21643
|
+
var LOCAL_EMBEDDING_RUNTIME_VERSION = "4.2.0";
|
|
21644
|
+
var RUNTIME_MANIFEST_FILE = "runtime.json";
|
|
21645
|
+
var VERIFY_RUNTIME_SCRIPT = `
|
|
21646
|
+
import { createRequire } from 'node:module'
|
|
21647
|
+
import path from 'node:path'
|
|
21648
|
+
import { pathToFileURL } from 'node:url'
|
|
21649
|
+
|
|
21650
|
+
const [, runtimeDir, transformersPackage] = process.argv
|
|
21651
|
+
const runtimeRequire = createRequire(path.join(runtimeDir, 'package.json'))
|
|
21652
|
+
const importFromRuntime = async (packageName) => {
|
|
21653
|
+
const resolved = runtimeRequire.resolve(packageName)
|
|
21654
|
+
return await import(pathToFileURL(resolved).href)
|
|
21655
|
+
}
|
|
21656
|
+
|
|
21657
|
+
const transformers = await importFromRuntime(transformersPackage)
|
|
21658
|
+
if (typeof transformers.pipeline !== 'function' || typeof transformers.env !== 'object') {
|
|
21659
|
+
throw new Error('Installed Transformers runtime does not expose the expected Node API.')
|
|
21660
|
+
}
|
|
21661
|
+
|
|
21662
|
+
const onnxRuntime = await importFromRuntime('onnxruntime-node')
|
|
21663
|
+
if (typeof onnxRuntime.listSupportedBackends !== 'function') {
|
|
21664
|
+
throw new Error('Installed ONNX runtime does not expose listSupportedBackends().')
|
|
21665
|
+
}
|
|
21666
|
+
`;
|
|
21667
|
+
function executeInstall(input) {
|
|
21668
|
+
return new Promise((resolve4, reject) => {
|
|
21669
|
+
execFile(
|
|
21670
|
+
input.command,
|
|
21671
|
+
input.args,
|
|
21672
|
+
{
|
|
21673
|
+
cwd: input.cwd,
|
|
21674
|
+
env: { ...process.env, npm_config_update_notifier: "false" },
|
|
21675
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
21676
|
+
shell: process.platform === "win32"
|
|
21677
|
+
},
|
|
21678
|
+
(error, _stdout, stderr) => {
|
|
21679
|
+
if (!error) {
|
|
21680
|
+
resolve4();
|
|
21681
|
+
return;
|
|
21682
|
+
}
|
|
21683
|
+
reject(new Error(stderr.trim() || error.message, { cause: error }));
|
|
21684
|
+
}
|
|
21685
|
+
);
|
|
21686
|
+
});
|
|
21687
|
+
}
|
|
21688
|
+
function verifyRuntimeInSubprocess(runtimeDir) {
|
|
21689
|
+
return new Promise((resolve4, reject) => {
|
|
21690
|
+
execFile(
|
|
21691
|
+
process.execPath,
|
|
21692
|
+
["--input-type=module", "--eval", VERIFY_RUNTIME_SCRIPT, runtimeDir, LOCAL_EMBEDDING_RUNTIME_PACKAGE],
|
|
21693
|
+
{
|
|
21694
|
+
env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" },
|
|
21695
|
+
maxBuffer: 10 * 1024 * 1024
|
|
21696
|
+
},
|
|
21697
|
+
(error, _stdout, stderr) => {
|
|
21698
|
+
if (!error) {
|
|
21699
|
+
resolve4();
|
|
21700
|
+
return;
|
|
21701
|
+
}
|
|
21702
|
+
reject(new Error(stderr.trim() || error.message, { cause: error }));
|
|
21703
|
+
}
|
|
21704
|
+
);
|
|
21705
|
+
});
|
|
21706
|
+
}
|
|
21707
|
+
function createRuntimeRequire(runtimeDir) {
|
|
21708
|
+
return createRequire2(path29.join(runtimeDir, "package.json"));
|
|
21709
|
+
}
|
|
21710
|
+
async function importFromRuntime(runtimeDir, packageName) {
|
|
21711
|
+
const resolved = createRuntimeRequire(runtimeDir).resolve(packageName);
|
|
21712
|
+
return await import(pathToFileURL(resolved).href);
|
|
21713
|
+
}
|
|
21714
|
+
async function verifyInstalledRuntime(runtimeDir) {
|
|
21715
|
+
const transformers = await importFromRuntime(runtimeDir, LOCAL_EMBEDDING_RUNTIME_PACKAGE);
|
|
21716
|
+
if (typeof transformers.pipeline !== "function" || typeof transformers.env !== "object") {
|
|
21717
|
+
throw new Error("Installed Transformers runtime does not expose the expected Node API.");
|
|
21718
|
+
}
|
|
21719
|
+
const onnxRuntime = await importFromRuntime(runtimeDir, "onnxruntime-node");
|
|
21720
|
+
if (typeof onnxRuntime.listSupportedBackends !== "function") {
|
|
21721
|
+
throw new Error("Installed ONNX runtime does not expose listSupportedBackends().");
|
|
21722
|
+
}
|
|
21723
|
+
}
|
|
21724
|
+
async function loadInstalledTransformers(runtimeDir) {
|
|
21725
|
+
return await importFromRuntime(
|
|
21726
|
+
runtimeDir,
|
|
21727
|
+
LOCAL_EMBEDDING_RUNTIME_PACKAGE
|
|
21728
|
+
);
|
|
21729
|
+
}
|
|
21730
|
+
var LocalEmbeddingRuntimeManager = class {
|
|
21731
|
+
baseDir;
|
|
21732
|
+
installMode;
|
|
21733
|
+
platform;
|
|
21734
|
+
arch;
|
|
21735
|
+
npmCommand;
|
|
21736
|
+
runInstall;
|
|
21737
|
+
verifyStagedRuntime;
|
|
21738
|
+
verifyRuntime;
|
|
21739
|
+
importTransformers;
|
|
21740
|
+
status;
|
|
21741
|
+
installPromise = null;
|
|
21742
|
+
transformersPromise = null;
|
|
21743
|
+
verifiedRuntimeDir = null;
|
|
21744
|
+
constructor(options2) {
|
|
21745
|
+
this.baseDir = options2.baseDir;
|
|
21746
|
+
this.installMode = options2.installMode;
|
|
21747
|
+
this.platform = options2.platform ?? process.platform;
|
|
21748
|
+
this.arch = options2.arch ?? process.arch;
|
|
21749
|
+
this.npmCommand = options2.npmCommand ?? (this.platform === "win32" ? "npm.cmd" : "npm");
|
|
21750
|
+
this.runInstall = options2.runInstall ?? executeInstall;
|
|
21751
|
+
this.verifyStagedRuntime = options2.verifyStagedRuntime ?? verifyRuntimeInSubprocess;
|
|
21752
|
+
this.verifyRuntime = options2.verifyRuntime ?? verifyInstalledRuntime;
|
|
21753
|
+
this.importTransformers = options2.importTransformers ?? loadInstalledTransformers;
|
|
21754
|
+
this.status = "idle";
|
|
21755
|
+
}
|
|
21756
|
+
getStatus() {
|
|
21757
|
+
return this.status;
|
|
21758
|
+
}
|
|
21759
|
+
getRuntimeDir() {
|
|
21760
|
+
return path29.join(this.baseDir, `transformers-${LOCAL_EMBEDDING_RUNTIME_VERSION}`, `${this.platform}-${this.arch}`);
|
|
21761
|
+
}
|
|
21762
|
+
async ensureInstalled() {
|
|
21763
|
+
if (!this.installPromise) {
|
|
21764
|
+
this.installPromise = this.ensureInstalledOnce().finally(() => {
|
|
21765
|
+
this.installPromise = null;
|
|
21766
|
+
});
|
|
21767
|
+
}
|
|
21768
|
+
return await this.installPromise;
|
|
21769
|
+
}
|
|
21770
|
+
async loadTransformers() {
|
|
21771
|
+
if (!this.transformersPromise) {
|
|
21772
|
+
this.transformersPromise = this.ensureInstalled().then((runtimeDir) => this.importTransformers(runtimeDir)).catch((error) => {
|
|
21773
|
+
this.transformersPromise = null;
|
|
21774
|
+
throw error;
|
|
21775
|
+
});
|
|
21776
|
+
}
|
|
21777
|
+
return await this.transformersPromise;
|
|
21778
|
+
}
|
|
21779
|
+
async ensureInstalledOnce() {
|
|
21780
|
+
const runtimeDir = this.getRuntimeDir();
|
|
21781
|
+
try {
|
|
21782
|
+
if (this.hasCompleteLayout(runtimeDir)) {
|
|
21783
|
+
try {
|
|
21784
|
+
await this.verifyOnce(runtimeDir);
|
|
21785
|
+
this.status = "ready";
|
|
21786
|
+
return runtimeDir;
|
|
21787
|
+
} catch (error) {
|
|
21788
|
+
if (this.installMode === "preinstalled") throw error;
|
|
21789
|
+
appLogger.warn("semantic-index", "Existing local embedding runtime is invalid; reinstalling", {
|
|
21790
|
+
packageVersion: LOCAL_EMBEDDING_RUNTIME_VERSION,
|
|
21791
|
+
platform: this.platform,
|
|
21792
|
+
arch: this.arch,
|
|
21793
|
+
error: error instanceof Error ? error.message : String(error)
|
|
21794
|
+
});
|
|
21795
|
+
fs34.rmSync(runtimeDir, { recursive: true, force: true });
|
|
21796
|
+
this.verifiedRuntimeDir = null;
|
|
21797
|
+
}
|
|
21798
|
+
}
|
|
21799
|
+
if (this.installMode === "preinstalled") {
|
|
21800
|
+
throw new Error(`Preinstalled local embedding runtime is missing or incomplete: ${runtimeDir}`);
|
|
21801
|
+
}
|
|
21802
|
+
this.status = "installing";
|
|
21803
|
+
appLogger.info("semantic-index", "Local embedding runtime installation started", {
|
|
21804
|
+
packageVersion: LOCAL_EMBEDDING_RUNTIME_VERSION,
|
|
21805
|
+
platform: this.platform,
|
|
21806
|
+
arch: this.arch
|
|
21807
|
+
});
|
|
21808
|
+
await this.installAtomically(runtimeDir);
|
|
21809
|
+
await this.verifyOnce(runtimeDir);
|
|
21810
|
+
this.status = "ready";
|
|
21811
|
+
appLogger.info("semantic-index", "Local embedding runtime installation completed", {
|
|
21812
|
+
packageVersion: LOCAL_EMBEDDING_RUNTIME_VERSION,
|
|
21813
|
+
platform: this.platform,
|
|
21814
|
+
arch: this.arch
|
|
21815
|
+
});
|
|
21816
|
+
return runtimeDir;
|
|
21817
|
+
} catch (error) {
|
|
21818
|
+
this.status = "error";
|
|
21819
|
+
appLogger.error("semantic-index", "Local embedding runtime installation failed", error);
|
|
21820
|
+
throw error;
|
|
21821
|
+
}
|
|
21822
|
+
}
|
|
21823
|
+
async installAtomically(runtimeDir) {
|
|
21824
|
+
const parentDir = path29.dirname(runtimeDir);
|
|
21825
|
+
fs34.mkdirSync(parentDir, { recursive: true });
|
|
21826
|
+
const stagingDir = fs34.mkdtempSync(path29.join(parentDir, ".installing-"));
|
|
21827
|
+
try {
|
|
21828
|
+
fs34.writeFileSync(
|
|
21829
|
+
path29.join(stagingDir, "package.json"),
|
|
21830
|
+
`${JSON.stringify({ private: true, type: "module" }, null, 2)}
|
|
21831
|
+
`,
|
|
21832
|
+
"utf8"
|
|
21833
|
+
);
|
|
21834
|
+
await this.runInstall({
|
|
21835
|
+
command: this.npmCommand,
|
|
21836
|
+
args: [
|
|
21837
|
+
"install",
|
|
21838
|
+
"--omit=dev",
|
|
21839
|
+
"--no-audit",
|
|
21840
|
+
"--no-fund",
|
|
21841
|
+
"--save-exact",
|
|
21842
|
+
"--loglevel=error",
|
|
21843
|
+
"--prefix",
|
|
21844
|
+
stagingDir,
|
|
21845
|
+
`${LOCAL_EMBEDDING_RUNTIME_PACKAGE}@${LOCAL_EMBEDDING_RUNTIME_VERSION}`
|
|
21846
|
+
],
|
|
21847
|
+
cwd: stagingDir
|
|
21848
|
+
});
|
|
21849
|
+
await this.verifyStagedRuntime(stagingDir);
|
|
21850
|
+
fs34.writeFileSync(path29.join(stagingDir, RUNTIME_MANIFEST_FILE), `${JSON.stringify(this.manifest(), null, 2)}
|
|
21851
|
+
`);
|
|
21852
|
+
if (fs34.existsSync(runtimeDir)) {
|
|
21853
|
+
if (this.hasCompleteLayout(runtimeDir)) return;
|
|
21854
|
+
fs34.rmSync(runtimeDir, { recursive: true, force: true });
|
|
21855
|
+
}
|
|
21856
|
+
try {
|
|
21857
|
+
fs34.renameSync(stagingDir, runtimeDir);
|
|
21858
|
+
} catch (error) {
|
|
21859
|
+
if (!this.hasCompleteLayout(runtimeDir)) throw error;
|
|
21860
|
+
}
|
|
21861
|
+
} finally {
|
|
21862
|
+
fs34.rmSync(stagingDir, { recursive: true, force: true });
|
|
21863
|
+
}
|
|
21864
|
+
}
|
|
21865
|
+
async verifyOnce(runtimeDir) {
|
|
21866
|
+
if (this.verifiedRuntimeDir === runtimeDir) return;
|
|
21867
|
+
await this.verifyRuntime(runtimeDir);
|
|
21868
|
+
this.verifiedRuntimeDir = runtimeDir;
|
|
21869
|
+
}
|
|
21870
|
+
hasCompleteLayout(runtimeDir) {
|
|
21871
|
+
const manifestPath = path29.join(runtimeDir, RUNTIME_MANIFEST_FILE);
|
|
21872
|
+
const packagePath = path29.join(runtimeDir, "node_modules", "@huggingface", "transformers", "package.json");
|
|
21873
|
+
if (!fs34.existsSync(manifestPath) || !fs34.existsSync(packagePath)) return false;
|
|
21874
|
+
try {
|
|
21875
|
+
const manifest = JSON.parse(fs34.readFileSync(manifestPath, "utf8"));
|
|
21876
|
+
const expected = this.manifest();
|
|
21877
|
+
return manifest.formatVersion === expected.formatVersion && manifest.packageName === expected.packageName && manifest.packageVersion === expected.packageVersion && manifest.platform === expected.platform && manifest.arch === expected.arch;
|
|
21878
|
+
} catch {
|
|
21879
|
+
return false;
|
|
21880
|
+
}
|
|
21881
|
+
}
|
|
21882
|
+
manifest() {
|
|
21883
|
+
return {
|
|
21884
|
+
formatVersion: 1,
|
|
21885
|
+
packageName: LOCAL_EMBEDDING_RUNTIME_PACKAGE,
|
|
21886
|
+
packageVersion: LOCAL_EMBEDDING_RUNTIME_VERSION,
|
|
21887
|
+
platform: this.platform,
|
|
21888
|
+
arch: this.arch
|
|
21889
|
+
};
|
|
21890
|
+
}
|
|
21891
|
+
};
|
|
21892
|
+
function createLocalEmbeddingRuntimeManager(options2) {
|
|
21893
|
+
return new LocalEmbeddingRuntimeManager(options2);
|
|
21894
|
+
}
|
|
21895
|
+
|
|
21026
21896
|
// ../../packages/node-runtime/src/services/merge-cache.ts
|
|
21027
|
-
import * as
|
|
21028
|
-
import * as
|
|
21897
|
+
import * as fs35 from "fs";
|
|
21898
|
+
import * as path30 from "path";
|
|
21029
21899
|
import * as crypto from "crypto";
|
|
21030
21900
|
var MergeSessionCache = class {
|
|
21031
21901
|
cache = /* @__PURE__ */ new Map();
|
|
@@ -21034,15 +21904,15 @@ var MergeSessionCache = class {
|
|
|
21034
21904
|
constructor(pathProvider, options2) {
|
|
21035
21905
|
this.tempDir = pathProvider.getTempDir();
|
|
21036
21906
|
this.nativeBinding = options2?.nativeBinding;
|
|
21037
|
-
if (!
|
|
21038
|
-
|
|
21907
|
+
if (!fs35.existsSync(this.tempDir)) {
|
|
21908
|
+
fs35.mkdirSync(this.tempDir, { recursive: true });
|
|
21039
21909
|
}
|
|
21040
21910
|
}
|
|
21041
21911
|
generateTempDbPath(filename) {
|
|
21042
21912
|
const ts = Date.now();
|
|
21043
21913
|
const rand = Math.random().toString(36).slice(2, 8);
|
|
21044
21914
|
const safeName = filename.replace(/[/\\?%*:|"<>]/g, "_").slice(0, 40);
|
|
21045
|
-
return
|
|
21915
|
+
return path30.join(this.tempDir, `merge_${safeName}_${ts}_${rand}.db`);
|
|
21046
21916
|
}
|
|
21047
21917
|
/** Store a parsed file's temp DB and return a handle for subsequent requests. */
|
|
21048
21918
|
store(filename, tempDbPath) {
|
|
@@ -21053,7 +21923,7 @@ var MergeSessionCache = class {
|
|
|
21053
21923
|
/** Open a TempDbReader for a given handle. Caller must close the reader. */
|
|
21054
21924
|
openReader(handle) {
|
|
21055
21925
|
const entry = this.cache.get(handle);
|
|
21056
|
-
if (!entry || !
|
|
21926
|
+
if (!entry || !fs35.existsSync(entry.tempDbPath)) return null;
|
|
21057
21927
|
const adapter = openBetterSqliteDatabase(entry.tempDbPath, {
|
|
21058
21928
|
readonly: true,
|
|
21059
21929
|
nativeBinding: this.nativeBinding
|
|
@@ -21098,7 +21968,7 @@ var MergeSessionCache = class {
|
|
|
21098
21968
|
};
|
|
21099
21969
|
|
|
21100
21970
|
// ../../packages/node-runtime/src/services/push-importer.ts
|
|
21101
|
-
import * as
|
|
21971
|
+
import * as fs36 from "fs";
|
|
21102
21972
|
var SYSTEM_SENDER_ID3 = "SYSTEM";
|
|
21103
21973
|
var SYSTEM_MEMBER_NAME2 = "\u7CFB\u7EDF\u6D88\u606F";
|
|
21104
21974
|
function isRecord5(value) {
|
|
@@ -21404,10 +22274,11 @@ function writeIncrementalImport(db, payload) {
|
|
|
21404
22274
|
}
|
|
21405
22275
|
if (writtenCount > 0) {
|
|
21406
22276
|
try {
|
|
22277
|
+
const sessionGapThreshold = getSessionIndexStats(db).gapThreshold;
|
|
21407
22278
|
if (minWrittenTs < preWriteMaxTs) {
|
|
21408
|
-
generateSessionIndex(db);
|
|
22279
|
+
generateSessionIndex(db, sessionGapThreshold);
|
|
21409
22280
|
} else {
|
|
21410
|
-
generateIncrementalSessionIndex(db);
|
|
22281
|
+
generateIncrementalSessionIndex(db, sessionGapThreshold);
|
|
21411
22282
|
}
|
|
21412
22283
|
} catch {
|
|
21413
22284
|
}
|
|
@@ -21465,7 +22336,7 @@ async function executePushImportUnlocked(deps, sessionId, payload) {
|
|
|
21465
22336
|
return { ok: false, reason: "invalid_payload", message: "sessionId contains invalid characters" };
|
|
21466
22337
|
}
|
|
21467
22338
|
const dbPath = deps.getDbPath(sessionId);
|
|
21468
|
-
const isNew = !
|
|
22339
|
+
const isNew = !fs36.existsSync(dbPath);
|
|
21469
22340
|
const validationError = validatePayload(payload, isNew);
|
|
21470
22341
|
if (validationError) {
|
|
21471
22342
|
return { ok: false, reason: "invalid_payload", message: validationError };
|
|
@@ -21526,10 +22397,10 @@ async function executePushImportUnlocked(deps, sessionId, payload) {
|
|
|
21526
22397
|
}
|
|
21527
22398
|
|
|
21528
22399
|
// ../../packages/node-runtime/src/services/import-idempotency.ts
|
|
21529
|
-
import { createHash as
|
|
22400
|
+
import { createHash as createHash5 } from "crypto";
|
|
21530
22401
|
var DEFAULT_IMPORT_IDEMPOTENCY_TTL_MS = 60 * 60 * 1e3;
|
|
21531
22402
|
function hashImportBody(body) {
|
|
21532
|
-
return
|
|
22403
|
+
return createHash5("sha256").update(JSON.stringify(body ?? "")).digest("hex");
|
|
21533
22404
|
}
|
|
21534
22405
|
var ImportIdempotencyCache = class {
|
|
21535
22406
|
constructor(ttlMs = DEFAULT_IMPORT_IDEMPOTENCY_TTL_MS) {
|
|
@@ -21581,6 +22452,7 @@ export {
|
|
|
21581
22452
|
findEntryFileInDirectory,
|
|
21582
22453
|
logNativeParserStatus,
|
|
21583
22454
|
LogLevel,
|
|
22455
|
+
createImportPerfLogger,
|
|
21584
22456
|
initPerfLog,
|
|
21585
22457
|
logPerf,
|
|
21586
22458
|
logPerfDetail,
|
|
@@ -21604,6 +22476,8 @@ export {
|
|
|
21604
22476
|
rebuildFtsIndex,
|
|
21605
22477
|
insertFtsEntries,
|
|
21606
22478
|
searchByFts,
|
|
22479
|
+
MESSAGE_INSERT_MAX_ROWS,
|
|
22480
|
+
MessageBatchInserter,
|
|
21607
22481
|
streamingImport,
|
|
21608
22482
|
analyzeNewImport,
|
|
21609
22483
|
streamParseFileInfo,
|
|
@@ -21611,8 +22485,13 @@ export {
|
|
|
21611
22485
|
incrementalImport,
|
|
21612
22486
|
isValidImportSessionId,
|
|
21613
22487
|
resolveAutoImportTarget,
|
|
22488
|
+
resolveAutoImportTargetPlan,
|
|
21614
22489
|
autoImportFile,
|
|
21615
22490
|
analyzeAutoImportFile,
|
|
22491
|
+
normalizeBatchConcurrency,
|
|
22492
|
+
resolveDefaultBatchConcurrency,
|
|
22493
|
+
runKeyedBatch,
|
|
22494
|
+
autoImportBatch,
|
|
21616
22495
|
IMPORT_LOCK_FILENAME,
|
|
21617
22496
|
IMPORT_IN_PROGRESS_ERROR_KEY,
|
|
21618
22497
|
ImportInProgressError,
|
|
@@ -21761,6 +22640,7 @@ export {
|
|
|
21761
22640
|
OpenAICompatibleEmbeddingProvider,
|
|
21762
22641
|
applyQueryInstruction,
|
|
21763
22642
|
clampTextChars,
|
|
22643
|
+
createTransformersPipelineFactory,
|
|
21764
22644
|
LocalEmbeddingProvider,
|
|
21765
22645
|
QWEN3_PROFILE,
|
|
21766
22646
|
BGE_BASE_PROFILE,
|
|
@@ -21786,6 +22666,9 @@ export {
|
|
|
21786
22666
|
SEMANTIC_INDEX_CONFIG_FILE,
|
|
21787
22667
|
createSemanticIndexService,
|
|
21788
22668
|
SemanticIndexService,
|
|
22669
|
+
LOCAL_EMBEDDING_RUNTIME_PACKAGE,
|
|
22670
|
+
LOCAL_EMBEDDING_RUNTIME_VERSION,
|
|
22671
|
+
createLocalEmbeddingRuntimeManager,
|
|
21789
22672
|
MergeSessionCache,
|
|
21790
22673
|
pushImport,
|
|
21791
22674
|
executePushImportUnlocked,
|
|
@@ -21793,4 +22676,4 @@ export {
|
|
|
21793
22676
|
hashImportBody,
|
|
21794
22677
|
ImportIdempotencyCache
|
|
21795
22678
|
};
|
|
21796
|
-
//# sourceMappingURL=chunk-
|
|
22679
|
+
//# sourceMappingURL=chunk-4LW6443F.mjs.map
|