chatlab-cli 0.34.0 → 0.34.2-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chat-command-XPY3WURC.mjs → chat-command-6IOH2QRF.mjs} +9 -9
- package/dist/{chunk-S5NCEESV.mjs → chunk-2XCNTG4O.mjs} +403 -76
- package/dist/chunk-2XCNTG4O.mjs.map +1 -0
- package/dist/{chunk-AQPDZAWB.mjs → chunk-4LW6443F.mjs} +1802 -712
- package/dist/chunk-4LW6443F.mjs.map +1 -0
- package/dist/{chunk-ISRLGLUL.mjs → chunk-4RLZFGO5.mjs} +2 -2
- package/dist/{chunk-IGEHMLHM.mjs → chunk-52P5ZRBD.mjs} +2 -2
- package/dist/{chunk-JTOHAMIW.mjs → chunk-652KEAX3.mjs} +2 -2
- package/dist/{chunk-DGF3S3TZ.mjs → chunk-72RWABYJ.mjs} +15 -9
- package/dist/chunk-72RWABYJ.mjs.map +1 -0
- package/dist/{chunk-JCSPDTZL.mjs → chunk-H6D4DZOC.mjs} +4 -4
- package/dist/{chunk-OLE2USEN.mjs → chunk-INBJ7SND.mjs} +53 -26
- package/dist/chunk-INBJ7SND.mjs.map +1 -0
- package/dist/{chunk-IGK6UMN2.mjs → chunk-JX6MFS6F.mjs} +43 -23
- package/dist/chunk-JX6MFS6F.mjs.map +1 -0
- package/dist/{chunk-UO2XVFY6.mjs → chunk-MENKFEPC.mjs} +147 -73
- package/dist/chunk-MENKFEPC.mjs.map +1 -0
- package/dist/{chunk-BQ3QN4QT.mjs → chunk-U6MVJSXX.mjs} +170 -66
- package/dist/chunk-U6MVJSXX.mjs.map +1 -0
- package/dist/chunk-UMBEHBIZ.mjs +41 -0
- package/dist/chunk-UMBEHBIZ.mjs.map +1 -0
- package/dist/{chunk-GFHSSCCR.mjs → chunk-VSOWNKPA.mjs} +1 -11
- package/dist/cli.mjs +10 -9
- package/dist/contacts-worker.mjs +3 -3
- package/dist/global-insight-worker.mjs +2 -2
- package/dist/{http-HV6XWU6X.mjs → http-HSK46XEU.mjs} +13 -12
- package/dist/{import-DP2XLZ2H.mjs → import-DF2WD2HE.mjs} +10 -8
- package/dist/index.mjs +13 -12
- package/dist/{mcp-QDPFBPGW.mjs → mcp-A2RND7KN.mjs} +12 -12
- package/dist/people-relationships-worker.mjs +3 -3
- package/dist/semantic-index-worker.mjs +1 -1
- package/dist/{service-KGL3RAIA.mjs → service-KZTHBDQ5.mjs} +2 -2
- package/dist/{src-B3A2QHP5.mjs → src-SHFHW2QB.mjs} +29 -7
- package/dist/{stream-import-KEGNP3QK.mjs → stream-import-GZ2QDE3Z.mjs} +10 -8
- package/dist/{undici-ZSXBT7EA.mjs → undici-MU5I3L6Q.mjs} +2 -2
- package/dist/{update-checker-LGMTEKMT.mjs → update-checker-3N7HQEP2.mjs} +2 -2
- package/dist/{worker-runtime-MDYKX3BW.mjs → worker-runtime-OWN56GG5.mjs} +14 -8
- package/dist/worker-runtime-OWN56GG5.mjs.map +1 -0
- package/dist-cli-web/assets/{ActionToolsPanel.vue_vue_type_script_setup_true_lang-JxPR2O6b.js → ActionToolsPanel.vue_vue_type_script_setup_true_lang-yr_T4ttU.js} +14 -14
- package/dist-cli-web/assets/{BufferResource--dSbJb0k.js → BufferResource-B-LHxriL.js} +1 -1
- package/dist-cli-web/assets/{CanvasRenderer-Cm_dmJXZ.js → CanvasRenderer-W3h_GGHM.js} +1 -1
- package/dist-cli-web/assets/{DataDirCleanupNotice-B0s5txX3.js → DataDirCleanupNotice-C6CMNhfs.js} +1 -1
- package/dist-cli-web/assets/{EChartWordcloud-CwMVJLBO.js → EChartWordcloud-Dt6rCzHf.js} +1 -1
- package/dist-cli-web/assets/{Filter-BjYScRpK.js → Filter-BpSopVKd.js} +1 -1
- package/dist-cli-web/assets/{LoadingState.vue_vue_type_script_setup_true_lang-BshQybD_.js → LoadingState.vue_vue_type_script_setup_true_lang-Bi_Uag3T.js} +1 -1
- package/dist-cli-web/assets/{MessageExportModal-DAJW2mFj.js → MessageExportModal-BoK-_h25.js} +1 -1
- package/dist-cli-web/assets/{PageHeader-BKAjCuqs.js → PageHeader-CD8B7Emf.js} +1 -1
- package/dist-cli-web/assets/{RelationshipGalaxyCanvas-mlZs7k4L.js → RelationshipGalaxyCanvas-BO3mo-Re.js} +7 -7
- package/dist-cli-web/assets/RelationshipGalaxyThreeCanvas-Brnpz6Ig.js +4108 -0
- package/dist-cli-web/assets/{RelationshipGalaxyThreeCanvas-CeYJABxI.css → RelationshipGalaxyThreeCanvas-CMuibfiA.css} +1 -1
- package/dist-cli-web/assets/{RenderTargetSystem-D00Ex44y.js → RenderTargetSystem-CNP4JLfK.js} +1 -1
- package/dist-cli-web/assets/{ThemeCard.vue_vue_type_script_setup_true_lang-Dsb6lzs-.js → ThemeCard.vue_vue_type_script_setup_true_lang-Na3LIoSJ.js} +1 -1
- package/dist-cli-web/assets/{TimeSelect.vue_vue_type_script_setup_true_lang-BfR3tlkz.js → TimeSelect.vue_vue_type_script_setup_true_lang-CKe1YZrV.js} +1 -1
- package/dist-cli-web/assets/{WebGLRenderer-CTTusm2n.js → WebGLRenderer-CIR5cTQN.js} +1 -1
- package/dist-cli-web/assets/{WebGPURenderer-pyv3EXor.js → WebGPURenderer-Dz3pCrzX.js} +1 -1
- package/dist-cli-web/assets/{annual-summary-time-range-C_d1oZrj.js → annual-summary-time-range-QhoA3uC6.js} +1 -1
- package/dist-cli-web/assets/{base-D91aVUwM.js → base-p9yfnTpk.js} +1 -1
- package/dist-cli-web/assets/{browserAll-C9fW936Y.js → browserAll-DMaXjf8N.js} +1 -1
- package/dist-cli-web/assets/cli-web-Dvy2aHxG.js +1 -0
- package/dist-cli-web/assets/{dateFormat-rY43adcv.js → dateFormat-5x3JKlIk.js} +1 -1
- package/dist-cli-web/assets/{fetch-CbnX5VoM.js → fetch-B0p6QRQ8.js} +1 -1
- package/dist-cli-web/assets/{fetch-DYNgtIcU.js → fetch-BI-Se2p7.js} +1 -1
- package/dist-cli-web/assets/{fetch-Kk2knkEo.js → fetch-BderOzzL.js} +1 -1
- package/dist-cli-web/assets/{fetch-DG3AeEjW.js → fetch-BtJWW2-O.js} +1 -1
- package/dist-cli-web/assets/fetch-CBaZNOpK.js +4 -0
- package/dist-cli-web/assets/{fetch-CNy9OQ2g.js → fetch-CH9Hm20R.js} +1 -1
- package/dist-cli-web/assets/{fetch-CrSz5KJy.js → fetch-CbpKKkdO.js} +1 -1
- package/dist-cli-web/assets/{fetch-PLrI3V67.js → fetch-Cne-uL1m.js} +1 -1
- package/dist-cli-web/assets/{fetch-DZPC3D8R.js → fetch-DRGykJYb.js} +1 -1
- package/dist-cli-web/assets/{fetch-CS3T-v-1.js → fetch-QLurhC_z.js} +1 -1
- package/dist-cli-web/assets/{index-JYfjMRb-.js → index-4NL2pYn0.js} +1 -1
- package/dist-cli-web/assets/{index-2-OaYdzt.js → index-B-KdENnr.js} +1 -1
- package/dist-cli-web/assets/{index-CGI3Y8pq.js → index-BDyixm5n.js} +1 -1
- package/dist-cli-web/assets/{index-BKo68Iu5.js → index-BaCd1avL.js} +1 -1
- package/dist-cli-web/assets/{index-ZTgfCZvY.js → index-CVBBDI3B.js} +3 -3
- package/dist-cli-web/assets/index-CXh01dqh.js +2 -0
- package/dist-cli-web/assets/index-CioAsn_2.js +137 -0
- package/dist-cli-web/assets/{index-DA3Q0TGT.js → index-Ct90lmPb.js} +1 -1
- package/dist-cli-web/assets/index-CumZ5uQ4.js +53 -0
- package/dist-cli-web/assets/index-CxGtGxLW.css +1 -0
- package/dist-cli-web/assets/{index-DsdMsKYT.js → index-DF2Dijfg.js} +1 -1
- package/dist-cli-web/assets/{index-3YO6RI5e.js → index-DYCPQMOl.js} +2 -2
- package/dist-cli-web/assets/{index-DFHYr_-u.js → index-Dp2uglLH.js} +1 -1
- package/dist-cli-web/assets/{index-BrkDgcJ6.js → index-IvxuxPNt.js} +1 -1
- package/dist-cli-web/assets/{people-page-header-D-LWaEqn.js → people-page-header-BRb9BrgP.js} +1 -1
- package/dist-cli-web/assets/telemetry-DPC69b73.js +1 -0
- package/dist-cli-web/assets/{webworkerAll-CPOVECup.js → webworkerAll-BoY19vZt.js} +1 -1
- package/dist-cli-web/index.html +1 -1
- package/package.json +3 -2
- package/dist/chunk-AQPDZAWB.mjs.map +0 -1
- package/dist/chunk-BQ3QN4QT.mjs.map +0 -1
- package/dist/chunk-DGF3S3TZ.mjs.map +0 -1
- package/dist/chunk-IGK6UMN2.mjs.map +0 -1
- package/dist/chunk-OLE2USEN.mjs.map +0 -1
- package/dist/chunk-S5NCEESV.mjs.map +0 -1
- package/dist/chunk-UO2XVFY6.mjs.map +0 -1
- package/dist/onnxruntime_binding-6Q6HXASN.node +0 -0
- package/dist/onnxruntime_binding-EKZT2NRK.node +0 -0
- package/dist/onnxruntime_binding-P6S7V3CI.node +0 -0
- package/dist/onnxruntime_binding-PJNNIIUO.node +0 -0
- package/dist/onnxruntime_binding-UN6SPTQK.node +0 -0
- package/dist/transformers.node-PUMHAODN.mjs +0 -42889
- package/dist/transformers.node-PUMHAODN.mjs.map +0 -1
- package/dist/worker-runtime-MDYKX3BW.mjs.map +0 -1
- package/dist-cli-web/assets/RelationshipGalaxyThreeCanvas-DgMG3jdU.js +0 -4108
- package/dist-cli-web/assets/cli-web-C7uGKz_E.js +0 -1
- package/dist-cli-web/assets/fetch-C6po4XYB.js +0 -3
- package/dist-cli-web/assets/index-2S80bo1s.js +0 -2
- package/dist-cli-web/assets/index-CW9H_fBd.js +0 -53
- package/dist-cli-web/assets/index-DmrrdYXH.css +0 -1
- package/dist-cli-web/assets/index-DmwA85vG.js +0 -173
- /package/dist/{chat-command-XPY3WURC.mjs.map → chat-command-6IOH2QRF.mjs.map} +0 -0
- /package/dist/{chunk-ISRLGLUL.mjs.map → chunk-4RLZFGO5.mjs.map} +0 -0
- /package/dist/{chunk-IGEHMLHM.mjs.map → chunk-52P5ZRBD.mjs.map} +0 -0
- /package/dist/{chunk-JTOHAMIW.mjs.map → chunk-652KEAX3.mjs.map} +0 -0
- /package/dist/{chunk-JCSPDTZL.mjs.map → chunk-H6D4DZOC.mjs.map} +0 -0
- /package/dist/{chunk-GFHSSCCR.mjs.map → chunk-VSOWNKPA.mjs.map} +0 -0
- /package/dist/{http-HV6XWU6X.mjs.map → http-HSK46XEU.mjs.map} +0 -0
- /package/dist/{import-DP2XLZ2H.mjs.map → import-DF2WD2HE.mjs.map} +0 -0
- /package/dist/{mcp-QDPFBPGW.mjs.map → mcp-A2RND7KN.mjs.map} +0 -0
- /package/dist/{service-KGL3RAIA.mjs.map → service-KZTHBDQ5.mjs.map} +0 -0
- /package/dist/{src-B3A2QHP5.mjs.map → src-SHFHW2QB.mjs.map} +0 -0
- /package/dist/{stream-import-KEGNP3QK.mjs.map → stream-import-GZ2QDE3Z.mjs.map} +0 -0
- /package/dist/{undici-ZSXBT7EA.mjs.map → undici-MU5I3L6Q.mjs.map} +0 -0
- /package/dist/{update-checker-LGMTEKMT.mjs.map → update-checker-3N7HQEP2.mjs.map} +0 -0
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
createEmptyContactsDiagnostics,
|
|
10
10
|
normalizeContactsTimeRangePreset,
|
|
11
11
|
resolveContactsTimeRange
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-52P5ZRBD.mjs";
|
|
13
13
|
import {
|
|
14
14
|
PEOPLE_RELATIONSHIPS_ALGORITHM_VERSION,
|
|
15
15
|
buildPeopleRelationshipsNeighborhoodGraph,
|
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
computePeopleRelationshipsSnapshot,
|
|
19
19
|
normalizePeopleRelationshipsTimeRangePreset,
|
|
20
20
|
resolvePeopleRelationshipsTimeRange
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-652KEAX3.mjs";
|
|
22
22
|
import {
|
|
23
23
|
ANNUAL_SUMMARY_ALGORITHM_VERSION
|
|
24
24
|
} from "./chunk-QZCNVAH6.mjs";
|
|
@@ -90,6 +90,7 @@ import {
|
|
|
90
90
|
needsStreamingThinkParsing,
|
|
91
91
|
normalizeAnnualSummaryRange,
|
|
92
92
|
normalizeBuiltinToolNames,
|
|
93
|
+
normalizeSessionGapThreshold,
|
|
93
94
|
openBetterSqliteDatabase,
|
|
94
95
|
readAnnualSummarySnapshot,
|
|
95
96
|
readJsonFile,
|
|
@@ -110,13 +111,13 @@ import {
|
|
|
110
111
|
writeAuthProfile,
|
|
111
112
|
writeJsonFileAtomically,
|
|
112
113
|
writeParseResultToDb
|
|
113
|
-
} from "./chunk-
|
|
114
|
+
} from "./chunk-INBJ7SND.mjs";
|
|
114
115
|
import {
|
|
115
116
|
__commonJS,
|
|
116
117
|
__export,
|
|
117
118
|
__require,
|
|
118
119
|
__toESM
|
|
119
|
-
} from "./chunk-
|
|
120
|
+
} from "./chunk-VSOWNKPA.mjs";
|
|
120
121
|
|
|
121
122
|
// ../../node_modules/stream-json/utils/Utf8Stream.js
|
|
122
123
|
var require_Utf8Stream = __commonJS({
|
|
@@ -726,8 +727,8 @@ var require_FilterBase = __commonJS({
|
|
|
726
727
|
var FilterBase = class _FilterBase extends Transform {
|
|
727
728
|
static stringFilter(string, separator) {
|
|
728
729
|
return (stack) => {
|
|
729
|
-
const
|
|
730
|
-
return
|
|
730
|
+
const path31 = stack.join(separator);
|
|
731
|
+
return path31.length === string.length && path31 === string || path31.length > string.length && path31.substr(0, string.length) === string && path31.substr(string.length, separator.length) === separator;
|
|
731
732
|
};
|
|
732
733
|
}
|
|
733
734
|
static regExpFilter(regExp, separator) {
|
|
@@ -1181,12 +1182,12 @@ var require_Assembler = __commonJS({
|
|
|
1181
1182
|
return (this.stack.length >> 1) + (this.done ? 0 : 1);
|
|
1182
1183
|
}
|
|
1183
1184
|
get path() {
|
|
1184
|
-
const
|
|
1185
|
+
const path31 = [];
|
|
1185
1186
|
for (let i = 0; i < this.stack.length; i += 2) {
|
|
1186
1187
|
const key = this.stack[i + 1];
|
|
1187
|
-
|
|
1188
|
+
path31.push(key === null ? this.stack[i].length : key);
|
|
1188
1189
|
}
|
|
1189
|
-
return
|
|
1190
|
+
return path31;
|
|
1190
1191
|
}
|
|
1191
1192
|
dropToLevel(level) {
|
|
1192
1193
|
if (level < this.depth) {
|
|
@@ -1468,7 +1469,7 @@ var require_pend = __commonJS({
|
|
|
1468
1469
|
var require_fd_slicer = __commonJS({
|
|
1469
1470
|
"../../packages/node-runtime/node_modules/yauzl/fd-slicer.js"(exports) {
|
|
1470
1471
|
"use strict";
|
|
1471
|
-
var
|
|
1472
|
+
var fs37 = __require("fs");
|
|
1472
1473
|
var util = __require("util");
|
|
1473
1474
|
var stream = __require("stream");
|
|
1474
1475
|
var Readable = stream.Readable;
|
|
@@ -1488,7 +1489,7 @@ var require_fd_slicer = __commonJS({
|
|
|
1488
1489
|
FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
|
|
1489
1490
|
var self = this;
|
|
1490
1491
|
self.pend.go(function(cb) {
|
|
1491
|
-
|
|
1492
|
+
fs37.read(self.fd, buffer, offset, length, position, function(err, bytesRead, buffer2) {
|
|
1492
1493
|
cb();
|
|
1493
1494
|
callback(err, bytesRead, buffer2);
|
|
1494
1495
|
});
|
|
@@ -1505,7 +1506,7 @@ var require_fd_slicer = __commonJS({
|
|
|
1505
1506
|
self.refCount -= 1;
|
|
1506
1507
|
if (self.refCount < 0) throw new Error("invalid unref");
|
|
1507
1508
|
if (self.refCount > 0) return;
|
|
1508
|
-
|
|
1509
|
+
fs37.close(self.fd, onCloseDone);
|
|
1509
1510
|
function onCloseDone(err) {
|
|
1510
1511
|
if (err) {
|
|
1511
1512
|
self.emit("error", err);
|
|
@@ -1537,7 +1538,7 @@ var require_fd_slicer = __commonJS({
|
|
|
1537
1538
|
}
|
|
1538
1539
|
self.context.pend.go(function(cb) {
|
|
1539
1540
|
var buffer = Buffer.allocUnsafe(toRead);
|
|
1540
|
-
|
|
1541
|
+
fs37.read(self.context.fd, buffer, 0, toRead, self.pos, function(err, bytesRead) {
|
|
1541
1542
|
if (err) {
|
|
1542
1543
|
self.destroy(err);
|
|
1543
1544
|
} else if (bytesRead === 0) {
|
|
@@ -1899,7 +1900,7 @@ var require_crc32 = __commonJS({
|
|
|
1899
1900
|
var require_yauzl = __commonJS({
|
|
1900
1901
|
"../../packages/node-runtime/node_modules/yauzl/index.js"(exports) {
|
|
1901
1902
|
"use strict";
|
|
1902
|
-
var
|
|
1903
|
+
var fs37 = __require("fs");
|
|
1903
1904
|
var zlib = __require("zlib");
|
|
1904
1905
|
var fd_slicer = require_fd_slicer();
|
|
1905
1906
|
var util = __require("util");
|
|
@@ -1924,9 +1925,9 @@ var require_yauzl = __commonJS({
|
|
|
1924
1925
|
exports.Entry = Entry;
|
|
1925
1926
|
exports.LocalFileHeader = LocalFileHeader;
|
|
1926
1927
|
exports.RandomAccessReader = RandomAccessReader;
|
|
1927
|
-
function openPromise2(
|
|
1928
|
+
function openPromise2(path31, options2) {
|
|
1928
1929
|
return new Promise((resolve4, reject) => {
|
|
1929
|
-
open(
|
|
1930
|
+
open(path31, { ...options2, lazyEntries: true }, function(err, zipfile) {
|
|
1930
1931
|
if (err) return reject(err);
|
|
1931
1932
|
resolve4(zipfile);
|
|
1932
1933
|
});
|
|
@@ -1956,7 +1957,7 @@ var require_yauzl = __commonJS({
|
|
|
1956
1957
|
});
|
|
1957
1958
|
});
|
|
1958
1959
|
}
|
|
1959
|
-
function open(
|
|
1960
|
+
function open(path31, options2, callback) {
|
|
1960
1961
|
if (typeof options2 === "function") {
|
|
1961
1962
|
callback = options2;
|
|
1962
1963
|
options2 = null;
|
|
@@ -1968,10 +1969,10 @@ var require_yauzl = __commonJS({
|
|
|
1968
1969
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
1969
1970
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
1970
1971
|
if (callback == null) callback = defaultCallback;
|
|
1971
|
-
|
|
1972
|
+
fs37.open(path31, "r", function(err, fd) {
|
|
1972
1973
|
if (err) return callback(err);
|
|
1973
1974
|
fromFd(fd, options2, function(err2, zipfile) {
|
|
1974
|
-
if (err2)
|
|
1975
|
+
if (err2) fs37.close(fd, defaultCallback);
|
|
1975
1976
|
callback(err2, zipfile);
|
|
1976
1977
|
});
|
|
1977
1978
|
});
|
|
@@ -1988,7 +1989,7 @@ var require_yauzl = __commonJS({
|
|
|
1988
1989
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
1989
1990
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
1990
1991
|
if (callback == null) callback = defaultCallback;
|
|
1991
|
-
|
|
1992
|
+
fs37.fstat(fd, function(err, stats) {
|
|
1992
1993
|
if (err) return callback(err);
|
|
1993
1994
|
var reader = new fd_slicer.FdSlicer(fd);
|
|
1994
1995
|
fromRandomAccessReader(reader, stats.size, options2, callback);
|
|
@@ -2767,9 +2768,9 @@ function optionsEnv(options2) {
|
|
|
2767
2768
|
function resolveChatLabTempRoot(options2 = {}) {
|
|
2768
2769
|
const envRoot = optionsEnv(options2)[CHATLAB_TEMP_ROOT_ENV]?.trim();
|
|
2769
2770
|
if (envRoot) return path.resolve(envRoot);
|
|
2770
|
-
const
|
|
2771
|
+
const platform = options2.platform ?? process.platform;
|
|
2771
2772
|
const systemTempDir = options2.systemTempDir ?? os.tmpdir();
|
|
2772
|
-
return path.resolve(
|
|
2773
|
+
return path.resolve(platform === "darwin" ? "/private/tmp/chatlab" : path.join(systemTempDir, "chatlab"));
|
|
2773
2774
|
}
|
|
2774
2775
|
function ensurePrivateOwnedDirectory(dirPath, label) {
|
|
2775
2776
|
fs.mkdirSync(dirPath, { recursive: true, mode: 448 });
|
|
@@ -2847,6 +2848,14 @@ function matchFilenameSignatures(filePath, patterns) {
|
|
|
2847
2848
|
const filename = path2.basename(filePath);
|
|
2848
2849
|
return patterns.some((pattern) => pattern.test(filename));
|
|
2849
2850
|
}
|
|
2851
|
+
function matchFilenameFallbackPatterns(filePath, patterns) {
|
|
2852
|
+
try {
|
|
2853
|
+
const content = fs2.readFileSync(filePath, "utf-8");
|
|
2854
|
+
return patterns.every((pattern) => pattern.test(content));
|
|
2855
|
+
} catch {
|
|
2856
|
+
return false;
|
|
2857
|
+
}
|
|
2858
|
+
}
|
|
2850
2859
|
function matchRequiredFields(headContent, fields) {
|
|
2851
2860
|
return fields.every((field) => {
|
|
2852
2861
|
const pattern = new RegExp(`"${field.replace(".", '"\\s*:\\s*.*"')}"\\s*:`);
|
|
@@ -2968,6 +2977,11 @@ var FormatSniffer = class {
|
|
|
2968
2977
|
return false;
|
|
2969
2978
|
}
|
|
2970
2979
|
}
|
|
2980
|
+
if (!headMatch && filenameMatch && signatures.filenameFallbackPatterns) {
|
|
2981
|
+
if (!filePath || !matchFilenameFallbackPatterns(filePath, signatures.filenameFallbackPatterns)) {
|
|
2982
|
+
return false;
|
|
2983
|
+
}
|
|
2984
|
+
}
|
|
2971
2985
|
if (signatures.requiredFields && signatures.requiredFields.length > 0) {
|
|
2972
2986
|
if (!matchRequiredFields(headContent, signatures.requiredFields)) {
|
|
2973
2987
|
return false;
|
|
@@ -3069,10 +3083,20 @@ function getNativeParserStatus() {
|
|
|
3069
3083
|
const module = loadNativeParser();
|
|
3070
3084
|
return { available: module !== null, disabled: false, error: module ? void 0 : cachedLoadError };
|
|
3071
3085
|
}
|
|
3086
|
+
function isNativeFormatAvailable(formatId) {
|
|
3087
|
+
const module = loadNativeParser();
|
|
3088
|
+
if (!module) return false;
|
|
3089
|
+
try {
|
|
3090
|
+
new module.NativeParser(formatId, "");
|
|
3091
|
+
return true;
|
|
3092
|
+
} catch {
|
|
3093
|
+
return false;
|
|
3094
|
+
}
|
|
3095
|
+
}
|
|
3072
3096
|
|
|
3073
3097
|
// ../../packages/parser/src/native/create-native-parser.ts
|
|
3074
3098
|
var PROGRESS_POLL_INTERVAL_MS = 200;
|
|
3075
|
-
async function* pumpNativeParser(adapter, options2, parser11,
|
|
3099
|
+
async function* pumpNativeParser(adapter, options2, parser11, fallbackAfterNativeFailure2) {
|
|
3076
3100
|
const { filePath, batchSize = 5e3, onProgress, onLog } = options2;
|
|
3077
3101
|
const totalBytes = getFileSize(filePath);
|
|
3078
3102
|
const initialProgress = createProgress("parsing", 0, totalBytes, 0, "");
|
|
@@ -3101,7 +3125,7 @@ async function* pumpNativeParser(adapter, options2, parser11, fallback) {
|
|
|
3101
3125
|
"warn",
|
|
3102
3126
|
`[NativeParser] Rust parse failed, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}`
|
|
3103
3127
|
);
|
|
3104
|
-
yield*
|
|
3128
|
+
yield* fallbackAfterNativeFailure2(options2);
|
|
3105
3129
|
return;
|
|
3106
3130
|
} finally {
|
|
3107
3131
|
clearInterval(progressTimer);
|
|
@@ -3122,9 +3146,12 @@ async function* pumpNativeParser(adapter, options2, parser11, fallback) {
|
|
|
3122
3146
|
yield { type: "progress", data: doneProgress };
|
|
3123
3147
|
onProgress?.(doneProgress);
|
|
3124
3148
|
onLog?.("info", `\u89E3\u6790\u5B8C\u6210: ${messagesProcessed} \u6761\u6D88\u606F, ${members.length} \u4E2A\u6210\u5458`);
|
|
3149
|
+
for (const message of adapter.completionLogs?.(metaJson) ?? []) {
|
|
3150
|
+
onLog?.("info", message);
|
|
3151
|
+
}
|
|
3125
3152
|
yield { type: "done", data: { messageCount: messagesProcessed, memberCount: members.length } };
|
|
3126
3153
|
}
|
|
3127
|
-
function createNativeFirstParser(adapter, fallback) {
|
|
3154
|
+
function createNativeFirstParser(adapter, fallback, fallbackAfterNativeFailure2 = fallback) {
|
|
3128
3155
|
return async function* (options2) {
|
|
3129
3156
|
const native = loadNativeParser();
|
|
3130
3157
|
if (!native) {
|
|
@@ -3146,7 +3173,7 @@ function createNativeFirstParser(adapter, fallback) {
|
|
|
3146
3173
|
yield* fallback(options2);
|
|
3147
3174
|
return;
|
|
3148
3175
|
}
|
|
3149
|
-
yield* pumpNativeParser(adapter, options2, parser11,
|
|
3176
|
+
yield* pumpNativeParser(adapter, options2, parser11, fallbackAfterNativeFailure2);
|
|
3150
3177
|
};
|
|
3151
3178
|
}
|
|
3152
3179
|
|
|
@@ -3662,6 +3689,12 @@ function slimMessage(msg) {
|
|
|
3662
3689
|
type: e.type
|
|
3663
3690
|
}));
|
|
3664
3691
|
}
|
|
3692
|
+
if (content?.reply && typeof content.reply === "object") {
|
|
3693
|
+
const reply = content.reply;
|
|
3694
|
+
if (reply.referencedMessageId !== void 0) {
|
|
3695
|
+
slimContent.reply = { referencedMessageId: reply.referencedMessageId };
|
|
3696
|
+
}
|
|
3697
|
+
}
|
|
3665
3698
|
const slimMsg = {
|
|
3666
3699
|
timestamp: msg.timestamp,
|
|
3667
3700
|
sender: { name: sender?.name || "" },
|
|
@@ -3669,12 +3702,12 @@ function slimMessage(msg) {
|
|
|
3669
3702
|
};
|
|
3670
3703
|
if (msg.id) slimMsg.id = msg.id;
|
|
3671
3704
|
if (msg.type) slimMsg.type = msg.type;
|
|
3672
|
-
if (msg.recalled) slimMsg.recalled = msg.recalled;
|
|
3673
|
-
if (msg.system) slimMsg.system = msg.system;
|
|
3674
|
-
if (msg.messageId) slimMsg.messageId = msg.messageId;
|
|
3705
|
+
if (msg.recalled !== void 0) slimMsg.recalled = msg.recalled;
|
|
3706
|
+
if (msg.system !== void 0) slimMsg.system = msg.system;
|
|
3707
|
+
if (msg.messageId !== void 0) slimMsg.messageId = msg.messageId;
|
|
3675
3708
|
if (msg.messageType !== void 0) slimMsg.messageType = msg.messageType;
|
|
3676
|
-
if (msg.isRecalled) slimMsg.isRecalled = msg.isRecalled;
|
|
3677
|
-
if (msg.isSystemMessage) slimMsg.isSystemMessage = msg.isSystemMessage;
|
|
3709
|
+
if (msg.isRecalled !== void 0) slimMsg.isRecalled = msg.isRecalled;
|
|
3710
|
+
if (msg.isSystemMessage !== void 0) slimMsg.isSystemMessage = msg.isSystemMessage;
|
|
3678
3711
|
if (sender?.uin) slimMsg.sender.uin = sender.uin;
|
|
3679
3712
|
if (sender?.uid) slimMsg.sender.uid = sender.uid;
|
|
3680
3713
|
if (rawMessage) {
|
|
@@ -3686,7 +3719,7 @@ function slimMessage(msg) {
|
|
|
3686
3719
|
}
|
|
3687
3720
|
return slimMsg;
|
|
3688
3721
|
}
|
|
3689
|
-
async function
|
|
3722
|
+
async function preprocessShuakamiQqJson(inputPath, onProgress) {
|
|
3690
3723
|
const totalBytes = getFileSize(inputPath);
|
|
3691
3724
|
let bytesRead = 0;
|
|
3692
3725
|
let messagesProcessed = 0;
|
|
@@ -3696,101 +3729,82 @@ async function preprocessQQJson(inputPath, onProgress) {
|
|
|
3696
3729
|
const outputPath = path5.join(tempDir, outputFilename);
|
|
3697
3730
|
onProgress?.(createProgress("parsing", 0, totalBytes, 0, ""));
|
|
3698
3731
|
const avatarsStr = readAvatarsFromFile(inputPath);
|
|
3732
|
+
const headContent = readFileHeadBytes(inputPath, 5e5);
|
|
3733
|
+
let chatInfo = { name: "\u672A\u77E5\u7FA4\u804A", type: "group" };
|
|
3734
|
+
let metadata;
|
|
3735
|
+
let statistics;
|
|
3736
|
+
try {
|
|
3737
|
+
const chatInfoJson = extractJsonObject(headContent, "chatInfo");
|
|
3738
|
+
if (chatInfoJson) chatInfo = JSON.parse(chatInfoJson);
|
|
3739
|
+
} catch {
|
|
3740
|
+
}
|
|
3741
|
+
if (!chatInfo.name) {
|
|
3742
|
+
chatInfo.name = path5.basename(inputPath).replace(/\.json$/i, "") || "\u672A\u77E5\u7FA4\u804A";
|
|
3743
|
+
}
|
|
3744
|
+
try {
|
|
3745
|
+
const metadataMatch = headContent.match(/"metadata"\s*:\s*(\{[^}]+\})/);
|
|
3746
|
+
if (metadataMatch) metadata = JSON.parse(metadataMatch[1]);
|
|
3747
|
+
} catch {
|
|
3748
|
+
}
|
|
3749
|
+
try {
|
|
3750
|
+
const statisticsMatch = headContent.match(/"statistics"\s*:\s*(\{[\s\S]*?\})\s*,\s*"messages"/);
|
|
3751
|
+
if (statisticsMatch) statistics = JSON.parse(statisticsMatch[1]);
|
|
3752
|
+
} catch {
|
|
3753
|
+
}
|
|
3699
3754
|
return new Promise((resolve4, reject) => {
|
|
3700
|
-
|
|
3701
|
-
|
|
3702
|
-
const
|
|
3703
|
-
|
|
3704
|
-
|
|
3705
|
-
let metadata;
|
|
3706
|
-
let statistics;
|
|
3707
|
-
headStream.on("data", (chunk) => {
|
|
3708
|
-
const str = typeof chunk === "string" ? chunk : chunk.toString("utf-8");
|
|
3709
|
-
if (headSize < maxHeadSize) {
|
|
3710
|
-
headChunks.push(str);
|
|
3711
|
-
headSize += str.length;
|
|
3712
|
-
} else {
|
|
3713
|
-
headStream.destroy();
|
|
3714
|
-
}
|
|
3755
|
+
onProgress?.(createProgress("parsing", 0, totalBytes, 0, ""));
|
|
3756
|
+
const readStream = fs6.createReadStream(inputPath, { encoding: "utf-8" });
|
|
3757
|
+
const writeStream = fs6.createWriteStream(outputPath, { encoding: "utf-8" });
|
|
3758
|
+
readStream.on("data", (chunk) => {
|
|
3759
|
+
bytesRead += typeof chunk === "string" ? Buffer.byteLength(chunk) : chunk.length;
|
|
3715
3760
|
});
|
|
3716
|
-
|
|
3717
|
-
|
|
3718
|
-
|
|
3719
|
-
|
|
3720
|
-
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
|
|
3724
|
-
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
|
|
3729
|
-
}
|
|
3730
|
-
} catch {
|
|
3761
|
+
const header = { metadata, chatInfo, statistics, messages: [] };
|
|
3762
|
+
const headerJson = JSON.stringify(header);
|
|
3763
|
+
writeStream.write(headerJson.slice(0, -2) + "\n");
|
|
3764
|
+
let isFirstMessage = true;
|
|
3765
|
+
const pipeline2 = chain2([readStream, parser3(), pick2({ filter: /^messages\.\d+$/ }), streamValues2()]);
|
|
3766
|
+
pipeline2.on("data", ({ value }) => {
|
|
3767
|
+
const slimMsg = slimMessage(value);
|
|
3768
|
+
const msgJson = JSON.stringify(slimMsg);
|
|
3769
|
+
if (isFirstMessage) {
|
|
3770
|
+
writeStream.write(msgJson);
|
|
3771
|
+
isFirstMessage = false;
|
|
3772
|
+
} else {
|
|
3773
|
+
writeStream.write(",\n" + msgJson);
|
|
3731
3774
|
}
|
|
3732
|
-
|
|
3733
|
-
|
|
3734
|
-
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
|
|
3775
|
+
messagesProcessed++;
|
|
3776
|
+
if (messagesProcessed % 1e4 === 0) {
|
|
3777
|
+
onProgress?.(
|
|
3778
|
+
createProgress(
|
|
3779
|
+
"parsing",
|
|
3780
|
+
bytesRead,
|
|
3781
|
+
totalBytes,
|
|
3782
|
+
messagesProcessed,
|
|
3783
|
+
`\u9884\u5904\u7406\uFF1A\u5DF2\u7CBE\u7B80 ${messagesProcessed} \u6761\u6D88\u606F...`
|
|
3784
|
+
)
|
|
3785
|
+
);
|
|
3738
3786
|
}
|
|
3739
|
-
|
|
3740
|
-
|
|
3741
|
-
|
|
3742
|
-
|
|
3743
|
-
|
|
3744
|
-
}
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
writeStream.
|
|
3748
|
-
|
|
3749
|
-
|
|
3750
|
-
pipeline2.on("data", ({ value }) => {
|
|
3751
|
-
const slimMsg = slimMessage(value);
|
|
3752
|
-
const msgJson = JSON.stringify(slimMsg);
|
|
3753
|
-
if (isFirstMessage) {
|
|
3754
|
-
writeStream.write(msgJson);
|
|
3755
|
-
isFirstMessage = false;
|
|
3756
|
-
} else {
|
|
3757
|
-
writeStream.write(",\n" + msgJson);
|
|
3758
|
-
}
|
|
3759
|
-
messagesProcessed++;
|
|
3760
|
-
if (messagesProcessed % 1e4 === 0) {
|
|
3761
|
-
onProgress?.(
|
|
3762
|
-
createProgress(
|
|
3763
|
-
"parsing",
|
|
3764
|
-
bytesRead,
|
|
3765
|
-
totalBytes,
|
|
3766
|
-
messagesProcessed,
|
|
3767
|
-
`\u9884\u5904\u7406\uFF1A\u5DF2\u7CBE\u7B80 ${messagesProcessed} \u6761\u6D88\u606F...`
|
|
3768
|
-
)
|
|
3769
|
-
);
|
|
3770
|
-
}
|
|
3771
|
-
});
|
|
3772
|
-
pipeline2.on("end", () => {
|
|
3773
|
-
writeStream.write("\n]");
|
|
3774
|
-
if (avatarsStr) {
|
|
3775
|
-
writeStream.write(',"avatars":' + avatarsStr);
|
|
3776
|
-
}
|
|
3777
|
-
writeStream.write("}");
|
|
3778
|
-
writeStream.end();
|
|
3779
|
-
writeStream.on("finish", () => {
|
|
3780
|
-
onProgress?.(createProgress("done", totalBytes, totalBytes, messagesProcessed, ""));
|
|
3781
|
-
resolve4(outputPath);
|
|
3782
|
-
});
|
|
3783
|
-
});
|
|
3784
|
-
pipeline2.on("error", (err) => {
|
|
3785
|
-
writeStream.destroy();
|
|
3786
|
-
if (fs6.existsSync(outputPath)) {
|
|
3787
|
-
fs6.unlinkSync(outputPath);
|
|
3788
|
-
}
|
|
3789
|
-
onProgress?.(createProgress("error", bytesRead, totalBytes, messagesProcessed, err.message));
|
|
3790
|
-
reject(err);
|
|
3787
|
+
});
|
|
3788
|
+
pipeline2.on("end", () => {
|
|
3789
|
+
writeStream.write("\n]");
|
|
3790
|
+
if (avatarsStr) {
|
|
3791
|
+
writeStream.write(',"avatars":' + avatarsStr);
|
|
3792
|
+
}
|
|
3793
|
+
writeStream.write("}");
|
|
3794
|
+
writeStream.end();
|
|
3795
|
+
writeStream.on("finish", () => {
|
|
3796
|
+
onProgress?.(createProgress("done", totalBytes, totalBytes, messagesProcessed, ""));
|
|
3797
|
+
resolve4(outputPath);
|
|
3791
3798
|
});
|
|
3792
3799
|
});
|
|
3793
|
-
|
|
3800
|
+
pipeline2.on("error", (err) => {
|
|
3801
|
+
writeStream.destroy();
|
|
3802
|
+
if (fs6.existsSync(outputPath)) {
|
|
3803
|
+
fs6.unlinkSync(outputPath);
|
|
3804
|
+
}
|
|
3805
|
+
onProgress?.(createProgress("error", bytesRead, totalBytes, messagesProcessed, err.message));
|
|
3806
|
+
reject(err);
|
|
3807
|
+
});
|
|
3794
3808
|
});
|
|
3795
3809
|
}
|
|
3796
3810
|
function cleanupTempFile(filePath) {
|
|
@@ -3801,14 +3815,85 @@ function cleanupTempFile(filePath) {
|
|
|
3801
3815
|
} catch {
|
|
3802
3816
|
}
|
|
3803
3817
|
}
|
|
3804
|
-
var
|
|
3818
|
+
var shuakamiQqPreprocessor = {
|
|
3805
3819
|
needsPreprocess(_filePath, fileSize) {
|
|
3806
3820
|
return fileSize > PREPROCESS_THRESHOLD;
|
|
3807
3821
|
},
|
|
3808
|
-
preprocess:
|
|
3822
|
+
preprocess: preprocessShuakamiQqJson,
|
|
3809
3823
|
cleanup: cleanupTempFile
|
|
3810
3824
|
};
|
|
3811
3825
|
|
|
3826
|
+
// ../../packages/parser/src/native/shuakami-qq-native.ts
|
|
3827
|
+
var shuakamiQqAdapter = {
|
|
3828
|
+
formatId: "shuakami-qq-exporter",
|
|
3829
|
+
label: "shuakami/qq-chat-exporter",
|
|
3830
|
+
mapMeta(metaJson) {
|
|
3831
|
+
const meta = metaJson;
|
|
3832
|
+
return {
|
|
3833
|
+
name: meta.name,
|
|
3834
|
+
platform: KNOWN_PLATFORMS.QQ,
|
|
3835
|
+
type: meta.chatType === "private" ? "private" /* PRIVATE */ : "group" /* GROUP */,
|
|
3836
|
+
groupAvatar: meta.groupAvatar ?? void 0
|
|
3837
|
+
};
|
|
3838
|
+
},
|
|
3839
|
+
mapMembers(members) {
|
|
3840
|
+
return members.map((member) => ({
|
|
3841
|
+
platformId: member.platformId,
|
|
3842
|
+
accountName: member.accountName,
|
|
3843
|
+
groupNickname: member.groupNickname,
|
|
3844
|
+
avatar: member.avatar
|
|
3845
|
+
}));
|
|
3846
|
+
},
|
|
3847
|
+
mapMessage(message) {
|
|
3848
|
+
return {
|
|
3849
|
+
platformMessageId: message.platformMessageId,
|
|
3850
|
+
senderPlatformId: message.senderPlatformId,
|
|
3851
|
+
senderAccountName: message.senderAccountName,
|
|
3852
|
+
senderGroupNickname: message.senderGroupNickname,
|
|
3853
|
+
timestamp: message.timestamp,
|
|
3854
|
+
type: message.messageType,
|
|
3855
|
+
content: message.content ?? null,
|
|
3856
|
+
replyToMessageId: message.replyToMessageId
|
|
3857
|
+
};
|
|
3858
|
+
},
|
|
3859
|
+
completionLogs(metaJson) {
|
|
3860
|
+
const skipped = metaJson.skippedMessages;
|
|
3861
|
+
return skipped > 0 ? [
|
|
3862
|
+
`[NativeParser] Skipped ${skipped} shuakami/qq-chat-exporter messages with a missing sender ID or invalid timestamp`
|
|
3863
|
+
] : [];
|
|
3864
|
+
}
|
|
3865
|
+
};
|
|
3866
|
+
async function* fallbackAfterNativeFailure(fallback, options2) {
|
|
3867
|
+
if (!shuakamiQqPreprocessor.needsPreprocess(options2.filePath, getFileSize(options2.filePath))) {
|
|
3868
|
+
yield* fallback(options2);
|
|
3869
|
+
return;
|
|
3870
|
+
}
|
|
3871
|
+
let slimFilePath = null;
|
|
3872
|
+
try {
|
|
3873
|
+
options2.onLog?.("info", "[NativeParser] Preprocessing large export before TypeScript fallback");
|
|
3874
|
+
slimFilePath = await shuakamiQqPreprocessor.preprocess(options2.filePath, options2.onProgress);
|
|
3875
|
+
} catch (error) {
|
|
3876
|
+
options2.onLog?.(
|
|
3877
|
+
"warn",
|
|
3878
|
+
`[NativeParser] Fallback preprocessing failed; retrying the original export: ${error instanceof Error ? error.message : String(error)}`
|
|
3879
|
+
);
|
|
3880
|
+
yield* fallback(options2);
|
|
3881
|
+
return;
|
|
3882
|
+
}
|
|
3883
|
+
try {
|
|
3884
|
+
yield* fallback({ ...options2, filePath: slimFilePath });
|
|
3885
|
+
} finally {
|
|
3886
|
+
if (slimFilePath) shuakamiQqPreprocessor.cleanup(slimFilePath);
|
|
3887
|
+
}
|
|
3888
|
+
}
|
|
3889
|
+
function withShuakamiQqNative(fallback) {
|
|
3890
|
+
return createNativeFirstParser(
|
|
3891
|
+
shuakamiQqAdapter,
|
|
3892
|
+
fallback,
|
|
3893
|
+
(options2) => fallbackAfterNativeFailure(fallback, options2)
|
|
3894
|
+
);
|
|
3895
|
+
}
|
|
3896
|
+
|
|
3812
3897
|
// ../../packages/parser/src/formats/shuakami-qq-exporter.ts
|
|
3813
3898
|
var { parser: parser4 } = import_stream_json3.default;
|
|
3814
3899
|
var { pick: pick3 } = import_Pick3.default;
|
|
@@ -3930,7 +4015,7 @@ function convertMessageType(messageType, content, isRecalled) {
|
|
|
3930
4015
|
if (messageType === 9) return 25 /* REPLY */;
|
|
3931
4016
|
return 0 /* TEXT */;
|
|
3932
4017
|
}
|
|
3933
|
-
async function*
|
|
4018
|
+
async function* parseShuakamiQqV4(options2) {
|
|
3934
4019
|
const { filePath, batchSize = 5e3, onProgress, onLog } = options2;
|
|
3935
4020
|
const totalBytes = getFileSize(filePath);
|
|
3936
4021
|
let bytesRead = 0;
|
|
@@ -3939,7 +4024,7 @@ async function* parseV4(options2) {
|
|
|
3939
4024
|
const initialProgress = createProgress("parsing", 0, totalBytes, 0, "");
|
|
3940
4025
|
yield { type: "progress", data: initialProgress };
|
|
3941
4026
|
onProgress?.(initialProgress);
|
|
3942
|
-
onLog?.("info", `\u5F00\u59CB\u89E3\u6790
|
|
4027
|
+
onLog?.("info", `\u5F00\u59CB\u89E3\u6790 shuakami/qq-chat-exporter \u6587\u4EF6\uFF0C\u5927\u5C0F: ${(totalBytes / 1024 / 1024).toFixed(2)} MB`);
|
|
3943
4028
|
const headContent = readFileHeadBytes(filePath, 5e5);
|
|
3944
4029
|
let chatInfo = { name: "\u672A\u77E5\u7FA4\u804A", type: "group" };
|
|
3945
4030
|
try {
|
|
@@ -4066,14 +4151,15 @@ async function* parseV4(options2) {
|
|
|
4066
4151
|
data: { messageCount: messagesProcessed, memberCount: memberMap.size }
|
|
4067
4152
|
};
|
|
4068
4153
|
}
|
|
4154
|
+
var parseShuakamiQqV4Accelerated = withShuakamiQqNative(parseShuakamiQqV4);
|
|
4069
4155
|
var parser_2 = {
|
|
4070
4156
|
feature: feature3,
|
|
4071
|
-
parse:
|
|
4157
|
+
parse: parseShuakamiQqV4Accelerated
|
|
4072
4158
|
};
|
|
4073
4159
|
var module_3 = {
|
|
4074
4160
|
feature: feature3,
|
|
4075
4161
|
parser: parser_2,
|
|
4076
|
-
preprocessor:
|
|
4162
|
+
preprocessor: shuakamiQqPreprocessor
|
|
4077
4163
|
};
|
|
4078
4164
|
var shuakami_qq_exporter_default = module_3;
|
|
4079
4165
|
|
|
@@ -4089,8 +4175,10 @@ var feature4 = {
|
|
|
4089
4175
|
// 比单文件版本优先级更高
|
|
4090
4176
|
extensions: [".json"],
|
|
4091
4177
|
signatures: {
|
|
4092
|
-
// 使用
|
|
4093
|
-
head: [/"format"\s*:\s*"chunked-jsonl"/],
|
|
4178
|
+
// V5 使用 metadata.format,0.1.x 将格式标识移至 chunked.format。
|
|
4179
|
+
head: [/"format"\s*:\s*"chunked-jsonl"/, /"chunked"\s*:\s*\{[^{}]*"format"\s*:\s*"jsonl"/],
|
|
4180
|
+
filename: [/^manifest\.json$/i],
|
|
4181
|
+
filenameFallbackPatterns: [/"chunked"\s*:\s*\{[\s\S]*?"format"\s*:\s*"jsonl"/],
|
|
4094
4182
|
requiredFields: ["metadata", "chatInfo"]
|
|
4095
4183
|
}
|
|
4096
4184
|
};
|
|
@@ -4221,9 +4309,10 @@ async function* parseChunkedJsonl(options2) {
|
|
|
4221
4309
|
yield { type: "error", data: new Error(`\u65E0\u6CD5\u8BFB\u53D6 manifest.json: ${errorMsg}`) };
|
|
4222
4310
|
return;
|
|
4223
4311
|
}
|
|
4224
|
-
if (manifest.metadata.format !== "chunked-jsonl") {
|
|
4225
|
-
|
|
4226
|
-
|
|
4312
|
+
if (manifest.metadata.format !== "chunked-jsonl" && manifest.chunked.format !== "jsonl") {
|
|
4313
|
+
const format = manifest.metadata.format ?? manifest.chunked.format ?? "unknown";
|
|
4314
|
+
onLog?.("error", `\u4E0D\u652F\u6301\u7684\u683C\u5F0F: ${format}`);
|
|
4315
|
+
yield { type: "error", data: new Error(`\u4E0D\u652F\u6301\u7684\u683C\u5F0F: ${format}`) };
|
|
4227
4316
|
return;
|
|
4228
4317
|
}
|
|
4229
4318
|
const totalBytes = calculateTotalBytes(manifest, baseDir);
|
|
@@ -5579,6 +5668,12 @@ var SYSTEM_MESSAGE_PATTERNS = [
|
|
|
5579
5668
|
/This message was deleted/i,
|
|
5580
5669
|
/disappearing messages/i
|
|
5581
5670
|
];
|
|
5671
|
+
var SYSTEM_MESSAGE_REGEX = new RegExp(SYSTEM_MESSAGE_PATTERNS.map((pattern) => pattern.source).join("|"), "i");
|
|
5672
|
+
var TIMESTAMP_SPACING = String.raw`(?:\s|\u2009|\u202F|\uFEFF|\u200E|\u200F|\u200B|\u200C|\u200D|\u2060)`;
|
|
5673
|
+
var COMMON_TIMESTAMP_REGEX = new RegExp(
|
|
5674
|
+
String.raw`^${TIMESTAMP_SPACING}*(\d{1,4})\/(\d{1,2})\/(\d{1,4})(?:,|${TIMESTAMP_SPACING})+(\d{1,2}):(\d{2})(?::(\d{2}))?(?:${TIMESTAMP_SPACING}+(AM|PM|A\.M\.|P\.M\.|上午|下午|午前|午後|오전|오후))?${TIMESTAMP_SPACING}*$`,
|
|
5675
|
+
"i"
|
|
5676
|
+
);
|
|
5582
5677
|
var AMPM_MARKERS = [
|
|
5583
5678
|
[/\bPM\b/i, true],
|
|
5584
5679
|
[/\bP\.M\.(?!\w)/i, true],
|
|
@@ -5651,7 +5746,7 @@ var WhatsAppTextAccumulator = class {
|
|
|
5651
5746
|
senderPlatformId,
|
|
5652
5747
|
senderAccountName: senderName,
|
|
5653
5748
|
timestamp: this.currentMessage.timestamp,
|
|
5654
|
-
type: detectMessageType2(content),
|
|
5749
|
+
type: detectMessageType2(content, this.currentMessage.isSystem),
|
|
5655
5750
|
content: content || null
|
|
5656
5751
|
});
|
|
5657
5752
|
if (this.currentMessage.sender) {
|
|
@@ -5662,10 +5757,11 @@ var WhatsAppTextAccumulator = class {
|
|
|
5662
5757
|
};
|
|
5663
5758
|
function createPendingMessage(timestamp, restContent) {
|
|
5664
5759
|
const senderMatch = restContent.match(SENDER_CONTENT_REGEX);
|
|
5665
|
-
|
|
5666
|
-
|
|
5760
|
+
const isSystem = isSystemMessage(restContent);
|
|
5761
|
+
if (senderMatch && !isSystem) {
|
|
5762
|
+
return { timestamp, sender: senderMatch[1].trim(), contentLines: [senderMatch[2]], isSystem };
|
|
5667
5763
|
}
|
|
5668
|
-
return { timestamp, sender: null, contentLines: [restContent] };
|
|
5764
|
+
return { timestamp, sender: null, contentLines: [restContent], isSystem };
|
|
5669
5765
|
}
|
|
5670
5766
|
function extractNameFromFileName(fileName) {
|
|
5671
5767
|
const basename10 = getBasename(fileName);
|
|
@@ -5689,9 +5785,9 @@ function cleanLine(line) {
|
|
|
5689
5785
|
return line.replace(/^(?:\uFEFF|\u200E|\u200F|\u200B|\u200C|\u200D|\u2060)+/, "").trim();
|
|
5690
5786
|
}
|
|
5691
5787
|
function isSystemMessage(content) {
|
|
5692
|
-
return
|
|
5788
|
+
return SYSTEM_MESSAGE_REGEX.test(content);
|
|
5693
5789
|
}
|
|
5694
|
-
function detectMessageType2(content) {
|
|
5790
|
+
function detectMessageType2(content, knownSystem = false) {
|
|
5695
5791
|
const trimmed = content.trim();
|
|
5696
5792
|
if (trimmed === "<\u7701\u7565\u5F71\u97F3\u5185\u5BB9>" || trimmed === "<\u5DF2\u7701\u7565\u591A\u5A92\u9AD4\u6A94\u6848>" || trimmed === "\u5716\u7247\u5DF2\u7565\u53BB" || trimmed === "\u5F71\u7247\u5DF2\u7565\u53BB" || trimmed === "\u97F3\u8A0A\u5DF2\u7565\u53BB" || trimmed === "image omitted" || trimmed === "video omitted" || trimmed === "audio omitted") {
|
|
5697
5793
|
return 1 /* IMAGE */;
|
|
@@ -5703,10 +5799,22 @@ function detectMessageType2(content) {
|
|
|
5703
5799
|
if (trimmed === "\u8FD9\u6761\u6D88\u606F\u5DF2\u5220\u9664" || trimmed.startsWith("\u6B64\u8A0A\u606F\u5DF2\u522A\u9664") || trimmed.startsWith("\u4F60\u5DF2\u522A\u9664\u6B64\u8A0A\u606F")) {
|
|
5704
5800
|
return 81 /* RECALL */;
|
|
5705
5801
|
}
|
|
5706
|
-
if (isSystemMessage(trimmed)) return 80 /* SYSTEM */;
|
|
5802
|
+
if (knownSystem || isSystemMessage(trimmed)) return 80 /* SYSTEM */;
|
|
5707
5803
|
return 0 /* TEXT */;
|
|
5708
5804
|
}
|
|
5709
5805
|
function parseFlexibleTimestamp(raw) {
|
|
5806
|
+
const commonMatch = raw.match(COMMON_TIMESTAMP_REGEX);
|
|
5807
|
+
if (commonMatch) {
|
|
5808
|
+
return buildTimestamp(
|
|
5809
|
+
Number.parseInt(commonMatch[1], 10),
|
|
5810
|
+
Number.parseInt(commonMatch[2], 10),
|
|
5811
|
+
Number.parseInt(commonMatch[3], 10),
|
|
5812
|
+
Number.parseInt(commonMatch[4], 10),
|
|
5813
|
+
Number.parseInt(commonMatch[5], 10),
|
|
5814
|
+
Number.parseInt(commonMatch[6] ?? "0", 10),
|
|
5815
|
+
parseAmPm(commonMatch[7])
|
|
5816
|
+
);
|
|
5817
|
+
}
|
|
5710
5818
|
let value = raw.replace(/(?:\u2009|\u202F|\uFEFF|\u200E|\u200F|\u200B|\u200C|\u200D|\u2060)/g, " ").trim();
|
|
5711
5819
|
let isPm = null;
|
|
5712
5820
|
for (const [pattern, pm] of AMPM_MARKERS) {
|
|
@@ -5720,23 +5828,30 @@ function parseFlexibleTimestamp(raw) {
|
|
|
5720
5828
|
if (!match) return null;
|
|
5721
5829
|
const dateParts = match[1].split("/").map((part) => Number.parseInt(part, 10));
|
|
5722
5830
|
const timeParts = match[2].split(":").map((part) => Number.parseInt(part, 10));
|
|
5831
|
+
return buildTimestamp(dateParts[0], dateParts[1], dateParts[2], timeParts[0], timeParts[1], timeParts[2] ?? 0, isPm);
|
|
5832
|
+
}
|
|
5833
|
+
function parseAmPm(marker) {
|
|
5834
|
+
if (!marker) return null;
|
|
5835
|
+
return /^(?:PM|P\.M\.|下午|午後|오후)$/i.test(marker);
|
|
5836
|
+
}
|
|
5837
|
+
function buildTimestamp(firstDatePart, secondDatePart, thirdDatePart, rawHour, minute, second, isPm) {
|
|
5723
5838
|
let year;
|
|
5724
5839
|
let month;
|
|
5725
5840
|
let day;
|
|
5726
|
-
if (
|
|
5727
|
-
;
|
|
5728
|
-
|
|
5729
|
-
|
|
5730
|
-
|
|
5731
|
-
|
|
5841
|
+
if (firstDatePart > 31) {
|
|
5842
|
+
year = firstDatePart;
|
|
5843
|
+
month = secondDatePart;
|
|
5844
|
+
day = thirdDatePart;
|
|
5845
|
+
} else if (thirdDatePart > 31) {
|
|
5846
|
+
day = firstDatePart;
|
|
5847
|
+
month = secondDatePart;
|
|
5848
|
+
year = thirdDatePart;
|
|
5732
5849
|
} else {
|
|
5733
|
-
;
|
|
5734
|
-
|
|
5735
|
-
year = 2e3 +
|
|
5850
|
+
month = firstDatePart;
|
|
5851
|
+
day = secondDatePart;
|
|
5852
|
+
year = 2e3 + thirdDatePart;
|
|
5736
5853
|
}
|
|
5737
|
-
let hour =
|
|
5738
|
-
const minute = timeParts[1];
|
|
5739
|
-
const second = timeParts[2] ?? 0;
|
|
5854
|
+
let hour = rawHour;
|
|
5740
5855
|
if (isPm === true && hour !== 12) hour += 12;
|
|
5741
5856
|
if (isPm === false && hour === 12) hour = 0;
|
|
5742
5857
|
const date = new Date(year, month - 1, day, hour, minute, second);
|
|
@@ -6020,13 +6135,14 @@ var feature12 = {
|
|
|
6020
6135
|
head: [/"is_geoblocked_for_viewer"\s*:/]
|
|
6021
6136
|
}
|
|
6022
6137
|
};
|
|
6138
|
+
var UTF8_DECODER = new TextDecoder("utf-8");
|
|
6023
6139
|
function decodeInstagramText(text) {
|
|
6024
6140
|
try {
|
|
6025
6141
|
const bytes = new Uint8Array(text.length);
|
|
6026
6142
|
for (let i = 0; i < text.length; i++) {
|
|
6027
6143
|
bytes[i] = text.charCodeAt(i);
|
|
6028
6144
|
}
|
|
6029
|
-
return
|
|
6145
|
+
return UTF8_DECODER.decode(bytes);
|
|
6030
6146
|
} catch {
|
|
6031
6147
|
return text;
|
|
6032
6148
|
}
|
|
@@ -6035,23 +6151,23 @@ function extractNameFromFilePath5(filePath) {
|
|
|
6035
6151
|
const basename10 = path9.basename(filePath);
|
|
6036
6152
|
return basename10.replace(/\.json$/i, "") || "\u672A\u77E5\u5BF9\u8BDD";
|
|
6037
6153
|
}
|
|
6154
|
+
var SYSTEM_MESSAGE_PATTERNS2 = [
|
|
6155
|
+
"You created the group",
|
|
6156
|
+
"created the group",
|
|
6157
|
+
"added",
|
|
6158
|
+
"to the group",
|
|
6159
|
+
"left the group",
|
|
6160
|
+
"removed",
|
|
6161
|
+
"named the group",
|
|
6162
|
+
"changed the group photo",
|
|
6163
|
+
"Reacted",
|
|
6164
|
+
"sent an attachment",
|
|
6165
|
+
"liked a message",
|
|
6166
|
+
"changed the theme",
|
|
6167
|
+
"set the nickname"
|
|
6168
|
+
];
|
|
6038
6169
|
function isSystemMessage2(content) {
|
|
6039
|
-
|
|
6040
|
-
"You created the group",
|
|
6041
|
-
"created the group",
|
|
6042
|
-
"added",
|
|
6043
|
-
"to the group",
|
|
6044
|
-
"left the group",
|
|
6045
|
-
"removed",
|
|
6046
|
-
"named the group",
|
|
6047
|
-
"changed the group photo",
|
|
6048
|
-
"Reacted",
|
|
6049
|
-
"sent an attachment",
|
|
6050
|
-
"liked a message",
|
|
6051
|
-
"changed the theme",
|
|
6052
|
-
"set the nickname"
|
|
6053
|
-
];
|
|
6054
|
-
return systemPatterns.some((p) => content.includes(p));
|
|
6170
|
+
return SYSTEM_MESSAGE_PATTERNS2.some((pattern) => content.includes(pattern));
|
|
6055
6171
|
}
|
|
6056
6172
|
function detectMessageType4(msg) {
|
|
6057
6173
|
const content = msg.content || "";
|
|
@@ -6135,9 +6251,9 @@ async function* parseInstagram(options2) {
|
|
|
6135
6251
|
}
|
|
6136
6252
|
const members = Array.from(memberMap.values());
|
|
6137
6253
|
yield { type: "members", data: members };
|
|
6138
|
-
const reversedMessages = [...data.messages].reverse();
|
|
6139
6254
|
const messageBatch = [];
|
|
6140
|
-
for (
|
|
6255
|
+
for (let index = data.messages.length - 1; index >= 0; index -= 1) {
|
|
6256
|
+
const msg = data.messages[index];
|
|
6141
6257
|
const senderName = decodeInstagramText(msg.sender_name);
|
|
6142
6258
|
const timestamp = Math.floor(msg.timestamp_ms / 1e3);
|
|
6143
6259
|
const type = detectMessageType4(msg);
|
|
@@ -6161,7 +6277,7 @@ async function* parseInstagram(options2) {
|
|
|
6161
6277
|
messageBatch.length = 0;
|
|
6162
6278
|
const progress = createProgress(
|
|
6163
6279
|
"parsing",
|
|
6164
|
-
Math.floor(messagesProcessed /
|
|
6280
|
+
Math.floor(messagesProcessed / data.messages.length * totalBytes),
|
|
6165
6281
|
totalBytes,
|
|
6166
6282
|
messagesProcessed,
|
|
6167
6283
|
`\u5DF2\u5904\u7406 ${messagesProcessed} \u6761\u6D88\u606F...`
|
|
@@ -6482,20 +6598,22 @@ var SPECIAL_MESSAGE_TYPES = {
|
|
|
6482
6598
|
"[\u8A18\u4E8B\u672C]": 0 /* TEXT */,
|
|
6483
6599
|
"[\u30CE\u30FC\u30C8]": 0 /* TEXT */
|
|
6484
6600
|
};
|
|
6601
|
+
var SPECIAL_MESSAGE_TYPE_ENTRIES = Object.entries(SPECIAL_MESSAGE_TYPES);
|
|
6485
6602
|
async function parseLineText(content, fileName, options2 = {}) {
|
|
6486
|
-
const lines = content.split("\n")
|
|
6603
|
+
const lines = content.split("\n");
|
|
6487
6604
|
const memberMap = /* @__PURE__ */ new Map();
|
|
6488
6605
|
const messages = [];
|
|
6489
|
-
const header = lines.length > 0 ? extractNameFromHeader(lines[0].trim()) : null;
|
|
6606
|
+
const header = lines.length > 0 ? extractNameFromHeader(lines[0].replace(/\r$/, "").trim()) : null;
|
|
6490
6607
|
const chatName = header?.name ?? extractNameFromFileName3(fileName);
|
|
6491
6608
|
let currentDate = null;
|
|
6492
6609
|
let lastMessage = null;
|
|
6493
6610
|
let quotedMultiline = false;
|
|
6494
6611
|
let lineIndex = header ? 3 : 0;
|
|
6495
6612
|
let useTabSeparator = Boolean(header);
|
|
6496
|
-
const yieldEvery = Math.max(1, options2.yieldEvery ??
|
|
6613
|
+
const yieldEvery = Math.max(1, options2.yieldEvery ?? 5e3);
|
|
6497
6614
|
if (!header) {
|
|
6498
|
-
for (const
|
|
6615
|
+
for (const rawLine of lines) {
|
|
6616
|
+
const line = rawLine.replace(/\r$/, "");
|
|
6499
6617
|
if (TAB_MESSAGE_PATTERN.test(line)) {
|
|
6500
6618
|
useTabSeparator = true;
|
|
6501
6619
|
break;
|
|
@@ -6508,7 +6626,7 @@ async function parseLineText(content, fileName, options2 = {}) {
|
|
|
6508
6626
|
}
|
|
6509
6627
|
for (; lineIndex < lines.length; lineIndex += 1) {
|
|
6510
6628
|
options2.checkCancelled?.();
|
|
6511
|
-
const line = lines[lineIndex];
|
|
6629
|
+
const line = lines[lineIndex].replace(/\r$/, "");
|
|
6512
6630
|
const date = parseDateLine(line);
|
|
6513
6631
|
if (date) {
|
|
6514
6632
|
currentDate = date;
|
|
@@ -6530,7 +6648,7 @@ async function parseLineText(content, fileName, options2 = {}) {
|
|
|
6530
6648
|
lastMessage = {
|
|
6531
6649
|
senderPlatformId: sender,
|
|
6532
6650
|
senderAccountName: sender,
|
|
6533
|
-
timestamp:
|
|
6651
|
+
timestamp: buildTimestamp2(currentDate, timeText, options2.now),
|
|
6534
6652
|
type: detectMessageType6(messageContent),
|
|
6535
6653
|
content: messageContent || null
|
|
6536
6654
|
};
|
|
@@ -6544,7 +6662,7 @@ async function parseLineText(content, fileName, options2 = {}) {
|
|
|
6544
6662
|
lastMessage = {
|
|
6545
6663
|
senderPlatformId: "system",
|
|
6546
6664
|
senderAccountName: "\u7CFB\u7D71",
|
|
6547
|
-
timestamp:
|
|
6665
|
+
timestamp: buildTimestamp2(currentDate, timeText, options2.now),
|
|
6548
6666
|
type: 80 /* SYSTEM */,
|
|
6549
6667
|
content: messageContent || null
|
|
6550
6668
|
};
|
|
@@ -6613,7 +6731,7 @@ function parseDateLine(line) {
|
|
|
6613
6731
|
}
|
|
6614
6732
|
return null;
|
|
6615
6733
|
}
|
|
6616
|
-
function
|
|
6734
|
+
function buildTimestamp2(currentDate, timeText, now) {
|
|
6617
6735
|
const { hours, minutes } = parseTime(timeText);
|
|
6618
6736
|
const value = currentDate ? new Date(currentDate) : now?.() ?? /* @__PURE__ */ new Date();
|
|
6619
6737
|
value.setHours(hours, minutes, 0, 0);
|
|
@@ -6637,7 +6755,7 @@ function parseTime(timeText) {
|
|
|
6637
6755
|
return { hours, minutes };
|
|
6638
6756
|
}
|
|
6639
6757
|
function detectMessageType6(content) {
|
|
6640
|
-
for (const [pattern, type] of
|
|
6758
|
+
for (const [pattern, type] of SPECIAL_MESSAGE_TYPE_ENTRIES) {
|
|
6641
6759
|
if (content === pattern || content.startsWith(pattern)) return type;
|
|
6642
6760
|
}
|
|
6643
6761
|
if (content.startsWith("[null]") && content.includes("maps.google.com")) return 8 /* LOCATION */;
|
|
@@ -6743,6 +6861,8 @@ var MAX_UNIX_SECONDS = 1e11;
|
|
|
6743
6861
|
var KNOWN_MESSAGE_TYPES = new Set(
|
|
6744
6862
|
Object.values(MessageType).filter((value) => typeof value === "number")
|
|
6745
6863
|
);
|
|
6864
|
+
var SYSTEM_SENDER_ID = "SYSTEM";
|
|
6865
|
+
var SYSTEM_SENDER_MESSAGE_TYPES = /* @__PURE__ */ new Set([80 /* SYSTEM */, 81 /* RECALL */]);
|
|
6746
6866
|
function createState(format, fileSize) {
|
|
6747
6867
|
return {
|
|
6748
6868
|
format,
|
|
@@ -6778,21 +6898,21 @@ function isRecord(value) {
|
|
|
6778
6898
|
function hasOwn(value, key) {
|
|
6779
6899
|
return Object.prototype.hasOwnProperty.call(value, key);
|
|
6780
6900
|
}
|
|
6781
|
-
function childLocation(location,
|
|
6782
|
-
return { ...location, path:
|
|
6901
|
+
function childLocation(location, path31) {
|
|
6902
|
+
return { ...location, path: path31 };
|
|
6783
6903
|
}
|
|
6784
|
-
function validateRequiredString(state, value,
|
|
6904
|
+
function validateRequiredString(state, value, path31, location) {
|
|
6785
6905
|
if (typeof value === "string" && value.trim().length > 0) return true;
|
|
6786
|
-
addIssue(state, "error", "INVALID_REQUIRED_STRING", "Expected a non-empty string.", childLocation(location,
|
|
6906
|
+
addIssue(state, "error", "INVALID_REQUIRED_STRING", "Expected a non-empty string.", childLocation(location, path31));
|
|
6787
6907
|
return false;
|
|
6788
6908
|
}
|
|
6789
|
-
function validateOptionalString(state, object, key,
|
|
6909
|
+
function validateOptionalString(state, object, key, path31, location) {
|
|
6790
6910
|
if (!hasOwn(object, key)) return;
|
|
6791
6911
|
if (typeof object[key] !== "string") {
|
|
6792
|
-
addIssue(state, "error", "INVALID_OPTIONAL_STRING", "Expected a string.", childLocation(location,
|
|
6912
|
+
addIssue(state, "error", "INVALID_OPTIONAL_STRING", "Expected a string.", childLocation(location, path31));
|
|
6793
6913
|
}
|
|
6794
6914
|
}
|
|
6795
|
-
function validateUnixSeconds(state, value,
|
|
6915
|
+
function validateUnixSeconds(state, value, path31, location) {
|
|
6796
6916
|
if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value < MAX_UNIX_SECONDS) {
|
|
6797
6917
|
return true;
|
|
6798
6918
|
}
|
|
@@ -6801,7 +6921,7 @@ function validateUnixSeconds(state, value, path30, location) {
|
|
|
6801
6921
|
"error",
|
|
6802
6922
|
"INVALID_UNIX_SECONDS",
|
|
6803
6923
|
"Expected a non-negative integer Unix timestamp in seconds, not milliseconds.",
|
|
6804
|
-
childLocation(location,
|
|
6924
|
+
childLocation(location, path31)
|
|
6805
6925
|
);
|
|
6806
6926
|
return false;
|
|
6807
6927
|
}
|
|
@@ -6862,30 +6982,30 @@ function validateHeader(state, chatlab, meta, location, basePath = "") {
|
|
|
6862
6982
|
}
|
|
6863
6983
|
validateOptionalString(state, meta, "sourceSessionId", `${basePath}meta.sourceSessionId`, location);
|
|
6864
6984
|
}
|
|
6865
|
-
function validateMember(state, value,
|
|
6985
|
+
function validateMember(state, value, path31, location) {
|
|
6866
6986
|
state.stats.memberCount++;
|
|
6867
6987
|
if (!isRecord(value)) {
|
|
6868
|
-
addIssue(state, "error", "INVALID_MEMBER", "Expected a member object.", childLocation(location,
|
|
6988
|
+
addIssue(state, "error", "INVALID_MEMBER", "Expected a member object.", childLocation(location, path31));
|
|
6869
6989
|
return;
|
|
6870
6990
|
}
|
|
6871
|
-
if (validateRequiredString(state, value.platformId, `${
|
|
6991
|
+
if (validateRequiredString(state, value.platformId, `${path31}.platformId`, location)) {
|
|
6872
6992
|
if (state.memberIds.has(value.platformId)) {
|
|
6873
6993
|
addIssue(state, "error", "DUPLICATE_MEMBER_ID", "Member platformId must be unique.", {
|
|
6874
6994
|
...location,
|
|
6875
|
-
path: `${
|
|
6995
|
+
path: `${path31}.platformId`
|
|
6876
6996
|
});
|
|
6877
6997
|
} else {
|
|
6878
6998
|
state.memberIds.add(value.platformId);
|
|
6879
6999
|
}
|
|
6880
7000
|
}
|
|
6881
|
-
validateRequiredString(state, value.accountName, `${
|
|
6882
|
-
validateOptionalString(state, value, "groupNickname", `${
|
|
6883
|
-
validateOptionalString(state, value, "avatar", `${
|
|
7001
|
+
validateRequiredString(state, value.accountName, `${path31}.accountName`, location);
|
|
7002
|
+
validateOptionalString(state, value, "groupNickname", `${path31}.groupNickname`, location);
|
|
7003
|
+
validateOptionalString(state, value, "avatar", `${path31}.avatar`, location);
|
|
6884
7004
|
if (hasOwn(value, "aliases")) {
|
|
6885
7005
|
if (!Array.isArray(value.aliases) || value.aliases.some((alias) => typeof alias !== "string")) {
|
|
6886
7006
|
addIssue(state, "error", "INVALID_ALIASES", "Expected aliases to be an array of strings.", {
|
|
6887
7007
|
...location,
|
|
6888
|
-
path: `${
|
|
7008
|
+
path: `${path31}.aliases`
|
|
6889
7009
|
});
|
|
6890
7010
|
}
|
|
6891
7011
|
}
|
|
@@ -6893,11 +7013,11 @@ function validateMember(state, value, path30, location) {
|
|
|
6893
7013
|
if (!Array.isArray(value.roles)) {
|
|
6894
7014
|
addIssue(state, "error", "INVALID_ROLES", "Expected roles to be an array.", {
|
|
6895
7015
|
...location,
|
|
6896
|
-
path: `${
|
|
7016
|
+
path: `${path31}.roles`
|
|
6897
7017
|
});
|
|
6898
7018
|
} else {
|
|
6899
7019
|
value.roles.forEach((role, index) => {
|
|
6900
|
-
const rolePath = `${
|
|
7020
|
+
const rolePath = `${path31}.roles[${index}]`;
|
|
6901
7021
|
if (!isRecord(role)) {
|
|
6902
7022
|
addIssue(state, "error", "INVALID_ROLE", "Expected a role object.", {
|
|
6903
7023
|
...location,
|
|
@@ -6911,47 +7031,57 @@ function validateMember(state, value, path30, location) {
|
|
|
6911
7031
|
}
|
|
6912
7032
|
}
|
|
6913
7033
|
}
|
|
6914
|
-
function validateMessage(state, value,
|
|
7034
|
+
function validateMessage(state, value, path31, location) {
|
|
6915
7035
|
state.stats.messageCount++;
|
|
6916
7036
|
if (!isRecord(value)) {
|
|
6917
|
-
addIssue(state, "error", "INVALID_MESSAGE", "Expected a message object.", childLocation(location,
|
|
7037
|
+
addIssue(state, "error", "INVALID_MESSAGE", "Expected a message object.", childLocation(location, path31));
|
|
6918
7038
|
return;
|
|
6919
7039
|
}
|
|
6920
|
-
|
|
6921
|
-
|
|
7040
|
+
const sender = validateRequiredString(state, value.sender, `${path31}.sender`, location) ? value.sender : void 0;
|
|
7041
|
+
if (sender !== void 0) {
|
|
7042
|
+
state.senderIds.add(sender);
|
|
6922
7043
|
}
|
|
6923
|
-
validateRequiredString(state, value.accountName, `${
|
|
6924
|
-
if (validateUnixSeconds(state, value.timestamp, `${
|
|
7044
|
+
validateRequiredString(state, value.accountName, `${path31}.accountName`, location);
|
|
7045
|
+
if (validateUnixSeconds(state, value.timestamp, `${path31}.timestamp`, location)) {
|
|
6925
7046
|
if (state.lastTimestamp !== void 0 && value.timestamp < state.lastTimestamp) {
|
|
6926
7047
|
addIssue(
|
|
6927
7048
|
state,
|
|
6928
7049
|
"warning",
|
|
6929
7050
|
"OUT_OF_ORDER_TIMESTAMP",
|
|
6930
7051
|
"Messages should be ordered by timestamp ascending.",
|
|
6931
|
-
childLocation(location, `${
|
|
7052
|
+
childLocation(location, `${path31}.timestamp`)
|
|
6932
7053
|
);
|
|
6933
7054
|
}
|
|
6934
7055
|
state.lastTimestamp = value.timestamp;
|
|
6935
7056
|
}
|
|
6936
|
-
|
|
7057
|
+
const messageType = typeof value.type === "number" && Number.isInteger(value.type) && KNOWN_MESSAGE_TYPES.has(value.type) ? value.type : void 0;
|
|
7058
|
+
if (messageType === void 0) {
|
|
6937
7059
|
addIssue(state, "error", "INVALID_MESSAGE_TYPE", "Expected a supported numeric ChatLab message type.", {
|
|
6938
7060
|
...location,
|
|
6939
|
-
path: `${
|
|
7061
|
+
path: `${path31}.type`
|
|
6940
7062
|
});
|
|
7063
|
+
} else if (sender === SYSTEM_SENDER_ID && !SYSTEM_SENDER_MESSAGE_TYPES.has(messageType)) {
|
|
7064
|
+
addIssue(
|
|
7065
|
+
state,
|
|
7066
|
+
"error",
|
|
7067
|
+
"INVALID_SYSTEM_SENDER_TYPE",
|
|
7068
|
+
"The reserved SYSTEM sender is only valid for system and recall messages.",
|
|
7069
|
+
childLocation(location, `${path31}.sender`)
|
|
7070
|
+
);
|
|
6941
7071
|
}
|
|
6942
7072
|
if (!hasOwn(value, "content") || typeof value.content !== "string" && value.content !== null) {
|
|
6943
7073
|
addIssue(state, "error", "INVALID_MESSAGE_CONTENT", "Expected content to be a string or null.", {
|
|
6944
7074
|
...location,
|
|
6945
|
-
path: `${
|
|
7075
|
+
path: `${path31}.content`
|
|
6946
7076
|
});
|
|
6947
7077
|
}
|
|
6948
|
-
validateOptionalString(state, value, "groupNickname", `${
|
|
7078
|
+
validateOptionalString(state, value, "groupNickname", `${path31}.groupNickname`, location);
|
|
6949
7079
|
if (hasOwn(value, "platformMessageId")) {
|
|
6950
|
-
if (validateRequiredString(state, value.platformMessageId, `${
|
|
7080
|
+
if (validateRequiredString(state, value.platformMessageId, `${path31}.platformMessageId`, location)) {
|
|
6951
7081
|
if (state.messageIds.has(value.platformMessageId)) {
|
|
6952
7082
|
addIssue(state, "error", "DUPLICATE_MESSAGE_ID", "platformMessageId must be unique.", {
|
|
6953
7083
|
...location,
|
|
6954
|
-
path: `${
|
|
7084
|
+
path: `${path31}.platformMessageId`
|
|
6955
7085
|
});
|
|
6956
7086
|
} else {
|
|
6957
7087
|
state.messageIds.add(value.platformMessageId);
|
|
@@ -6959,7 +7089,7 @@ function validateMessage(state, value, path30, location) {
|
|
|
6959
7089
|
}
|
|
6960
7090
|
}
|
|
6961
7091
|
if (hasOwn(value, "replyToMessageId")) {
|
|
6962
|
-
if (validateRequiredString(state, value.replyToMessageId, `${
|
|
7092
|
+
if (validateRequiredString(state, value.replyToMessageId, `${path31}.replyToMessageId`, location)) {
|
|
6963
7093
|
state.replyTargetIds.add(value.replyToMessageId);
|
|
6964
7094
|
}
|
|
6965
7095
|
}
|
|
@@ -6986,7 +7116,7 @@ function finalizeState(state, requireExplicitMembers, enforceRequiredSections =
|
|
|
6986
7116
|
}
|
|
6987
7117
|
if (state.memberIds.size > 0) {
|
|
6988
7118
|
for (const senderId of state.senderIds) {
|
|
6989
|
-
if (!state.memberIds.has(senderId)) {
|
|
7119
|
+
if (senderId !== SYSTEM_SENDER_ID && !state.memberIds.has(senderId)) {
|
|
6990
7120
|
addIssue(
|
|
6991
7121
|
state,
|
|
6992
7122
|
"error",
|
|
@@ -7125,6 +7255,154 @@ async function validateChatLabFile(filePath) {
|
|
|
7125
7255
|
return finalizeState(state, false, false);
|
|
7126
7256
|
}
|
|
7127
7257
|
|
|
7258
|
+
// ../../packages/parser/src/chatlab-demo-timeline.ts
|
|
7259
|
+
var SECONDS_PER_MINUTE = 60;
|
|
7260
|
+
var DEFAULT_DEMO_TIME_ZONE_OFFSET_MINUTES = 8 * 60;
|
|
7261
|
+
var MAX_TIME_ZONE_OFFSET_MINUTES = 14 * 60;
|
|
7262
|
+
function rebaseChatLabDemoDocuments(jsonDocuments, now = /* @__PURE__ */ new Date(), targetTimeZone) {
|
|
7263
|
+
if (jsonDocuments.length === 0) throw new Error("Demo timeline requires at least one document");
|
|
7264
|
+
if (!Number.isFinite(now.getTime())) throw new Error("Demo timeline requires a valid import time");
|
|
7265
|
+
const documents = jsonDocuments.map((json, index) => parseDemoDocument(json, index));
|
|
7266
|
+
const timeZoneOffsetMinutes = resolveTimeZoneOffsetMinutes(documents);
|
|
7267
|
+
const sourceLatestTimestamp = Math.max(
|
|
7268
|
+
...documents.flatMap((document) => document.messages.map((message) => message.timestamp))
|
|
7269
|
+
);
|
|
7270
|
+
const sourceLatestWallTime = new Date((sourceLatestTimestamp + timeZoneOffsetMinutes * SECONDS_PER_MINUTE) * 1e3);
|
|
7271
|
+
const latestTimestamp = resolveTargetLatestTimestamp(now, sourceLatestWallTime, targetTimeZone);
|
|
7272
|
+
const offsetSeconds = latestTimestamp - sourceLatestTimestamp;
|
|
7273
|
+
const exportedAt = Math.floor(now.getTime() / 1e3);
|
|
7274
|
+
for (const document of documents) {
|
|
7275
|
+
document.chatlab.exportedAt = exportedAt;
|
|
7276
|
+
for (const message of document.messages) {
|
|
7277
|
+
message.timestamp += offsetSeconds;
|
|
7278
|
+
}
|
|
7279
|
+
}
|
|
7280
|
+
return {
|
|
7281
|
+
documents: documents.map((document) => JSON.stringify(document)),
|
|
7282
|
+
offsetSeconds,
|
|
7283
|
+
latestTimestamp
|
|
7284
|
+
};
|
|
7285
|
+
}
|
|
7286
|
+
function resolveTargetLatestTimestamp(now, sourceWallTime, targetTimeZone) {
|
|
7287
|
+
if (!targetTimeZone) {
|
|
7288
|
+
const targetLatestTime = new Date(now);
|
|
7289
|
+
targetLatestTime.setDate(targetLatestTime.getDate() - 1);
|
|
7290
|
+
targetLatestTime.setHours(
|
|
7291
|
+
sourceWallTime.getUTCHours(),
|
|
7292
|
+
sourceWallTime.getUTCMinutes(),
|
|
7293
|
+
sourceWallTime.getUTCSeconds(),
|
|
7294
|
+
0
|
|
7295
|
+
);
|
|
7296
|
+
return Math.floor(targetLatestTime.getTime() / 1e3);
|
|
7297
|
+
}
|
|
7298
|
+
let formatter;
|
|
7299
|
+
try {
|
|
7300
|
+
formatter = new Intl.DateTimeFormat("en-US-u-ca-gregory-nu-latn", {
|
|
7301
|
+
timeZone: targetTimeZone,
|
|
7302
|
+
year: "numeric",
|
|
7303
|
+
month: "2-digit",
|
|
7304
|
+
day: "2-digit",
|
|
7305
|
+
hour: "2-digit",
|
|
7306
|
+
minute: "2-digit",
|
|
7307
|
+
second: "2-digit",
|
|
7308
|
+
hourCycle: "h23"
|
|
7309
|
+
});
|
|
7310
|
+
} catch {
|
|
7311
|
+
throw new Error("Demo timeline has an invalid target time zone");
|
|
7312
|
+
}
|
|
7313
|
+
const currentDate = getZonedDateTimeParts(now, formatter);
|
|
7314
|
+
const yesterday = new Date(Date.UTC(currentDate.year, currentDate.month - 1, currentDate.day - 1));
|
|
7315
|
+
const targetParts = {
|
|
7316
|
+
year: yesterday.getUTCFullYear(),
|
|
7317
|
+
month: yesterday.getUTCMonth() + 1,
|
|
7318
|
+
day: yesterday.getUTCDate(),
|
|
7319
|
+
hour: sourceWallTime.getUTCHours(),
|
|
7320
|
+
minute: sourceWallTime.getUTCMinutes(),
|
|
7321
|
+
second: sourceWallTime.getUTCSeconds()
|
|
7322
|
+
};
|
|
7323
|
+
return Math.floor(zonedWallTimeToUnixMillis(targetParts, formatter) / 1e3);
|
|
7324
|
+
}
|
|
7325
|
+
function getZonedDateTimeParts(date, formatter) {
|
|
7326
|
+
const values = new Map(formatter.formatToParts(date).map((part) => [part.type, part.value]));
|
|
7327
|
+
const read = (type) => Number(values.get(type));
|
|
7328
|
+
const parts = {
|
|
7329
|
+
year: read("year"),
|
|
7330
|
+
month: read("month"),
|
|
7331
|
+
day: read("day"),
|
|
7332
|
+
hour: read("hour"),
|
|
7333
|
+
minute: read("minute"),
|
|
7334
|
+
second: read("second")
|
|
7335
|
+
};
|
|
7336
|
+
if (Object.values(parts).some((value) => !Number.isInteger(value))) {
|
|
7337
|
+
throw new Error("Demo timeline could not resolve the target time zone");
|
|
7338
|
+
}
|
|
7339
|
+
return parts;
|
|
7340
|
+
}
|
|
7341
|
+
function zonedWallTimeToUnixMillis(target, formatter) {
|
|
7342
|
+
const targetAsUtc = dateTimePartsToUtcMillis(target);
|
|
7343
|
+
let candidate = targetAsUtc;
|
|
7344
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
7345
|
+
const actual2 = getZonedDateTimeParts(new Date(candidate), formatter);
|
|
7346
|
+
const correction = targetAsUtc - dateTimePartsToUtcMillis(actual2);
|
|
7347
|
+
candidate += correction;
|
|
7348
|
+
if (correction === 0) return candidate;
|
|
7349
|
+
}
|
|
7350
|
+
const actual = getZonedDateTimeParts(new Date(candidate), formatter);
|
|
7351
|
+
if (dateTimePartsToUtcMillis(actual) !== targetAsUtc) {
|
|
7352
|
+
throw new Error("Demo timeline target wall time does not exist in the selected time zone");
|
|
7353
|
+
}
|
|
7354
|
+
return candidate;
|
|
7355
|
+
}
|
|
7356
|
+
function dateTimePartsToUtcMillis(parts) {
|
|
7357
|
+
return Date.UTC(parts.year, parts.month - 1, parts.day, parts.hour, parts.minute, parts.second);
|
|
7358
|
+
}
|
|
7359
|
+
function parseDemoDocument(json, index) {
|
|
7360
|
+
let value;
|
|
7361
|
+
try {
|
|
7362
|
+
value = JSON.parse(json);
|
|
7363
|
+
} catch {
|
|
7364
|
+
throw new Error(`Demo document ${index + 1} is not valid JSON`);
|
|
7365
|
+
}
|
|
7366
|
+
if (!isRecord2(value) || !isRecord2(value.chatlab) || !Array.isArray(value.messages)) {
|
|
7367
|
+
throw new Error(`Demo document ${index + 1} is not a ChatLab JSON document`);
|
|
7368
|
+
}
|
|
7369
|
+
if (value.messages.length === 0) {
|
|
7370
|
+
throw new Error(`Demo document ${index + 1} has no messages`);
|
|
7371
|
+
}
|
|
7372
|
+
const messages = value.messages.map((message, messageIndex) => {
|
|
7373
|
+
if (!isRecord2(message) || !isUnixSeconds(message.timestamp)) {
|
|
7374
|
+
throw new Error(`Demo document ${index + 1} has an invalid timestamp at message ${messageIndex + 1}`);
|
|
7375
|
+
}
|
|
7376
|
+
return message;
|
|
7377
|
+
});
|
|
7378
|
+
return {
|
|
7379
|
+
...value,
|
|
7380
|
+
chatlab: value.chatlab,
|
|
7381
|
+
messages
|
|
7382
|
+
};
|
|
7383
|
+
}
|
|
7384
|
+
function resolveTimeZoneOffsetMinutes(documents) {
|
|
7385
|
+
const offsets = /* @__PURE__ */ new Set();
|
|
7386
|
+
for (const document of documents) {
|
|
7387
|
+
const configured = document.chatlab.demoTimeline?.timeZoneOffsetMinutes;
|
|
7388
|
+
const offset = configured ?? DEFAULT_DEMO_TIME_ZONE_OFFSET_MINUTES;
|
|
7389
|
+
if (typeof offset !== "number" || !Number.isInteger(offset) || Math.abs(offset) > MAX_TIME_ZONE_OFFSET_MINUTES) {
|
|
7390
|
+
throw new Error("Demo timeline has an invalid time zone offset");
|
|
7391
|
+
}
|
|
7392
|
+
offsets.add(offset);
|
|
7393
|
+
}
|
|
7394
|
+
if (offsets.size !== 1) {
|
|
7395
|
+
throw new Error("Demo documents must use the same time zone offset");
|
|
7396
|
+
}
|
|
7397
|
+
return offsets.values().next().value;
|
|
7398
|
+
}
|
|
7399
|
+
function isUnixSeconds(value) {
|
|
7400
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
7401
|
+
}
|
|
7402
|
+
function isRecord2(value) {
|
|
7403
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
7404
|
+
}
|
|
7405
|
+
|
|
7128
7406
|
// ../../packages/parser/src/index.ts
|
|
7129
7407
|
var sniffer = createSniffer();
|
|
7130
7408
|
sniffer.registerAll(formats);
|
|
@@ -7262,95 +7540,113 @@ var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
|
7262
7540
|
LogLevel2["INFO"] = "INFO";
|
|
7263
7541
|
return LogLevel2;
|
|
7264
7542
|
})(LogLevel || {});
|
|
7265
|
-
|
|
7266
|
-
|
|
7267
|
-
|
|
7268
|
-
|
|
7269
|
-
|
|
7270
|
-
|
|
7271
|
-
if (!
|
|
7272
|
-
|
|
7543
|
+
function createImportPerfLogger(logDir) {
|
|
7544
|
+
let lastLogTime = Date.now();
|
|
7545
|
+
let lastMessageCount = 0;
|
|
7546
|
+
let currentLogFile = null;
|
|
7547
|
+
let errorCount = 0;
|
|
7548
|
+
const append = (line) => {
|
|
7549
|
+
if (!currentLogFile) return;
|
|
7550
|
+
try {
|
|
7551
|
+
fs19.appendFileSync(currentLogFile, line, "utf-8");
|
|
7552
|
+
} catch {
|
|
7273
7553
|
}
|
|
7274
|
-
|
|
7275
|
-
|
|
7554
|
+
};
|
|
7555
|
+
const writeLogLine = (level, message) => {
|
|
7556
|
+
append(`[${(/* @__PURE__ */ new Date()).toISOString()}] [${level}] ${message}
|
|
7557
|
+
`);
|
|
7558
|
+
};
|
|
7559
|
+
return {
|
|
7560
|
+
init(sessionId) {
|
|
7561
|
+
try {
|
|
7562
|
+
if (!fs19.existsSync(logDir)) fs19.mkdirSync(logDir, { recursive: true });
|
|
7563
|
+
currentLogFile = path12.join(logDir, `import_${sessionId}_${Date.now()}.log`);
|
|
7564
|
+
fs19.writeFileSync(currentLogFile, `=== Import Log ===
|
|
7276
7565
|
Start time: ${(/* @__PURE__ */ new Date()).toISOString()}
|
|
7277
7566
|
|
|
7278
7567
|
`, "utf-8");
|
|
7279
|
-
|
|
7280
|
-
|
|
7568
|
+
} catch {
|
|
7569
|
+
currentLogFile = null;
|
|
7570
|
+
}
|
|
7571
|
+
},
|
|
7572
|
+
perf(event, messagesProcessed, batchSize) {
|
|
7573
|
+
const now = Date.now();
|
|
7574
|
+
const duration = now - lastLogTime;
|
|
7575
|
+
const messagesDelta = messagesProcessed - lastMessageCount;
|
|
7576
|
+
const speed = duration > 0 ? Math.round(messagesDelta / duration * 1e3) : 0;
|
|
7577
|
+
let memory = 0;
|
|
7578
|
+
try {
|
|
7579
|
+
memory = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
|
|
7580
|
+
} catch {
|
|
7581
|
+
}
|
|
7582
|
+
append(
|
|
7583
|
+
`[${(/* @__PURE__ */ new Date()).toISOString()}] ${event} | messages: ${messagesProcessed.toLocaleString()} | elapsed: ${duration}ms | speed: ${speed.toLocaleString()}/s | memory: ${memory}MB` + (batchSize ? ` | batch: ${batchSize}` : "") + "\n"
|
|
7584
|
+
);
|
|
7585
|
+
lastLogTime = now;
|
|
7586
|
+
lastMessageCount = messagesProcessed;
|
|
7587
|
+
},
|
|
7588
|
+
perfDetail(detail) {
|
|
7589
|
+
append(` ${detail}
|
|
7590
|
+
`);
|
|
7591
|
+
},
|
|
7592
|
+
reset() {
|
|
7593
|
+
lastLogTime = Date.now();
|
|
7594
|
+
lastMessageCount = 0;
|
|
7595
|
+
currentLogFile = null;
|
|
7596
|
+
errorCount = 0;
|
|
7597
|
+
},
|
|
7598
|
+
getCurrentLogFile() {
|
|
7599
|
+
return currentLogFile;
|
|
7600
|
+
},
|
|
7601
|
+
error(message, error) {
|
|
7602
|
+
errorCount++;
|
|
7603
|
+
writeLogLine("ERROR" /* ERROR */, `${message}${error ? `: ${error.message}` : ""}`);
|
|
7604
|
+
},
|
|
7605
|
+
info(message) {
|
|
7606
|
+
writeLogLine("INFO" /* INFO */, message);
|
|
7607
|
+
},
|
|
7608
|
+
getErrorCount() {
|
|
7609
|
+
return errorCount;
|
|
7610
|
+
},
|
|
7611
|
+
summary(totalMessages, totalMembers) {
|
|
7612
|
+
append(`
|
|
7613
|
+
=== Import Summary ===
|
|
7614
|
+
End time: ${(/* @__PURE__ */ new Date()).toISOString()}
|
|
7615
|
+
Total messages: ${totalMessages.toLocaleString()}
|
|
7616
|
+
Total members: ${totalMembers.toLocaleString()}
|
|
7617
|
+
Errors: ${errorCount}
|
|
7618
|
+
`);
|
|
7619
|
+
}
|
|
7620
|
+
};
|
|
7621
|
+
}
|
|
7622
|
+
var defaultLogger = createImportPerfLogger("");
|
|
7623
|
+
function initPerfLog(sessionId, logDir) {
|
|
7624
|
+
defaultLogger = createImportPerfLogger(logDir);
|
|
7625
|
+
defaultLogger.init(sessionId);
|
|
7281
7626
|
}
|
|
7282
7627
|
function logPerf(event, messagesProcessed, batchSize) {
|
|
7283
|
-
|
|
7284
|
-
const duration = now - lastLogTime;
|
|
7285
|
-
const messagesDelta = messagesProcessed - lastMessageCount;
|
|
7286
|
-
const speed = duration > 0 ? Math.round(messagesDelta / duration * 1e3) : 0;
|
|
7287
|
-
let memory = 0;
|
|
7288
|
-
try {
|
|
7289
|
-
const used = process.memoryUsage();
|
|
7290
|
-
memory = Math.round(used.heapUsed / 1024 / 1024);
|
|
7291
|
-
} catch {
|
|
7292
|
-
}
|
|
7293
|
-
const logLine = `[${(/* @__PURE__ */ new Date()).toISOString()}] ${event} | messages: ${messagesProcessed.toLocaleString()} | elapsed: ${duration}ms | speed: ${speed.toLocaleString()}/s | memory: ${memory}MB` + (batchSize ? ` | batch: ${batchSize}` : "") + "\n";
|
|
7294
|
-
if (currentLogFile) {
|
|
7295
|
-
try {
|
|
7296
|
-
fs19.appendFileSync(currentLogFile, logLine, "utf-8");
|
|
7297
|
-
} catch {
|
|
7298
|
-
}
|
|
7299
|
-
}
|
|
7300
|
-
lastLogTime = now;
|
|
7301
|
-
lastMessageCount = messagesProcessed;
|
|
7628
|
+
defaultLogger.perf(event, messagesProcessed, batchSize);
|
|
7302
7629
|
}
|
|
7303
7630
|
function logPerfDetail(detail) {
|
|
7304
|
-
|
|
7305
|
-
try {
|
|
7306
|
-
fs19.appendFileSync(currentLogFile, ` ${detail}
|
|
7307
|
-
`, "utf-8");
|
|
7308
|
-
} catch {
|
|
7309
|
-
}
|
|
7310
|
-
}
|
|
7631
|
+
defaultLogger.perfDetail(detail);
|
|
7311
7632
|
}
|
|
7312
7633
|
function resetPerfLog() {
|
|
7313
|
-
|
|
7314
|
-
lastMessageCount = 0;
|
|
7315
|
-
currentLogFile = null;
|
|
7316
|
-
errorCount = 0;
|
|
7634
|
+
defaultLogger.reset();
|
|
7317
7635
|
}
|
|
7318
7636
|
function getCurrentLogFile() {
|
|
7319
|
-
return
|
|
7320
|
-
}
|
|
7321
|
-
function writeLogLine(level, message) {
|
|
7322
|
-
if (!currentLogFile) return;
|
|
7323
|
-
const logLine = `[${(/* @__PURE__ */ new Date()).toISOString()}] [${level}] ${message}
|
|
7324
|
-
`;
|
|
7325
|
-
try {
|
|
7326
|
-
fs19.appendFileSync(currentLogFile, logLine, "utf-8");
|
|
7327
|
-
} catch {
|
|
7328
|
-
}
|
|
7637
|
+
return defaultLogger.getCurrentLogFile();
|
|
7329
7638
|
}
|
|
7330
7639
|
function logError(message, error) {
|
|
7331
|
-
|
|
7332
|
-
const errorDetail = error ? `: ${error.message}` : "";
|
|
7333
|
-
writeLogLine("ERROR" /* ERROR */, `${message}${errorDetail}`);
|
|
7640
|
+
defaultLogger.error(message, error);
|
|
7334
7641
|
}
|
|
7335
7642
|
function logInfo(message) {
|
|
7336
|
-
|
|
7643
|
+
defaultLogger.info(message);
|
|
7337
7644
|
}
|
|
7338
7645
|
function getErrorCount() {
|
|
7339
|
-
return
|
|
7646
|
+
return defaultLogger.getErrorCount();
|
|
7340
7647
|
}
|
|
7341
7648
|
function logSummary(totalMessages, totalMembers) {
|
|
7342
|
-
|
|
7343
|
-
const summary = `
|
|
7344
|
-
=== Import Summary ===
|
|
7345
|
-
End time: ${(/* @__PURE__ */ new Date()).toISOString()}
|
|
7346
|
-
Total messages: ${totalMessages.toLocaleString()}
|
|
7347
|
-
Total members: ${totalMembers.toLocaleString()}
|
|
7348
|
-
Errors: ${errorCount}
|
|
7349
|
-
`;
|
|
7350
|
-
try {
|
|
7351
|
-
fs19.appendFileSync(currentLogFile, summary, "utf-8");
|
|
7352
|
-
} catch {
|
|
7353
|
-
}
|
|
7649
|
+
defaultLogger.summary(totalMessages, totalMembers);
|
|
7354
7650
|
}
|
|
7355
7651
|
|
|
7356
7652
|
// ../../packages/node-runtime/src/nlp/word-frequency.ts
|
|
@@ -7593,16 +7889,27 @@ function createFtsTable(db) {
|
|
|
7593
7889
|
function buildFtsIndex(db) {
|
|
7594
7890
|
createFtsTable(db);
|
|
7595
7891
|
const insertFts = db.prepare("INSERT INTO message_fts(rowid, content) VALUES (?, ?)");
|
|
7596
|
-
const countRow = db.prepare("SELECT COUNT(*) as total FROM message WHERE type = 0 AND content IS NOT NULL AND content != ''").get();
|
|
7597
|
-
const total = countRow?.total ?? 0;
|
|
7598
7892
|
let indexed = 0;
|
|
7599
|
-
let
|
|
7600
|
-
|
|
7601
|
-
|
|
7602
|
-
|
|
7603
|
-
|
|
7604
|
-
|
|
7605
|
-
|
|
7893
|
+
let lastId = null;
|
|
7894
|
+
const selectFirstBatch = db.prepare(
|
|
7895
|
+
`SELECT id, content FROM message
|
|
7896
|
+
WHERE type = 0
|
|
7897
|
+
AND content IS NOT NULL
|
|
7898
|
+
AND content != ''
|
|
7899
|
+
ORDER BY id ASC
|
|
7900
|
+
LIMIT ?`
|
|
7901
|
+
);
|
|
7902
|
+
const selectNextBatch = db.prepare(
|
|
7903
|
+
`SELECT id, content FROM message
|
|
7904
|
+
WHERE type = 0
|
|
7905
|
+
AND content IS NOT NULL
|
|
7906
|
+
AND content != ''
|
|
7907
|
+
AND id > ?
|
|
7908
|
+
ORDER BY id ASC
|
|
7909
|
+
LIMIT ?`
|
|
7910
|
+
);
|
|
7911
|
+
while (true) {
|
|
7912
|
+
const rows = lastId === null ? selectFirstBatch.all(BATCH_SIZE) : selectNextBatch.all(lastId, BATCH_SIZE);
|
|
7606
7913
|
if (rows.length === 0) break;
|
|
7607
7914
|
db.transaction(() => {
|
|
7608
7915
|
for (const row of rows) {
|
|
@@ -7613,7 +7920,7 @@ function buildFtsIndex(db) {
|
|
|
7613
7920
|
}
|
|
7614
7921
|
});
|
|
7615
7922
|
indexed += rows.length;
|
|
7616
|
-
|
|
7923
|
+
lastId = rows[rows.length - 1].id;
|
|
7617
7924
|
}
|
|
7618
7925
|
return { indexed };
|
|
7619
7926
|
}
|
|
@@ -7655,8 +7962,62 @@ function searchByFts(db, keywords, limit = 1e3, offset = 0) {
|
|
|
7655
7962
|
}
|
|
7656
7963
|
}
|
|
7657
7964
|
|
|
7965
|
+
// ../../packages/node-runtime/src/import/message-batch-inserter.ts
|
|
7966
|
+
var MESSAGE_COLUMN_COUNT = 8;
|
|
7967
|
+
var SQLITE_LEGACY_VARIABLE_LIMIT = 999;
|
|
7968
|
+
var MESSAGE_INSERT_MAX_ROWS = Math.floor(SQLITE_LEGACY_VARIABLE_LIMIT / MESSAGE_COLUMN_COUNT);
|
|
7969
|
+
var MessageBatchInserter = class {
|
|
7970
|
+
constructor(db) {
|
|
7971
|
+
this.db = db;
|
|
7972
|
+
}
|
|
7973
|
+
db;
|
|
7974
|
+
statementCache = /* @__PURE__ */ new Map();
|
|
7975
|
+
insert(rows) {
|
|
7976
|
+
let statementCount = 0;
|
|
7977
|
+
for (let offset = 0; offset < rows.length; offset += MESSAGE_INSERT_MAX_ROWS) {
|
|
7978
|
+
const batch = rows.slice(offset, offset + MESSAGE_INSERT_MAX_ROWS);
|
|
7979
|
+
const statement = this.getStatement(batch.length);
|
|
7980
|
+
statement.run(...batch.flatMap(toParams));
|
|
7981
|
+
statementCount++;
|
|
7982
|
+
}
|
|
7983
|
+
return statementCount;
|
|
7984
|
+
}
|
|
7985
|
+
getStatement(rowCount) {
|
|
7986
|
+
const cached = this.statementCache.get(rowCount);
|
|
7987
|
+
if (cached) return cached;
|
|
7988
|
+
const values = Array.from({ length: rowCount }, () => "(?, ?, ?, ?, ?, ?, ?, ?)").join(", ");
|
|
7989
|
+
const statement = this.db.prepare(
|
|
7990
|
+
`INSERT INTO message (
|
|
7991
|
+
sender_id,
|
|
7992
|
+
sender_account_name,
|
|
7993
|
+
sender_group_nickname,
|
|
7994
|
+
ts,
|
|
7995
|
+
type,
|
|
7996
|
+
content,
|
|
7997
|
+
reply_to_message_id,
|
|
7998
|
+
platform_message_id
|
|
7999
|
+
) VALUES ${values}`
|
|
8000
|
+
);
|
|
8001
|
+
this.statementCache.set(rowCount, statement);
|
|
8002
|
+
return statement;
|
|
8003
|
+
}
|
|
8004
|
+
};
|
|
8005
|
+
function toParams(row) {
|
|
8006
|
+
return [
|
|
8007
|
+
row.senderId,
|
|
8008
|
+
row.senderAccountName,
|
|
8009
|
+
row.senderGroupNickname,
|
|
8010
|
+
row.timestamp,
|
|
8011
|
+
row.type,
|
|
8012
|
+
row.content,
|
|
8013
|
+
row.replyToMessageId,
|
|
8014
|
+
row.platformMessageId
|
|
8015
|
+
];
|
|
8016
|
+
}
|
|
8017
|
+
|
|
7658
8018
|
// ../../packages/node-runtime/src/import/streaming-importer.ts
|
|
7659
8019
|
import * as fs21 from "fs";
|
|
8020
|
+
import { performance } from "perf_hooks";
|
|
7660
8021
|
|
|
7661
8022
|
// ../../packages/node-runtime/src/import/message-deduplicator.ts
|
|
7662
8023
|
function generateFallbackMessageKey(message) {
|
|
@@ -7688,15 +8049,25 @@ function registerMessageAndCheckDuplicate(message, state) {
|
|
|
7688
8049
|
// ../../packages/node-runtime/src/import/streaming-importer.ts
|
|
7689
8050
|
var BATCH_COMMIT_SIZE = 5e4;
|
|
7690
8051
|
var CHECKPOINT_INTERVAL = 2e5;
|
|
8052
|
+
var SYSTEM_SENDER_ID2 = "SYSTEM";
|
|
8053
|
+
var SYSTEM_MEMBER_NAME = "\u7CFB\u7EDF\u6D88\u606F";
|
|
8054
|
+
var RESERVED_SYSTEM_SENDER_FORMATS = /* @__PURE__ */ new Set(["chatlab", "chatlab-jsonl"]);
|
|
8055
|
+
var defaultNow = () => performance.now();
|
|
8056
|
+
function elapsedMs(startedAt, now) {
|
|
8057
|
+
return now() - startedAt;
|
|
8058
|
+
}
|
|
7691
8059
|
var yieldToEventLoop = () => new Promise((resolve4) => setImmediate(resolve4));
|
|
7692
8060
|
function defaultGenerateSessionId() {
|
|
7693
8061
|
const ts = Date.now();
|
|
7694
8062
|
const rand = Math.random().toString(36).substring(2, 8);
|
|
7695
8063
|
return `chat_${ts}_${rand}`;
|
|
7696
8064
|
}
|
|
7697
|
-
function
|
|
8065
|
+
function normalizeSystemMemberName(formatId, platformId, name) {
|
|
8066
|
+
return RESERVED_SYSTEM_SENDER_FORMATS.has(formatId) && platformId === SYSTEM_SENDER_ID2 ? SYSTEM_MEMBER_NAME : name;
|
|
8067
|
+
}
|
|
8068
|
+
function prepareMessageForCreate(message, senderAccountName = message.senderAccountName) {
|
|
7698
8069
|
if (!message.senderPlatformId) return { skipCounter: "skippedNoSenderId" };
|
|
7699
|
-
if (!
|
|
8070
|
+
if (!senderAccountName) return { skipCounter: "skippedNoAccountName" };
|
|
7700
8071
|
if (message.timestamp === void 0 || message.timestamp === null || isNaN(message.timestamp)) {
|
|
7701
8072
|
return { skipCounter: "skippedInvalidTimestamp" };
|
|
7702
8073
|
}
|
|
@@ -7717,28 +8088,38 @@ function prepareMessageForCreate(message) {
|
|
|
7717
8088
|
};
|
|
7718
8089
|
}
|
|
7719
8090
|
async function streamingImport(filePath, deps, formatOptions, externalSessionId) {
|
|
8091
|
+
const now = deps.now ?? defaultNow;
|
|
8092
|
+
const totalStartedAt = now();
|
|
8093
|
+
const detectionStartedAt = now();
|
|
7720
8094
|
if (formatOptions?.formatId) {
|
|
7721
8095
|
const formatId = formatOptions.formatId;
|
|
7722
8096
|
const feature15 = getFormatFeatureById(formatId);
|
|
7723
8097
|
if (!feature15) {
|
|
7724
8098
|
return { success: false, error: "error.unknown_format_id" };
|
|
7725
8099
|
}
|
|
7726
|
-
return streamImportSingle(filePath, deps, feature15, formatOptions, externalSessionId
|
|
8100
|
+
return streamImportSingle(filePath, deps, feature15, formatOptions, externalSessionId, {
|
|
8101
|
+
totalStartedAt,
|
|
8102
|
+
detectionMs: elapsedMs(detectionStartedAt, now)
|
|
8103
|
+
});
|
|
7727
8104
|
}
|
|
7728
8105
|
const candidates = detectAllFormats(filePath);
|
|
7729
8106
|
if (candidates.length === 0) {
|
|
7730
8107
|
return { success: false, error: "error.unrecognized_format" };
|
|
7731
8108
|
}
|
|
8109
|
+
const timingContext = {
|
|
8110
|
+
totalStartedAt,
|
|
8111
|
+
detectionMs: elapsedMs(detectionStartedAt, now)
|
|
8112
|
+
};
|
|
7732
8113
|
if (candidates.length > 1) {
|
|
7733
|
-
return streamImportWithFallback(filePath, deps, candidates, formatOptions, externalSessionId);
|
|
8114
|
+
return streamImportWithFallback(filePath, deps, candidates, formatOptions, externalSessionId, timingContext);
|
|
7734
8115
|
}
|
|
7735
|
-
return streamImportSingle(filePath, deps, candidates[0], formatOptions, externalSessionId);
|
|
8116
|
+
return streamImportSingle(filePath, deps, candidates[0], formatOptions, externalSessionId, timingContext);
|
|
7736
8117
|
}
|
|
7737
|
-
async function streamImportWithFallback(filePath, deps, candidates, formatOptions, externalSessionId) {
|
|
8118
|
+
async function streamImportWithFallback(filePath, deps, candidates, formatOptions, externalSessionId, timingContext) {
|
|
7738
8119
|
for (let i = 0; i < candidates.length; i++) {
|
|
7739
8120
|
const candidate = candidates[i];
|
|
7740
8121
|
deps.logger?.info(`[StreamImport] Trying format ${i + 1}/${candidates.length}: ${candidate.name} (${candidate.id})`);
|
|
7741
|
-
const result = await streamImportSingle(filePath, deps, candidate, formatOptions, externalSessionId);
|
|
8122
|
+
const result = await streamImportSingle(filePath, deps, candidate, formatOptions, externalSessionId, timingContext);
|
|
7742
8123
|
if (result.success) {
|
|
7743
8124
|
if (i > 0) {
|
|
7744
8125
|
deps.logger?.info(
|
|
@@ -7752,9 +8133,32 @@ async function streamImportWithFallback(filePath, deps, candidates, formatOption
|
|
|
7752
8133
|
}
|
|
7753
8134
|
return { success: false, error: "error.no_messages" };
|
|
7754
8135
|
}
|
|
7755
|
-
async function streamImportSingle(filePath, deps, formatFeature, formatOptions, externalSessionId) {
|
|
8136
|
+
async function streamImportSingle(filePath, deps, formatFeature, formatOptions, externalSessionId, timingContext) {
|
|
7756
8137
|
const { onProgress, logger } = deps;
|
|
7757
8138
|
const genId = deps.generateSessionId ?? defaultGenerateSessionId;
|
|
8139
|
+
const now = deps.now ?? defaultNow;
|
|
8140
|
+
const totalStartedAt = timingContext?.totalStartedAt ?? now();
|
|
8141
|
+
const timings = {
|
|
8142
|
+
detectionMs: timingContext?.detectionMs ?? 0,
|
|
8143
|
+
preprocessingMs: 0,
|
|
8144
|
+
databaseSetupMs: 0,
|
|
8145
|
+
parserMs: 0,
|
|
8146
|
+
metaWriteMs: 0,
|
|
8147
|
+
memberWriteMs: 0,
|
|
8148
|
+
messageWriteMs: 0,
|
|
8149
|
+
nicknameHistoryMs: 0,
|
|
8150
|
+
indexCreationMs: 0,
|
|
8151
|
+
ftsMs: 0,
|
|
8152
|
+
checkpointMs: 0,
|
|
8153
|
+
sessionIndexMs: 0,
|
|
8154
|
+
postImportHookMs: 0,
|
|
8155
|
+
totalMs: 0
|
|
8156
|
+
};
|
|
8157
|
+
const rssStartBytes = process.memoryUsage().rss;
|
|
8158
|
+
let rssPeakBytes = rssStartBytes;
|
|
8159
|
+
const sampleRss = () => {
|
|
8160
|
+
rssPeakBytes = Math.max(rssPeakBytes, process.memoryUsage().rss);
|
|
8161
|
+
};
|
|
7758
8162
|
logger?.reset();
|
|
7759
8163
|
const sessionId = externalSessionId || genId();
|
|
7760
8164
|
logger?.init(sessionId);
|
|
@@ -7764,8 +8168,18 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
7764
8168
|
logger?.perf("Import started", 0);
|
|
7765
8169
|
let actualFilePath = filePath;
|
|
7766
8170
|
let tempFilePath = null;
|
|
7767
|
-
const
|
|
7768
|
-
|
|
8171
|
+
const preprocessing = deps.preprocessing ?? { getPreprocessor, needsPreprocess, isNativeFormatAvailable };
|
|
8172
|
+
const getImportPreprocessor = preprocessing.getPreprocessor;
|
|
8173
|
+
const shouldPreprocess = preprocessing.needsPreprocess;
|
|
8174
|
+
const nativeFormatAvailable = preprocessing.isNativeFormatAvailable;
|
|
8175
|
+
const preprocessor = getImportPreprocessor(filePath);
|
|
8176
|
+
const needsLargeFilePreprocess = preprocessor && shouldPreprocess(filePath);
|
|
8177
|
+
const nativeCanParseOriginal = needsLargeFilePreprocess && nativeFormatAvailable(formatFeature.id);
|
|
8178
|
+
if (nativeCanParseOriginal) {
|
|
8179
|
+
logger?.info(
|
|
8180
|
+
`[NativeParser] Kernel ${formatFeature.id} is available; skipping large-file preprocessing and parsing the original export`
|
|
8181
|
+
);
|
|
8182
|
+
} else if (needsLargeFilePreprocess) {
|
|
7769
8183
|
logger?.info("File needs preprocessing, simplifying large file...");
|
|
7770
8184
|
onProgress({
|
|
7771
8185
|
stage: "parsing",
|
|
@@ -7775,6 +8189,7 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
7775
8189
|
percentage: 0,
|
|
7776
8190
|
message: ""
|
|
7777
8191
|
});
|
|
8192
|
+
const preprocessingStartedAt = now();
|
|
7778
8193
|
try {
|
|
7779
8194
|
tempFilePath = await preprocessor.preprocess(filePath, (progress) => {
|
|
7780
8195
|
onProgress({ ...progress, message: "" });
|
|
@@ -7785,45 +8200,100 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
7785
8200
|
const errorMsg = `Preprocessing failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
7786
8201
|
logger?.error(errorMsg, err instanceof Error ? err : void 0);
|
|
7787
8202
|
return { success: false, error: errorMsg };
|
|
8203
|
+
} finally {
|
|
8204
|
+
timings.preprocessingMs = elapsedMs(preprocessingStartedAt, now);
|
|
8205
|
+
sampleRss();
|
|
7788
8206
|
}
|
|
7789
8207
|
}
|
|
7790
|
-
const
|
|
7791
|
-
const
|
|
7792
|
-
|
|
7793
|
-
|
|
7794
|
-
|
|
7795
|
-
|
|
7796
|
-
|
|
7797
|
-
|
|
7798
|
-
|
|
7799
|
-
|
|
7800
|
-
|
|
7801
|
-
|
|
7802
|
-
|
|
7803
|
-
|
|
7804
|
-
|
|
7805
|
-
|
|
7806
|
-
|
|
7807
|
-
|
|
7808
|
-
|
|
7809
|
-
|
|
7810
|
-
|
|
7811
|
-
|
|
7812
|
-
|
|
7813
|
-
|
|
8208
|
+
const databaseSetupStartedAt = now();
|
|
8209
|
+
const databaseSetup = (() => {
|
|
8210
|
+
let db2;
|
|
8211
|
+
try {
|
|
8212
|
+
db2 = deps.openDatabase(sessionId);
|
|
8213
|
+
return {
|
|
8214
|
+
ok: true,
|
|
8215
|
+
db: db2,
|
|
8216
|
+
insertMeta: db2.prepare(
|
|
8217
|
+
`INSERT INTO meta (name, platform, type, imported_at, group_id, group_avatar, owner_id) VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
8218
|
+
),
|
|
8219
|
+
insertMember: db2.prepare(
|
|
8220
|
+
`INSERT INTO member (platform_id, account_name, group_nickname, aliases, avatar, roles)
|
|
8221
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
8222
|
+
ON CONFLICT(platform_id) DO UPDATE SET
|
|
8223
|
+
account_name = COALESCE(NULLIF(excluded.account_name, ''), account_name),
|
|
8224
|
+
group_nickname = COALESCE(NULLIF(excluded.group_nickname, ''), group_nickname),
|
|
8225
|
+
aliases = CASE WHEN excluded.aliases != '[]' THEN excluded.aliases ELSE aliases END,
|
|
8226
|
+
avatar = COALESCE(NULLIF(excluded.avatar, ''), avatar),
|
|
8227
|
+
roles = CASE WHEN excluded.roles != '[]' THEN excluded.roles ELSE roles END`
|
|
8228
|
+
),
|
|
8229
|
+
getMemberId: db2.prepare(`SELECT id FROM member WHERE platform_id = ?`),
|
|
8230
|
+
insertNameHistory: db2.prepare(
|
|
8231
|
+
`INSERT INTO member_name_history (member_id, name_type, name, start_ts, end_ts) VALUES (?, ?, ?, ?, ?)`
|
|
8232
|
+
),
|
|
8233
|
+
updateMemberAccountName: db2.prepare(`UPDATE member SET account_name = ? WHERE platform_id = ?`),
|
|
8234
|
+
updateMemberGroupNickname: db2.prepare(`UPDATE member SET group_nickname = ? WHERE platform_id = ?`)
|
|
8235
|
+
};
|
|
8236
|
+
} catch (error) {
|
|
8237
|
+
try {
|
|
8238
|
+
db2?.close();
|
|
8239
|
+
} catch {
|
|
8240
|
+
}
|
|
8241
|
+
return { ok: false, error, databaseOpened: db2 !== void 0 };
|
|
8242
|
+
}
|
|
8243
|
+
})();
|
|
8244
|
+
timings.databaseSetupMs = elapsedMs(databaseSetupStartedAt, now);
|
|
8245
|
+
sampleRss();
|
|
8246
|
+
if (!databaseSetup.ok) {
|
|
8247
|
+
if (tempFilePath && preprocessor) preprocessor.cleanup(tempFilePath);
|
|
8248
|
+
if (databaseSetup.databaseOpened) deps.deleteDatabase(sessionId);
|
|
8249
|
+
const error = databaseSetup.error;
|
|
8250
|
+
logger?.error("Import failed during database setup", error instanceof Error ? error : void 0);
|
|
8251
|
+
throw error;
|
|
8252
|
+
}
|
|
8253
|
+
const {
|
|
8254
|
+
db,
|
|
8255
|
+
insertMeta,
|
|
8256
|
+
insertMember,
|
|
8257
|
+
getMemberId,
|
|
8258
|
+
insertNameHistory,
|
|
8259
|
+
updateMemberAccountName,
|
|
8260
|
+
updateMemberGroupNickname
|
|
8261
|
+
} = databaseSetup;
|
|
7814
8262
|
const memberIdMap = /* @__PURE__ */ new Map();
|
|
7815
8263
|
const accountNameTracker = /* @__PURE__ */ new Map();
|
|
7816
8264
|
const groupNicknameTracker = /* @__PURE__ */ new Map();
|
|
7817
8265
|
let metaInserted = false;
|
|
8266
|
+
let importedPlatform = formatFeature.platform;
|
|
7818
8267
|
let messageCountInBatch = 0;
|
|
7819
8268
|
let totalMessageCount = 0;
|
|
7820
8269
|
let duplicateCount = 0;
|
|
7821
8270
|
let lastCheckpointCount = 0;
|
|
7822
8271
|
let inTransaction = false;
|
|
8272
|
+
let messageTransactionCount = 0;
|
|
8273
|
+
let messageTransactionMs = 0;
|
|
8274
|
+
let messageInsertStatementCount = 0;
|
|
8275
|
+
const messageBatchInserter = new MessageBatchInserter(db);
|
|
7823
8276
|
const beginTransaction = () => {
|
|
7824
8277
|
if (!inTransaction) {
|
|
7825
|
-
|
|
7826
|
-
|
|
8278
|
+
const startedAt = now();
|
|
8279
|
+
try {
|
|
8280
|
+
db.exec("BEGIN TRANSACTION");
|
|
8281
|
+
inTransaction = true;
|
|
8282
|
+
messageTransactionCount++;
|
|
8283
|
+
} finally {
|
|
8284
|
+
messageTransactionMs += elapsedMs(startedAt, now);
|
|
8285
|
+
}
|
|
8286
|
+
}
|
|
8287
|
+
};
|
|
8288
|
+
const commitTransaction = () => {
|
|
8289
|
+
if (inTransaction) {
|
|
8290
|
+
const startedAt = now();
|
|
8291
|
+
try {
|
|
8292
|
+
db.exec("COMMIT");
|
|
8293
|
+
inTransaction = false;
|
|
8294
|
+
} finally {
|
|
8295
|
+
messageTransactionMs += elapsedMs(startedAt, now);
|
|
8296
|
+
}
|
|
7827
8297
|
}
|
|
7828
8298
|
};
|
|
7829
8299
|
const doCheckpoint = () => {
|
|
@@ -7832,13 +8302,20 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
7832
8302
|
} catch {
|
|
7833
8303
|
}
|
|
7834
8304
|
};
|
|
8305
|
+
const measureCheckpoint = () => {
|
|
8306
|
+
const startedAt = now();
|
|
8307
|
+
try {
|
|
8308
|
+
doCheckpoint();
|
|
8309
|
+
} finally {
|
|
8310
|
+
timings.checkpointMs += elapsedMs(startedAt, now);
|
|
8311
|
+
}
|
|
8312
|
+
};
|
|
7835
8313
|
const commitAndBeginNew = () => {
|
|
7836
8314
|
if (inTransaction) {
|
|
7837
|
-
|
|
7838
|
-
inTransaction = false;
|
|
8315
|
+
commitTransaction();
|
|
7839
8316
|
logger?.perf("Commit transaction", totalMessageCount, BATCH_COMMIT_SIZE);
|
|
7840
8317
|
if (totalMessageCount - lastCheckpointCount >= CHECKPOINT_INTERVAL) {
|
|
7841
|
-
|
|
8318
|
+
measureCheckpoint();
|
|
7842
8319
|
logger?.perf("WAL checkpoint", totalMessageCount);
|
|
7843
8320
|
lastCheckpointCount = totalMessageCount;
|
|
7844
8321
|
}
|
|
@@ -7853,7 +8330,6 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
7853
8330
|
}
|
|
7854
8331
|
beginTransaction();
|
|
7855
8332
|
};
|
|
7856
|
-
beginTransaction();
|
|
7857
8333
|
let shouldDeleteDb = false;
|
|
7858
8334
|
let importError = null;
|
|
7859
8335
|
const callbackStats = {
|
|
@@ -7871,7 +8347,18 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
7871
8347
|
};
|
|
7872
8348
|
const dedupState = createMessageDedupState();
|
|
7873
8349
|
logger?.info("Starting streamParseFile...");
|
|
8350
|
+
const parsePipelineStartedAt = now();
|
|
8351
|
+
let parserTimingFinished = false;
|
|
8352
|
+
const finishParserTiming = () => {
|
|
8353
|
+
if (parserTimingFinished) return;
|
|
8354
|
+
const parsePipelineMs = elapsedMs(parsePipelineStartedAt, now);
|
|
8355
|
+
const writeCallbackMs = timings.metaWriteMs + timings.memberWriteMs + timings.messageWriteMs + messageTransactionMs + timings.checkpointMs;
|
|
8356
|
+
timings.parserMs = Math.max(0, parsePipelineMs - writeCallbackMs);
|
|
8357
|
+
parserTimingFinished = true;
|
|
8358
|
+
sampleRss();
|
|
8359
|
+
};
|
|
7874
8360
|
try {
|
|
8361
|
+
beginTransaction();
|
|
7875
8362
|
await streamParseFile(
|
|
7876
8363
|
actualFilePath,
|
|
7877
8364
|
{
|
|
@@ -7890,123 +8377,139 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
7890
8377
|
}
|
|
7891
8378
|
},
|
|
7892
8379
|
onMeta: (meta) => {
|
|
7893
|
-
|
|
7894
|
-
|
|
7895
|
-
|
|
7896
|
-
|
|
7897
|
-
|
|
7898
|
-
meta.platform
|
|
7899
|
-
|
|
7900
|
-
|
|
7901
|
-
|
|
7902
|
-
|
|
7903
|
-
|
|
7904
|
-
|
|
7905
|
-
|
|
8380
|
+
const startedAt = now();
|
|
8381
|
+
try {
|
|
8382
|
+
callbackStats.onMetaCalls++;
|
|
8383
|
+
importedPlatform = meta.platform || importedPlatform;
|
|
8384
|
+
if (!metaInserted) {
|
|
8385
|
+
logger?.info(`Writing meta: name=${meta.name}, type=${meta.type}, platform=${meta.platform}`);
|
|
8386
|
+
insertMeta.run(
|
|
8387
|
+
meta.name,
|
|
8388
|
+
meta.platform,
|
|
8389
|
+
meta.type,
|
|
8390
|
+
Math.floor(Date.now() / 1e3),
|
|
8391
|
+
meta.groupId || null,
|
|
8392
|
+
meta.groupAvatar || null,
|
|
8393
|
+
meta.ownerId || null
|
|
8394
|
+
);
|
|
8395
|
+
metaInserted = true;
|
|
8396
|
+
}
|
|
8397
|
+
} finally {
|
|
8398
|
+
timings.metaWriteMs += elapsedMs(startedAt, now);
|
|
8399
|
+
sampleRss();
|
|
7906
8400
|
}
|
|
7907
8401
|
},
|
|
7908
8402
|
onMembers: (members) => {
|
|
7909
|
-
|
|
7910
|
-
|
|
7911
|
-
|
|
7912
|
-
|
|
7913
|
-
|
|
7914
|
-
|
|
7915
|
-
|
|
7916
|
-
|
|
7917
|
-
|
|
7918
|
-
|
|
7919
|
-
|
|
7920
|
-
|
|
7921
|
-
|
|
7922
|
-
|
|
8403
|
+
const startedAt = now();
|
|
8404
|
+
try {
|
|
8405
|
+
callbackStats.onMembersCalls++;
|
|
8406
|
+
callbackStats.totalMembersReceived += members.length;
|
|
8407
|
+
logger?.info(`Received member batch: ${members.length} members`);
|
|
8408
|
+
for (const member of members) {
|
|
8409
|
+
const accountName = normalizeSystemMemberName(formatFeature.id, member.platformId, member.accountName);
|
|
8410
|
+
const groupNickname = normalizeSystemMemberName(formatFeature.id, member.platformId, member.groupNickname);
|
|
8411
|
+
insertMember.run(
|
|
8412
|
+
member.platformId,
|
|
8413
|
+
accountName || null,
|
|
8414
|
+
groupNickname || null,
|
|
8415
|
+
member.aliases ? JSON.stringify(member.aliases) : "[]",
|
|
8416
|
+
member.avatar || null,
|
|
8417
|
+
member.roles ? JSON.stringify(member.roles) : "[]"
|
|
8418
|
+
);
|
|
8419
|
+
const row = getMemberId.get(member.platformId);
|
|
8420
|
+
if (row) memberIdMap.set(member.platformId, row.id);
|
|
8421
|
+
}
|
|
8422
|
+
} finally {
|
|
8423
|
+
timings.memberWriteMs += elapsedMs(startedAt, now);
|
|
8424
|
+
sampleRss();
|
|
7923
8425
|
}
|
|
7924
8426
|
},
|
|
7925
8427
|
onMessageBatch: (messages) => {
|
|
7926
|
-
|
|
7927
|
-
|
|
7928
|
-
|
|
7929
|
-
|
|
7930
|
-
|
|
7931
|
-
|
|
7932
|
-
|
|
7933
|
-
|
|
7934
|
-
|
|
7935
|
-
|
|
7936
|
-
|
|
7937
|
-
|
|
7938
|
-
|
|
7939
|
-
|
|
7940
|
-
if ("skipCounter" in prepared) {
|
|
7941
|
-
callbackStats[prepared.skipCounter]++;
|
|
7942
|
-
continue;
|
|
7943
|
-
}
|
|
7944
|
-
const dedupMessage = prepared.message;
|
|
7945
|
-
if (registerMessageAndCheckDuplicate(dedupMessage, dedupState)) {
|
|
7946
|
-
duplicateCount++;
|
|
7947
|
-
continue;
|
|
8428
|
+
const startedAt = now();
|
|
8429
|
+
const transactionMsBefore = messageTransactionMs;
|
|
8430
|
+
const checkpointMsBefore = timings.checkpointMs;
|
|
8431
|
+
try {
|
|
8432
|
+
const pendingRows = [];
|
|
8433
|
+
const flushPendingRows = () => {
|
|
8434
|
+
if (pendingRows.length === 0) return;
|
|
8435
|
+
messageInsertStatementCount += messageBatchInserter.insert(pendingRows);
|
|
8436
|
+
pendingRows.length = 0;
|
|
8437
|
+
};
|
|
8438
|
+
callbackStats.onMessageBatchCalls++;
|
|
8439
|
+
callbackStats.totalMessagesReceived += messages.length;
|
|
8440
|
+
if (callbackStats.onMessageBatchCalls <= 3 || callbackStats.onMessageBatchCalls % 10 === 0) {
|
|
8441
|
+
logger?.info(`Received message batch #${callbackStats.onMessageBatchCalls}: ${messages.length} messages`);
|
|
7948
8442
|
}
|
|
7949
|
-
|
|
7950
|
-
|
|
7951
|
-
|
|
8443
|
+
for (const msg of messages) {
|
|
8444
|
+
const senderAccountName = normalizeSystemMemberName(
|
|
8445
|
+
formatFeature.id,
|
|
7952
8446
|
msg.senderPlatformId,
|
|
7953
|
-
msg.senderAccountName
|
|
7954
|
-
msg.senderGroupNickname || null,
|
|
7955
|
-
"[]",
|
|
7956
|
-
null,
|
|
7957
|
-
"[]"
|
|
8447
|
+
msg.senderAccountName
|
|
7958
8448
|
);
|
|
7959
|
-
const
|
|
7960
|
-
|
|
7961
|
-
|
|
7962
|
-
|
|
7963
|
-
|
|
7964
|
-
|
|
7965
|
-
|
|
7966
|
-
|
|
7967
|
-
|
|
7968
|
-
|
|
7969
|
-
|
|
7970
|
-
|
|
7971
|
-
|
|
7972
|
-
|
|
7973
|
-
|
|
7974
|
-
|
|
7975
|
-
|
|
7976
|
-
|
|
7977
|
-
|
|
7978
|
-
|
|
7979
|
-
|
|
7980
|
-
|
|
7981
|
-
|
|
7982
|
-
|
|
7983
|
-
|
|
7984
|
-
|
|
7985
|
-
|
|
7986
|
-
|
|
7987
|
-
|
|
7988
|
-
|
|
7989
|
-
|
|
7990
|
-
|
|
7991
|
-
|
|
7992
|
-
|
|
7993
|
-
|
|
7994
|
-
|
|
7995
|
-
|
|
7996
|
-
|
|
7997
|
-
|
|
7998
|
-
|
|
7999
|
-
|
|
8449
|
+
const senderGroupNickname = normalizeSystemMemberName(
|
|
8450
|
+
formatFeature.id,
|
|
8451
|
+
msg.senderPlatformId,
|
|
8452
|
+
msg.senderGroupNickname
|
|
8453
|
+
);
|
|
8454
|
+
const prepared = prepareMessageForCreate(msg, senderAccountName);
|
|
8455
|
+
if ("skipCounter" in prepared) {
|
|
8456
|
+
callbackStats[prepared.skipCounter]++;
|
|
8457
|
+
continue;
|
|
8458
|
+
}
|
|
8459
|
+
const dedupMessage = prepared.message;
|
|
8460
|
+
if (registerMessageAndCheckDuplicate(dedupMessage, dedupState)) {
|
|
8461
|
+
duplicateCount++;
|
|
8462
|
+
continue;
|
|
8463
|
+
}
|
|
8464
|
+
if (!memberIdMap.has(msg.senderPlatformId)) {
|
|
8465
|
+
insertMember.run(
|
|
8466
|
+
msg.senderPlatformId,
|
|
8467
|
+
senderAccountName || null,
|
|
8468
|
+
senderGroupNickname || null,
|
|
8469
|
+
"[]",
|
|
8470
|
+
null,
|
|
8471
|
+
"[]"
|
|
8472
|
+
);
|
|
8473
|
+
const row = getMemberId.get(msg.senderPlatformId);
|
|
8474
|
+
if (row) memberIdMap.set(msg.senderPlatformId, row.id);
|
|
8475
|
+
}
|
|
8476
|
+
const senderId = memberIdMap.get(msg.senderPlatformId);
|
|
8477
|
+
if (senderId === void 0) continue;
|
|
8478
|
+
pendingRows.push({
|
|
8479
|
+
senderId,
|
|
8480
|
+
senderAccountName: senderAccountName || null,
|
|
8481
|
+
senderGroupNickname: senderGroupNickname || null,
|
|
8482
|
+
timestamp: dedupMessage.timestamp,
|
|
8483
|
+
type: dedupMessage.type,
|
|
8484
|
+
content: dedupMessage.content,
|
|
8485
|
+
replyToMessageId: msg.replyToMessageId || null,
|
|
8486
|
+
platformMessageId: msg.platformMessageId || null
|
|
8487
|
+
});
|
|
8488
|
+
messageCountInBatch++;
|
|
8489
|
+
totalMessageCount++;
|
|
8490
|
+
trackNickname(accountNameTracker, msg.senderPlatformId, senderAccountName, msg.timestamp);
|
|
8491
|
+
trackNickname(groupNicknameTracker, msg.senderPlatformId, senderGroupNickname, msg.timestamp);
|
|
8492
|
+
if (messageCountInBatch >= BATCH_COMMIT_SIZE) {
|
|
8493
|
+
flushPendingRows();
|
|
8494
|
+
commitAndBeginNew();
|
|
8495
|
+
messageCountInBatch = 0;
|
|
8496
|
+
}
|
|
8000
8497
|
}
|
|
8498
|
+
flushPendingRows();
|
|
8499
|
+
} finally {
|
|
8500
|
+
const nestedStageMs = messageTransactionMs - transactionMsBefore + (timings.checkpointMs - checkpointMsBefore);
|
|
8501
|
+
timings.messageWriteMs += Math.max(0, elapsedMs(startedAt, now) - nestedStageMs);
|
|
8502
|
+
sampleRss();
|
|
8001
8503
|
}
|
|
8002
8504
|
}
|
|
8003
8505
|
},
|
|
8004
8506
|
formatFeature.id
|
|
8005
8507
|
);
|
|
8508
|
+
finishParserTiming();
|
|
8006
8509
|
if (inTransaction) {
|
|
8007
|
-
|
|
8008
|
-
inTransaction = false;
|
|
8510
|
+
commitTransaction();
|
|
8009
8511
|
}
|
|
8512
|
+
timings.messageWriteMs += messageTransactionMs;
|
|
8010
8513
|
onProgress({
|
|
8011
8514
|
stage: "saving",
|
|
8012
8515
|
bytesRead: 0,
|
|
@@ -8017,6 +8520,7 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8017
8520
|
});
|
|
8018
8521
|
await yieldToEventLoop();
|
|
8019
8522
|
logger?.perf("Writing nickname history", totalMessageCount);
|
|
8523
|
+
const nicknameHistoryStartedAt = now();
|
|
8020
8524
|
db.exec("BEGIN TRANSACTION");
|
|
8021
8525
|
let historyCount = 0;
|
|
8022
8526
|
flushNicknameHistory(accountNameTracker, "account_name", memberIdMap, insertNameHistory, updateMemberAccountName);
|
|
@@ -8029,6 +8533,8 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8029
8533
|
);
|
|
8030
8534
|
historyCount = countHistory(accountNameTracker) + countHistory(groupNicknameTracker);
|
|
8031
8535
|
db.exec("COMMIT");
|
|
8536
|
+
timings.nicknameHistoryMs = elapsedMs(nicknameHistoryStartedAt, now);
|
|
8537
|
+
sampleRss();
|
|
8032
8538
|
logger?.perf(`Nickname history written (${historyCount} entries)`, totalMessageCount);
|
|
8033
8539
|
onProgress({
|
|
8034
8540
|
stage: "indexing",
|
|
@@ -8040,13 +8546,20 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8040
8546
|
});
|
|
8041
8547
|
await yieldToEventLoop();
|
|
8042
8548
|
logger?.perf("Creating indexes", totalMessageCount);
|
|
8549
|
+
const indexCreationStartedAt = now();
|
|
8043
8550
|
db.exec(CHAT_DB_INDEXES);
|
|
8551
|
+
timings.indexCreationMs = elapsedMs(indexCreationStartedAt, now);
|
|
8552
|
+
sampleRss();
|
|
8044
8553
|
logger?.perf("Indexes created", totalMessageCount);
|
|
8554
|
+
const ftsStartedAt = now();
|
|
8045
8555
|
try {
|
|
8046
8556
|
buildFtsIndex(db);
|
|
8047
8557
|
logger?.perf("FTS index built", totalMessageCount);
|
|
8048
8558
|
} catch (ftsError) {
|
|
8049
8559
|
logger?.error("FTS index build failed (non-fatal)", ftsError instanceof Error ? ftsError : void 0);
|
|
8560
|
+
} finally {
|
|
8561
|
+
timings.ftsMs = elapsedMs(ftsStartedAt, now);
|
|
8562
|
+
sampleRss();
|
|
8050
8563
|
}
|
|
8051
8564
|
onProgress({
|
|
8052
8565
|
stage: "indexing",
|
|
@@ -8057,18 +8570,36 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8057
8570
|
message: ""
|
|
8058
8571
|
});
|
|
8059
8572
|
await yieldToEventLoop();
|
|
8060
|
-
|
|
8573
|
+
measureCheckpoint();
|
|
8574
|
+
sampleRss();
|
|
8061
8575
|
logger?.perf("WAL checkpoint done", totalMessageCount);
|
|
8576
|
+
const sessionIndexStartedAt = now();
|
|
8577
|
+
const sessionGapThreshold = normalizeSessionGapThreshold(deps.sessionGapThreshold);
|
|
8062
8578
|
try {
|
|
8063
|
-
generateSessionIndex
|
|
8579
|
+
const generateSessionIndex2 = deps.generateSessionIndex ?? generateSessionIndex;
|
|
8580
|
+
generateSessionIndex2(db, sessionGapThreshold);
|
|
8581
|
+
logger?.perfDetail(`[SessionIndex] Built with gap threshold ${sessionGapThreshold}s`);
|
|
8064
8582
|
logger?.perf("Session index built", totalMessageCount);
|
|
8065
|
-
} catch {
|
|
8583
|
+
} catch (error) {
|
|
8584
|
+
logger?.error("Session index build failed (non-fatal)", error instanceof Error ? error : void 0);
|
|
8585
|
+
} finally {
|
|
8586
|
+
timings.sessionIndexMs = elapsedMs(sessionIndexStartedAt, now);
|
|
8587
|
+
sampleRss();
|
|
8066
8588
|
}
|
|
8067
|
-
|
|
8068
|
-
|
|
8069
|
-
|
|
8070
|
-
|
|
8589
|
+
if (deps.postImportHook) {
|
|
8590
|
+
const postImportHookStartedAt = now();
|
|
8591
|
+
try {
|
|
8592
|
+
await deps.postImportHook(db, sessionId);
|
|
8593
|
+
logger?.perf("Post-import hook done", totalMessageCount);
|
|
8594
|
+
} catch {
|
|
8595
|
+
} finally {
|
|
8596
|
+
timings.postImportHookMs = elapsedMs(postImportHookStartedAt, now);
|
|
8597
|
+
sampleRss();
|
|
8598
|
+
}
|
|
8071
8599
|
}
|
|
8600
|
+
logger?.perfDetail(
|
|
8601
|
+
`[Stages] parser=${timings.parserMs.toFixed(1)}ms | message-write=${timings.messageWriteMs.toFixed(1)}ms | indexes=${timings.indexCreationMs.toFixed(1)}ms | fts=${timings.ftsMs.toFixed(1)}ms | session-index=${timings.sessionIndexMs.toFixed(1)}ms | hook=${timings.postImportHookMs.toFixed(1)}ms`
|
|
8602
|
+
);
|
|
8072
8603
|
logger?.perf("Import completed", totalMessageCount);
|
|
8073
8604
|
logger?.info(`=== Parser Callback Stats ===`);
|
|
8074
8605
|
logger?.info(`onProgress calls: ${callbackStats.onProgressCalls}`);
|
|
@@ -8100,6 +8631,7 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8100
8631
|
importError = "error.no_messages";
|
|
8101
8632
|
}
|
|
8102
8633
|
} catch (error) {
|
|
8634
|
+
finishParserTiming();
|
|
8103
8635
|
logger?.error("Import failed", error instanceof Error ? error : void 0);
|
|
8104
8636
|
if (inTransaction) {
|
|
8105
8637
|
try {
|
|
@@ -8117,7 +8649,10 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8117
8649
|
if (shouldDeleteDb) {
|
|
8118
8650
|
deps.deleteDatabase(sessionId);
|
|
8119
8651
|
}
|
|
8652
|
+
sampleRss();
|
|
8120
8653
|
}
|
|
8654
|
+
timings.totalMs = elapsedMs(totalStartedAt, now);
|
|
8655
|
+
const bytesPerMegabyte = 1024 * 1024;
|
|
8121
8656
|
const diagnostics = {
|
|
8122
8657
|
logFile: logger?.getCurrentLogFile() ?? null,
|
|
8123
8658
|
detectedFormat: formatFeature ? `${formatFeature.name} (${formatFeature.id})` : null,
|
|
@@ -8130,12 +8665,21 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8130
8665
|
noAccountName: callbackStats.skippedNoAccountName,
|
|
8131
8666
|
invalidTimestamp: callbackStats.skippedInvalidTimestamp,
|
|
8132
8667
|
noType: callbackStats.skippedNoType
|
|
8668
|
+
},
|
|
8669
|
+
performance: {
|
|
8670
|
+
timings,
|
|
8671
|
+
messageBatchCount: callbackStats.onMessageBatchCalls,
|
|
8672
|
+
messageTransactionCount,
|
|
8673
|
+
messageInsertStatementCount,
|
|
8674
|
+
rssStartMb: rssStartBytes / bytesPerMegabyte,
|
|
8675
|
+
rssSampledPeakMb: rssPeakBytes / bytesPerMegabyte,
|
|
8676
|
+
rssSampledDeltaMb: (rssPeakBytes - rssStartBytes) / bytesPerMegabyte
|
|
8133
8677
|
}
|
|
8134
8678
|
};
|
|
8135
8679
|
if (importError) {
|
|
8136
|
-
return { success: false, error: importError, diagnostics };
|
|
8680
|
+
return { success: false, platform: importedPlatform, error: importError, diagnostics };
|
|
8137
8681
|
}
|
|
8138
|
-
return { success: true, sessionId, diagnostics };
|
|
8682
|
+
return { success: true, sessionId, platform: importedPlatform, diagnostics };
|
|
8139
8683
|
}
|
|
8140
8684
|
async function analyzeNewImport(filePath, onProgress, options2) {
|
|
8141
8685
|
const formatFeature = options2?.formatId ? getFormatFeatureById(options2.formatId) : detectFormat(filePath);
|
|
@@ -8396,11 +8940,13 @@ async function analyzeIncrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8396
8940
|
let totalInFile = 0;
|
|
8397
8941
|
let newMessageCount = 0;
|
|
8398
8942
|
let duplicateCount = 0;
|
|
8943
|
+
let platform = formatFeature.platform;
|
|
8399
8944
|
await streamParseFile(
|
|
8400
8945
|
filePath,
|
|
8401
8946
|
{
|
|
8402
8947
|
formatOptions: options2?.chatIndex === void 0 ? void 0 : { chatIndex: options2.chatIndex },
|
|
8403
|
-
onMeta: () => {
|
|
8948
|
+
onMeta: (meta) => {
|
|
8949
|
+
platform = meta.platform;
|
|
8404
8950
|
},
|
|
8405
8951
|
onMembers: () => {
|
|
8406
8952
|
},
|
|
@@ -8422,7 +8968,7 @@ async function analyzeIncrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8422
8968
|
},
|
|
8423
8969
|
options2?.formatId
|
|
8424
8970
|
);
|
|
8425
|
-
return { newMessageCount, duplicateCount, totalInFile };
|
|
8971
|
+
return { newMessageCount, duplicateCount, totalInFile, platform };
|
|
8426
8972
|
}
|
|
8427
8973
|
async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
8428
8974
|
const formatFeature = options2?.formatId ? getFormatFeatureById(options2.formatId) : detectFormat(filePath);
|
|
@@ -8439,7 +8985,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8439
8985
|
const memberUpdateMode = options2?.memberUpdateMode ?? "upsert";
|
|
8440
8986
|
try {
|
|
8441
8987
|
let trackError2 = function(index, reason, detail) {
|
|
8442
|
-
|
|
8988
|
+
errorCount++;
|
|
8443
8989
|
errorReasonCounts[reason] = (errorReasonCounts[reason] || 0) + 1;
|
|
8444
8990
|
if (errorSamples.length < MAX_ERROR_SAMPLES) {
|
|
8445
8991
|
errorSamples.push({ index, reason, detail });
|
|
@@ -8488,7 +9034,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8488
9034
|
let metaUpdated = false;
|
|
8489
9035
|
let membersAdded = 0;
|
|
8490
9036
|
let membersUpdated = 0;
|
|
8491
|
-
let
|
|
9037
|
+
let errorCount = 0;
|
|
8492
9038
|
const errorReasonCounts = {};
|
|
8493
9039
|
const errorSamples = [];
|
|
8494
9040
|
const MAX_ERROR_SAMPLES = 5;
|
|
@@ -8513,10 +9059,12 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8513
9059
|
if (memberUpdateMode === "none") return;
|
|
8514
9060
|
for (const m of members) {
|
|
8515
9061
|
const existed = memberIdMap.has(m.platformId);
|
|
9062
|
+
const accountName = normalizeSystemMemberName(formatFeature.id, m.platformId, m.accountName);
|
|
9063
|
+
const groupNickname = normalizeSystemMemberName(formatFeature.id, m.platformId, m.groupNickname);
|
|
8516
9064
|
upsertMember.run(
|
|
8517
9065
|
m.platformId,
|
|
8518
|
-
|
|
8519
|
-
|
|
9066
|
+
accountName || null,
|
|
9067
|
+
groupNickname || null,
|
|
8520
9068
|
m.aliases ? JSON.stringify(m.aliases) : "[]",
|
|
8521
9069
|
m.avatar || null,
|
|
8522
9070
|
m.roles ? JSON.stringify(m.roles) : "[]"
|
|
@@ -8553,12 +9101,22 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8553
9101
|
duplicateCount++;
|
|
8554
9102
|
continue;
|
|
8555
9103
|
}
|
|
9104
|
+
const senderAccountName = normalizeSystemMemberName(
|
|
9105
|
+
formatFeature.id,
|
|
9106
|
+
msg.senderPlatformId,
|
|
9107
|
+
msg.senderAccountName
|
|
9108
|
+
);
|
|
9109
|
+
const senderGroupNickname = normalizeSystemMemberName(
|
|
9110
|
+
formatFeature.id,
|
|
9111
|
+
msg.senderPlatformId,
|
|
9112
|
+
msg.senderGroupNickname
|
|
9113
|
+
);
|
|
8556
9114
|
let memberId = memberIdMap.get(msg.senderPlatformId);
|
|
8557
9115
|
if (!memberId) {
|
|
8558
9116
|
insertMemberMinimal.run(
|
|
8559
9117
|
msg.senderPlatformId,
|
|
8560
|
-
|
|
8561
|
-
|
|
9118
|
+
senderAccountName || null,
|
|
9119
|
+
senderGroupNickname || null,
|
|
8562
9120
|
null
|
|
8563
9121
|
);
|
|
8564
9122
|
const row = getMemberId.get(msg.senderPlatformId);
|
|
@@ -8571,8 +9129,8 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8571
9129
|
if (!memberId) continue;
|
|
8572
9130
|
const msgResult = insertMessage.run(
|
|
8573
9131
|
memberId,
|
|
8574
|
-
|
|
8575
|
-
|
|
9132
|
+
senderAccountName || null,
|
|
9133
|
+
senderGroupNickname || null,
|
|
8576
9134
|
timestamp,
|
|
8577
9135
|
msg.type,
|
|
8578
9136
|
msg.content || null,
|
|
@@ -8612,10 +9170,11 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8612
9170
|
}
|
|
8613
9171
|
if (newMessageCount > 0) {
|
|
8614
9172
|
try {
|
|
9173
|
+
const sessionGapThreshold = getSessionIndexStats(db).gapThreshold;
|
|
8615
9174
|
if (minWrittenTs < preWriteMaxTs) {
|
|
8616
|
-
generateSessionIndex(db);
|
|
9175
|
+
generateSessionIndex(db, sessionGapThreshold);
|
|
8617
9176
|
} else {
|
|
8618
|
-
generateIncrementalSessionIndex(db);
|
|
9177
|
+
generateIncrementalSessionIndex(db, sessionGapThreshold);
|
|
8619
9178
|
}
|
|
8620
9179
|
} catch {
|
|
8621
9180
|
}
|
|
@@ -8627,7 +9186,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8627
9186
|
MAX(ts) as lastTimestamp
|
|
8628
9187
|
FROM message`
|
|
8629
9188
|
).get();
|
|
8630
|
-
const memberCountRow = db.prepare(
|
|
9189
|
+
const memberCountRow = db.prepare(`SELECT COUNT(*) as count FROM member WHERE COALESCE(account_name, '') != ?`).get(SYSTEM_MEMBER_NAME);
|
|
8631
9190
|
try {
|
|
8632
9191
|
await deps.postImportHook?.(db, sessionId);
|
|
8633
9192
|
} catch {
|
|
@@ -8648,7 +9207,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8648
9207
|
receivedCount: processedCount,
|
|
8649
9208
|
writtenCount: newMessageCount,
|
|
8650
9209
|
duplicateCount,
|
|
8651
|
-
errorCount
|
|
9210
|
+
errorCount,
|
|
8652
9211
|
errorReasonCounts,
|
|
8653
9212
|
errorSample: errorSamples
|
|
8654
9213
|
},
|
|
@@ -8682,6 +9241,7 @@ function isValidImportSessionId(sessionId) {
|
|
|
8682
9241
|
}
|
|
8683
9242
|
|
|
8684
9243
|
// ../../packages/node-runtime/src/import/auto-import-matcher.ts
|
|
9244
|
+
import { createHash as createHash2 } from "crypto";
|
|
8685
9245
|
var MATCH_WINDOW_SIZE = 5;
|
|
8686
9246
|
function buildPrivateIdentity(ownerId, memberIds) {
|
|
8687
9247
|
if (!ownerId) return null;
|
|
@@ -8700,6 +9260,9 @@ function addMessageWindow(recentKeys, windows, message) {
|
|
|
8700
9260
|
if (recentKeys.length === MATCH_WINDOW_SIZE) windows.add(recentKeys.join(""));
|
|
8701
9261
|
}
|
|
8702
9262
|
async function resolveAutoImportTarget(filePath, deps, formatOptions) {
|
|
9263
|
+
return (await resolveAutoImportTargetPlan(filePath, deps, formatOptions)).decision;
|
|
9264
|
+
}
|
|
9265
|
+
async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
8703
9266
|
let sourceMeta = null;
|
|
8704
9267
|
const sourceMemberIds = /* @__PURE__ */ new Set();
|
|
8705
9268
|
const sourceWindows = /* @__PURE__ */ new Set();
|
|
@@ -8730,6 +9293,8 @@ async function resolveAutoImportTarget(filePath, deps, formatOptions) {
|
|
|
8730
9293
|
const meta = sourceMeta;
|
|
8731
9294
|
const privateIdentity = meta.type === "private" ? buildPrivateIdentity(meta.ownerId, sourceMemberIds) : null;
|
|
8732
9295
|
const hasStableIdentity = Boolean(meta.groupId || privateIdentity);
|
|
9296
|
+
const stableIdentity = meta.groupId ? `group\0${meta.groupId}` : privateIdentity ? `private\0${privateIdentity}` : null;
|
|
9297
|
+
const stableConcurrencyKey = stableIdentity ? `source:${createHash2("sha256").update(`${meta.platform}\0${meta.type}\0${stableIdentity}`).digest("hex")}` : null;
|
|
8733
9298
|
const stableMatches = [];
|
|
8734
9299
|
const trailingMatches = [];
|
|
8735
9300
|
for (const sessionId of deps.listSessionIds()) {
|
|
@@ -8774,24 +9339,62 @@ async function resolveAutoImportTarget(filePath, deps, formatOptions) {
|
|
|
8774
9339
|
}
|
|
8775
9340
|
if (hasStableIdentity) {
|
|
8776
9341
|
if (meta.sourceSessionId && stableMatches.includes(meta.sourceSessionId)) {
|
|
8777
|
-
return {
|
|
9342
|
+
return {
|
|
9343
|
+
decision: { action: "incremental", sessionId: meta.sourceSessionId, matchedBy: "source-session-id" },
|
|
9344
|
+
concurrencyKey: `session:${meta.sourceSessionId}`,
|
|
9345
|
+
exclusive: false,
|
|
9346
|
+
coalesceCreate: false
|
|
9347
|
+
};
|
|
8778
9348
|
}
|
|
8779
9349
|
if (stableMatches.length === 1) {
|
|
8780
|
-
return {
|
|
9350
|
+
return {
|
|
9351
|
+
decision: { action: "incremental", sessionId: stableMatches[0], matchedBy: "stable-id" },
|
|
9352
|
+
concurrencyKey: `session:${stableMatches[0]}`,
|
|
9353
|
+
exclusive: false,
|
|
9354
|
+
coalesceCreate: false
|
|
9355
|
+
};
|
|
8781
9356
|
}
|
|
8782
9357
|
}
|
|
8783
9358
|
if (meta.sourceSessionId && trailingMatches.includes(meta.sourceSessionId)) {
|
|
8784
|
-
return {
|
|
9359
|
+
return {
|
|
9360
|
+
decision: { action: "incremental", sessionId: meta.sourceSessionId, matchedBy: "source-session-id" },
|
|
9361
|
+
concurrencyKey: `session:${meta.sourceSessionId}`,
|
|
9362
|
+
exclusive: false,
|
|
9363
|
+
coalesceCreate: false
|
|
9364
|
+
};
|
|
8785
9365
|
}
|
|
8786
9366
|
if (trailingMatches.length === 1) {
|
|
8787
|
-
return {
|
|
9367
|
+
return {
|
|
9368
|
+
decision: { action: "incremental", sessionId: trailingMatches[0], matchedBy: "trailing-messages" },
|
|
9369
|
+
concurrencyKey: `session:${trailingMatches[0]}`,
|
|
9370
|
+
exclusive: false,
|
|
9371
|
+
coalesceCreate: false
|
|
9372
|
+
};
|
|
8788
9373
|
}
|
|
8789
9374
|
const ambiguous = stableMatches.length > 1 || trailingMatches.length > 1;
|
|
8790
|
-
|
|
9375
|
+
const decision = { action: "create", reason: ambiguous ? "ambiguous" : "no-match" };
|
|
9376
|
+
if (!ambiguous && stableConcurrencyKey) {
|
|
9377
|
+
return {
|
|
9378
|
+
decision,
|
|
9379
|
+
concurrencyKey: stableConcurrencyKey,
|
|
9380
|
+
exclusive: false,
|
|
9381
|
+
coalesceCreate: true
|
|
9382
|
+
};
|
|
9383
|
+
}
|
|
9384
|
+
return {
|
|
9385
|
+
decision,
|
|
9386
|
+
concurrencyKey: "unresolved",
|
|
9387
|
+
exclusive: true,
|
|
9388
|
+
coalesceCreate: false
|
|
9389
|
+
};
|
|
8791
9390
|
}
|
|
8792
9391
|
|
|
8793
9392
|
// ../../packages/node-runtime/src/import/auto-importer.ts
|
|
8794
9393
|
async function planAutoImport(filePath, deps, options2) {
|
|
9394
|
+
if (options2.resolvedDecision) {
|
|
9395
|
+
const decision2 = options2.resolvedDecision;
|
|
9396
|
+
return decision2.action === "incremental" ? { action: "incremental", sessionId: decision2.sessionId, matchedBy: decision2.matchedBy } : { action: "create", reason: decision2.reason };
|
|
9397
|
+
}
|
|
8795
9398
|
if (options2.explicitSessionId) {
|
|
8796
9399
|
if (!isValidImportSessionId(options2.explicitSessionId)) {
|
|
8797
9400
|
throw new Error("sessionId contains invalid characters");
|
|
@@ -8807,11 +9410,17 @@ async function planAutoImport(filePath, deps, options2) {
|
|
|
8807
9410
|
}
|
|
8808
9411
|
function mapCreateResult(result, createReason) {
|
|
8809
9412
|
if (!result.success || !result.sessionId) {
|
|
8810
|
-
return {
|
|
9413
|
+
return {
|
|
9414
|
+
success: false,
|
|
9415
|
+
...result.platform !== void 0 ? { platform: result.platform } : {},
|
|
9416
|
+
error: result.error,
|
|
9417
|
+
diagnostics: result.diagnostics
|
|
9418
|
+
};
|
|
8811
9419
|
}
|
|
8812
9420
|
return {
|
|
8813
9421
|
success: true,
|
|
8814
9422
|
sessionId: result.sessionId,
|
|
9423
|
+
platform: result.platform,
|
|
8815
9424
|
importMode: "created",
|
|
8816
9425
|
...createReason ? { createReason } : {},
|
|
8817
9426
|
newMessageCount: result.diagnostics?.messagesWritten ?? 0,
|
|
@@ -8825,12 +9434,15 @@ function mapCreateResult(result, createReason) {
|
|
|
8825
9434
|
};
|
|
8826
9435
|
}
|
|
8827
9436
|
function mapIncrementalResult(sessionId, result, matchedBy) {
|
|
8828
|
-
|
|
8829
|
-
return {
|
|
8830
|
-
success: true,
|
|
9437
|
+
const incrementalResult = {
|
|
8831
9438
|
sessionId,
|
|
8832
9439
|
importMode: "incremental",
|
|
8833
|
-
...matchedBy ? { matchedBy } : {}
|
|
9440
|
+
...matchedBy ? { matchedBy } : {}
|
|
9441
|
+
};
|
|
9442
|
+
if (!result.success) return { success: false, ...incrementalResult, error: result.error };
|
|
9443
|
+
return {
|
|
9444
|
+
success: true,
|
|
9445
|
+
...incrementalResult,
|
|
8834
9446
|
newMessageCount: result.newMessageCount,
|
|
8835
9447
|
duplicateCount: result.batch?.duplicateCount ?? 0,
|
|
8836
9448
|
batch: result.batch,
|
|
@@ -8839,17 +9451,18 @@ function mapIncrementalResult(sessionId, result, matchedBy) {
|
|
|
8839
9451
|
};
|
|
8840
9452
|
}
|
|
8841
9453
|
async function autoImportFile(filePath, deps, options2 = {}) {
|
|
9454
|
+
let plan;
|
|
8842
9455
|
try {
|
|
8843
9456
|
if (!options2.explicitSessionId) {
|
|
8844
9457
|
appLogger.info("import", "Automatic session matching started", {
|
|
8845
9458
|
candidateCount: deps.listSessionIds().length
|
|
8846
9459
|
});
|
|
8847
9460
|
}
|
|
8848
|
-
|
|
9461
|
+
plan = await planAutoImport(filePath, deps, options2);
|
|
8849
9462
|
if (plan.action === "incremental") {
|
|
8850
9463
|
const result2 = mapIncrementalResult(
|
|
8851
9464
|
plan.sessionId,
|
|
8852
|
-
await deps.appendSession(plan.sessionId, filePath, options2.formatOptions),
|
|
9465
|
+
await deps.appendSession(plan.sessionId, filePath, options2.formatOptions, deps.onProgress),
|
|
8853
9466
|
plan.matchedBy
|
|
8854
9467
|
);
|
|
8855
9468
|
appLogger.info("import", "Incremental import completed", {
|
|
@@ -8862,7 +9475,7 @@ async function autoImportFile(filePath, deps, options2 = {}) {
|
|
|
8862
9475
|
return result2;
|
|
8863
9476
|
}
|
|
8864
9477
|
const result = mapCreateResult(
|
|
8865
|
-
await deps.createSession(filePath, options2.formatOptions, plan.sessionId),
|
|
9478
|
+
await deps.createSession(filePath, options2.formatOptions, plan.sessionId, deps.onProgress),
|
|
8866
9479
|
plan.reason
|
|
8867
9480
|
);
|
|
8868
9481
|
appLogger.info("import", "Import created a new session", {
|
|
@@ -8873,7 +9486,15 @@ async function autoImportFile(filePath, deps, options2 = {}) {
|
|
|
8873
9486
|
return result;
|
|
8874
9487
|
} catch (error) {
|
|
8875
9488
|
appLogger.error("import", "Automatic import failed", error);
|
|
8876
|
-
return {
|
|
9489
|
+
return {
|
|
9490
|
+
success: false,
|
|
9491
|
+
...plan?.action === "incremental" ? {
|
|
9492
|
+
sessionId: plan.sessionId,
|
|
9493
|
+
importMode: "incremental",
|
|
9494
|
+
...plan.matchedBy ? { matchedBy: plan.matchedBy } : {}
|
|
9495
|
+
} : {},
|
|
9496
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9497
|
+
};
|
|
8877
9498
|
}
|
|
8878
9499
|
}
|
|
8879
9500
|
async function analyzeAutoImportFile(filePath, deps, options2 = {}) {
|
|
@@ -8924,6 +9545,221 @@ async function analyzeAutoImportFile(filePath, deps, options2 = {}) {
|
|
|
8924
9545
|
}
|
|
8925
9546
|
}
|
|
8926
9547
|
|
|
9548
|
+
// ../../packages/node-runtime/src/import/batch-coordinator.ts
|
|
9549
|
+
import { cpus, totalmem } from "os";
|
|
9550
|
+
function normalizeBatchConcurrency(value, taskCount) {
|
|
9551
|
+
if (taskCount <= 0) return 0;
|
|
9552
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 1) return 1;
|
|
9553
|
+
return Math.min(value, taskCount);
|
|
9554
|
+
}
|
|
9555
|
+
function resolveDefaultBatchConcurrency(taskCount, resources = {}) {
|
|
9556
|
+
if (taskCount <= 0) return 0;
|
|
9557
|
+
const cpuCount = resources.cpuCount ?? cpus().length;
|
|
9558
|
+
const totalMemoryBytes = resources.totalMemoryBytes ?? totalmem();
|
|
9559
|
+
const lowResourceRuntime = cpuCount < 4 || totalMemoryBytes < 4 * 1024 * 1024 * 1024;
|
|
9560
|
+
return Math.min(taskCount, lowResourceRuntime ? 1 : 2);
|
|
9561
|
+
}
|
|
9562
|
+
async function runKeyedBatch(tasks, options2) {
|
|
9563
|
+
if (tasks.length === 0) return [];
|
|
9564
|
+
const concurrency = normalizeBatchConcurrency(options2.concurrency, tasks.length);
|
|
9565
|
+
const results = Array(tasks.length);
|
|
9566
|
+
const pending = new Set(tasks.map((_, index) => index));
|
|
9567
|
+
const activeKeys = /* @__PURE__ */ new Set();
|
|
9568
|
+
let activeCount = 0;
|
|
9569
|
+
let exclusiveActive = false;
|
|
9570
|
+
return new Promise((resolve4) => {
|
|
9571
|
+
const finishIfDone = () => {
|
|
9572
|
+
if (activeCount !== 0 || pending.size !== 0) return false;
|
|
9573
|
+
options2.signal?.removeEventListener("abort", schedule);
|
|
9574
|
+
resolve4(results);
|
|
9575
|
+
return true;
|
|
9576
|
+
};
|
|
9577
|
+
const cancelPending = () => {
|
|
9578
|
+
if (!options2.signal?.aborted) return;
|
|
9579
|
+
for (const index of pending) {
|
|
9580
|
+
pending.delete(index);
|
|
9581
|
+
const result = { status: "cancelled" };
|
|
9582
|
+
results[index] = result;
|
|
9583
|
+
options2.onTaskComplete?.(tasks[index].value, index, result);
|
|
9584
|
+
}
|
|
9585
|
+
};
|
|
9586
|
+
const findRunnableIndex = () => {
|
|
9587
|
+
if (exclusiveActive) return void 0;
|
|
9588
|
+
for (const index of pending) {
|
|
9589
|
+
const task = tasks[index];
|
|
9590
|
+
if (task.exclusive) return activeCount === 0 ? index : void 0;
|
|
9591
|
+
if (!activeKeys.has(task.key)) return index;
|
|
9592
|
+
}
|
|
9593
|
+
return void 0;
|
|
9594
|
+
};
|
|
9595
|
+
function schedule() {
|
|
9596
|
+
cancelPending();
|
|
9597
|
+
if (finishIfDone()) return;
|
|
9598
|
+
while (activeCount < concurrency && !options2.signal?.aborted) {
|
|
9599
|
+
const index = findRunnableIndex();
|
|
9600
|
+
if (index === void 0) break;
|
|
9601
|
+
const task = tasks[index];
|
|
9602
|
+
pending.delete(index);
|
|
9603
|
+
activeCount++;
|
|
9604
|
+
activeKeys.add(task.key);
|
|
9605
|
+
if (task.exclusive) exclusiveActive = true;
|
|
9606
|
+
options2.onTaskStart?.(task.value, index);
|
|
9607
|
+
void Promise.resolve().then(() => options2.run(task.value, index)).then(
|
|
9608
|
+
(value) => ({ status: "success", value }),
|
|
9609
|
+
(error) => ({
|
|
9610
|
+
status: "failed",
|
|
9611
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9612
|
+
})
|
|
9613
|
+
).then((result) => {
|
|
9614
|
+
results[index] = result;
|
|
9615
|
+
options2.onTaskComplete?.(task.value, index, result);
|
|
9616
|
+
}).finally(() => {
|
|
9617
|
+
activeCount--;
|
|
9618
|
+
activeKeys.delete(task.key);
|
|
9619
|
+
if (task.exclusive) exclusiveActive = false;
|
|
9620
|
+
schedule();
|
|
9621
|
+
});
|
|
9622
|
+
}
|
|
9623
|
+
}
|
|
9624
|
+
options2.signal?.addEventListener("abort", schedule, { once: true });
|
|
9625
|
+
schedule();
|
|
9626
|
+
});
|
|
9627
|
+
}
|
|
9628
|
+
|
|
9629
|
+
// ../../packages/node-runtime/src/import/auto-import-batch.ts
|
|
9630
|
+
function mapTaskResult(item, result) {
|
|
9631
|
+
if (result.status === "cancelled") return { id: item.id, status: "cancelled" };
|
|
9632
|
+
if (result.status === "failed") return { id: item.id, status: "failed", error: result.error };
|
|
9633
|
+
if (!result.value.success) {
|
|
9634
|
+
return {
|
|
9635
|
+
id: item.id,
|
|
9636
|
+
status: "failed",
|
|
9637
|
+
error: result.value.error ?? "Import failed",
|
|
9638
|
+
result: result.value
|
|
9639
|
+
};
|
|
9640
|
+
}
|
|
9641
|
+
return { id: item.id, status: "success", result: result.value };
|
|
9642
|
+
}
|
|
9643
|
+
function preserveSerialTargetMatching(planned) {
|
|
9644
|
+
let priorPlannedWrite = false;
|
|
9645
|
+
for (const plannedItem of planned) {
|
|
9646
|
+
let plan = plannedItem.plan;
|
|
9647
|
+
if (priorPlannedWrite && !plannedItem.item.options?.explicitSessionId && !plan.exclusive) {
|
|
9648
|
+
plan = {
|
|
9649
|
+
...plan,
|
|
9650
|
+
concurrencyKey: `unresolved:${plannedItem.itemIndex}`,
|
|
9651
|
+
exclusive: true,
|
|
9652
|
+
coalesceCreate: false
|
|
9653
|
+
};
|
|
9654
|
+
}
|
|
9655
|
+
plannedItem.plan = plan;
|
|
9656
|
+
priorPlannedWrite = true;
|
|
9657
|
+
}
|
|
9658
|
+
}
|
|
9659
|
+
async function autoImportBatch(items, deps, options2 = {}) {
|
|
9660
|
+
if (items.length === 0) return [];
|
|
9661
|
+
const resolvePlan = options2.resolveTargetPlan ?? resolveAutoImportTargetPlan;
|
|
9662
|
+
const concurrency = normalizeBatchConcurrency(options2.concurrency, items.length);
|
|
9663
|
+
appLogger.info("import-batch", "Batch import started", {
|
|
9664
|
+
itemCount: items.length,
|
|
9665
|
+
concurrency
|
|
9666
|
+
});
|
|
9667
|
+
const preflight = await runKeyedBatch(
|
|
9668
|
+
items.map((item, index) => ({ value: { item, itemIndex: index }, key: `preflight:${index}` })),
|
|
9669
|
+
{
|
|
9670
|
+
concurrency,
|
|
9671
|
+
signal: options2.signal,
|
|
9672
|
+
onTaskStart: ({ item, itemIndex }) => options2.onItemStart?.(item, itemIndex),
|
|
9673
|
+
run: async ({ item, itemIndex }) => {
|
|
9674
|
+
const itemOptions = item.options ?? {};
|
|
9675
|
+
if (itemOptions.explicitSessionId) {
|
|
9676
|
+
const sessionId = itemOptions.explicitSessionId;
|
|
9677
|
+
const decision = deps.sessionExists(sessionId) ? { action: "incremental", sessionId, matchedBy: "source-session-id" } : { action: "create", reason: "no-match" };
|
|
9678
|
+
return {
|
|
9679
|
+
item,
|
|
9680
|
+
itemIndex,
|
|
9681
|
+
plan: {
|
|
9682
|
+
decision,
|
|
9683
|
+
concurrencyKey: `session:${sessionId}`,
|
|
9684
|
+
exclusive: false,
|
|
9685
|
+
coalesceCreate: decision.action === "create"
|
|
9686
|
+
}
|
|
9687
|
+
};
|
|
9688
|
+
}
|
|
9689
|
+
return {
|
|
9690
|
+
item,
|
|
9691
|
+
itemIndex,
|
|
9692
|
+
plan: await resolvePlan(
|
|
9693
|
+
item.filePath,
|
|
9694
|
+
{
|
|
9695
|
+
...deps,
|
|
9696
|
+
onProgress: (progress) => options2.onItemProgress?.(item, itemIndex, progress)
|
|
9697
|
+
},
|
|
9698
|
+
itemOptions.formatOptions
|
|
9699
|
+
)
|
|
9700
|
+
};
|
|
9701
|
+
}
|
|
9702
|
+
}
|
|
9703
|
+
);
|
|
9704
|
+
const finalResults = Array(items.length);
|
|
9705
|
+
const planned = [];
|
|
9706
|
+
for (let index = 0; index < preflight.length; index++) {
|
|
9707
|
+
const result = preflight[index];
|
|
9708
|
+
if (result.status === "success") {
|
|
9709
|
+
planned.push(result.value);
|
|
9710
|
+
continue;
|
|
9711
|
+
}
|
|
9712
|
+
const mapped = result.status === "cancelled" ? { id: items[index].id, status: "cancelled" } : { id: items[index].id, status: "failed", error: result.error };
|
|
9713
|
+
finalResults[index] = mapped;
|
|
9714
|
+
options2.onItemComplete?.(items[index], index, mapped);
|
|
9715
|
+
}
|
|
9716
|
+
preserveSerialTargetMatching(planned);
|
|
9717
|
+
const targetSessionByKey = /* @__PURE__ */ new Map();
|
|
9718
|
+
await runKeyedBatch(
|
|
9719
|
+
planned.map((plannedItem) => ({
|
|
9720
|
+
value: plannedItem,
|
|
9721
|
+
key: plannedItem.plan.concurrencyKey,
|
|
9722
|
+
exclusive: plannedItem.plan.exclusive
|
|
9723
|
+
})),
|
|
9724
|
+
{
|
|
9725
|
+
concurrency,
|
|
9726
|
+
signal: options2.signal,
|
|
9727
|
+
onTaskComplete: ({ item, itemIndex }, _plannedIndex, taskResult) => {
|
|
9728
|
+
const mapped = mapTaskResult(item, taskResult);
|
|
9729
|
+
finalResults[itemIndex] = mapped;
|
|
9730
|
+
options2.onItemComplete?.(item, itemIndex, mapped);
|
|
9731
|
+
},
|
|
9732
|
+
run: async ({ item, itemIndex, plan }) => {
|
|
9733
|
+
const targetSessionId = plan.coalesceCreate ? targetSessionByKey.get(plan.concurrencyKey) : void 0;
|
|
9734
|
+
const decision = targetSessionId ? { action: "incremental", sessionId: targetSessionId, matchedBy: "trailing-messages" } : plan.decision;
|
|
9735
|
+
const result = await autoImportFile(
|
|
9736
|
+
item.filePath,
|
|
9737
|
+
{
|
|
9738
|
+
...deps,
|
|
9739
|
+
onProgress: (progress) => options2.onItemProgress?.(item, itemIndex, progress)
|
|
9740
|
+
},
|
|
9741
|
+
{
|
|
9742
|
+
...item.options,
|
|
9743
|
+
resolvedDecision: plan.exclusive || item.options?.explicitSessionId && !targetSessionId ? void 0 : decision
|
|
9744
|
+
}
|
|
9745
|
+
);
|
|
9746
|
+
if (!plan.exclusive && result.success && result.sessionId) {
|
|
9747
|
+
targetSessionByKey.set(plan.concurrencyKey, result.sessionId);
|
|
9748
|
+
}
|
|
9749
|
+
return result;
|
|
9750
|
+
}
|
|
9751
|
+
}
|
|
9752
|
+
);
|
|
9753
|
+
const completedResults = finalResults;
|
|
9754
|
+
appLogger.info("import-batch", "Batch import completed", {
|
|
9755
|
+
itemCount: items.length,
|
|
9756
|
+
successCount: completedResults.filter((result) => result.status === "success").length,
|
|
9757
|
+
failedCount: completedResults.filter((result) => result.status === "failed").length,
|
|
9758
|
+
cancelledCount: completedResults.filter((result) => result.status === "cancelled").length
|
|
9759
|
+
});
|
|
9760
|
+
return completedResults;
|
|
9761
|
+
}
|
|
9762
|
+
|
|
8927
9763
|
// ../../packages/node-runtime/src/import/import-lock.ts
|
|
8928
9764
|
import * as fs22 from "fs";
|
|
8929
9765
|
import * as path14 from "path";
|
|
@@ -9758,8 +10594,8 @@ var AssistantManager = class {
|
|
|
9758
10594
|
}
|
|
9759
10595
|
// ==================== Init ====================
|
|
9760
10596
|
init() {
|
|
9761
|
-
const { fs:
|
|
9762
|
-
|
|
10597
|
+
const { fs: fs37, assistantsDir } = this.deps;
|
|
10598
|
+
fs37.ensureDir(assistantsDir);
|
|
9763
10599
|
const { generalCreated, generalUpdated } = this.syncGeneralAssistants();
|
|
9764
10600
|
this.loadAll();
|
|
9765
10601
|
this.initialized = true;
|
|
@@ -9771,25 +10607,25 @@ var AssistantManager = class {
|
|
|
9771
10607
|
return { total: this.cache.size, generalCreated, generalUpdated };
|
|
9772
10608
|
}
|
|
9773
10609
|
syncGeneralAssistants() {
|
|
9774
|
-
const { fs:
|
|
10610
|
+
const { fs: fs37, assistantsDir } = this.deps;
|
|
9775
10611
|
let generalCreated = false;
|
|
9776
10612
|
let generalUpdated = false;
|
|
9777
10613
|
for (const id of this.generalIds) {
|
|
9778
10614
|
const config = this.getBuiltinConfig(id);
|
|
9779
10615
|
if (!config) continue;
|
|
9780
|
-
const filePath =
|
|
9781
|
-
if (!
|
|
9782
|
-
|
|
10616
|
+
const filePath = fs37.joinPath(assistantsDir, `${id}.md`);
|
|
10617
|
+
if (!fs37.fileExists(filePath)) {
|
|
10618
|
+
fs37.writeFile(filePath, serializeAssistant(this.withBuiltinTracking(config)));
|
|
9783
10619
|
generalCreated = true;
|
|
9784
10620
|
continue;
|
|
9785
10621
|
}
|
|
9786
10622
|
try {
|
|
9787
|
-
const existing = parseAssistantFile(
|
|
10623
|
+
const existing = parseAssistantFile(fs37.readFile(filePath), filePath);
|
|
9788
10624
|
if (!existing) continue;
|
|
9789
10625
|
if (existing.builtinId !== id) {
|
|
9790
10626
|
try {
|
|
9791
10627
|
const backupId = this.backupAssistantOccupyingReservedId(existing);
|
|
9792
|
-
|
|
10628
|
+
fs37.writeFile(filePath, serializeAssistant(this.withBuiltinTracking(config)));
|
|
9793
10629
|
generalCreated = true;
|
|
9794
10630
|
this.deps.logger?.info("AssistantManager", "Backed up assistant occupying reserved default ID", {
|
|
9795
10631
|
reservedId: id,
|
|
@@ -9805,7 +10641,7 @@ var AssistantManager = class {
|
|
|
9805
10641
|
continue;
|
|
9806
10642
|
}
|
|
9807
10643
|
if (!this.shouldUpgradeBuiltin(existing, config)) continue;
|
|
9808
|
-
|
|
10644
|
+
fs37.writeFile(filePath, serializeAssistant(this.withBuiltinTracking(config, existing.id)));
|
|
9809
10645
|
generalUpdated = true;
|
|
9810
10646
|
this.deps.logger?.info("AssistantManager", "Updated unmodified default assistant template", {
|
|
9811
10647
|
id,
|
|
@@ -9821,14 +10657,14 @@ var AssistantManager = class {
|
|
|
9821
10657
|
return { generalCreated, generalUpdated };
|
|
9822
10658
|
}
|
|
9823
10659
|
backupAssistantOccupyingReservedId(config) {
|
|
9824
|
-
const { fs:
|
|
10660
|
+
const { fs: fs37, assistantsDir } = this.deps;
|
|
9825
10661
|
const baseId = this.deps.generateId?.() || `custom_${Date.now().toString(36)}`;
|
|
9826
10662
|
let backupId = baseId;
|
|
9827
10663
|
let suffix = 2;
|
|
9828
|
-
while (this.generalIds.includes(backupId) ||
|
|
10664
|
+
while (this.generalIds.includes(backupId) || fs37.fileExists(fs37.joinPath(assistantsDir, `${backupId}.md`))) {
|
|
9829
10665
|
backupId = `${baseId}_${suffix++}`;
|
|
9830
10666
|
}
|
|
9831
|
-
|
|
10667
|
+
fs37.writeFile(fs37.joinPath(assistantsDir, `${backupId}.md`), serializeAssistant({ ...config, id: backupId }));
|
|
9832
10668
|
return backupId;
|
|
9833
10669
|
}
|
|
9834
10670
|
getConfigDigest(config) {
|
|
@@ -9865,13 +10701,13 @@ var AssistantManager = class {
|
|
|
9865
10701
|
return { isUnmodified, isOutdated };
|
|
9866
10702
|
}
|
|
9867
10703
|
loadAll() {
|
|
9868
|
-
const { fs:
|
|
10704
|
+
const { fs: fs37, assistantsDir } = this.deps;
|
|
9869
10705
|
this.cache.clear();
|
|
9870
|
-
const files =
|
|
10706
|
+
const files = fs37.listFiles(assistantsDir, ".md");
|
|
9871
10707
|
for (const file of files) {
|
|
9872
10708
|
try {
|
|
9873
|
-
const filePath =
|
|
9874
|
-
const content =
|
|
10709
|
+
const filePath = fs37.joinPath(assistantsDir, file);
|
|
10710
|
+
const content = fs37.readFile(filePath);
|
|
9875
10711
|
const config = parseAssistantFile(content, filePath);
|
|
9876
10712
|
if (config) {
|
|
9877
10713
|
this.cache.set(config.id, config);
|
|
@@ -10094,7 +10930,7 @@ import Database from "better-sqlite3";
|
|
|
10094
10930
|
import * as fs26 from "fs";
|
|
10095
10931
|
import * as path20 from "path";
|
|
10096
10932
|
var DEFAULT_GENERAL_ID = DEFAULT_GENERAL_ASSISTANT_ID;
|
|
10097
|
-
var
|
|
10933
|
+
var defaultLogger2 = {
|
|
10098
10934
|
warn(_category, message, extra) {
|
|
10099
10935
|
console.warn(`[AI Chats] ${message}`, extra ?? "");
|
|
10100
10936
|
}
|
|
@@ -10107,7 +10943,7 @@ var AIChatManager = class {
|
|
|
10107
10943
|
pendingDebugContextMap = /* @__PURE__ */ new Map();
|
|
10108
10944
|
constructor(aiDataDir, options2) {
|
|
10109
10945
|
this.aiDataDir = aiDataDir;
|
|
10110
|
-
this.logger = options2?.logger ??
|
|
10946
|
+
this.logger = options2?.logger ?? defaultLogger2;
|
|
10111
10947
|
this.nativeBinding = options2?.nativeBinding;
|
|
10112
10948
|
}
|
|
10113
10949
|
ensureDir(dir) {
|
|
@@ -10361,16 +11197,16 @@ var AIChatManager = class {
|
|
|
10361
11197
|
if (allRows.length === 0) return [];
|
|
10362
11198
|
const rowMap = new Map(allRows.map((row) => [row.id, row]));
|
|
10363
11199
|
let currentId = leafMessageId ?? this.getActiveMessageId(aiChatId);
|
|
10364
|
-
const
|
|
11200
|
+
const path31 = [];
|
|
10365
11201
|
const seen = /* @__PURE__ */ new Set();
|
|
10366
11202
|
while (currentId && !seen.has(currentId)) {
|
|
10367
11203
|
seen.add(currentId);
|
|
10368
11204
|
const row = rowMap.get(currentId);
|
|
10369
11205
|
if (!row) break;
|
|
10370
|
-
|
|
11206
|
+
path31.push(row);
|
|
10371
11207
|
currentId = row.parentId;
|
|
10372
11208
|
}
|
|
10373
|
-
return
|
|
11209
|
+
return path31.length > 0 ? path31.reverse() : allRows;
|
|
10374
11210
|
}
|
|
10375
11211
|
// ==================== 生命周期 ====================
|
|
10376
11212
|
close() {
|
|
@@ -10940,21 +11776,21 @@ var SkillManagerCore = class {
|
|
|
10940
11776
|
}
|
|
10941
11777
|
// ==================== Init ====================
|
|
10942
11778
|
init() {
|
|
10943
|
-
const { fs:
|
|
10944
|
-
|
|
11779
|
+
const { fs: fs37, skillsDir } = this.deps;
|
|
11780
|
+
fs37.ensureDir(skillsDir);
|
|
10945
11781
|
this.loadAll();
|
|
10946
11782
|
this.initialized = true;
|
|
10947
11783
|
this.deps.logger?.info("SkillManager", "Initialized", { total: this.cache.size });
|
|
10948
11784
|
return { total: this.cache.size };
|
|
10949
11785
|
}
|
|
10950
11786
|
loadAll() {
|
|
10951
|
-
const { fs:
|
|
11787
|
+
const { fs: fs37, skillsDir } = this.deps;
|
|
10952
11788
|
this.cache.clear();
|
|
10953
|
-
const files =
|
|
11789
|
+
const files = fs37.listFiles(skillsDir, ".md");
|
|
10954
11790
|
for (const file of files) {
|
|
10955
11791
|
try {
|
|
10956
|
-
const filePath =
|
|
10957
|
-
const content =
|
|
11792
|
+
const filePath = fs37.joinPath(skillsDir, file);
|
|
11793
|
+
const content = fs37.readFile(filePath);
|
|
10958
11794
|
const def = parseSkillFile(content, filePath);
|
|
10959
11795
|
if (def) {
|
|
10960
11796
|
this.cache.set(def.id, def);
|
|
@@ -11109,7 +11945,7 @@ var SkillManagerCore = class {
|
|
|
11109
11945
|
import * as fs27 from "fs";
|
|
11110
11946
|
import * as path21 from "path";
|
|
11111
11947
|
var SKILLS_DIR_NAME = "skills";
|
|
11112
|
-
var
|
|
11948
|
+
var defaultLogger3 = {
|
|
11113
11949
|
info: () => {
|
|
11114
11950
|
},
|
|
11115
11951
|
warn: () => {
|
|
@@ -11143,7 +11979,7 @@ var SkillManager = class {
|
|
|
11143
11979
|
core;
|
|
11144
11980
|
logger;
|
|
11145
11981
|
constructor(aiDataDir, logger) {
|
|
11146
|
-
this.logger = logger ??
|
|
11982
|
+
this.logger = logger ?? defaultLogger3;
|
|
11147
11983
|
this.core = new SkillManagerCore({
|
|
11148
11984
|
fs: nodeFs,
|
|
11149
11985
|
skillsDir: path21.join(aiDataDir, SKILLS_DIR_NAME),
|
|
@@ -11566,7 +12402,7 @@ STRICT RULES:
|
|
|
11566
12402
|
- Omit: pleasantries, filler, redundant back-and-forth, detailed tables (summarize their conclusions instead).
|
|
11567
12403
|
|
|
11568
12404
|
{messages}`;
|
|
11569
|
-
var
|
|
12405
|
+
var defaultLogger4 = {
|
|
11570
12406
|
info: () => {
|
|
11571
12407
|
},
|
|
11572
12408
|
warn: () => {
|
|
@@ -11574,7 +12410,7 @@ var defaultLogger3 = {
|
|
|
11574
12410
|
error: () => {
|
|
11575
12411
|
}
|
|
11576
12412
|
};
|
|
11577
|
-
async function checkAndCompress(aiChatId, config, systemPrompt, llmAdapter, convManager, logger =
|
|
12413
|
+
async function checkAndCompress(aiChatId, config, systemPrompt, llmAdapter, convManager, logger = defaultLogger4) {
|
|
11578
12414
|
if (!config.enabled) {
|
|
11579
12415
|
return { compressed: false, reason: "skipped_disabled" };
|
|
11580
12416
|
}
|
|
@@ -11768,7 +12604,7 @@ function createCompressionLlmAdapter(options2) {
|
|
|
11768
12604
|
|
|
11769
12605
|
// ../../packages/node-runtime/src/ai/preprocessor/pipeline.ts
|
|
11770
12606
|
var MERGE_WINDOW_DEFAULT = 180;
|
|
11771
|
-
var
|
|
12607
|
+
var defaultLogger5 = {
|
|
11772
12608
|
info: () => {
|
|
11773
12609
|
},
|
|
11774
12610
|
warn: () => {
|
|
@@ -11785,10 +12621,10 @@ function emptyStats(count) {
|
|
|
11785
12621
|
desensitizeRulesApplied: 0
|
|
11786
12622
|
};
|
|
11787
12623
|
}
|
|
11788
|
-
function preprocessMessages(messages, config, logger =
|
|
12624
|
+
function preprocessMessages(messages, config, logger = defaultLogger5) {
|
|
11789
12625
|
return preprocessMessagesWithStats(messages, config, logger).messages;
|
|
11790
12626
|
}
|
|
11791
|
-
function preprocessMessagesWithStats(messages, config, logger =
|
|
12627
|
+
function preprocessMessagesWithStats(messages, config, logger = defaultLogger5) {
|
|
11792
12628
|
if (!config || !hasAnyEnabled(config)) return { messages, stats: emptyStats(messages.length) };
|
|
11793
12629
|
if (messages.length === 0) return { messages, stats: emptyStats(0) };
|
|
11794
12630
|
const stats = emptyStats(messages.length);
|
|
@@ -11970,7 +12806,7 @@ function applyCompiledRules(text, compiledRules) {
|
|
|
11970
12806
|
}
|
|
11971
12807
|
return result;
|
|
11972
12808
|
}
|
|
11973
|
-
function desensitizeText(text, rules, logger =
|
|
12809
|
+
function desensitizeText(text, rules, logger = defaultLogger5) {
|
|
11974
12810
|
const enabledRules = rules.filter((r) => r.enabled);
|
|
11975
12811
|
if (enabledRules.length === 0 || !text) return text;
|
|
11976
12812
|
return applyCompiledRules(text, compileRules(enabledRules, logger));
|
|
@@ -13008,7 +13844,7 @@ var PLAN_INTENTS = [
|
|
|
13008
13844
|
"mixed"
|
|
13009
13845
|
];
|
|
13010
13846
|
var STREAM_MARKER_HOLDBACK_CHARS = Math.max("</draft>".length, "</plan>".length, "<json>".length) - 1;
|
|
13011
|
-
function
|
|
13847
|
+
function isRecord3(value) {
|
|
13012
13848
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
13013
13849
|
}
|
|
13014
13850
|
function isIntent(value) {
|
|
@@ -13030,10 +13866,10 @@ function extractJsonObject4(text) {
|
|
|
13030
13866
|
function parsePlanJson(rawText, availableTools) {
|
|
13031
13867
|
try {
|
|
13032
13868
|
const parsed = JSON.parse(extractJsonObject4(rawText));
|
|
13033
|
-
if (!
|
|
13869
|
+
if (!isRecord3(parsed)) return null;
|
|
13034
13870
|
const title = normalizeString(parsed.title);
|
|
13035
13871
|
if (!title || !isIntent(parsed.intent) || !Array.isArray(parsed.steps)) return null;
|
|
13036
|
-
const steps = parsed.steps.filter(
|
|
13872
|
+
const steps = parsed.steps.filter(isRecord3).map((step) => {
|
|
13037
13873
|
const goal = normalizeString(step.goal);
|
|
13038
13874
|
const evidenceNeeded = normalizeString(step.evidenceNeeded);
|
|
13039
13875
|
if (!goal || !evidenceNeeded) return null;
|
|
@@ -15012,9 +15848,9 @@ var localeMap = {
|
|
|
15012
15848
|
"ja-JP": ja_JP_default,
|
|
15013
15849
|
"zh-TW": zh_TW_default
|
|
15014
15850
|
};
|
|
15015
|
-
function getNestedValue(obj,
|
|
15851
|
+
function getNestedValue(obj, path31) {
|
|
15016
15852
|
let current = obj;
|
|
15017
|
-
for (const segment2 of
|
|
15853
|
+
for (const segment2 of path31.split(".")) {
|
|
15018
15854
|
if (current == null || typeof current !== "object") return void 0;
|
|
15019
15855
|
current = current[segment2];
|
|
15020
15856
|
}
|
|
@@ -15542,6 +16378,10 @@ var LLMConfigStore = class {
|
|
|
15542
16378
|
hasActiveConfig() {
|
|
15543
16379
|
return this.getDefaultAssistantConfig() !== null;
|
|
15544
16380
|
}
|
|
16381
|
+
hasConfiguredModel() {
|
|
16382
|
+
const config = this.getDefaultAssistantConfig();
|
|
16383
|
+
return typeof config?.model === "string" && config.model.trim().length > 0;
|
|
16384
|
+
}
|
|
15545
16385
|
};
|
|
15546
16386
|
|
|
15547
16387
|
// ../../packages/node-runtime/src/ai/file-config-storage.ts
|
|
@@ -17116,7 +17956,7 @@ function setOwnerAndApplyProfile(adapter, preferences, sessionId, ownerPlatformI
|
|
|
17116
17956
|
});
|
|
17117
17957
|
return { sessionId, platform: meta.platform, ownerId: ownerPlatformId, updatedSessionIds, updatedSessionOwnerIds };
|
|
17118
17958
|
}
|
|
17119
|
-
function applyProfileToOtherSessions(adapter,
|
|
17959
|
+
function applyProfileToOtherSessions(adapter, platform, profile, excludeSessionId) {
|
|
17120
17960
|
const updated = [];
|
|
17121
17961
|
for (const id of adapter.listSessionIds()) {
|
|
17122
17962
|
if (id === excludeSessionId) continue;
|
|
@@ -17124,8 +17964,8 @@ function applyProfileToOtherSessions(adapter, platform2, profile, excludeSession
|
|
|
17124
17964
|
const db = adapter.openReadonly(id);
|
|
17125
17965
|
if (!db || !isChatSessionDb(db)) continue;
|
|
17126
17966
|
const meta = getSessionMeta(db);
|
|
17127
|
-
if (!meta || meta.platform !==
|
|
17128
|
-
const result = matchOwnerProfile(
|
|
17967
|
+
if (!meta || meta.platform !== platform || meta.ownerId) continue;
|
|
17968
|
+
const result = matchOwnerProfile(platform, profile, getMembersWithAliases(db));
|
|
17129
17969
|
if (result.type === "exact" || result.type === "name") {
|
|
17130
17970
|
updateSessionOwnerId(adapter.ensureWritable(id), result.platformId);
|
|
17131
17971
|
updated.push({ id, ownerId: result.platformId });
|
|
@@ -17180,11 +18020,11 @@ function writeContactOverrides(snapshotDir, overrides) {
|
|
|
17180
18020
|
fs29.renameSync(tmpPath, getContactOverridesPath(snapshotDir));
|
|
17181
18021
|
}
|
|
17182
18022
|
function normalizeContactOverrides(value) {
|
|
17183
|
-
const input =
|
|
17184
|
-
const manualFriendsInput =
|
|
18023
|
+
const input = isRecord4(value) ? value : {};
|
|
18024
|
+
const manualFriendsInput = isRecord4(input.manualFriends) ? input.manualFriends : {};
|
|
17185
18025
|
const manualFriends = {};
|
|
17186
18026
|
for (const [key, entry] of Object.entries(manualFriendsInput)) {
|
|
17187
|
-
if (!key || !
|
|
18027
|
+
if (!key || !isRecord4(entry)) continue;
|
|
17188
18028
|
const createdAt = Number.isFinite(entry.createdAt) ? Number(entry.createdAt) : Date.now();
|
|
17189
18029
|
const updatedAt = Number.isFinite(entry.updatedAt) ? Number(entry.updatedAt) : createdAt;
|
|
17190
18030
|
manualFriends[key] = { key, createdAt, updatedAt };
|
|
@@ -17194,7 +18034,7 @@ function normalizeContactOverrides(value) {
|
|
|
17194
18034
|
manualFriends
|
|
17195
18035
|
};
|
|
17196
18036
|
}
|
|
17197
|
-
function
|
|
18037
|
+
function isRecord4(value) {
|
|
17198
18038
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
17199
18039
|
}
|
|
17200
18040
|
|
|
@@ -18727,11 +19567,130 @@ var SemanticIndexConfigStore = class {
|
|
|
18727
19567
|
}
|
|
18728
19568
|
};
|
|
18729
19569
|
|
|
19570
|
+
// ../../packages/node-runtime/src/semantic-index/embedding/text.ts
|
|
19571
|
+
function applyQueryInstruction(instruction, query) {
|
|
19572
|
+
const trimmed = instruction.trim();
|
|
19573
|
+
if (!trimmed) return query;
|
|
19574
|
+
if (trimmed.endsWith(":") || trimmed.endsWith("\uFF1A")) return `${trimmed}${query}`;
|
|
19575
|
+
return `Instruct: ${trimmed}
|
|
19576
|
+
Query: ${query}`;
|
|
19577
|
+
}
|
|
19578
|
+
function clampTextChars(text, maxChars) {
|
|
19579
|
+
if (maxChars === void 0 || text.length <= maxChars) return text;
|
|
19580
|
+
return text.slice(0, maxChars);
|
|
19581
|
+
}
|
|
19582
|
+
|
|
19583
|
+
// ../../packages/node-runtime/src/semantic-index/embedding/local.ts
|
|
19584
|
+
var SOCKS_PROXY_PROTOCOLS = /* @__PURE__ */ new Set(["socks:", "socks4:", "socks5:"]);
|
|
19585
|
+
var LOCAL_ONNX_SESSION_OPTIONS = { intraOpNumThreads: 1, interOpNumThreads: 1 };
|
|
19586
|
+
var HUGGING_FACE_MODEL_HOST = "https://huggingface.co/";
|
|
19587
|
+
var HF_MIRROR_MODEL_HOST = "https://hf-mirror.com/";
|
|
19588
|
+
function resolveModelDownloadRemoteHost(source) {
|
|
19589
|
+
return source === "hf-mirror" ? HF_MIRROR_MODEL_HOST : HUGGING_FACE_MODEL_HOST;
|
|
19590
|
+
}
|
|
19591
|
+
async function createProxyFetch(proxyUrl) {
|
|
19592
|
+
const parsed = new URL(proxyUrl);
|
|
19593
|
+
if (SOCKS_PROXY_PROTOCOLS.has(parsed.protocol)) {
|
|
19594
|
+
throw new Error(
|
|
19595
|
+
`SOCKS proxy is not supported for local embedding model downloads: ${parsed.protocol}//${parsed.host}`
|
|
19596
|
+
);
|
|
19597
|
+
}
|
|
19598
|
+
const { fetch: undiciFetch, ProxyAgent } = await import("./undici-MU5I3L6Q.mjs");
|
|
19599
|
+
const dispatcher = new ProxyAgent(proxyUrl);
|
|
19600
|
+
return ((input, init) => {
|
|
19601
|
+
const nextInit = { ...init, dispatcher };
|
|
19602
|
+
return undiciFetch(input, nextInit);
|
|
19603
|
+
});
|
|
19604
|
+
}
|
|
19605
|
+
function createTransformersPipelineFactory(loadTransformers) {
|
|
19606
|
+
return async ({ modelId, dtype, cacheDir, modelDownloadProxyUrl, modelDownloadSource }) => {
|
|
19607
|
+
const transformers = await loadTransformers();
|
|
19608
|
+
transformers.env.remoteHost = resolveModelDownloadRemoteHost(modelDownloadSource);
|
|
19609
|
+
if (cacheDir) {
|
|
19610
|
+
transformers.env.cacheDir = cacheDir;
|
|
19611
|
+
transformers.env.allowRemoteModels = true;
|
|
19612
|
+
}
|
|
19613
|
+
if (modelDownloadProxyUrl) {
|
|
19614
|
+
transformers.env.fetch = await createProxyFetch(modelDownloadProxyUrl);
|
|
19615
|
+
}
|
|
19616
|
+
const extractor = await transformers.pipeline("feature-extraction", modelId, {
|
|
19617
|
+
...dtype ? { dtype } : {},
|
|
19618
|
+
session_options: LOCAL_ONNX_SESSION_OPTIONS
|
|
19619
|
+
});
|
|
19620
|
+
return async (texts, options2) => {
|
|
19621
|
+
const out = await extractor(texts, { pooling: options2.pooling, normalize: options2.normalize });
|
|
19622
|
+
return out.tolist();
|
|
19623
|
+
};
|
|
19624
|
+
};
|
|
19625
|
+
}
|
|
19626
|
+
var defaultPipelineFactory = createTransformersPipelineFactory(() => import("@huggingface/transformers"));
|
|
19627
|
+
var LocalEmbeddingProvider = class {
|
|
19628
|
+
modelId;
|
|
19629
|
+
dim;
|
|
19630
|
+
maxTokens;
|
|
19631
|
+
documentBatchSize;
|
|
19632
|
+
profile;
|
|
19633
|
+
options;
|
|
19634
|
+
extractorPromise = null;
|
|
19635
|
+
constructor(profile, options2 = {}) {
|
|
19636
|
+
this.profile = profile;
|
|
19637
|
+
this.options = options2;
|
|
19638
|
+
this.modelId = profile.modelId;
|
|
19639
|
+
this.dim = profile.dim;
|
|
19640
|
+
this.maxTokens = profile.maxTokens;
|
|
19641
|
+
this.documentBatchSize = profile.maxBatchSize ?? 1;
|
|
19642
|
+
}
|
|
19643
|
+
getExtractor() {
|
|
19644
|
+
if (!this.extractorPromise) {
|
|
19645
|
+
const factory = this.options.pipelineFactory ?? defaultPipelineFactory;
|
|
19646
|
+
this.extractorPromise = factory({
|
|
19647
|
+
modelId: this.profile.modelId,
|
|
19648
|
+
dtype: this.profile.dtype,
|
|
19649
|
+
cacheDir: this.options.cacheDir,
|
|
19650
|
+
modelDownloadProxyUrl: this.options.modelDownloadProxyUrl,
|
|
19651
|
+
modelDownloadSource: this.options.modelDownloadSource,
|
|
19652
|
+
sessionOptions: LOCAL_ONNX_SESSION_OPTIONS
|
|
19653
|
+
}).catch((error) => {
|
|
19654
|
+
this.extractorPromise = null;
|
|
19655
|
+
throw error;
|
|
19656
|
+
});
|
|
19657
|
+
}
|
|
19658
|
+
return this.extractorPromise;
|
|
19659
|
+
}
|
|
19660
|
+
toFloat32(vector) {
|
|
19661
|
+
if (vector.length !== this.profile.dim) {
|
|
19662
|
+
throw new Error(`model ${this.profile.modelId} returned dim ${vector.length}, expected ${this.profile.dim}`);
|
|
19663
|
+
}
|
|
19664
|
+
return Float32Array.from(vector);
|
|
19665
|
+
}
|
|
19666
|
+
async embedDocuments(texts) {
|
|
19667
|
+
if (texts.length === 0) return [];
|
|
19668
|
+
const extractor = await this.getExtractor();
|
|
19669
|
+
const batchSize = this.profile.maxBatchSize ?? texts.length;
|
|
19670
|
+
const clamped = texts.map((t2) => clampTextChars(t2, this.profile.maxTextChars));
|
|
19671
|
+
const result = [];
|
|
19672
|
+
for (let i = 0; i < clamped.length; i += batchSize) {
|
|
19673
|
+
const batch = clamped.slice(i, i + batchSize);
|
|
19674
|
+
const vectors = await extractor(batch, { pooling: this.profile.pooling, normalize: this.profile.normalize });
|
|
19675
|
+
for (const v of vectors) result.push(this.toFloat32(v));
|
|
19676
|
+
}
|
|
19677
|
+
return result;
|
|
19678
|
+
}
|
|
19679
|
+
async embedQuery(text) {
|
|
19680
|
+
const withInstruction = applyQueryInstruction(this.profile.queryInstruction, text);
|
|
19681
|
+
const [vector] = await this.embedDocuments([withInstruction]);
|
|
19682
|
+
return vector;
|
|
19683
|
+
}
|
|
19684
|
+
async preload() {
|
|
19685
|
+
await this.getExtractor();
|
|
19686
|
+
}
|
|
19687
|
+
};
|
|
19688
|
+
|
|
18730
19689
|
// ../../packages/node-runtime/src/semantic-index/service.ts
|
|
18731
19690
|
import path28 from "path";
|
|
18732
19691
|
|
|
18733
19692
|
// ../../packages/node-runtime/src/semantic-index/chunker-config.ts
|
|
18734
|
-
import { createHash as
|
|
19693
|
+
import { createHash as createHash3 } from "crypto";
|
|
18735
19694
|
var CHUNKER_VERSION = "v1.3";
|
|
18736
19695
|
var STRATEGY_ID = "balanced";
|
|
18737
19696
|
var DEFAULT_CHUNKER_CONFIG = {
|
|
@@ -18746,7 +19705,7 @@ var DEFAULT_CHUNKER_CONFIG = {
|
|
|
18746
19705
|
semanticVoidSkipThreshold: 20
|
|
18747
19706
|
};
|
|
18748
19707
|
function sha256Hex(input) {
|
|
18749
|
-
return
|
|
19708
|
+
return createHash3("sha256").update(input, "utf-8").digest("hex");
|
|
18750
19709
|
}
|
|
18751
19710
|
function computeChunkerConfigHash(config) {
|
|
18752
19711
|
const canonical = JSON.stringify({
|
|
@@ -18885,128 +19844,6 @@ var OpenAICompatibleEmbeddingProvider = class {
|
|
|
18885
19844
|
}
|
|
18886
19845
|
};
|
|
18887
19846
|
|
|
18888
|
-
// ../../packages/node-runtime/src/semantic-index/embedding/text.ts
|
|
18889
|
-
function applyQueryInstruction(instruction, query) {
|
|
18890
|
-
const trimmed = instruction.trim();
|
|
18891
|
-
if (!trimmed) return query;
|
|
18892
|
-
if (trimmed.endsWith(":") || trimmed.endsWith("\uFF1A")) return `${trimmed}${query}`;
|
|
18893
|
-
return `Instruct: ${trimmed}
|
|
18894
|
-
Query: ${query}`;
|
|
18895
|
-
}
|
|
18896
|
-
function clampTextChars(text, maxChars) {
|
|
18897
|
-
if (maxChars === void 0 || text.length <= maxChars) return text;
|
|
18898
|
-
return text.slice(0, maxChars);
|
|
18899
|
-
}
|
|
18900
|
-
|
|
18901
|
-
// ../../packages/node-runtime/src/semantic-index/embedding/local.ts
|
|
18902
|
-
var SOCKS_PROXY_PROTOCOLS = /* @__PURE__ */ new Set(["socks:", "socks4:", "socks5:"]);
|
|
18903
|
-
var LOCAL_ONNX_SESSION_OPTIONS = { intraOpNumThreads: 1, interOpNumThreads: 1 };
|
|
18904
|
-
var HUGGING_FACE_MODEL_HOST = "https://huggingface.co/";
|
|
18905
|
-
var HF_MIRROR_MODEL_HOST = "https://hf-mirror.com/";
|
|
18906
|
-
function resolveModelDownloadRemoteHost(source) {
|
|
18907
|
-
return source === "hf-mirror" ? HF_MIRROR_MODEL_HOST : HUGGING_FACE_MODEL_HOST;
|
|
18908
|
-
}
|
|
18909
|
-
async function createProxyFetch(proxyUrl) {
|
|
18910
|
-
const parsed = new URL(proxyUrl);
|
|
18911
|
-
if (SOCKS_PROXY_PROTOCOLS.has(parsed.protocol)) {
|
|
18912
|
-
throw new Error(
|
|
18913
|
-
`SOCKS proxy is not supported for local embedding model downloads: ${parsed.protocol}//${parsed.host}`
|
|
18914
|
-
);
|
|
18915
|
-
}
|
|
18916
|
-
const { fetch: undiciFetch, ProxyAgent } = await import("./undici-ZSXBT7EA.mjs");
|
|
18917
|
-
const dispatcher = new ProxyAgent(proxyUrl);
|
|
18918
|
-
return ((input, init) => {
|
|
18919
|
-
const nextInit = { ...init, dispatcher };
|
|
18920
|
-
return undiciFetch(input, nextInit);
|
|
18921
|
-
});
|
|
18922
|
-
}
|
|
18923
|
-
var defaultPipelineFactory = async ({
|
|
18924
|
-
modelId,
|
|
18925
|
-
dtype,
|
|
18926
|
-
cacheDir,
|
|
18927
|
-
modelDownloadProxyUrl,
|
|
18928
|
-
modelDownloadSource
|
|
18929
|
-
}) => {
|
|
18930
|
-
const transformers = await import("./transformers.node-PUMHAODN.mjs");
|
|
18931
|
-
transformers.env.remoteHost = resolveModelDownloadRemoteHost(modelDownloadSource);
|
|
18932
|
-
if (cacheDir) {
|
|
18933
|
-
transformers.env.cacheDir = cacheDir;
|
|
18934
|
-
transformers.env.allowRemoteModels = true;
|
|
18935
|
-
}
|
|
18936
|
-
if (modelDownloadProxyUrl) {
|
|
18937
|
-
transformers.env.fetch = await createProxyFetch(modelDownloadProxyUrl);
|
|
18938
|
-
}
|
|
18939
|
-
const extractor = await transformers.pipeline("feature-extraction", modelId, {
|
|
18940
|
-
...dtype ? { dtype } : {},
|
|
18941
|
-
session_options: LOCAL_ONNX_SESSION_OPTIONS
|
|
18942
|
-
});
|
|
18943
|
-
return async (texts, options2) => {
|
|
18944
|
-
const out = await extractor(texts, { pooling: options2.pooling, normalize: options2.normalize });
|
|
18945
|
-
return out.tolist();
|
|
18946
|
-
};
|
|
18947
|
-
};
|
|
18948
|
-
var LocalEmbeddingProvider = class {
|
|
18949
|
-
modelId;
|
|
18950
|
-
dim;
|
|
18951
|
-
maxTokens;
|
|
18952
|
-
documentBatchSize;
|
|
18953
|
-
profile;
|
|
18954
|
-
options;
|
|
18955
|
-
extractorPromise = null;
|
|
18956
|
-
constructor(profile, options2 = {}) {
|
|
18957
|
-
this.profile = profile;
|
|
18958
|
-
this.options = options2;
|
|
18959
|
-
this.modelId = profile.modelId;
|
|
18960
|
-
this.dim = profile.dim;
|
|
18961
|
-
this.maxTokens = profile.maxTokens;
|
|
18962
|
-
this.documentBatchSize = profile.maxBatchSize ?? 1;
|
|
18963
|
-
}
|
|
18964
|
-
getExtractor() {
|
|
18965
|
-
if (!this.extractorPromise) {
|
|
18966
|
-
const factory = this.options.pipelineFactory ?? defaultPipelineFactory;
|
|
18967
|
-
this.extractorPromise = factory({
|
|
18968
|
-
modelId: this.profile.modelId,
|
|
18969
|
-
dtype: this.profile.dtype,
|
|
18970
|
-
cacheDir: this.options.cacheDir,
|
|
18971
|
-
modelDownloadProxyUrl: this.options.modelDownloadProxyUrl,
|
|
18972
|
-
modelDownloadSource: this.options.modelDownloadSource,
|
|
18973
|
-
sessionOptions: LOCAL_ONNX_SESSION_OPTIONS
|
|
18974
|
-
}).catch((error) => {
|
|
18975
|
-
this.extractorPromise = null;
|
|
18976
|
-
throw error;
|
|
18977
|
-
});
|
|
18978
|
-
}
|
|
18979
|
-
return this.extractorPromise;
|
|
18980
|
-
}
|
|
18981
|
-
toFloat32(vector) {
|
|
18982
|
-
if (vector.length !== this.profile.dim) {
|
|
18983
|
-
throw new Error(`model ${this.profile.modelId} returned dim ${vector.length}, expected ${this.profile.dim}`);
|
|
18984
|
-
}
|
|
18985
|
-
return Float32Array.from(vector);
|
|
18986
|
-
}
|
|
18987
|
-
async embedDocuments(texts) {
|
|
18988
|
-
if (texts.length === 0) return [];
|
|
18989
|
-
const extractor = await this.getExtractor();
|
|
18990
|
-
const batchSize = this.profile.maxBatchSize ?? texts.length;
|
|
18991
|
-
const clamped = texts.map((t2) => clampTextChars(t2, this.profile.maxTextChars));
|
|
18992
|
-
const result = [];
|
|
18993
|
-
for (let i = 0; i < clamped.length; i += batchSize) {
|
|
18994
|
-
const batch = clamped.slice(i, i + batchSize);
|
|
18995
|
-
const vectors = await extractor(batch, { pooling: this.profile.pooling, normalize: this.profile.normalize });
|
|
18996
|
-
for (const v of vectors) result.push(this.toFloat32(v));
|
|
18997
|
-
}
|
|
18998
|
-
return result;
|
|
18999
|
-
}
|
|
19000
|
-
async embedQuery(text) {
|
|
19001
|
-
const withInstruction = applyQueryInstruction(this.profile.queryInstruction, text);
|
|
19002
|
-
const [vector] = await this.embedDocuments([withInstruction]);
|
|
19003
|
-
return vector;
|
|
19004
|
-
}
|
|
19005
|
-
async preload() {
|
|
19006
|
-
await this.getExtractor();
|
|
19007
|
-
}
|
|
19008
|
-
};
|
|
19009
|
-
|
|
19010
19847
|
// ../../packages/node-runtime/src/semantic-index/embedding/profiles.ts
|
|
19011
19848
|
var QWEN3_PROFILE = {
|
|
19012
19849
|
displayName: "Qwen3 Embedding 0.6B",
|
|
@@ -19070,39 +19907,7 @@ function createEmbedder(config, deps = {}) {
|
|
|
19070
19907
|
|
|
19071
19908
|
// ../../packages/node-runtime/src/semantic-index/store.ts
|
|
19072
19909
|
import Database2 from "better-sqlite3";
|
|
19073
|
-
|
|
19074
|
-
// ../../node_modules/sqlite-vec/index.mjs
|
|
19075
|
-
import { fileURLToPath } from "url";
|
|
19076
|
-
import { arch, platform } from "process";
|
|
19077
|
-
var BASE_PACKAGE_NAME = "sqlite-vec";
|
|
19078
|
-
var ENTRYPOINT_BASE_NAME = "vec0";
|
|
19079
|
-
var supportedPlatforms = [["darwin", "x64"], ["linux", "x64"], ["darwin", "arm64"], ["win32", "x64"], ["linux", "arm64"]];
|
|
19080
|
-
var invalidPlatformErrorMessage = `Unsupported platform for ${BASE_PACKAGE_NAME}, on a ${platform}-${arch} machine. Supported platforms are (${supportedPlatforms.map(([p, a]) => `${p}-${a}`).join(",")}). Consult the ${BASE_PACKAGE_NAME} NPM package README for details.`;
|
|
19081
|
-
function validPlatform(platform2, arch2) {
|
|
19082
|
-
return supportedPlatforms.find(([p, a]) => platform2 === p && arch2 === a) !== void 0;
|
|
19083
|
-
}
|
|
19084
|
-
function extensionSuffix(platform2) {
|
|
19085
|
-
if (platform2 === "win32") return "dll";
|
|
19086
|
-
if (platform2 === "darwin") return "dylib";
|
|
19087
|
-
return "so";
|
|
19088
|
-
}
|
|
19089
|
-
function platformPackageName(platform2, arch2) {
|
|
19090
|
-
const os3 = platform2 === "win32" ? "windows" : platform2;
|
|
19091
|
-
return `${BASE_PACKAGE_NAME}-${os3}-${arch2}`;
|
|
19092
|
-
}
|
|
19093
|
-
function getLoadablePath() {
|
|
19094
|
-
if (!validPlatform(platform, arch)) {
|
|
19095
|
-
throw new Error(
|
|
19096
|
-
invalidPlatformErrorMessage
|
|
19097
|
-
);
|
|
19098
|
-
}
|
|
19099
|
-
const packageName = platformPackageName(platform, arch);
|
|
19100
|
-
const loadablePath = fileURLToPath(import.meta.resolve(packageName + "/" + ENTRYPOINT_BASE_NAME + "." + extensionSuffix(platform)));
|
|
19101
|
-
return loadablePath;
|
|
19102
|
-
}
|
|
19103
|
-
function load(db) {
|
|
19104
|
-
db.loadExtension(getLoadablePath());
|
|
19105
|
-
}
|
|
19910
|
+
import * as sqliteVec from "sqlite-vec";
|
|
19106
19911
|
|
|
19107
19912
|
// ../../packages/node-runtime/src/semantic-index/schema.ts
|
|
19108
19913
|
var CHUNK_VECTOR_INDEX_TABLE = `
|
|
@@ -19168,7 +19973,7 @@ function vecTableSchema(dim) {
|
|
|
19168
19973
|
}
|
|
19169
19974
|
|
|
19170
19975
|
// ../../packages/node-runtime/src/semantic-index/store.ts
|
|
19171
|
-
var defaultLoadSqliteVec = (db) => load(db);
|
|
19976
|
+
var defaultLoadSqliteVec = (db) => sqliteVec.load(db);
|
|
19172
19977
|
function rowToRecord(row) {
|
|
19173
19978
|
return {
|
|
19174
19979
|
chunkId: row.chunk_id,
|
|
@@ -19614,7 +20419,7 @@ var SemanticIndexJobQueue = class {
|
|
|
19614
20419
|
};
|
|
19615
20420
|
|
|
19616
20421
|
// ../../packages/node-runtime/src/semantic-index/chunker.ts
|
|
19617
|
-
import { createHash as
|
|
20422
|
+
import { createHash as createHash4 } from "crypto";
|
|
19618
20423
|
var VOID_MESSAGE_TYPES = /* @__PURE__ */ new Set([
|
|
19619
20424
|
1 /* IMAGE */,
|
|
19620
20425
|
2 /* VOICE */,
|
|
@@ -19697,7 +20502,7 @@ function sumEffectiveChars(messages) {
|
|
|
19697
20502
|
return total;
|
|
19698
20503
|
}
|
|
19699
20504
|
function sha256Hex2(input) {
|
|
19700
|
-
return
|
|
20505
|
+
return createHash4("sha256").update(input, "utf-8").digest("hex");
|
|
19701
20506
|
}
|
|
19702
20507
|
var EMBEDDING_HEADER_BUDGET_RATIO = 0.25;
|
|
19703
20508
|
var SOURCE_TITLE_HEADER_SHARE = 1 / 3;
|
|
@@ -20273,7 +21078,8 @@ function createSemanticIndexService(params) {
|
|
|
20273
21078
|
sessionAdapter: params.sessionAdapter,
|
|
20274
21079
|
nativeBinding: params.nativeBinding,
|
|
20275
21080
|
loadSqliteVec: params.loadSqliteVec,
|
|
20276
|
-
embedderFactoryDeps: params.embedderFactoryDeps
|
|
21081
|
+
embedderFactoryDeps: params.embedderFactoryDeps,
|
|
21082
|
+
getLocalEmbeddingRuntimeStatus: params.getLocalEmbeddingRuntimeStatus
|
|
20277
21083
|
});
|
|
20278
21084
|
}
|
|
20279
21085
|
var SemanticIndexService = class {
|
|
@@ -20328,7 +21134,8 @@ var SemanticIndexService = class {
|
|
|
20328
21134
|
* 传入 apiKey 时写入固定 auth profile,config 内只保存引用,绝不持久化明文。
|
|
20329
21135
|
*/
|
|
20330
21136
|
getModelStatus() {
|
|
20331
|
-
return this.modelPreloadStatus;
|
|
21137
|
+
if (this.modelPreloadStatus !== "downloading") return this.modelPreloadStatus;
|
|
21138
|
+
return this.options.getLocalEmbeddingRuntimeStatus?.() === "installing" ? "installing-runtime" : "downloading-model";
|
|
20332
21139
|
}
|
|
20333
21140
|
setConfig(config, options2) {
|
|
20334
21141
|
const saved = persistSemanticIndexConfig(this.configStore, config, {
|
|
@@ -20342,11 +21149,17 @@ var SemanticIndexService = class {
|
|
|
20342
21149
|
if (!saved.enabled) {
|
|
20343
21150
|
for (const state of this.stateStore.listEnabled()) this.queue.pause(state.dbPathHash);
|
|
20344
21151
|
}
|
|
20345
|
-
|
|
20346
|
-
|
|
20347
|
-
|
|
20348
|
-
|
|
20349
|
-
|
|
21152
|
+
this.startLocalModelPreload(saved, preloadGeneration);
|
|
21153
|
+
return saved;
|
|
21154
|
+
}
|
|
21155
|
+
startLocalModelPreload(config, preloadGeneration, options2 = {}) {
|
|
21156
|
+
if (!config.enabled || !isSemanticIndexConfigured(config) || config.mode !== "local") return;
|
|
21157
|
+
const modelId = config.local.modelId;
|
|
21158
|
+
const downloadSource = config.local.downloadSource ?? DEFAULT_SEMANTIC_INDEX_MODEL_DOWNLOAD_SOURCE;
|
|
21159
|
+
const cacheDir = this.options.modelsCacheDir;
|
|
21160
|
+
this.modelPreloadStatus = "downloading";
|
|
21161
|
+
const run = () => {
|
|
21162
|
+
if (preloadGeneration !== this.modelPreloadGeneration) return;
|
|
20350
21163
|
appLogger.info("semantic-index", "local embedding model preload started", { modelId, downloadSource, cacheDir });
|
|
20351
21164
|
void this.getEmbedder().preload?.().then(() => {
|
|
20352
21165
|
if (preloadGeneration !== this.modelPreloadGeneration) return;
|
|
@@ -20365,8 +21178,9 @@ var SemanticIndexService = class {
|
|
|
20365
21178
|
error
|
|
20366
21179
|
);
|
|
20367
21180
|
});
|
|
20368
|
-
}
|
|
20369
|
-
|
|
21181
|
+
};
|
|
21182
|
+
if (options2.defer) queueMicrotask(run);
|
|
21183
|
+
else run();
|
|
20370
21184
|
}
|
|
20371
21185
|
currentModelId() {
|
|
20372
21186
|
return this.configStore.resolveModelId();
|
|
@@ -20742,6 +21556,8 @@ ${text}`;
|
|
|
20742
21556
|
this.stateStore.setIndexStatus(state.dbPathHash, "paused");
|
|
20743
21557
|
}
|
|
20744
21558
|
}
|
|
21559
|
+
const config = this.configStore.get();
|
|
21560
|
+
this.startLocalModelPreload(config, ++this.modelPreloadGeneration, { defer: true });
|
|
20745
21561
|
}
|
|
20746
21562
|
// ---------- 队列执行 ----------
|
|
20747
21563
|
resolveSource(db) {
|
|
@@ -20817,9 +21633,269 @@ ${text}`;
|
|
|
20817
21633
|
}
|
|
20818
21634
|
};
|
|
20819
21635
|
|
|
21636
|
+
// ../../packages/node-runtime/src/semantic-index/embedding/local-runtime.ts
|
|
21637
|
+
import { execFile } from "child_process";
|
|
21638
|
+
import fs34 from "fs";
|
|
21639
|
+
import { createRequire as createRequire2 } from "module";
|
|
21640
|
+
import path29 from "path";
|
|
21641
|
+
import { pathToFileURL } from "url";
|
|
21642
|
+
var LOCAL_EMBEDDING_RUNTIME_PACKAGE = "@huggingface/transformers";
|
|
21643
|
+
var LOCAL_EMBEDDING_RUNTIME_VERSION = "4.2.0";
|
|
21644
|
+
var RUNTIME_MANIFEST_FILE = "runtime.json";
|
|
21645
|
+
var VERIFY_RUNTIME_SCRIPT = `
|
|
21646
|
+
import { createRequire } from 'node:module'
|
|
21647
|
+
import path from 'node:path'
|
|
21648
|
+
import { pathToFileURL } from 'node:url'
|
|
21649
|
+
|
|
21650
|
+
const [, runtimeDir, transformersPackage] = process.argv
|
|
21651
|
+
const runtimeRequire = createRequire(path.join(runtimeDir, 'package.json'))
|
|
21652
|
+
const importFromRuntime = async (packageName) => {
|
|
21653
|
+
const resolved = runtimeRequire.resolve(packageName)
|
|
21654
|
+
return await import(pathToFileURL(resolved).href)
|
|
21655
|
+
}
|
|
21656
|
+
|
|
21657
|
+
const transformers = await importFromRuntime(transformersPackage)
|
|
21658
|
+
if (typeof transformers.pipeline !== 'function' || typeof transformers.env !== 'object') {
|
|
21659
|
+
throw new Error('Installed Transformers runtime does not expose the expected Node API.')
|
|
21660
|
+
}
|
|
21661
|
+
|
|
21662
|
+
const onnxRuntime = await importFromRuntime('onnxruntime-node')
|
|
21663
|
+
if (typeof onnxRuntime.listSupportedBackends !== 'function') {
|
|
21664
|
+
throw new Error('Installed ONNX runtime does not expose listSupportedBackends().')
|
|
21665
|
+
}
|
|
21666
|
+
`;
|
|
21667
|
+
function executeInstall(input) {
|
|
21668
|
+
return new Promise((resolve4, reject) => {
|
|
21669
|
+
execFile(
|
|
21670
|
+
input.command,
|
|
21671
|
+
input.args,
|
|
21672
|
+
{
|
|
21673
|
+
cwd: input.cwd,
|
|
21674
|
+
env: { ...process.env, npm_config_update_notifier: "false" },
|
|
21675
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
21676
|
+
shell: process.platform === "win32"
|
|
21677
|
+
},
|
|
21678
|
+
(error, _stdout, stderr) => {
|
|
21679
|
+
if (!error) {
|
|
21680
|
+
resolve4();
|
|
21681
|
+
return;
|
|
21682
|
+
}
|
|
21683
|
+
reject(new Error(stderr.trim() || error.message, { cause: error }));
|
|
21684
|
+
}
|
|
21685
|
+
);
|
|
21686
|
+
});
|
|
21687
|
+
}
|
|
21688
|
+
function verifyRuntimeInSubprocess(runtimeDir) {
|
|
21689
|
+
return new Promise((resolve4, reject) => {
|
|
21690
|
+
execFile(
|
|
21691
|
+
process.execPath,
|
|
21692
|
+
["--input-type=module", "--eval", VERIFY_RUNTIME_SCRIPT, runtimeDir, LOCAL_EMBEDDING_RUNTIME_PACKAGE],
|
|
21693
|
+
{
|
|
21694
|
+
env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" },
|
|
21695
|
+
maxBuffer: 10 * 1024 * 1024
|
|
21696
|
+
},
|
|
21697
|
+
(error, _stdout, stderr) => {
|
|
21698
|
+
if (!error) {
|
|
21699
|
+
resolve4();
|
|
21700
|
+
return;
|
|
21701
|
+
}
|
|
21702
|
+
reject(new Error(stderr.trim() || error.message, { cause: error }));
|
|
21703
|
+
}
|
|
21704
|
+
);
|
|
21705
|
+
});
|
|
21706
|
+
}
|
|
21707
|
+
function createRuntimeRequire(runtimeDir) {
|
|
21708
|
+
return createRequire2(path29.join(runtimeDir, "package.json"));
|
|
21709
|
+
}
|
|
21710
|
+
async function importFromRuntime(runtimeDir, packageName) {
|
|
21711
|
+
const resolved = createRuntimeRequire(runtimeDir).resolve(packageName);
|
|
21712
|
+
return await import(pathToFileURL(resolved).href);
|
|
21713
|
+
}
|
|
21714
|
+
async function verifyInstalledRuntime(runtimeDir) {
|
|
21715
|
+
const transformers = await importFromRuntime(runtimeDir, LOCAL_EMBEDDING_RUNTIME_PACKAGE);
|
|
21716
|
+
if (typeof transformers.pipeline !== "function" || typeof transformers.env !== "object") {
|
|
21717
|
+
throw new Error("Installed Transformers runtime does not expose the expected Node API.");
|
|
21718
|
+
}
|
|
21719
|
+
const onnxRuntime = await importFromRuntime(runtimeDir, "onnxruntime-node");
|
|
21720
|
+
if (typeof onnxRuntime.listSupportedBackends !== "function") {
|
|
21721
|
+
throw new Error("Installed ONNX runtime does not expose listSupportedBackends().");
|
|
21722
|
+
}
|
|
21723
|
+
}
|
|
21724
|
+
async function loadInstalledTransformers(runtimeDir) {
|
|
21725
|
+
return await importFromRuntime(
|
|
21726
|
+
runtimeDir,
|
|
21727
|
+
LOCAL_EMBEDDING_RUNTIME_PACKAGE
|
|
21728
|
+
);
|
|
21729
|
+
}
|
|
21730
|
+
var LocalEmbeddingRuntimeManager = class {
|
|
21731
|
+
baseDir;
|
|
21732
|
+
installMode;
|
|
21733
|
+
platform;
|
|
21734
|
+
arch;
|
|
21735
|
+
npmCommand;
|
|
21736
|
+
runInstall;
|
|
21737
|
+
verifyStagedRuntime;
|
|
21738
|
+
verifyRuntime;
|
|
21739
|
+
importTransformers;
|
|
21740
|
+
status;
|
|
21741
|
+
installPromise = null;
|
|
21742
|
+
transformersPromise = null;
|
|
21743
|
+
verifiedRuntimeDir = null;
|
|
21744
|
+
constructor(options2) {
|
|
21745
|
+
this.baseDir = options2.baseDir;
|
|
21746
|
+
this.installMode = options2.installMode;
|
|
21747
|
+
this.platform = options2.platform ?? process.platform;
|
|
21748
|
+
this.arch = options2.arch ?? process.arch;
|
|
21749
|
+
this.npmCommand = options2.npmCommand ?? (this.platform === "win32" ? "npm.cmd" : "npm");
|
|
21750
|
+
this.runInstall = options2.runInstall ?? executeInstall;
|
|
21751
|
+
this.verifyStagedRuntime = options2.verifyStagedRuntime ?? verifyRuntimeInSubprocess;
|
|
21752
|
+
this.verifyRuntime = options2.verifyRuntime ?? verifyInstalledRuntime;
|
|
21753
|
+
this.importTransformers = options2.importTransformers ?? loadInstalledTransformers;
|
|
21754
|
+
this.status = "idle";
|
|
21755
|
+
}
|
|
21756
|
+
getStatus() {
|
|
21757
|
+
return this.status;
|
|
21758
|
+
}
|
|
21759
|
+
getRuntimeDir() {
|
|
21760
|
+
return path29.join(this.baseDir, `transformers-${LOCAL_EMBEDDING_RUNTIME_VERSION}`, `${this.platform}-${this.arch}`);
|
|
21761
|
+
}
|
|
21762
|
+
async ensureInstalled() {
|
|
21763
|
+
if (!this.installPromise) {
|
|
21764
|
+
this.installPromise = this.ensureInstalledOnce().finally(() => {
|
|
21765
|
+
this.installPromise = null;
|
|
21766
|
+
});
|
|
21767
|
+
}
|
|
21768
|
+
return await this.installPromise;
|
|
21769
|
+
}
|
|
21770
|
+
async loadTransformers() {
|
|
21771
|
+
if (!this.transformersPromise) {
|
|
21772
|
+
this.transformersPromise = this.ensureInstalled().then((runtimeDir) => this.importTransformers(runtimeDir)).catch((error) => {
|
|
21773
|
+
this.transformersPromise = null;
|
|
21774
|
+
throw error;
|
|
21775
|
+
});
|
|
21776
|
+
}
|
|
21777
|
+
return await this.transformersPromise;
|
|
21778
|
+
}
|
|
21779
|
+
async ensureInstalledOnce() {
|
|
21780
|
+
const runtimeDir = this.getRuntimeDir();
|
|
21781
|
+
try {
|
|
21782
|
+
if (this.hasCompleteLayout(runtimeDir)) {
|
|
21783
|
+
try {
|
|
21784
|
+
await this.verifyOnce(runtimeDir);
|
|
21785
|
+
this.status = "ready";
|
|
21786
|
+
return runtimeDir;
|
|
21787
|
+
} catch (error) {
|
|
21788
|
+
if (this.installMode === "preinstalled") throw error;
|
|
21789
|
+
appLogger.warn("semantic-index", "Existing local embedding runtime is invalid; reinstalling", {
|
|
21790
|
+
packageVersion: LOCAL_EMBEDDING_RUNTIME_VERSION,
|
|
21791
|
+
platform: this.platform,
|
|
21792
|
+
arch: this.arch,
|
|
21793
|
+
error: error instanceof Error ? error.message : String(error)
|
|
21794
|
+
});
|
|
21795
|
+
fs34.rmSync(runtimeDir, { recursive: true, force: true });
|
|
21796
|
+
this.verifiedRuntimeDir = null;
|
|
21797
|
+
}
|
|
21798
|
+
}
|
|
21799
|
+
if (this.installMode === "preinstalled") {
|
|
21800
|
+
throw new Error(`Preinstalled local embedding runtime is missing or incomplete: ${runtimeDir}`);
|
|
21801
|
+
}
|
|
21802
|
+
this.status = "installing";
|
|
21803
|
+
appLogger.info("semantic-index", "Local embedding runtime installation started", {
|
|
21804
|
+
packageVersion: LOCAL_EMBEDDING_RUNTIME_VERSION,
|
|
21805
|
+
platform: this.platform,
|
|
21806
|
+
arch: this.arch
|
|
21807
|
+
});
|
|
21808
|
+
await this.installAtomically(runtimeDir);
|
|
21809
|
+
await this.verifyOnce(runtimeDir);
|
|
21810
|
+
this.status = "ready";
|
|
21811
|
+
appLogger.info("semantic-index", "Local embedding runtime installation completed", {
|
|
21812
|
+
packageVersion: LOCAL_EMBEDDING_RUNTIME_VERSION,
|
|
21813
|
+
platform: this.platform,
|
|
21814
|
+
arch: this.arch
|
|
21815
|
+
});
|
|
21816
|
+
return runtimeDir;
|
|
21817
|
+
} catch (error) {
|
|
21818
|
+
this.status = "error";
|
|
21819
|
+
appLogger.error("semantic-index", "Local embedding runtime installation failed", error);
|
|
21820
|
+
throw error;
|
|
21821
|
+
}
|
|
21822
|
+
}
|
|
21823
|
+
async installAtomically(runtimeDir) {
|
|
21824
|
+
const parentDir = path29.dirname(runtimeDir);
|
|
21825
|
+
fs34.mkdirSync(parentDir, { recursive: true });
|
|
21826
|
+
const stagingDir = fs34.mkdtempSync(path29.join(parentDir, ".installing-"));
|
|
21827
|
+
try {
|
|
21828
|
+
fs34.writeFileSync(
|
|
21829
|
+
path29.join(stagingDir, "package.json"),
|
|
21830
|
+
`${JSON.stringify({ private: true, type: "module" }, null, 2)}
|
|
21831
|
+
`,
|
|
21832
|
+
"utf8"
|
|
21833
|
+
);
|
|
21834
|
+
await this.runInstall({
|
|
21835
|
+
command: this.npmCommand,
|
|
21836
|
+
args: [
|
|
21837
|
+
"install",
|
|
21838
|
+
"--omit=dev",
|
|
21839
|
+
"--no-audit",
|
|
21840
|
+
"--no-fund",
|
|
21841
|
+
"--save-exact",
|
|
21842
|
+
"--loglevel=error",
|
|
21843
|
+
"--prefix",
|
|
21844
|
+
stagingDir,
|
|
21845
|
+
`${LOCAL_EMBEDDING_RUNTIME_PACKAGE}@${LOCAL_EMBEDDING_RUNTIME_VERSION}`
|
|
21846
|
+
],
|
|
21847
|
+
cwd: stagingDir
|
|
21848
|
+
});
|
|
21849
|
+
await this.verifyStagedRuntime(stagingDir);
|
|
21850
|
+
fs34.writeFileSync(path29.join(stagingDir, RUNTIME_MANIFEST_FILE), `${JSON.stringify(this.manifest(), null, 2)}
|
|
21851
|
+
`);
|
|
21852
|
+
if (fs34.existsSync(runtimeDir)) {
|
|
21853
|
+
if (this.hasCompleteLayout(runtimeDir)) return;
|
|
21854
|
+
fs34.rmSync(runtimeDir, { recursive: true, force: true });
|
|
21855
|
+
}
|
|
21856
|
+
try {
|
|
21857
|
+
fs34.renameSync(stagingDir, runtimeDir);
|
|
21858
|
+
} catch (error) {
|
|
21859
|
+
if (!this.hasCompleteLayout(runtimeDir)) throw error;
|
|
21860
|
+
}
|
|
21861
|
+
} finally {
|
|
21862
|
+
fs34.rmSync(stagingDir, { recursive: true, force: true });
|
|
21863
|
+
}
|
|
21864
|
+
}
|
|
21865
|
+
async verifyOnce(runtimeDir) {
|
|
21866
|
+
if (this.verifiedRuntimeDir === runtimeDir) return;
|
|
21867
|
+
await this.verifyRuntime(runtimeDir);
|
|
21868
|
+
this.verifiedRuntimeDir = runtimeDir;
|
|
21869
|
+
}
|
|
21870
|
+
hasCompleteLayout(runtimeDir) {
|
|
21871
|
+
const manifestPath = path29.join(runtimeDir, RUNTIME_MANIFEST_FILE);
|
|
21872
|
+
const packagePath = path29.join(runtimeDir, "node_modules", "@huggingface", "transformers", "package.json");
|
|
21873
|
+
if (!fs34.existsSync(manifestPath) || !fs34.existsSync(packagePath)) return false;
|
|
21874
|
+
try {
|
|
21875
|
+
const manifest = JSON.parse(fs34.readFileSync(manifestPath, "utf8"));
|
|
21876
|
+
const expected = this.manifest();
|
|
21877
|
+
return manifest.formatVersion === expected.formatVersion && manifest.packageName === expected.packageName && manifest.packageVersion === expected.packageVersion && manifest.platform === expected.platform && manifest.arch === expected.arch;
|
|
21878
|
+
} catch {
|
|
21879
|
+
return false;
|
|
21880
|
+
}
|
|
21881
|
+
}
|
|
21882
|
+
manifest() {
|
|
21883
|
+
return {
|
|
21884
|
+
formatVersion: 1,
|
|
21885
|
+
packageName: LOCAL_EMBEDDING_RUNTIME_PACKAGE,
|
|
21886
|
+
packageVersion: LOCAL_EMBEDDING_RUNTIME_VERSION,
|
|
21887
|
+
platform: this.platform,
|
|
21888
|
+
arch: this.arch
|
|
21889
|
+
};
|
|
21890
|
+
}
|
|
21891
|
+
};
|
|
21892
|
+
function createLocalEmbeddingRuntimeManager(options2) {
|
|
21893
|
+
return new LocalEmbeddingRuntimeManager(options2);
|
|
21894
|
+
}
|
|
21895
|
+
|
|
20820
21896
|
// ../../packages/node-runtime/src/services/merge-cache.ts
|
|
20821
|
-
import * as
|
|
20822
|
-
import * as
|
|
21897
|
+
import * as fs35 from "fs";
|
|
21898
|
+
import * as path30 from "path";
|
|
20823
21899
|
import * as crypto from "crypto";
|
|
20824
21900
|
var MergeSessionCache = class {
|
|
20825
21901
|
cache = /* @__PURE__ */ new Map();
|
|
@@ -20828,15 +21904,15 @@ var MergeSessionCache = class {
|
|
|
20828
21904
|
constructor(pathProvider, options2) {
|
|
20829
21905
|
this.tempDir = pathProvider.getTempDir();
|
|
20830
21906
|
this.nativeBinding = options2?.nativeBinding;
|
|
20831
|
-
if (!
|
|
20832
|
-
|
|
21907
|
+
if (!fs35.existsSync(this.tempDir)) {
|
|
21908
|
+
fs35.mkdirSync(this.tempDir, { recursive: true });
|
|
20833
21909
|
}
|
|
20834
21910
|
}
|
|
20835
21911
|
generateTempDbPath(filename) {
|
|
20836
21912
|
const ts = Date.now();
|
|
20837
21913
|
const rand = Math.random().toString(36).slice(2, 8);
|
|
20838
21914
|
const safeName = filename.replace(/[/\\?%*:|"<>]/g, "_").slice(0, 40);
|
|
20839
|
-
return
|
|
21915
|
+
return path30.join(this.tempDir, `merge_${safeName}_${ts}_${rand}.db`);
|
|
20840
21916
|
}
|
|
20841
21917
|
/** Store a parsed file's temp DB and return a handle for subsequent requests. */
|
|
20842
21918
|
store(filename, tempDbPath) {
|
|
@@ -20847,7 +21923,7 @@ var MergeSessionCache = class {
|
|
|
20847
21923
|
/** Open a TempDbReader for a given handle. Caller must close the reader. */
|
|
20848
21924
|
openReader(handle) {
|
|
20849
21925
|
const entry = this.cache.get(handle);
|
|
20850
|
-
if (!entry || !
|
|
21926
|
+
if (!entry || !fs35.existsSync(entry.tempDbPath)) return null;
|
|
20851
21927
|
const adapter = openBetterSqliteDatabase(entry.tempDbPath, {
|
|
20852
21928
|
readonly: true,
|
|
20853
21929
|
nativeBinding: this.nativeBinding
|
|
@@ -20892,21 +21968,21 @@ var MergeSessionCache = class {
|
|
|
20892
21968
|
};
|
|
20893
21969
|
|
|
20894
21970
|
// ../../packages/node-runtime/src/services/push-importer.ts
|
|
20895
|
-
import * as
|
|
20896
|
-
var
|
|
20897
|
-
var
|
|
20898
|
-
function
|
|
21971
|
+
import * as fs36 from "fs";
|
|
21972
|
+
var SYSTEM_SENDER_ID3 = "SYSTEM";
|
|
21973
|
+
var SYSTEM_MEMBER_NAME2 = "\u7CFB\u7EDF\u6D88\u606F";
|
|
21974
|
+
function isRecord5(value) {
|
|
20899
21975
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
20900
21976
|
}
|
|
20901
21977
|
function validateOptionalString2(value, field) {
|
|
20902
21978
|
return value !== void 0 && typeof value !== "string" ? `${field} must be a string` : null;
|
|
20903
21979
|
}
|
|
20904
21980
|
function validatePayload(payload, isNew) {
|
|
20905
|
-
if (!
|
|
21981
|
+
if (!isRecord5(payload)) return "payload must be an object";
|
|
20906
21982
|
const messages = payload.messages;
|
|
20907
21983
|
if (!Array.isArray(messages) || messages.length === 0) return "messages is required and must be a non-empty array";
|
|
20908
21984
|
if (payload.chatlab !== void 0) {
|
|
20909
|
-
if (!
|
|
21985
|
+
if (!isRecord5(payload.chatlab)) return "chatlab must be an object";
|
|
20910
21986
|
if (typeof payload.chatlab.version !== "string" || payload.chatlab.version.length === 0)
|
|
20911
21987
|
return "chatlab.version must be a string";
|
|
20912
21988
|
if (typeof payload.chatlab.exportedAt !== "number" || !Number.isFinite(payload.chatlab.exportedAt))
|
|
@@ -20919,7 +21995,7 @@ function validatePayload(payload, isNew) {
|
|
|
20919
21995
|
if (!payload.meta) return "meta is required for new sessions";
|
|
20920
21996
|
}
|
|
20921
21997
|
if (payload.meta !== void 0) {
|
|
20922
|
-
if (!
|
|
21998
|
+
if (!isRecord5(payload.meta)) return "meta must be an object";
|
|
20923
21999
|
const metaFields = ["name", "platform", "type", "groupId", "groupAvatar", "ownerId"];
|
|
20924
22000
|
for (const field of metaFields) {
|
|
20925
22001
|
const error = validateOptionalString2(payload.meta[field], `meta.${field}`);
|
|
@@ -20929,7 +22005,7 @@ function validatePayload(payload, isNew) {
|
|
|
20929
22005
|
if (isNew && !payload.meta.platform) return "meta.platform is required";
|
|
20930
22006
|
if (isNew && !payload.meta.type) return "meta.type is required";
|
|
20931
22007
|
}
|
|
20932
|
-
if (payload.options !== void 0 && !
|
|
22008
|
+
if (payload.options !== void 0 && !isRecord5(payload.options)) return "options must be an object";
|
|
20933
22009
|
const { metaUpdateMode, memberUpdateMode } = payload.options ?? {};
|
|
20934
22010
|
if (metaUpdateMode !== void 0 && metaUpdateMode !== "patch" && metaUpdateMode !== "none") {
|
|
20935
22011
|
return `options.metaUpdateMode must be 'patch' or 'none'`;
|
|
@@ -20941,7 +22017,7 @@ function validatePayload(payload, isNew) {
|
|
|
20941
22017
|
if (payload.members) {
|
|
20942
22018
|
for (let i = 0; i < payload.members.length; i++) {
|
|
20943
22019
|
const member = payload.members[i];
|
|
20944
|
-
if (!
|
|
22020
|
+
if (!isRecord5(member)) return `members[${i}] must be an object`;
|
|
20945
22021
|
if (typeof member.platformId !== "string" || member.platformId.length === 0)
|
|
20946
22022
|
return `members[${i}].platformId must be a string`;
|
|
20947
22023
|
for (const field of ["accountName", "groupNickname", "avatar"]) {
|
|
@@ -20952,7 +22028,7 @@ function validatePayload(payload, isNew) {
|
|
|
20952
22028
|
if (!Array.isArray(member.roles)) return `members[${i}].roles must be an array`;
|
|
20953
22029
|
for (let j = 0; j < member.roles.length; j++) {
|
|
20954
22030
|
const role = member.roles[j];
|
|
20955
|
-
if (!
|
|
22031
|
+
if (!isRecord5(role) || typeof role.id !== "string" || role.id.length === 0)
|
|
20956
22032
|
return `members[${i}].roles[${j}].id must be a string`;
|
|
20957
22033
|
}
|
|
20958
22034
|
}
|
|
@@ -20960,7 +22036,7 @@ function validatePayload(payload, isNew) {
|
|
|
20960
22036
|
}
|
|
20961
22037
|
for (let i = 0; i < messages.length; i++) {
|
|
20962
22038
|
const msg = messages[i];
|
|
20963
|
-
if (!
|
|
22039
|
+
if (!isRecord5(msg)) return `messages[${i}] must be an object`;
|
|
20964
22040
|
if (typeof msg.sender !== "string" || msg.sender.length === 0) return `messages[${i}].sender must be a string`;
|
|
20965
22041
|
if (typeof msg.timestamp !== "number" || !Number.isFinite(msg.timestamp) || msg.timestamp <= 0)
|
|
20966
22042
|
return `messages[${i}].timestamp must be a positive number`;
|
|
@@ -20980,17 +22056,17 @@ function validatePayload(payload, isNew) {
|
|
|
20980
22056
|
}
|
|
20981
22057
|
function queryStats(db) {
|
|
20982
22058
|
const row = db.prepare("SELECT COUNT(*) as total, MIN(ts) as first, MAX(ts) as last FROM message").get();
|
|
20983
|
-
const memberRow = db.prepare(`SELECT COUNT(*) as cnt FROM member WHERE COALESCE(account_name, '') != ?`).get(
|
|
22059
|
+
const memberRow = db.prepare(`SELECT COUNT(*) as cnt FROM member WHERE COALESCE(account_name, '') != ?`).get(SYSTEM_MEMBER_NAME2);
|
|
20984
22060
|
return { totalCount: row.total, memberCount: memberRow.cnt, firstTimestamp: row.first, lastTimestamp: row.last };
|
|
20985
22061
|
}
|
|
20986
22062
|
function normalizeSenderAccountName(sender, accountName) {
|
|
20987
|
-
return sender ===
|
|
22063
|
+
return sender === SYSTEM_SENDER_ID3 ? SYSTEM_MEMBER_NAME2 : accountName;
|
|
20988
22064
|
}
|
|
20989
22065
|
function normalizeSenderGroupNickname(sender, groupNickname) {
|
|
20990
|
-
return sender ===
|
|
22066
|
+
return sender === SYSTEM_SENDER_ID3 ? SYSTEM_MEMBER_NAME2 : groupNickname;
|
|
20991
22067
|
}
|
|
20992
22068
|
function isCountedMember(platformId) {
|
|
20993
|
-
return platformId !==
|
|
22069
|
+
return platformId !== SYSTEM_SENDER_ID3;
|
|
20994
22070
|
}
|
|
20995
22071
|
function registerPushMessageAndCheckDuplicate(message, dedupState) {
|
|
20996
22072
|
return registerMessageAndCheckDuplicate(
|
|
@@ -21198,10 +22274,11 @@ function writeIncrementalImport(db, payload) {
|
|
|
21198
22274
|
}
|
|
21199
22275
|
if (writtenCount > 0) {
|
|
21200
22276
|
try {
|
|
22277
|
+
const sessionGapThreshold = getSessionIndexStats(db).gapThreshold;
|
|
21201
22278
|
if (minWrittenTs < preWriteMaxTs) {
|
|
21202
|
-
generateSessionIndex(db);
|
|
22279
|
+
generateSessionIndex(db, sessionGapThreshold);
|
|
21203
22280
|
} else {
|
|
21204
|
-
generateIncrementalSessionIndex(db);
|
|
22281
|
+
generateIncrementalSessionIndex(db, sessionGapThreshold);
|
|
21205
22282
|
}
|
|
21206
22283
|
} catch {
|
|
21207
22284
|
}
|
|
@@ -21259,7 +22336,7 @@ async function executePushImportUnlocked(deps, sessionId, payload) {
|
|
|
21259
22336
|
return { ok: false, reason: "invalid_payload", message: "sessionId contains invalid characters" };
|
|
21260
22337
|
}
|
|
21261
22338
|
const dbPath = deps.getDbPath(sessionId);
|
|
21262
|
-
const isNew = !
|
|
22339
|
+
const isNew = !fs36.existsSync(dbPath);
|
|
21263
22340
|
const validationError = validatePayload(payload, isNew);
|
|
21264
22341
|
if (validationError) {
|
|
21265
22342
|
return { ok: false, reason: "invalid_payload", message: validationError };
|
|
@@ -21320,10 +22397,10 @@ async function executePushImportUnlocked(deps, sessionId, payload) {
|
|
|
21320
22397
|
}
|
|
21321
22398
|
|
|
21322
22399
|
// ../../packages/node-runtime/src/services/import-idempotency.ts
|
|
21323
|
-
import { createHash as
|
|
22400
|
+
import { createHash as createHash5 } from "crypto";
|
|
21324
22401
|
var DEFAULT_IMPORT_IDEMPOTENCY_TTL_MS = 60 * 60 * 1e3;
|
|
21325
22402
|
function hashImportBody(body) {
|
|
21326
|
-
return
|
|
22403
|
+
return createHash5("sha256").update(JSON.stringify(body ?? "")).digest("hex");
|
|
21327
22404
|
}
|
|
21328
22405
|
var ImportIdempotencyCache = class {
|
|
21329
22406
|
constructor(ttlMs = DEFAULT_IMPORT_IDEMPOTENCY_TTL_MS) {
|
|
@@ -21366,6 +22443,7 @@ export {
|
|
|
21366
22443
|
createChatLabTempDir,
|
|
21367
22444
|
removeChatLabTempDir,
|
|
21368
22445
|
validateChatLabFile,
|
|
22446
|
+
rebaseChatLabDemoDocuments,
|
|
21369
22447
|
detectFormat,
|
|
21370
22448
|
detectAllFormats,
|
|
21371
22449
|
getSupportedFormats,
|
|
@@ -21374,6 +22452,7 @@ export {
|
|
|
21374
22452
|
findEntryFileInDirectory,
|
|
21375
22453
|
logNativeParserStatus,
|
|
21376
22454
|
LogLevel,
|
|
22455
|
+
createImportPerfLogger,
|
|
21377
22456
|
initPerfLog,
|
|
21378
22457
|
logPerf,
|
|
21379
22458
|
logPerfDetail,
|
|
@@ -21397,6 +22476,8 @@ export {
|
|
|
21397
22476
|
rebuildFtsIndex,
|
|
21398
22477
|
insertFtsEntries,
|
|
21399
22478
|
searchByFts,
|
|
22479
|
+
MESSAGE_INSERT_MAX_ROWS,
|
|
22480
|
+
MessageBatchInserter,
|
|
21400
22481
|
streamingImport,
|
|
21401
22482
|
analyzeNewImport,
|
|
21402
22483
|
streamParseFileInfo,
|
|
@@ -21404,8 +22485,13 @@ export {
|
|
|
21404
22485
|
incrementalImport,
|
|
21405
22486
|
isValidImportSessionId,
|
|
21406
22487
|
resolveAutoImportTarget,
|
|
22488
|
+
resolveAutoImportTargetPlan,
|
|
21407
22489
|
autoImportFile,
|
|
21408
22490
|
analyzeAutoImportFile,
|
|
22491
|
+
normalizeBatchConcurrency,
|
|
22492
|
+
resolveDefaultBatchConcurrency,
|
|
22493
|
+
runKeyedBatch,
|
|
22494
|
+
autoImportBatch,
|
|
21409
22495
|
IMPORT_LOCK_FILENAME,
|
|
21410
22496
|
IMPORT_IN_PROGRESS_ERROR_KEY,
|
|
21411
22497
|
ImportInProgressError,
|
|
@@ -21554,6 +22640,7 @@ export {
|
|
|
21554
22640
|
OpenAICompatibleEmbeddingProvider,
|
|
21555
22641
|
applyQueryInstruction,
|
|
21556
22642
|
clampTextChars,
|
|
22643
|
+
createTransformersPipelineFactory,
|
|
21557
22644
|
LocalEmbeddingProvider,
|
|
21558
22645
|
QWEN3_PROFILE,
|
|
21559
22646
|
BGE_BASE_PROFILE,
|
|
@@ -21579,6 +22666,9 @@ export {
|
|
|
21579
22666
|
SEMANTIC_INDEX_CONFIG_FILE,
|
|
21580
22667
|
createSemanticIndexService,
|
|
21581
22668
|
SemanticIndexService,
|
|
22669
|
+
LOCAL_EMBEDDING_RUNTIME_PACKAGE,
|
|
22670
|
+
LOCAL_EMBEDDING_RUNTIME_VERSION,
|
|
22671
|
+
createLocalEmbeddingRuntimeManager,
|
|
21582
22672
|
MergeSessionCache,
|
|
21583
22673
|
pushImport,
|
|
21584
22674
|
executePushImportUnlocked,
|
|
@@ -21586,4 +22676,4 @@ export {
|
|
|
21586
22676
|
hashImportBody,
|
|
21587
22677
|
ImportIdempotencyCache
|
|
21588
22678
|
};
|
|
21589
|
-
//# sourceMappingURL=chunk-
|
|
22679
|
+
//# sourceMappingURL=chunk-4LW6443F.mjs.map
|