chatlab-cli 0.34.2-beta.2 → 0.34.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chat-command-6IOH2QRF.mjs → chat-command-QH7DXCAJ.mjs} +8 -8
- package/dist/{chunk-JX6MFS6F.mjs → chunk-2Q3HHVTS.mjs} +15 -15
- package/dist/{chunk-INBJ7SND.mjs → chunk-2WYXD5ZP.mjs} +117 -416
- package/dist/chunk-2WYXD5ZP.mjs.map +1 -0
- package/dist/{chunk-U6MVJSXX.mjs → chunk-3X26JJIP.mjs} +15 -9
- package/dist/chunk-3X26JJIP.mjs.map +1 -0
- package/dist/{chunk-UMBEHBIZ.mjs → chunk-6U2QZO3N.mjs} +2 -2
- package/dist/{chunk-52P5ZRBD.mjs → chunk-AWGXYFIC.mjs} +2 -2
- package/dist/{chunk-4LW6443F.mjs → chunk-AWXHYTD5.mjs} +795 -876
- package/dist/chunk-AWXHYTD5.mjs.map +1 -0
- package/dist/{chunk-4RLZFGO5.mjs → chunk-C2PD3BS7.mjs} +2 -2
- package/dist/{chunk-H6D4DZOC.mjs → chunk-JYBPJWE2.mjs} +7 -5
- package/dist/{chunk-H6D4DZOC.mjs.map → chunk-JYBPJWE2.mjs.map} +1 -1
- package/dist/{chunk-652KEAX3.mjs → chunk-SWUTRDGZ.mjs} +2 -2
- package/dist/{chunk-72RWABYJ.mjs → chunk-WJVE5Q4Y.mjs} +3 -3
- package/dist/chunk-WJVE5Q4Y.mjs.map +1 -0
- package/dist/{chunk-2XCNTG4O.mjs → chunk-XIWVJAXH.mjs} +670 -54
- package/dist/chunk-XIWVJAXH.mjs.map +1 -0
- package/dist/{chunk-MENKFEPC.mjs → chunk-ZVGDOSGG.mjs} +29 -41
- package/dist/chunk-ZVGDOSGG.mjs.map +1 -0
- package/dist/cli.mjs +9 -9
- package/dist/contacts-worker.mjs +2 -2
- package/dist/global-insight-worker.mjs +1 -1
- package/dist/{http-HSK46XEU.mjs → http-D2F4QTQW.mjs} +12 -12
- package/dist/{import-DF2WD2HE.mjs → import-H7X2JU2O.mjs} +7 -7
- package/dist/index.mjs +12 -12
- package/dist/{mcp-A2RND7KN.mjs → mcp-UURXRGDF.mjs} +8 -8
- package/dist/people-relationships-worker.mjs +2 -2
- package/dist/semantic-index-worker.mjs +1 -1
- package/dist/{src-SHFHW2QB.mjs → src-MHCO4HYY.mjs} +22 -38
- package/dist/{stream-import-GZ2QDE3Z.mjs → stream-import-BHXACUCA.mjs} +7 -7
- package/dist/{worker-runtime-OWN56GG5.mjs → worker-runtime-JOUCP2L4.mjs} +5 -5
- package/dist-cli-web/assets/ActionToolsPanel.vue_vue_type_script_setup_true_lang-kN3uxTW8.js +110 -0
- package/dist-cli-web/assets/{BufferResource-B-LHxriL.js → BufferResource-Dak23r3B.js} +1 -1
- package/dist-cli-web/assets/{CanvasRenderer-W3h_GGHM.js → CanvasRenderer-Cmjtvski.js} +1 -1
- package/dist-cli-web/assets/DataDirCleanupNotice-B5tyDzK-.js +1 -0
- package/dist-cli-web/assets/{EChartWordcloud-Dt6rCzHf.js → EChartWordcloud-Bd3L3G8I.js} +1 -1
- package/dist-cli-web/assets/{Filter-BpSopVKd.js → Filter-CHKZVYB3.js} +1 -1
- package/dist-cli-web/assets/LoadingState.vue_vue_type_script_setup_true_lang-CKf_gC1k.js +1 -0
- package/dist-cli-web/assets/{MessageExportModal-BoK-_h25.js → MessageExportModal-ODYCcSnk.js} +1 -1
- package/dist-cli-web/assets/{PageHeader-CD8B7Emf.js → PageHeader-Bn7Jn8JW.js} +1 -1
- package/dist-cli-web/assets/{RelationshipGalaxyCanvas-BO3mo-Re.js → RelationshipGalaxyCanvas-mJhEPzsY.js} +3 -3
- package/dist-cli-web/assets/{RelationshipGalaxyThreeCanvas-Brnpz6Ig.js → RelationshipGalaxyThreeCanvas-Bf76yzmQ.js} +1 -1
- package/dist-cli-web/assets/{RenderTargetSystem-CNP4JLfK.js → RenderTargetSystem-1rM7OF-M.js} +1 -1
- package/dist-cli-web/assets/{ThemeCard.vue_vue_type_script_setup_true_lang-Na3LIoSJ.js → ThemeCard.vue_vue_type_script_setup_true_lang-CADZpPia.js} +1 -1
- package/dist-cli-web/assets/TimeSelect.vue_vue_type_script_setup_true_lang-I3FYjhfs.js +1 -0
- package/dist-cli-web/assets/{WebGLRenderer-CIR5cTQN.js → WebGLRenderer-Y0jQtBvK.js} +1 -1
- package/dist-cli-web/assets/{WebGPURenderer-Dz3pCrzX.js → WebGPURenderer-BcUvCPhd.js} +1 -1
- package/dist-cli-web/assets/{annual-summary-time-range-QhoA3uC6.js → annual-summary-time-range-BIADd8qE.js} +1 -1
- package/dist-cli-web/assets/{base-p9yfnTpk.js → base-BpxxC5Z5.js} +1 -1
- package/dist-cli-web/assets/{browserAll-DMaXjf8N.js → browserAll-BsR1UaCC.js} +1 -1
- package/dist-cli-web/assets/{cli-web-Dvy2aHxG.js → cli-web-BiWUOncc.js} +1 -1
- package/dist-cli-web/assets/{dateFormat-5x3JKlIk.js → dateFormat-B6JnNCxp.js} +1 -1
- package/dist-cli-web/assets/{fetch-BtJWW2-O.js → fetch-BAJ_VjWE.js} +1 -1
- package/dist-cli-web/assets/{fetch-B0p6QRQ8.js → fetch-BNcHe2-O.js} +1 -1
- package/dist-cli-web/assets/{fetch-BI-Se2p7.js → fetch-CPloHijF.js} +1 -1
- package/dist-cli-web/assets/{fetch-CBaZNOpK.js → fetch-Clj_RVUD.js} +1 -1
- package/dist-cli-web/assets/{fetch-BderOzzL.js → fetch-D7vgaLpy.js} +1 -1
- package/dist-cli-web/assets/{fetch-DRGykJYb.js → fetch-DGvqctIb.js} +1 -1
- package/dist-cli-web/assets/fetch-DSxgaVBO.js +1 -0
- package/dist-cli-web/assets/{fetch-QLurhC_z.js → fetch-DtnhbqTN.js} +1 -1
- package/dist-cli-web/assets/{fetch-Cne-uL1m.js → fetch-bdY0RMM3.js} +1 -1
- package/dist-cli-web/assets/{fetch-CH9Hm20R.js → fetch-bhHY8rO7.js} +1 -1
- package/dist-cli-web/assets/index-B6Gq11lw.css +1 -0
- package/dist-cli-web/assets/index-BJ2lywiG.js +1 -0
- package/dist-cli-web/assets/{index-CXh01dqh.js → index-BT3_fSP6.js} +2 -2
- package/dist-cli-web/assets/{index-Dp2uglLH.js → index-BVZhP1J5.js} +1 -1
- package/dist-cli-web/assets/index-Br8r4pu8.js +1 -0
- package/dist-cli-web/assets/index-Byok4VaA.css +1 -0
- package/dist-cli-web/assets/index-CKnnxOci.js +1 -0
- package/dist-cli-web/assets/index-CXnYfvTv.js +137 -0
- package/dist-cli-web/assets/index-CsFLLk3T.css +1 -0
- package/dist-cli-web/assets/index-CzoOQ7zg.js +1 -0
- package/dist-cli-web/assets/{index-DF2Dijfg.js → index-D-dTTUGj.js} +1 -1
- package/dist-cli-web/assets/index-DEbvEKzi.js +2 -0
- package/dist-cli-web/assets/{index-IvxuxPNt.js → index-DMYFpctK.js} +1 -1
- package/dist-cli-web/assets/index-KO7uPd8W.js +64 -0
- package/dist-cli-web/assets/index-lkmJ6dI7.js +1 -0
- package/dist-cli-web/assets/{index-CumZ5uQ4.js → index-y7xPDpsW.js} +4 -4
- package/dist-cli-web/assets/{people-page-header-BRb9BrgP.js → people-page-header-gjTyODWs.js} +1 -1
- package/dist-cli-web/assets/{webworkerAll-BoY19vZt.js → webworkerAll-CfZWlukP.js} +1 -1
- package/dist-cli-web/index.html +2 -2
- package/package.json +1 -1
- package/dist/chunk-2XCNTG4O.mjs.map +0 -1
- package/dist/chunk-4LW6443F.mjs.map +0 -1
- package/dist/chunk-72RWABYJ.mjs.map +0 -1
- package/dist/chunk-INBJ7SND.mjs.map +0 -1
- package/dist/chunk-MENKFEPC.mjs.map +0 -1
- package/dist/chunk-U6MVJSXX.mjs.map +0 -1
- package/dist-cli-web/assets/ActionToolsPanel.vue_vue_type_script_setup_true_lang-yr_T4ttU.js +0 -110
- package/dist-cli-web/assets/DataDirCleanupNotice-C6CMNhfs.js +0 -1
- package/dist-cli-web/assets/LoadingState.vue_vue_type_script_setup_true_lang-Bi_Uag3T.js +0 -1
- package/dist-cli-web/assets/TimeSelect.vue_vue_type_script_setup_true_lang-CKe1YZrV.js +0 -1
- package/dist-cli-web/assets/fetch-CbpKKkdO.js +0 -1
- package/dist-cli-web/assets/index-4NL2pYn0.js +0 -1
- package/dist-cli-web/assets/index-B-KdENnr.js +0 -1
- package/dist-cli-web/assets/index-BDyixm5n.js +0 -1
- package/dist-cli-web/assets/index-BaCd1avL.js +0 -1
- package/dist-cli-web/assets/index-CVBBDI3B.js +0 -64
- package/dist-cli-web/assets/index-CioAsn_2.js +0 -137
- package/dist-cli-web/assets/index-Ct90lmPb.js +0 -1
- package/dist-cli-web/assets/index-DYCPQMOl.js +0 -2
- package/dist-cli-web/assets/index-DrjoMv9s.css +0 -1
- package/dist-cli-web/assets/index-Jhwca9kg.css +0 -1
- package/dist-cli-web/assets/index-vtF05fh7.css +0 -1
- /package/dist/{chat-command-6IOH2QRF.mjs.map → chat-command-QH7DXCAJ.mjs.map} +0 -0
- /package/dist/{chunk-JX6MFS6F.mjs.map → chunk-2Q3HHVTS.mjs.map} +0 -0
- /package/dist/{chunk-UMBEHBIZ.mjs.map → chunk-6U2QZO3N.mjs.map} +0 -0
- /package/dist/{chunk-52P5ZRBD.mjs.map → chunk-AWGXYFIC.mjs.map} +0 -0
- /package/dist/{chunk-4RLZFGO5.mjs.map → chunk-C2PD3BS7.mjs.map} +0 -0
- /package/dist/{chunk-652KEAX3.mjs.map → chunk-SWUTRDGZ.mjs.map} +0 -0
- /package/dist/{http-HSK46XEU.mjs.map → http-D2F4QTQW.mjs.map} +0 -0
- /package/dist/{import-DF2WD2HE.mjs.map → import-H7X2JU2O.mjs.map} +0 -0
- /package/dist/{mcp-A2RND7KN.mjs.map → mcp-UURXRGDF.mjs.map} +0 -0
- /package/dist/{src-SHFHW2QB.mjs.map → src-MHCO4HYY.mjs.map} +0 -0
- /package/dist/{stream-import-GZ2QDE3Z.mjs.map → stream-import-BHXACUCA.mjs.map} +0 -0
- /package/dist/{worker-runtime-OWN56GG5.mjs.map → worker-runtime-JOUCP2L4.mjs.map} +0 -0
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
createEmptyContactsDiagnostics,
|
|
10
10
|
normalizeContactsTimeRangePreset,
|
|
11
11
|
resolveContactsTimeRange
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-AWGXYFIC.mjs";
|
|
13
13
|
import {
|
|
14
14
|
PEOPLE_RELATIONSHIPS_ALGORITHM_VERSION,
|
|
15
15
|
buildPeopleRelationshipsNeighborhoodGraph,
|
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
computePeopleRelationshipsSnapshot,
|
|
19
19
|
normalizePeopleRelationshipsTimeRangePreset,
|
|
20
20
|
resolvePeopleRelationshipsTimeRange
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-SWUTRDGZ.mjs";
|
|
22
22
|
import {
|
|
23
23
|
ANNUAL_SUMMARY_ALGORITHM_VERSION
|
|
24
24
|
} from "./chunk-QZCNVAH6.mjs";
|
|
@@ -33,20 +33,14 @@ import {
|
|
|
33
33
|
CHAT_DB_SCHEMA,
|
|
34
34
|
DEFAULT_GENERAL_ASSISTANT_ID,
|
|
35
35
|
DataDirCompatibilityError,
|
|
36
|
-
FTS_TABLE_SCHEMA,
|
|
37
36
|
GENERAL_ASSISTANT_IDS,
|
|
38
37
|
KNOWN_PLATFORMS,
|
|
39
38
|
MessageType,
|
|
40
39
|
StreamingThinkTagParser,
|
|
41
40
|
appLogger,
|
|
42
|
-
batchSegmentChineseWithStats,
|
|
43
|
-
batchSegmentWithFrequency,
|
|
44
|
-
buildExcludeKeywordsConditions,
|
|
45
41
|
buildSessionInfo,
|
|
46
42
|
cleanupAnnualSummarySnapshotTempFiles,
|
|
47
|
-
clearJiebaInstance,
|
|
48
43
|
clearSessionIndex,
|
|
49
|
-
collectPosTagStats,
|
|
50
44
|
deleteAnnualSummarySnapshots,
|
|
51
45
|
deleteAuthProfile,
|
|
52
46
|
deleteMember,
|
|
@@ -65,7 +59,6 @@ import {
|
|
|
65
59
|
getDbFileVersion,
|
|
66
60
|
getGlobalInsightDir,
|
|
67
61
|
getGlobalInsightFactsCacheDir,
|
|
68
|
-
getJieba,
|
|
69
62
|
getMemberNameHistory,
|
|
70
63
|
getMembersPaginated,
|
|
71
64
|
getMembersWithAliases,
|
|
@@ -97,12 +90,9 @@ import {
|
|
|
97
90
|
renameSession,
|
|
98
91
|
resolveApiKey,
|
|
99
92
|
saveSegmentSummary,
|
|
100
|
-
segment,
|
|
101
93
|
snapshotPathProvider,
|
|
102
94
|
toAnnualSummaryLocalDate,
|
|
103
95
|
toAnnualSummaryRangeKey,
|
|
104
|
-
tokenizeForFts,
|
|
105
|
-
tokenizeQueryForFts,
|
|
106
96
|
truncateToolResultText,
|
|
107
97
|
updateMemberAliases,
|
|
108
98
|
updateSessionOwnerId,
|
|
@@ -111,7 +101,7 @@ import {
|
|
|
111
101
|
writeAuthProfile,
|
|
112
102
|
writeJsonFileAtomically,
|
|
113
103
|
writeParseResultToDb
|
|
114
|
-
} from "./chunk-
|
|
104
|
+
} from "./chunk-2WYXD5ZP.mjs";
|
|
115
105
|
import {
|
|
116
106
|
__commonJS,
|
|
117
107
|
__export,
|
|
@@ -727,8 +717,8 @@ var require_FilterBase = __commonJS({
|
|
|
727
717
|
var FilterBase = class _FilterBase extends Transform {
|
|
728
718
|
static stringFilter(string, separator) {
|
|
729
719
|
return (stack) => {
|
|
730
|
-
const
|
|
731
|
-
return
|
|
720
|
+
const path30 = stack.join(separator);
|
|
721
|
+
return path30.length === string.length && path30 === string || path30.length > string.length && path30.substr(0, string.length) === string && path30.substr(string.length, separator.length) === separator;
|
|
732
722
|
};
|
|
733
723
|
}
|
|
734
724
|
static regExpFilter(regExp, separator) {
|
|
@@ -1182,12 +1172,12 @@ var require_Assembler = __commonJS({
|
|
|
1182
1172
|
return (this.stack.length >> 1) + (this.done ? 0 : 1);
|
|
1183
1173
|
}
|
|
1184
1174
|
get path() {
|
|
1185
|
-
const
|
|
1175
|
+
const path30 = [];
|
|
1186
1176
|
for (let i = 0; i < this.stack.length; i += 2) {
|
|
1187
1177
|
const key = this.stack[i + 1];
|
|
1188
|
-
|
|
1178
|
+
path30.push(key === null ? this.stack[i].length : key);
|
|
1189
1179
|
}
|
|
1190
|
-
return
|
|
1180
|
+
return path30;
|
|
1191
1181
|
}
|
|
1192
1182
|
dropToLevel(level) {
|
|
1193
1183
|
if (level < this.depth) {
|
|
@@ -1469,7 +1459,7 @@ var require_pend = __commonJS({
|
|
|
1469
1459
|
var require_fd_slicer = __commonJS({
|
|
1470
1460
|
"../../packages/node-runtime/node_modules/yauzl/fd-slicer.js"(exports) {
|
|
1471
1461
|
"use strict";
|
|
1472
|
-
var
|
|
1462
|
+
var fs36 = __require("fs");
|
|
1473
1463
|
var util = __require("util");
|
|
1474
1464
|
var stream = __require("stream");
|
|
1475
1465
|
var Readable = stream.Readable;
|
|
@@ -1489,7 +1479,7 @@ var require_fd_slicer = __commonJS({
|
|
|
1489
1479
|
FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
|
|
1490
1480
|
var self = this;
|
|
1491
1481
|
self.pend.go(function(cb) {
|
|
1492
|
-
|
|
1482
|
+
fs36.read(self.fd, buffer, offset, length, position, function(err, bytesRead, buffer2) {
|
|
1493
1483
|
cb();
|
|
1494
1484
|
callback(err, bytesRead, buffer2);
|
|
1495
1485
|
});
|
|
@@ -1506,7 +1496,7 @@ var require_fd_slicer = __commonJS({
|
|
|
1506
1496
|
self.refCount -= 1;
|
|
1507
1497
|
if (self.refCount < 0) throw new Error("invalid unref");
|
|
1508
1498
|
if (self.refCount > 0) return;
|
|
1509
|
-
|
|
1499
|
+
fs36.close(self.fd, onCloseDone);
|
|
1510
1500
|
function onCloseDone(err) {
|
|
1511
1501
|
if (err) {
|
|
1512
1502
|
self.emit("error", err);
|
|
@@ -1538,7 +1528,7 @@ var require_fd_slicer = __commonJS({
|
|
|
1538
1528
|
}
|
|
1539
1529
|
self.context.pend.go(function(cb) {
|
|
1540
1530
|
var buffer = Buffer.allocUnsafe(toRead);
|
|
1541
|
-
|
|
1531
|
+
fs36.read(self.context.fd, buffer, 0, toRead, self.pos, function(err, bytesRead) {
|
|
1542
1532
|
if (err) {
|
|
1543
1533
|
self.destroy(err);
|
|
1544
1534
|
} else if (bytesRead === 0) {
|
|
@@ -1900,7 +1890,7 @@ var require_crc32 = __commonJS({
|
|
|
1900
1890
|
var require_yauzl = __commonJS({
|
|
1901
1891
|
"../../packages/node-runtime/node_modules/yauzl/index.js"(exports) {
|
|
1902
1892
|
"use strict";
|
|
1903
|
-
var
|
|
1893
|
+
var fs36 = __require("fs");
|
|
1904
1894
|
var zlib = __require("zlib");
|
|
1905
1895
|
var fd_slicer = require_fd_slicer();
|
|
1906
1896
|
var util = __require("util");
|
|
@@ -1925,9 +1915,9 @@ var require_yauzl = __commonJS({
|
|
|
1925
1915
|
exports.Entry = Entry;
|
|
1926
1916
|
exports.LocalFileHeader = LocalFileHeader;
|
|
1927
1917
|
exports.RandomAccessReader = RandomAccessReader;
|
|
1928
|
-
function openPromise2(
|
|
1918
|
+
function openPromise2(path30, options2) {
|
|
1929
1919
|
return new Promise((resolve4, reject) => {
|
|
1930
|
-
open(
|
|
1920
|
+
open(path30, { ...options2, lazyEntries: true }, function(err, zipfile) {
|
|
1931
1921
|
if (err) return reject(err);
|
|
1932
1922
|
resolve4(zipfile);
|
|
1933
1923
|
});
|
|
@@ -1957,7 +1947,7 @@ var require_yauzl = __commonJS({
|
|
|
1957
1947
|
});
|
|
1958
1948
|
});
|
|
1959
1949
|
}
|
|
1960
|
-
function open(
|
|
1950
|
+
function open(path30, options2, callback) {
|
|
1961
1951
|
if (typeof options2 === "function") {
|
|
1962
1952
|
callback = options2;
|
|
1963
1953
|
options2 = null;
|
|
@@ -1969,10 +1959,10 @@ var require_yauzl = __commonJS({
|
|
|
1969
1959
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
1970
1960
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
1971
1961
|
if (callback == null) callback = defaultCallback;
|
|
1972
|
-
|
|
1962
|
+
fs36.open(path30, "r", function(err, fd) {
|
|
1973
1963
|
if (err) return callback(err);
|
|
1974
1964
|
fromFd(fd, options2, function(err2, zipfile) {
|
|
1975
|
-
if (err2)
|
|
1965
|
+
if (err2) fs36.close(fd, defaultCallback);
|
|
1976
1966
|
callback(err2, zipfile);
|
|
1977
1967
|
});
|
|
1978
1968
|
});
|
|
@@ -1989,7 +1979,7 @@ var require_yauzl = __commonJS({
|
|
|
1989
1979
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
1990
1980
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
1991
1981
|
if (callback == null) callback = defaultCallback;
|
|
1992
|
-
|
|
1982
|
+
fs36.fstat(fd, function(err, stats) {
|
|
1993
1983
|
if (err) return callback(err);
|
|
1994
1984
|
var reader = new fd_slicer.FdSlicer(fd);
|
|
1995
1985
|
fromRandomAccessReader(reader, stats.size, options2, callback);
|
|
@@ -2830,6 +2820,39 @@ import * as path11 from "path";
|
|
|
2830
2820
|
// ../../packages/parser/src/sniffer.ts
|
|
2831
2821
|
import * as fs2 from "fs";
|
|
2832
2822
|
import * as path2 from "path";
|
|
2823
|
+
|
|
2824
|
+
// ../../packages/parser/src/format-ids.ts
|
|
2825
|
+
var PARSER_FORMAT_IDS = {
|
|
2826
|
+
CHATLAB: "chatlab",
|
|
2827
|
+
CHATLAB_JSONL: "chatlab-jsonl",
|
|
2828
|
+
QQ_SHUAKAMI: "qq-shuakami",
|
|
2829
|
+
QQ_SHUAKAMI_CHUNKED: "qq-shuakami-chunked",
|
|
2830
|
+
WEFLOW: "weflow",
|
|
2831
|
+
ECHOTRACE: "echotrace",
|
|
2832
|
+
DISCORD_TYRRRZ: "discord-tyrrrz",
|
|
2833
|
+
TELEGRAM_NATIVE: "telegram-native",
|
|
2834
|
+
TELEGRAM_NATIVE_SINGLE: "telegram-native-single",
|
|
2835
|
+
GOOGLE_CHAT_NATIVE: "google-chat-native",
|
|
2836
|
+
INSTAGRAM_NATIVE: "instagram-native",
|
|
2837
|
+
WHATSAPP_NATIVE: "whatsapp-native",
|
|
2838
|
+
QQ_NATIVE: "qq-native",
|
|
2839
|
+
LINE_NATIVE: "line-native"
|
|
2840
|
+
};
|
|
2841
|
+
var LEGACY_PARSER_FORMAT_ID_ALIASES = {
|
|
2842
|
+
"shuakami-qq-exporter": PARSER_FORMAT_IDS.QQ_SHUAKAMI,
|
|
2843
|
+
"shuakami-qq-exporter-chunked": PARSER_FORMAT_IDS.QQ_SHUAKAMI_CHUNKED,
|
|
2844
|
+
"ycccccccy-echotrace": PARSER_FORMAT_IDS.ECHOTRACE,
|
|
2845
|
+
"tyrrrz-discord-exporter": PARSER_FORMAT_IDS.DISCORD_TYRRRZ,
|
|
2846
|
+
"whatsapp-native-txt": PARSER_FORMAT_IDS.WHATSAPP_NATIVE,
|
|
2847
|
+
"qq-native-txt": PARSER_FORMAT_IDS.QQ_NATIVE,
|
|
2848
|
+
"line-native-txt": PARSER_FORMAT_IDS.LINE_NATIVE,
|
|
2849
|
+
"google-chat-takeout": PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE
|
|
2850
|
+
};
|
|
2851
|
+
function normalizeParserFormatId(formatId) {
|
|
2852
|
+
return LEGACY_PARSER_FORMAT_ID_ALIASES[formatId] ?? formatId;
|
|
2853
|
+
}
|
|
2854
|
+
|
|
2855
|
+
// ../../packages/parser/src/sniffer.ts
|
|
2833
2856
|
var HEAD_SIZE = 64 * 1024;
|
|
2834
2857
|
function readFileHead(filePath, size = HEAD_SIZE) {
|
|
2835
2858
|
const fd = fs2.openSync(filePath, "r");
|
|
@@ -2947,9 +2970,17 @@ var FormatSniffer = class {
|
|
|
2947
2970
|
* 根据格式 ID 获取解析器
|
|
2948
2971
|
*/
|
|
2949
2972
|
getParserById(formatId) {
|
|
2950
|
-
const
|
|
2973
|
+
const canonicalId = normalizeParserFormatId(formatId);
|
|
2974
|
+
const module = this.formats.find((m) => m.feature.id === canonicalId);
|
|
2951
2975
|
return module?.parser || null;
|
|
2952
2976
|
}
|
|
2977
|
+
/**
|
|
2978
|
+
* Resolve canonical and released legacy format IDs to one canonical feature.
|
|
2979
|
+
*/
|
|
2980
|
+
getFeatureById(formatId) {
|
|
2981
|
+
const canonicalId = normalizeParserFormatId(formatId);
|
|
2982
|
+
return this.formats.find((module) => module.feature.id === canonicalId)?.feature ?? null;
|
|
2983
|
+
}
|
|
2953
2984
|
/**
|
|
2954
2985
|
* 获取所有支持的格式
|
|
2955
2986
|
*/
|
|
@@ -3087,12 +3118,18 @@ function isNativeFormatAvailable(formatId) {
|
|
|
3087
3118
|
const module = loadNativeParser();
|
|
3088
3119
|
if (!module) return false;
|
|
3089
3120
|
try {
|
|
3090
|
-
new module.NativeParser(formatId, "");
|
|
3121
|
+
new module.NativeParser(resolveNativeKernelId(formatId), "");
|
|
3091
3122
|
return true;
|
|
3092
3123
|
} catch {
|
|
3093
3124
|
return false;
|
|
3094
3125
|
}
|
|
3095
3126
|
}
|
|
3127
|
+
function resolveNativeKernelId(formatId) {
|
|
3128
|
+
const canonicalId = normalizeParserFormatId(formatId);
|
|
3129
|
+
if (canonicalId === PARSER_FORMAT_IDS.QQ_SHUAKAMI) return "shuakami-qq-exporter";
|
|
3130
|
+
if (canonicalId === PARSER_FORMAT_IDS.ECHOTRACE) return "weflow";
|
|
3131
|
+
return canonicalId;
|
|
3132
|
+
}
|
|
3096
3133
|
|
|
3097
3134
|
// ../../packages/parser/src/native/create-native-parser.ts
|
|
3098
3135
|
var PROGRESS_POLL_INTERVAL_MS = 200;
|
|
@@ -3161,7 +3198,7 @@ function createNativeFirstParser(adapter, fallback, fallbackAfterNativeFailure2
|
|
|
3161
3198
|
let parser11;
|
|
3162
3199
|
try {
|
|
3163
3200
|
parser11 = new native.NativeParser(
|
|
3164
|
-
adapter.
|
|
3201
|
+
adapter.kernelId,
|
|
3165
3202
|
options2.filePath,
|
|
3166
3203
|
options2.formatOptions ? JSON.stringify(options2.formatOptions) : void 0
|
|
3167
3204
|
);
|
|
@@ -3197,7 +3234,7 @@ function toParsedMember(member, fromHead) {
|
|
|
3197
3234
|
};
|
|
3198
3235
|
}
|
|
3199
3236
|
var chatlabAdapter = {
|
|
3200
|
-
|
|
3237
|
+
kernelId: "chatlab",
|
|
3201
3238
|
label: "ChatLab JSON",
|
|
3202
3239
|
mapMeta(metaJson) {
|
|
3203
3240
|
const meta = metaJson;
|
|
@@ -3246,7 +3283,7 @@ function extractNameFromFilePath(filePath) {
|
|
|
3246
3283
|
return name || "\u672A\u77E5\u7FA4\u804A";
|
|
3247
3284
|
}
|
|
3248
3285
|
var feature = {
|
|
3249
|
-
id:
|
|
3286
|
+
id: PARSER_FORMAT_IDS.CHATLAB,
|
|
3250
3287
|
name: "ChatLab JSON",
|
|
3251
3288
|
platform: KNOWN_PLATFORMS.UNKNOWN,
|
|
3252
3289
|
// ChatLab 格式可能包含多平台数据
|
|
@@ -3428,7 +3465,7 @@ function parseLine(line) {
|
|
|
3428
3465
|
}
|
|
3429
3466
|
}
|
|
3430
3467
|
var feature2 = {
|
|
3431
|
-
id:
|
|
3468
|
+
id: PARSER_FORMAT_IDS.CHATLAB_JSONL,
|
|
3432
3469
|
name: "ChatLab JSONL",
|
|
3433
3470
|
platform: KNOWN_PLATFORMS.UNKNOWN,
|
|
3434
3471
|
priority: 51,
|
|
@@ -3825,7 +3862,7 @@ var shuakamiQqPreprocessor = {
|
|
|
3825
3862
|
|
|
3826
3863
|
// ../../packages/parser/src/native/shuakami-qq-native.ts
|
|
3827
3864
|
var shuakamiQqAdapter = {
|
|
3828
|
-
|
|
3865
|
+
kernelId: "shuakami-qq-exporter",
|
|
3829
3866
|
label: "shuakami/qq-chat-exporter",
|
|
3830
3867
|
mapMeta(metaJson) {
|
|
3831
3868
|
const meta = metaJson;
|
|
@@ -3833,7 +3870,8 @@ var shuakamiQqAdapter = {
|
|
|
3833
3870
|
name: meta.name,
|
|
3834
3871
|
platform: KNOWN_PLATFORMS.QQ,
|
|
3835
3872
|
type: meta.chatType === "private" ? "private" /* PRIVATE */ : "group" /* GROUP */,
|
|
3836
|
-
groupAvatar: meta.groupAvatar ?? void 0
|
|
3873
|
+
groupAvatar: meta.groupAvatar ?? void 0,
|
|
3874
|
+
...meta.ownerId ? { ownerId: meta.ownerId } : {}
|
|
3837
3875
|
};
|
|
3838
3876
|
},
|
|
3839
3877
|
mapMembers(members) {
|
|
@@ -3900,7 +3938,7 @@ var { pick: pick3 } = import_Pick3.default;
|
|
|
3900
3938
|
var { streamValues: streamValues3 } = import_StreamValues3.default;
|
|
3901
3939
|
var { chain: chain3 } = import_stream_chain3.default;
|
|
3902
3940
|
var feature3 = {
|
|
3903
|
-
id:
|
|
3941
|
+
id: PARSER_FORMAT_IDS.QQ_SHUAKAMI,
|
|
3904
3942
|
name: "shuakami/qq-chat-exporter",
|
|
3905
3943
|
platform: KNOWN_PLATFORMS.QQ,
|
|
3906
3944
|
priority: 10,
|
|
@@ -4168,7 +4206,7 @@ import * as fs8 from "fs";
|
|
|
4168
4206
|
import * as path7 from "path";
|
|
4169
4207
|
import * as readline2 from "readline";
|
|
4170
4208
|
var feature4 = {
|
|
4171
|
-
id:
|
|
4209
|
+
id: PARSER_FORMAT_IDS.QQ_SHUAKAMI_CHUNKED,
|
|
4172
4210
|
name: "shuakami/qq-chat-exporter (chunked)",
|
|
4173
4211
|
platform: KNOWN_PLATFORMS.QQ,
|
|
4174
4212
|
priority: 5,
|
|
@@ -4341,11 +4379,10 @@ async function* parseChunkedJsonl(options2) {
|
|
|
4341
4379
|
const meta = {
|
|
4342
4380
|
name: manifest.chatInfo.name || "\u672A\u77E5\u7FA4\u804A",
|
|
4343
4381
|
platform: KNOWN_PLATFORMS.QQ,
|
|
4344
|
-
type: chatType
|
|
4345
|
-
ownerId: manifest.chatInfo.selfUin || manifest.chatInfo.selfUid
|
|
4382
|
+
type: chatType
|
|
4346
4383
|
};
|
|
4347
4384
|
yield { type: "meta", data: meta };
|
|
4348
|
-
onLog?.("info", `Meta \u4FE1\u606F: name=${meta.name}, type=${meta.type}
|
|
4385
|
+
onLog?.("info", `Meta \u4FE1\u606F: name=${meta.name}, type=${meta.type}`);
|
|
4349
4386
|
const memberMap = /* @__PURE__ */ new Map();
|
|
4350
4387
|
let currentBatch = [];
|
|
4351
4388
|
for (let chunkIndex = 0; chunkIndex < manifest.chunked.chunks.length; chunkIndex++) {
|
|
@@ -4484,7 +4521,7 @@ import * as path8 from "path";
|
|
|
4484
4521
|
|
|
4485
4522
|
// ../../packages/parser/src/native/weflow-native.ts
|
|
4486
4523
|
var weflowAdapter = {
|
|
4487
|
-
|
|
4524
|
+
kernelId: "weflow",
|
|
4488
4525
|
label: "WeFlow export",
|
|
4489
4526
|
mapMeta(metaJson) {
|
|
4490
4527
|
const meta = metaJson;
|
|
@@ -4558,7 +4595,7 @@ function extractNameFromFilePath4(filePath) {
|
|
|
4558
4595
|
return name || "\u672A\u77E5\u804A\u5929";
|
|
4559
4596
|
}
|
|
4560
4597
|
var feature5 = {
|
|
4561
|
-
id:
|
|
4598
|
+
id: PARSER_FORMAT_IDS.WEFLOW,
|
|
4562
4599
|
name: "WeFlow \u5BFC\u51FA",
|
|
4563
4600
|
platform: KNOWN_PLATFORMS.WECHAT,
|
|
4564
4601
|
priority: 15,
|
|
@@ -4947,7 +4984,7 @@ var weflow_default = module_5;
|
|
|
4947
4984
|
|
|
4948
4985
|
// ../../packages/parser/src/formats/ycccccccy-echotrace.ts
|
|
4949
4986
|
var feature6 = {
|
|
4950
|
-
id:
|
|
4987
|
+
id: PARSER_FORMAT_IDS.ECHOTRACE,
|
|
4951
4988
|
name: "ycccccccy/echotrace \u5BFC\u51FA",
|
|
4952
4989
|
platform: KNOWN_PLATFORMS.WECHAT,
|
|
4953
4990
|
priority: 16,
|
|
@@ -5066,7 +5103,7 @@ function formatSticker(sticker) {
|
|
|
5066
5103
|
return AttachmentMarkers.sticker(sticker.name);
|
|
5067
5104
|
}
|
|
5068
5105
|
var feature7 = {
|
|
5069
|
-
id:
|
|
5106
|
+
id: PARSER_FORMAT_IDS.DISCORD_TYRRRZ,
|
|
5070
5107
|
name: "Tyrrrz/DiscordChatExporter",
|
|
5071
5108
|
platform: KNOWN_PLATFORMS.DISCORD,
|
|
5072
5109
|
priority: 20,
|
|
@@ -5311,7 +5348,7 @@ var { parser: parser7 } = import_stream_json6.default;
|
|
|
5311
5348
|
var { pick: pick6 } = import_Pick6.default;
|
|
5312
5349
|
var { streamValues: streamValues6 } = import_StreamValues6.default;
|
|
5313
5350
|
var feature8 = {
|
|
5314
|
-
id:
|
|
5351
|
+
id: PARSER_FORMAT_IDS.TELEGRAM_NATIVE,
|
|
5315
5352
|
name: "Telegram \u5B98\u65B9\u5BFC\u51FA (JSON)",
|
|
5316
5353
|
platform: KNOWN_PLATFORMS.TELEGRAM,
|
|
5317
5354
|
priority: 22,
|
|
@@ -5476,7 +5513,7 @@ var TELEGRAM_SINGLE_HEAD_SIGNATURES = [/^\s*\{\s*(?:\r?\n\s*)?"name"\s*:/];
|
|
|
5476
5513
|
// ../../packages/parser/src/formats/telegram-native-single.ts
|
|
5477
5514
|
var { parser: parser8 } = import_stream_json7.default;
|
|
5478
5515
|
var feature9 = {
|
|
5479
|
-
id:
|
|
5516
|
+
id: PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE,
|
|
5480
5517
|
name: "Telegram \u5355\u804A\u5929\u5BFC\u51FA (JSON)",
|
|
5481
5518
|
platform: KNOWN_PLATFORMS.TELEGRAM,
|
|
5482
5519
|
priority: 23,
|
|
@@ -5861,7 +5898,7 @@ function buildTimestamp(firstDatePart, secondDatePart, thirdDatePart, rawHour, m
|
|
|
5861
5898
|
|
|
5862
5899
|
// ../../packages/parser/src/formats/whatsapp-native-txt.ts
|
|
5863
5900
|
var feature10 = {
|
|
5864
|
-
id:
|
|
5901
|
+
id: PARSER_FORMAT_IDS.WHATSAPP_NATIVE,
|
|
5865
5902
|
name: "WhatsApp \u5B98\u65B9\u5BFC\u51FA (TXT)",
|
|
5866
5903
|
platform: KNOWN_PLATFORMS.WHATSAPP,
|
|
5867
5904
|
priority: 25,
|
|
@@ -6067,7 +6104,7 @@ function detectMessageType3(content) {
|
|
|
6067
6104
|
|
|
6068
6105
|
// ../../packages/parser/src/formats/qq-native-txt.ts
|
|
6069
6106
|
var feature11 = {
|
|
6070
|
-
id:
|
|
6107
|
+
id: PARSER_FORMAT_IDS.QQ_NATIVE,
|
|
6071
6108
|
name: "QQ \u5B98\u65B9\u5BFC\u51FA (TXT)",
|
|
6072
6109
|
platform: KNOWN_PLATFORMS.QQ,
|
|
6073
6110
|
priority: 30,
|
|
@@ -6123,7 +6160,7 @@ var qq_native_txt_default = module_11;
|
|
|
6123
6160
|
import * as fs15 from "fs";
|
|
6124
6161
|
import * as path9 from "path";
|
|
6125
6162
|
var feature12 = {
|
|
6126
|
-
id:
|
|
6163
|
+
id: PARSER_FORMAT_IDS.INSTAGRAM_NATIVE,
|
|
6127
6164
|
name: "Instagram \u5B98\u65B9\u5BFC\u51FA",
|
|
6128
6165
|
platform: KNOWN_PLATFORMS.INSTAGRAM,
|
|
6129
6166
|
priority: 25,
|
|
@@ -6333,7 +6370,7 @@ var ENGLISH_MONTHS = /* @__PURE__ */ new Map([
|
|
|
6333
6370
|
["december", 11]
|
|
6334
6371
|
]);
|
|
6335
6372
|
var feature13 = {
|
|
6336
|
-
id:
|
|
6373
|
+
id: PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE,
|
|
6337
6374
|
name: "Google Chat Takeout",
|
|
6338
6375
|
platform: KNOWN_PLATFORMS.GOOGLE_CHAT,
|
|
6339
6376
|
priority: 24,
|
|
@@ -6767,7 +6804,7 @@ function detectMessageType6(content) {
|
|
|
6767
6804
|
|
|
6768
6805
|
// ../../packages/parser/src/formats/line-native-txt.ts
|
|
6769
6806
|
var feature14 = {
|
|
6770
|
-
id:
|
|
6807
|
+
id: PARSER_FORMAT_IDS.LINE_NATIVE,
|
|
6771
6808
|
name: "LINE \u5B98\u65B9\u5BFC\u51FA TXT",
|
|
6772
6809
|
platform: KNOWN_PLATFORMS.LINE,
|
|
6773
6810
|
priority: 35,
|
|
@@ -6898,21 +6935,21 @@ function isRecord(value) {
|
|
|
6898
6935
|
function hasOwn(value, key) {
|
|
6899
6936
|
return Object.prototype.hasOwnProperty.call(value, key);
|
|
6900
6937
|
}
|
|
6901
|
-
function childLocation(location,
|
|
6902
|
-
return { ...location, path:
|
|
6938
|
+
function childLocation(location, path30) {
|
|
6939
|
+
return { ...location, path: path30 };
|
|
6903
6940
|
}
|
|
6904
|
-
function validateRequiredString(state, value,
|
|
6941
|
+
function validateRequiredString(state, value, path30, location) {
|
|
6905
6942
|
if (typeof value === "string" && value.trim().length > 0) return true;
|
|
6906
|
-
addIssue(state, "error", "INVALID_REQUIRED_STRING", "Expected a non-empty string.", childLocation(location,
|
|
6943
|
+
addIssue(state, "error", "INVALID_REQUIRED_STRING", "Expected a non-empty string.", childLocation(location, path30));
|
|
6907
6944
|
return false;
|
|
6908
6945
|
}
|
|
6909
|
-
function validateOptionalString(state, object, key,
|
|
6946
|
+
function validateOptionalString(state, object, key, path30, location) {
|
|
6910
6947
|
if (!hasOwn(object, key)) return;
|
|
6911
6948
|
if (typeof object[key] !== "string") {
|
|
6912
|
-
addIssue(state, "error", "INVALID_OPTIONAL_STRING", "Expected a string.", childLocation(location,
|
|
6949
|
+
addIssue(state, "error", "INVALID_OPTIONAL_STRING", "Expected a string.", childLocation(location, path30));
|
|
6913
6950
|
}
|
|
6914
6951
|
}
|
|
6915
|
-
function validateUnixSeconds(state, value,
|
|
6952
|
+
function validateUnixSeconds(state, value, path30, location) {
|
|
6916
6953
|
if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value < MAX_UNIX_SECONDS) {
|
|
6917
6954
|
return true;
|
|
6918
6955
|
}
|
|
@@ -6921,7 +6958,7 @@ function validateUnixSeconds(state, value, path31, location) {
|
|
|
6921
6958
|
"error",
|
|
6922
6959
|
"INVALID_UNIX_SECONDS",
|
|
6923
6960
|
"Expected a non-negative integer Unix timestamp in seconds, not milliseconds.",
|
|
6924
|
-
childLocation(location,
|
|
6961
|
+
childLocation(location, path30)
|
|
6925
6962
|
);
|
|
6926
6963
|
return false;
|
|
6927
6964
|
}
|
|
@@ -6982,30 +7019,30 @@ function validateHeader(state, chatlab, meta, location, basePath = "") {
|
|
|
6982
7019
|
}
|
|
6983
7020
|
validateOptionalString(state, meta, "sourceSessionId", `${basePath}meta.sourceSessionId`, location);
|
|
6984
7021
|
}
|
|
6985
|
-
function validateMember(state, value,
|
|
7022
|
+
function validateMember(state, value, path30, location) {
|
|
6986
7023
|
state.stats.memberCount++;
|
|
6987
7024
|
if (!isRecord(value)) {
|
|
6988
|
-
addIssue(state, "error", "INVALID_MEMBER", "Expected a member object.", childLocation(location,
|
|
7025
|
+
addIssue(state, "error", "INVALID_MEMBER", "Expected a member object.", childLocation(location, path30));
|
|
6989
7026
|
return;
|
|
6990
7027
|
}
|
|
6991
|
-
if (validateRequiredString(state, value.platformId, `${
|
|
7028
|
+
if (validateRequiredString(state, value.platformId, `${path30}.platformId`, location)) {
|
|
6992
7029
|
if (state.memberIds.has(value.platformId)) {
|
|
6993
7030
|
addIssue(state, "error", "DUPLICATE_MEMBER_ID", "Member platformId must be unique.", {
|
|
6994
7031
|
...location,
|
|
6995
|
-
path: `${
|
|
7032
|
+
path: `${path30}.platformId`
|
|
6996
7033
|
});
|
|
6997
7034
|
} else {
|
|
6998
7035
|
state.memberIds.add(value.platformId);
|
|
6999
7036
|
}
|
|
7000
7037
|
}
|
|
7001
|
-
validateRequiredString(state, value.accountName, `${
|
|
7002
|
-
validateOptionalString(state, value, "groupNickname", `${
|
|
7003
|
-
validateOptionalString(state, value, "avatar", `${
|
|
7038
|
+
validateRequiredString(state, value.accountName, `${path30}.accountName`, location);
|
|
7039
|
+
validateOptionalString(state, value, "groupNickname", `${path30}.groupNickname`, location);
|
|
7040
|
+
validateOptionalString(state, value, "avatar", `${path30}.avatar`, location);
|
|
7004
7041
|
if (hasOwn(value, "aliases")) {
|
|
7005
7042
|
if (!Array.isArray(value.aliases) || value.aliases.some((alias) => typeof alias !== "string")) {
|
|
7006
7043
|
addIssue(state, "error", "INVALID_ALIASES", "Expected aliases to be an array of strings.", {
|
|
7007
7044
|
...location,
|
|
7008
|
-
path: `${
|
|
7045
|
+
path: `${path30}.aliases`
|
|
7009
7046
|
});
|
|
7010
7047
|
}
|
|
7011
7048
|
}
|
|
@@ -7013,11 +7050,11 @@ function validateMember(state, value, path31, location) {
|
|
|
7013
7050
|
if (!Array.isArray(value.roles)) {
|
|
7014
7051
|
addIssue(state, "error", "INVALID_ROLES", "Expected roles to be an array.", {
|
|
7015
7052
|
...location,
|
|
7016
|
-
path: `${
|
|
7053
|
+
path: `${path30}.roles`
|
|
7017
7054
|
});
|
|
7018
7055
|
} else {
|
|
7019
7056
|
value.roles.forEach((role, index) => {
|
|
7020
|
-
const rolePath = `${
|
|
7057
|
+
const rolePath = `${path30}.roles[${index}]`;
|
|
7021
7058
|
if (!isRecord(role)) {
|
|
7022
7059
|
addIssue(state, "error", "INVALID_ROLE", "Expected a role object.", {
|
|
7023
7060
|
...location,
|
|
@@ -7031,25 +7068,25 @@ function validateMember(state, value, path31, location) {
|
|
|
7031
7068
|
}
|
|
7032
7069
|
}
|
|
7033
7070
|
}
|
|
7034
|
-
function validateMessage(state, value,
|
|
7071
|
+
function validateMessage(state, value, path30, location) {
|
|
7035
7072
|
state.stats.messageCount++;
|
|
7036
7073
|
if (!isRecord(value)) {
|
|
7037
|
-
addIssue(state, "error", "INVALID_MESSAGE", "Expected a message object.", childLocation(location,
|
|
7074
|
+
addIssue(state, "error", "INVALID_MESSAGE", "Expected a message object.", childLocation(location, path30));
|
|
7038
7075
|
return;
|
|
7039
7076
|
}
|
|
7040
|
-
const sender = validateRequiredString(state, value.sender, `${
|
|
7077
|
+
const sender = validateRequiredString(state, value.sender, `${path30}.sender`, location) ? value.sender : void 0;
|
|
7041
7078
|
if (sender !== void 0) {
|
|
7042
7079
|
state.senderIds.add(sender);
|
|
7043
7080
|
}
|
|
7044
|
-
validateRequiredString(state, value.accountName, `${
|
|
7045
|
-
if (validateUnixSeconds(state, value.timestamp, `${
|
|
7081
|
+
validateRequiredString(state, value.accountName, `${path30}.accountName`, location);
|
|
7082
|
+
if (validateUnixSeconds(state, value.timestamp, `${path30}.timestamp`, location)) {
|
|
7046
7083
|
if (state.lastTimestamp !== void 0 && value.timestamp < state.lastTimestamp) {
|
|
7047
7084
|
addIssue(
|
|
7048
7085
|
state,
|
|
7049
7086
|
"warning",
|
|
7050
7087
|
"OUT_OF_ORDER_TIMESTAMP",
|
|
7051
7088
|
"Messages should be ordered by timestamp ascending.",
|
|
7052
|
-
childLocation(location, `${
|
|
7089
|
+
childLocation(location, `${path30}.timestamp`)
|
|
7053
7090
|
);
|
|
7054
7091
|
}
|
|
7055
7092
|
state.lastTimestamp = value.timestamp;
|
|
@@ -7058,7 +7095,7 @@ function validateMessage(state, value, path31, location) {
|
|
|
7058
7095
|
if (messageType === void 0) {
|
|
7059
7096
|
addIssue(state, "error", "INVALID_MESSAGE_TYPE", "Expected a supported numeric ChatLab message type.", {
|
|
7060
7097
|
...location,
|
|
7061
|
-
path: `${
|
|
7098
|
+
path: `${path30}.type`
|
|
7062
7099
|
});
|
|
7063
7100
|
} else if (sender === SYSTEM_SENDER_ID && !SYSTEM_SENDER_MESSAGE_TYPES.has(messageType)) {
|
|
7064
7101
|
addIssue(
|
|
@@ -7066,22 +7103,22 @@ function validateMessage(state, value, path31, location) {
|
|
|
7066
7103
|
"error",
|
|
7067
7104
|
"INVALID_SYSTEM_SENDER_TYPE",
|
|
7068
7105
|
"The reserved SYSTEM sender is only valid for system and recall messages.",
|
|
7069
|
-
childLocation(location, `${
|
|
7106
|
+
childLocation(location, `${path30}.sender`)
|
|
7070
7107
|
);
|
|
7071
7108
|
}
|
|
7072
7109
|
if (!hasOwn(value, "content") || typeof value.content !== "string" && value.content !== null) {
|
|
7073
7110
|
addIssue(state, "error", "INVALID_MESSAGE_CONTENT", "Expected content to be a string or null.", {
|
|
7074
7111
|
...location,
|
|
7075
|
-
path: `${
|
|
7112
|
+
path: `${path30}.content`
|
|
7076
7113
|
});
|
|
7077
7114
|
}
|
|
7078
|
-
validateOptionalString(state, value, "groupNickname", `${
|
|
7115
|
+
validateOptionalString(state, value, "groupNickname", `${path30}.groupNickname`, location);
|
|
7079
7116
|
if (hasOwn(value, "platformMessageId")) {
|
|
7080
|
-
if (validateRequiredString(state, value.platformMessageId, `${
|
|
7117
|
+
if (validateRequiredString(state, value.platformMessageId, `${path30}.platformMessageId`, location)) {
|
|
7081
7118
|
if (state.messageIds.has(value.platformMessageId)) {
|
|
7082
7119
|
addIssue(state, "error", "DUPLICATE_MESSAGE_ID", "platformMessageId must be unique.", {
|
|
7083
7120
|
...location,
|
|
7084
|
-
path: `${
|
|
7121
|
+
path: `${path30}.platformMessageId`
|
|
7085
7122
|
});
|
|
7086
7123
|
} else {
|
|
7087
7124
|
state.messageIds.add(value.platformMessageId);
|
|
@@ -7089,7 +7126,7 @@ function validateMessage(state, value, path31, location) {
|
|
|
7089
7126
|
}
|
|
7090
7127
|
}
|
|
7091
7128
|
if (hasOwn(value, "replyToMessageId")) {
|
|
7092
|
-
if (validateRequiredString(state, value.replyToMessageId, `${
|
|
7129
|
+
if (validateRequiredString(state, value.replyToMessageId, `${path30}.replyToMessageId`, location)) {
|
|
7093
7130
|
state.replyTargetIds.add(value.replyToMessageId);
|
|
7094
7131
|
}
|
|
7095
7132
|
}
|
|
@@ -7416,8 +7453,7 @@ function getSupportedFormats() {
|
|
|
7416
7453
|
return sniffer.getSupportedFormats();
|
|
7417
7454
|
}
|
|
7418
7455
|
function getFormatFeatureById(formatId) {
|
|
7419
|
-
|
|
7420
|
-
return all.find((f) => f.id === formatId) ?? null;
|
|
7456
|
+
return sniffer.getFeatureById(formatId);
|
|
7421
7457
|
}
|
|
7422
7458
|
function getPreprocessor(filePath) {
|
|
7423
7459
|
const feature15 = sniffer.sniff(filePath);
|
|
@@ -7649,319 +7685,6 @@ function logSummary(totalMessages, totalMembers) {
|
|
|
7649
7685
|
defaultLogger.summary(totalMessages, totalMembers);
|
|
7650
7686
|
}
|
|
7651
7687
|
|
|
7652
|
-
// ../../packages/node-runtime/src/nlp/word-frequency.ts
|
|
7653
|
-
function buildMessageQuery(timeFilter, memberId, excludeKeywords) {
|
|
7654
|
-
const conditions = [];
|
|
7655
|
-
const params = [];
|
|
7656
|
-
if (timeFilter?.startTs !== void 0) {
|
|
7657
|
-
conditions.push("msg.ts >= ?");
|
|
7658
|
-
params.push(timeFilter.startTs);
|
|
7659
|
-
}
|
|
7660
|
-
if (timeFilter?.endTs !== void 0) {
|
|
7661
|
-
conditions.push("msg.ts <= ?");
|
|
7662
|
-
params.push(timeFilter.endTs);
|
|
7663
|
-
}
|
|
7664
|
-
if (memberId !== void 0 && memberId !== null) {
|
|
7665
|
-
conditions.push("msg.sender_id = ?");
|
|
7666
|
-
params.push(memberId);
|
|
7667
|
-
}
|
|
7668
|
-
if (excludeKeywords && excludeKeywords.length > 0) {
|
|
7669
|
-
const exclude = buildExcludeKeywordsConditions(excludeKeywords);
|
|
7670
|
-
conditions.push(...exclude.conditions);
|
|
7671
|
-
params.push(...exclude.params);
|
|
7672
|
-
}
|
|
7673
|
-
conditions.push("COALESCE(m.account_name, '') != '\u7CFB\u7EDF\u6D88\u606F'");
|
|
7674
|
-
conditions.push("msg.type = 0");
|
|
7675
|
-
conditions.push("msg.content IS NOT NULL");
|
|
7676
|
-
conditions.push("TRIM(msg.content) != ''");
|
|
7677
|
-
return {
|
|
7678
|
-
clause: ` WHERE ${conditions.join(" AND ")}`,
|
|
7679
|
-
params
|
|
7680
|
-
};
|
|
7681
|
-
}
|
|
7682
|
-
function computeWordFrequency(db, params) {
|
|
7683
|
-
const {
|
|
7684
|
-
locale,
|
|
7685
|
-
timeFilter,
|
|
7686
|
-
memberId,
|
|
7687
|
-
topN = 100,
|
|
7688
|
-
minWordLength,
|
|
7689
|
-
minCount = 2,
|
|
7690
|
-
posFilterMode = "meaningful",
|
|
7691
|
-
customPosTags,
|
|
7692
|
-
enableStopwords = true,
|
|
7693
|
-
dictType = "default",
|
|
7694
|
-
excludeWords,
|
|
7695
|
-
excludeKeywords
|
|
7696
|
-
} = params;
|
|
7697
|
-
const { clause, params: filterParams } = buildMessageQuery(timeFilter, memberId, excludeKeywords);
|
|
7698
|
-
const messages = db.prepare(`SELECT msg.content FROM message msg JOIN member m ON msg.sender_id = m.id${clause}`).all(...filterParams);
|
|
7699
|
-
if (messages.length === 0) {
|
|
7700
|
-
return { words: [], totalWords: 0, totalMessages: 0, uniqueWords: 0 };
|
|
7701
|
-
}
|
|
7702
|
-
const texts = messages.map((m) => m.content);
|
|
7703
|
-
const segmentOptions = {
|
|
7704
|
-
minLength: minWordLength,
|
|
7705
|
-
minCount,
|
|
7706
|
-
topN,
|
|
7707
|
-
posFilterMode,
|
|
7708
|
-
customPosTags,
|
|
7709
|
-
enableStopwords,
|
|
7710
|
-
dictType,
|
|
7711
|
-
excludeWords
|
|
7712
|
-
};
|
|
7713
|
-
let posTagStats;
|
|
7714
|
-
let result;
|
|
7715
|
-
if (locale.startsWith("zh") && posFilterMode !== "all") {
|
|
7716
|
-
const combined = batchSegmentChineseWithStats(texts, locale, segmentOptions);
|
|
7717
|
-
result = { words: combined.words, uniqueWords: combined.uniqueWords, totalWords: combined.totalWords };
|
|
7718
|
-
posTagStats = [...combined.posTagStats.entries()].map(([tag, count]) => ({ tag, count }));
|
|
7719
|
-
} else {
|
|
7720
|
-
if (locale.startsWith("zh")) {
|
|
7721
|
-
const posStatsMap = collectPosTagStats(texts, minWordLength ?? 2, enableStopwords, dictType);
|
|
7722
|
-
posTagStats = [...posStatsMap.entries()].map(([tag, count]) => ({ tag, count }));
|
|
7723
|
-
}
|
|
7724
|
-
result = batchSegmentWithFrequency(texts, locale, segmentOptions);
|
|
7725
|
-
}
|
|
7726
|
-
let topNTotalWords = 0;
|
|
7727
|
-
for (const count of result.words.values()) topNTotalWords += count;
|
|
7728
|
-
const words = [...result.words.entries()].map(([word, count]) => ({
|
|
7729
|
-
word,
|
|
7730
|
-
count,
|
|
7731
|
-
percentage: topNTotalWords > 0 ? Math.round(count / topNTotalWords * 1e4) / 100 : 0
|
|
7732
|
-
}));
|
|
7733
|
-
return {
|
|
7734
|
-
words,
|
|
7735
|
-
totalWords: result.totalWords,
|
|
7736
|
-
totalMessages: messages.length,
|
|
7737
|
-
uniqueWords: result.uniqueWords,
|
|
7738
|
-
posTagStats
|
|
7739
|
-
};
|
|
7740
|
-
}
|
|
7741
|
-
function segmentText(text, locale, minLength) {
|
|
7742
|
-
return segment(text, locale, { minLength });
|
|
7743
|
-
}
|
|
7744
|
-
|
|
7745
|
-
// ../../packages/node-runtime/src/nlp/dict-manager.ts
|
|
7746
|
-
import * as fs20 from "fs";
|
|
7747
|
-
import * as path13 from "path";
|
|
7748
|
-
import { createHash } from "crypto";
|
|
7749
|
-
var DICT_DOWNLOAD_URL_BASE = "https://chatlab.fun/assets/nlp";
|
|
7750
|
-
var DICT_SHA256 = {
|
|
7751
|
-
"zh-CN": "139519822fe8ab9e10d9d07e68ea0451045380aedaf54ecc51e2a28c6b42a13f",
|
|
7752
|
-
"zh-TW": "a63ec7e388f16f1b486dcd948a9f1a3b492be5d9b6bdab786a95e59966786dfd"
|
|
7753
|
-
};
|
|
7754
|
-
var AVAILABLE_DICTS = [
|
|
7755
|
-
{ id: "zh-CN", label: "\u7B80\u4F53\u4E2D\u6587", locale: "zh-CN" },
|
|
7756
|
-
{ id: "zh-TW", label: "\u7E41\u9AD4\u4E2D\u6587", locale: "zh-TW" }
|
|
7757
|
-
];
|
|
7758
|
-
function getDictFilePath(nlpDir, dictId) {
|
|
7759
|
-
return path13.join(nlpDir, `${dictId}.dict`);
|
|
7760
|
-
}
|
|
7761
|
-
function isDictDownloaded(nlpDir, dictId) {
|
|
7762
|
-
return fs20.existsSync(getDictFilePath(nlpDir, dictId));
|
|
7763
|
-
}
|
|
7764
|
-
function getDictList(nlpDir) {
|
|
7765
|
-
return AVAILABLE_DICTS.map((d) => {
|
|
7766
|
-
const filePath = getDictFilePath(nlpDir, d.id);
|
|
7767
|
-
const downloaded = fs20.existsSync(filePath);
|
|
7768
|
-
let fileSize;
|
|
7769
|
-
if (downloaded) {
|
|
7770
|
-
try {
|
|
7771
|
-
fileSize = fs20.statSync(filePath).size;
|
|
7772
|
-
} catch {
|
|
7773
|
-
}
|
|
7774
|
-
}
|
|
7775
|
-
return { ...d, downloaded, fileSize };
|
|
7776
|
-
});
|
|
7777
|
-
}
|
|
7778
|
-
function loadDictBuffer(nlpDir, dictId) {
|
|
7779
|
-
const filePath = getDictFilePath(nlpDir, dictId);
|
|
7780
|
-
if (!fs20.existsSync(filePath)) return null;
|
|
7781
|
-
try {
|
|
7782
|
-
return fs20.readFileSync(filePath);
|
|
7783
|
-
} catch (error) {
|
|
7784
|
-
console.error(`[NLP DictManager] Failed to read dict file: ${filePath}`, error);
|
|
7785
|
-
return null;
|
|
7786
|
-
}
|
|
7787
|
-
}
|
|
7788
|
-
async function downloadDict(nlpDir, dictId, onProgress) {
|
|
7789
|
-
if (!AVAILABLE_DICTS.find((d) => d.id === dictId)) {
|
|
7790
|
-
return { success: false, error: `Unknown dict: ${dictId}` };
|
|
7791
|
-
}
|
|
7792
|
-
fs20.mkdirSync(nlpDir, { recursive: true });
|
|
7793
|
-
const url = `${DICT_DOWNLOAD_URL_BASE}/${dictId}.dict`;
|
|
7794
|
-
const filePath = getDictFilePath(nlpDir, dictId);
|
|
7795
|
-
const tmpPath = filePath + ".tmp";
|
|
7796
|
-
try {
|
|
7797
|
-
const response = await fetch(url, { signal: AbortSignal.timeout(12e4) });
|
|
7798
|
-
if (!response.ok) throw new Error(`Download failed: HTTP ${response.status}`);
|
|
7799
|
-
const total = Number(response.headers.get("content-length") || 0);
|
|
7800
|
-
let buffer;
|
|
7801
|
-
if (!response.body) {
|
|
7802
|
-
buffer = Buffer.from(await response.arrayBuffer());
|
|
7803
|
-
} else {
|
|
7804
|
-
const reader = response.body.getReader();
|
|
7805
|
-
const chunks = [];
|
|
7806
|
-
let loaded = 0;
|
|
7807
|
-
while (true) {
|
|
7808
|
-
const { done, value } = await reader.read();
|
|
7809
|
-
if (done) break;
|
|
7810
|
-
const chunk = Buffer.from(value);
|
|
7811
|
-
chunks.push(chunk);
|
|
7812
|
-
loaded += chunk.length;
|
|
7813
|
-
if (total > 0 && onProgress) onProgress(Math.round(loaded / total * 100));
|
|
7814
|
-
}
|
|
7815
|
-
buffer = Buffer.concat(chunks);
|
|
7816
|
-
}
|
|
7817
|
-
const MIN_DICT_SIZE = 1e6;
|
|
7818
|
-
if (buffer.length < MIN_DICT_SIZE) {
|
|
7819
|
-
return { success: false, error: `Downloaded file is invalid (${buffer.length} bytes)` };
|
|
7820
|
-
}
|
|
7821
|
-
const head = buffer.subarray(0, 50).toString("utf-8").trim();
|
|
7822
|
-
if (head.startsWith("<!") || head.startsWith("<html")) {
|
|
7823
|
-
return { success: false, error: "Downloaded file is HTML, not a dictionary file" };
|
|
7824
|
-
}
|
|
7825
|
-
const expectedSha256 = DICT_SHA256[dictId];
|
|
7826
|
-
if (!expectedSha256) {
|
|
7827
|
-
return { success: false, error: `Missing SHA256 checksum for dict: ${dictId}` };
|
|
7828
|
-
}
|
|
7829
|
-
const actualSha256 = createHash("sha256").update(buffer).digest("hex");
|
|
7830
|
-
if (actualSha256 !== expectedSha256) {
|
|
7831
|
-
return { success: false, error: "Dictionary integrity check failed (SHA256 mismatch)" };
|
|
7832
|
-
}
|
|
7833
|
-
fs20.writeFileSync(tmpPath, buffer);
|
|
7834
|
-
if (fs20.existsSync(filePath)) fs20.unlinkSync(filePath);
|
|
7835
|
-
fs20.renameSync(tmpPath, filePath);
|
|
7836
|
-
clearJiebaInstance(dictId);
|
|
7837
|
-
console.log(`[NLP DictManager] Dict downloaded: ${dictId} (${fs20.statSync(filePath).size} bytes)`);
|
|
7838
|
-
return { success: true };
|
|
7839
|
-
} catch (error) {
|
|
7840
|
-
if (fs20.existsSync(tmpPath)) {
|
|
7841
|
-
try {
|
|
7842
|
-
fs20.unlinkSync(tmpPath);
|
|
7843
|
-
} catch {
|
|
7844
|
-
}
|
|
7845
|
-
}
|
|
7846
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
7847
|
-
console.error(`[NLP DictManager] Download failed for ${dictId}:`, msg);
|
|
7848
|
-
return { success: false, error: msg };
|
|
7849
|
-
}
|
|
7850
|
-
}
|
|
7851
|
-
function deleteDict(nlpDir, dictId) {
|
|
7852
|
-
const filePath = getDictFilePath(nlpDir, dictId);
|
|
7853
|
-
if (!fs20.existsSync(filePath)) return { success: true };
|
|
7854
|
-
try {
|
|
7855
|
-
fs20.unlinkSync(filePath);
|
|
7856
|
-
clearJiebaInstance(dictId);
|
|
7857
|
-
console.log(`[NLP DictManager] Dict deleted: ${dictId}`);
|
|
7858
|
-
return { success: true };
|
|
7859
|
-
} catch (error) {
|
|
7860
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
7861
|
-
console.error(`[NLP DictManager] Delete failed for ${dictId}:`, msg);
|
|
7862
|
-
return { success: false, error: msg };
|
|
7863
|
-
}
|
|
7864
|
-
}
|
|
7865
|
-
async function ensureDefaultDict(nlpDir) {
|
|
7866
|
-
if (isDictDownloaded(nlpDir, "zh-CN")) return;
|
|
7867
|
-
console.log("[NLP DictManager] zh-CN dict not found, starting background download...");
|
|
7868
|
-
const result = await downloadDict(nlpDir, "zh-CN");
|
|
7869
|
-
if (result.success) {
|
|
7870
|
-
console.log("[NLP DictManager] zh-CN dict auto-downloaded successfully");
|
|
7871
|
-
} else {
|
|
7872
|
-
console.warn("[NLP DictManager] zh-CN dict auto-download failed:", result.error);
|
|
7873
|
-
}
|
|
7874
|
-
}
|
|
7875
|
-
|
|
7876
|
-
// ../../packages/node-runtime/src/fts/index.ts
|
|
7877
|
-
var BATCH_SIZE = 5e3;
|
|
7878
|
-
function hasFtsTable(db) {
|
|
7879
|
-
try {
|
|
7880
|
-
const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='message_fts'").get();
|
|
7881
|
-
return !!row;
|
|
7882
|
-
} catch {
|
|
7883
|
-
return false;
|
|
7884
|
-
}
|
|
7885
|
-
}
|
|
7886
|
-
function createFtsTable(db) {
|
|
7887
|
-
db.exec(FTS_TABLE_SCHEMA);
|
|
7888
|
-
}
|
|
7889
|
-
function buildFtsIndex(db) {
|
|
7890
|
-
createFtsTable(db);
|
|
7891
|
-
const insertFts = db.prepare("INSERT INTO message_fts(rowid, content) VALUES (?, ?)");
|
|
7892
|
-
let indexed = 0;
|
|
7893
|
-
let lastId = null;
|
|
7894
|
-
const selectFirstBatch = db.prepare(
|
|
7895
|
-
`SELECT id, content FROM message
|
|
7896
|
-
WHERE type = 0
|
|
7897
|
-
AND content IS NOT NULL
|
|
7898
|
-
AND content != ''
|
|
7899
|
-
ORDER BY id ASC
|
|
7900
|
-
LIMIT ?`
|
|
7901
|
-
);
|
|
7902
|
-
const selectNextBatch = db.prepare(
|
|
7903
|
-
`SELECT id, content FROM message
|
|
7904
|
-
WHERE type = 0
|
|
7905
|
-
AND content IS NOT NULL
|
|
7906
|
-
AND content != ''
|
|
7907
|
-
AND id > ?
|
|
7908
|
-
ORDER BY id ASC
|
|
7909
|
-
LIMIT ?`
|
|
7910
|
-
);
|
|
7911
|
-
while (true) {
|
|
7912
|
-
const rows = lastId === null ? selectFirstBatch.all(BATCH_SIZE) : selectNextBatch.all(lastId, BATCH_SIZE);
|
|
7913
|
-
if (rows.length === 0) break;
|
|
7914
|
-
db.transaction(() => {
|
|
7915
|
-
for (const row of rows) {
|
|
7916
|
-
const tokens = tokenizeForFts(row.content);
|
|
7917
|
-
if (tokens) {
|
|
7918
|
-
insertFts.run(row.id, tokens);
|
|
7919
|
-
}
|
|
7920
|
-
}
|
|
7921
|
-
});
|
|
7922
|
-
indexed += rows.length;
|
|
7923
|
-
lastId = rows[rows.length - 1].id;
|
|
7924
|
-
}
|
|
7925
|
-
return { indexed };
|
|
7926
|
-
}
|
|
7927
|
-
function rebuildFtsIndex(db) {
|
|
7928
|
-
if (hasFtsTable(db)) {
|
|
7929
|
-
db.exec("DROP TABLE message_fts");
|
|
7930
|
-
}
|
|
7931
|
-
return buildFtsIndex(db);
|
|
7932
|
-
}
|
|
7933
|
-
function insertFtsEntries(db, entries) {
|
|
7934
|
-
if (!hasFtsTable(db)) return;
|
|
7935
|
-
const insertFts = db.prepare("INSERT INTO message_fts(rowid, content) VALUES (?, ?)");
|
|
7936
|
-
db.transaction(() => {
|
|
7937
|
-
for (const entry of entries) {
|
|
7938
|
-
if (entry.content) {
|
|
7939
|
-
const tokens = tokenizeForFts(entry.content);
|
|
7940
|
-
if (tokens) {
|
|
7941
|
-
insertFts.run(entry.id, tokens);
|
|
7942
|
-
}
|
|
7943
|
-
}
|
|
7944
|
-
}
|
|
7945
|
-
});
|
|
7946
|
-
}
|
|
7947
|
-
function searchByFts(db, keywords, limit = 1e3, offset = 0) {
|
|
7948
|
-
if (keywords.length === 0) return { rowids: [], total: 0 };
|
|
7949
|
-
const matchQuery = tokenizeQueryForFts(keywords);
|
|
7950
|
-
if (!matchQuery) return { rowids: [], total: 0 };
|
|
7951
|
-
try {
|
|
7952
|
-
const countRow = db.prepare("SELECT COUNT(*) as total FROM message_fts WHERE content MATCH ?").get(matchQuery);
|
|
7953
|
-
const total = countRow?.total ?? 0;
|
|
7954
|
-
const rows = db.prepare(`SELECT rowid FROM message_fts WHERE content MATCH ? ORDER BY rank LIMIT ? OFFSET ?`).all(matchQuery, limit, offset);
|
|
7955
|
-
return {
|
|
7956
|
-
rowids: rows.map((r) => r.rowid),
|
|
7957
|
-
total
|
|
7958
|
-
};
|
|
7959
|
-
} catch (error) {
|
|
7960
|
-
console.error("[FTS] Search failed, query:", matchQuery, error);
|
|
7961
|
-
return { rowids: [], total: 0 };
|
|
7962
|
-
}
|
|
7963
|
-
}
|
|
7964
|
-
|
|
7965
7688
|
// ../../packages/node-runtime/src/import/message-batch-inserter.ts
|
|
7966
7689
|
var MESSAGE_COLUMN_COUNT = 8;
|
|
7967
7690
|
var SQLITE_LEGACY_VARIABLE_LIMIT = 999;
|
|
@@ -8016,10 +7739,24 @@ function toParams(row) {
|
|
|
8016
7739
|
}
|
|
8017
7740
|
|
|
8018
7741
|
// ../../packages/node-runtime/src/import/streaming-importer.ts
|
|
8019
|
-
import * as
|
|
7742
|
+
import * as fs20 from "fs";
|
|
8020
7743
|
import { performance } from "perf_hooks";
|
|
8021
7744
|
|
|
8022
7745
|
// ../../packages/node-runtime/src/import/message-deduplicator.ts
|
|
7746
|
+
var SCOPED_PLATFORM_MESSAGE_ID_PATTERN = /^(__chatlab_message_scope__\d+__)(.*)$/;
|
|
7747
|
+
function parsePlatformMessageId(id) {
|
|
7748
|
+
const scoped = SCOPED_PLATFORM_MESSAGE_ID_PATTERN.exec(id);
|
|
7749
|
+
if (!scoped) return { rawId: id };
|
|
7750
|
+
try {
|
|
7751
|
+
return { rawId: decodeURIComponent(scoped[2]), scope: scoped[1] };
|
|
7752
|
+
} catch {
|
|
7753
|
+
return { rawId: id };
|
|
7754
|
+
}
|
|
7755
|
+
}
|
|
7756
|
+
function applyPlatformMessageIdScope(id, scope) {
|
|
7757
|
+
if (!id || !scope) return id;
|
|
7758
|
+
return `${scope}${encodeURIComponent(parsePlatformMessageId(id).rawId)}`;
|
|
7759
|
+
}
|
|
8023
7760
|
function generateFallbackMessageKey(message) {
|
|
8024
7761
|
const contentKey = generateMessageKey(message.timestamp, message.senderPlatformId, message.content);
|
|
8025
7762
|
return JSON.stringify([contentKey, message.type, message.replyToMessageId || null]);
|
|
@@ -8148,7 +7885,6 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8148
7885
|
messageWriteMs: 0,
|
|
8149
7886
|
nicknameHistoryMs: 0,
|
|
8150
7887
|
indexCreationMs: 0,
|
|
8151
|
-
ftsMs: 0,
|
|
8152
7888
|
checkpointMs: 0,
|
|
8153
7889
|
sessionIndexMs: 0,
|
|
8154
7890
|
postImportHookMs: 0,
|
|
@@ -8551,16 +8287,6 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8551
8287
|
timings.indexCreationMs = elapsedMs(indexCreationStartedAt, now);
|
|
8552
8288
|
sampleRss();
|
|
8553
8289
|
logger?.perf("Indexes created", totalMessageCount);
|
|
8554
|
-
const ftsStartedAt = now();
|
|
8555
|
-
try {
|
|
8556
|
-
buildFtsIndex(db);
|
|
8557
|
-
logger?.perf("FTS index built", totalMessageCount);
|
|
8558
|
-
} catch (ftsError) {
|
|
8559
|
-
logger?.error("FTS index build failed (non-fatal)", ftsError instanceof Error ? ftsError : void 0);
|
|
8560
|
-
} finally {
|
|
8561
|
-
timings.ftsMs = elapsedMs(ftsStartedAt, now);
|
|
8562
|
-
sampleRss();
|
|
8563
|
-
}
|
|
8564
8290
|
onProgress({
|
|
8565
8291
|
stage: "indexing",
|
|
8566
8292
|
bytesRead: 0,
|
|
@@ -8598,7 +8324,7 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8598
8324
|
}
|
|
8599
8325
|
}
|
|
8600
8326
|
logger?.perfDetail(
|
|
8601
|
-
`[Stages] parser=${timings.parserMs.toFixed(1)}ms | message-write=${timings.messageWriteMs.toFixed(1)}ms | indexes=${timings.indexCreationMs.toFixed(1)}ms |
|
|
8327
|
+
`[Stages] parser=${timings.parserMs.toFixed(1)}ms | message-write=${timings.messageWriteMs.toFixed(1)}ms | indexes=${timings.indexCreationMs.toFixed(1)}ms | session-index=${timings.sessionIndexMs.toFixed(1)}ms | hook=${timings.postImportHookMs.toFixed(1)}ms`
|
|
8602
8328
|
);
|
|
8603
8329
|
logger?.perf("Import completed", totalMessageCount);
|
|
8604
8330
|
logger?.info(`=== Parser Callback Stats ===`);
|
|
@@ -8735,7 +8461,7 @@ async function streamParseFileInfo(filePath, deps) {
|
|
|
8735
8461
|
if (!formatFeature) {
|
|
8736
8462
|
throw new Error("Unrecognized file format");
|
|
8737
8463
|
}
|
|
8738
|
-
const fileSize =
|
|
8464
|
+
const fileSize = fs20.statSync(filePath).size;
|
|
8739
8465
|
deps.onProgress({
|
|
8740
8466
|
stage: "parsing",
|
|
8741
8467
|
bytesRead: 0,
|
|
@@ -8752,8 +8478,10 @@ async function streamParseFileInfo(filePath, deps) {
|
|
|
8752
8478
|
"INSERT OR IGNORE INTO member (platform_id, account_name, group_nickname, avatar) VALUES (?, ?, ?, ?)"
|
|
8753
8479
|
);
|
|
8754
8480
|
const insertMessage = db.prepare(
|
|
8755
|
-
`INSERT INTO message (
|
|
8756
|
-
|
|
8481
|
+
`INSERT INTO message (
|
|
8482
|
+
platform_message_id, sender_platform_id, sender_account_name, sender_group_nickname,
|
|
8483
|
+
timestamp, type, content, reply_to_message_id
|
|
8484
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
8757
8485
|
);
|
|
8758
8486
|
let meta = { name: "Unknown", platform: formatFeature.platform, type: 0 };
|
|
8759
8487
|
const memberSet = /* @__PURE__ */ new Set();
|
|
@@ -8795,12 +8523,14 @@ async function streamParseFileInfo(filePath, deps) {
|
|
|
8795
8523
|
insertMember.run(msg.senderPlatformId, msg.senderAccountName || null, msg.senderGroupNickname || null, null);
|
|
8796
8524
|
}
|
|
8797
8525
|
insertMessage.run(
|
|
8526
|
+
msg.platformMessageId || null,
|
|
8798
8527
|
msg.senderPlatformId,
|
|
8799
8528
|
msg.senderAccountName || null,
|
|
8800
8529
|
msg.senderGroupNickname || null,
|
|
8801
8530
|
msg.timestamp,
|
|
8802
8531
|
msg.type,
|
|
8803
|
-
msg.content || null
|
|
8532
|
+
msg.content || null,
|
|
8533
|
+
msg.replyToMessageId || null
|
|
8804
8534
|
);
|
|
8805
8535
|
messageCount++;
|
|
8806
8536
|
}
|
|
@@ -8824,7 +8554,7 @@ async function streamParseFileInfo(filePath, deps) {
|
|
|
8824
8554
|
}
|
|
8825
8555
|
db.close();
|
|
8826
8556
|
try {
|
|
8827
|
-
if (
|
|
8557
|
+
if (fs20.existsSync(tempDbPath)) fs20.unlinkSync(tempDbPath);
|
|
8828
8558
|
} catch {
|
|
8829
8559
|
}
|
|
8830
8560
|
throw error;
|
|
@@ -8886,40 +8616,112 @@ function countHistory(tracker) {
|
|
|
8886
8616
|
}
|
|
8887
8617
|
|
|
8888
8618
|
// ../../packages/node-runtime/src/import/incremental-importer.ts
|
|
8889
|
-
|
|
8890
|
-
|
|
8891
|
-
|
|
8892
|
-
|
|
8893
|
-
|
|
8894
|
-
|
|
8895
|
-
|
|
8896
|
-
}
|
|
8897
|
-
|
|
8898
|
-
|
|
8899
|
-
|
|
8900
|
-
|
|
8901
|
-
|
|
8902
|
-
|
|
8903
|
-
|
|
8904
|
-
|
|
8905
|
-
|
|
8906
|
-
|
|
8907
|
-
|
|
8908
|
-
|
|
8909
|
-
|
|
8910
|
-
|
|
8911
|
-
|
|
8912
|
-
|
|
8619
|
+
var PLATFORM_ID_QUERY_CHUNK_SIZE = 500;
|
|
8620
|
+
function applyMessageIdScope(message, scope) {
|
|
8621
|
+
if (!scope) return message;
|
|
8622
|
+
return {
|
|
8623
|
+
...message,
|
|
8624
|
+
platformMessageId: applyPlatformMessageIdScope(message.platformMessageId, scope),
|
|
8625
|
+
replyToMessageId: applyPlatformMessageIdScope(message.replyToMessageId, scope)
|
|
8626
|
+
};
|
|
8627
|
+
}
|
|
8628
|
+
function createPlatformIdMapper(mappings) {
|
|
8629
|
+
const mapping = new Map(mappings?.map(({ sourceId, targetId }) => [sourceId, targetId]));
|
|
8630
|
+
return (platformId) => mapping.get(platformId) ?? platformId;
|
|
8631
|
+
}
|
|
8632
|
+
function applySenderPlatformIdMapping(message, mapPlatformId) {
|
|
8633
|
+
const senderPlatformId = mapPlatformId(message.senderPlatformId);
|
|
8634
|
+
return senderPlatformId === message.senderPlatformId ? message : { ...message, senderPlatformId };
|
|
8635
|
+
}
|
|
8636
|
+
function createExistingMessageLookup(db) {
|
|
8637
|
+
const maxMessageId = db.prepare("SELECT MAX(id) AS max_id FROM message").get()?.max_id ?? 0;
|
|
8638
|
+
const hasFallbackOnlyMessages = Boolean(
|
|
8639
|
+
db.prepare(
|
|
8640
|
+
`SELECT 1 FROM message
|
|
8641
|
+
WHERE id <= ? AND (platform_message_id IS NULL OR platform_message_id = '')
|
|
8642
|
+
LIMIT 1`
|
|
8643
|
+
).get(maxMessageId)
|
|
8644
|
+
);
|
|
8645
|
+
return {
|
|
8646
|
+
db,
|
|
8647
|
+
maxMessageId,
|
|
8648
|
+
hasFallbackOnlyMessages,
|
|
8649
|
+
consumedFallbackOnlyKeys: /* @__PURE__ */ new Set(),
|
|
8650
|
+
bridgedPlatformMessageIds: /* @__PURE__ */ new Set(),
|
|
8651
|
+
candidateStatement: db.prepare(
|
|
8652
|
+
`SELECT msg.type, msg.content, msg.reply_to_message_id, msg.platform_message_id
|
|
8653
|
+
FROM member m
|
|
8654
|
+
JOIN message msg ON msg.sender_id = m.id
|
|
8655
|
+
WHERE m.platform_id = ? AND msg.ts = ? AND msg.id <= ?`
|
|
8656
|
+
),
|
|
8657
|
+
platformIdStatements: /* @__PURE__ */ new Map()
|
|
8658
|
+
};
|
|
8659
|
+
}
|
|
8660
|
+
function findExistingPlatformMessageIds(lookup, platformMessageIds) {
|
|
8661
|
+
const uniqueIds = [...new Set([...platformMessageIds].filter((id) => Boolean(id)))];
|
|
8662
|
+
const existingIds = /* @__PURE__ */ new Set();
|
|
8663
|
+
for (let start = 0; start < uniqueIds.length; start += PLATFORM_ID_QUERY_CHUNK_SIZE) {
|
|
8664
|
+
const chunk = uniqueIds.slice(start, start + PLATFORM_ID_QUERY_CHUNK_SIZE);
|
|
8665
|
+
let statement = lookup.platformIdStatements.get(chunk.length);
|
|
8666
|
+
if (!statement) {
|
|
8667
|
+
const placeholders = chunk.map(() => "?").join(", ");
|
|
8668
|
+
statement = lookup.db.prepare(
|
|
8669
|
+
`SELECT platform_message_id
|
|
8670
|
+
FROM message
|
|
8671
|
+
WHERE id <= ? AND platform_message_id IN (${placeholders})`
|
|
8672
|
+
);
|
|
8673
|
+
lookup.platformIdStatements.set(chunk.length, statement);
|
|
8674
|
+
}
|
|
8675
|
+
const rows = statement.all(lookup.maxMessageId, ...chunk);
|
|
8676
|
+
for (const row of rows) existingIds.add(row.platform_message_id);
|
|
8913
8677
|
}
|
|
8914
|
-
return
|
|
8678
|
+
return existingIds;
|
|
8915
8679
|
}
|
|
8916
|
-
function
|
|
8917
|
-
|
|
8918
|
-
|
|
8919
|
-
|
|
8920
|
-
|
|
8680
|
+
function getExistingCandidates(lookup, message, cache) {
|
|
8681
|
+
const cacheKey = JSON.stringify([message.senderPlatformId, message.timestamp]);
|
|
8682
|
+
const cached = cache.get(cacheKey);
|
|
8683
|
+
if (cached) return cached;
|
|
8684
|
+
const rows = lookup.candidateStatement.all(
|
|
8685
|
+
message.senderPlatformId,
|
|
8686
|
+
message.timestamp,
|
|
8687
|
+
lookup.maxMessageId
|
|
8688
|
+
);
|
|
8689
|
+
cache.set(cacheKey, rows);
|
|
8690
|
+
return rows;
|
|
8691
|
+
}
|
|
8692
|
+
function candidateFallbackKey(message, candidate) {
|
|
8693
|
+
return generateFallbackMessageKey({
|
|
8694
|
+
timestamp: message.timestamp,
|
|
8695
|
+
senderPlatformId: message.senderPlatformId,
|
|
8696
|
+
type: candidate.type,
|
|
8697
|
+
content: candidate.content,
|
|
8698
|
+
replyToMessageId: candidate.reply_to_message_id ?? void 0
|
|
8921
8699
|
});
|
|
8922
8700
|
}
|
|
8701
|
+
function isExistingMessageDuplicate(lookup, message, existingBatchPlatformIds, candidateCache) {
|
|
8702
|
+
if (message.platformMessageId && (existingBatchPlatformIds.has(message.platformMessageId) || lookup.bridgedPlatformMessageIds.has(message.platformMessageId))) {
|
|
8703
|
+
return true;
|
|
8704
|
+
}
|
|
8705
|
+
const fallbackKey = generateFallbackMessageKey(message);
|
|
8706
|
+
if (message.platformMessageId) {
|
|
8707
|
+
if (!lookup.hasFallbackOnlyMessages || lookup.consumedFallbackOnlyKeys.has(fallbackKey)) return false;
|
|
8708
|
+
const matchesFallbackOnly = getExistingCandidates(lookup, message, candidateCache).some(
|
|
8709
|
+
(candidate) => !candidate.platform_message_id && candidateFallbackKey(message, candidate) === fallbackKey
|
|
8710
|
+
);
|
|
8711
|
+
if (matchesFallbackOnly) {
|
|
8712
|
+
lookup.consumedFallbackOnlyKeys.add(fallbackKey);
|
|
8713
|
+
lookup.bridgedPlatformMessageIds.add(message.platformMessageId);
|
|
8714
|
+
}
|
|
8715
|
+
return matchesFallbackOnly;
|
|
8716
|
+
}
|
|
8717
|
+
return getExistingCandidates(lookup, message, candidateCache).some(
|
|
8718
|
+
(candidate) => candidateFallbackKey(message, candidate) === fallbackKey
|
|
8719
|
+
);
|
|
8720
|
+
}
|
|
8721
|
+
function isDuplicate(lookup, incomingState, message, existingBatchPlatformIds, candidateCache) {
|
|
8722
|
+
if (isExistingMessageDuplicate(lookup, message, existingBatchPlatformIds, candidateCache)) return true;
|
|
8723
|
+
return registerMessageAndCheckDuplicate(message, incomingState);
|
|
8724
|
+
}
|
|
8923
8725
|
function normalizeImportTimestamp(timestamp) {
|
|
8924
8726
|
const value = typeof timestamp === "string" && timestamp.trim() !== "" ? Number(timestamp) : timestamp;
|
|
8925
8727
|
return typeof value === "number" && value > 0 && Number.isFinite(value) ? value : null;
|
|
@@ -8935,39 +8737,59 @@ async function analyzeIncrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8935
8737
|
} catch {
|
|
8936
8738
|
return { error: "error.session_not_found", newMessageCount: 0, duplicateCount: 0, totalInFile: 0 };
|
|
8937
8739
|
}
|
|
8938
|
-
const { existingPlatformMsgIds, existingKeys, existingFallbackOnlyKeys } = loadExistingDedup(db);
|
|
8939
|
-
db.close();
|
|
8940
8740
|
let totalInFile = 0;
|
|
8941
8741
|
let newMessageCount = 0;
|
|
8942
8742
|
let duplicateCount = 0;
|
|
8943
8743
|
let platform = formatFeature.platform;
|
|
8944
|
-
|
|
8945
|
-
|
|
8946
|
-
|
|
8947
|
-
|
|
8948
|
-
|
|
8949
|
-
|
|
8950
|
-
|
|
8951
|
-
|
|
8952
|
-
|
|
8953
|
-
|
|
8954
|
-
|
|
8955
|
-
|
|
8956
|
-
|
|
8957
|
-
|
|
8958
|
-
|
|
8959
|
-
|
|
8960
|
-
|
|
8961
|
-
|
|
8962
|
-
|
|
8963
|
-
|
|
8964
|
-
|
|
8744
|
+
const mapPlatformId = createPlatformIdMapper(options2?.senderPlatformIdMappings);
|
|
8745
|
+
try {
|
|
8746
|
+
const existingLookup = createExistingMessageLookup(db);
|
|
8747
|
+
const incomingState = createMessageDedupState();
|
|
8748
|
+
await streamParseFile(
|
|
8749
|
+
filePath,
|
|
8750
|
+
{
|
|
8751
|
+
formatOptions: options2?.chatIndex === void 0 ? void 0 : { chatIndex: options2.chatIndex },
|
|
8752
|
+
onMeta: (meta) => {
|
|
8753
|
+
platform = meta.platform;
|
|
8754
|
+
},
|
|
8755
|
+
onMembers: () => {
|
|
8756
|
+
},
|
|
8757
|
+
onProgress: (progress) => {
|
|
8758
|
+
deps.onProgress(progress);
|
|
8759
|
+
},
|
|
8760
|
+
onLog: deps.onParserLog,
|
|
8761
|
+
onMessageBatch: (batch) => {
|
|
8762
|
+
const scopedBatch = batch.map(
|
|
8763
|
+
(message) => applyMessageIdScope(applySenderPlatformIdMapping(message, mapPlatformId), options2?.platformMessageIdScope)
|
|
8764
|
+
);
|
|
8765
|
+
const existingBatchPlatformIds = findExistingPlatformMessageIds(
|
|
8766
|
+
existingLookup,
|
|
8767
|
+
scopedBatch.map((message) => message.platformMessageId)
|
|
8768
|
+
);
|
|
8769
|
+
const candidateCache = /* @__PURE__ */ new Map();
|
|
8770
|
+
for (const msg of scopedBatch) {
|
|
8771
|
+
totalInFile++;
|
|
8772
|
+
const timestamp = normalizeImportTimestamp(msg.timestamp);
|
|
8773
|
+
if (timestamp === null) continue;
|
|
8774
|
+
if (isDuplicate(
|
|
8775
|
+
existingLookup,
|
|
8776
|
+
incomingState,
|
|
8777
|
+
{ ...msg, timestamp },
|
|
8778
|
+
existingBatchPlatformIds,
|
|
8779
|
+
candidateCache
|
|
8780
|
+
)) {
|
|
8781
|
+
duplicateCount++;
|
|
8782
|
+
} else {
|
|
8783
|
+
newMessageCount++;
|
|
8784
|
+
}
|
|
8965
8785
|
}
|
|
8966
8786
|
}
|
|
8967
|
-
}
|
|
8968
|
-
|
|
8969
|
-
|
|
8970
|
-
|
|
8787
|
+
},
|
|
8788
|
+
options2?.formatId
|
|
8789
|
+
);
|
|
8790
|
+
} finally {
|
|
8791
|
+
db.close();
|
|
8792
|
+
}
|
|
8971
8793
|
return { newMessageCount, duplicateCount, totalInFile, platform };
|
|
8972
8794
|
}
|
|
8973
8795
|
async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
@@ -8983,6 +8805,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8983
8805
|
}
|
|
8984
8806
|
const metaUpdateMode = options2?.metaUpdateMode ?? "patch";
|
|
8985
8807
|
const memberUpdateMode = options2?.memberUpdateMode ?? "upsert";
|
|
8808
|
+
const mapPlatformId = createPlatformIdMapper(options2?.senderPlatformIdMappings);
|
|
8986
8809
|
try {
|
|
8987
8810
|
let trackError2 = function(index, reason, detail) {
|
|
8988
8811
|
errorCount++;
|
|
@@ -8992,7 +8815,8 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8992
8815
|
}
|
|
8993
8816
|
};
|
|
8994
8817
|
var trackError = trackError2;
|
|
8995
|
-
const
|
|
8818
|
+
const existingLookup = createExistingMessageLookup(db);
|
|
8819
|
+
const incomingState = createMessageDedupState();
|
|
8996
8820
|
const memberIdMap = /* @__PURE__ */ new Map();
|
|
8997
8821
|
const existingMembers = db.prepare("SELECT id, platform_id FROM member").all();
|
|
8998
8822
|
for (const m of existingMembers) {
|
|
@@ -9038,8 +8862,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9038
8862
|
const errorReasonCounts = {};
|
|
9039
8863
|
const errorSamples = [];
|
|
9040
8864
|
const MAX_ERROR_SAMPLES = 5;
|
|
9041
|
-
const
|
|
9042
|
-
const newFtsEntries = [];
|
|
8865
|
+
const BATCH_SIZE = 5e3;
|
|
9043
8866
|
await streamParseFile(
|
|
9044
8867
|
filePath,
|
|
9045
8868
|
{
|
|
@@ -9050,7 +8873,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9050
8873
|
meta.name || "",
|
|
9051
8874
|
meta.groupId || "",
|
|
9052
8875
|
meta.groupAvatar || "",
|
|
9053
|
-
meta.ownerId
|
|
8876
|
+
meta.ownerId ? mapPlatformId(meta.ownerId) : "",
|
|
9054
8877
|
Math.floor(Date.now() / 1e3)
|
|
9055
8878
|
);
|
|
9056
8879
|
metaUpdated = true;
|
|
@@ -9058,11 +8881,12 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9058
8881
|
onMembers: (members) => {
|
|
9059
8882
|
if (memberUpdateMode === "none") return;
|
|
9060
8883
|
for (const m of members) {
|
|
9061
|
-
const
|
|
9062
|
-
const
|
|
9063
|
-
const
|
|
8884
|
+
const platformId = mapPlatformId(m.platformId);
|
|
8885
|
+
const existed = memberIdMap.has(platformId);
|
|
8886
|
+
const accountName = normalizeSystemMemberName(formatFeature.id, platformId, m.accountName);
|
|
8887
|
+
const groupNickname = normalizeSystemMemberName(formatFeature.id, platformId, m.groupNickname);
|
|
9064
8888
|
upsertMember.run(
|
|
9065
|
-
|
|
8889
|
+
platformId,
|
|
9066
8890
|
accountName || null,
|
|
9067
8891
|
groupNickname || null,
|
|
9068
8892
|
m.aliases ? JSON.stringify(m.aliases) : "[]",
|
|
@@ -9070,8 +8894,8 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9070
8894
|
m.roles ? JSON.stringify(m.roles) : "[]"
|
|
9071
8895
|
);
|
|
9072
8896
|
if (!existed) {
|
|
9073
|
-
const row = getMemberId.get(
|
|
9074
|
-
if (row) memberIdMap.set(
|
|
8897
|
+
const row = getMemberId.get(platformId);
|
|
8898
|
+
if (row) memberIdMap.set(platformId, row.id);
|
|
9075
8899
|
membersAdded++;
|
|
9076
8900
|
} else {
|
|
9077
8901
|
membersUpdated++;
|
|
@@ -9081,8 +8905,17 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9081
8905
|
onProgress: (progress) => {
|
|
9082
8906
|
deps.onProgress(progress);
|
|
9083
8907
|
},
|
|
8908
|
+
onLog: deps.onParserLog,
|
|
9084
8909
|
onMessageBatch: (batch) => {
|
|
9085
|
-
|
|
8910
|
+
const scopedBatch = batch.map(
|
|
8911
|
+
(message) => applyMessageIdScope(applySenderPlatformIdMapping(message, mapPlatformId), options2?.platformMessageIdScope)
|
|
8912
|
+
);
|
|
8913
|
+
const existingBatchPlatformIds = findExistingPlatformMessageIds(
|
|
8914
|
+
existingLookup,
|
|
8915
|
+
scopedBatch.map((message) => message.platformMessageId)
|
|
8916
|
+
);
|
|
8917
|
+
const candidateCache = /* @__PURE__ */ new Map();
|
|
8918
|
+
for (const msg of scopedBatch) {
|
|
9086
8919
|
processedCount++;
|
|
9087
8920
|
if (!msg.senderPlatformId) {
|
|
9088
8921
|
trackError2(processedCount, "MISSING_SENDER", "sender field is empty");
|
|
@@ -9097,7 +8930,13 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9097
8930
|
trackError2(processedCount, "INVALID_TIMESTAMP", `timestamp value: ${msg.timestamp}`);
|
|
9098
8931
|
continue;
|
|
9099
8932
|
}
|
|
9100
|
-
if (isDuplicate(
|
|
8933
|
+
if (isDuplicate(
|
|
8934
|
+
existingLookup,
|
|
8935
|
+
incomingState,
|
|
8936
|
+
{ ...msg, timestamp },
|
|
8937
|
+
existingBatchPlatformIds,
|
|
8938
|
+
candidateCache
|
|
8939
|
+
)) {
|
|
9101
8940
|
duplicateCount++;
|
|
9102
8941
|
continue;
|
|
9103
8942
|
}
|
|
@@ -9127,7 +8966,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9127
8966
|
}
|
|
9128
8967
|
}
|
|
9129
8968
|
if (!memberId) continue;
|
|
9130
|
-
|
|
8969
|
+
insertMessage.run(
|
|
9131
8970
|
memberId,
|
|
9132
8971
|
senderAccountName || null,
|
|
9133
8972
|
senderGroupNickname || null,
|
|
@@ -9137,14 +8976,10 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9137
8976
|
msg.replyToMessageId || null,
|
|
9138
8977
|
msg.platformMessageId || null
|
|
9139
8978
|
);
|
|
9140
|
-
newFtsEntries.push({
|
|
9141
|
-
id: Number(msgResult.lastInsertRowid ?? 0),
|
|
9142
|
-
content: msg.content || null
|
|
9143
|
-
});
|
|
9144
8979
|
if (timestamp < minWrittenTs) minWrittenTs = timestamp;
|
|
9145
8980
|
newMessageCount++;
|
|
9146
8981
|
}
|
|
9147
|
-
if (processedCount %
|
|
8982
|
+
if (processedCount % BATCH_SIZE === 0) {
|
|
9148
8983
|
deps.onProgress({
|
|
9149
8984
|
stage: "saving",
|
|
9150
8985
|
bytesRead: 0,
|
|
@@ -9162,12 +8997,6 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9162
8997
|
if (!metaUpdated) {
|
|
9163
8998
|
db.prepare("UPDATE meta SET imported_at = ?").run(Math.floor(Date.now() / 1e3));
|
|
9164
8999
|
}
|
|
9165
|
-
if (newFtsEntries.length > 0 && hasFtsTable(db)) {
|
|
9166
|
-
try {
|
|
9167
|
-
insertFtsEntries(db, newFtsEntries);
|
|
9168
|
-
} catch {
|
|
9169
|
-
}
|
|
9170
|
-
}
|
|
9171
9000
|
if (newMessageCount > 0) {
|
|
9172
9001
|
try {
|
|
9173
9002
|
const sessionGapThreshold = getSessionIndexStats(db).gapThreshold;
|
|
@@ -9241,23 +9070,112 @@ function isValidImportSessionId(sessionId) {
|
|
|
9241
9070
|
}
|
|
9242
9071
|
|
|
9243
9072
|
// ../../packages/node-runtime/src/import/auto-import-matcher.ts
|
|
9244
|
-
import { createHash
|
|
9073
|
+
import { createHash } from "crypto";
|
|
9245
9074
|
var MATCH_WINDOW_SIZE = 5;
|
|
9246
|
-
function buildPrivateIdentity(ownerId, memberIds) {
|
|
9247
|
-
|
|
9248
|
-
const
|
|
9249
|
-
|
|
9075
|
+
function buildPrivateIdentity(platform, ownerId, memberIds) {
|
|
9076
|
+
const sortedIds = [...new Set(memberIds)].filter((id) => id && id.toLowerCase() !== "system").sort();
|
|
9077
|
+
const validOwnerId = ownerId && ownerId.toLowerCase() !== "system" ? ownerId : null;
|
|
9078
|
+
if (validOwnerId) {
|
|
9079
|
+
return sortedIds.length > 0 ? `owner\0${validOwnerId}\0members\0${sortedIds.join("\0")}` : null;
|
|
9080
|
+
}
|
|
9081
|
+
return platform === "qq" && sortedIds.length >= 2 ? `members\0${sortedIds.join("\0")}` : null;
|
|
9250
9082
|
}
|
|
9251
9083
|
function isBusinessMessage(type, content) {
|
|
9252
9084
|
return type !== 80 /* SYSTEM */ && type !== 81 /* RECALL */ && Boolean(content?.trim());
|
|
9253
9085
|
}
|
|
9254
|
-
function addMessageWindow(recentKeys, windows, message) {
|
|
9086
|
+
function addMessageWindow(recentKeys, windows, recentPlatformIdKeys, platformIdWindows, platformIdSenders, recentExactPlatformIdKeys, exactPlatformIdWindows, exactPlatformIdSenders, message) {
|
|
9255
9087
|
if (!message.senderPlatformId || !isBusinessMessage(message.type, message.content)) return;
|
|
9256
9088
|
const timestamp = normalizeImportTimestamp(message.timestamp);
|
|
9257
9089
|
if (timestamp === null) return;
|
|
9258
9090
|
recentKeys.push(generateMessageKey(timestamp, message.senderPlatformId, message.content));
|
|
9259
9091
|
if (recentKeys.length > MATCH_WINDOW_SIZE) recentKeys.shift();
|
|
9260
9092
|
if (recentKeys.length === MATCH_WINDOW_SIZE) windows.add(recentKeys.join(""));
|
|
9093
|
+
const platformIdKey = generatePlatformMessageKey(message, timestamp);
|
|
9094
|
+
const exactPlatformIdKey = generatePlatformMessageKey(message, timestamp, false);
|
|
9095
|
+
if (!platformIdKey) {
|
|
9096
|
+
recentPlatformIdKeys.length = 0;
|
|
9097
|
+
recentExactPlatformIdKeys.length = 0;
|
|
9098
|
+
return;
|
|
9099
|
+
}
|
|
9100
|
+
recordPlatformMessageSender(platformIdSenders, platformIdKey, message.senderPlatformId);
|
|
9101
|
+
recordPlatformMessageSender(exactPlatformIdSenders, exactPlatformIdKey ?? platformIdKey, message.senderPlatformId);
|
|
9102
|
+
recentPlatformIdKeys.push(platformIdKey);
|
|
9103
|
+
if (recentPlatformIdKeys.length > MATCH_WINDOW_SIZE) recentPlatformIdKeys.shift();
|
|
9104
|
+
if (recentPlatformIdKeys.length === MATCH_WINDOW_SIZE) platformIdWindows.add(recentPlatformIdKeys.join(""));
|
|
9105
|
+
recentExactPlatformIdKeys.push(exactPlatformIdKey ?? platformIdKey);
|
|
9106
|
+
if (recentExactPlatformIdKeys.length > MATCH_WINDOW_SIZE) recentExactPlatformIdKeys.shift();
|
|
9107
|
+
if (recentExactPlatformIdKeys.length === MATCH_WINDOW_SIZE) {
|
|
9108
|
+
exactPlatformIdWindows.add(recentExactPlatformIdKeys.join(""));
|
|
9109
|
+
}
|
|
9110
|
+
}
|
|
9111
|
+
function recordPlatformMessageSender(senders, messageKey, senderPlatformId) {
|
|
9112
|
+
if (!senders.has(messageKey)) {
|
|
9113
|
+
senders.set(messageKey, senderPlatformId);
|
|
9114
|
+
} else if (senders.get(messageKey) !== senderPlatformId) {
|
|
9115
|
+
senders.set(messageKey, null);
|
|
9116
|
+
}
|
|
9117
|
+
}
|
|
9118
|
+
function isUsablePlatformId(id) {
|
|
9119
|
+
return Boolean(id && id.toLowerCase() !== "system");
|
|
9120
|
+
}
|
|
9121
|
+
function deriveSenderPlatformIdMappings(sourceMemberIds, targetMemberIds, sourceSenders, targetSenders, sourceOwnerId, targetOwnerId, allowSingleRemainingPair = false) {
|
|
9122
|
+
const sourceIds = new Set([...sourceMemberIds].filter(isUsablePlatformId));
|
|
9123
|
+
const targetIds = new Set([...targetMemberIds].filter(isUsablePlatformId));
|
|
9124
|
+
if (isUsablePlatformId(sourceOwnerId)) sourceIds.add(sourceOwnerId);
|
|
9125
|
+
if (isUsablePlatformId(targetOwnerId)) targetIds.add(targetOwnerId);
|
|
9126
|
+
const forward = /* @__PURE__ */ new Map();
|
|
9127
|
+
const reverse = /* @__PURE__ */ new Map();
|
|
9128
|
+
let hasDrift = false;
|
|
9129
|
+
const addPair = (sourceId, targetId) => {
|
|
9130
|
+
if (!sourceIds.has(sourceId) || !targetIds.has(targetId)) return false;
|
|
9131
|
+
const existingTarget = forward.get(sourceId);
|
|
9132
|
+
const existingSource = reverse.get(targetId);
|
|
9133
|
+
if (existingTarget && existingTarget !== targetId || existingSource && existingSource !== sourceId) {
|
|
9134
|
+
return false;
|
|
9135
|
+
}
|
|
9136
|
+
forward.set(sourceId, targetId);
|
|
9137
|
+
reverse.set(targetId, sourceId);
|
|
9138
|
+
hasDrift ||= sourceId !== targetId;
|
|
9139
|
+
return true;
|
|
9140
|
+
};
|
|
9141
|
+
if (isUsablePlatformId(sourceOwnerId) && isUsablePlatformId(targetOwnerId) && !addPair(sourceOwnerId, targetOwnerId)) {
|
|
9142
|
+
return null;
|
|
9143
|
+
}
|
|
9144
|
+
for (let index = 0; index < sourceSenders.length; index++) {
|
|
9145
|
+
if (!addPair(sourceSenders[index], targetSenders[index])) return null;
|
|
9146
|
+
}
|
|
9147
|
+
if (!hasDrift) return [];
|
|
9148
|
+
for (const sourceId of sourceIds) {
|
|
9149
|
+
if (!forward.has(sourceId) && targetIds.has(sourceId) && !reverse.has(sourceId)) {
|
|
9150
|
+
addPair(sourceId, sourceId);
|
|
9151
|
+
}
|
|
9152
|
+
}
|
|
9153
|
+
const remainingSourceIds = [...sourceIds].filter((id) => !forward.has(id));
|
|
9154
|
+
const remainingTargetIds = [...targetIds].filter((id) => !reverse.has(id));
|
|
9155
|
+
if (allowSingleRemainingPair && remainingSourceIds.length === 1 && remainingTargetIds.length === 1) {
|
|
9156
|
+
addPair(remainingSourceIds[0], remainingTargetIds[0]);
|
|
9157
|
+
}
|
|
9158
|
+
if ([...sourceIds].some((id) => !forward.has(id))) return null;
|
|
9159
|
+
return [...forward].filter(([sourceId, targetId]) => sourceId !== targetId).map(([sourceId, targetId]) => ({ sourceId, targetId }));
|
|
9160
|
+
}
|
|
9161
|
+
function getCommonPlatformMessageIdScope(ids) {
|
|
9162
|
+
let commonScope;
|
|
9163
|
+
for (const id of ids) {
|
|
9164
|
+
if (!id) return void 0;
|
|
9165
|
+
const { scope } = parsePlatformMessageId(id);
|
|
9166
|
+
if (!scope || commonScope && scope !== commonScope) return void 0;
|
|
9167
|
+
commonScope = scope;
|
|
9168
|
+
}
|
|
9169
|
+
return commonScope;
|
|
9170
|
+
}
|
|
9171
|
+
function generatePlatformMessageKey(message, timestamp, normalizeScope = true) {
|
|
9172
|
+
if (!message.platformMessageId) return null;
|
|
9173
|
+
const parsedId = parsePlatformMessageId(message.platformMessageId);
|
|
9174
|
+
return generateMessageKey(
|
|
9175
|
+
timestamp,
|
|
9176
|
+
normalizeScope ? parsedId.rawId : message.platformMessageId,
|
|
9177
|
+
JSON.stringify([message.type, message.content || null])
|
|
9178
|
+
);
|
|
9261
9179
|
}
|
|
9262
9180
|
async function resolveAutoImportTarget(filePath, deps, formatOptions) {
|
|
9263
9181
|
return (await resolveAutoImportTargetPlan(filePath, deps, formatOptions)).decision;
|
|
@@ -9266,7 +9184,14 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9266
9184
|
let sourceMeta = null;
|
|
9267
9185
|
const sourceMemberIds = /* @__PURE__ */ new Set();
|
|
9268
9186
|
const sourceWindows = /* @__PURE__ */ new Set();
|
|
9187
|
+
const sourcePlatformIdWindows = /* @__PURE__ */ new Set();
|
|
9188
|
+
const sourcePlatformIdSenders = /* @__PURE__ */ new Map();
|
|
9189
|
+
const sourceExactPlatformIdWindows = /* @__PURE__ */ new Set();
|
|
9190
|
+
const sourceExactPlatformIdSenders = /* @__PURE__ */ new Map();
|
|
9269
9191
|
const recentSourceKeys = [];
|
|
9192
|
+
const recentSourcePlatformIdKeys = [];
|
|
9193
|
+
const recentSourceExactPlatformIdKeys = [];
|
|
9194
|
+
let sourceHasScopedPlatformMessageIds = false;
|
|
9270
9195
|
const { formatId, ...parserOptions } = formatOptions ?? {};
|
|
9271
9196
|
await streamParseFile(
|
|
9272
9197
|
filePath,
|
|
@@ -9282,8 +9207,21 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9282
9207
|
},
|
|
9283
9208
|
onMessageBatch: (messages) => {
|
|
9284
9209
|
for (const message of messages) {
|
|
9210
|
+
sourceHasScopedPlatformMessageIds ||= Boolean(
|
|
9211
|
+
message.platformMessageId && parsePlatformMessageId(message.platformMessageId).scope || message.replyToMessageId && parsePlatformMessageId(message.replyToMessageId).scope
|
|
9212
|
+
);
|
|
9285
9213
|
sourceMemberIds.add(message.senderPlatformId);
|
|
9286
|
-
addMessageWindow(
|
|
9214
|
+
addMessageWindow(
|
|
9215
|
+
recentSourceKeys,
|
|
9216
|
+
sourceWindows,
|
|
9217
|
+
recentSourcePlatformIdKeys,
|
|
9218
|
+
sourcePlatformIdWindows,
|
|
9219
|
+
sourcePlatformIdSenders,
|
|
9220
|
+
recentSourceExactPlatformIdKeys,
|
|
9221
|
+
sourceExactPlatformIdWindows,
|
|
9222
|
+
sourceExactPlatformIdSenders,
|
|
9223
|
+
message
|
|
9224
|
+
);
|
|
9287
9225
|
}
|
|
9288
9226
|
}
|
|
9289
9227
|
},
|
|
@@ -9291,10 +9229,12 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9291
9229
|
);
|
|
9292
9230
|
if (!sourceMeta) throw new Error("Import source did not provide metadata");
|
|
9293
9231
|
const meta = sourceMeta;
|
|
9294
|
-
const
|
|
9232
|
+
const sourceHasValidOwner = Boolean(meta.ownerId && meta.ownerId.toLowerCase() !== "system");
|
|
9233
|
+
const matchQqPrivateByMembers = meta.platform === "qq" && meta.type === "private" && !sourceHasValidOwner;
|
|
9234
|
+
const privateIdentity = meta.type === "private" ? buildPrivateIdentity(meta.platform, meta.ownerId, sourceMemberIds) : null;
|
|
9295
9235
|
const hasStableIdentity = Boolean(meta.groupId || privateIdentity);
|
|
9296
9236
|
const stableIdentity = meta.groupId ? `group\0${meta.groupId}` : privateIdentity ? `private\0${privateIdentity}` : null;
|
|
9297
|
-
const stableConcurrencyKey = stableIdentity ? `source:${
|
|
9237
|
+
const stableConcurrencyKey = stableIdentity ? `source:${createHash("sha256").update(`${meta.platform}\0${meta.type}\0${stableIdentity}`).digest("hex")}` : null;
|
|
9298
9238
|
const stableMatches = [];
|
|
9299
9239
|
const trailingMatches = [];
|
|
9300
9240
|
for (const sessionId of deps.listSessionIds()) {
|
|
@@ -9305,17 +9245,18 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9305
9245
|
const candidate = getSessionMeta(db);
|
|
9306
9246
|
if (candidate?.platform !== meta.platform || candidate.type !== meta.type) continue;
|
|
9307
9247
|
const groupMatches = Boolean(meta.groupId && candidate.groupId === meta.groupId);
|
|
9308
|
-
const memberRows = privateIdentity ? db.prepare("SELECT platform_id FROM member WHERE LOWER(platform_id) != 'system'").all() : [];
|
|
9248
|
+
const memberRows = privateIdentity || sourcePlatformIdWindows.size > 0 ? db.prepare("SELECT platform_id FROM member WHERE LOWER(platform_id) != 'system'").all() : [];
|
|
9309
9249
|
const candidatePrivateIdentity = privateIdentity ? buildPrivateIdentity(
|
|
9310
|
-
candidate.
|
|
9250
|
+
candidate.platform,
|
|
9251
|
+
matchQqPrivateByMembers ? null : candidate.ownerId,
|
|
9311
9252
|
memberRows.map((row) => row.platform_id)
|
|
9312
9253
|
) : null;
|
|
9313
9254
|
if (groupMatches || privateIdentity && candidatePrivateIdentity === privateIdentity) {
|
|
9314
9255
|
stableMatches.push(sessionId);
|
|
9315
9256
|
}
|
|
9316
|
-
if (sourceWindows.size > 0) {
|
|
9257
|
+
if (sourceWindows.size > 0 || sourcePlatformIdWindows.size > 0) {
|
|
9317
9258
|
const rows = db.prepare(
|
|
9318
|
-
`SELECT msg.ts, member.platform_id, msg.type, msg.content
|
|
9259
|
+
`SELECT msg.ts, member.platform_id, msg.type, msg.content, msg.platform_message_id
|
|
9319
9260
|
FROM message msg
|
|
9320
9261
|
JOIN member ON member.id = msg.sender_id
|
|
9321
9262
|
WHERE msg.type NOT IN (?, ?)
|
|
@@ -9324,8 +9265,54 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9324
9265
|
LIMIT ?`
|
|
9325
9266
|
).all(80 /* SYSTEM */, 81 /* RECALL */, MATCH_WINDOW_SIZE);
|
|
9326
9267
|
if (rows.length === MATCH_WINDOW_SIZE) {
|
|
9327
|
-
const
|
|
9328
|
-
|
|
9268
|
+
const orderedRows = rows.reverse();
|
|
9269
|
+
const signature = orderedRows.map((row) => generateMessageKey(row.ts, row.platform_id, row.content)).join("");
|
|
9270
|
+
const platformIdKeys = orderedRows.map(
|
|
9271
|
+
(row) => generatePlatformMessageKey(
|
|
9272
|
+
{
|
|
9273
|
+
platformMessageId: row.platform_message_id || void 0,
|
|
9274
|
+
type: row.type,
|
|
9275
|
+
content: row.content
|
|
9276
|
+
},
|
|
9277
|
+
row.ts
|
|
9278
|
+
)
|
|
9279
|
+
);
|
|
9280
|
+
const platformIdSignature = platformIdKeys.every((key) => key !== null) ? platformIdKeys.join("") : null;
|
|
9281
|
+
const exactPlatformIdKeys = orderedRows.map(
|
|
9282
|
+
(row) => generatePlatformMessageKey(
|
|
9283
|
+
{
|
|
9284
|
+
platformMessageId: row.platform_message_id || void 0,
|
|
9285
|
+
type: row.type,
|
|
9286
|
+
content: row.content
|
|
9287
|
+
},
|
|
9288
|
+
row.ts,
|
|
9289
|
+
false
|
|
9290
|
+
)
|
|
9291
|
+
);
|
|
9292
|
+
const exactPlatformIdSignature = exactPlatformIdKeys.every((key) => key !== null) ? exactPlatformIdKeys.join("") : null;
|
|
9293
|
+
const platformIdsMatch = sourceHasScopedPlatformMessageIds ? exactPlatformIdSignature !== null && sourceExactPlatformIdWindows.has(exactPlatformIdSignature) : platformIdSignature !== null && sourcePlatformIdWindows.has(platformIdSignature);
|
|
9294
|
+
const contentMatches = !sourceHasScopedPlatformMessageIds && sourceWindows.has(signature);
|
|
9295
|
+
const matchedPlatformIdKeys = sourceHasScopedPlatformMessageIds ? exactPlatformIdKeys : platformIdKeys;
|
|
9296
|
+
const sourceSenderLookup = sourceHasScopedPlatformMessageIds ? sourceExactPlatformIdSenders : sourcePlatformIdSenders;
|
|
9297
|
+
const sourceSenders = platformIdsMatch ? matchedPlatformIdKeys.map((key) => key ? sourceSenderLookup.get(key) : void 0) : [];
|
|
9298
|
+
const senderPlatformIdMappings = platformIdsMatch && sourceSenders.every((sender) => Boolean(sender)) ? deriveSenderPlatformIdMappings(
|
|
9299
|
+
sourceMemberIds,
|
|
9300
|
+
memberRows.map((row) => row.platform_id),
|
|
9301
|
+
sourceSenders,
|
|
9302
|
+
orderedRows.map((row) => row.platform_id),
|
|
9303
|
+
meta.ownerId,
|
|
9304
|
+
candidate.ownerId,
|
|
9305
|
+
meta.type === "private"
|
|
9306
|
+
) : null;
|
|
9307
|
+
const safePlatformIdsMatch = platformIdsMatch && senderPlatformIdMappings !== null;
|
|
9308
|
+
if (contentMatches || safePlatformIdsMatch) {
|
|
9309
|
+
const platformMessageIdScope = sourceHasScopedPlatformMessageIds ? void 0 : getCommonPlatformMessageIdScope(orderedRows.map((row) => row.platform_message_id));
|
|
9310
|
+
trailingMatches.push({
|
|
9311
|
+
sessionId,
|
|
9312
|
+
...platformMessageIdScope ? { platformMessageIdScope } : {},
|
|
9313
|
+
...safePlatformIdsMatch && senderPlatformIdMappings.length > 0 ? { senderPlatformIdMappings } : {}
|
|
9314
|
+
});
|
|
9315
|
+
}
|
|
9329
9316
|
}
|
|
9330
9317
|
}
|
|
9331
9318
|
} catch (error) {
|
|
@@ -9339,34 +9326,62 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9339
9326
|
}
|
|
9340
9327
|
if (hasStableIdentity) {
|
|
9341
9328
|
if (meta.sourceSessionId && stableMatches.includes(meta.sourceSessionId)) {
|
|
9329
|
+
const trailingMatch = trailingMatches.find((match) => match.sessionId === meta.sourceSessionId);
|
|
9342
9330
|
return {
|
|
9343
|
-
decision: {
|
|
9331
|
+
decision: {
|
|
9332
|
+
action: "incremental",
|
|
9333
|
+
sessionId: meta.sourceSessionId,
|
|
9334
|
+
matchedBy: "source-session-id",
|
|
9335
|
+
...trailingMatch?.platformMessageIdScope ? { platformMessageIdScope: trailingMatch.platformMessageIdScope } : {},
|
|
9336
|
+
...trailingMatch?.senderPlatformIdMappings ? { senderPlatformIdMappings: trailingMatch.senderPlatformIdMappings } : {}
|
|
9337
|
+
},
|
|
9344
9338
|
concurrencyKey: `session:${meta.sourceSessionId}`,
|
|
9345
9339
|
exclusive: false,
|
|
9346
9340
|
coalesceCreate: false
|
|
9347
9341
|
};
|
|
9348
9342
|
}
|
|
9349
9343
|
if (stableMatches.length === 1) {
|
|
9344
|
+
const trailingMatch = trailingMatches.find((match) => match.sessionId === stableMatches[0]);
|
|
9350
9345
|
return {
|
|
9351
|
-
decision: {
|
|
9346
|
+
decision: {
|
|
9347
|
+
action: "incremental",
|
|
9348
|
+
sessionId: stableMatches[0],
|
|
9349
|
+
matchedBy: "stable-id",
|
|
9350
|
+
...trailingMatch?.platformMessageIdScope ? { platformMessageIdScope: trailingMatch.platformMessageIdScope } : {},
|
|
9351
|
+
...trailingMatch?.senderPlatformIdMappings ? { senderPlatformIdMappings: trailingMatch.senderPlatformIdMappings } : {}
|
|
9352
|
+
},
|
|
9352
9353
|
concurrencyKey: `session:${stableMatches[0]}`,
|
|
9353
9354
|
exclusive: false,
|
|
9354
9355
|
coalesceCreate: false
|
|
9355
9356
|
};
|
|
9356
9357
|
}
|
|
9357
9358
|
}
|
|
9358
|
-
|
|
9359
|
+
const sourceSessionTrailingMatch = meta.sourceSessionId ? trailingMatches.find((match) => match.sessionId === meta.sourceSessionId) : void 0;
|
|
9360
|
+
if (meta.sourceSessionId && sourceSessionTrailingMatch) {
|
|
9359
9361
|
return {
|
|
9360
|
-
decision: {
|
|
9362
|
+
decision: {
|
|
9363
|
+
action: "incremental",
|
|
9364
|
+
sessionId: meta.sourceSessionId,
|
|
9365
|
+
matchedBy: "source-session-id",
|
|
9366
|
+
...sourceSessionTrailingMatch.platformMessageIdScope ? { platformMessageIdScope: sourceSessionTrailingMatch.platformMessageIdScope } : {},
|
|
9367
|
+
...sourceSessionTrailingMatch.senderPlatformIdMappings ? { senderPlatformIdMappings: sourceSessionTrailingMatch.senderPlatformIdMappings } : {}
|
|
9368
|
+
},
|
|
9361
9369
|
concurrencyKey: `session:${meta.sourceSessionId}`,
|
|
9362
9370
|
exclusive: false,
|
|
9363
9371
|
coalesceCreate: false
|
|
9364
9372
|
};
|
|
9365
9373
|
}
|
|
9366
9374
|
if (trailingMatches.length === 1) {
|
|
9375
|
+
const [trailingMatch] = trailingMatches;
|
|
9367
9376
|
return {
|
|
9368
|
-
decision: {
|
|
9369
|
-
|
|
9377
|
+
decision: {
|
|
9378
|
+
action: "incremental",
|
|
9379
|
+
sessionId: trailingMatch.sessionId,
|
|
9380
|
+
matchedBy: "trailing-messages",
|
|
9381
|
+
...trailingMatch.platformMessageIdScope ? { platformMessageIdScope: trailingMatch.platformMessageIdScope } : {},
|
|
9382
|
+
...trailingMatch.senderPlatformIdMappings ? { senderPlatformIdMappings: trailingMatch.senderPlatformIdMappings } : {}
|
|
9383
|
+
},
|
|
9384
|
+
concurrencyKey: `session:${trailingMatch.sessionId}`,
|
|
9370
9385
|
exclusive: false,
|
|
9371
9386
|
coalesceCreate: false
|
|
9372
9387
|
};
|
|
@@ -9393,7 +9408,13 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9393
9408
|
async function planAutoImport(filePath, deps, options2) {
|
|
9394
9409
|
if (options2.resolvedDecision) {
|
|
9395
9410
|
const decision2 = options2.resolvedDecision;
|
|
9396
|
-
return decision2.action === "incremental" ? {
|
|
9411
|
+
return decision2.action === "incremental" ? {
|
|
9412
|
+
action: "incremental",
|
|
9413
|
+
sessionId: decision2.sessionId,
|
|
9414
|
+
matchedBy: decision2.matchedBy,
|
|
9415
|
+
...decision2.platformMessageIdScope ? { platformMessageIdScope: decision2.platformMessageIdScope } : {},
|
|
9416
|
+
...decision2.senderPlatformIdMappings ? { senderPlatformIdMappings: decision2.senderPlatformIdMappings } : {}
|
|
9417
|
+
} : { action: "create", reason: decision2.reason };
|
|
9397
9418
|
}
|
|
9398
9419
|
if (options2.explicitSessionId) {
|
|
9399
9420
|
if (!isValidImportSessionId(options2.explicitSessionId)) {
|
|
@@ -9406,7 +9427,13 @@ async function planAutoImport(filePath, deps, options2) {
|
|
|
9406
9427
|
deps,
|
|
9407
9428
|
options2.formatOptions
|
|
9408
9429
|
);
|
|
9409
|
-
return decision.action === "incremental" ? {
|
|
9430
|
+
return decision.action === "incremental" ? {
|
|
9431
|
+
action: "incremental",
|
|
9432
|
+
sessionId: decision.sessionId,
|
|
9433
|
+
matchedBy: decision.matchedBy,
|
|
9434
|
+
...decision.platformMessageIdScope ? { platformMessageIdScope: decision.platformMessageIdScope } : {},
|
|
9435
|
+
...decision.senderPlatformIdMappings ? { senderPlatformIdMappings: decision.senderPlatformIdMappings } : {}
|
|
9436
|
+
} : { action: "create", reason: decision.reason };
|
|
9410
9437
|
}
|
|
9411
9438
|
function mapCreateResult(result, createReason) {
|
|
9412
9439
|
if (!result.success || !result.sessionId) {
|
|
@@ -9462,7 +9489,10 @@ async function autoImportFile(filePath, deps, options2 = {}) {
|
|
|
9462
9489
|
if (plan.action === "incremental") {
|
|
9463
9490
|
const result2 = mapIncrementalResult(
|
|
9464
9491
|
plan.sessionId,
|
|
9465
|
-
await deps.appendSession(plan.sessionId, filePath, options2.formatOptions, deps.onProgress
|
|
9492
|
+
await deps.appendSession(plan.sessionId, filePath, options2.formatOptions, deps.onProgress, {
|
|
9493
|
+
platformMessageIdScope: plan.platformMessageIdScope,
|
|
9494
|
+
senderPlatformIdMappings: plan.senderPlatformIdMappings
|
|
9495
|
+
}),
|
|
9466
9496
|
plan.matchedBy
|
|
9467
9497
|
);
|
|
9468
9498
|
appLogger.info("import", "Incremental import completed", {
|
|
@@ -9501,7 +9531,10 @@ async function analyzeAutoImportFile(filePath, deps, options2 = {}) {
|
|
|
9501
9531
|
try {
|
|
9502
9532
|
const plan = await planAutoImport(filePath, deps, options2);
|
|
9503
9533
|
if (plan.action === "incremental") {
|
|
9504
|
-
const analysis2 = await deps.analyzeAppendSession(plan.sessionId, filePath, options2.formatOptions
|
|
9534
|
+
const analysis2 = await deps.analyzeAppendSession(plan.sessionId, filePath, options2.formatOptions, {
|
|
9535
|
+
platformMessageIdScope: plan.platformMessageIdScope,
|
|
9536
|
+
senderPlatformIdMappings: plan.senderPlatformIdMappings
|
|
9537
|
+
});
|
|
9505
9538
|
if (analysis2.error) return { success: false, error: analysis2.error };
|
|
9506
9539
|
const result2 = {
|
|
9507
9540
|
success: true,
|
|
@@ -9761,8 +9794,8 @@ async function autoImportBatch(items, deps, options2 = {}) {
|
|
|
9761
9794
|
}
|
|
9762
9795
|
|
|
9763
9796
|
// ../../packages/node-runtime/src/import/import-lock.ts
|
|
9764
|
-
import * as
|
|
9765
|
-
import * as
|
|
9797
|
+
import * as fs21 from "fs";
|
|
9798
|
+
import * as path13 from "path";
|
|
9766
9799
|
import { randomUUID } from "crypto";
|
|
9767
9800
|
var IMPORT_LOCK_FILENAME = ".chatlab-import.lock";
|
|
9768
9801
|
var IMPORT_IN_PROGRESS_ERROR_KEY = "error.import_in_progress";
|
|
@@ -9776,7 +9809,7 @@ var ImportInProgressError = class extends Error {
|
|
|
9776
9809
|
};
|
|
9777
9810
|
function parseOwner(lockPath) {
|
|
9778
9811
|
try {
|
|
9779
|
-
const parsed = JSON.parse(
|
|
9812
|
+
const parsed = JSON.parse(fs21.readFileSync(lockPath, "utf-8"));
|
|
9780
9813
|
if (typeof parsed.pid !== "number" || !Number.isInteger(parsed.pid) || parsed.pid <= 0 || typeof parsed.token !== "string" || parsed.token.length === 0 || typeof parsed.startedAt !== "number") {
|
|
9781
9814
|
return null;
|
|
9782
9815
|
}
|
|
@@ -9799,13 +9832,13 @@ function removeStaleLock(lockPath) {
|
|
|
9799
9832
|
if (isProcessRunning(owner.pid)) return false;
|
|
9800
9833
|
} else {
|
|
9801
9834
|
try {
|
|
9802
|
-
if (Date.now() -
|
|
9835
|
+
if (Date.now() - fs21.statSync(lockPath).mtimeMs < 5e3) return false;
|
|
9803
9836
|
} catch {
|
|
9804
9837
|
return true;
|
|
9805
9838
|
}
|
|
9806
9839
|
}
|
|
9807
9840
|
try {
|
|
9808
|
-
|
|
9841
|
+
fs21.unlinkSync(lockPath);
|
|
9809
9842
|
appLogger.warn("import-lock", "Removed stale import lock", { ownerPid: owner?.pid });
|
|
9810
9843
|
return true;
|
|
9811
9844
|
} catch {
|
|
@@ -9814,7 +9847,7 @@ function removeStaleLock(lockPath) {
|
|
|
9814
9847
|
}
|
|
9815
9848
|
function releaseLock(lockPath, fd, owner) {
|
|
9816
9849
|
try {
|
|
9817
|
-
|
|
9850
|
+
fs21.closeSync(fd);
|
|
9818
9851
|
} catch (error) {
|
|
9819
9852
|
appLogger.error("import-lock", "Failed to close import lock", error);
|
|
9820
9853
|
}
|
|
@@ -9823,19 +9856,19 @@ function releaseLock(lockPath, fd, owner) {
|
|
|
9823
9856
|
appLogger.warn("import-lock", "Import lock ownership changed before release");
|
|
9824
9857
|
return;
|
|
9825
9858
|
}
|
|
9826
|
-
|
|
9859
|
+
fs21.unlinkSync(lockPath);
|
|
9827
9860
|
appLogger.info("import-lock", "Import lock released");
|
|
9828
9861
|
} catch (error) {
|
|
9829
9862
|
appLogger.error("import-lock", "Failed to release import lock", error);
|
|
9830
9863
|
}
|
|
9831
9864
|
}
|
|
9832
9865
|
async function withDataDirImportLock(userDataDir, task) {
|
|
9833
|
-
|
|
9834
|
-
const lockPath =
|
|
9866
|
+
fs21.mkdirSync(userDataDir, { recursive: true });
|
|
9867
|
+
const lockPath = path13.join(userDataDir, IMPORT_LOCK_FILENAME);
|
|
9835
9868
|
let fd = null;
|
|
9836
9869
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
9837
9870
|
try {
|
|
9838
|
-
fd =
|
|
9871
|
+
fd = fs21.openSync(lockPath, "wx");
|
|
9839
9872
|
break;
|
|
9840
9873
|
} catch (error) {
|
|
9841
9874
|
const code = error.code;
|
|
@@ -9854,8 +9887,8 @@ async function withDataDirImportLock(userDataDir, task) {
|
|
|
9854
9887
|
startedAt: Date.now()
|
|
9855
9888
|
};
|
|
9856
9889
|
try {
|
|
9857
|
-
|
|
9858
|
-
|
|
9890
|
+
fs21.writeFileSync(fd, JSON.stringify(owner), "utf-8");
|
|
9891
|
+
fs21.fsyncSync(fd);
|
|
9859
9892
|
appLogger.info("import-lock", "Import lock acquired");
|
|
9860
9893
|
return await task();
|
|
9861
9894
|
} finally {
|
|
@@ -9875,8 +9908,8 @@ var ArchiveImportError = class extends Error {
|
|
|
9875
9908
|
|
|
9876
9909
|
// ../../packages/node-runtime/src/import/archive/archive-reader.ts
|
|
9877
9910
|
var import_yauzl = __toESM(require_yauzl(), 1);
|
|
9878
|
-
import * as
|
|
9879
|
-
import * as
|
|
9911
|
+
import * as fs22 from "fs";
|
|
9912
|
+
import * as path14 from "path";
|
|
9880
9913
|
import { pipeline } from "stream/promises";
|
|
9881
9914
|
var DEFAULT_MAX_ENTRIES = 1e5;
|
|
9882
9915
|
var DEFAULT_MAX_ENTRY_BYTES = 20 * 1024 * 1024 * 1024;
|
|
@@ -9995,8 +10028,8 @@ var ZipArchiveReader = class {
|
|
|
9995
10028
|
await this.visitEntries(async (entry, openStream) => {
|
|
9996
10029
|
const destination = targets.get(entry.name);
|
|
9997
10030
|
if (!destination || entry.isDirectory) return;
|
|
9998
|
-
|
|
9999
|
-
await pipeline(await openStream(),
|
|
10031
|
+
fs22.mkdirSync(path14.dirname(destination), { recursive: true });
|
|
10032
|
+
await pipeline(await openStream(), fs22.createWriteStream(destination, { flags: "wx" }));
|
|
10000
10033
|
remaining.delete(entry.name);
|
|
10001
10034
|
});
|
|
10002
10035
|
if (remaining.size > 0) {
|
|
@@ -10013,8 +10046,8 @@ var import_stream_chain8 = __toESM(require_stream_chain(), 1);
|
|
|
10013
10046
|
var import_stream_json9 = __toESM(require_stream_json(), 1);
|
|
10014
10047
|
var import_Pick8 = __toESM(require_Pick(), 1);
|
|
10015
10048
|
var import_StreamValues8 = __toESM(require_StreamValues(), 1);
|
|
10016
|
-
import * as
|
|
10017
|
-
import * as
|
|
10049
|
+
import * as fs23 from "fs";
|
|
10050
|
+
import * as path15 from "path";
|
|
10018
10051
|
var { chain: chain8 } = import_stream_chain8.default;
|
|
10019
10052
|
var { parser: parser10 } = import_stream_json9.default;
|
|
10020
10053
|
var { pick: pick8 } = import_Pick8.default;
|
|
@@ -10064,7 +10097,7 @@ function deriveConversationName(draft, ownerEmail) {
|
|
|
10064
10097
|
return other?.name?.trim() || other?.email?.trim() || members[0]?.name?.trim() || draft.chatId;
|
|
10065
10098
|
}
|
|
10066
10099
|
var GoogleChatTakeoutResolver = class {
|
|
10067
|
-
id =
|
|
10100
|
+
id = PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE;
|
|
10068
10101
|
platform = "google-chat";
|
|
10069
10102
|
detect(entries) {
|
|
10070
10103
|
const products = /* @__PURE__ */ new Set();
|
|
@@ -10143,16 +10176,16 @@ var GoogleChatTakeoutResolver = class {
|
|
|
10143
10176
|
`Google Chat conversation is missing required files: ${chat.chatId}`
|
|
10144
10177
|
);
|
|
10145
10178
|
}
|
|
10146
|
-
|
|
10179
|
+
fs23.mkdirSync(targetDir, { recursive: true });
|
|
10147
10180
|
await reader.pipeEntries(
|
|
10148
10181
|
/* @__PURE__ */ new Map([
|
|
10149
|
-
[userInfoEntry,
|
|
10150
|
-
[groupInfoEntry,
|
|
10151
|
-
[messagesEntry,
|
|
10182
|
+
[userInfoEntry, path15.join(targetDir, "user_info.json")],
|
|
10183
|
+
[groupInfoEntry, path15.join(targetDir, "group_info.json")],
|
|
10184
|
+
[messagesEntry, path15.join(targetDir, "messages.json")]
|
|
10152
10185
|
])
|
|
10153
10186
|
);
|
|
10154
|
-
const manifestPath =
|
|
10155
|
-
|
|
10187
|
+
const manifestPath = path15.join(targetDir, "google-chat-import.json");
|
|
10188
|
+
fs23.writeFileSync(
|
|
10156
10189
|
manifestPath,
|
|
10157
10190
|
JSON.stringify(
|
|
10158
10191
|
{
|
|
@@ -10175,8 +10208,8 @@ var GoogleChatTakeoutResolver = class {
|
|
|
10175
10208
|
};
|
|
10176
10209
|
|
|
10177
10210
|
// ../../packages/node-runtime/src/import/archive/source-manager.ts
|
|
10178
|
-
import * as
|
|
10179
|
-
import * as
|
|
10211
|
+
import * as fs24 from "fs";
|
|
10212
|
+
import * as path16 from "path";
|
|
10180
10213
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
10181
10214
|
var DEFAULT_TTL_MS = 30 * 60 * 1e3;
|
|
10182
10215
|
var ArchiveImportSourceManager = class {
|
|
@@ -10191,7 +10224,7 @@ var ArchiveImportSourceManager = class {
|
|
|
10191
10224
|
this.ttlMs = options2.ttlMs ?? DEFAULT_TTL_MS;
|
|
10192
10225
|
this.now = options2.now ?? Date.now;
|
|
10193
10226
|
this.resolvers = options2.resolvers ?? [new GoogleChatTakeoutResolver()];
|
|
10194
|
-
|
|
10227
|
+
fs24.mkdirSync(this.tempRoot, { recursive: true });
|
|
10195
10228
|
}
|
|
10196
10229
|
prepareLocalArchive(archivePath) {
|
|
10197
10230
|
return this.prepareArchive(archivePath, false);
|
|
@@ -10248,7 +10281,7 @@ var ArchiveImportSourceManager = class {
|
|
|
10248
10281
|
`Archive conversation was not found: ${chatId}`
|
|
10249
10282
|
);
|
|
10250
10283
|
}
|
|
10251
|
-
const materializedDir =
|
|
10284
|
+
const materializedDir = fs24.mkdtempSync(path16.join(this.tempRoot, `materialized-${sourceId}-`));
|
|
10252
10285
|
try {
|
|
10253
10286
|
const reader = new ZipArchiveReader(record.archivePath);
|
|
10254
10287
|
const materialized = await record.resolver.materialize(reader, chat, materializedDir);
|
|
@@ -10299,7 +10332,7 @@ var ArchiveImportSourceManager = class {
|
|
|
10299
10332
|
}
|
|
10300
10333
|
removePath(targetPath) {
|
|
10301
10334
|
try {
|
|
10302
|
-
|
|
10335
|
+
fs24.rmSync(targetPath, { recursive: true, force: true });
|
|
10303
10336
|
} catch {
|
|
10304
10337
|
}
|
|
10305
10338
|
}
|
|
@@ -10369,12 +10402,12 @@ ${rawErrorLabel}: ${details}`;
|
|
|
10369
10402
|
}
|
|
10370
10403
|
|
|
10371
10404
|
// ../../packages/node-runtime/src/ai/assistant-parser.ts
|
|
10372
|
-
import * as
|
|
10405
|
+
import * as path17 from "path";
|
|
10373
10406
|
import matter from "gray-matter";
|
|
10374
10407
|
function parseAssistantFile(content, filePath) {
|
|
10375
10408
|
try {
|
|
10376
10409
|
const { data: fm, content: body } = matter(content);
|
|
10377
|
-
const id = fm.id ??
|
|
10410
|
+
const id = fm.id ?? path17.basename(filePath, ".md");
|
|
10378
10411
|
const name = fm.name;
|
|
10379
10412
|
if (!name) return null;
|
|
10380
10413
|
return {
|
|
@@ -10594,8 +10627,8 @@ var AssistantManager = class {
|
|
|
10594
10627
|
}
|
|
10595
10628
|
// ==================== Init ====================
|
|
10596
10629
|
init() {
|
|
10597
|
-
const { fs:
|
|
10598
|
-
|
|
10630
|
+
const { fs: fs36, assistantsDir } = this.deps;
|
|
10631
|
+
fs36.ensureDir(assistantsDir);
|
|
10599
10632
|
const { generalCreated, generalUpdated } = this.syncGeneralAssistants();
|
|
10600
10633
|
this.loadAll();
|
|
10601
10634
|
this.initialized = true;
|
|
@@ -10607,25 +10640,25 @@ var AssistantManager = class {
|
|
|
10607
10640
|
return { total: this.cache.size, generalCreated, generalUpdated };
|
|
10608
10641
|
}
|
|
10609
10642
|
syncGeneralAssistants() {
|
|
10610
|
-
const { fs:
|
|
10643
|
+
const { fs: fs36, assistantsDir } = this.deps;
|
|
10611
10644
|
let generalCreated = false;
|
|
10612
10645
|
let generalUpdated = false;
|
|
10613
10646
|
for (const id of this.generalIds) {
|
|
10614
10647
|
const config = this.getBuiltinConfig(id);
|
|
10615
10648
|
if (!config) continue;
|
|
10616
|
-
const filePath =
|
|
10617
|
-
if (!
|
|
10618
|
-
|
|
10649
|
+
const filePath = fs36.joinPath(assistantsDir, `${id}.md`);
|
|
10650
|
+
if (!fs36.fileExists(filePath)) {
|
|
10651
|
+
fs36.writeFile(filePath, serializeAssistant(this.withBuiltinTracking(config)));
|
|
10619
10652
|
generalCreated = true;
|
|
10620
10653
|
continue;
|
|
10621
10654
|
}
|
|
10622
10655
|
try {
|
|
10623
|
-
const existing = parseAssistantFile(
|
|
10656
|
+
const existing = parseAssistantFile(fs36.readFile(filePath), filePath);
|
|
10624
10657
|
if (!existing) continue;
|
|
10625
10658
|
if (existing.builtinId !== id) {
|
|
10626
10659
|
try {
|
|
10627
10660
|
const backupId = this.backupAssistantOccupyingReservedId(existing);
|
|
10628
|
-
|
|
10661
|
+
fs36.writeFile(filePath, serializeAssistant(this.withBuiltinTracking(config)));
|
|
10629
10662
|
generalCreated = true;
|
|
10630
10663
|
this.deps.logger?.info("AssistantManager", "Backed up assistant occupying reserved default ID", {
|
|
10631
10664
|
reservedId: id,
|
|
@@ -10641,7 +10674,7 @@ var AssistantManager = class {
|
|
|
10641
10674
|
continue;
|
|
10642
10675
|
}
|
|
10643
10676
|
if (!this.shouldUpgradeBuiltin(existing, config)) continue;
|
|
10644
|
-
|
|
10677
|
+
fs36.writeFile(filePath, serializeAssistant(this.withBuiltinTracking(config, existing.id)));
|
|
10645
10678
|
generalUpdated = true;
|
|
10646
10679
|
this.deps.logger?.info("AssistantManager", "Updated unmodified default assistant template", {
|
|
10647
10680
|
id,
|
|
@@ -10657,14 +10690,14 @@ var AssistantManager = class {
|
|
|
10657
10690
|
return { generalCreated, generalUpdated };
|
|
10658
10691
|
}
|
|
10659
10692
|
backupAssistantOccupyingReservedId(config) {
|
|
10660
|
-
const { fs:
|
|
10693
|
+
const { fs: fs36, assistantsDir } = this.deps;
|
|
10661
10694
|
const baseId = this.deps.generateId?.() || `custom_${Date.now().toString(36)}`;
|
|
10662
10695
|
let backupId = baseId;
|
|
10663
10696
|
let suffix = 2;
|
|
10664
|
-
while (this.generalIds.includes(backupId) ||
|
|
10697
|
+
while (this.generalIds.includes(backupId) || fs36.fileExists(fs36.joinPath(assistantsDir, `${backupId}.md`))) {
|
|
10665
10698
|
backupId = `${baseId}_${suffix++}`;
|
|
10666
10699
|
}
|
|
10667
|
-
|
|
10700
|
+
fs36.writeFile(fs36.joinPath(assistantsDir, `${backupId}.md`), serializeAssistant({ ...config, id: backupId }));
|
|
10668
10701
|
return backupId;
|
|
10669
10702
|
}
|
|
10670
10703
|
getConfigDigest(config) {
|
|
@@ -10701,13 +10734,13 @@ var AssistantManager = class {
|
|
|
10701
10734
|
return { isUnmodified, isOutdated };
|
|
10702
10735
|
}
|
|
10703
10736
|
loadAll() {
|
|
10704
|
-
const { fs:
|
|
10737
|
+
const { fs: fs36, assistantsDir } = this.deps;
|
|
10705
10738
|
this.cache.clear();
|
|
10706
|
-
const files =
|
|
10739
|
+
const files = fs36.listFiles(assistantsDir, ".md");
|
|
10707
10740
|
for (const file of files) {
|
|
10708
10741
|
try {
|
|
10709
|
-
const filePath =
|
|
10710
|
-
const content =
|
|
10742
|
+
const filePath = fs36.joinPath(assistantsDir, file);
|
|
10743
|
+
const content = fs36.readFile(filePath);
|
|
10711
10744
|
const config = parseAssistantFile(content, filePath);
|
|
10712
10745
|
if (config) {
|
|
10713
10746
|
this.cache.set(config.id, config);
|
|
@@ -10884,12 +10917,12 @@ var AssistantManager = class {
|
|
|
10884
10917
|
};
|
|
10885
10918
|
|
|
10886
10919
|
// ../../packages/node-runtime/src/ai/skill-parser.ts
|
|
10887
|
-
import * as
|
|
10920
|
+
import * as path18 from "path";
|
|
10888
10921
|
import matter2 from "gray-matter";
|
|
10889
10922
|
function parseSkillFile(content, filePath) {
|
|
10890
10923
|
try {
|
|
10891
10924
|
const { data: fm, content: prompt } = matter2(content);
|
|
10892
|
-
const id = fm.id ??
|
|
10925
|
+
const id = fm.id ?? path18.basename(filePath, ".md");
|
|
10893
10926
|
const name = fm.name;
|
|
10894
10927
|
if (!name) return null;
|
|
10895
10928
|
return {
|
|
@@ -10908,7 +10941,7 @@ function parseSkillFile(content, filePath) {
|
|
|
10908
10941
|
function extractSkillId(content, filePath) {
|
|
10909
10942
|
try {
|
|
10910
10943
|
const { data: fm } = matter2(content);
|
|
10911
|
-
return fm.id ??
|
|
10944
|
+
return fm.id ?? path18.basename(filePath, ".md");
|
|
10912
10945
|
} catch {
|
|
10913
10946
|
return null;
|
|
10914
10947
|
}
|
|
@@ -10927,8 +10960,8 @@ function validateChatScope(raw) {
|
|
|
10927
10960
|
|
|
10928
10961
|
// ../../packages/node-runtime/src/ai/chats.ts
|
|
10929
10962
|
import Database from "better-sqlite3";
|
|
10930
|
-
import * as
|
|
10931
|
-
import * as
|
|
10963
|
+
import * as fs25 from "fs";
|
|
10964
|
+
import * as path19 from "path";
|
|
10932
10965
|
var DEFAULT_GENERAL_ID = DEFAULT_GENERAL_ASSISTANT_ID;
|
|
10933
10966
|
var defaultLogger2 = {
|
|
10934
10967
|
warn(_category, message, extra) {
|
|
@@ -10947,14 +10980,14 @@ var AIChatManager = class {
|
|
|
10947
10980
|
this.nativeBinding = options2?.nativeBinding;
|
|
10948
10981
|
}
|
|
10949
10982
|
ensureDir(dir) {
|
|
10950
|
-
if (!
|
|
10951
|
-
|
|
10983
|
+
if (!fs25.existsSync(dir)) {
|
|
10984
|
+
fs25.mkdirSync(dir, { recursive: true });
|
|
10952
10985
|
}
|
|
10953
10986
|
}
|
|
10954
10987
|
getDb() {
|
|
10955
10988
|
if (this.db) return this.db;
|
|
10956
10989
|
this.ensureDir(this.aiDataDir);
|
|
10957
|
-
const dbPath =
|
|
10990
|
+
const dbPath = path19.join(this.aiDataDir, "conversations.db");
|
|
10958
10991
|
this.db = this.nativeBinding ? new Database(dbPath, { nativeBinding: this.nativeBinding }) : new Database(dbPath);
|
|
10959
10992
|
this.db.pragma("journal_mode = WAL");
|
|
10960
10993
|
this.db.exec(`
|
|
@@ -11197,16 +11230,16 @@ var AIChatManager = class {
|
|
|
11197
11230
|
if (allRows.length === 0) return [];
|
|
11198
11231
|
const rowMap = new Map(allRows.map((row) => [row.id, row]));
|
|
11199
11232
|
let currentId = leafMessageId ?? this.getActiveMessageId(aiChatId);
|
|
11200
|
-
const
|
|
11233
|
+
const path30 = [];
|
|
11201
11234
|
const seen = /* @__PURE__ */ new Set();
|
|
11202
11235
|
while (currentId && !seen.has(currentId)) {
|
|
11203
11236
|
seen.add(currentId);
|
|
11204
11237
|
const row = rowMap.get(currentId);
|
|
11205
11238
|
if (!row) break;
|
|
11206
|
-
|
|
11239
|
+
path30.push(row);
|
|
11207
11240
|
currentId = row.parentId;
|
|
11208
11241
|
}
|
|
11209
|
-
return
|
|
11242
|
+
return path30.length > 0 ? path30.reverse() : allRows;
|
|
11210
11243
|
}
|
|
11211
11244
|
// ==================== 生命周期 ====================
|
|
11212
11245
|
close() {
|
|
@@ -11776,21 +11809,21 @@ var SkillManagerCore = class {
|
|
|
11776
11809
|
}
|
|
11777
11810
|
// ==================== Init ====================
|
|
11778
11811
|
init() {
|
|
11779
|
-
const { fs:
|
|
11780
|
-
|
|
11812
|
+
const { fs: fs36, skillsDir } = this.deps;
|
|
11813
|
+
fs36.ensureDir(skillsDir);
|
|
11781
11814
|
this.loadAll();
|
|
11782
11815
|
this.initialized = true;
|
|
11783
11816
|
this.deps.logger?.info("SkillManager", "Initialized", { total: this.cache.size });
|
|
11784
11817
|
return { total: this.cache.size };
|
|
11785
11818
|
}
|
|
11786
11819
|
loadAll() {
|
|
11787
|
-
const { fs:
|
|
11820
|
+
const { fs: fs36, skillsDir } = this.deps;
|
|
11788
11821
|
this.cache.clear();
|
|
11789
|
-
const files =
|
|
11822
|
+
const files = fs36.listFiles(skillsDir, ".md");
|
|
11790
11823
|
for (const file of files) {
|
|
11791
11824
|
try {
|
|
11792
|
-
const filePath =
|
|
11793
|
-
const content =
|
|
11825
|
+
const filePath = fs36.joinPath(skillsDir, file);
|
|
11826
|
+
const content = fs36.readFile(filePath);
|
|
11794
11827
|
const def = parseSkillFile(content, filePath);
|
|
11795
11828
|
if (def) {
|
|
11796
11829
|
this.cache.set(def.id, def);
|
|
@@ -11942,8 +11975,8 @@ var SkillManagerCore = class {
|
|
|
11942
11975
|
};
|
|
11943
11976
|
|
|
11944
11977
|
// ../../packages/node-runtime/src/ai/skill-manager.ts
|
|
11945
|
-
import * as
|
|
11946
|
-
import * as
|
|
11978
|
+
import * as fs26 from "fs";
|
|
11979
|
+
import * as path20 from "path";
|
|
11947
11980
|
var SKILLS_DIR_NAME = "skills";
|
|
11948
11981
|
var defaultLogger3 = {
|
|
11949
11982
|
info: () => {
|
|
@@ -11953,26 +11986,26 @@ var defaultLogger3 = {
|
|
|
11953
11986
|
};
|
|
11954
11987
|
var nodeFs = {
|
|
11955
11988
|
ensureDir(dir) {
|
|
11956
|
-
|
|
11989
|
+
fs26.mkdirSync(dir, { recursive: true });
|
|
11957
11990
|
},
|
|
11958
11991
|
listFiles(dir, ext) {
|
|
11959
|
-
if (!
|
|
11960
|
-
return
|
|
11992
|
+
if (!fs26.existsSync(dir)) return [];
|
|
11993
|
+
return fs26.readdirSync(dir).filter((file) => file.endsWith(ext));
|
|
11961
11994
|
},
|
|
11962
11995
|
readFile(filePath) {
|
|
11963
|
-
return
|
|
11996
|
+
return fs26.readFileSync(filePath, "utf-8");
|
|
11964
11997
|
},
|
|
11965
11998
|
writeFile(filePath, content) {
|
|
11966
|
-
|
|
11999
|
+
fs26.writeFileSync(filePath, content, "utf-8");
|
|
11967
12000
|
},
|
|
11968
12001
|
deleteFile(filePath) {
|
|
11969
|
-
|
|
12002
|
+
fs26.unlinkSync(filePath);
|
|
11970
12003
|
},
|
|
11971
12004
|
fileExists(filePath) {
|
|
11972
|
-
return
|
|
12005
|
+
return fs26.existsSync(filePath);
|
|
11973
12006
|
},
|
|
11974
12007
|
joinPath(...parts) {
|
|
11975
|
-
return
|
|
12008
|
+
return path20.join(...parts);
|
|
11976
12009
|
}
|
|
11977
12010
|
};
|
|
11978
12011
|
var SkillManager = class {
|
|
@@ -11982,7 +12015,7 @@ var SkillManager = class {
|
|
|
11982
12015
|
this.logger = logger ?? defaultLogger3;
|
|
11983
12016
|
this.core = new SkillManagerCore({
|
|
11984
12017
|
fs: nodeFs,
|
|
11985
|
-
skillsDir:
|
|
12018
|
+
skillsDir: path20.join(aiDataDir, SKILLS_DIR_NAME),
|
|
11986
12019
|
builtinRawSkills: [],
|
|
11987
12020
|
logger: {
|
|
11988
12021
|
info: () => {
|
|
@@ -15848,11 +15881,11 @@ var localeMap = {
|
|
|
15848
15881
|
"ja-JP": ja_JP_default,
|
|
15849
15882
|
"zh-TW": zh_TW_default
|
|
15850
15883
|
};
|
|
15851
|
-
function getNestedValue(obj,
|
|
15884
|
+
function getNestedValue(obj, path30) {
|
|
15852
15885
|
let current = obj;
|
|
15853
|
-
for (const
|
|
15886
|
+
for (const segment of path30.split(".")) {
|
|
15854
15887
|
if (current == null || typeof current !== "object") return void 0;
|
|
15855
|
-
current = current[
|
|
15888
|
+
current = current[segment];
|
|
15856
15889
|
}
|
|
15857
15890
|
return current;
|
|
15858
15891
|
}
|
|
@@ -16086,8 +16119,8 @@ async function generateSessionSummary(deps, segmentId, options2 = {}) {
|
|
|
16086
16119
|
const segments = splitIntoSegments(validMessages, MAX_CONTENT_PER_CALL);
|
|
16087
16120
|
log?.info("Summary", `Long session segmented: ${segments.length} segments`);
|
|
16088
16121
|
const subSummaries = [];
|
|
16089
|
-
for (const
|
|
16090
|
-
const segContent = formatMessages(
|
|
16122
|
+
for (const segment of segments) {
|
|
16123
|
+
const segContent = formatMessages(segment);
|
|
16091
16124
|
const sub = await deps.llmComplete(
|
|
16092
16125
|
deps.t("summary.systemPromptDirect"),
|
|
16093
16126
|
buildSubSummaryPrompt(segContent, locale, strategy),
|
|
@@ -16385,10 +16418,10 @@ var LLMConfigStore = class {
|
|
|
16385
16418
|
};
|
|
16386
16419
|
|
|
16387
16420
|
// ../../packages/node-runtime/src/ai/file-config-storage.ts
|
|
16388
|
-
import * as
|
|
16421
|
+
import * as path21 from "path";
|
|
16389
16422
|
function createFileConfigStorage(baseDir) {
|
|
16390
16423
|
const heldLocks = /* @__PURE__ */ new Set();
|
|
16391
|
-
const getFilePath = (key) =>
|
|
16424
|
+
const getFilePath = (key) => path21.join(baseDir, `${key}.json`);
|
|
16392
16425
|
return {
|
|
16393
16426
|
readJson(key) {
|
|
16394
16427
|
return readJsonFile(getFilePath(key));
|
|
@@ -17344,8 +17377,8 @@ function exportWithFormat(params, openDatabase) {
|
|
|
17344
17377
|
}
|
|
17345
17378
|
|
|
17346
17379
|
// ../../packages/node-runtime/src/merger/temp-db.ts
|
|
17347
|
-
import * as
|
|
17348
|
-
import * as
|
|
17380
|
+
import * as fs27 from "fs";
|
|
17381
|
+
import * as path22 from "path";
|
|
17349
17382
|
var TEMP_DB_SCHEMA = `
|
|
17350
17383
|
CREATE TABLE IF NOT EXISTS meta (
|
|
17351
17384
|
name TEXT NOT NULL,
|
|
@@ -17365,12 +17398,14 @@ var TEMP_DB_SCHEMA = `
|
|
|
17365
17398
|
|
|
17366
17399
|
CREATE TABLE IF NOT EXISTS message (
|
|
17367
17400
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
17401
|
+
platform_message_id TEXT,
|
|
17368
17402
|
sender_platform_id TEXT NOT NULL,
|
|
17369
17403
|
sender_account_name TEXT,
|
|
17370
17404
|
sender_group_nickname TEXT,
|
|
17371
17405
|
timestamp INTEGER NOT NULL,
|
|
17372
17406
|
type INTEGER NOT NULL,
|
|
17373
|
-
content TEXT
|
|
17407
|
+
content TEXT,
|
|
17408
|
+
reply_to_message_id TEXT
|
|
17374
17409
|
);
|
|
17375
17410
|
|
|
17376
17411
|
CREATE INDEX IF NOT EXISTS idx_message_ts ON message(timestamp);
|
|
@@ -17401,8 +17436,10 @@ var TempDbWriter = class {
|
|
|
17401
17436
|
}
|
|
17402
17437
|
writeMessages(messages) {
|
|
17403
17438
|
const insert = this.db.prepare(
|
|
17404
|
-
`INSERT INTO message (
|
|
17405
|
-
|
|
17439
|
+
`INSERT INTO message (
|
|
17440
|
+
platform_message_id, sender_platform_id, sender_account_name, sender_group_nickname,
|
|
17441
|
+
timestamp, type, content, reply_to_message_id
|
|
17442
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
17406
17443
|
);
|
|
17407
17444
|
const memberInsert = this.db.prepare(
|
|
17408
17445
|
"INSERT OR IGNORE INTO member (platform_id, account_name, group_nickname, avatar) VALUES (?, ?, ?, ?)"
|
|
@@ -17413,12 +17450,14 @@ var TempDbWriter = class {
|
|
|
17413
17450
|
memberInsert.run(msg.senderPlatformId, msg.senderAccountName || null, msg.senderGroupNickname || null, null);
|
|
17414
17451
|
}
|
|
17415
17452
|
insert.run(
|
|
17453
|
+
msg.platformMessageId || null,
|
|
17416
17454
|
msg.senderPlatformId,
|
|
17417
17455
|
msg.senderAccountName || null,
|
|
17418
17456
|
msg.senderGroupNickname || null,
|
|
17419
17457
|
msg.timestamp,
|
|
17420
17458
|
msg.type,
|
|
17421
|
-
msg.content || null
|
|
17459
|
+
msg.content || null,
|
|
17460
|
+
msg.replyToMessageId || null
|
|
17422
17461
|
);
|
|
17423
17462
|
this.messageCount++;
|
|
17424
17463
|
}
|
|
@@ -17450,7 +17489,8 @@ var TempDbReader = class {
|
|
|
17450
17489
|
platform: row.platform,
|
|
17451
17490
|
type: row.type,
|
|
17452
17491
|
groupId: row.group_id || void 0,
|
|
17453
|
-
groupAvatar: row.group_avatar || void 0
|
|
17492
|
+
groupAvatar: row.group_avatar || void 0,
|
|
17493
|
+
ownerId: row.owner_id || void 0
|
|
17454
17494
|
};
|
|
17455
17495
|
}
|
|
17456
17496
|
getMembers() {
|
|
@@ -17468,20 +17508,23 @@ var TempDbReader = class {
|
|
|
17468
17508
|
}
|
|
17469
17509
|
streamMessages(batchSize, callback) {
|
|
17470
17510
|
const stmt = this.db.prepare(`
|
|
17471
|
-
SELECT sender_platform_id, sender_account_name, sender_group_nickname,
|
|
17472
|
-
|
|
17511
|
+
SELECT platform_message_id, sender_platform_id, sender_account_name, sender_group_nickname,
|
|
17512
|
+
timestamp, type, content, reply_to_message_id
|
|
17513
|
+
FROM message ORDER BY timestamp ASC, id ASC LIMIT ? OFFSET ?
|
|
17473
17514
|
`);
|
|
17474
17515
|
let offset = 0;
|
|
17475
17516
|
while (true) {
|
|
17476
17517
|
const rows = stmt.all(batchSize, offset);
|
|
17477
17518
|
if (rows.length === 0) break;
|
|
17478
17519
|
const messages = rows.map((r) => ({
|
|
17520
|
+
platformMessageId: r.platform_message_id || void 0,
|
|
17479
17521
|
senderPlatformId: r.sender_platform_id,
|
|
17480
17522
|
senderAccountName: r.sender_account_name || r.sender_platform_id,
|
|
17481
17523
|
senderGroupNickname: r.sender_group_nickname || void 0,
|
|
17482
17524
|
timestamp: r.timestamp,
|
|
17483
17525
|
type: r.type,
|
|
17484
|
-
content: r.content
|
|
17526
|
+
content: r.content,
|
|
17527
|
+
replyToMessageId: r.reply_to_message_id || void 0
|
|
17485
17528
|
}));
|
|
17486
17529
|
callback(messages);
|
|
17487
17530
|
offset += batchSize;
|
|
@@ -17489,16 +17532,19 @@ var TempDbReader = class {
|
|
|
17489
17532
|
}
|
|
17490
17533
|
getAllMessages() {
|
|
17491
17534
|
const rows = this.db.prepare(
|
|
17492
|
-
`SELECT sender_platform_id, sender_account_name, sender_group_nickname,
|
|
17493
|
-
|
|
17535
|
+
`SELECT platform_message_id, sender_platform_id, sender_account_name, sender_group_nickname,
|
|
17536
|
+
timestamp, type, content, reply_to_message_id
|
|
17537
|
+
FROM message ORDER BY timestamp ASC, id ASC`
|
|
17494
17538
|
).all();
|
|
17495
17539
|
return rows.map((r) => ({
|
|
17540
|
+
platformMessageId: r.platform_message_id || void 0,
|
|
17496
17541
|
senderPlatformId: r.sender_platform_id,
|
|
17497
17542
|
senderAccountName: r.sender_account_name || r.sender_platform_id,
|
|
17498
17543
|
senderGroupNickname: r.sender_group_nickname || void 0,
|
|
17499
17544
|
timestamp: r.timestamp,
|
|
17500
17545
|
type: r.type,
|
|
17501
|
-
content: r.content
|
|
17546
|
+
content: r.content,
|
|
17547
|
+
replyToMessageId: r.reply_to_message_id || void 0
|
|
17502
17548
|
}));
|
|
17503
17549
|
}
|
|
17504
17550
|
close() {
|
|
@@ -17519,7 +17565,8 @@ function createDataSourceFromReader(reader) {
|
|
|
17519
17565
|
platform: meta.platform,
|
|
17520
17566
|
type: meta.type,
|
|
17521
17567
|
groupId: meta.groupId,
|
|
17522
|
-
groupAvatar: meta.groupAvatar
|
|
17568
|
+
groupAvatar: meta.groupAvatar,
|
|
17569
|
+
ownerId: meta.ownerId
|
|
17523
17570
|
};
|
|
17524
17571
|
},
|
|
17525
17572
|
getMembers() {
|
|
@@ -17538,11 +17585,13 @@ function createDataSourceFromReader(reader) {
|
|
|
17538
17585
|
callback(
|
|
17539
17586
|
messages.map((msg) => ({
|
|
17540
17587
|
senderPlatformId: msg.senderPlatformId,
|
|
17588
|
+
platformMessageId: msg.platformMessageId,
|
|
17541
17589
|
senderAccountName: msg.senderAccountName,
|
|
17542
17590
|
senderGroupNickname: msg.senderGroupNickname,
|
|
17543
17591
|
timestamp: msg.timestamp,
|
|
17544
17592
|
type: msg.type,
|
|
17545
|
-
content: msg.content
|
|
17593
|
+
content: msg.content,
|
|
17594
|
+
replyToMessageId: msg.replyToMessageId
|
|
17546
17595
|
}))
|
|
17547
17596
|
);
|
|
17548
17597
|
});
|
|
@@ -17555,8 +17604,9 @@ function exportSessionToJson(db) {
|
|
|
17555
17604
|
const members = db.prepare("SELECT platform_id, account_name, group_nickname, avatar FROM member").all();
|
|
17556
17605
|
const messages = db.prepare(
|
|
17557
17606
|
`SELECT m.platform_id as sender, msg.sender_account_name as accountName,
|
|
17558
|
-
msg.sender_group_nickname as groupNickname, msg.ts as timestamp, msg.type, msg.content
|
|
17559
|
-
|
|
17607
|
+
msg.sender_group_nickname as groupNickname, msg.ts as timestamp, msg.type, msg.content,
|
|
17608
|
+
msg.platform_message_id as platformMessageId, msg.reply_to_message_id as replyToMessageId
|
|
17609
|
+
FROM message msg JOIN member m ON msg.sender_id = m.id ORDER BY msg.ts, msg.id`
|
|
17560
17610
|
).all();
|
|
17561
17611
|
return {
|
|
17562
17612
|
chatlab: {
|
|
@@ -17570,7 +17620,8 @@ function exportSessionToJson(db) {
|
|
|
17570
17620
|
platform: meta.platform,
|
|
17571
17621
|
type: meta.type,
|
|
17572
17622
|
groupId: meta.group_id,
|
|
17573
|
-
groupAvatar: meta.group_avatar
|
|
17623
|
+
groupAvatar: meta.group_avatar,
|
|
17624
|
+
ownerId: meta.owner_id
|
|
17574
17625
|
},
|
|
17575
17626
|
members: members.map((m) => ({
|
|
17576
17627
|
platformId: m.platform_id,
|
|
@@ -17579,32 +17630,34 @@ function exportSessionToJson(db) {
|
|
|
17579
17630
|
avatar: m.avatar
|
|
17580
17631
|
})),
|
|
17581
17632
|
messages: messages.map((msg) => ({
|
|
17633
|
+
platformMessageId: msg.platformMessageId || void 0,
|
|
17582
17634
|
sender: msg.sender,
|
|
17583
17635
|
accountName: msg.accountName || msg.sender,
|
|
17584
17636
|
groupNickname: msg.groupNickname || void 0,
|
|
17585
17637
|
timestamp: msg.timestamp,
|
|
17586
17638
|
type: msg.type,
|
|
17587
|
-
content: msg.content ?? null
|
|
17639
|
+
content: msg.content ?? null,
|
|
17640
|
+
replyToMessageId: msg.replyToMessageId || void 0
|
|
17588
17641
|
}))
|
|
17589
17642
|
};
|
|
17590
17643
|
}
|
|
17591
17644
|
function deleteTempDatabase(dbPath) {
|
|
17592
17645
|
try {
|
|
17593
|
-
if (
|
|
17646
|
+
if (fs27.existsSync(dbPath)) fs27.unlinkSync(dbPath);
|
|
17594
17647
|
const walPath = dbPath + "-wal";
|
|
17595
17648
|
const shmPath = dbPath + "-shm";
|
|
17596
|
-
if (
|
|
17597
|
-
if (
|
|
17649
|
+
if (fs27.existsSync(walPath)) fs27.unlinkSync(walPath);
|
|
17650
|
+
if (fs27.existsSync(shmPath)) fs27.unlinkSync(shmPath);
|
|
17598
17651
|
} catch {
|
|
17599
17652
|
}
|
|
17600
17653
|
}
|
|
17601
17654
|
function cleanupTempDatabases(tempDir) {
|
|
17602
17655
|
try {
|
|
17603
|
-
if (!
|
|
17604
|
-
const files =
|
|
17656
|
+
if (!fs27.existsSync(tempDir)) return;
|
|
17657
|
+
const files = fs27.readdirSync(tempDir);
|
|
17605
17658
|
for (const file of files) {
|
|
17606
17659
|
if (file.startsWith("merge_") && file.endsWith(".db")) {
|
|
17607
|
-
deleteTempDatabase(
|
|
17660
|
+
deleteTempDatabase(path22.join(tempDir, file));
|
|
17608
17661
|
}
|
|
17609
17662
|
}
|
|
17610
17663
|
} catch {
|
|
@@ -17732,10 +17785,7 @@ __export(session_index_service_exports, {
|
|
|
17732
17785
|
clearIndex: () => clearIndex,
|
|
17733
17786
|
generateIncrementalIndex: () => generateIncrementalIndex,
|
|
17734
17787
|
generateIndex: () => generateIndex,
|
|
17735
|
-
getAllIndexStats: () => getAllIndexStats
|
|
17736
|
-
getFtsStatus: () => getFtsStatus,
|
|
17737
|
-
rebuildFts: () => rebuildFts,
|
|
17738
|
-
searchFts: () => searchFts
|
|
17788
|
+
getAllIndexStats: () => getAllIndexStats
|
|
17739
17789
|
});
|
|
17740
17790
|
function generateIndex(adapter, sessionId, gapThreshold = 1800) {
|
|
17741
17791
|
const db = adapter.ensureWritable(sessionId);
|
|
@@ -17749,18 +17799,6 @@ function clearIndex(adapter, sessionId) {
|
|
|
17749
17799
|
const db = adapter.ensureWritable(sessionId);
|
|
17750
17800
|
clearSessionIndex(db);
|
|
17751
17801
|
}
|
|
17752
|
-
function getFtsStatus(adapter, sessionId) {
|
|
17753
|
-
const db = adapter.ensureReadonly(sessionId);
|
|
17754
|
-
return hasFtsTable(db);
|
|
17755
|
-
}
|
|
17756
|
-
function searchFts(adapter, sessionId, keywords, limit, offset) {
|
|
17757
|
-
const db = adapter.ensureReadonly(sessionId);
|
|
17758
|
-
return searchByFts(db, keywords, limit, offset);
|
|
17759
|
-
}
|
|
17760
|
-
function rebuildFts(adapter, sessionId) {
|
|
17761
|
-
const db = adapter.ensureWritable(sessionId);
|
|
17762
|
-
return rebuildFtsIndex(db);
|
|
17763
|
-
}
|
|
17764
17802
|
function getAllIndexStats(adapter) {
|
|
17765
17803
|
const ids = adapter.listSessionIds();
|
|
17766
17804
|
return ids.map((sessionId) => {
|
|
@@ -17989,8 +18027,8 @@ function clearSessionOwner(adapter, sessionId) {
|
|
|
17989
18027
|
}
|
|
17990
18028
|
|
|
17991
18029
|
// ../../packages/node-runtime/src/services/contacts/overrides.ts
|
|
17992
|
-
import
|
|
17993
|
-
import
|
|
18030
|
+
import fs28 from "fs";
|
|
18031
|
+
import path23 from "path";
|
|
17994
18032
|
var CONTACT_OVERRIDES_FILE_NAME = "contact-overrides.json";
|
|
17995
18033
|
var CONTACT_OVERRIDES_TMP_PREFIX = "contact-overrides.tmp-";
|
|
17996
18034
|
var CONTACT_OVERRIDES_VERSION = 1;
|
|
@@ -18001,23 +18039,23 @@ function createEmptyContactOverridesFile() {
|
|
|
18001
18039
|
};
|
|
18002
18040
|
}
|
|
18003
18041
|
function getContactOverridesPath(snapshotDir) {
|
|
18004
|
-
return
|
|
18042
|
+
return path23.join(snapshotDir, CONTACT_OVERRIDES_FILE_NAME);
|
|
18005
18043
|
}
|
|
18006
18044
|
function readContactOverrides(snapshotDir) {
|
|
18007
18045
|
const filePath = getContactOverridesPath(snapshotDir);
|
|
18008
|
-
if (!
|
|
18046
|
+
if (!fs28.existsSync(filePath)) return createEmptyContactOverridesFile();
|
|
18009
18047
|
try {
|
|
18010
|
-
return normalizeContactOverrides(JSON.parse(
|
|
18048
|
+
return normalizeContactOverrides(JSON.parse(fs28.readFileSync(filePath, "utf-8")));
|
|
18011
18049
|
} catch (error) {
|
|
18012
18050
|
appLogger.warn("contacts", "contact overrides file is unreadable", error);
|
|
18013
18051
|
return createEmptyContactOverridesFile();
|
|
18014
18052
|
}
|
|
18015
18053
|
}
|
|
18016
18054
|
function writeContactOverrides(snapshotDir, overrides) {
|
|
18017
|
-
if (!
|
|
18018
|
-
const tmpPath =
|
|
18019
|
-
|
|
18020
|
-
|
|
18055
|
+
if (!fs28.existsSync(snapshotDir)) fs28.mkdirSync(snapshotDir, { recursive: true });
|
|
18056
|
+
const tmpPath = path23.join(snapshotDir, `${CONTACT_OVERRIDES_TMP_PREFIX}${process.pid}-${Date.now()}`);
|
|
18057
|
+
fs28.writeFileSync(tmpPath, JSON.stringify(normalizeContactOverrides(overrides), null, 2), "utf-8");
|
|
18058
|
+
fs28.renameSync(tmpPath, getContactOverridesPath(snapshotDir));
|
|
18021
18059
|
}
|
|
18022
18060
|
function normalizeContactOverrides(value) {
|
|
18023
18061
|
const input = isRecord4(value) ? value : {};
|
|
@@ -18052,23 +18090,23 @@ function buildContactsSignature(adapter, timeRangePreset) {
|
|
|
18052
18090
|
}
|
|
18053
18091
|
|
|
18054
18092
|
// ../../packages/node-runtime/src/services/contacts/snapshot.ts
|
|
18055
|
-
import
|
|
18056
|
-
import
|
|
18093
|
+
import fs29 from "fs";
|
|
18094
|
+
import path24 from "path";
|
|
18057
18095
|
var CONTACTS_SNAPSHOT_TMP_PREFIX = "contacts-snapshot.tmp-";
|
|
18058
18096
|
function getContactsSnapshotPath(snapshotDir, timeRangePreset) {
|
|
18059
18097
|
const preset = normalizeContactsTimeRangePreset(timeRangePreset);
|
|
18060
|
-
return
|
|
18098
|
+
return path24.join(snapshotDir, `contacts-snapshot-${preset}.json`);
|
|
18061
18099
|
}
|
|
18062
18100
|
function readContactsSnapshot(snapshotDir, timeRangePreset, options2 = {}) {
|
|
18063
18101
|
const snapshotPath = getContactsSnapshotPath(snapshotDir, timeRangePreset);
|
|
18064
|
-
if (!
|
|
18102
|
+
if (!fs29.existsSync(snapshotPath)) return null;
|
|
18065
18103
|
try {
|
|
18066
|
-
return JSON.parse(
|
|
18104
|
+
return JSON.parse(fs29.readFileSync(snapshotPath, "utf-8"));
|
|
18067
18105
|
} catch (error) {
|
|
18068
18106
|
const ts = options2.now?.() ?? Date.now();
|
|
18069
|
-
const backupPath =
|
|
18107
|
+
const backupPath = path24.join(snapshotDir, `contacts-snapshot.corrupt-${ts}.json`);
|
|
18070
18108
|
try {
|
|
18071
|
-
|
|
18109
|
+
fs29.renameSync(snapshotPath, backupPath);
|
|
18072
18110
|
} catch (renameError) {
|
|
18073
18111
|
appLogger.warn("contacts", "failed to backup corrupt contacts snapshot", renameError);
|
|
18074
18112
|
}
|
|
@@ -18077,17 +18115,17 @@ function readContactsSnapshot(snapshotDir, timeRangePreset, options2 = {}) {
|
|
|
18077
18115
|
}
|
|
18078
18116
|
}
|
|
18079
18117
|
function writeContactsSnapshot(snapshotDir, snapshot) {
|
|
18080
|
-
if (!
|
|
18081
|
-
const tmpPath =
|
|
18082
|
-
|
|
18083
|
-
|
|
18118
|
+
if (!fs29.existsSync(snapshotDir)) fs29.mkdirSync(snapshotDir, { recursive: true });
|
|
18119
|
+
const tmpPath = path24.join(snapshotDir, `${CONTACTS_SNAPSHOT_TMP_PREFIX}${process.pid}-${Date.now()}`);
|
|
18120
|
+
fs29.writeFileSync(tmpPath, JSON.stringify(snapshot, null, 2), "utf-8");
|
|
18121
|
+
fs29.renameSync(tmpPath, getContactsSnapshotPath(snapshotDir, snapshot.timeRange.preset));
|
|
18084
18122
|
}
|
|
18085
18123
|
function cleanupContactsSnapshotTempFiles(snapshotDir) {
|
|
18086
|
-
if (!
|
|
18087
|
-
for (const name of
|
|
18124
|
+
if (!fs29.existsSync(snapshotDir)) return;
|
|
18125
|
+
for (const name of fs29.readdirSync(snapshotDir)) {
|
|
18088
18126
|
if (!name.startsWith(CONTACTS_SNAPSHOT_TMP_PREFIX)) continue;
|
|
18089
18127
|
try {
|
|
18090
|
-
|
|
18128
|
+
fs29.rmSync(path24.join(snapshotDir, name), { force: true });
|
|
18091
18129
|
} catch (error) {
|
|
18092
18130
|
appLogger.warn("contacts", "failed to remove contacts snapshot temp file", error);
|
|
18093
18131
|
}
|
|
@@ -18096,8 +18134,8 @@ function cleanupContactsSnapshotTempFiles(snapshotDir) {
|
|
|
18096
18134
|
|
|
18097
18135
|
// ../../packages/node-runtime/src/services/contacts/worker-runner.ts
|
|
18098
18136
|
import { Worker } from "worker_threads";
|
|
18099
|
-
import { existsSync as
|
|
18100
|
-
function resolveDefaultContactsWorkerEntryUrl(currentModuleUrl = import.meta.url, entryExists = (url) =>
|
|
18137
|
+
import { existsSync as existsSync8 } from "fs";
|
|
18138
|
+
function resolveDefaultContactsWorkerEntryUrl(currentModuleUrl = import.meta.url, entryExists = (url) => existsSync8(url)) {
|
|
18101
18139
|
const moduleUrl = typeof currentModuleUrl === "string" ? currentModuleUrl : currentModuleUrl.href;
|
|
18102
18140
|
if (moduleUrl.endsWith(".ts")) return new URL("./worker-entry.ts", moduleUrl);
|
|
18103
18141
|
if (moduleUrl.endsWith(".mjs")) return new URL("./contacts-worker.mjs", moduleUrl);
|
|
@@ -18585,23 +18623,23 @@ function buildPeopleRelationshipsSignature(adapter, timeRangePreset) {
|
|
|
18585
18623
|
}
|
|
18586
18624
|
|
|
18587
18625
|
// ../../packages/node-runtime/src/services/people/relationships/snapshot.ts
|
|
18588
|
-
import
|
|
18589
|
-
import
|
|
18626
|
+
import fs30 from "fs";
|
|
18627
|
+
import path25 from "path";
|
|
18590
18628
|
var PEOPLE_RELATIONSHIPS_SNAPSHOT_TMP_PREFIX = "people-relationships-snapshot.tmp-";
|
|
18591
18629
|
function getPeopleRelationshipsSnapshotPath(snapshotDir, timeRangePreset) {
|
|
18592
18630
|
const preset = normalizePeopleRelationshipsTimeRangePreset(timeRangePreset);
|
|
18593
|
-
return
|
|
18631
|
+
return path25.join(snapshotDir, `graph-snapshot-${preset}.json`);
|
|
18594
18632
|
}
|
|
18595
18633
|
function readPeopleRelationshipsSnapshot(snapshotDir, timeRangePreset, options2 = {}) {
|
|
18596
18634
|
const snapshotPath = getPeopleRelationshipsSnapshotPath(snapshotDir, timeRangePreset);
|
|
18597
|
-
if (!
|
|
18635
|
+
if (!fs30.existsSync(snapshotPath)) return null;
|
|
18598
18636
|
try {
|
|
18599
|
-
return JSON.parse(
|
|
18637
|
+
return JSON.parse(fs30.readFileSync(snapshotPath, "utf-8"));
|
|
18600
18638
|
} catch (error) {
|
|
18601
18639
|
const ts = options2.now?.() ?? Date.now();
|
|
18602
|
-
const backupPath =
|
|
18640
|
+
const backupPath = path25.join(snapshotDir, `graph-snapshot.corrupt-${ts}.json`);
|
|
18603
18641
|
try {
|
|
18604
|
-
|
|
18642
|
+
fs30.renameSync(snapshotPath, backupPath);
|
|
18605
18643
|
} catch (renameError) {
|
|
18606
18644
|
appLogger.warn("people-relationships", "failed to backup corrupt people relationships snapshot", renameError);
|
|
18607
18645
|
}
|
|
@@ -18610,17 +18648,17 @@ function readPeopleRelationshipsSnapshot(snapshotDir, timeRangePreset, options2
|
|
|
18610
18648
|
}
|
|
18611
18649
|
}
|
|
18612
18650
|
function writePeopleRelationshipsSnapshot(snapshotDir, snapshot) {
|
|
18613
|
-
if (!
|
|
18614
|
-
const tmpPath =
|
|
18615
|
-
|
|
18616
|
-
|
|
18651
|
+
if (!fs30.existsSync(snapshotDir)) fs30.mkdirSync(snapshotDir, { recursive: true });
|
|
18652
|
+
const tmpPath = path25.join(snapshotDir, `${PEOPLE_RELATIONSHIPS_SNAPSHOT_TMP_PREFIX}${process.pid}-${Date.now()}`);
|
|
18653
|
+
fs30.writeFileSync(tmpPath, JSON.stringify(snapshot, null, 2), "utf-8");
|
|
18654
|
+
fs30.renameSync(tmpPath, getPeopleRelationshipsSnapshotPath(snapshotDir, snapshot.timeRange.preset));
|
|
18617
18655
|
}
|
|
18618
18656
|
function cleanupPeopleRelationshipsSnapshotTempFiles(snapshotDir) {
|
|
18619
|
-
if (!
|
|
18620
|
-
for (const name of
|
|
18657
|
+
if (!fs30.existsSync(snapshotDir)) return;
|
|
18658
|
+
for (const name of fs30.readdirSync(snapshotDir)) {
|
|
18621
18659
|
if (!name.startsWith(PEOPLE_RELATIONSHIPS_SNAPSHOT_TMP_PREFIX)) continue;
|
|
18622
18660
|
try {
|
|
18623
|
-
|
|
18661
|
+
fs30.rmSync(path25.join(snapshotDir, name), { force: true });
|
|
18624
18662
|
} catch (error) {
|
|
18625
18663
|
appLogger.warn("people-relationships", "failed to remove people relationships snapshot temp file", error);
|
|
18626
18664
|
}
|
|
@@ -18628,9 +18666,9 @@ function cleanupPeopleRelationshipsSnapshotTempFiles(snapshotDir) {
|
|
|
18628
18666
|
}
|
|
18629
18667
|
|
|
18630
18668
|
// ../../packages/node-runtime/src/services/people/relationships/worker-runner.ts
|
|
18631
|
-
import { existsSync as
|
|
18669
|
+
import { existsSync as existsSync9 } from "fs";
|
|
18632
18670
|
import { Worker as Worker2 } from "worker_threads";
|
|
18633
|
-
function resolveDefaultPeopleRelationshipsWorkerEntryUrl(currentModuleUrl = import.meta.url, entryExists = (url) =>
|
|
18671
|
+
function resolveDefaultPeopleRelationshipsWorkerEntryUrl(currentModuleUrl = import.meta.url, entryExists = (url) => existsSync9(url)) {
|
|
18634
18672
|
const moduleUrl = typeof currentModuleUrl === "string" ? currentModuleUrl : currentModuleUrl.href;
|
|
18635
18673
|
if (moduleUrl.endsWith(".ts")) return new URL("./worker-entry.ts", moduleUrl);
|
|
18636
18674
|
if (moduleUrl.endsWith(".mjs")) return new URL("./people-relationships-worker.mjs", moduleUrl);
|
|
@@ -19155,7 +19193,7 @@ __export(relationships_exports, {
|
|
|
19155
19193
|
});
|
|
19156
19194
|
|
|
19157
19195
|
// ../../packages/node-runtime/src/services/global-insight/service.ts
|
|
19158
|
-
import
|
|
19196
|
+
import fs31 from "fs";
|
|
19159
19197
|
|
|
19160
19198
|
// ../../packages/node-runtime/src/services/global-insight/signature.ts
|
|
19161
19199
|
function buildAnnualSummarySignature(adapter, range) {
|
|
@@ -19172,9 +19210,9 @@ function buildAnnualSummarySignature(adapter, range) {
|
|
|
19172
19210
|
}
|
|
19173
19211
|
|
|
19174
19212
|
// ../../packages/node-runtime/src/services/global-insight/worker-runner.ts
|
|
19175
|
-
import { existsSync as
|
|
19213
|
+
import { existsSync as existsSync10 } from "fs";
|
|
19176
19214
|
import { Worker as Worker3 } from "worker_threads";
|
|
19177
|
-
function resolveDefaultAnnualSummaryWorkerEntryUrl(currentModuleUrl = import.meta.url, entryExists = (url) =>
|
|
19215
|
+
function resolveDefaultAnnualSummaryWorkerEntryUrl(currentModuleUrl = import.meta.url, entryExists = (url) => existsSync10(url)) {
|
|
19178
19216
|
const moduleUrl = typeof currentModuleUrl === "string" ? currentModuleUrl : currentModuleUrl.href;
|
|
19179
19217
|
if (moduleUrl.endsWith(".ts")) return new URL("./worker-entry.ts", moduleUrl);
|
|
19180
19218
|
if (moduleUrl.endsWith(".mjs")) return new URL("./global-insight-worker.mjs", moduleUrl);
|
|
@@ -19409,7 +19447,7 @@ var DefaultGlobalInsightService = class {
|
|
|
19409
19447
|
}
|
|
19410
19448
|
getSnapshot(range) {
|
|
19411
19449
|
const key = toAnnualSummaryRangeKey(range);
|
|
19412
|
-
if (this.snapshots.has(key) && !
|
|
19450
|
+
if (this.snapshots.has(key) && !fs31.existsSync(getAnnualSummarySnapshotPath(this.snapshotDir, range))) {
|
|
19413
19451
|
this.snapshots.delete(key);
|
|
19414
19452
|
}
|
|
19415
19453
|
if (!this.snapshots.has(key)) {
|
|
@@ -19457,8 +19495,8 @@ __export(global_insight_exports, {
|
|
|
19457
19495
|
});
|
|
19458
19496
|
|
|
19459
19497
|
// ../../packages/node-runtime/src/semantic-index/config.ts
|
|
19460
|
-
import
|
|
19461
|
-
import
|
|
19498
|
+
import fs32 from "fs";
|
|
19499
|
+
import path26 from "path";
|
|
19462
19500
|
var DEFAULT_SEMANTIC_INDEX_MODEL_DOWNLOAD_SOURCE = "huggingface";
|
|
19463
19501
|
var SEMANTIC_INDEX_CONFIG_VERSION = 1;
|
|
19464
19502
|
var SEARCH_MAX_RESULTS_DEFAULT = 10;
|
|
@@ -19547,9 +19585,9 @@ var SemanticIndexConfigStore = class {
|
|
|
19547
19585
|
return this.set({ ...this.get(), enabled });
|
|
19548
19586
|
}
|
|
19549
19587
|
get() {
|
|
19550
|
-
if (!
|
|
19588
|
+
if (!fs32.existsSync(this.filePath)) return defaultSemanticIndexConfig();
|
|
19551
19589
|
try {
|
|
19552
|
-
const raw = JSON.parse(
|
|
19590
|
+
const raw = JSON.parse(fs32.readFileSync(this.filePath, "utf-8"));
|
|
19553
19591
|
return normalizeConfig(raw);
|
|
19554
19592
|
} catch {
|
|
19555
19593
|
return defaultSemanticIndexConfig();
|
|
@@ -19557,9 +19595,9 @@ var SemanticIndexConfigStore = class {
|
|
|
19557
19595
|
}
|
|
19558
19596
|
set(config) {
|
|
19559
19597
|
const normalized = normalizeConfig(config);
|
|
19560
|
-
const dir =
|
|
19561
|
-
if (!
|
|
19562
|
-
|
|
19598
|
+
const dir = path26.dirname(this.filePath);
|
|
19599
|
+
if (!fs32.existsSync(dir)) fs32.mkdirSync(dir, { recursive: true });
|
|
19600
|
+
fs32.writeFileSync(this.filePath, JSON.stringify(normalized, null, 2), "utf-8");
|
|
19563
19601
|
return normalized;
|
|
19564
19602
|
}
|
|
19565
19603
|
resolveModelId() {
|
|
@@ -19687,10 +19725,10 @@ var LocalEmbeddingProvider = class {
|
|
|
19687
19725
|
};
|
|
19688
19726
|
|
|
19689
19727
|
// ../../packages/node-runtime/src/semantic-index/service.ts
|
|
19690
|
-
import
|
|
19728
|
+
import path27 from "path";
|
|
19691
19729
|
|
|
19692
19730
|
// ../../packages/node-runtime/src/semantic-index/chunker-config.ts
|
|
19693
|
-
import { createHash as
|
|
19731
|
+
import { createHash as createHash2 } from "crypto";
|
|
19694
19732
|
var CHUNKER_VERSION = "v1.3";
|
|
19695
19733
|
var STRATEGY_ID = "balanced";
|
|
19696
19734
|
var DEFAULT_CHUNKER_CONFIG = {
|
|
@@ -19705,7 +19743,7 @@ var DEFAULT_CHUNKER_CONFIG = {
|
|
|
19705
19743
|
semanticVoidSkipThreshold: 20
|
|
19706
19744
|
};
|
|
19707
19745
|
function sha256Hex(input) {
|
|
19708
|
-
return
|
|
19746
|
+
return createHash2("sha256").update(input, "utf-8").digest("hex");
|
|
19709
19747
|
}
|
|
19710
19748
|
function computeChunkerConfigHash(config) {
|
|
19711
19749
|
const canonical = JSON.stringify({
|
|
@@ -20097,32 +20135,6 @@ var EmbeddingIndexStore = class {
|
|
|
20097
20135
|
).all(toFloat32Buffer(params.embedding), params.dbPathHash, params.modelId, params.k);
|
|
20098
20136
|
return rows.map((row) => ({ chunkId: row.chunk_id, distance: row.distance, record: rowToRecord(row) }));
|
|
20099
20137
|
}
|
|
20100
|
-
/**
|
|
20101
|
-
* FTS message -> chunk 映射。按 (ts, id) 复合查找:取 (start_ts, start_message_id) <= (messageTs, messageId)
|
|
20102
|
-
* 的最大值,再校验 end 覆盖。
|
|
20103
|
-
* - 用 ts 而非 ID 做主键:回填旧消息后 ID 不再单调
|
|
20104
|
-
* - 用 (ts, id) 复合而非仅 ts:同秒多 chunk 时 ts 相同,需 id 打破 tie
|
|
20105
|
-
*/
|
|
20106
|
-
mapMessageToChunk(params) {
|
|
20107
|
-
const row = this.db.prepare(
|
|
20108
|
-
`SELECT ${CHUNK_COLUMNS} FROM chunk_vector_index
|
|
20109
|
-
WHERE db_path_hash = ? AND model_id = ? AND strategy_id = ?
|
|
20110
|
-
AND (start_ts < CAST(? AS INTEGER)
|
|
20111
|
-
OR (start_ts = CAST(? AS INTEGER) AND start_message_id <= CAST(? AS INTEGER)))
|
|
20112
|
-
ORDER BY start_ts DESC, start_message_id DESC LIMIT 1`
|
|
20113
|
-
).get(
|
|
20114
|
-
params.dbPathHash,
|
|
20115
|
-
params.modelId,
|
|
20116
|
-
params.strategyId,
|
|
20117
|
-
params.messageTs,
|
|
20118
|
-
params.messageTs,
|
|
20119
|
-
params.messageId
|
|
20120
|
-
);
|
|
20121
|
-
if (!row) return null;
|
|
20122
|
-
if (row.end_ts < params.messageTs) return null;
|
|
20123
|
-
if (row.end_ts === params.messageTs && row.end_message_id < params.messageId) return null;
|
|
20124
|
-
return rowToRecord(row);
|
|
20125
|
-
}
|
|
20126
20138
|
/** 删除某聊天库(全模型)的所有 chunk 元数据与向量,返回删除的元数据行数 */
|
|
20127
20139
|
deleteByDbPathHash(dbPathHash) {
|
|
20128
20140
|
return this.db.transaction(() => {
|
|
@@ -20419,7 +20431,7 @@ var SemanticIndexJobQueue = class {
|
|
|
20419
20431
|
};
|
|
20420
20432
|
|
|
20421
20433
|
// ../../packages/node-runtime/src/semantic-index/chunker.ts
|
|
20422
|
-
import { createHash as
|
|
20434
|
+
import { createHash as createHash3 } from "crypto";
|
|
20423
20435
|
var VOID_MESSAGE_TYPES = /* @__PURE__ */ new Set([
|
|
20424
20436
|
1 /* IMAGE */,
|
|
20425
20437
|
2 /* VOICE */,
|
|
@@ -20502,7 +20514,7 @@ function sumEffectiveChars(messages) {
|
|
|
20502
20514
|
return total;
|
|
20503
20515
|
}
|
|
20504
20516
|
function sha256Hex2(input) {
|
|
20505
|
-
return
|
|
20517
|
+
return createHash3("sha256").update(input, "utf-8").digest("hex");
|
|
20506
20518
|
}
|
|
20507
20519
|
var EMBEDDING_HEADER_BUDGET_RATIO = 0.25;
|
|
20508
20520
|
var SOURCE_TITLE_HEADER_SHARE = 1 / 3;
|
|
@@ -20773,21 +20785,7 @@ async function runWarmup(options2) {
|
|
|
20773
20785
|
}
|
|
20774
20786
|
}
|
|
20775
20787
|
|
|
20776
|
-
// ../../packages/node-runtime/src/semantic-index/retrieval/
|
|
20777
|
-
function reciprocalRankFusion(rankedLists, k = 60) {
|
|
20778
|
-
const scores = /* @__PURE__ */ new Map();
|
|
20779
|
-
const firstSeen = /* @__PURE__ */ new Map();
|
|
20780
|
-
let seq = 0;
|
|
20781
|
-
for (const list of rankedLists) {
|
|
20782
|
-
list.forEach((id, rank) => {
|
|
20783
|
-
scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank));
|
|
20784
|
-
if (!firstSeen.has(id)) firstSeen.set(id, seq++);
|
|
20785
|
-
});
|
|
20786
|
-
}
|
|
20787
|
-
return [...scores.entries()].map(([id, score]) => ({ id, score })).sort((a, b) => b.score - a.score || firstSeen.get(a.id) - firstSeen.get(b.id));
|
|
20788
|
-
}
|
|
20789
|
-
|
|
20790
|
-
// ../../packages/node-runtime/src/semantic-index/retrieval/hybrid-search.ts
|
|
20788
|
+
// ../../packages/node-runtime/src/semantic-index/retrieval/semantic-search.ts
|
|
20791
20789
|
function overlapsTimeRangeMs(record, filter) {
|
|
20792
20790
|
if (!filter) return true;
|
|
20793
20791
|
if (filter.startTs != null && record.endTs < filter.startTs) return false;
|
|
@@ -20795,46 +20793,22 @@ function overlapsTimeRangeMs(record, filter) {
|
|
|
20795
20793
|
return true;
|
|
20796
20794
|
}
|
|
20797
20795
|
var TIME_FILTER_POOL_MULTIPLIER = 10;
|
|
20798
|
-
async function
|
|
20799
|
-
const { embedder, store
|
|
20800
|
-
const { query, dbPathHash, modelId,
|
|
20796
|
+
async function semanticSearch(deps, params) {
|
|
20797
|
+
const { embedder, store } = deps;
|
|
20798
|
+
const { query, dbPathHash, modelId, dim, finalTopK = 5, timeRangeMs } = params;
|
|
20801
20799
|
if (!query.trim()) return [];
|
|
20802
20800
|
const poolFactor = timeRangeMs ? TIME_FILTER_POOL_MULTIPLIER : 1;
|
|
20803
20801
|
const denseTopN = Math.max((params.denseTopN ?? 40) * poolFactor, 40);
|
|
20804
|
-
const ftsTopN = Math.max((params.ftsTopN ?? 40) * poolFactor, 40);
|
|
20805
|
-
const records = /* @__PURE__ */ new Map();
|
|
20806
20802
|
const queryVector = await embedder.embedQuery(query);
|
|
20807
20803
|
const dense = store.queryDense({ dbPathHash, modelId, dim, embedding: queryVector, k: denseTopN });
|
|
20808
|
-
const
|
|
20809
|
-
const denseRankById = /* @__PURE__ */ new Map();
|
|
20804
|
+
const results = [];
|
|
20810
20805
|
for (const hit of dense) {
|
|
20811
20806
|
if (!overlapsTimeRangeMs(hit.record, timeRangeMs)) continue;
|
|
20812
|
-
denseRankById.set(hit.chunkId, denseIds.length);
|
|
20813
|
-
denseIds.push(hit.chunkId);
|
|
20814
|
-
records.set(hit.chunkId, hit.record);
|
|
20815
|
-
}
|
|
20816
|
-
const ftsIds = [];
|
|
20817
|
-
const ftsRankById = /* @__PURE__ */ new Map();
|
|
20818
|
-
for (const { id: messageId, ts: messageTs } of fts.search(query, ftsTopN)) {
|
|
20819
|
-
const record = store.mapMessageToChunk({ dbPathHash, modelId, strategyId, messageId, messageTs });
|
|
20820
|
-
if (!record || ftsRankById.has(record.chunkId)) continue;
|
|
20821
|
-
if (!overlapsTimeRangeMs(record, timeRangeMs)) continue;
|
|
20822
|
-
ftsRankById.set(record.chunkId, ftsIds.length);
|
|
20823
|
-
ftsIds.push(record.chunkId);
|
|
20824
|
-
if (!records.has(record.chunkId)) records.set(record.chunkId, record);
|
|
20825
|
-
}
|
|
20826
|
-
const fused = reciprocalRankFusion([denseIds, ftsIds], rrfK);
|
|
20827
|
-
const results = [];
|
|
20828
|
-
for (const { id, score } of fused) {
|
|
20829
|
-
const record = records.get(id) ?? store.getChunkById(id);
|
|
20830
|
-
if (!record) continue;
|
|
20831
|
-
if (!overlapsTimeRangeMs(record, timeRangeMs)) continue;
|
|
20832
20807
|
results.push({
|
|
20833
|
-
chunkId:
|
|
20834
|
-
score,
|
|
20835
|
-
record,
|
|
20836
|
-
denseRank:
|
|
20837
|
-
ftsRank: ftsRankById.get(id) ?? null
|
|
20808
|
+
chunkId: hit.chunkId,
|
|
20809
|
+
score: 1 / (1 + Math.max(0, hit.distance)),
|
|
20810
|
+
record: hit.record,
|
|
20811
|
+
denseRank: results.length
|
|
20838
20812
|
});
|
|
20839
20813
|
if (results.length >= finalTopK) break;
|
|
20840
20814
|
}
|
|
@@ -21014,35 +20988,6 @@ function createChatDbMessageRangeReader(db) {
|
|
|
21014
20988
|
};
|
|
21015
20989
|
}
|
|
21016
20990
|
|
|
21017
|
-
// ../../packages/node-runtime/src/semantic-index/chat-db/fts-searcher.ts
|
|
21018
|
-
function extractFtsKeywords(query) {
|
|
21019
|
-
const trimmed = query.trim();
|
|
21020
|
-
if (!trimmed) return [];
|
|
21021
|
-
try {
|
|
21022
|
-
const tokens = getJieba().cut(trimmed, false).map((t2) => t2.trim()).filter((t2) => t2.length > 0);
|
|
21023
|
-
return [...new Set(tokens)];
|
|
21024
|
-
} catch {
|
|
21025
|
-
return [...new Set(trimmed.split(/\s+/).filter((t2) => t2.length > 0))];
|
|
21026
|
-
}
|
|
21027
|
-
}
|
|
21028
|
-
function createChatDbFtsSearcher(db) {
|
|
21029
|
-
return {
|
|
21030
|
-
search(query, topN) {
|
|
21031
|
-
const keywords = extractFtsKeywords(query);
|
|
21032
|
-
if (keywords.length === 0) return [];
|
|
21033
|
-
const { rowids } = searchByFts(db, keywords, topN, 0);
|
|
21034
|
-
if (rowids.length === 0) return [];
|
|
21035
|
-
const placeholders = rowids.map(() => "?").join(", ");
|
|
21036
|
-
const tsRows = db.prepare(`SELECT id, ts FROM message WHERE id IN (${placeholders})`).all(...rowids);
|
|
21037
|
-
const tsById = new Map(tsRows.map((r) => [r.id, r.ts * 1e3]));
|
|
21038
|
-
return rowids.flatMap((id) => {
|
|
21039
|
-
const ts = tsById.get(id);
|
|
21040
|
-
return ts !== void 0 ? [{ id, ts }] : [];
|
|
21041
|
-
});
|
|
21042
|
-
}
|
|
21043
|
-
};
|
|
21044
|
-
}
|
|
21045
|
-
|
|
21046
20991
|
// ../../packages/node-runtime/src/semantic-index/service.ts
|
|
21047
20992
|
var SEMANTIC_INDEX_AUTH_PROFILE = "semantic-index-embedding";
|
|
21048
20993
|
function persistSemanticIndexConfig(configStore, config, options2) {
|
|
@@ -21064,16 +21009,16 @@ function resolveSearchToolEvidenceTokens(maxResultTokens) {
|
|
|
21064
21009
|
return maxResultTokens && maxResultTokens > 0 ? Math.min(SEARCH_TOOL_EVIDENCE_TOKENS, maxResultTokens) : SEARCH_TOOL_EVIDENCE_TOKENS;
|
|
21065
21010
|
}
|
|
21066
21011
|
function sessionIdFromDbPath(dbPath) {
|
|
21067
|
-
return
|
|
21012
|
+
return path27.basename(dbPath, ".db");
|
|
21068
21013
|
}
|
|
21069
21014
|
var SEMANTIC_INDEX_DB_FILE = "embedding_index.db";
|
|
21070
21015
|
var SEMANTIC_INDEX_CONFIG_FILE = "semantic-index-config.json";
|
|
21071
21016
|
function createSemanticIndexService(params) {
|
|
21072
21017
|
const { pathProvider } = params;
|
|
21073
21018
|
return new SemanticIndexService({
|
|
21074
|
-
vectorDbPath:
|
|
21075
|
-
configPath:
|
|
21076
|
-
modelsCacheDir:
|
|
21019
|
+
vectorDbPath: path27.join(pathProvider.getVectorDir(), SEMANTIC_INDEX_DB_FILE),
|
|
21020
|
+
configPath: path27.join(pathProvider.getAiDataDir(), SEMANTIC_INDEX_CONFIG_FILE),
|
|
21021
|
+
modelsCacheDir: path27.join(pathProvider.getAiDataDir(), "models", "semantic-index"),
|
|
21077
21022
|
modelDownloadProxyUrl: params.modelDownloadProxyUrl,
|
|
21078
21023
|
sessionAdapter: params.sessionAdapter,
|
|
21079
21024
|
nativeBinding: params.nativeBinding,
|
|
@@ -21363,20 +21308,17 @@ var SemanticIndexService = class {
|
|
|
21363
21308
|
const db = this.sessionAdapter.openReadonly(sessionId);
|
|
21364
21309
|
if (!db) return { available: false, reason: "not-found", blocks: [], coverage: 0, partial: false, hitCount: 0 };
|
|
21365
21310
|
const embedder = this.getEmbedder();
|
|
21366
|
-
const fts = createChatDbFtsSearcher(db);
|
|
21367
21311
|
const finalTopK = options2?.finalTopK ?? 5;
|
|
21368
21312
|
const candidateTopN = Math.max(40, finalTopK * 4);
|
|
21369
|
-
const hits = await
|
|
21370
|
-
{ embedder, store: this.store
|
|
21313
|
+
const hits = await semanticSearch(
|
|
21314
|
+
{ embedder, store: this.store },
|
|
21371
21315
|
{
|
|
21372
21316
|
query,
|
|
21373
21317
|
dbPathHash: hash,
|
|
21374
21318
|
modelId,
|
|
21375
|
-
strategyId: STRATEGY_ID,
|
|
21376
21319
|
dim,
|
|
21377
21320
|
finalTopK,
|
|
21378
21321
|
denseTopN: candidateTopN,
|
|
21379
|
-
ftsTopN: candidateTopN,
|
|
21380
21322
|
timeRangeMs: options2?.timeRangeMs
|
|
21381
21323
|
}
|
|
21382
21324
|
);
|
|
@@ -21635,9 +21577,9 @@ ${text}`;
|
|
|
21635
21577
|
|
|
21636
21578
|
// ../../packages/node-runtime/src/semantic-index/embedding/local-runtime.ts
|
|
21637
21579
|
import { execFile } from "child_process";
|
|
21638
|
-
import
|
|
21580
|
+
import fs33 from "fs";
|
|
21639
21581
|
import { createRequire as createRequire2 } from "module";
|
|
21640
|
-
import
|
|
21582
|
+
import path28 from "path";
|
|
21641
21583
|
import { pathToFileURL } from "url";
|
|
21642
21584
|
var LOCAL_EMBEDDING_RUNTIME_PACKAGE = "@huggingface/transformers";
|
|
21643
21585
|
var LOCAL_EMBEDDING_RUNTIME_VERSION = "4.2.0";
|
|
@@ -21705,7 +21647,7 @@ function verifyRuntimeInSubprocess(runtimeDir) {
|
|
|
21705
21647
|
});
|
|
21706
21648
|
}
|
|
21707
21649
|
function createRuntimeRequire(runtimeDir) {
|
|
21708
|
-
return createRequire2(
|
|
21650
|
+
return createRequire2(path28.join(runtimeDir, "package.json"));
|
|
21709
21651
|
}
|
|
21710
21652
|
async function importFromRuntime(runtimeDir, packageName) {
|
|
21711
21653
|
const resolved = createRuntimeRequire(runtimeDir).resolve(packageName);
|
|
@@ -21757,7 +21699,7 @@ var LocalEmbeddingRuntimeManager = class {
|
|
|
21757
21699
|
return this.status;
|
|
21758
21700
|
}
|
|
21759
21701
|
getRuntimeDir() {
|
|
21760
|
-
return
|
|
21702
|
+
return path28.join(this.baseDir, `transformers-${LOCAL_EMBEDDING_RUNTIME_VERSION}`, `${this.platform}-${this.arch}`);
|
|
21761
21703
|
}
|
|
21762
21704
|
async ensureInstalled() {
|
|
21763
21705
|
if (!this.installPromise) {
|
|
@@ -21792,7 +21734,7 @@ var LocalEmbeddingRuntimeManager = class {
|
|
|
21792
21734
|
arch: this.arch,
|
|
21793
21735
|
error: error instanceof Error ? error.message : String(error)
|
|
21794
21736
|
});
|
|
21795
|
-
|
|
21737
|
+
fs33.rmSync(runtimeDir, { recursive: true, force: true });
|
|
21796
21738
|
this.verifiedRuntimeDir = null;
|
|
21797
21739
|
}
|
|
21798
21740
|
}
|
|
@@ -21821,12 +21763,12 @@ var LocalEmbeddingRuntimeManager = class {
|
|
|
21821
21763
|
}
|
|
21822
21764
|
}
|
|
21823
21765
|
async installAtomically(runtimeDir) {
|
|
21824
|
-
const parentDir =
|
|
21825
|
-
|
|
21826
|
-
const stagingDir =
|
|
21766
|
+
const parentDir = path28.dirname(runtimeDir);
|
|
21767
|
+
fs33.mkdirSync(parentDir, { recursive: true });
|
|
21768
|
+
const stagingDir = fs33.mkdtempSync(path28.join(parentDir, ".installing-"));
|
|
21827
21769
|
try {
|
|
21828
|
-
|
|
21829
|
-
|
|
21770
|
+
fs33.writeFileSync(
|
|
21771
|
+
path28.join(stagingDir, "package.json"),
|
|
21830
21772
|
`${JSON.stringify({ private: true, type: "module" }, null, 2)}
|
|
21831
21773
|
`,
|
|
21832
21774
|
"utf8"
|
|
@@ -21847,19 +21789,19 @@ var LocalEmbeddingRuntimeManager = class {
|
|
|
21847
21789
|
cwd: stagingDir
|
|
21848
21790
|
});
|
|
21849
21791
|
await this.verifyStagedRuntime(stagingDir);
|
|
21850
|
-
|
|
21792
|
+
fs33.writeFileSync(path28.join(stagingDir, RUNTIME_MANIFEST_FILE), `${JSON.stringify(this.manifest(), null, 2)}
|
|
21851
21793
|
`);
|
|
21852
|
-
if (
|
|
21794
|
+
if (fs33.existsSync(runtimeDir)) {
|
|
21853
21795
|
if (this.hasCompleteLayout(runtimeDir)) return;
|
|
21854
|
-
|
|
21796
|
+
fs33.rmSync(runtimeDir, { recursive: true, force: true });
|
|
21855
21797
|
}
|
|
21856
21798
|
try {
|
|
21857
|
-
|
|
21799
|
+
fs33.renameSync(stagingDir, runtimeDir);
|
|
21858
21800
|
} catch (error) {
|
|
21859
21801
|
if (!this.hasCompleteLayout(runtimeDir)) throw error;
|
|
21860
21802
|
}
|
|
21861
21803
|
} finally {
|
|
21862
|
-
|
|
21804
|
+
fs33.rmSync(stagingDir, { recursive: true, force: true });
|
|
21863
21805
|
}
|
|
21864
21806
|
}
|
|
21865
21807
|
async verifyOnce(runtimeDir) {
|
|
@@ -21868,11 +21810,11 @@ var LocalEmbeddingRuntimeManager = class {
|
|
|
21868
21810
|
this.verifiedRuntimeDir = runtimeDir;
|
|
21869
21811
|
}
|
|
21870
21812
|
hasCompleteLayout(runtimeDir) {
|
|
21871
|
-
const manifestPath =
|
|
21872
|
-
const packagePath =
|
|
21873
|
-
if (!
|
|
21813
|
+
const manifestPath = path28.join(runtimeDir, RUNTIME_MANIFEST_FILE);
|
|
21814
|
+
const packagePath = path28.join(runtimeDir, "node_modules", "@huggingface", "transformers", "package.json");
|
|
21815
|
+
if (!fs33.existsSync(manifestPath) || !fs33.existsSync(packagePath)) return false;
|
|
21874
21816
|
try {
|
|
21875
|
-
const manifest = JSON.parse(
|
|
21817
|
+
const manifest = JSON.parse(fs33.readFileSync(manifestPath, "utf8"));
|
|
21876
21818
|
const expected = this.manifest();
|
|
21877
21819
|
return manifest.formatVersion === expected.formatVersion && manifest.packageName === expected.packageName && manifest.packageVersion === expected.packageVersion && manifest.platform === expected.platform && manifest.arch === expected.arch;
|
|
21878
21820
|
} catch {
|
|
@@ -21894,8 +21836,8 @@ function createLocalEmbeddingRuntimeManager(options2) {
|
|
|
21894
21836
|
}
|
|
21895
21837
|
|
|
21896
21838
|
// ../../packages/node-runtime/src/services/merge-cache.ts
|
|
21897
|
-
import * as
|
|
21898
|
-
import * as
|
|
21839
|
+
import * as fs34 from "fs";
|
|
21840
|
+
import * as path29 from "path";
|
|
21899
21841
|
import * as crypto from "crypto";
|
|
21900
21842
|
var MergeSessionCache = class {
|
|
21901
21843
|
cache = /* @__PURE__ */ new Map();
|
|
@@ -21904,15 +21846,15 @@ var MergeSessionCache = class {
|
|
|
21904
21846
|
constructor(pathProvider, options2) {
|
|
21905
21847
|
this.tempDir = pathProvider.getTempDir();
|
|
21906
21848
|
this.nativeBinding = options2?.nativeBinding;
|
|
21907
|
-
if (!
|
|
21908
|
-
|
|
21849
|
+
if (!fs34.existsSync(this.tempDir)) {
|
|
21850
|
+
fs34.mkdirSync(this.tempDir, { recursive: true });
|
|
21909
21851
|
}
|
|
21910
21852
|
}
|
|
21911
21853
|
generateTempDbPath(filename) {
|
|
21912
21854
|
const ts = Date.now();
|
|
21913
21855
|
const rand = Math.random().toString(36).slice(2, 8);
|
|
21914
21856
|
const safeName = filename.replace(/[/\\?%*:|"<>]/g, "_").slice(0, 40);
|
|
21915
|
-
return
|
|
21857
|
+
return path29.join(this.tempDir, `merge_${safeName}_${ts}_${rand}.db`);
|
|
21916
21858
|
}
|
|
21917
21859
|
/** Store a parsed file's temp DB and return a handle for subsequent requests. */
|
|
21918
21860
|
store(filename, tempDbPath) {
|
|
@@ -21923,7 +21865,7 @@ var MergeSessionCache = class {
|
|
|
21923
21865
|
/** Open a TempDbReader for a given handle. Caller must close the reader. */
|
|
21924
21866
|
openReader(handle) {
|
|
21925
21867
|
const entry = this.cache.get(handle);
|
|
21926
|
-
if (!entry || !
|
|
21868
|
+
if (!entry || !fs34.existsSync(entry.tempDbPath)) return null;
|
|
21927
21869
|
const adapter = openBetterSqliteDatabase(entry.tempDbPath, {
|
|
21928
21870
|
readonly: true,
|
|
21929
21871
|
nativeBinding: this.nativeBinding
|
|
@@ -21968,7 +21910,7 @@ var MergeSessionCache = class {
|
|
|
21968
21910
|
};
|
|
21969
21911
|
|
|
21970
21912
|
// ../../packages/node-runtime/src/services/push-importer.ts
|
|
21971
|
-
import * as
|
|
21913
|
+
import * as fs35 from "fs";
|
|
21972
21914
|
var SYSTEM_SENDER_ID3 = "SYSTEM";
|
|
21973
21915
|
var SYSTEM_MEMBER_NAME2 = "\u7CFB\u7EDF\u6D88\u606F";
|
|
21974
21916
|
function isRecord5(value) {
|
|
@@ -22118,7 +22060,6 @@ function writeMessages(db, messages, dedupState) {
|
|
|
22118
22060
|
let duplicateCount = 0;
|
|
22119
22061
|
let membersAdded = 0;
|
|
22120
22062
|
let minWrittenTs = Infinity;
|
|
22121
|
-
const ftsEntries = [];
|
|
22122
22063
|
const sorted = [...messages].sort((a, b) => a.timestamp - b.timestamp);
|
|
22123
22064
|
db.transaction(() => {
|
|
22124
22065
|
for (const msg of sorted) {
|
|
@@ -22141,7 +22082,7 @@ function writeMessages(db, messages, dedupState) {
|
|
|
22141
22082
|
}
|
|
22142
22083
|
}
|
|
22143
22084
|
if (!memberId) continue;
|
|
22144
|
-
|
|
22085
|
+
insertMsg.run(
|
|
22145
22086
|
memberId,
|
|
22146
22087
|
normalizeSenderAccountName(msg.sender, msg.accountName) || null,
|
|
22147
22088
|
normalizeSenderGroupNickname(msg.sender, msg.groupNickname) || null,
|
|
@@ -22151,12 +22092,11 @@ function writeMessages(db, messages, dedupState) {
|
|
|
22151
22092
|
msg.replyToMessageId || null,
|
|
22152
22093
|
msg.platformMessageId || null
|
|
22153
22094
|
);
|
|
22154
|
-
ftsEntries.push({ id: Number(result.lastInsertRowid), content: msg.content ?? null });
|
|
22155
22095
|
if (msg.timestamp < minWrittenTs) minWrittenTs = msg.timestamp;
|
|
22156
22096
|
writtenCount++;
|
|
22157
22097
|
}
|
|
22158
22098
|
});
|
|
22159
|
-
return { writtenCount, duplicateCount, membersAdded, minWrittenTs
|
|
22099
|
+
return { writtenCount, duplicateCount, membersAdded, minWrittenTs };
|
|
22160
22100
|
}
|
|
22161
22101
|
function fullImport(db, meta, members, messages) {
|
|
22162
22102
|
db.exec(CHAT_DB_SCHEMA);
|
|
@@ -22202,7 +22142,6 @@ function fullImport(db, meta, members, messages) {
|
|
|
22202
22142
|
replyToMessageId: m.replyToMessageId
|
|
22203
22143
|
}))
|
|
22204
22144
|
);
|
|
22205
|
-
buildFtsIndex(db);
|
|
22206
22145
|
try {
|
|
22207
22146
|
generateSessionIndex(db);
|
|
22208
22147
|
} catch {
|
|
@@ -22263,15 +22202,8 @@ function writeIncrementalImport(db, payload) {
|
|
|
22263
22202
|
writtenCount,
|
|
22264
22203
|
duplicateCount,
|
|
22265
22204
|
membersAdded: autoCreatedMembersAdded,
|
|
22266
|
-
minWrittenTs
|
|
22267
|
-
ftsEntries
|
|
22205
|
+
minWrittenTs
|
|
22268
22206
|
} = writeMessages(db, payload.messages, dedupState);
|
|
22269
|
-
if (ftsEntries.length > 0) {
|
|
22270
|
-
try {
|
|
22271
|
-
insertFtsEntries(db, ftsEntries);
|
|
22272
|
-
} catch {
|
|
22273
|
-
}
|
|
22274
|
-
}
|
|
22275
22207
|
if (writtenCount > 0) {
|
|
22276
22208
|
try {
|
|
22277
22209
|
const sessionGapThreshold = getSessionIndexStats(db).gapThreshold;
|
|
@@ -22336,7 +22268,7 @@ async function executePushImportUnlocked(deps, sessionId, payload) {
|
|
|
22336
22268
|
return { ok: false, reason: "invalid_payload", message: "sessionId contains invalid characters" };
|
|
22337
22269
|
}
|
|
22338
22270
|
const dbPath = deps.getDbPath(sessionId);
|
|
22339
|
-
const isNew = !
|
|
22271
|
+
const isNew = !fs35.existsSync(dbPath);
|
|
22340
22272
|
const validationError = validatePayload(payload, isNew);
|
|
22341
22273
|
if (validationError) {
|
|
22342
22274
|
return { ok: false, reason: "invalid_payload", message: validationError };
|
|
@@ -22397,10 +22329,10 @@ async function executePushImportUnlocked(deps, sessionId, payload) {
|
|
|
22397
22329
|
}
|
|
22398
22330
|
|
|
22399
22331
|
// ../../packages/node-runtime/src/services/import-idempotency.ts
|
|
22400
|
-
import { createHash as
|
|
22332
|
+
import { createHash as createHash4 } from "crypto";
|
|
22401
22333
|
var DEFAULT_IMPORT_IDEMPOTENCY_TTL_MS = 60 * 60 * 1e3;
|
|
22402
22334
|
function hashImportBody(body) {
|
|
22403
|
-
return
|
|
22335
|
+
return createHash4("sha256").update(JSON.stringify(body ?? "")).digest("hex");
|
|
22404
22336
|
}
|
|
22405
22337
|
var ImportIdempotencyCache = class {
|
|
22406
22338
|
constructor(ttlMs = DEFAULT_IMPORT_IDEMPOTENCY_TTL_MS) {
|
|
@@ -22442,6 +22374,7 @@ export {
|
|
|
22442
22374
|
getChatLabTempScopeDir,
|
|
22443
22375
|
createChatLabTempDir,
|
|
22444
22376
|
removeChatLabTempDir,
|
|
22377
|
+
PARSER_FORMAT_IDS,
|
|
22445
22378
|
validateChatLabFile,
|
|
22446
22379
|
rebaseChatLabDemoDocuments,
|
|
22447
22380
|
detectFormat,
|
|
@@ -22462,22 +22395,11 @@ export {
|
|
|
22462
22395
|
logInfo,
|
|
22463
22396
|
getErrorCount,
|
|
22464
22397
|
logSummary,
|
|
22465
|
-
computeWordFrequency,
|
|
22466
|
-
segmentText,
|
|
22467
|
-
isDictDownloaded,
|
|
22468
|
-
getDictList,
|
|
22469
|
-
loadDictBuffer,
|
|
22470
|
-
downloadDict,
|
|
22471
|
-
deleteDict,
|
|
22472
|
-
ensureDefaultDict,
|
|
22473
|
-
hasFtsTable,
|
|
22474
|
-
createFtsTable,
|
|
22475
|
-
buildFtsIndex,
|
|
22476
|
-
rebuildFtsIndex,
|
|
22477
|
-
insertFtsEntries,
|
|
22478
|
-
searchByFts,
|
|
22479
22398
|
MESSAGE_INSERT_MAX_ROWS,
|
|
22480
22399
|
MessageBatchInserter,
|
|
22400
|
+
generateFallbackMessageKey,
|
|
22401
|
+
createMessageDedupState,
|
|
22402
|
+
registerMessageAndCheckDuplicate,
|
|
22481
22403
|
streamingImport,
|
|
22482
22404
|
analyzeNewImport,
|
|
22483
22405
|
streamParseFileInfo,
|
|
@@ -22651,14 +22573,11 @@ export {
|
|
|
22651
22573
|
isSemanticVoid,
|
|
22652
22574
|
chunkMessages,
|
|
22653
22575
|
runWarmup,
|
|
22654
|
-
|
|
22655
|
-
hybridSearch,
|
|
22576
|
+
semanticSearch,
|
|
22656
22577
|
formatEvidenceMessages,
|
|
22657
22578
|
assembleEvidence,
|
|
22658
22579
|
createChatDbMessageSource,
|
|
22659
22580
|
createChatDbMessageRangeReader,
|
|
22660
|
-
extractFtsKeywords,
|
|
22661
|
-
createChatDbFtsSearcher,
|
|
22662
22581
|
SEMANTIC_INDEX_AUTH_PROFILE,
|
|
22663
22582
|
persistSemanticIndexConfig,
|
|
22664
22583
|
resolveSemanticIndexApiKeySet,
|
|
@@ -22676,4 +22595,4 @@ export {
|
|
|
22676
22595
|
hashImportBody,
|
|
22677
22596
|
ImportIdempotencyCache
|
|
22678
22597
|
};
|
|
22679
|
-
//# sourceMappingURL=chunk-
|
|
22598
|
+
//# sourceMappingURL=chunk-AWXHYTD5.mjs.map
|