chatlab-cli 0.34.2 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chat-command-6IOH2QRF.mjs → chat-command-24RYGJAS.mjs} +8 -8
- package/dist/{chunk-INBJ7SND.mjs → chunk-22YJOBA3.mjs} +589 -513
- package/dist/chunk-22YJOBA3.mjs.map +1 -0
- package/dist/{chunk-4LW6443F.mjs → chunk-4GWWKHWA.mjs} +833 -886
- package/dist/chunk-4GWWKHWA.mjs.map +1 -0
- package/dist/{chunk-MENKFEPC.mjs → chunk-7IGBPZMI.mjs} +56 -48
- package/dist/chunk-7IGBPZMI.mjs.map +1 -0
- package/dist/{chunk-72RWABYJ.mjs → chunk-BRZ6YIBR.mjs} +3 -3
- package/dist/chunk-BRZ6YIBR.mjs.map +1 -0
- package/dist/{chunk-2XCNTG4O.mjs → chunk-DC7IUDNV.mjs} +677 -69
- package/dist/chunk-DC7IUDNV.mjs.map +1 -0
- package/dist/{chunk-H6D4DZOC.mjs → chunk-DEFEU567.mjs} +12 -18
- package/dist/chunk-DEFEU567.mjs.map +1 -0
- package/dist/{chunk-4RLZFGO5.mjs → chunk-GQMCDWZQ.mjs} +2 -2
- package/dist/{chunk-52P5ZRBD.mjs → chunk-KIMJXKZO.mjs} +2 -2
- package/dist/{chunk-652KEAX3.mjs → chunk-KXDJ7N6N.mjs} +2 -2
- package/dist/{chunk-U6MVJSXX.mjs → chunk-QKEARRFH.mjs} +15 -9
- package/dist/chunk-QKEARRFH.mjs.map +1 -0
- package/dist/{chunk-UMBEHBIZ.mjs → chunk-U36TQMAF.mjs} +2 -2
- package/dist/{chunk-JX6MFS6F.mjs → chunk-ZBPPLXM4.mjs} +15 -15
- package/dist/cli.mjs +9 -9
- package/dist/contacts-worker.mjs +2 -2
- package/dist/global-insight-worker.mjs +1 -1
- package/dist/{http-HSK46XEU.mjs → http-3K4YMWDE.mjs} +12 -12
- package/dist/{import-DF2WD2HE.mjs → import-D4LV565A.mjs} +7 -7
- package/dist/index.mjs +12 -12
- package/dist/{mcp-A2RND7KN.mjs → mcp-KL22FKQ2.mjs} +8 -8
- package/dist/people-relationships-worker.mjs +2 -2
- package/dist/semantic-index-worker.mjs +1 -1
- package/dist/{src-SHFHW2QB.mjs → src-COIB6Z4P.mjs} +24 -38
- package/dist/{stream-import-GZ2QDE3Z.mjs → stream-import-FSXCFG4M.mjs} +7 -7
- package/dist/{worker-runtime-OWN56GG5.mjs → worker-runtime-DPS2LURI.mjs} +5 -5
- package/dist-cli-web/assets/AISettingsTab-rQC2ocFH.js +1 -0
- package/dist-cli-web/assets/AboutTab-DMPziKjj.js +1 -0
- package/dist-cli-web/assets/ActionToolsPanel.vue_vue_type_script_setup_true_lang-CnRLt_1j.js +110 -0
- package/dist-cli-web/assets/Alert--bTQXRcZ.js +1 -0
- package/dist-cli-web/assets/ApiSettingsTab-CtXl4I28.js +2 -0
- package/dist-cli-web/assets/BatchManageTab-1KaCPweN.js +1 -0
- package/dist-cli-web/assets/{BufferResource-BdKXeRZS.js → BufferResource-BXSJADxx.js} +1 -1
- package/dist-cli-web/assets/CanvasRenderer-Cq--8UQ_.js +1 -0
- package/dist-cli-web/assets/ChatRecordDrawer-BC0F7ok1.js +11 -0
- package/dist-cli-web/assets/ChatRecordDrawer-Tt6UduRC.css +1 -0
- package/dist-cli-web/assets/Checkbox-CwajNPGF.js +1 -0
- package/dist-cli-web/assets/DataDirCleanupNotice-D0Oruka2.js +1 -0
- package/dist-cli-web/assets/DatePicker.vue_vue_type_script_setup_true_lang-C4CsoIlq.js +1 -0
- package/dist-cli-web/assets/DebugToolsPanel-DVDwS-wz.js +1 -0
- package/dist-cli-web/assets/EChartWordcloud-KV0MdsDE.js +1 -0
- package/dist-cli-web/assets/{Filter-B2eEAPxI.js → Filter-DVaqcFmI.js} +1 -1
- package/dist-cli-web/assets/GlobalTaskBar-COW_uBX6.css +1 -0
- package/dist-cli-web/assets/GlobalTaskBar-DpPMX0jx.js +1 -0
- package/dist-cli-web/assets/GroupRelationshipGalaxyView-CLnO3phW.js +1 -0
- package/dist-cli-web/assets/Label-Bt2U4lGy.js +1 -0
- package/dist-cli-web/assets/LoadingState.vue_vue_type_script_setup_true_lang-Cnshyn0f.js +1 -0
- package/dist-cli-web/assets/MessageExportModal-BozaR9UZ.js +1 -0
- package/dist-cli-web/assets/PageHeader-CEe0xTl3.js +1 -0
- package/dist-cli-web/assets/PageTabs.vue_vue_type_script_setup_true_lang-D2DZrrnv.js +1 -0
- package/dist-cli-web/assets/{RelationshipGalaxyCanvas-CwQ63TPi.js → RelationshipGalaxyCanvas-6EamagxG.js} +3 -3
- package/dist-cli-web/assets/{RelationshipGalaxyThreeCanvas-BLqISQe5.js → RelationshipGalaxyThreeCanvas-B6GB4PWS.js} +166 -166
- package/dist-cli-web/assets/{RelationshipGalaxyThreeCanvas-CMuibfiA.css → RelationshipGalaxyThreeCanvas-TYGOUDiv.css} +1 -1
- package/dist-cli-web/assets/{RenderTargetSystem-eC_VrDAr.js → RenderTargetSystem-mYuj4uBS.js} +1 -1
- package/dist-cli-web/assets/RovingFocusItem-B2y6BG1a.js +1 -0
- package/dist-cli-web/assets/ScreenCaptureModal-BvAtSNNR.js +1 -0
- package/dist-cli-web/assets/SectionTabs.vue_vue_type_script_setup_true_lang-ueEF4zHE.js +1 -0
- package/dist-cli-web/assets/Select-Cm4I8u9n.js +1 -0
- package/dist-cli-web/assets/SettingsModal-CjVax3I6.css +1 -0
- package/dist-cli-web/assets/SettingsModal-Cwshzj7i.js +2 -0
- package/dist-cli-web/assets/StorageTab-D7HzH16Y.js +1 -0
- package/dist-cli-web/assets/Switch-BVw4e_19.js +1 -0
- package/dist-cli-web/assets/Tabs-Dqwkh3Ea.js +1 -0
- package/dist-cli-web/assets/Tabs-OrJkANIA.js +1 -0
- package/dist-cli-web/assets/ThemeCard.vue_vue_type_script_setup_true_lang-DPIvbq75.js +1 -0
- package/dist-cli-web/assets/TimeSelect.vue_vue_type_script_setup_true_lang-dEdo6CqM.js +1 -0
- package/dist-cli-web/assets/VisuallyHiddenInput-BOdkR_GC.js +1 -0
- package/dist-cli-web/assets/{WebGLRenderer-CRp_HC84.js → WebGLRenderer-BB8ukBI_.js} +2 -2
- package/dist-cli-web/assets/{WebGPURenderer-WIUnEhpi.js → WebGPURenderer-DDOzF1Kc.js} +2 -2
- package/dist-cli-web/assets/aiChat-CzFSFuOJ.js +8 -0
- package/dist-cli-web/assets/annual-summary-time-range-CmrAqcuj.js +1 -0
- package/dist-cli-web/assets/base-CawUl6ET.js +1 -0
- package/dist-cli-web/assets/{browserAll-B29Ox2in.js → browserAll-BRqlnNxq.js} +2 -2
- package/dist-cli-web/assets/chatlabSiteLocale-CrCNb8QP.js +1 -0
- package/dist-cli-web/assets/clamp-jFYhBC76.js +1 -0
- package/dist-cli-web/assets/cli-web-ClAJ6Tao.js +1 -0
- package/dist-cli-web/assets/{dateFormat-T7kMS5IP.js → dateFormat-DXuDxIye.js} +1 -1
- package/dist-cli-web/assets/fetch-3_xX8Yi8.js +1 -0
- package/dist-cli-web/assets/{fetch-DsHf-61a.js → fetch-Bcy2T8Fe.js} +1 -1
- package/dist-cli-web/assets/{fetch-DvrVGAzR.js → fetch-CGu53cxT.js} +1 -1
- package/dist-cli-web/assets/{fetch-BBgZQob_.js → fetch-CP1afnT0.js} +1 -1
- package/dist-cli-web/assets/{fetch-D9sn99zX.js → fetch-DWAIk_6_.js} +1 -1
- package/dist-cli-web/assets/{fetch-BKBlpBl4.js → fetch-DijSOvop.js} +1 -1
- package/dist-cli-web/assets/{fetch-CYaGyohV.js → fetch-Dy38lkAa.js} +1 -1
- package/dist-cli-web/assets/{fetch-L6PKD8-N.js → fetch-DzKlTaLz.js} +1 -1
- package/dist-cli-web/assets/{fetch-D8hJZa8o.js → fetch-LY-yerw1.js} +1 -1
- package/dist-cli-web/assets/{fetch-BLi6Jsjg.js → fetch-q480XYZQ.js} +1 -1
- package/dist-cli-web/assets/index-0dp58uL4.css +1 -0
- package/dist-cli-web/assets/index-1X7zdLgJ.css +1 -0
- package/dist-cli-web/assets/index-4enwdBcq.css +1 -0
- package/dist-cli-web/assets/index-AudHHbMs.js +1 -0
- package/dist-cli-web/assets/{index-0qcENyvH.js → index-BRV_D8dv.js} +3 -3
- package/dist-cli-web/assets/index-CF4DTbvD.js +1 -0
- package/dist-cli-web/assets/index-CMLoht5R.js +1 -0
- package/dist-cli-web/assets/index-CMPHZH6B.css +1 -0
- package/dist-cli-web/assets/index-CNzv7aBW.js +1 -0
- package/dist-cli-web/assets/index-CXUTiy-V.js +1 -0
- package/dist-cli-web/assets/index-D5m0P4ss.js +36 -0
- package/dist-cli-web/assets/index-Di6FmCdk.js +64 -0
- package/dist-cli-web/assets/index-DkOrt7_X.js +2 -0
- package/dist-cli-web/assets/index-Dlb6fC68.js +1 -0
- package/dist-cli-web/assets/index-IA7YUjQX.js +2 -0
- package/dist-cli-web/assets/index-kzU381y1.js +1 -0
- package/dist-cli-web/assets/index-z8Vogc-b.js +137 -0
- package/dist-cli-web/assets/lazy-overlay-visibility-Bd9ZFOgR.js +1 -0
- package/dist-cli-web/assets/member-select-utils-BXZ3oHct.js +1 -0
- package/dist-cli-web/assets/people-page-header-Bh3KXD4g.js +1 -0
- package/dist-cli-web/assets/prompt-DcAcgQiB.js +1 -0
- package/dist-cli-web/assets/relationship-galaxy-connections-Cg4G5yYE.js +1 -0
- package/dist-cli-web/assets/semantic-index-config-builder-CK6v64Y1.js +1 -0
- package/dist-cli-web/assets/service-YZnJyEMq.js +1 -0
- package/dist-cli-web/assets/useFormControl-B-aXrJZN.js +1 -0
- package/dist-cli-web/assets/useTable-DPEMaGby.js +1 -0
- package/dist-cli-web/assets/useToast-B36MzFFy.js +1 -0
- package/dist-cli-web/assets/vendor-echarts-DfAMQxdr.js +11 -0
- package/dist-cli-web/assets/{vendor-echarts-wordcloud-Ca8n6qS6.js → vendor-echarts-wordcloud-BsaEJTPY.js} +1 -1
- package/dist-cli-web/assets/vendor-zrender-PFmZe1EY.js +8 -0
- package/dist-cli-web/assets/{webworkerAll-B2qvP2qR.js → webworkerAll-Dwqmy_YO.js} +1 -1
- package/dist-cli-web/index.html +2 -2
- package/package.json +1 -1
- package/dist/chunk-2XCNTG4O.mjs.map +0 -1
- package/dist/chunk-4LW6443F.mjs.map +0 -1
- package/dist/chunk-72RWABYJ.mjs.map +0 -1
- package/dist/chunk-H6D4DZOC.mjs.map +0 -1
- package/dist/chunk-INBJ7SND.mjs.map +0 -1
- package/dist/chunk-MENKFEPC.mjs.map +0 -1
- package/dist/chunk-U6MVJSXX.mjs.map +0 -1
- package/dist-cli-web/assets/ActionToolsPanel.vue_vue_type_script_setup_true_lang-B06LYYS9.js +0 -110
- package/dist-cli-web/assets/CanvasRenderer-DDRdIyZ8.js +0 -1
- package/dist-cli-web/assets/DataDirCleanupNotice-CP8P1lsL.js +0 -1
- package/dist-cli-web/assets/EChartWordcloud-CqknPGui.js +0 -1
- package/dist-cli-web/assets/LoadingState.vue_vue_type_script_setup_true_lang-BFaXTz1c.js +0 -1
- package/dist-cli-web/assets/MessageExportModal-CZge6qub.js +0 -1
- package/dist-cli-web/assets/PageHeader-CxKbcLDc.js +0 -1
- package/dist-cli-web/assets/ThemeCard.vue_vue_type_script_setup_true_lang-DVmpDz05.js +0 -1
- package/dist-cli-web/assets/TimeSelect.vue_vue_type_script_setup_true_lang-CCH1UQGc.js +0 -1
- package/dist-cli-web/assets/annual-summary-time-range-R3pLPEuN.js +0 -1
- package/dist-cli-web/assets/base-CLwWn3Rs.js +0 -1
- package/dist-cli-web/assets/cli-web-CVLb5Pkq.js +0 -1
- package/dist-cli-web/assets/fetch-Cm2WR2Cv.js +0 -1
- package/dist-cli-web/assets/index-AkF_KqVI.js +0 -1
- package/dist-cli-web/assets/index-BVOFyB9E.js +0 -137
- package/dist-cli-web/assets/index-BkdOH41D.js +0 -1
- package/dist-cli-web/assets/index-C4TuoOLM.js +0 -2
- package/dist-cli-web/assets/index-C6Mk-Xqt.js +0 -1
- package/dist-cli-web/assets/index-CkgUsaeX.js +0 -1
- package/dist-cli-web/assets/index-CuAy1Xmt.js +0 -64
- package/dist-cli-web/assets/index-D_0qeKKc.js +0 -53
- package/dist-cli-web/assets/index-DeR0Vukh.js +0 -1
- package/dist-cli-web/assets/index-DrjoMv9s.css +0 -1
- package/dist-cli-web/assets/index-Dx-XvfIK.js +0 -2
- package/dist-cli-web/assets/index-Jhwca9kg.css +0 -1
- package/dist-cli-web/assets/index-_DrviTnP.js +0 -1
- package/dist-cli-web/assets/index-vtF05fh7.css +0 -1
- package/dist-cli-web/assets/index-xN0sjOLK.js +0 -1
- package/dist-cli-web/assets/people-page-header-CCNJOrmK.js +0 -1
- package/dist-cli-web/assets/vendor-echarts-B-i_56Av.js +0 -11
- package/dist-cli-web/assets/vendor-zrender-5DcYjSZk.js +0 -8
- /package/dist/{chat-command-6IOH2QRF.mjs.map → chat-command-24RYGJAS.mjs.map} +0 -0
- /package/dist/{chunk-4RLZFGO5.mjs.map → chunk-GQMCDWZQ.mjs.map} +0 -0
- /package/dist/{chunk-52P5ZRBD.mjs.map → chunk-KIMJXKZO.mjs.map} +0 -0
- /package/dist/{chunk-652KEAX3.mjs.map → chunk-KXDJ7N6N.mjs.map} +0 -0
- /package/dist/{chunk-UMBEHBIZ.mjs.map → chunk-U36TQMAF.mjs.map} +0 -0
- /package/dist/{chunk-JX6MFS6F.mjs.map → chunk-ZBPPLXM4.mjs.map} +0 -0
- /package/dist/{http-HSK46XEU.mjs.map → http-3K4YMWDE.mjs.map} +0 -0
- /package/dist/{import-DF2WD2HE.mjs.map → import-D4LV565A.mjs.map} +0 -0
- /package/dist/{mcp-A2RND7KN.mjs.map → mcp-KL22FKQ2.mjs.map} +0 -0
- /package/dist/{src-SHFHW2QB.mjs.map → src-COIB6Z4P.mjs.map} +0 -0
- /package/dist/{stream-import-GZ2QDE3Z.mjs.map → stream-import-FSXCFG4M.mjs.map} +0 -0
- /package/dist/{worker-runtime-OWN56GG5.mjs.map → worker-runtime-DPS2LURI.mjs.map} +0 -0
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
createEmptyContactsDiagnostics,
|
|
10
10
|
normalizeContactsTimeRangePreset,
|
|
11
11
|
resolveContactsTimeRange
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-KIMJXKZO.mjs";
|
|
13
13
|
import {
|
|
14
14
|
PEOPLE_RELATIONSHIPS_ALGORITHM_VERSION,
|
|
15
15
|
buildPeopleRelationshipsNeighborhoodGraph,
|
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
computePeopleRelationshipsSnapshot,
|
|
19
19
|
normalizePeopleRelationshipsTimeRangePreset,
|
|
20
20
|
resolvePeopleRelationshipsTimeRange
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-KXDJ7N6N.mjs";
|
|
22
22
|
import {
|
|
23
23
|
ANNUAL_SUMMARY_ALGORITHM_VERSION
|
|
24
24
|
} from "./chunk-QZCNVAH6.mjs";
|
|
@@ -26,6 +26,8 @@ import {
|
|
|
26
26
|
BUILTIN_MODELS,
|
|
27
27
|
BUILTIN_PROVIDERS,
|
|
28
28
|
BUILTIN_TOOL_CATALOG,
|
|
29
|
+
CACHE_KEY_MEMBERS,
|
|
30
|
+
CACHE_KEY_OVERVIEW,
|
|
29
31
|
CHART_CAPABILITY_SKILL_ID,
|
|
30
32
|
CHATLAB_FORMAT_VERSION,
|
|
31
33
|
CHATLAB_SUPPORTED_FORMAT_VERSIONS,
|
|
@@ -33,20 +35,14 @@ import {
|
|
|
33
35
|
CHAT_DB_SCHEMA,
|
|
34
36
|
DEFAULT_GENERAL_ASSISTANT_ID,
|
|
35
37
|
DataDirCompatibilityError,
|
|
36
|
-
FTS_TABLE_SCHEMA,
|
|
37
38
|
GENERAL_ASSISTANT_IDS,
|
|
38
39
|
KNOWN_PLATFORMS,
|
|
39
40
|
MessageType,
|
|
40
41
|
StreamingThinkTagParser,
|
|
41
42
|
appLogger,
|
|
42
|
-
batchSegmentChineseWithStats,
|
|
43
|
-
batchSegmentWithFrequency,
|
|
44
|
-
buildExcludeKeywordsConditions,
|
|
45
43
|
buildSessionInfo,
|
|
46
44
|
cleanupAnnualSummarySnapshotTempFiles,
|
|
47
|
-
clearJiebaInstance,
|
|
48
45
|
clearSessionIndex,
|
|
49
|
-
collectPosTagStats,
|
|
50
46
|
deleteAnnualSummarySnapshots,
|
|
51
47
|
deleteAuthProfile,
|
|
52
48
|
deleteMember,
|
|
@@ -65,7 +61,6 @@ import {
|
|
|
65
61
|
getDbFileVersion,
|
|
66
62
|
getGlobalInsightDir,
|
|
67
63
|
getGlobalInsightFactsCacheDir,
|
|
68
|
-
getJieba,
|
|
69
64
|
getMemberNameHistory,
|
|
70
65
|
getMembersPaginated,
|
|
71
66
|
getMembersWithAliases,
|
|
@@ -77,8 +72,11 @@ import {
|
|
|
77
72
|
getSessionIndexStats,
|
|
78
73
|
getSessionInfo,
|
|
79
74
|
getSessionMeta,
|
|
75
|
+
getSessionOverview,
|
|
80
76
|
getSummaryCount,
|
|
81
77
|
getThinkingCompat,
|
|
78
|
+
getValidatedOverviewCache,
|
|
79
|
+
invalidateCache,
|
|
82
80
|
isChatSessionDb,
|
|
83
81
|
isNameMatchPlatform,
|
|
84
82
|
isReasoningModel,
|
|
@@ -97,12 +95,9 @@ import {
|
|
|
97
95
|
renameSession,
|
|
98
96
|
resolveApiKey,
|
|
99
97
|
saveSegmentSummary,
|
|
100
|
-
segment,
|
|
101
98
|
snapshotPathProvider,
|
|
102
99
|
toAnnualSummaryLocalDate,
|
|
103
100
|
toAnnualSummaryRangeKey,
|
|
104
|
-
tokenizeForFts,
|
|
105
|
-
tokenizeQueryForFts,
|
|
106
101
|
truncateToolResultText,
|
|
107
102
|
updateMemberAliases,
|
|
108
103
|
updateSessionOwnerId,
|
|
@@ -111,7 +106,7 @@ import {
|
|
|
111
106
|
writeAuthProfile,
|
|
112
107
|
writeJsonFileAtomically,
|
|
113
108
|
writeParseResultToDb
|
|
114
|
-
} from "./chunk-
|
|
109
|
+
} from "./chunk-22YJOBA3.mjs";
|
|
115
110
|
import {
|
|
116
111
|
__commonJS,
|
|
117
112
|
__export,
|
|
@@ -727,8 +722,8 @@ var require_FilterBase = __commonJS({
|
|
|
727
722
|
var FilterBase = class _FilterBase extends Transform {
|
|
728
723
|
static stringFilter(string, separator) {
|
|
729
724
|
return (stack) => {
|
|
730
|
-
const
|
|
731
|
-
return
|
|
725
|
+
const path30 = stack.join(separator);
|
|
726
|
+
return path30.length === string.length && path30 === string || path30.length > string.length && path30.substr(0, string.length) === string && path30.substr(string.length, separator.length) === separator;
|
|
732
727
|
};
|
|
733
728
|
}
|
|
734
729
|
static regExpFilter(regExp, separator) {
|
|
@@ -1182,12 +1177,12 @@ var require_Assembler = __commonJS({
|
|
|
1182
1177
|
return (this.stack.length >> 1) + (this.done ? 0 : 1);
|
|
1183
1178
|
}
|
|
1184
1179
|
get path() {
|
|
1185
|
-
const
|
|
1180
|
+
const path30 = [];
|
|
1186
1181
|
for (let i = 0; i < this.stack.length; i += 2) {
|
|
1187
1182
|
const key = this.stack[i + 1];
|
|
1188
|
-
|
|
1183
|
+
path30.push(key === null ? this.stack[i].length : key);
|
|
1189
1184
|
}
|
|
1190
|
-
return
|
|
1185
|
+
return path30;
|
|
1191
1186
|
}
|
|
1192
1187
|
dropToLevel(level) {
|
|
1193
1188
|
if (level < this.depth) {
|
|
@@ -1469,7 +1464,7 @@ var require_pend = __commonJS({
|
|
|
1469
1464
|
var require_fd_slicer = __commonJS({
|
|
1470
1465
|
"../../packages/node-runtime/node_modules/yauzl/fd-slicer.js"(exports) {
|
|
1471
1466
|
"use strict";
|
|
1472
|
-
var
|
|
1467
|
+
var fs36 = __require("fs");
|
|
1473
1468
|
var util = __require("util");
|
|
1474
1469
|
var stream = __require("stream");
|
|
1475
1470
|
var Readable = stream.Readable;
|
|
@@ -1489,7 +1484,7 @@ var require_fd_slicer = __commonJS({
|
|
|
1489
1484
|
FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
|
|
1490
1485
|
var self = this;
|
|
1491
1486
|
self.pend.go(function(cb) {
|
|
1492
|
-
|
|
1487
|
+
fs36.read(self.fd, buffer, offset, length, position, function(err, bytesRead, buffer2) {
|
|
1493
1488
|
cb();
|
|
1494
1489
|
callback(err, bytesRead, buffer2);
|
|
1495
1490
|
});
|
|
@@ -1506,7 +1501,7 @@ var require_fd_slicer = __commonJS({
|
|
|
1506
1501
|
self.refCount -= 1;
|
|
1507
1502
|
if (self.refCount < 0) throw new Error("invalid unref");
|
|
1508
1503
|
if (self.refCount > 0) return;
|
|
1509
|
-
|
|
1504
|
+
fs36.close(self.fd, onCloseDone);
|
|
1510
1505
|
function onCloseDone(err) {
|
|
1511
1506
|
if (err) {
|
|
1512
1507
|
self.emit("error", err);
|
|
@@ -1538,7 +1533,7 @@ var require_fd_slicer = __commonJS({
|
|
|
1538
1533
|
}
|
|
1539
1534
|
self.context.pend.go(function(cb) {
|
|
1540
1535
|
var buffer = Buffer.allocUnsafe(toRead);
|
|
1541
|
-
|
|
1536
|
+
fs36.read(self.context.fd, buffer, 0, toRead, self.pos, function(err, bytesRead) {
|
|
1542
1537
|
if (err) {
|
|
1543
1538
|
self.destroy(err);
|
|
1544
1539
|
} else if (bytesRead === 0) {
|
|
@@ -1900,7 +1895,7 @@ var require_crc32 = __commonJS({
|
|
|
1900
1895
|
var require_yauzl = __commonJS({
|
|
1901
1896
|
"../../packages/node-runtime/node_modules/yauzl/index.js"(exports) {
|
|
1902
1897
|
"use strict";
|
|
1903
|
-
var
|
|
1898
|
+
var fs36 = __require("fs");
|
|
1904
1899
|
var zlib = __require("zlib");
|
|
1905
1900
|
var fd_slicer = require_fd_slicer();
|
|
1906
1901
|
var util = __require("util");
|
|
@@ -1925,9 +1920,9 @@ var require_yauzl = __commonJS({
|
|
|
1925
1920
|
exports.Entry = Entry;
|
|
1926
1921
|
exports.LocalFileHeader = LocalFileHeader;
|
|
1927
1922
|
exports.RandomAccessReader = RandomAccessReader;
|
|
1928
|
-
function openPromise2(
|
|
1923
|
+
function openPromise2(path30, options2) {
|
|
1929
1924
|
return new Promise((resolve4, reject) => {
|
|
1930
|
-
open(
|
|
1925
|
+
open(path30, { ...options2, lazyEntries: true }, function(err, zipfile) {
|
|
1931
1926
|
if (err) return reject(err);
|
|
1932
1927
|
resolve4(zipfile);
|
|
1933
1928
|
});
|
|
@@ -1957,7 +1952,7 @@ var require_yauzl = __commonJS({
|
|
|
1957
1952
|
});
|
|
1958
1953
|
});
|
|
1959
1954
|
}
|
|
1960
|
-
function open(
|
|
1955
|
+
function open(path30, options2, callback) {
|
|
1961
1956
|
if (typeof options2 === "function") {
|
|
1962
1957
|
callback = options2;
|
|
1963
1958
|
options2 = null;
|
|
@@ -1969,10 +1964,10 @@ var require_yauzl = __commonJS({
|
|
|
1969
1964
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
1970
1965
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
1971
1966
|
if (callback == null) callback = defaultCallback;
|
|
1972
|
-
|
|
1967
|
+
fs36.open(path30, "r", function(err, fd) {
|
|
1973
1968
|
if (err) return callback(err);
|
|
1974
1969
|
fromFd(fd, options2, function(err2, zipfile) {
|
|
1975
|
-
if (err2)
|
|
1970
|
+
if (err2) fs36.close(fd, defaultCallback);
|
|
1976
1971
|
callback(err2, zipfile);
|
|
1977
1972
|
});
|
|
1978
1973
|
});
|
|
@@ -1989,7 +1984,7 @@ var require_yauzl = __commonJS({
|
|
|
1989
1984
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
1990
1985
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
1991
1986
|
if (callback == null) callback = defaultCallback;
|
|
1992
|
-
|
|
1987
|
+
fs36.fstat(fd, function(err, stats) {
|
|
1993
1988
|
if (err) return callback(err);
|
|
1994
1989
|
var reader = new fd_slicer.FdSlicer(fd);
|
|
1995
1990
|
fromRandomAccessReader(reader, stats.size, options2, callback);
|
|
@@ -2830,6 +2825,39 @@ import * as path11 from "path";
|
|
|
2830
2825
|
// ../../packages/parser/src/sniffer.ts
|
|
2831
2826
|
import * as fs2 from "fs";
|
|
2832
2827
|
import * as path2 from "path";
|
|
2828
|
+
|
|
2829
|
+
// ../../packages/parser/src/format-ids.ts
|
|
2830
|
+
var PARSER_FORMAT_IDS = {
|
|
2831
|
+
CHATLAB: "chatlab",
|
|
2832
|
+
CHATLAB_JSONL: "chatlab-jsonl",
|
|
2833
|
+
QQ_SHUAKAMI: "qq-shuakami",
|
|
2834
|
+
QQ_SHUAKAMI_CHUNKED: "qq-shuakami-chunked",
|
|
2835
|
+
WEFLOW: "weflow",
|
|
2836
|
+
ECHOTRACE: "echotrace",
|
|
2837
|
+
DISCORD_TYRRRZ: "discord-tyrrrz",
|
|
2838
|
+
TELEGRAM_NATIVE: "telegram-native",
|
|
2839
|
+
TELEGRAM_NATIVE_SINGLE: "telegram-native-single",
|
|
2840
|
+
GOOGLE_CHAT_NATIVE: "google-chat-native",
|
|
2841
|
+
INSTAGRAM_NATIVE: "instagram-native",
|
|
2842
|
+
WHATSAPP_NATIVE: "whatsapp-native",
|
|
2843
|
+
QQ_NATIVE: "qq-native",
|
|
2844
|
+
LINE_NATIVE: "line-native"
|
|
2845
|
+
};
|
|
2846
|
+
var LEGACY_PARSER_FORMAT_ID_ALIASES = {
|
|
2847
|
+
"shuakami-qq-exporter": PARSER_FORMAT_IDS.QQ_SHUAKAMI,
|
|
2848
|
+
"shuakami-qq-exporter-chunked": PARSER_FORMAT_IDS.QQ_SHUAKAMI_CHUNKED,
|
|
2849
|
+
"ycccccccy-echotrace": PARSER_FORMAT_IDS.ECHOTRACE,
|
|
2850
|
+
"tyrrrz-discord-exporter": PARSER_FORMAT_IDS.DISCORD_TYRRRZ,
|
|
2851
|
+
"whatsapp-native-txt": PARSER_FORMAT_IDS.WHATSAPP_NATIVE,
|
|
2852
|
+
"qq-native-txt": PARSER_FORMAT_IDS.QQ_NATIVE,
|
|
2853
|
+
"line-native-txt": PARSER_FORMAT_IDS.LINE_NATIVE,
|
|
2854
|
+
"google-chat-takeout": PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE
|
|
2855
|
+
};
|
|
2856
|
+
function normalizeParserFormatId(formatId) {
|
|
2857
|
+
return LEGACY_PARSER_FORMAT_ID_ALIASES[formatId] ?? formatId;
|
|
2858
|
+
}
|
|
2859
|
+
|
|
2860
|
+
// ../../packages/parser/src/sniffer.ts
|
|
2833
2861
|
var HEAD_SIZE = 64 * 1024;
|
|
2834
2862
|
function readFileHead(filePath, size = HEAD_SIZE) {
|
|
2835
2863
|
const fd = fs2.openSync(filePath, "r");
|
|
@@ -2947,9 +2975,17 @@ var FormatSniffer = class {
|
|
|
2947
2975
|
* 根据格式 ID 获取解析器
|
|
2948
2976
|
*/
|
|
2949
2977
|
getParserById(formatId) {
|
|
2950
|
-
const
|
|
2978
|
+
const canonicalId = normalizeParserFormatId(formatId);
|
|
2979
|
+
const module = this.formats.find((m) => m.feature.id === canonicalId);
|
|
2951
2980
|
return module?.parser || null;
|
|
2952
2981
|
}
|
|
2982
|
+
/**
|
|
2983
|
+
* Resolve canonical and released legacy format IDs to one canonical feature.
|
|
2984
|
+
*/
|
|
2985
|
+
getFeatureById(formatId) {
|
|
2986
|
+
const canonicalId = normalizeParserFormatId(formatId);
|
|
2987
|
+
return this.formats.find((module) => module.feature.id === canonicalId)?.feature ?? null;
|
|
2988
|
+
}
|
|
2953
2989
|
/**
|
|
2954
2990
|
* 获取所有支持的格式
|
|
2955
2991
|
*/
|
|
@@ -3087,12 +3123,18 @@ function isNativeFormatAvailable(formatId) {
|
|
|
3087
3123
|
const module = loadNativeParser();
|
|
3088
3124
|
if (!module) return false;
|
|
3089
3125
|
try {
|
|
3090
|
-
new module.NativeParser(formatId, "");
|
|
3126
|
+
new module.NativeParser(resolveNativeKernelId(formatId), "");
|
|
3091
3127
|
return true;
|
|
3092
3128
|
} catch {
|
|
3093
3129
|
return false;
|
|
3094
3130
|
}
|
|
3095
3131
|
}
|
|
3132
|
+
function resolveNativeKernelId(formatId) {
|
|
3133
|
+
const canonicalId = normalizeParserFormatId(formatId);
|
|
3134
|
+
if (canonicalId === PARSER_FORMAT_IDS.QQ_SHUAKAMI) return "shuakami-qq-exporter";
|
|
3135
|
+
if (canonicalId === PARSER_FORMAT_IDS.ECHOTRACE) return "weflow";
|
|
3136
|
+
return canonicalId;
|
|
3137
|
+
}
|
|
3096
3138
|
|
|
3097
3139
|
// ../../packages/parser/src/native/create-native-parser.ts
|
|
3098
3140
|
var PROGRESS_POLL_INTERVAL_MS = 200;
|
|
@@ -3161,7 +3203,7 @@ function createNativeFirstParser(adapter, fallback, fallbackAfterNativeFailure2
|
|
|
3161
3203
|
let parser11;
|
|
3162
3204
|
try {
|
|
3163
3205
|
parser11 = new native.NativeParser(
|
|
3164
|
-
adapter.
|
|
3206
|
+
adapter.kernelId,
|
|
3165
3207
|
options2.filePath,
|
|
3166
3208
|
options2.formatOptions ? JSON.stringify(options2.formatOptions) : void 0
|
|
3167
3209
|
);
|
|
@@ -3197,7 +3239,7 @@ function toParsedMember(member, fromHead) {
|
|
|
3197
3239
|
};
|
|
3198
3240
|
}
|
|
3199
3241
|
var chatlabAdapter = {
|
|
3200
|
-
|
|
3242
|
+
kernelId: "chatlab",
|
|
3201
3243
|
label: "ChatLab JSON",
|
|
3202
3244
|
mapMeta(metaJson) {
|
|
3203
3245
|
const meta = metaJson;
|
|
@@ -3246,7 +3288,7 @@ function extractNameFromFilePath(filePath) {
|
|
|
3246
3288
|
return name || "\u672A\u77E5\u7FA4\u804A";
|
|
3247
3289
|
}
|
|
3248
3290
|
var feature = {
|
|
3249
|
-
id:
|
|
3291
|
+
id: PARSER_FORMAT_IDS.CHATLAB,
|
|
3250
3292
|
name: "ChatLab JSON",
|
|
3251
3293
|
platform: KNOWN_PLATFORMS.UNKNOWN,
|
|
3252
3294
|
// ChatLab 格式可能包含多平台数据
|
|
@@ -3428,7 +3470,7 @@ function parseLine(line) {
|
|
|
3428
3470
|
}
|
|
3429
3471
|
}
|
|
3430
3472
|
var feature2 = {
|
|
3431
|
-
id:
|
|
3473
|
+
id: PARSER_FORMAT_IDS.CHATLAB_JSONL,
|
|
3432
3474
|
name: "ChatLab JSONL",
|
|
3433
3475
|
platform: KNOWN_PLATFORMS.UNKNOWN,
|
|
3434
3476
|
priority: 51,
|
|
@@ -3825,7 +3867,7 @@ var shuakamiQqPreprocessor = {
|
|
|
3825
3867
|
|
|
3826
3868
|
// ../../packages/parser/src/native/shuakami-qq-native.ts
|
|
3827
3869
|
var shuakamiQqAdapter = {
|
|
3828
|
-
|
|
3870
|
+
kernelId: "shuakami-qq-exporter",
|
|
3829
3871
|
label: "shuakami/qq-chat-exporter",
|
|
3830
3872
|
mapMeta(metaJson) {
|
|
3831
3873
|
const meta = metaJson;
|
|
@@ -3833,7 +3875,8 @@ var shuakamiQqAdapter = {
|
|
|
3833
3875
|
name: meta.name,
|
|
3834
3876
|
platform: KNOWN_PLATFORMS.QQ,
|
|
3835
3877
|
type: meta.chatType === "private" ? "private" /* PRIVATE */ : "group" /* GROUP */,
|
|
3836
|
-
groupAvatar: meta.groupAvatar ?? void 0
|
|
3878
|
+
groupAvatar: meta.groupAvatar ?? void 0,
|
|
3879
|
+
...meta.ownerId ? { ownerId: meta.ownerId } : {}
|
|
3837
3880
|
};
|
|
3838
3881
|
},
|
|
3839
3882
|
mapMembers(members) {
|
|
@@ -3900,7 +3943,7 @@ var { pick: pick3 } = import_Pick3.default;
|
|
|
3900
3943
|
var { streamValues: streamValues3 } = import_StreamValues3.default;
|
|
3901
3944
|
var { chain: chain3 } = import_stream_chain3.default;
|
|
3902
3945
|
var feature3 = {
|
|
3903
|
-
id:
|
|
3946
|
+
id: PARSER_FORMAT_IDS.QQ_SHUAKAMI,
|
|
3904
3947
|
name: "shuakami/qq-chat-exporter",
|
|
3905
3948
|
platform: KNOWN_PLATFORMS.QQ,
|
|
3906
3949
|
priority: 10,
|
|
@@ -4168,7 +4211,7 @@ import * as fs8 from "fs";
|
|
|
4168
4211
|
import * as path7 from "path";
|
|
4169
4212
|
import * as readline2 from "readline";
|
|
4170
4213
|
var feature4 = {
|
|
4171
|
-
id:
|
|
4214
|
+
id: PARSER_FORMAT_IDS.QQ_SHUAKAMI_CHUNKED,
|
|
4172
4215
|
name: "shuakami/qq-chat-exporter (chunked)",
|
|
4173
4216
|
platform: KNOWN_PLATFORMS.QQ,
|
|
4174
4217
|
priority: 5,
|
|
@@ -4341,11 +4384,10 @@ async function* parseChunkedJsonl(options2) {
|
|
|
4341
4384
|
const meta = {
|
|
4342
4385
|
name: manifest.chatInfo.name || "\u672A\u77E5\u7FA4\u804A",
|
|
4343
4386
|
platform: KNOWN_PLATFORMS.QQ,
|
|
4344
|
-
type: chatType
|
|
4345
|
-
ownerId: manifest.chatInfo.selfUin || manifest.chatInfo.selfUid
|
|
4387
|
+
type: chatType
|
|
4346
4388
|
};
|
|
4347
4389
|
yield { type: "meta", data: meta };
|
|
4348
|
-
onLog?.("info", `Meta \u4FE1\u606F: name=${meta.name}, type=${meta.type}
|
|
4390
|
+
onLog?.("info", `Meta \u4FE1\u606F: name=${meta.name}, type=${meta.type}`);
|
|
4349
4391
|
const memberMap = /* @__PURE__ */ new Map();
|
|
4350
4392
|
let currentBatch = [];
|
|
4351
4393
|
for (let chunkIndex = 0; chunkIndex < manifest.chunked.chunks.length; chunkIndex++) {
|
|
@@ -4484,7 +4526,7 @@ import * as path8 from "path";
|
|
|
4484
4526
|
|
|
4485
4527
|
// ../../packages/parser/src/native/weflow-native.ts
|
|
4486
4528
|
var weflowAdapter = {
|
|
4487
|
-
|
|
4529
|
+
kernelId: "weflow",
|
|
4488
4530
|
label: "WeFlow export",
|
|
4489
4531
|
mapMeta(metaJson) {
|
|
4490
4532
|
const meta = metaJson;
|
|
@@ -4558,7 +4600,7 @@ function extractNameFromFilePath4(filePath) {
|
|
|
4558
4600
|
return name || "\u672A\u77E5\u804A\u5929";
|
|
4559
4601
|
}
|
|
4560
4602
|
var feature5 = {
|
|
4561
|
-
id:
|
|
4603
|
+
id: PARSER_FORMAT_IDS.WEFLOW,
|
|
4562
4604
|
name: "WeFlow \u5BFC\u51FA",
|
|
4563
4605
|
platform: KNOWN_PLATFORMS.WECHAT,
|
|
4564
4606
|
priority: 15,
|
|
@@ -4947,7 +4989,7 @@ var weflow_default = module_5;
|
|
|
4947
4989
|
|
|
4948
4990
|
// ../../packages/parser/src/formats/ycccccccy-echotrace.ts
|
|
4949
4991
|
var feature6 = {
|
|
4950
|
-
id:
|
|
4992
|
+
id: PARSER_FORMAT_IDS.ECHOTRACE,
|
|
4951
4993
|
name: "ycccccccy/echotrace \u5BFC\u51FA",
|
|
4952
4994
|
platform: KNOWN_PLATFORMS.WECHAT,
|
|
4953
4995
|
priority: 16,
|
|
@@ -5066,7 +5108,7 @@ function formatSticker(sticker) {
|
|
|
5066
5108
|
return AttachmentMarkers.sticker(sticker.name);
|
|
5067
5109
|
}
|
|
5068
5110
|
var feature7 = {
|
|
5069
|
-
id:
|
|
5111
|
+
id: PARSER_FORMAT_IDS.DISCORD_TYRRRZ,
|
|
5070
5112
|
name: "Tyrrrz/DiscordChatExporter",
|
|
5071
5113
|
platform: KNOWN_PLATFORMS.DISCORD,
|
|
5072
5114
|
priority: 20,
|
|
@@ -5311,7 +5353,7 @@ var { parser: parser7 } = import_stream_json6.default;
|
|
|
5311
5353
|
var { pick: pick6 } = import_Pick6.default;
|
|
5312
5354
|
var { streamValues: streamValues6 } = import_StreamValues6.default;
|
|
5313
5355
|
var feature8 = {
|
|
5314
|
-
id:
|
|
5356
|
+
id: PARSER_FORMAT_IDS.TELEGRAM_NATIVE,
|
|
5315
5357
|
name: "Telegram \u5B98\u65B9\u5BFC\u51FA (JSON)",
|
|
5316
5358
|
platform: KNOWN_PLATFORMS.TELEGRAM,
|
|
5317
5359
|
priority: 22,
|
|
@@ -5476,7 +5518,7 @@ var TELEGRAM_SINGLE_HEAD_SIGNATURES = [/^\s*\{\s*(?:\r?\n\s*)?"name"\s*:/];
|
|
|
5476
5518
|
// ../../packages/parser/src/formats/telegram-native-single.ts
|
|
5477
5519
|
var { parser: parser8 } = import_stream_json7.default;
|
|
5478
5520
|
var feature9 = {
|
|
5479
|
-
id:
|
|
5521
|
+
id: PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE,
|
|
5480
5522
|
name: "Telegram \u5355\u804A\u5929\u5BFC\u51FA (JSON)",
|
|
5481
5523
|
platform: KNOWN_PLATFORMS.TELEGRAM,
|
|
5482
5524
|
priority: 23,
|
|
@@ -5861,7 +5903,7 @@ function buildTimestamp(firstDatePart, secondDatePart, thirdDatePart, rawHour, m
|
|
|
5861
5903
|
|
|
5862
5904
|
// ../../packages/parser/src/formats/whatsapp-native-txt.ts
|
|
5863
5905
|
var feature10 = {
|
|
5864
|
-
id:
|
|
5906
|
+
id: PARSER_FORMAT_IDS.WHATSAPP_NATIVE,
|
|
5865
5907
|
name: "WhatsApp \u5B98\u65B9\u5BFC\u51FA (TXT)",
|
|
5866
5908
|
platform: KNOWN_PLATFORMS.WHATSAPP,
|
|
5867
5909
|
priority: 25,
|
|
@@ -6067,7 +6109,7 @@ function detectMessageType3(content) {
|
|
|
6067
6109
|
|
|
6068
6110
|
// ../../packages/parser/src/formats/qq-native-txt.ts
|
|
6069
6111
|
var feature11 = {
|
|
6070
|
-
id:
|
|
6112
|
+
id: PARSER_FORMAT_IDS.QQ_NATIVE,
|
|
6071
6113
|
name: "QQ \u5B98\u65B9\u5BFC\u51FA (TXT)",
|
|
6072
6114
|
platform: KNOWN_PLATFORMS.QQ,
|
|
6073
6115
|
priority: 30,
|
|
@@ -6123,7 +6165,7 @@ var qq_native_txt_default = module_11;
|
|
|
6123
6165
|
import * as fs15 from "fs";
|
|
6124
6166
|
import * as path9 from "path";
|
|
6125
6167
|
var feature12 = {
|
|
6126
|
-
id:
|
|
6168
|
+
id: PARSER_FORMAT_IDS.INSTAGRAM_NATIVE,
|
|
6127
6169
|
name: "Instagram \u5B98\u65B9\u5BFC\u51FA",
|
|
6128
6170
|
platform: KNOWN_PLATFORMS.INSTAGRAM,
|
|
6129
6171
|
priority: 25,
|
|
@@ -6333,7 +6375,7 @@ var ENGLISH_MONTHS = /* @__PURE__ */ new Map([
|
|
|
6333
6375
|
["december", 11]
|
|
6334
6376
|
]);
|
|
6335
6377
|
var feature13 = {
|
|
6336
|
-
id:
|
|
6378
|
+
id: PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE,
|
|
6337
6379
|
name: "Google Chat Takeout",
|
|
6338
6380
|
platform: KNOWN_PLATFORMS.GOOGLE_CHAT,
|
|
6339
6381
|
priority: 24,
|
|
@@ -6767,7 +6809,7 @@ function detectMessageType6(content) {
|
|
|
6767
6809
|
|
|
6768
6810
|
// ../../packages/parser/src/formats/line-native-txt.ts
|
|
6769
6811
|
var feature14 = {
|
|
6770
|
-
id:
|
|
6812
|
+
id: PARSER_FORMAT_IDS.LINE_NATIVE,
|
|
6771
6813
|
name: "LINE \u5B98\u65B9\u5BFC\u51FA TXT",
|
|
6772
6814
|
platform: KNOWN_PLATFORMS.LINE,
|
|
6773
6815
|
priority: 35,
|
|
@@ -6898,21 +6940,21 @@ function isRecord(value) {
|
|
|
6898
6940
|
function hasOwn(value, key) {
|
|
6899
6941
|
return Object.prototype.hasOwnProperty.call(value, key);
|
|
6900
6942
|
}
|
|
6901
|
-
function childLocation(location,
|
|
6902
|
-
return { ...location, path:
|
|
6943
|
+
function childLocation(location, path30) {
|
|
6944
|
+
return { ...location, path: path30 };
|
|
6903
6945
|
}
|
|
6904
|
-
function validateRequiredString(state, value,
|
|
6946
|
+
function validateRequiredString(state, value, path30, location) {
|
|
6905
6947
|
if (typeof value === "string" && value.trim().length > 0) return true;
|
|
6906
|
-
addIssue(state, "error", "INVALID_REQUIRED_STRING", "Expected a non-empty string.", childLocation(location,
|
|
6948
|
+
addIssue(state, "error", "INVALID_REQUIRED_STRING", "Expected a non-empty string.", childLocation(location, path30));
|
|
6907
6949
|
return false;
|
|
6908
6950
|
}
|
|
6909
|
-
function validateOptionalString(state, object, key,
|
|
6951
|
+
function validateOptionalString(state, object, key, path30, location) {
|
|
6910
6952
|
if (!hasOwn(object, key)) return;
|
|
6911
6953
|
if (typeof object[key] !== "string") {
|
|
6912
|
-
addIssue(state, "error", "INVALID_OPTIONAL_STRING", "Expected a string.", childLocation(location,
|
|
6954
|
+
addIssue(state, "error", "INVALID_OPTIONAL_STRING", "Expected a string.", childLocation(location, path30));
|
|
6913
6955
|
}
|
|
6914
6956
|
}
|
|
6915
|
-
function validateUnixSeconds(state, value,
|
|
6957
|
+
function validateUnixSeconds(state, value, path30, location) {
|
|
6916
6958
|
if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value < MAX_UNIX_SECONDS) {
|
|
6917
6959
|
return true;
|
|
6918
6960
|
}
|
|
@@ -6921,7 +6963,7 @@ function validateUnixSeconds(state, value, path31, location) {
|
|
|
6921
6963
|
"error",
|
|
6922
6964
|
"INVALID_UNIX_SECONDS",
|
|
6923
6965
|
"Expected a non-negative integer Unix timestamp in seconds, not milliseconds.",
|
|
6924
|
-
childLocation(location,
|
|
6966
|
+
childLocation(location, path30)
|
|
6925
6967
|
);
|
|
6926
6968
|
return false;
|
|
6927
6969
|
}
|
|
@@ -6982,30 +7024,30 @@ function validateHeader(state, chatlab, meta, location, basePath = "") {
|
|
|
6982
7024
|
}
|
|
6983
7025
|
validateOptionalString(state, meta, "sourceSessionId", `${basePath}meta.sourceSessionId`, location);
|
|
6984
7026
|
}
|
|
6985
|
-
function validateMember(state, value,
|
|
7027
|
+
function validateMember(state, value, path30, location) {
|
|
6986
7028
|
state.stats.memberCount++;
|
|
6987
7029
|
if (!isRecord(value)) {
|
|
6988
|
-
addIssue(state, "error", "INVALID_MEMBER", "Expected a member object.", childLocation(location,
|
|
7030
|
+
addIssue(state, "error", "INVALID_MEMBER", "Expected a member object.", childLocation(location, path30));
|
|
6989
7031
|
return;
|
|
6990
7032
|
}
|
|
6991
|
-
if (validateRequiredString(state, value.platformId, `${
|
|
7033
|
+
if (validateRequiredString(state, value.platformId, `${path30}.platformId`, location)) {
|
|
6992
7034
|
if (state.memberIds.has(value.platformId)) {
|
|
6993
7035
|
addIssue(state, "error", "DUPLICATE_MEMBER_ID", "Member platformId must be unique.", {
|
|
6994
7036
|
...location,
|
|
6995
|
-
path: `${
|
|
7037
|
+
path: `${path30}.platformId`
|
|
6996
7038
|
});
|
|
6997
7039
|
} else {
|
|
6998
7040
|
state.memberIds.add(value.platformId);
|
|
6999
7041
|
}
|
|
7000
7042
|
}
|
|
7001
|
-
validateRequiredString(state, value.accountName, `${
|
|
7002
|
-
validateOptionalString(state, value, "groupNickname", `${
|
|
7003
|
-
validateOptionalString(state, value, "avatar", `${
|
|
7043
|
+
validateRequiredString(state, value.accountName, `${path30}.accountName`, location);
|
|
7044
|
+
validateOptionalString(state, value, "groupNickname", `${path30}.groupNickname`, location);
|
|
7045
|
+
validateOptionalString(state, value, "avatar", `${path30}.avatar`, location);
|
|
7004
7046
|
if (hasOwn(value, "aliases")) {
|
|
7005
7047
|
if (!Array.isArray(value.aliases) || value.aliases.some((alias) => typeof alias !== "string")) {
|
|
7006
7048
|
addIssue(state, "error", "INVALID_ALIASES", "Expected aliases to be an array of strings.", {
|
|
7007
7049
|
...location,
|
|
7008
|
-
path: `${
|
|
7050
|
+
path: `${path30}.aliases`
|
|
7009
7051
|
});
|
|
7010
7052
|
}
|
|
7011
7053
|
}
|
|
@@ -7013,11 +7055,11 @@ function validateMember(state, value, path31, location) {
|
|
|
7013
7055
|
if (!Array.isArray(value.roles)) {
|
|
7014
7056
|
addIssue(state, "error", "INVALID_ROLES", "Expected roles to be an array.", {
|
|
7015
7057
|
...location,
|
|
7016
|
-
path: `${
|
|
7058
|
+
path: `${path30}.roles`
|
|
7017
7059
|
});
|
|
7018
7060
|
} else {
|
|
7019
7061
|
value.roles.forEach((role, index) => {
|
|
7020
|
-
const rolePath = `${
|
|
7062
|
+
const rolePath = `${path30}.roles[${index}]`;
|
|
7021
7063
|
if (!isRecord(role)) {
|
|
7022
7064
|
addIssue(state, "error", "INVALID_ROLE", "Expected a role object.", {
|
|
7023
7065
|
...location,
|
|
@@ -7031,25 +7073,25 @@ function validateMember(state, value, path31, location) {
|
|
|
7031
7073
|
}
|
|
7032
7074
|
}
|
|
7033
7075
|
}
|
|
7034
|
-
function validateMessage(state, value,
|
|
7076
|
+
function validateMessage(state, value, path30, location) {
|
|
7035
7077
|
state.stats.messageCount++;
|
|
7036
7078
|
if (!isRecord(value)) {
|
|
7037
|
-
addIssue(state, "error", "INVALID_MESSAGE", "Expected a message object.", childLocation(location,
|
|
7079
|
+
addIssue(state, "error", "INVALID_MESSAGE", "Expected a message object.", childLocation(location, path30));
|
|
7038
7080
|
return;
|
|
7039
7081
|
}
|
|
7040
|
-
const sender = validateRequiredString(state, value.sender, `${
|
|
7082
|
+
const sender = validateRequiredString(state, value.sender, `${path30}.sender`, location) ? value.sender : void 0;
|
|
7041
7083
|
if (sender !== void 0) {
|
|
7042
7084
|
state.senderIds.add(sender);
|
|
7043
7085
|
}
|
|
7044
|
-
validateRequiredString(state, value.accountName, `${
|
|
7045
|
-
if (validateUnixSeconds(state, value.timestamp, `${
|
|
7086
|
+
validateRequiredString(state, value.accountName, `${path30}.accountName`, location);
|
|
7087
|
+
if (validateUnixSeconds(state, value.timestamp, `${path30}.timestamp`, location)) {
|
|
7046
7088
|
if (state.lastTimestamp !== void 0 && value.timestamp < state.lastTimestamp) {
|
|
7047
7089
|
addIssue(
|
|
7048
7090
|
state,
|
|
7049
7091
|
"warning",
|
|
7050
7092
|
"OUT_OF_ORDER_TIMESTAMP",
|
|
7051
7093
|
"Messages should be ordered by timestamp ascending.",
|
|
7052
|
-
childLocation(location, `${
|
|
7094
|
+
childLocation(location, `${path30}.timestamp`)
|
|
7053
7095
|
);
|
|
7054
7096
|
}
|
|
7055
7097
|
state.lastTimestamp = value.timestamp;
|
|
@@ -7058,7 +7100,7 @@ function validateMessage(state, value, path31, location) {
|
|
|
7058
7100
|
if (messageType === void 0) {
|
|
7059
7101
|
addIssue(state, "error", "INVALID_MESSAGE_TYPE", "Expected a supported numeric ChatLab message type.", {
|
|
7060
7102
|
...location,
|
|
7061
|
-
path: `${
|
|
7103
|
+
path: `${path30}.type`
|
|
7062
7104
|
});
|
|
7063
7105
|
} else if (sender === SYSTEM_SENDER_ID && !SYSTEM_SENDER_MESSAGE_TYPES.has(messageType)) {
|
|
7064
7106
|
addIssue(
|
|
@@ -7066,22 +7108,22 @@ function validateMessage(state, value, path31, location) {
|
|
|
7066
7108
|
"error",
|
|
7067
7109
|
"INVALID_SYSTEM_SENDER_TYPE",
|
|
7068
7110
|
"The reserved SYSTEM sender is only valid for system and recall messages.",
|
|
7069
|
-
childLocation(location, `${
|
|
7111
|
+
childLocation(location, `${path30}.sender`)
|
|
7070
7112
|
);
|
|
7071
7113
|
}
|
|
7072
7114
|
if (!hasOwn(value, "content") || typeof value.content !== "string" && value.content !== null) {
|
|
7073
7115
|
addIssue(state, "error", "INVALID_MESSAGE_CONTENT", "Expected content to be a string or null.", {
|
|
7074
7116
|
...location,
|
|
7075
|
-
path: `${
|
|
7117
|
+
path: `${path30}.content`
|
|
7076
7118
|
});
|
|
7077
7119
|
}
|
|
7078
|
-
validateOptionalString(state, value, "groupNickname", `${
|
|
7120
|
+
validateOptionalString(state, value, "groupNickname", `${path30}.groupNickname`, location);
|
|
7079
7121
|
if (hasOwn(value, "platformMessageId")) {
|
|
7080
|
-
if (validateRequiredString(state, value.platformMessageId, `${
|
|
7122
|
+
if (validateRequiredString(state, value.platformMessageId, `${path30}.platformMessageId`, location)) {
|
|
7081
7123
|
if (state.messageIds.has(value.platformMessageId)) {
|
|
7082
7124
|
addIssue(state, "error", "DUPLICATE_MESSAGE_ID", "platformMessageId must be unique.", {
|
|
7083
7125
|
...location,
|
|
7084
|
-
path: `${
|
|
7126
|
+
path: `${path30}.platformMessageId`
|
|
7085
7127
|
});
|
|
7086
7128
|
} else {
|
|
7087
7129
|
state.messageIds.add(value.platformMessageId);
|
|
@@ -7089,7 +7131,7 @@ function validateMessage(state, value, path31, location) {
|
|
|
7089
7131
|
}
|
|
7090
7132
|
}
|
|
7091
7133
|
if (hasOwn(value, "replyToMessageId")) {
|
|
7092
|
-
if (validateRequiredString(state, value.replyToMessageId, `${
|
|
7134
|
+
if (validateRequiredString(state, value.replyToMessageId, `${path30}.replyToMessageId`, location)) {
|
|
7093
7135
|
state.replyTargetIds.add(value.replyToMessageId);
|
|
7094
7136
|
}
|
|
7095
7137
|
}
|
|
@@ -7416,8 +7458,7 @@ function getSupportedFormats() {
|
|
|
7416
7458
|
return sniffer.getSupportedFormats();
|
|
7417
7459
|
}
|
|
7418
7460
|
function getFormatFeatureById(formatId) {
|
|
7419
|
-
|
|
7420
|
-
return all.find((f) => f.id === formatId) ?? null;
|
|
7461
|
+
return sniffer.getFeatureById(formatId);
|
|
7421
7462
|
}
|
|
7422
7463
|
function getPreprocessor(filePath) {
|
|
7423
7464
|
const feature15 = sniffer.sniff(filePath);
|
|
@@ -7649,319 +7690,6 @@ function logSummary(totalMessages, totalMembers) {
|
|
|
7649
7690
|
defaultLogger.summary(totalMessages, totalMembers);
|
|
7650
7691
|
}
|
|
7651
7692
|
|
|
7652
|
-
// ../../packages/node-runtime/src/nlp/word-frequency.ts
|
|
7653
|
-
function buildMessageQuery(timeFilter, memberId, excludeKeywords) {
|
|
7654
|
-
const conditions = [];
|
|
7655
|
-
const params = [];
|
|
7656
|
-
if (timeFilter?.startTs !== void 0) {
|
|
7657
|
-
conditions.push("msg.ts >= ?");
|
|
7658
|
-
params.push(timeFilter.startTs);
|
|
7659
|
-
}
|
|
7660
|
-
if (timeFilter?.endTs !== void 0) {
|
|
7661
|
-
conditions.push("msg.ts <= ?");
|
|
7662
|
-
params.push(timeFilter.endTs);
|
|
7663
|
-
}
|
|
7664
|
-
if (memberId !== void 0 && memberId !== null) {
|
|
7665
|
-
conditions.push("msg.sender_id = ?");
|
|
7666
|
-
params.push(memberId);
|
|
7667
|
-
}
|
|
7668
|
-
if (excludeKeywords && excludeKeywords.length > 0) {
|
|
7669
|
-
const exclude = buildExcludeKeywordsConditions(excludeKeywords);
|
|
7670
|
-
conditions.push(...exclude.conditions);
|
|
7671
|
-
params.push(...exclude.params);
|
|
7672
|
-
}
|
|
7673
|
-
conditions.push("COALESCE(m.account_name, '') != '\u7CFB\u7EDF\u6D88\u606F'");
|
|
7674
|
-
conditions.push("msg.type = 0");
|
|
7675
|
-
conditions.push("msg.content IS NOT NULL");
|
|
7676
|
-
conditions.push("TRIM(msg.content) != ''");
|
|
7677
|
-
return {
|
|
7678
|
-
clause: ` WHERE ${conditions.join(" AND ")}`,
|
|
7679
|
-
params
|
|
7680
|
-
};
|
|
7681
|
-
}
|
|
7682
|
-
function computeWordFrequency(db, params) {
|
|
7683
|
-
const {
|
|
7684
|
-
locale,
|
|
7685
|
-
timeFilter,
|
|
7686
|
-
memberId,
|
|
7687
|
-
topN = 100,
|
|
7688
|
-
minWordLength,
|
|
7689
|
-
minCount = 2,
|
|
7690
|
-
posFilterMode = "meaningful",
|
|
7691
|
-
customPosTags,
|
|
7692
|
-
enableStopwords = true,
|
|
7693
|
-
dictType = "default",
|
|
7694
|
-
excludeWords,
|
|
7695
|
-
excludeKeywords
|
|
7696
|
-
} = params;
|
|
7697
|
-
const { clause, params: filterParams } = buildMessageQuery(timeFilter, memberId, excludeKeywords);
|
|
7698
|
-
const messages = db.prepare(`SELECT msg.content FROM message msg JOIN member m ON msg.sender_id = m.id${clause}`).all(...filterParams);
|
|
7699
|
-
if (messages.length === 0) {
|
|
7700
|
-
return { words: [], totalWords: 0, totalMessages: 0, uniqueWords: 0 };
|
|
7701
|
-
}
|
|
7702
|
-
const texts = messages.map((m) => m.content);
|
|
7703
|
-
const segmentOptions = {
|
|
7704
|
-
minLength: minWordLength,
|
|
7705
|
-
minCount,
|
|
7706
|
-
topN,
|
|
7707
|
-
posFilterMode,
|
|
7708
|
-
customPosTags,
|
|
7709
|
-
enableStopwords,
|
|
7710
|
-
dictType,
|
|
7711
|
-
excludeWords
|
|
7712
|
-
};
|
|
7713
|
-
let posTagStats;
|
|
7714
|
-
let result;
|
|
7715
|
-
if (locale.startsWith("zh") && posFilterMode !== "all") {
|
|
7716
|
-
const combined = batchSegmentChineseWithStats(texts, locale, segmentOptions);
|
|
7717
|
-
result = { words: combined.words, uniqueWords: combined.uniqueWords, totalWords: combined.totalWords };
|
|
7718
|
-
posTagStats = [...combined.posTagStats.entries()].map(([tag, count]) => ({ tag, count }));
|
|
7719
|
-
} else {
|
|
7720
|
-
if (locale.startsWith("zh")) {
|
|
7721
|
-
const posStatsMap = collectPosTagStats(texts, minWordLength ?? 2, enableStopwords, dictType);
|
|
7722
|
-
posTagStats = [...posStatsMap.entries()].map(([tag, count]) => ({ tag, count }));
|
|
7723
|
-
}
|
|
7724
|
-
result = batchSegmentWithFrequency(texts, locale, segmentOptions);
|
|
7725
|
-
}
|
|
7726
|
-
let topNTotalWords = 0;
|
|
7727
|
-
for (const count of result.words.values()) topNTotalWords += count;
|
|
7728
|
-
const words = [...result.words.entries()].map(([word, count]) => ({
|
|
7729
|
-
word,
|
|
7730
|
-
count,
|
|
7731
|
-
percentage: topNTotalWords > 0 ? Math.round(count / topNTotalWords * 1e4) / 100 : 0
|
|
7732
|
-
}));
|
|
7733
|
-
return {
|
|
7734
|
-
words,
|
|
7735
|
-
totalWords: result.totalWords,
|
|
7736
|
-
totalMessages: messages.length,
|
|
7737
|
-
uniqueWords: result.uniqueWords,
|
|
7738
|
-
posTagStats
|
|
7739
|
-
};
|
|
7740
|
-
}
|
|
7741
|
-
function segmentText(text, locale, minLength) {
|
|
7742
|
-
return segment(text, locale, { minLength });
|
|
7743
|
-
}
|
|
7744
|
-
|
|
7745
|
-
// ../../packages/node-runtime/src/nlp/dict-manager.ts
|
|
7746
|
-
import * as fs20 from "fs";
|
|
7747
|
-
import * as path13 from "path";
|
|
7748
|
-
import { createHash } from "crypto";
|
|
7749
|
-
var DICT_DOWNLOAD_URL_BASE = "https://chatlab.fun/assets/nlp";
|
|
7750
|
-
var DICT_SHA256 = {
|
|
7751
|
-
"zh-CN": "139519822fe8ab9e10d9d07e68ea0451045380aedaf54ecc51e2a28c6b42a13f",
|
|
7752
|
-
"zh-TW": "a63ec7e388f16f1b486dcd948a9f1a3b492be5d9b6bdab786a95e59966786dfd"
|
|
7753
|
-
};
|
|
7754
|
-
var AVAILABLE_DICTS = [
|
|
7755
|
-
{ id: "zh-CN", label: "\u7B80\u4F53\u4E2D\u6587", locale: "zh-CN" },
|
|
7756
|
-
{ id: "zh-TW", label: "\u7E41\u9AD4\u4E2D\u6587", locale: "zh-TW" }
|
|
7757
|
-
];
|
|
7758
|
-
function getDictFilePath(nlpDir, dictId) {
|
|
7759
|
-
return path13.join(nlpDir, `${dictId}.dict`);
|
|
7760
|
-
}
|
|
7761
|
-
function isDictDownloaded(nlpDir, dictId) {
|
|
7762
|
-
return fs20.existsSync(getDictFilePath(nlpDir, dictId));
|
|
7763
|
-
}
|
|
7764
|
-
function getDictList(nlpDir) {
|
|
7765
|
-
return AVAILABLE_DICTS.map((d) => {
|
|
7766
|
-
const filePath = getDictFilePath(nlpDir, d.id);
|
|
7767
|
-
const downloaded = fs20.existsSync(filePath);
|
|
7768
|
-
let fileSize;
|
|
7769
|
-
if (downloaded) {
|
|
7770
|
-
try {
|
|
7771
|
-
fileSize = fs20.statSync(filePath).size;
|
|
7772
|
-
} catch {
|
|
7773
|
-
}
|
|
7774
|
-
}
|
|
7775
|
-
return { ...d, downloaded, fileSize };
|
|
7776
|
-
});
|
|
7777
|
-
}
|
|
7778
|
-
function loadDictBuffer(nlpDir, dictId) {
|
|
7779
|
-
const filePath = getDictFilePath(nlpDir, dictId);
|
|
7780
|
-
if (!fs20.existsSync(filePath)) return null;
|
|
7781
|
-
try {
|
|
7782
|
-
return fs20.readFileSync(filePath);
|
|
7783
|
-
} catch (error) {
|
|
7784
|
-
console.error(`[NLP DictManager] Failed to read dict file: ${filePath}`, error);
|
|
7785
|
-
return null;
|
|
7786
|
-
}
|
|
7787
|
-
}
|
|
7788
|
-
async function downloadDict(nlpDir, dictId, onProgress) {
|
|
7789
|
-
if (!AVAILABLE_DICTS.find((d) => d.id === dictId)) {
|
|
7790
|
-
return { success: false, error: `Unknown dict: ${dictId}` };
|
|
7791
|
-
}
|
|
7792
|
-
fs20.mkdirSync(nlpDir, { recursive: true });
|
|
7793
|
-
const url = `${DICT_DOWNLOAD_URL_BASE}/${dictId}.dict`;
|
|
7794
|
-
const filePath = getDictFilePath(nlpDir, dictId);
|
|
7795
|
-
const tmpPath = filePath + ".tmp";
|
|
7796
|
-
try {
|
|
7797
|
-
const response = await fetch(url, { signal: AbortSignal.timeout(12e4) });
|
|
7798
|
-
if (!response.ok) throw new Error(`Download failed: HTTP ${response.status}`);
|
|
7799
|
-
const total = Number(response.headers.get("content-length") || 0);
|
|
7800
|
-
let buffer;
|
|
7801
|
-
if (!response.body) {
|
|
7802
|
-
buffer = Buffer.from(await response.arrayBuffer());
|
|
7803
|
-
} else {
|
|
7804
|
-
const reader = response.body.getReader();
|
|
7805
|
-
const chunks = [];
|
|
7806
|
-
let loaded = 0;
|
|
7807
|
-
while (true) {
|
|
7808
|
-
const { done, value } = await reader.read();
|
|
7809
|
-
if (done) break;
|
|
7810
|
-
const chunk = Buffer.from(value);
|
|
7811
|
-
chunks.push(chunk);
|
|
7812
|
-
loaded += chunk.length;
|
|
7813
|
-
if (total > 0 && onProgress) onProgress(Math.round(loaded / total * 100));
|
|
7814
|
-
}
|
|
7815
|
-
buffer = Buffer.concat(chunks);
|
|
7816
|
-
}
|
|
7817
|
-
const MIN_DICT_SIZE = 1e6;
|
|
7818
|
-
if (buffer.length < MIN_DICT_SIZE) {
|
|
7819
|
-
return { success: false, error: `Downloaded file is invalid (${buffer.length} bytes)` };
|
|
7820
|
-
}
|
|
7821
|
-
const head = buffer.subarray(0, 50).toString("utf-8").trim();
|
|
7822
|
-
if (head.startsWith("<!") || head.startsWith("<html")) {
|
|
7823
|
-
return { success: false, error: "Downloaded file is HTML, not a dictionary file" };
|
|
7824
|
-
}
|
|
7825
|
-
const expectedSha256 = DICT_SHA256[dictId];
|
|
7826
|
-
if (!expectedSha256) {
|
|
7827
|
-
return { success: false, error: `Missing SHA256 checksum for dict: ${dictId}` };
|
|
7828
|
-
}
|
|
7829
|
-
const actualSha256 = createHash("sha256").update(buffer).digest("hex");
|
|
7830
|
-
if (actualSha256 !== expectedSha256) {
|
|
7831
|
-
return { success: false, error: "Dictionary integrity check failed (SHA256 mismatch)" };
|
|
7832
|
-
}
|
|
7833
|
-
fs20.writeFileSync(tmpPath, buffer);
|
|
7834
|
-
if (fs20.existsSync(filePath)) fs20.unlinkSync(filePath);
|
|
7835
|
-
fs20.renameSync(tmpPath, filePath);
|
|
7836
|
-
clearJiebaInstance(dictId);
|
|
7837
|
-
console.log(`[NLP DictManager] Dict downloaded: ${dictId} (${fs20.statSync(filePath).size} bytes)`);
|
|
7838
|
-
return { success: true };
|
|
7839
|
-
} catch (error) {
|
|
7840
|
-
if (fs20.existsSync(tmpPath)) {
|
|
7841
|
-
try {
|
|
7842
|
-
fs20.unlinkSync(tmpPath);
|
|
7843
|
-
} catch {
|
|
7844
|
-
}
|
|
7845
|
-
}
|
|
7846
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
7847
|
-
console.error(`[NLP DictManager] Download failed for ${dictId}:`, msg);
|
|
7848
|
-
return { success: false, error: msg };
|
|
7849
|
-
}
|
|
7850
|
-
}
|
|
7851
|
-
function deleteDict(nlpDir, dictId) {
|
|
7852
|
-
const filePath = getDictFilePath(nlpDir, dictId);
|
|
7853
|
-
if (!fs20.existsSync(filePath)) return { success: true };
|
|
7854
|
-
try {
|
|
7855
|
-
fs20.unlinkSync(filePath);
|
|
7856
|
-
clearJiebaInstance(dictId);
|
|
7857
|
-
console.log(`[NLP DictManager] Dict deleted: ${dictId}`);
|
|
7858
|
-
return { success: true };
|
|
7859
|
-
} catch (error) {
|
|
7860
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
7861
|
-
console.error(`[NLP DictManager] Delete failed for ${dictId}:`, msg);
|
|
7862
|
-
return { success: false, error: msg };
|
|
7863
|
-
}
|
|
7864
|
-
}
|
|
7865
|
-
async function ensureDefaultDict(nlpDir) {
|
|
7866
|
-
if (isDictDownloaded(nlpDir, "zh-CN")) return;
|
|
7867
|
-
console.log("[NLP DictManager] zh-CN dict not found, starting background download...");
|
|
7868
|
-
const result = await downloadDict(nlpDir, "zh-CN");
|
|
7869
|
-
if (result.success) {
|
|
7870
|
-
console.log("[NLP DictManager] zh-CN dict auto-downloaded successfully");
|
|
7871
|
-
} else {
|
|
7872
|
-
console.warn("[NLP DictManager] zh-CN dict auto-download failed:", result.error);
|
|
7873
|
-
}
|
|
7874
|
-
}
|
|
7875
|
-
|
|
7876
|
-
// ../../packages/node-runtime/src/fts/index.ts
|
|
7877
|
-
var BATCH_SIZE = 5e3;
|
|
7878
|
-
function hasFtsTable(db) {
|
|
7879
|
-
try {
|
|
7880
|
-
const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='message_fts'").get();
|
|
7881
|
-
return !!row;
|
|
7882
|
-
} catch {
|
|
7883
|
-
return false;
|
|
7884
|
-
}
|
|
7885
|
-
}
|
|
7886
|
-
function createFtsTable(db) {
|
|
7887
|
-
db.exec(FTS_TABLE_SCHEMA);
|
|
7888
|
-
}
|
|
7889
|
-
function buildFtsIndex(db) {
|
|
7890
|
-
createFtsTable(db);
|
|
7891
|
-
const insertFts = db.prepare("INSERT INTO message_fts(rowid, content) VALUES (?, ?)");
|
|
7892
|
-
let indexed = 0;
|
|
7893
|
-
let lastId = null;
|
|
7894
|
-
const selectFirstBatch = db.prepare(
|
|
7895
|
-
`SELECT id, content FROM message
|
|
7896
|
-
WHERE type = 0
|
|
7897
|
-
AND content IS NOT NULL
|
|
7898
|
-
AND content != ''
|
|
7899
|
-
ORDER BY id ASC
|
|
7900
|
-
LIMIT ?`
|
|
7901
|
-
);
|
|
7902
|
-
const selectNextBatch = db.prepare(
|
|
7903
|
-
`SELECT id, content FROM message
|
|
7904
|
-
WHERE type = 0
|
|
7905
|
-
AND content IS NOT NULL
|
|
7906
|
-
AND content != ''
|
|
7907
|
-
AND id > ?
|
|
7908
|
-
ORDER BY id ASC
|
|
7909
|
-
LIMIT ?`
|
|
7910
|
-
);
|
|
7911
|
-
while (true) {
|
|
7912
|
-
const rows = lastId === null ? selectFirstBatch.all(BATCH_SIZE) : selectNextBatch.all(lastId, BATCH_SIZE);
|
|
7913
|
-
if (rows.length === 0) break;
|
|
7914
|
-
db.transaction(() => {
|
|
7915
|
-
for (const row of rows) {
|
|
7916
|
-
const tokens = tokenizeForFts(row.content);
|
|
7917
|
-
if (tokens) {
|
|
7918
|
-
insertFts.run(row.id, tokens);
|
|
7919
|
-
}
|
|
7920
|
-
}
|
|
7921
|
-
});
|
|
7922
|
-
indexed += rows.length;
|
|
7923
|
-
lastId = rows[rows.length - 1].id;
|
|
7924
|
-
}
|
|
7925
|
-
return { indexed };
|
|
7926
|
-
}
|
|
7927
|
-
function rebuildFtsIndex(db) {
|
|
7928
|
-
if (hasFtsTable(db)) {
|
|
7929
|
-
db.exec("DROP TABLE message_fts");
|
|
7930
|
-
}
|
|
7931
|
-
return buildFtsIndex(db);
|
|
7932
|
-
}
|
|
7933
|
-
function insertFtsEntries(db, entries) {
|
|
7934
|
-
if (!hasFtsTable(db)) return;
|
|
7935
|
-
const insertFts = db.prepare("INSERT INTO message_fts(rowid, content) VALUES (?, ?)");
|
|
7936
|
-
db.transaction(() => {
|
|
7937
|
-
for (const entry of entries) {
|
|
7938
|
-
if (entry.content) {
|
|
7939
|
-
const tokens = tokenizeForFts(entry.content);
|
|
7940
|
-
if (tokens) {
|
|
7941
|
-
insertFts.run(entry.id, tokens);
|
|
7942
|
-
}
|
|
7943
|
-
}
|
|
7944
|
-
}
|
|
7945
|
-
});
|
|
7946
|
-
}
|
|
7947
|
-
function searchByFts(db, keywords, limit = 1e3, offset = 0) {
|
|
7948
|
-
if (keywords.length === 0) return { rowids: [], total: 0 };
|
|
7949
|
-
const matchQuery = tokenizeQueryForFts(keywords);
|
|
7950
|
-
if (!matchQuery) return { rowids: [], total: 0 };
|
|
7951
|
-
try {
|
|
7952
|
-
const countRow = db.prepare("SELECT COUNT(*) as total FROM message_fts WHERE content MATCH ?").get(matchQuery);
|
|
7953
|
-
const total = countRow?.total ?? 0;
|
|
7954
|
-
const rows = db.prepare(`SELECT rowid FROM message_fts WHERE content MATCH ? ORDER BY rank LIMIT ? OFFSET ?`).all(matchQuery, limit, offset);
|
|
7955
|
-
return {
|
|
7956
|
-
rowids: rows.map((r) => r.rowid),
|
|
7957
|
-
total
|
|
7958
|
-
};
|
|
7959
|
-
} catch (error) {
|
|
7960
|
-
console.error("[FTS] Search failed, query:", matchQuery, error);
|
|
7961
|
-
return { rowids: [], total: 0 };
|
|
7962
|
-
}
|
|
7963
|
-
}
|
|
7964
|
-
|
|
7965
7693
|
// ../../packages/node-runtime/src/import/message-batch-inserter.ts
|
|
7966
7694
|
var MESSAGE_COLUMN_COUNT = 8;
|
|
7967
7695
|
var SQLITE_LEGACY_VARIABLE_LIMIT = 999;
|
|
@@ -8016,10 +7744,24 @@ function toParams(row) {
|
|
|
8016
7744
|
}
|
|
8017
7745
|
|
|
8018
7746
|
// ../../packages/node-runtime/src/import/streaming-importer.ts
|
|
8019
|
-
import * as
|
|
7747
|
+
import * as fs20 from "fs";
|
|
8020
7748
|
import { performance } from "perf_hooks";
|
|
8021
7749
|
|
|
8022
7750
|
// ../../packages/node-runtime/src/import/message-deduplicator.ts
|
|
7751
|
+
var SCOPED_PLATFORM_MESSAGE_ID_PATTERN = /^(__chatlab_message_scope__\d+__)(.*)$/;
|
|
7752
|
+
function parsePlatformMessageId(id) {
|
|
7753
|
+
const scoped = SCOPED_PLATFORM_MESSAGE_ID_PATTERN.exec(id);
|
|
7754
|
+
if (!scoped) return { rawId: id };
|
|
7755
|
+
try {
|
|
7756
|
+
return { rawId: decodeURIComponent(scoped[2]), scope: scoped[1] };
|
|
7757
|
+
} catch {
|
|
7758
|
+
return { rawId: id };
|
|
7759
|
+
}
|
|
7760
|
+
}
|
|
7761
|
+
function applyPlatformMessageIdScope(id, scope) {
|
|
7762
|
+
if (!id || !scope) return id;
|
|
7763
|
+
return `${scope}${encodeURIComponent(parsePlatformMessageId(id).rawId)}`;
|
|
7764
|
+
}
|
|
8023
7765
|
function generateFallbackMessageKey(message) {
|
|
8024
7766
|
const contentKey = generateMessageKey(message.timestamp, message.senderPlatformId, message.content);
|
|
8025
7767
|
return JSON.stringify([contentKey, message.type, message.replyToMessageId || null]);
|
|
@@ -8148,7 +7890,6 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8148
7890
|
messageWriteMs: 0,
|
|
8149
7891
|
nicknameHistoryMs: 0,
|
|
8150
7892
|
indexCreationMs: 0,
|
|
8151
|
-
ftsMs: 0,
|
|
8152
7893
|
checkpointMs: 0,
|
|
8153
7894
|
sessionIndexMs: 0,
|
|
8154
7895
|
postImportHookMs: 0,
|
|
@@ -8551,16 +8292,6 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8551
8292
|
timings.indexCreationMs = elapsedMs(indexCreationStartedAt, now);
|
|
8552
8293
|
sampleRss();
|
|
8553
8294
|
logger?.perf("Indexes created", totalMessageCount);
|
|
8554
|
-
const ftsStartedAt = now();
|
|
8555
|
-
try {
|
|
8556
|
-
buildFtsIndex(db);
|
|
8557
|
-
logger?.perf("FTS index built", totalMessageCount);
|
|
8558
|
-
} catch (ftsError) {
|
|
8559
|
-
logger?.error("FTS index build failed (non-fatal)", ftsError instanceof Error ? ftsError : void 0);
|
|
8560
|
-
} finally {
|
|
8561
|
-
timings.ftsMs = elapsedMs(ftsStartedAt, now);
|
|
8562
|
-
sampleRss();
|
|
8563
|
-
}
|
|
8564
8295
|
onProgress({
|
|
8565
8296
|
stage: "indexing",
|
|
8566
8297
|
bytesRead: 0,
|
|
@@ -8598,7 +8329,7 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
|
|
|
8598
8329
|
}
|
|
8599
8330
|
}
|
|
8600
8331
|
logger?.perfDetail(
|
|
8601
|
-
`[Stages] parser=${timings.parserMs.toFixed(1)}ms | message-write=${timings.messageWriteMs.toFixed(1)}ms | indexes=${timings.indexCreationMs.toFixed(1)}ms |
|
|
8332
|
+
`[Stages] parser=${timings.parserMs.toFixed(1)}ms | message-write=${timings.messageWriteMs.toFixed(1)}ms | indexes=${timings.indexCreationMs.toFixed(1)}ms | session-index=${timings.sessionIndexMs.toFixed(1)}ms | hook=${timings.postImportHookMs.toFixed(1)}ms`
|
|
8602
8333
|
);
|
|
8603
8334
|
logger?.perf("Import completed", totalMessageCount);
|
|
8604
8335
|
logger?.info(`=== Parser Callback Stats ===`);
|
|
@@ -8735,7 +8466,7 @@ async function streamParseFileInfo(filePath, deps) {
|
|
|
8735
8466
|
if (!formatFeature) {
|
|
8736
8467
|
throw new Error("Unrecognized file format");
|
|
8737
8468
|
}
|
|
8738
|
-
const fileSize =
|
|
8469
|
+
const fileSize = fs20.statSync(filePath).size;
|
|
8739
8470
|
deps.onProgress({
|
|
8740
8471
|
stage: "parsing",
|
|
8741
8472
|
bytesRead: 0,
|
|
@@ -8752,8 +8483,10 @@ async function streamParseFileInfo(filePath, deps) {
|
|
|
8752
8483
|
"INSERT OR IGNORE INTO member (platform_id, account_name, group_nickname, avatar) VALUES (?, ?, ?, ?)"
|
|
8753
8484
|
);
|
|
8754
8485
|
const insertMessage = db.prepare(
|
|
8755
|
-
`INSERT INTO message (
|
|
8756
|
-
|
|
8486
|
+
`INSERT INTO message (
|
|
8487
|
+
platform_message_id, sender_platform_id, sender_account_name, sender_group_nickname,
|
|
8488
|
+
timestamp, type, content, reply_to_message_id
|
|
8489
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
8757
8490
|
);
|
|
8758
8491
|
let meta = { name: "Unknown", platform: formatFeature.platform, type: 0 };
|
|
8759
8492
|
const memberSet = /* @__PURE__ */ new Set();
|
|
@@ -8795,12 +8528,14 @@ async function streamParseFileInfo(filePath, deps) {
|
|
|
8795
8528
|
insertMember.run(msg.senderPlatformId, msg.senderAccountName || null, msg.senderGroupNickname || null, null);
|
|
8796
8529
|
}
|
|
8797
8530
|
insertMessage.run(
|
|
8531
|
+
msg.platformMessageId || null,
|
|
8798
8532
|
msg.senderPlatformId,
|
|
8799
8533
|
msg.senderAccountName || null,
|
|
8800
8534
|
msg.senderGroupNickname || null,
|
|
8801
8535
|
msg.timestamp,
|
|
8802
8536
|
msg.type,
|
|
8803
|
-
msg.content || null
|
|
8537
|
+
msg.content || null,
|
|
8538
|
+
msg.replyToMessageId || null
|
|
8804
8539
|
);
|
|
8805
8540
|
messageCount++;
|
|
8806
8541
|
}
|
|
@@ -8824,7 +8559,7 @@ async function streamParseFileInfo(filePath, deps) {
|
|
|
8824
8559
|
}
|
|
8825
8560
|
db.close();
|
|
8826
8561
|
try {
|
|
8827
|
-
if (
|
|
8562
|
+
if (fs20.existsSync(tempDbPath)) fs20.unlinkSync(tempDbPath);
|
|
8828
8563
|
} catch {
|
|
8829
8564
|
}
|
|
8830
8565
|
throw error;
|
|
@@ -8886,40 +8621,112 @@ function countHistory(tracker) {
|
|
|
8886
8621
|
}
|
|
8887
8622
|
|
|
8888
8623
|
// ../../packages/node-runtime/src/import/incremental-importer.ts
|
|
8889
|
-
|
|
8890
|
-
|
|
8891
|
-
|
|
8892
|
-
|
|
8893
|
-
|
|
8894
|
-
|
|
8895
|
-
|
|
8896
|
-
}
|
|
8897
|
-
|
|
8898
|
-
|
|
8899
|
-
|
|
8900
|
-
|
|
8901
|
-
|
|
8902
|
-
|
|
8903
|
-
|
|
8904
|
-
|
|
8905
|
-
|
|
8906
|
-
|
|
8907
|
-
|
|
8908
|
-
|
|
8909
|
-
|
|
8910
|
-
|
|
8911
|
-
|
|
8912
|
-
|
|
8624
|
+
var PLATFORM_ID_QUERY_CHUNK_SIZE = 500;
|
|
8625
|
+
function applyMessageIdScope(message, scope) {
|
|
8626
|
+
if (!scope) return message;
|
|
8627
|
+
return {
|
|
8628
|
+
...message,
|
|
8629
|
+
platformMessageId: applyPlatformMessageIdScope(message.platformMessageId, scope),
|
|
8630
|
+
replyToMessageId: applyPlatformMessageIdScope(message.replyToMessageId, scope)
|
|
8631
|
+
};
|
|
8632
|
+
}
|
|
8633
|
+
function createPlatformIdMapper(mappings) {
|
|
8634
|
+
const mapping = new Map(mappings?.map(({ sourceId, targetId }) => [sourceId, targetId]));
|
|
8635
|
+
return (platformId) => mapping.get(platformId) ?? platformId;
|
|
8636
|
+
}
|
|
8637
|
+
function applySenderPlatformIdMapping(message, mapPlatformId) {
|
|
8638
|
+
const senderPlatformId = mapPlatformId(message.senderPlatformId);
|
|
8639
|
+
return senderPlatformId === message.senderPlatformId ? message : { ...message, senderPlatformId };
|
|
8640
|
+
}
|
|
8641
|
+
function createExistingMessageLookup(db) {
|
|
8642
|
+
const maxMessageId = db.prepare("SELECT MAX(id) AS max_id FROM message").get()?.max_id ?? 0;
|
|
8643
|
+
const hasFallbackOnlyMessages = Boolean(
|
|
8644
|
+
db.prepare(
|
|
8645
|
+
`SELECT 1 FROM message
|
|
8646
|
+
WHERE id <= ? AND (platform_message_id IS NULL OR platform_message_id = '')
|
|
8647
|
+
LIMIT 1`
|
|
8648
|
+
).get(maxMessageId)
|
|
8649
|
+
);
|
|
8650
|
+
return {
|
|
8651
|
+
db,
|
|
8652
|
+
maxMessageId,
|
|
8653
|
+
hasFallbackOnlyMessages,
|
|
8654
|
+
consumedFallbackOnlyKeys: /* @__PURE__ */ new Set(),
|
|
8655
|
+
bridgedPlatformMessageIds: /* @__PURE__ */ new Set(),
|
|
8656
|
+
candidateStatement: db.prepare(
|
|
8657
|
+
`SELECT msg.type, msg.content, msg.reply_to_message_id, msg.platform_message_id
|
|
8658
|
+
FROM member m
|
|
8659
|
+
JOIN message msg ON msg.sender_id = m.id
|
|
8660
|
+
WHERE m.platform_id = ? AND msg.ts = ? AND msg.id <= ?`
|
|
8661
|
+
),
|
|
8662
|
+
platformIdStatements: /* @__PURE__ */ new Map()
|
|
8663
|
+
};
|
|
8664
|
+
}
|
|
8665
|
+
function findExistingPlatformMessageIds(lookup, platformMessageIds) {
|
|
8666
|
+
const uniqueIds = [...new Set([...platformMessageIds].filter((id) => Boolean(id)))];
|
|
8667
|
+
const existingIds = /* @__PURE__ */ new Set();
|
|
8668
|
+
for (let start = 0; start < uniqueIds.length; start += PLATFORM_ID_QUERY_CHUNK_SIZE) {
|
|
8669
|
+
const chunk = uniqueIds.slice(start, start + PLATFORM_ID_QUERY_CHUNK_SIZE);
|
|
8670
|
+
let statement = lookup.platformIdStatements.get(chunk.length);
|
|
8671
|
+
if (!statement) {
|
|
8672
|
+
const placeholders = chunk.map(() => "?").join(", ");
|
|
8673
|
+
statement = lookup.db.prepare(
|
|
8674
|
+
`SELECT platform_message_id
|
|
8675
|
+
FROM message
|
|
8676
|
+
WHERE id <= ? AND platform_message_id IN (${placeholders})`
|
|
8677
|
+
);
|
|
8678
|
+
lookup.platformIdStatements.set(chunk.length, statement);
|
|
8679
|
+
}
|
|
8680
|
+
const rows = statement.all(lookup.maxMessageId, ...chunk);
|
|
8681
|
+
for (const row of rows) existingIds.add(row.platform_message_id);
|
|
8913
8682
|
}
|
|
8914
|
-
return
|
|
8683
|
+
return existingIds;
|
|
8915
8684
|
}
|
|
8916
|
-
function
|
|
8917
|
-
|
|
8918
|
-
|
|
8919
|
-
|
|
8920
|
-
|
|
8685
|
+
function getExistingCandidates(lookup, message, cache) {
|
|
8686
|
+
const cacheKey = JSON.stringify([message.senderPlatformId, message.timestamp]);
|
|
8687
|
+
const cached = cache.get(cacheKey);
|
|
8688
|
+
if (cached) return cached;
|
|
8689
|
+
const rows = lookup.candidateStatement.all(
|
|
8690
|
+
message.senderPlatformId,
|
|
8691
|
+
message.timestamp,
|
|
8692
|
+
lookup.maxMessageId
|
|
8693
|
+
);
|
|
8694
|
+
cache.set(cacheKey, rows);
|
|
8695
|
+
return rows;
|
|
8696
|
+
}
|
|
8697
|
+
function candidateFallbackKey(message, candidate) {
|
|
8698
|
+
return generateFallbackMessageKey({
|
|
8699
|
+
timestamp: message.timestamp,
|
|
8700
|
+
senderPlatformId: message.senderPlatformId,
|
|
8701
|
+
type: candidate.type,
|
|
8702
|
+
content: candidate.content,
|
|
8703
|
+
replyToMessageId: candidate.reply_to_message_id ?? void 0
|
|
8921
8704
|
});
|
|
8922
8705
|
}
|
|
8706
|
+
function isExistingMessageDuplicate(lookup, message, existingBatchPlatformIds, candidateCache) {
|
|
8707
|
+
if (message.platformMessageId && (existingBatchPlatformIds.has(message.platformMessageId) || lookup.bridgedPlatformMessageIds.has(message.platformMessageId))) {
|
|
8708
|
+
return true;
|
|
8709
|
+
}
|
|
8710
|
+
const fallbackKey = generateFallbackMessageKey(message);
|
|
8711
|
+
if (message.platformMessageId) {
|
|
8712
|
+
if (!lookup.hasFallbackOnlyMessages || lookup.consumedFallbackOnlyKeys.has(fallbackKey)) return false;
|
|
8713
|
+
const matchesFallbackOnly = getExistingCandidates(lookup, message, candidateCache).some(
|
|
8714
|
+
(candidate) => !candidate.platform_message_id && candidateFallbackKey(message, candidate) === fallbackKey
|
|
8715
|
+
);
|
|
8716
|
+
if (matchesFallbackOnly) {
|
|
8717
|
+
lookup.consumedFallbackOnlyKeys.add(fallbackKey);
|
|
8718
|
+
lookup.bridgedPlatformMessageIds.add(message.platformMessageId);
|
|
8719
|
+
}
|
|
8720
|
+
return matchesFallbackOnly;
|
|
8721
|
+
}
|
|
8722
|
+
return getExistingCandidates(lookup, message, candidateCache).some(
|
|
8723
|
+
(candidate) => candidateFallbackKey(message, candidate) === fallbackKey
|
|
8724
|
+
);
|
|
8725
|
+
}
|
|
8726
|
+
function isDuplicate(lookup, incomingState, message, existingBatchPlatformIds, candidateCache) {
|
|
8727
|
+
if (isExistingMessageDuplicate(lookup, message, existingBatchPlatformIds, candidateCache)) return true;
|
|
8728
|
+
return registerMessageAndCheckDuplicate(message, incomingState);
|
|
8729
|
+
}
|
|
8923
8730
|
function normalizeImportTimestamp(timestamp) {
|
|
8924
8731
|
const value = typeof timestamp === "string" && timestamp.trim() !== "" ? Number(timestamp) : timestamp;
|
|
8925
8732
|
return typeof value === "number" && value > 0 && Number.isFinite(value) ? value : null;
|
|
@@ -8935,39 +8742,59 @@ async function analyzeIncrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8935
8742
|
} catch {
|
|
8936
8743
|
return { error: "error.session_not_found", newMessageCount: 0, duplicateCount: 0, totalInFile: 0 };
|
|
8937
8744
|
}
|
|
8938
|
-
const { existingPlatformMsgIds, existingKeys, existingFallbackOnlyKeys } = loadExistingDedup(db);
|
|
8939
|
-
db.close();
|
|
8940
8745
|
let totalInFile = 0;
|
|
8941
8746
|
let newMessageCount = 0;
|
|
8942
8747
|
let duplicateCount = 0;
|
|
8943
8748
|
let platform = formatFeature.platform;
|
|
8944
|
-
|
|
8945
|
-
|
|
8946
|
-
|
|
8947
|
-
|
|
8948
|
-
|
|
8949
|
-
|
|
8950
|
-
|
|
8951
|
-
|
|
8952
|
-
|
|
8953
|
-
|
|
8954
|
-
|
|
8955
|
-
|
|
8956
|
-
|
|
8957
|
-
|
|
8958
|
-
|
|
8959
|
-
|
|
8960
|
-
|
|
8961
|
-
|
|
8962
|
-
|
|
8963
|
-
|
|
8964
|
-
|
|
8749
|
+
const mapPlatformId = createPlatformIdMapper(options2?.senderPlatformIdMappings);
|
|
8750
|
+
try {
|
|
8751
|
+
const existingLookup = createExistingMessageLookup(db);
|
|
8752
|
+
const incomingState = createMessageDedupState();
|
|
8753
|
+
await streamParseFile(
|
|
8754
|
+
filePath,
|
|
8755
|
+
{
|
|
8756
|
+
formatOptions: options2?.chatIndex === void 0 ? void 0 : { chatIndex: options2.chatIndex },
|
|
8757
|
+
onMeta: (meta) => {
|
|
8758
|
+
platform = meta.platform;
|
|
8759
|
+
},
|
|
8760
|
+
onMembers: () => {
|
|
8761
|
+
},
|
|
8762
|
+
onProgress: (progress) => {
|
|
8763
|
+
deps.onProgress(progress);
|
|
8764
|
+
},
|
|
8765
|
+
onLog: deps.onParserLog,
|
|
8766
|
+
onMessageBatch: (batch) => {
|
|
8767
|
+
const scopedBatch = batch.map(
|
|
8768
|
+
(message) => applyMessageIdScope(applySenderPlatformIdMapping(message, mapPlatformId), options2?.platformMessageIdScope)
|
|
8769
|
+
);
|
|
8770
|
+
const existingBatchPlatformIds = findExistingPlatformMessageIds(
|
|
8771
|
+
existingLookup,
|
|
8772
|
+
scopedBatch.map((message) => message.platformMessageId)
|
|
8773
|
+
);
|
|
8774
|
+
const candidateCache = /* @__PURE__ */ new Map();
|
|
8775
|
+
for (const msg of scopedBatch) {
|
|
8776
|
+
totalInFile++;
|
|
8777
|
+
const timestamp = normalizeImportTimestamp(msg.timestamp);
|
|
8778
|
+
if (timestamp === null) continue;
|
|
8779
|
+
if (isDuplicate(
|
|
8780
|
+
existingLookup,
|
|
8781
|
+
incomingState,
|
|
8782
|
+
{ ...msg, timestamp },
|
|
8783
|
+
existingBatchPlatformIds,
|
|
8784
|
+
candidateCache
|
|
8785
|
+
)) {
|
|
8786
|
+
duplicateCount++;
|
|
8787
|
+
} else {
|
|
8788
|
+
newMessageCount++;
|
|
8789
|
+
}
|
|
8965
8790
|
}
|
|
8966
8791
|
}
|
|
8967
|
-
}
|
|
8968
|
-
|
|
8969
|
-
|
|
8970
|
-
|
|
8792
|
+
},
|
|
8793
|
+
options2?.formatId
|
|
8794
|
+
);
|
|
8795
|
+
} finally {
|
|
8796
|
+
db.close();
|
|
8797
|
+
}
|
|
8971
8798
|
return { newMessageCount, duplicateCount, totalInFile, platform };
|
|
8972
8799
|
}
|
|
8973
8800
|
async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
@@ -8983,6 +8810,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8983
8810
|
}
|
|
8984
8811
|
const metaUpdateMode = options2?.metaUpdateMode ?? "patch";
|
|
8985
8812
|
const memberUpdateMode = options2?.memberUpdateMode ?? "upsert";
|
|
8813
|
+
const mapPlatformId = createPlatformIdMapper(options2?.senderPlatformIdMappings);
|
|
8986
8814
|
try {
|
|
8987
8815
|
let trackError2 = function(index, reason, detail) {
|
|
8988
8816
|
errorCount++;
|
|
@@ -8992,7 +8820,8 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
8992
8820
|
}
|
|
8993
8821
|
};
|
|
8994
8822
|
var trackError = trackError2;
|
|
8995
|
-
const
|
|
8823
|
+
const existingLookup = createExistingMessageLookup(db);
|
|
8824
|
+
const incomingState = createMessageDedupState();
|
|
8996
8825
|
const memberIdMap = /* @__PURE__ */ new Map();
|
|
8997
8826
|
const existingMembers = db.prepare("SELECT id, platform_id FROM member").all();
|
|
8998
8827
|
for (const m of existingMembers) {
|
|
@@ -9038,8 +8867,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9038
8867
|
const errorReasonCounts = {};
|
|
9039
8868
|
const errorSamples = [];
|
|
9040
8869
|
const MAX_ERROR_SAMPLES = 5;
|
|
9041
|
-
const
|
|
9042
|
-
const newFtsEntries = [];
|
|
8870
|
+
const BATCH_SIZE = 5e3;
|
|
9043
8871
|
await streamParseFile(
|
|
9044
8872
|
filePath,
|
|
9045
8873
|
{
|
|
@@ -9050,7 +8878,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9050
8878
|
meta.name || "",
|
|
9051
8879
|
meta.groupId || "",
|
|
9052
8880
|
meta.groupAvatar || "",
|
|
9053
|
-
meta.ownerId
|
|
8881
|
+
meta.ownerId ? mapPlatformId(meta.ownerId) : "",
|
|
9054
8882
|
Math.floor(Date.now() / 1e3)
|
|
9055
8883
|
);
|
|
9056
8884
|
metaUpdated = true;
|
|
@@ -9058,11 +8886,12 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9058
8886
|
onMembers: (members) => {
|
|
9059
8887
|
if (memberUpdateMode === "none") return;
|
|
9060
8888
|
for (const m of members) {
|
|
9061
|
-
const
|
|
9062
|
-
const
|
|
9063
|
-
const
|
|
8889
|
+
const platformId = mapPlatformId(m.platformId);
|
|
8890
|
+
const existed = memberIdMap.has(platformId);
|
|
8891
|
+
const accountName = normalizeSystemMemberName(formatFeature.id, platformId, m.accountName);
|
|
8892
|
+
const groupNickname = normalizeSystemMemberName(formatFeature.id, platformId, m.groupNickname);
|
|
9064
8893
|
upsertMember.run(
|
|
9065
|
-
|
|
8894
|
+
platformId,
|
|
9066
8895
|
accountName || null,
|
|
9067
8896
|
groupNickname || null,
|
|
9068
8897
|
m.aliases ? JSON.stringify(m.aliases) : "[]",
|
|
@@ -9070,8 +8899,8 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9070
8899
|
m.roles ? JSON.stringify(m.roles) : "[]"
|
|
9071
8900
|
);
|
|
9072
8901
|
if (!existed) {
|
|
9073
|
-
const row = getMemberId.get(
|
|
9074
|
-
if (row) memberIdMap.set(
|
|
8902
|
+
const row = getMemberId.get(platformId);
|
|
8903
|
+
if (row) memberIdMap.set(platformId, row.id);
|
|
9075
8904
|
membersAdded++;
|
|
9076
8905
|
} else {
|
|
9077
8906
|
membersUpdated++;
|
|
@@ -9081,8 +8910,17 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9081
8910
|
onProgress: (progress) => {
|
|
9082
8911
|
deps.onProgress(progress);
|
|
9083
8912
|
},
|
|
8913
|
+
onLog: deps.onParserLog,
|
|
9084
8914
|
onMessageBatch: (batch) => {
|
|
9085
|
-
|
|
8915
|
+
const scopedBatch = batch.map(
|
|
8916
|
+
(message) => applyMessageIdScope(applySenderPlatformIdMapping(message, mapPlatformId), options2?.platformMessageIdScope)
|
|
8917
|
+
);
|
|
8918
|
+
const existingBatchPlatformIds = findExistingPlatformMessageIds(
|
|
8919
|
+
existingLookup,
|
|
8920
|
+
scopedBatch.map((message) => message.platformMessageId)
|
|
8921
|
+
);
|
|
8922
|
+
const candidateCache = /* @__PURE__ */ new Map();
|
|
8923
|
+
for (const msg of scopedBatch) {
|
|
9086
8924
|
processedCount++;
|
|
9087
8925
|
if (!msg.senderPlatformId) {
|
|
9088
8926
|
trackError2(processedCount, "MISSING_SENDER", "sender field is empty");
|
|
@@ -9097,7 +8935,13 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9097
8935
|
trackError2(processedCount, "INVALID_TIMESTAMP", `timestamp value: ${msg.timestamp}`);
|
|
9098
8936
|
continue;
|
|
9099
8937
|
}
|
|
9100
|
-
if (isDuplicate(
|
|
8938
|
+
if (isDuplicate(
|
|
8939
|
+
existingLookup,
|
|
8940
|
+
incomingState,
|
|
8941
|
+
{ ...msg, timestamp },
|
|
8942
|
+
existingBatchPlatformIds,
|
|
8943
|
+
candidateCache
|
|
8944
|
+
)) {
|
|
9101
8945
|
duplicateCount++;
|
|
9102
8946
|
continue;
|
|
9103
8947
|
}
|
|
@@ -9127,7 +8971,7 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9127
8971
|
}
|
|
9128
8972
|
}
|
|
9129
8973
|
if (!memberId) continue;
|
|
9130
|
-
|
|
8974
|
+
insertMessage.run(
|
|
9131
8975
|
memberId,
|
|
9132
8976
|
senderAccountName || null,
|
|
9133
8977
|
senderGroupNickname || null,
|
|
@@ -9137,14 +8981,10 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9137
8981
|
msg.replyToMessageId || null,
|
|
9138
8982
|
msg.platformMessageId || null
|
|
9139
8983
|
);
|
|
9140
|
-
newFtsEntries.push({
|
|
9141
|
-
id: Number(msgResult.lastInsertRowid ?? 0),
|
|
9142
|
-
content: msg.content || null
|
|
9143
|
-
});
|
|
9144
8984
|
if (timestamp < minWrittenTs) minWrittenTs = timestamp;
|
|
9145
8985
|
newMessageCount++;
|
|
9146
8986
|
}
|
|
9147
|
-
if (processedCount %
|
|
8987
|
+
if (processedCount % BATCH_SIZE === 0) {
|
|
9148
8988
|
deps.onProgress({
|
|
9149
8989
|
stage: "saving",
|
|
9150
8990
|
bytesRead: 0,
|
|
@@ -9162,12 +9002,6 @@ async function incrementalImport(sessionId, filePath, deps, options2) {
|
|
|
9162
9002
|
if (!metaUpdated) {
|
|
9163
9003
|
db.prepare("UPDATE meta SET imported_at = ?").run(Math.floor(Date.now() / 1e3));
|
|
9164
9004
|
}
|
|
9165
|
-
if (newFtsEntries.length > 0 && hasFtsTable(db)) {
|
|
9166
|
-
try {
|
|
9167
|
-
insertFtsEntries(db, newFtsEntries);
|
|
9168
|
-
} catch {
|
|
9169
|
-
}
|
|
9170
|
-
}
|
|
9171
9005
|
if (newMessageCount > 0) {
|
|
9172
9006
|
try {
|
|
9173
9007
|
const sessionGapThreshold = getSessionIndexStats(db).gapThreshold;
|
|
@@ -9241,23 +9075,112 @@ function isValidImportSessionId(sessionId) {
|
|
|
9241
9075
|
}
|
|
9242
9076
|
|
|
9243
9077
|
// ../../packages/node-runtime/src/import/auto-import-matcher.ts
|
|
9244
|
-
import { createHash
|
|
9078
|
+
import { createHash } from "crypto";
|
|
9245
9079
|
var MATCH_WINDOW_SIZE = 5;
|
|
9246
|
-
function buildPrivateIdentity(ownerId, memberIds) {
|
|
9247
|
-
|
|
9248
|
-
const
|
|
9249
|
-
|
|
9080
|
+
function buildPrivateIdentity(platform, ownerId, memberIds) {
|
|
9081
|
+
const sortedIds = [...new Set(memberIds)].filter((id) => id && id.toLowerCase() !== "system").sort();
|
|
9082
|
+
const validOwnerId = ownerId && ownerId.toLowerCase() !== "system" ? ownerId : null;
|
|
9083
|
+
if (validOwnerId) {
|
|
9084
|
+
return sortedIds.length > 0 ? `owner\0${validOwnerId}\0members\0${sortedIds.join("\0")}` : null;
|
|
9085
|
+
}
|
|
9086
|
+
return platform === "qq" && sortedIds.length >= 2 ? `members\0${sortedIds.join("\0")}` : null;
|
|
9250
9087
|
}
|
|
9251
9088
|
function isBusinessMessage(type, content) {
|
|
9252
9089
|
return type !== 80 /* SYSTEM */ && type !== 81 /* RECALL */ && Boolean(content?.trim());
|
|
9253
9090
|
}
|
|
9254
|
-
function addMessageWindow(recentKeys, windows, message) {
|
|
9091
|
+
function addMessageWindow(recentKeys, windows, recentPlatformIdKeys, platformIdWindows, platformIdSenders, recentExactPlatformIdKeys, exactPlatformIdWindows, exactPlatformIdSenders, message) {
|
|
9255
9092
|
if (!message.senderPlatformId || !isBusinessMessage(message.type, message.content)) return;
|
|
9256
9093
|
const timestamp = normalizeImportTimestamp(message.timestamp);
|
|
9257
9094
|
if (timestamp === null) return;
|
|
9258
9095
|
recentKeys.push(generateMessageKey(timestamp, message.senderPlatformId, message.content));
|
|
9259
9096
|
if (recentKeys.length > MATCH_WINDOW_SIZE) recentKeys.shift();
|
|
9260
9097
|
if (recentKeys.length === MATCH_WINDOW_SIZE) windows.add(recentKeys.join(""));
|
|
9098
|
+
const platformIdKey = generatePlatformMessageKey(message, timestamp);
|
|
9099
|
+
const exactPlatformIdKey = generatePlatformMessageKey(message, timestamp, false);
|
|
9100
|
+
if (!platformIdKey) {
|
|
9101
|
+
recentPlatformIdKeys.length = 0;
|
|
9102
|
+
recentExactPlatformIdKeys.length = 0;
|
|
9103
|
+
return;
|
|
9104
|
+
}
|
|
9105
|
+
recordPlatformMessageSender(platformIdSenders, platformIdKey, message.senderPlatformId);
|
|
9106
|
+
recordPlatformMessageSender(exactPlatformIdSenders, exactPlatformIdKey ?? platformIdKey, message.senderPlatformId);
|
|
9107
|
+
recentPlatformIdKeys.push(platformIdKey);
|
|
9108
|
+
if (recentPlatformIdKeys.length > MATCH_WINDOW_SIZE) recentPlatformIdKeys.shift();
|
|
9109
|
+
if (recentPlatformIdKeys.length === MATCH_WINDOW_SIZE) platformIdWindows.add(recentPlatformIdKeys.join(""));
|
|
9110
|
+
recentExactPlatformIdKeys.push(exactPlatformIdKey ?? platformIdKey);
|
|
9111
|
+
if (recentExactPlatformIdKeys.length > MATCH_WINDOW_SIZE) recentExactPlatformIdKeys.shift();
|
|
9112
|
+
if (recentExactPlatformIdKeys.length === MATCH_WINDOW_SIZE) {
|
|
9113
|
+
exactPlatformIdWindows.add(recentExactPlatformIdKeys.join(""));
|
|
9114
|
+
}
|
|
9115
|
+
}
|
|
9116
|
+
function recordPlatformMessageSender(senders, messageKey, senderPlatformId) {
|
|
9117
|
+
if (!senders.has(messageKey)) {
|
|
9118
|
+
senders.set(messageKey, senderPlatformId);
|
|
9119
|
+
} else if (senders.get(messageKey) !== senderPlatformId) {
|
|
9120
|
+
senders.set(messageKey, null);
|
|
9121
|
+
}
|
|
9122
|
+
}
|
|
9123
|
+
function isUsablePlatformId(id) {
|
|
9124
|
+
return Boolean(id && id.toLowerCase() !== "system");
|
|
9125
|
+
}
|
|
9126
|
+
function deriveSenderPlatformIdMappings(sourceMemberIds, targetMemberIds, sourceSenders, targetSenders, sourceOwnerId, targetOwnerId, allowSingleRemainingPair = false) {
|
|
9127
|
+
const sourceIds = new Set([...sourceMemberIds].filter(isUsablePlatformId));
|
|
9128
|
+
const targetIds = new Set([...targetMemberIds].filter(isUsablePlatformId));
|
|
9129
|
+
if (isUsablePlatformId(sourceOwnerId)) sourceIds.add(sourceOwnerId);
|
|
9130
|
+
if (isUsablePlatformId(targetOwnerId)) targetIds.add(targetOwnerId);
|
|
9131
|
+
const forward = /* @__PURE__ */ new Map();
|
|
9132
|
+
const reverse = /* @__PURE__ */ new Map();
|
|
9133
|
+
let hasDrift = false;
|
|
9134
|
+
const addPair = (sourceId, targetId) => {
|
|
9135
|
+
if (!sourceIds.has(sourceId) || !targetIds.has(targetId)) return false;
|
|
9136
|
+
const existingTarget = forward.get(sourceId);
|
|
9137
|
+
const existingSource = reverse.get(targetId);
|
|
9138
|
+
if (existingTarget && existingTarget !== targetId || existingSource && existingSource !== sourceId) {
|
|
9139
|
+
return false;
|
|
9140
|
+
}
|
|
9141
|
+
forward.set(sourceId, targetId);
|
|
9142
|
+
reverse.set(targetId, sourceId);
|
|
9143
|
+
hasDrift ||= sourceId !== targetId;
|
|
9144
|
+
return true;
|
|
9145
|
+
};
|
|
9146
|
+
if (isUsablePlatformId(sourceOwnerId) && isUsablePlatformId(targetOwnerId) && !addPair(sourceOwnerId, targetOwnerId)) {
|
|
9147
|
+
return null;
|
|
9148
|
+
}
|
|
9149
|
+
for (let index = 0; index < sourceSenders.length; index++) {
|
|
9150
|
+
if (!addPair(sourceSenders[index], targetSenders[index])) return null;
|
|
9151
|
+
}
|
|
9152
|
+
if (!hasDrift) return [];
|
|
9153
|
+
for (const sourceId of sourceIds) {
|
|
9154
|
+
if (!forward.has(sourceId) && targetIds.has(sourceId) && !reverse.has(sourceId)) {
|
|
9155
|
+
addPair(sourceId, sourceId);
|
|
9156
|
+
}
|
|
9157
|
+
}
|
|
9158
|
+
const remainingSourceIds = [...sourceIds].filter((id) => !forward.has(id));
|
|
9159
|
+
const remainingTargetIds = [...targetIds].filter((id) => !reverse.has(id));
|
|
9160
|
+
if (allowSingleRemainingPair && remainingSourceIds.length === 1 && remainingTargetIds.length === 1) {
|
|
9161
|
+
addPair(remainingSourceIds[0], remainingTargetIds[0]);
|
|
9162
|
+
}
|
|
9163
|
+
if ([...sourceIds].some((id) => !forward.has(id))) return null;
|
|
9164
|
+
return [...forward].filter(([sourceId, targetId]) => sourceId !== targetId).map(([sourceId, targetId]) => ({ sourceId, targetId }));
|
|
9165
|
+
}
|
|
9166
|
+
function getCommonPlatformMessageIdScope(ids) {
|
|
9167
|
+
let commonScope;
|
|
9168
|
+
for (const id of ids) {
|
|
9169
|
+
if (!id) return void 0;
|
|
9170
|
+
const { scope } = parsePlatformMessageId(id);
|
|
9171
|
+
if (!scope || commonScope && scope !== commonScope) return void 0;
|
|
9172
|
+
commonScope = scope;
|
|
9173
|
+
}
|
|
9174
|
+
return commonScope;
|
|
9175
|
+
}
|
|
9176
|
+
function generatePlatformMessageKey(message, timestamp, normalizeScope = true) {
|
|
9177
|
+
if (!message.platformMessageId) return null;
|
|
9178
|
+
const parsedId = parsePlatformMessageId(message.platformMessageId);
|
|
9179
|
+
return generateMessageKey(
|
|
9180
|
+
timestamp,
|
|
9181
|
+
normalizeScope ? parsedId.rawId : message.platformMessageId,
|
|
9182
|
+
JSON.stringify([message.type, message.content || null])
|
|
9183
|
+
);
|
|
9261
9184
|
}
|
|
9262
9185
|
async function resolveAutoImportTarget(filePath, deps, formatOptions) {
|
|
9263
9186
|
return (await resolveAutoImportTargetPlan(filePath, deps, formatOptions)).decision;
|
|
@@ -9266,7 +9189,14 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9266
9189
|
let sourceMeta = null;
|
|
9267
9190
|
const sourceMemberIds = /* @__PURE__ */ new Set();
|
|
9268
9191
|
const sourceWindows = /* @__PURE__ */ new Set();
|
|
9192
|
+
const sourcePlatformIdWindows = /* @__PURE__ */ new Set();
|
|
9193
|
+
const sourcePlatformIdSenders = /* @__PURE__ */ new Map();
|
|
9194
|
+
const sourceExactPlatformIdWindows = /* @__PURE__ */ new Set();
|
|
9195
|
+
const sourceExactPlatformIdSenders = /* @__PURE__ */ new Map();
|
|
9269
9196
|
const recentSourceKeys = [];
|
|
9197
|
+
const recentSourcePlatformIdKeys = [];
|
|
9198
|
+
const recentSourceExactPlatformIdKeys = [];
|
|
9199
|
+
let sourceHasScopedPlatformMessageIds = false;
|
|
9270
9200
|
const { formatId, ...parserOptions } = formatOptions ?? {};
|
|
9271
9201
|
await streamParseFile(
|
|
9272
9202
|
filePath,
|
|
@@ -9282,8 +9212,21 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9282
9212
|
},
|
|
9283
9213
|
onMessageBatch: (messages) => {
|
|
9284
9214
|
for (const message of messages) {
|
|
9215
|
+
sourceHasScopedPlatformMessageIds ||= Boolean(
|
|
9216
|
+
message.platformMessageId && parsePlatformMessageId(message.platformMessageId).scope || message.replyToMessageId && parsePlatformMessageId(message.replyToMessageId).scope
|
|
9217
|
+
);
|
|
9285
9218
|
sourceMemberIds.add(message.senderPlatformId);
|
|
9286
|
-
addMessageWindow(
|
|
9219
|
+
addMessageWindow(
|
|
9220
|
+
recentSourceKeys,
|
|
9221
|
+
sourceWindows,
|
|
9222
|
+
recentSourcePlatformIdKeys,
|
|
9223
|
+
sourcePlatformIdWindows,
|
|
9224
|
+
sourcePlatformIdSenders,
|
|
9225
|
+
recentSourceExactPlatformIdKeys,
|
|
9226
|
+
sourceExactPlatformIdWindows,
|
|
9227
|
+
sourceExactPlatformIdSenders,
|
|
9228
|
+
message
|
|
9229
|
+
);
|
|
9287
9230
|
}
|
|
9288
9231
|
}
|
|
9289
9232
|
},
|
|
@@ -9291,10 +9234,12 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9291
9234
|
);
|
|
9292
9235
|
if (!sourceMeta) throw new Error("Import source did not provide metadata");
|
|
9293
9236
|
const meta = sourceMeta;
|
|
9294
|
-
const
|
|
9237
|
+
const sourceHasValidOwner = Boolean(meta.ownerId && meta.ownerId.toLowerCase() !== "system");
|
|
9238
|
+
const matchQqPrivateByMembers = meta.platform === "qq" && meta.type === "private" && !sourceHasValidOwner;
|
|
9239
|
+
const privateIdentity = meta.type === "private" ? buildPrivateIdentity(meta.platform, meta.ownerId, sourceMemberIds) : null;
|
|
9295
9240
|
const hasStableIdentity = Boolean(meta.groupId || privateIdentity);
|
|
9296
9241
|
const stableIdentity = meta.groupId ? `group\0${meta.groupId}` : privateIdentity ? `private\0${privateIdentity}` : null;
|
|
9297
|
-
const stableConcurrencyKey = stableIdentity ? `source:${
|
|
9242
|
+
const stableConcurrencyKey = stableIdentity ? `source:${createHash("sha256").update(`${meta.platform}\0${meta.type}\0${stableIdentity}`).digest("hex")}` : null;
|
|
9298
9243
|
const stableMatches = [];
|
|
9299
9244
|
const trailingMatches = [];
|
|
9300
9245
|
for (const sessionId of deps.listSessionIds()) {
|
|
@@ -9305,17 +9250,18 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9305
9250
|
const candidate = getSessionMeta(db);
|
|
9306
9251
|
if (candidate?.platform !== meta.platform || candidate.type !== meta.type) continue;
|
|
9307
9252
|
const groupMatches = Boolean(meta.groupId && candidate.groupId === meta.groupId);
|
|
9308
|
-
const memberRows = privateIdentity ? db.prepare("SELECT platform_id FROM member WHERE LOWER(platform_id) != 'system'").all() : [];
|
|
9253
|
+
const memberRows = privateIdentity || sourcePlatformIdWindows.size > 0 ? db.prepare("SELECT platform_id FROM member WHERE LOWER(platform_id) != 'system'").all() : [];
|
|
9309
9254
|
const candidatePrivateIdentity = privateIdentity ? buildPrivateIdentity(
|
|
9310
|
-
candidate.
|
|
9255
|
+
candidate.platform,
|
|
9256
|
+
matchQqPrivateByMembers ? null : candidate.ownerId,
|
|
9311
9257
|
memberRows.map((row) => row.platform_id)
|
|
9312
9258
|
) : null;
|
|
9313
9259
|
if (groupMatches || privateIdentity && candidatePrivateIdentity === privateIdentity) {
|
|
9314
9260
|
stableMatches.push(sessionId);
|
|
9315
9261
|
}
|
|
9316
|
-
if (sourceWindows.size > 0) {
|
|
9262
|
+
if (sourceWindows.size > 0 || sourcePlatformIdWindows.size > 0) {
|
|
9317
9263
|
const rows = db.prepare(
|
|
9318
|
-
`SELECT msg.ts, member.platform_id, msg.type, msg.content
|
|
9264
|
+
`SELECT msg.ts, member.platform_id, msg.type, msg.content, msg.platform_message_id
|
|
9319
9265
|
FROM message msg
|
|
9320
9266
|
JOIN member ON member.id = msg.sender_id
|
|
9321
9267
|
WHERE msg.type NOT IN (?, ?)
|
|
@@ -9324,8 +9270,54 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9324
9270
|
LIMIT ?`
|
|
9325
9271
|
).all(80 /* SYSTEM */, 81 /* RECALL */, MATCH_WINDOW_SIZE);
|
|
9326
9272
|
if (rows.length === MATCH_WINDOW_SIZE) {
|
|
9327
|
-
const
|
|
9328
|
-
|
|
9273
|
+
const orderedRows = rows.reverse();
|
|
9274
|
+
const signature = orderedRows.map((row) => generateMessageKey(row.ts, row.platform_id, row.content)).join("");
|
|
9275
|
+
const platformIdKeys = orderedRows.map(
|
|
9276
|
+
(row) => generatePlatformMessageKey(
|
|
9277
|
+
{
|
|
9278
|
+
platformMessageId: row.platform_message_id || void 0,
|
|
9279
|
+
type: row.type,
|
|
9280
|
+
content: row.content
|
|
9281
|
+
},
|
|
9282
|
+
row.ts
|
|
9283
|
+
)
|
|
9284
|
+
);
|
|
9285
|
+
const platformIdSignature = platformIdKeys.every((key) => key !== null) ? platformIdKeys.join("") : null;
|
|
9286
|
+
const exactPlatformIdKeys = orderedRows.map(
|
|
9287
|
+
(row) => generatePlatformMessageKey(
|
|
9288
|
+
{
|
|
9289
|
+
platformMessageId: row.platform_message_id || void 0,
|
|
9290
|
+
type: row.type,
|
|
9291
|
+
content: row.content
|
|
9292
|
+
},
|
|
9293
|
+
row.ts,
|
|
9294
|
+
false
|
|
9295
|
+
)
|
|
9296
|
+
);
|
|
9297
|
+
const exactPlatformIdSignature = exactPlatformIdKeys.every((key) => key !== null) ? exactPlatformIdKeys.join("") : null;
|
|
9298
|
+
const platformIdsMatch = sourceHasScopedPlatformMessageIds ? exactPlatformIdSignature !== null && sourceExactPlatformIdWindows.has(exactPlatformIdSignature) : platformIdSignature !== null && sourcePlatformIdWindows.has(platformIdSignature);
|
|
9299
|
+
const contentMatches = !sourceHasScopedPlatformMessageIds && sourceWindows.has(signature);
|
|
9300
|
+
const matchedPlatformIdKeys = sourceHasScopedPlatformMessageIds ? exactPlatformIdKeys : platformIdKeys;
|
|
9301
|
+
const sourceSenderLookup = sourceHasScopedPlatformMessageIds ? sourceExactPlatformIdSenders : sourcePlatformIdSenders;
|
|
9302
|
+
const sourceSenders = platformIdsMatch ? matchedPlatformIdKeys.map((key) => key ? sourceSenderLookup.get(key) : void 0) : [];
|
|
9303
|
+
const senderPlatformIdMappings = platformIdsMatch && sourceSenders.every((sender) => Boolean(sender)) ? deriveSenderPlatformIdMappings(
|
|
9304
|
+
sourceMemberIds,
|
|
9305
|
+
memberRows.map((row) => row.platform_id),
|
|
9306
|
+
sourceSenders,
|
|
9307
|
+
orderedRows.map((row) => row.platform_id),
|
|
9308
|
+
meta.ownerId,
|
|
9309
|
+
candidate.ownerId,
|
|
9310
|
+
meta.type === "private"
|
|
9311
|
+
) : null;
|
|
9312
|
+
const safePlatformIdsMatch = platformIdsMatch && senderPlatformIdMappings !== null;
|
|
9313
|
+
if (contentMatches || safePlatformIdsMatch) {
|
|
9314
|
+
const platformMessageIdScope = sourceHasScopedPlatformMessageIds ? void 0 : getCommonPlatformMessageIdScope(orderedRows.map((row) => row.platform_message_id));
|
|
9315
|
+
trailingMatches.push({
|
|
9316
|
+
sessionId,
|
|
9317
|
+
...platformMessageIdScope ? { platformMessageIdScope } : {},
|
|
9318
|
+
...safePlatformIdsMatch && senderPlatformIdMappings.length > 0 ? { senderPlatformIdMappings } : {}
|
|
9319
|
+
});
|
|
9320
|
+
}
|
|
9329
9321
|
}
|
|
9330
9322
|
}
|
|
9331
9323
|
} catch (error) {
|
|
@@ -9339,34 +9331,62 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9339
9331
|
}
|
|
9340
9332
|
if (hasStableIdentity) {
|
|
9341
9333
|
if (meta.sourceSessionId && stableMatches.includes(meta.sourceSessionId)) {
|
|
9334
|
+
const trailingMatch = trailingMatches.find((match) => match.sessionId === meta.sourceSessionId);
|
|
9342
9335
|
return {
|
|
9343
|
-
decision: {
|
|
9336
|
+
decision: {
|
|
9337
|
+
action: "incremental",
|
|
9338
|
+
sessionId: meta.sourceSessionId,
|
|
9339
|
+
matchedBy: "source-session-id",
|
|
9340
|
+
...trailingMatch?.platformMessageIdScope ? { platformMessageIdScope: trailingMatch.platformMessageIdScope } : {},
|
|
9341
|
+
...trailingMatch?.senderPlatformIdMappings ? { senderPlatformIdMappings: trailingMatch.senderPlatformIdMappings } : {}
|
|
9342
|
+
},
|
|
9344
9343
|
concurrencyKey: `session:${meta.sourceSessionId}`,
|
|
9345
9344
|
exclusive: false,
|
|
9346
9345
|
coalesceCreate: false
|
|
9347
9346
|
};
|
|
9348
9347
|
}
|
|
9349
9348
|
if (stableMatches.length === 1) {
|
|
9349
|
+
const trailingMatch = trailingMatches.find((match) => match.sessionId === stableMatches[0]);
|
|
9350
9350
|
return {
|
|
9351
|
-
decision: {
|
|
9351
|
+
decision: {
|
|
9352
|
+
action: "incremental",
|
|
9353
|
+
sessionId: stableMatches[0],
|
|
9354
|
+
matchedBy: "stable-id",
|
|
9355
|
+
...trailingMatch?.platformMessageIdScope ? { platformMessageIdScope: trailingMatch.platformMessageIdScope } : {},
|
|
9356
|
+
...trailingMatch?.senderPlatformIdMappings ? { senderPlatformIdMappings: trailingMatch.senderPlatformIdMappings } : {}
|
|
9357
|
+
},
|
|
9352
9358
|
concurrencyKey: `session:${stableMatches[0]}`,
|
|
9353
9359
|
exclusive: false,
|
|
9354
9360
|
coalesceCreate: false
|
|
9355
9361
|
};
|
|
9356
9362
|
}
|
|
9357
9363
|
}
|
|
9358
|
-
|
|
9364
|
+
const sourceSessionTrailingMatch = meta.sourceSessionId ? trailingMatches.find((match) => match.sessionId === meta.sourceSessionId) : void 0;
|
|
9365
|
+
if (meta.sourceSessionId && sourceSessionTrailingMatch) {
|
|
9359
9366
|
return {
|
|
9360
|
-
decision: {
|
|
9367
|
+
decision: {
|
|
9368
|
+
action: "incremental",
|
|
9369
|
+
sessionId: meta.sourceSessionId,
|
|
9370
|
+
matchedBy: "source-session-id",
|
|
9371
|
+
...sourceSessionTrailingMatch.platformMessageIdScope ? { platformMessageIdScope: sourceSessionTrailingMatch.platformMessageIdScope } : {},
|
|
9372
|
+
...sourceSessionTrailingMatch.senderPlatformIdMappings ? { senderPlatformIdMappings: sourceSessionTrailingMatch.senderPlatformIdMappings } : {}
|
|
9373
|
+
},
|
|
9361
9374
|
concurrencyKey: `session:${meta.sourceSessionId}`,
|
|
9362
9375
|
exclusive: false,
|
|
9363
9376
|
coalesceCreate: false
|
|
9364
9377
|
};
|
|
9365
9378
|
}
|
|
9366
9379
|
if (trailingMatches.length === 1) {
|
|
9380
|
+
const [trailingMatch] = trailingMatches;
|
|
9367
9381
|
return {
|
|
9368
|
-
decision: {
|
|
9369
|
-
|
|
9382
|
+
decision: {
|
|
9383
|
+
action: "incremental",
|
|
9384
|
+
sessionId: trailingMatch.sessionId,
|
|
9385
|
+
matchedBy: "trailing-messages",
|
|
9386
|
+
...trailingMatch.platformMessageIdScope ? { platformMessageIdScope: trailingMatch.platformMessageIdScope } : {},
|
|
9387
|
+
...trailingMatch.senderPlatformIdMappings ? { senderPlatformIdMappings: trailingMatch.senderPlatformIdMappings } : {}
|
|
9388
|
+
},
|
|
9389
|
+
concurrencyKey: `session:${trailingMatch.sessionId}`,
|
|
9370
9390
|
exclusive: false,
|
|
9371
9391
|
coalesceCreate: false
|
|
9372
9392
|
};
|
|
@@ -9393,7 +9413,13 @@ async function resolveAutoImportTargetPlan(filePath, deps, formatOptions) {
|
|
|
9393
9413
|
async function planAutoImport(filePath, deps, options2) {
|
|
9394
9414
|
if (options2.resolvedDecision) {
|
|
9395
9415
|
const decision2 = options2.resolvedDecision;
|
|
9396
|
-
return decision2.action === "incremental" ? {
|
|
9416
|
+
return decision2.action === "incremental" ? {
|
|
9417
|
+
action: "incremental",
|
|
9418
|
+
sessionId: decision2.sessionId,
|
|
9419
|
+
matchedBy: decision2.matchedBy,
|
|
9420
|
+
...decision2.platformMessageIdScope ? { platformMessageIdScope: decision2.platformMessageIdScope } : {},
|
|
9421
|
+
...decision2.senderPlatformIdMappings ? { senderPlatformIdMappings: decision2.senderPlatformIdMappings } : {}
|
|
9422
|
+
} : { action: "create", reason: decision2.reason };
|
|
9397
9423
|
}
|
|
9398
9424
|
if (options2.explicitSessionId) {
|
|
9399
9425
|
if (!isValidImportSessionId(options2.explicitSessionId)) {
|
|
@@ -9406,7 +9432,13 @@ async function planAutoImport(filePath, deps, options2) {
|
|
|
9406
9432
|
deps,
|
|
9407
9433
|
options2.formatOptions
|
|
9408
9434
|
);
|
|
9409
|
-
return decision.action === "incremental" ? {
|
|
9435
|
+
return decision.action === "incremental" ? {
|
|
9436
|
+
action: "incremental",
|
|
9437
|
+
sessionId: decision.sessionId,
|
|
9438
|
+
matchedBy: decision.matchedBy,
|
|
9439
|
+
...decision.platformMessageIdScope ? { platformMessageIdScope: decision.platformMessageIdScope } : {},
|
|
9440
|
+
...decision.senderPlatformIdMappings ? { senderPlatformIdMappings: decision.senderPlatformIdMappings } : {}
|
|
9441
|
+
} : { action: "create", reason: decision.reason };
|
|
9410
9442
|
}
|
|
9411
9443
|
function mapCreateResult(result, createReason) {
|
|
9412
9444
|
if (!result.success || !result.sessionId) {
|
|
@@ -9462,7 +9494,10 @@ async function autoImportFile(filePath, deps, options2 = {}) {
|
|
|
9462
9494
|
if (plan.action === "incremental") {
|
|
9463
9495
|
const result2 = mapIncrementalResult(
|
|
9464
9496
|
plan.sessionId,
|
|
9465
|
-
await deps.appendSession(plan.sessionId, filePath, options2.formatOptions, deps.onProgress
|
|
9497
|
+
await deps.appendSession(plan.sessionId, filePath, options2.formatOptions, deps.onProgress, {
|
|
9498
|
+
platformMessageIdScope: plan.platformMessageIdScope,
|
|
9499
|
+
senderPlatformIdMappings: plan.senderPlatformIdMappings
|
|
9500
|
+
}),
|
|
9466
9501
|
plan.matchedBy
|
|
9467
9502
|
);
|
|
9468
9503
|
appLogger.info("import", "Incremental import completed", {
|
|
@@ -9501,7 +9536,10 @@ async function analyzeAutoImportFile(filePath, deps, options2 = {}) {
|
|
|
9501
9536
|
try {
|
|
9502
9537
|
const plan = await planAutoImport(filePath, deps, options2);
|
|
9503
9538
|
if (plan.action === "incremental") {
|
|
9504
|
-
const analysis2 = await deps.analyzeAppendSession(plan.sessionId, filePath, options2.formatOptions
|
|
9539
|
+
const analysis2 = await deps.analyzeAppendSession(plan.sessionId, filePath, options2.formatOptions, {
|
|
9540
|
+
platformMessageIdScope: plan.platformMessageIdScope,
|
|
9541
|
+
senderPlatformIdMappings: plan.senderPlatformIdMappings
|
|
9542
|
+
});
|
|
9505
9543
|
if (analysis2.error) return { success: false, error: analysis2.error };
|
|
9506
9544
|
const result2 = {
|
|
9507
9545
|
success: true,
|
|
@@ -9761,8 +9799,8 @@ async function autoImportBatch(items, deps, options2 = {}) {
|
|
|
9761
9799
|
}
|
|
9762
9800
|
|
|
9763
9801
|
// ../../packages/node-runtime/src/import/import-lock.ts
|
|
9764
|
-
import * as
|
|
9765
|
-
import * as
|
|
9802
|
+
import * as fs21 from "fs";
|
|
9803
|
+
import * as path13 from "path";
|
|
9766
9804
|
import { randomUUID } from "crypto";
|
|
9767
9805
|
var IMPORT_LOCK_FILENAME = ".chatlab-import.lock";
|
|
9768
9806
|
var IMPORT_IN_PROGRESS_ERROR_KEY = "error.import_in_progress";
|
|
@@ -9776,7 +9814,7 @@ var ImportInProgressError = class extends Error {
|
|
|
9776
9814
|
};
|
|
9777
9815
|
function parseOwner(lockPath) {
|
|
9778
9816
|
try {
|
|
9779
|
-
const parsed = JSON.parse(
|
|
9817
|
+
const parsed = JSON.parse(fs21.readFileSync(lockPath, "utf-8"));
|
|
9780
9818
|
if (typeof parsed.pid !== "number" || !Number.isInteger(parsed.pid) || parsed.pid <= 0 || typeof parsed.token !== "string" || parsed.token.length === 0 || typeof parsed.startedAt !== "number") {
|
|
9781
9819
|
return null;
|
|
9782
9820
|
}
|
|
@@ -9799,13 +9837,13 @@ function removeStaleLock(lockPath) {
|
|
|
9799
9837
|
if (isProcessRunning(owner.pid)) return false;
|
|
9800
9838
|
} else {
|
|
9801
9839
|
try {
|
|
9802
|
-
if (Date.now() -
|
|
9840
|
+
if (Date.now() - fs21.statSync(lockPath).mtimeMs < 5e3) return false;
|
|
9803
9841
|
} catch {
|
|
9804
9842
|
return true;
|
|
9805
9843
|
}
|
|
9806
9844
|
}
|
|
9807
9845
|
try {
|
|
9808
|
-
|
|
9846
|
+
fs21.unlinkSync(lockPath);
|
|
9809
9847
|
appLogger.warn("import-lock", "Removed stale import lock", { ownerPid: owner?.pid });
|
|
9810
9848
|
return true;
|
|
9811
9849
|
} catch {
|
|
@@ -9814,7 +9852,7 @@ function removeStaleLock(lockPath) {
|
|
|
9814
9852
|
}
|
|
9815
9853
|
function releaseLock(lockPath, fd, owner) {
|
|
9816
9854
|
try {
|
|
9817
|
-
|
|
9855
|
+
fs21.closeSync(fd);
|
|
9818
9856
|
} catch (error) {
|
|
9819
9857
|
appLogger.error("import-lock", "Failed to close import lock", error);
|
|
9820
9858
|
}
|
|
@@ -9823,19 +9861,19 @@ function releaseLock(lockPath, fd, owner) {
|
|
|
9823
9861
|
appLogger.warn("import-lock", "Import lock ownership changed before release");
|
|
9824
9862
|
return;
|
|
9825
9863
|
}
|
|
9826
|
-
|
|
9864
|
+
fs21.unlinkSync(lockPath);
|
|
9827
9865
|
appLogger.info("import-lock", "Import lock released");
|
|
9828
9866
|
} catch (error) {
|
|
9829
9867
|
appLogger.error("import-lock", "Failed to release import lock", error);
|
|
9830
9868
|
}
|
|
9831
9869
|
}
|
|
9832
9870
|
async function withDataDirImportLock(userDataDir, task) {
|
|
9833
|
-
|
|
9834
|
-
const lockPath =
|
|
9871
|
+
fs21.mkdirSync(userDataDir, { recursive: true });
|
|
9872
|
+
const lockPath = path13.join(userDataDir, IMPORT_LOCK_FILENAME);
|
|
9835
9873
|
let fd = null;
|
|
9836
9874
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
9837
9875
|
try {
|
|
9838
|
-
fd =
|
|
9876
|
+
fd = fs21.openSync(lockPath, "wx");
|
|
9839
9877
|
break;
|
|
9840
9878
|
} catch (error) {
|
|
9841
9879
|
const code = error.code;
|
|
@@ -9854,8 +9892,8 @@ async function withDataDirImportLock(userDataDir, task) {
|
|
|
9854
9892
|
startedAt: Date.now()
|
|
9855
9893
|
};
|
|
9856
9894
|
try {
|
|
9857
|
-
|
|
9858
|
-
|
|
9895
|
+
fs21.writeFileSync(fd, JSON.stringify(owner), "utf-8");
|
|
9896
|
+
fs21.fsyncSync(fd);
|
|
9859
9897
|
appLogger.info("import-lock", "Import lock acquired");
|
|
9860
9898
|
return await task();
|
|
9861
9899
|
} finally {
|
|
@@ -9875,8 +9913,8 @@ var ArchiveImportError = class extends Error {
|
|
|
9875
9913
|
|
|
9876
9914
|
// ../../packages/node-runtime/src/import/archive/archive-reader.ts
|
|
9877
9915
|
var import_yauzl = __toESM(require_yauzl(), 1);
|
|
9878
|
-
import * as
|
|
9879
|
-
import * as
|
|
9916
|
+
import * as fs22 from "fs";
|
|
9917
|
+
import * as path14 from "path";
|
|
9880
9918
|
import { pipeline } from "stream/promises";
|
|
9881
9919
|
var DEFAULT_MAX_ENTRIES = 1e5;
|
|
9882
9920
|
var DEFAULT_MAX_ENTRY_BYTES = 20 * 1024 * 1024 * 1024;
|
|
@@ -9995,8 +10033,8 @@ var ZipArchiveReader = class {
|
|
|
9995
10033
|
await this.visitEntries(async (entry, openStream) => {
|
|
9996
10034
|
const destination = targets.get(entry.name);
|
|
9997
10035
|
if (!destination || entry.isDirectory) return;
|
|
9998
|
-
|
|
9999
|
-
await pipeline(await openStream(),
|
|
10036
|
+
fs22.mkdirSync(path14.dirname(destination), { recursive: true });
|
|
10037
|
+
await pipeline(await openStream(), fs22.createWriteStream(destination, { flags: "wx" }));
|
|
10000
10038
|
remaining.delete(entry.name);
|
|
10001
10039
|
});
|
|
10002
10040
|
if (remaining.size > 0) {
|
|
@@ -10013,8 +10051,8 @@ var import_stream_chain8 = __toESM(require_stream_chain(), 1);
|
|
|
10013
10051
|
var import_stream_json9 = __toESM(require_stream_json(), 1);
|
|
10014
10052
|
var import_Pick8 = __toESM(require_Pick(), 1);
|
|
10015
10053
|
var import_StreamValues8 = __toESM(require_StreamValues(), 1);
|
|
10016
|
-
import * as
|
|
10017
|
-
import * as
|
|
10054
|
+
import * as fs23 from "fs";
|
|
10055
|
+
import * as path15 from "path";
|
|
10018
10056
|
var { chain: chain8 } = import_stream_chain8.default;
|
|
10019
10057
|
var { parser: parser10 } = import_stream_json9.default;
|
|
10020
10058
|
var { pick: pick8 } = import_Pick8.default;
|
|
@@ -10064,7 +10102,7 @@ function deriveConversationName(draft, ownerEmail) {
|
|
|
10064
10102
|
return other?.name?.trim() || other?.email?.trim() || members[0]?.name?.trim() || draft.chatId;
|
|
10065
10103
|
}
|
|
10066
10104
|
var GoogleChatTakeoutResolver = class {
|
|
10067
|
-
id =
|
|
10105
|
+
id = PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE;
|
|
10068
10106
|
platform = "google-chat";
|
|
10069
10107
|
detect(entries) {
|
|
10070
10108
|
const products = /* @__PURE__ */ new Set();
|
|
@@ -10143,16 +10181,16 @@ var GoogleChatTakeoutResolver = class {
|
|
|
10143
10181
|
`Google Chat conversation is missing required files: ${chat.chatId}`
|
|
10144
10182
|
);
|
|
10145
10183
|
}
|
|
10146
|
-
|
|
10184
|
+
fs23.mkdirSync(targetDir, { recursive: true });
|
|
10147
10185
|
await reader.pipeEntries(
|
|
10148
10186
|
/* @__PURE__ */ new Map([
|
|
10149
|
-
[userInfoEntry,
|
|
10150
|
-
[groupInfoEntry,
|
|
10151
|
-
[messagesEntry,
|
|
10187
|
+
[userInfoEntry, path15.join(targetDir, "user_info.json")],
|
|
10188
|
+
[groupInfoEntry, path15.join(targetDir, "group_info.json")],
|
|
10189
|
+
[messagesEntry, path15.join(targetDir, "messages.json")]
|
|
10152
10190
|
])
|
|
10153
10191
|
);
|
|
10154
|
-
const manifestPath =
|
|
10155
|
-
|
|
10192
|
+
const manifestPath = path15.join(targetDir, "google-chat-import.json");
|
|
10193
|
+
fs23.writeFileSync(
|
|
10156
10194
|
manifestPath,
|
|
10157
10195
|
JSON.stringify(
|
|
10158
10196
|
{
|
|
@@ -10175,8 +10213,8 @@ var GoogleChatTakeoutResolver = class {
|
|
|
10175
10213
|
};
|
|
10176
10214
|
|
|
10177
10215
|
// ../../packages/node-runtime/src/import/archive/source-manager.ts
|
|
10178
|
-
import * as
|
|
10179
|
-
import * as
|
|
10216
|
+
import * as fs24 from "fs";
|
|
10217
|
+
import * as path16 from "path";
|
|
10180
10218
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
10181
10219
|
var DEFAULT_TTL_MS = 30 * 60 * 1e3;
|
|
10182
10220
|
var ArchiveImportSourceManager = class {
|
|
@@ -10191,7 +10229,7 @@ var ArchiveImportSourceManager = class {
|
|
|
10191
10229
|
this.ttlMs = options2.ttlMs ?? DEFAULT_TTL_MS;
|
|
10192
10230
|
this.now = options2.now ?? Date.now;
|
|
10193
10231
|
this.resolvers = options2.resolvers ?? [new GoogleChatTakeoutResolver()];
|
|
10194
|
-
|
|
10232
|
+
fs24.mkdirSync(this.tempRoot, { recursive: true });
|
|
10195
10233
|
}
|
|
10196
10234
|
prepareLocalArchive(archivePath) {
|
|
10197
10235
|
return this.prepareArchive(archivePath, false);
|
|
@@ -10248,7 +10286,7 @@ var ArchiveImportSourceManager = class {
|
|
|
10248
10286
|
`Archive conversation was not found: ${chatId}`
|
|
10249
10287
|
);
|
|
10250
10288
|
}
|
|
10251
|
-
const materializedDir =
|
|
10289
|
+
const materializedDir = fs24.mkdtempSync(path16.join(this.tempRoot, `materialized-${sourceId}-`));
|
|
10252
10290
|
try {
|
|
10253
10291
|
const reader = new ZipArchiveReader(record.archivePath);
|
|
10254
10292
|
const materialized = await record.resolver.materialize(reader, chat, materializedDir);
|
|
@@ -10299,7 +10337,7 @@ var ArchiveImportSourceManager = class {
|
|
|
10299
10337
|
}
|
|
10300
10338
|
removePath(targetPath) {
|
|
10301
10339
|
try {
|
|
10302
|
-
|
|
10340
|
+
fs24.rmSync(targetPath, { recursive: true, force: true });
|
|
10303
10341
|
} catch {
|
|
10304
10342
|
}
|
|
10305
10343
|
}
|
|
@@ -10369,12 +10407,12 @@ ${rawErrorLabel}: ${details}`;
|
|
|
10369
10407
|
}
|
|
10370
10408
|
|
|
10371
10409
|
// ../../packages/node-runtime/src/ai/assistant-parser.ts
|
|
10372
|
-
import * as
|
|
10410
|
+
import * as path17 from "path";
|
|
10373
10411
|
import matter from "gray-matter";
|
|
10374
10412
|
function parseAssistantFile(content, filePath) {
|
|
10375
10413
|
try {
|
|
10376
10414
|
const { data: fm, content: body } = matter(content);
|
|
10377
|
-
const id = fm.id ??
|
|
10415
|
+
const id = fm.id ?? path17.basename(filePath, ".md");
|
|
10378
10416
|
const name = fm.name;
|
|
10379
10417
|
if (!name) return null;
|
|
10380
10418
|
return {
|
|
@@ -10594,8 +10632,8 @@ var AssistantManager = class {
|
|
|
10594
10632
|
}
|
|
10595
10633
|
// ==================== Init ====================
|
|
10596
10634
|
init() {
|
|
10597
|
-
const { fs:
|
|
10598
|
-
|
|
10635
|
+
const { fs: fs36, assistantsDir } = this.deps;
|
|
10636
|
+
fs36.ensureDir(assistantsDir);
|
|
10599
10637
|
const { generalCreated, generalUpdated } = this.syncGeneralAssistants();
|
|
10600
10638
|
this.loadAll();
|
|
10601
10639
|
this.initialized = true;
|
|
@@ -10607,25 +10645,25 @@ var AssistantManager = class {
|
|
|
10607
10645
|
return { total: this.cache.size, generalCreated, generalUpdated };
|
|
10608
10646
|
}
|
|
10609
10647
|
syncGeneralAssistants() {
|
|
10610
|
-
const { fs:
|
|
10648
|
+
const { fs: fs36, assistantsDir } = this.deps;
|
|
10611
10649
|
let generalCreated = false;
|
|
10612
10650
|
let generalUpdated = false;
|
|
10613
10651
|
for (const id of this.generalIds) {
|
|
10614
10652
|
const config = this.getBuiltinConfig(id);
|
|
10615
10653
|
if (!config) continue;
|
|
10616
|
-
const filePath =
|
|
10617
|
-
if (!
|
|
10618
|
-
|
|
10654
|
+
const filePath = fs36.joinPath(assistantsDir, `${id}.md`);
|
|
10655
|
+
if (!fs36.fileExists(filePath)) {
|
|
10656
|
+
fs36.writeFile(filePath, serializeAssistant(this.withBuiltinTracking(config)));
|
|
10619
10657
|
generalCreated = true;
|
|
10620
10658
|
continue;
|
|
10621
10659
|
}
|
|
10622
10660
|
try {
|
|
10623
|
-
const existing = parseAssistantFile(
|
|
10661
|
+
const existing = parseAssistantFile(fs36.readFile(filePath), filePath);
|
|
10624
10662
|
if (!existing) continue;
|
|
10625
10663
|
if (existing.builtinId !== id) {
|
|
10626
10664
|
try {
|
|
10627
10665
|
const backupId = this.backupAssistantOccupyingReservedId(existing);
|
|
10628
|
-
|
|
10666
|
+
fs36.writeFile(filePath, serializeAssistant(this.withBuiltinTracking(config)));
|
|
10629
10667
|
generalCreated = true;
|
|
10630
10668
|
this.deps.logger?.info("AssistantManager", "Backed up assistant occupying reserved default ID", {
|
|
10631
10669
|
reservedId: id,
|
|
@@ -10641,7 +10679,7 @@ var AssistantManager = class {
|
|
|
10641
10679
|
continue;
|
|
10642
10680
|
}
|
|
10643
10681
|
if (!this.shouldUpgradeBuiltin(existing, config)) continue;
|
|
10644
|
-
|
|
10682
|
+
fs36.writeFile(filePath, serializeAssistant(this.withBuiltinTracking(config, existing.id)));
|
|
10645
10683
|
generalUpdated = true;
|
|
10646
10684
|
this.deps.logger?.info("AssistantManager", "Updated unmodified default assistant template", {
|
|
10647
10685
|
id,
|
|
@@ -10657,14 +10695,14 @@ var AssistantManager = class {
|
|
|
10657
10695
|
return { generalCreated, generalUpdated };
|
|
10658
10696
|
}
|
|
10659
10697
|
backupAssistantOccupyingReservedId(config) {
|
|
10660
|
-
const { fs:
|
|
10698
|
+
const { fs: fs36, assistantsDir } = this.deps;
|
|
10661
10699
|
const baseId = this.deps.generateId?.() || `custom_${Date.now().toString(36)}`;
|
|
10662
10700
|
let backupId = baseId;
|
|
10663
10701
|
let suffix = 2;
|
|
10664
|
-
while (this.generalIds.includes(backupId) ||
|
|
10702
|
+
while (this.generalIds.includes(backupId) || fs36.fileExists(fs36.joinPath(assistantsDir, `${backupId}.md`))) {
|
|
10665
10703
|
backupId = `${baseId}_${suffix++}`;
|
|
10666
10704
|
}
|
|
10667
|
-
|
|
10705
|
+
fs36.writeFile(fs36.joinPath(assistantsDir, `${backupId}.md`), serializeAssistant({ ...config, id: backupId }));
|
|
10668
10706
|
return backupId;
|
|
10669
10707
|
}
|
|
10670
10708
|
getConfigDigest(config) {
|
|
@@ -10701,13 +10739,13 @@ var AssistantManager = class {
|
|
|
10701
10739
|
return { isUnmodified, isOutdated };
|
|
10702
10740
|
}
|
|
10703
10741
|
loadAll() {
|
|
10704
|
-
const { fs:
|
|
10742
|
+
const { fs: fs36, assistantsDir } = this.deps;
|
|
10705
10743
|
this.cache.clear();
|
|
10706
|
-
const files =
|
|
10744
|
+
const files = fs36.listFiles(assistantsDir, ".md");
|
|
10707
10745
|
for (const file of files) {
|
|
10708
10746
|
try {
|
|
10709
|
-
const filePath =
|
|
10710
|
-
const content =
|
|
10747
|
+
const filePath = fs36.joinPath(assistantsDir, file);
|
|
10748
|
+
const content = fs36.readFile(filePath);
|
|
10711
10749
|
const config = parseAssistantFile(content, filePath);
|
|
10712
10750
|
if (config) {
|
|
10713
10751
|
this.cache.set(config.id, config);
|
|
@@ -10884,12 +10922,12 @@ var AssistantManager = class {
|
|
|
10884
10922
|
};
|
|
10885
10923
|
|
|
10886
10924
|
// ../../packages/node-runtime/src/ai/skill-parser.ts
|
|
10887
|
-
import * as
|
|
10925
|
+
import * as path18 from "path";
|
|
10888
10926
|
import matter2 from "gray-matter";
|
|
10889
10927
|
function parseSkillFile(content, filePath) {
|
|
10890
10928
|
try {
|
|
10891
10929
|
const { data: fm, content: prompt } = matter2(content);
|
|
10892
|
-
const id = fm.id ??
|
|
10930
|
+
const id = fm.id ?? path18.basename(filePath, ".md");
|
|
10893
10931
|
const name = fm.name;
|
|
10894
10932
|
if (!name) return null;
|
|
10895
10933
|
return {
|
|
@@ -10908,7 +10946,7 @@ function parseSkillFile(content, filePath) {
|
|
|
10908
10946
|
function extractSkillId(content, filePath) {
|
|
10909
10947
|
try {
|
|
10910
10948
|
const { data: fm } = matter2(content);
|
|
10911
|
-
return fm.id ??
|
|
10949
|
+
return fm.id ?? path18.basename(filePath, ".md");
|
|
10912
10950
|
} catch {
|
|
10913
10951
|
return null;
|
|
10914
10952
|
}
|
|
@@ -10927,8 +10965,8 @@ function validateChatScope(raw) {
|
|
|
10927
10965
|
|
|
10928
10966
|
// ../../packages/node-runtime/src/ai/chats.ts
|
|
10929
10967
|
import Database from "better-sqlite3";
|
|
10930
|
-
import * as
|
|
10931
|
-
import * as
|
|
10968
|
+
import * as fs25 from "fs";
|
|
10969
|
+
import * as path19 from "path";
|
|
10932
10970
|
var DEFAULT_GENERAL_ID = DEFAULT_GENERAL_ASSISTANT_ID;
|
|
10933
10971
|
var defaultLogger2 = {
|
|
10934
10972
|
warn(_category, message, extra) {
|
|
@@ -10947,14 +10985,14 @@ var AIChatManager = class {
|
|
|
10947
10985
|
this.nativeBinding = options2?.nativeBinding;
|
|
10948
10986
|
}
|
|
10949
10987
|
ensureDir(dir) {
|
|
10950
|
-
if (!
|
|
10951
|
-
|
|
10988
|
+
if (!fs25.existsSync(dir)) {
|
|
10989
|
+
fs25.mkdirSync(dir, { recursive: true });
|
|
10952
10990
|
}
|
|
10953
10991
|
}
|
|
10954
10992
|
getDb() {
|
|
10955
10993
|
if (this.db) return this.db;
|
|
10956
10994
|
this.ensureDir(this.aiDataDir);
|
|
10957
|
-
const dbPath =
|
|
10995
|
+
const dbPath = path19.join(this.aiDataDir, "conversations.db");
|
|
10958
10996
|
this.db = this.nativeBinding ? new Database(dbPath, { nativeBinding: this.nativeBinding }) : new Database(dbPath);
|
|
10959
10997
|
this.db.pragma("journal_mode = WAL");
|
|
10960
10998
|
this.db.exec(`
|
|
@@ -11197,16 +11235,16 @@ var AIChatManager = class {
|
|
|
11197
11235
|
if (allRows.length === 0) return [];
|
|
11198
11236
|
const rowMap = new Map(allRows.map((row) => [row.id, row]));
|
|
11199
11237
|
let currentId = leafMessageId ?? this.getActiveMessageId(aiChatId);
|
|
11200
|
-
const
|
|
11238
|
+
const path30 = [];
|
|
11201
11239
|
const seen = /* @__PURE__ */ new Set();
|
|
11202
11240
|
while (currentId && !seen.has(currentId)) {
|
|
11203
11241
|
seen.add(currentId);
|
|
11204
11242
|
const row = rowMap.get(currentId);
|
|
11205
11243
|
if (!row) break;
|
|
11206
|
-
|
|
11244
|
+
path30.push(row);
|
|
11207
11245
|
currentId = row.parentId;
|
|
11208
11246
|
}
|
|
11209
|
-
return
|
|
11247
|
+
return path30.length > 0 ? path30.reverse() : allRows;
|
|
11210
11248
|
}
|
|
11211
11249
|
// ==================== 生命周期 ====================
|
|
11212
11250
|
close() {
|
|
@@ -11776,21 +11814,21 @@ var SkillManagerCore = class {
|
|
|
11776
11814
|
}
|
|
11777
11815
|
// ==================== Init ====================
|
|
11778
11816
|
init() {
|
|
11779
|
-
const { fs:
|
|
11780
|
-
|
|
11817
|
+
const { fs: fs36, skillsDir } = this.deps;
|
|
11818
|
+
fs36.ensureDir(skillsDir);
|
|
11781
11819
|
this.loadAll();
|
|
11782
11820
|
this.initialized = true;
|
|
11783
11821
|
this.deps.logger?.info("SkillManager", "Initialized", { total: this.cache.size });
|
|
11784
11822
|
return { total: this.cache.size };
|
|
11785
11823
|
}
|
|
11786
11824
|
loadAll() {
|
|
11787
|
-
const { fs:
|
|
11825
|
+
const { fs: fs36, skillsDir } = this.deps;
|
|
11788
11826
|
this.cache.clear();
|
|
11789
|
-
const files =
|
|
11827
|
+
const files = fs36.listFiles(skillsDir, ".md");
|
|
11790
11828
|
for (const file of files) {
|
|
11791
11829
|
try {
|
|
11792
|
-
const filePath =
|
|
11793
|
-
const content =
|
|
11830
|
+
const filePath = fs36.joinPath(skillsDir, file);
|
|
11831
|
+
const content = fs36.readFile(filePath);
|
|
11794
11832
|
const def = parseSkillFile(content, filePath);
|
|
11795
11833
|
if (def) {
|
|
11796
11834
|
this.cache.set(def.id, def);
|
|
@@ -11942,8 +11980,8 @@ var SkillManagerCore = class {
|
|
|
11942
11980
|
};
|
|
11943
11981
|
|
|
11944
11982
|
// ../../packages/node-runtime/src/ai/skill-manager.ts
|
|
11945
|
-
import * as
|
|
11946
|
-
import * as
|
|
11983
|
+
import * as fs26 from "fs";
|
|
11984
|
+
import * as path20 from "path";
|
|
11947
11985
|
var SKILLS_DIR_NAME = "skills";
|
|
11948
11986
|
var defaultLogger3 = {
|
|
11949
11987
|
info: () => {
|
|
@@ -11953,26 +11991,26 @@ var defaultLogger3 = {
|
|
|
11953
11991
|
};
|
|
11954
11992
|
var nodeFs = {
|
|
11955
11993
|
ensureDir(dir) {
|
|
11956
|
-
|
|
11994
|
+
fs26.mkdirSync(dir, { recursive: true });
|
|
11957
11995
|
},
|
|
11958
11996
|
listFiles(dir, ext) {
|
|
11959
|
-
if (!
|
|
11960
|
-
return
|
|
11997
|
+
if (!fs26.existsSync(dir)) return [];
|
|
11998
|
+
return fs26.readdirSync(dir).filter((file) => file.endsWith(ext));
|
|
11961
11999
|
},
|
|
11962
12000
|
readFile(filePath) {
|
|
11963
|
-
return
|
|
12001
|
+
return fs26.readFileSync(filePath, "utf-8");
|
|
11964
12002
|
},
|
|
11965
12003
|
writeFile(filePath, content) {
|
|
11966
|
-
|
|
12004
|
+
fs26.writeFileSync(filePath, content, "utf-8");
|
|
11967
12005
|
},
|
|
11968
12006
|
deleteFile(filePath) {
|
|
11969
|
-
|
|
12007
|
+
fs26.unlinkSync(filePath);
|
|
11970
12008
|
},
|
|
11971
12009
|
fileExists(filePath) {
|
|
11972
|
-
return
|
|
12010
|
+
return fs26.existsSync(filePath);
|
|
11973
12011
|
},
|
|
11974
12012
|
joinPath(...parts) {
|
|
11975
|
-
return
|
|
12013
|
+
return path20.join(...parts);
|
|
11976
12014
|
}
|
|
11977
12015
|
};
|
|
11978
12016
|
var SkillManager = class {
|
|
@@ -11982,7 +12020,7 @@ var SkillManager = class {
|
|
|
11982
12020
|
this.logger = logger ?? defaultLogger3;
|
|
11983
12021
|
this.core = new SkillManagerCore({
|
|
11984
12022
|
fs: nodeFs,
|
|
11985
|
-
skillsDir:
|
|
12023
|
+
skillsDir: path20.join(aiDataDir, SKILLS_DIR_NAME),
|
|
11986
12024
|
builtinRawSkills: [],
|
|
11987
12025
|
logger: {
|
|
11988
12026
|
info: () => {
|
|
@@ -12278,6 +12316,13 @@ function wrapWithChartSchemaGate(tool, state) {
|
|
|
12278
12316
|
};
|
|
12279
12317
|
}
|
|
12280
12318
|
|
|
12319
|
+
// ../../packages/node-runtime/src/ai/compression/types.ts
|
|
12320
|
+
var DEFAULT_CONTEXT_COMPRESSION_CONFIG = Object.freeze({
|
|
12321
|
+
tokenThresholdPercent: 75,
|
|
12322
|
+
bufferSizePercent: 20,
|
|
12323
|
+
maxToolResultPercent: 50
|
|
12324
|
+
});
|
|
12325
|
+
|
|
12281
12326
|
// ../../packages/node-runtime/src/ai/agent/history.ts
|
|
12282
12327
|
function createEmptyPiUsage() {
|
|
12283
12328
|
return {
|
|
@@ -12411,9 +12456,6 @@ var defaultLogger4 = {
|
|
|
12411
12456
|
}
|
|
12412
12457
|
};
|
|
12413
12458
|
async function checkAndCompress(aiChatId, config, systemPrompt, llmAdapter, convManager, logger = defaultLogger4) {
|
|
12414
|
-
if (!config.enabled) {
|
|
12415
|
-
return { compressed: false, reason: "skipped_disabled" };
|
|
12416
|
-
}
|
|
12417
12459
|
try {
|
|
12418
12460
|
const contextWindow = llmAdapter.contextWindow || DEFAULT_CONTEXT_WINDOW;
|
|
12419
12461
|
const thresholdTokens = Math.floor(contextWindow * (config.tokenThresholdPercent / 100) * 0.95);
|
|
@@ -12507,7 +12549,7 @@ async function manualCompress(aiChatId, config, systemPrompt, llmAdapter, convMa
|
|
|
12507
12549
|
if (messageCount < 5) {
|
|
12508
12550
|
return { compressed: false, reason: "skipped_idempotent" };
|
|
12509
12551
|
}
|
|
12510
|
-
const overrideConfig = { ...config,
|
|
12552
|
+
const overrideConfig = { ...config, tokenThresholdPercent: 0 };
|
|
12511
12553
|
return checkAndCompress(aiChatId, overrideConfig, systemPrompt, llmAdapter, convManager, logger);
|
|
12512
12554
|
}
|
|
12513
12555
|
function replayedToolResultTexts(blocks) {
|
|
@@ -15848,11 +15890,11 @@ var localeMap = {
|
|
|
15848
15890
|
"ja-JP": ja_JP_default,
|
|
15849
15891
|
"zh-TW": zh_TW_default
|
|
15850
15892
|
};
|
|
15851
|
-
function getNestedValue(obj,
|
|
15893
|
+
function getNestedValue(obj, path30) {
|
|
15852
15894
|
let current = obj;
|
|
15853
|
-
for (const
|
|
15895
|
+
for (const segment of path30.split(".")) {
|
|
15854
15896
|
if (current == null || typeof current !== "object") return void 0;
|
|
15855
|
-
current = current[
|
|
15897
|
+
current = current[segment];
|
|
15856
15898
|
}
|
|
15857
15899
|
return current;
|
|
15858
15900
|
}
|
|
@@ -16086,8 +16128,8 @@ async function generateSessionSummary(deps, segmentId, options2 = {}) {
|
|
|
16086
16128
|
const segments = splitIntoSegments(validMessages, MAX_CONTENT_PER_CALL);
|
|
16087
16129
|
log?.info("Summary", `Long session segmented: ${segments.length} segments`);
|
|
16088
16130
|
const subSummaries = [];
|
|
16089
|
-
for (const
|
|
16090
|
-
const segContent = formatMessages(
|
|
16131
|
+
for (const segment of segments) {
|
|
16132
|
+
const segContent = formatMessages(segment);
|
|
16091
16133
|
const sub = await deps.llmComplete(
|
|
16092
16134
|
deps.t("summary.systemPromptDirect"),
|
|
16093
16135
|
buildSubSummaryPrompt(segContent, locale, strategy),
|
|
@@ -16385,10 +16427,10 @@ var LLMConfigStore = class {
|
|
|
16385
16427
|
};
|
|
16386
16428
|
|
|
16387
16429
|
// ../../packages/node-runtime/src/ai/file-config-storage.ts
|
|
16388
|
-
import * as
|
|
16430
|
+
import * as path21 from "path";
|
|
16389
16431
|
function createFileConfigStorage(baseDir) {
|
|
16390
16432
|
const heldLocks = /* @__PURE__ */ new Set();
|
|
16391
|
-
const getFilePath = (key) =>
|
|
16433
|
+
const getFilePath = (key) => path21.join(baseDir, `${key}.json`);
|
|
16392
16434
|
return {
|
|
16393
16435
|
readJson(key) {
|
|
16394
16436
|
return readJsonFile(getFilePath(key));
|
|
@@ -17344,8 +17386,8 @@ function exportWithFormat(params, openDatabase) {
|
|
|
17344
17386
|
}
|
|
17345
17387
|
|
|
17346
17388
|
// ../../packages/node-runtime/src/merger/temp-db.ts
|
|
17347
|
-
import * as
|
|
17348
|
-
import * as
|
|
17389
|
+
import * as fs27 from "fs";
|
|
17390
|
+
import * as path22 from "path";
|
|
17349
17391
|
var TEMP_DB_SCHEMA = `
|
|
17350
17392
|
CREATE TABLE IF NOT EXISTS meta (
|
|
17351
17393
|
name TEXT NOT NULL,
|
|
@@ -17365,12 +17407,14 @@ var TEMP_DB_SCHEMA = `
|
|
|
17365
17407
|
|
|
17366
17408
|
CREATE TABLE IF NOT EXISTS message (
|
|
17367
17409
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
17410
|
+
platform_message_id TEXT,
|
|
17368
17411
|
sender_platform_id TEXT NOT NULL,
|
|
17369
17412
|
sender_account_name TEXT,
|
|
17370
17413
|
sender_group_nickname TEXT,
|
|
17371
17414
|
timestamp INTEGER NOT NULL,
|
|
17372
17415
|
type INTEGER NOT NULL,
|
|
17373
|
-
content TEXT
|
|
17416
|
+
content TEXT,
|
|
17417
|
+
reply_to_message_id TEXT
|
|
17374
17418
|
);
|
|
17375
17419
|
|
|
17376
17420
|
CREATE INDEX IF NOT EXISTS idx_message_ts ON message(timestamp);
|
|
@@ -17401,8 +17445,10 @@ var TempDbWriter = class {
|
|
|
17401
17445
|
}
|
|
17402
17446
|
writeMessages(messages) {
|
|
17403
17447
|
const insert = this.db.prepare(
|
|
17404
|
-
`INSERT INTO message (
|
|
17405
|
-
|
|
17448
|
+
`INSERT INTO message (
|
|
17449
|
+
platform_message_id, sender_platform_id, sender_account_name, sender_group_nickname,
|
|
17450
|
+
timestamp, type, content, reply_to_message_id
|
|
17451
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
17406
17452
|
);
|
|
17407
17453
|
const memberInsert = this.db.prepare(
|
|
17408
17454
|
"INSERT OR IGNORE INTO member (platform_id, account_name, group_nickname, avatar) VALUES (?, ?, ?, ?)"
|
|
@@ -17413,12 +17459,14 @@ var TempDbWriter = class {
|
|
|
17413
17459
|
memberInsert.run(msg.senderPlatformId, msg.senderAccountName || null, msg.senderGroupNickname || null, null);
|
|
17414
17460
|
}
|
|
17415
17461
|
insert.run(
|
|
17462
|
+
msg.platformMessageId || null,
|
|
17416
17463
|
msg.senderPlatformId,
|
|
17417
17464
|
msg.senderAccountName || null,
|
|
17418
17465
|
msg.senderGroupNickname || null,
|
|
17419
17466
|
msg.timestamp,
|
|
17420
17467
|
msg.type,
|
|
17421
|
-
msg.content || null
|
|
17468
|
+
msg.content || null,
|
|
17469
|
+
msg.replyToMessageId || null
|
|
17422
17470
|
);
|
|
17423
17471
|
this.messageCount++;
|
|
17424
17472
|
}
|
|
@@ -17450,7 +17498,8 @@ var TempDbReader = class {
|
|
|
17450
17498
|
platform: row.platform,
|
|
17451
17499
|
type: row.type,
|
|
17452
17500
|
groupId: row.group_id || void 0,
|
|
17453
|
-
groupAvatar: row.group_avatar || void 0
|
|
17501
|
+
groupAvatar: row.group_avatar || void 0,
|
|
17502
|
+
ownerId: row.owner_id || void 0
|
|
17454
17503
|
};
|
|
17455
17504
|
}
|
|
17456
17505
|
getMembers() {
|
|
@@ -17468,20 +17517,23 @@ var TempDbReader = class {
|
|
|
17468
17517
|
}
|
|
17469
17518
|
streamMessages(batchSize, callback) {
|
|
17470
17519
|
const stmt = this.db.prepare(`
|
|
17471
|
-
SELECT sender_platform_id, sender_account_name, sender_group_nickname,
|
|
17472
|
-
|
|
17520
|
+
SELECT platform_message_id, sender_platform_id, sender_account_name, sender_group_nickname,
|
|
17521
|
+
timestamp, type, content, reply_to_message_id
|
|
17522
|
+
FROM message ORDER BY timestamp ASC, id ASC LIMIT ? OFFSET ?
|
|
17473
17523
|
`);
|
|
17474
17524
|
let offset = 0;
|
|
17475
17525
|
while (true) {
|
|
17476
17526
|
const rows = stmt.all(batchSize, offset);
|
|
17477
17527
|
if (rows.length === 0) break;
|
|
17478
17528
|
const messages = rows.map((r) => ({
|
|
17529
|
+
platformMessageId: r.platform_message_id || void 0,
|
|
17479
17530
|
senderPlatformId: r.sender_platform_id,
|
|
17480
17531
|
senderAccountName: r.sender_account_name || r.sender_platform_id,
|
|
17481
17532
|
senderGroupNickname: r.sender_group_nickname || void 0,
|
|
17482
17533
|
timestamp: r.timestamp,
|
|
17483
17534
|
type: r.type,
|
|
17484
|
-
content: r.content
|
|
17535
|
+
content: r.content,
|
|
17536
|
+
replyToMessageId: r.reply_to_message_id || void 0
|
|
17485
17537
|
}));
|
|
17486
17538
|
callback(messages);
|
|
17487
17539
|
offset += batchSize;
|
|
@@ -17489,16 +17541,19 @@ var TempDbReader = class {
|
|
|
17489
17541
|
}
|
|
17490
17542
|
getAllMessages() {
|
|
17491
17543
|
const rows = this.db.prepare(
|
|
17492
|
-
`SELECT sender_platform_id, sender_account_name, sender_group_nickname,
|
|
17493
|
-
|
|
17544
|
+
`SELECT platform_message_id, sender_platform_id, sender_account_name, sender_group_nickname,
|
|
17545
|
+
timestamp, type, content, reply_to_message_id
|
|
17546
|
+
FROM message ORDER BY timestamp ASC, id ASC`
|
|
17494
17547
|
).all();
|
|
17495
17548
|
return rows.map((r) => ({
|
|
17549
|
+
platformMessageId: r.platform_message_id || void 0,
|
|
17496
17550
|
senderPlatformId: r.sender_platform_id,
|
|
17497
17551
|
senderAccountName: r.sender_account_name || r.sender_platform_id,
|
|
17498
17552
|
senderGroupNickname: r.sender_group_nickname || void 0,
|
|
17499
17553
|
timestamp: r.timestamp,
|
|
17500
17554
|
type: r.type,
|
|
17501
|
-
content: r.content
|
|
17555
|
+
content: r.content,
|
|
17556
|
+
replyToMessageId: r.reply_to_message_id || void 0
|
|
17502
17557
|
}));
|
|
17503
17558
|
}
|
|
17504
17559
|
close() {
|
|
@@ -17519,7 +17574,8 @@ function createDataSourceFromReader(reader) {
|
|
|
17519
17574
|
platform: meta.platform,
|
|
17520
17575
|
type: meta.type,
|
|
17521
17576
|
groupId: meta.groupId,
|
|
17522
|
-
groupAvatar: meta.groupAvatar
|
|
17577
|
+
groupAvatar: meta.groupAvatar,
|
|
17578
|
+
ownerId: meta.ownerId
|
|
17523
17579
|
};
|
|
17524
17580
|
},
|
|
17525
17581
|
getMembers() {
|
|
@@ -17538,11 +17594,13 @@ function createDataSourceFromReader(reader) {
|
|
|
17538
17594
|
callback(
|
|
17539
17595
|
messages.map((msg) => ({
|
|
17540
17596
|
senderPlatformId: msg.senderPlatformId,
|
|
17597
|
+
platformMessageId: msg.platformMessageId,
|
|
17541
17598
|
senderAccountName: msg.senderAccountName,
|
|
17542
17599
|
senderGroupNickname: msg.senderGroupNickname,
|
|
17543
17600
|
timestamp: msg.timestamp,
|
|
17544
17601
|
type: msg.type,
|
|
17545
|
-
content: msg.content
|
|
17602
|
+
content: msg.content,
|
|
17603
|
+
replyToMessageId: msg.replyToMessageId
|
|
17546
17604
|
}))
|
|
17547
17605
|
);
|
|
17548
17606
|
});
|
|
@@ -17555,8 +17613,9 @@ function exportSessionToJson(db) {
|
|
|
17555
17613
|
const members = db.prepare("SELECT platform_id, account_name, group_nickname, avatar FROM member").all();
|
|
17556
17614
|
const messages = db.prepare(
|
|
17557
17615
|
`SELECT m.platform_id as sender, msg.sender_account_name as accountName,
|
|
17558
|
-
msg.sender_group_nickname as groupNickname, msg.ts as timestamp, msg.type, msg.content
|
|
17559
|
-
|
|
17616
|
+
msg.sender_group_nickname as groupNickname, msg.ts as timestamp, msg.type, msg.content,
|
|
17617
|
+
msg.platform_message_id as platformMessageId, msg.reply_to_message_id as replyToMessageId
|
|
17618
|
+
FROM message msg JOIN member m ON msg.sender_id = m.id ORDER BY msg.ts, msg.id`
|
|
17560
17619
|
).all();
|
|
17561
17620
|
return {
|
|
17562
17621
|
chatlab: {
|
|
@@ -17570,7 +17629,8 @@ function exportSessionToJson(db) {
|
|
|
17570
17629
|
platform: meta.platform,
|
|
17571
17630
|
type: meta.type,
|
|
17572
17631
|
groupId: meta.group_id,
|
|
17573
|
-
groupAvatar: meta.group_avatar
|
|
17632
|
+
groupAvatar: meta.group_avatar,
|
|
17633
|
+
ownerId: meta.owner_id
|
|
17574
17634
|
},
|
|
17575
17635
|
members: members.map((m) => ({
|
|
17576
17636
|
platformId: m.platform_id,
|
|
@@ -17579,32 +17639,34 @@ function exportSessionToJson(db) {
|
|
|
17579
17639
|
avatar: m.avatar
|
|
17580
17640
|
})),
|
|
17581
17641
|
messages: messages.map((msg) => ({
|
|
17642
|
+
platformMessageId: msg.platformMessageId || void 0,
|
|
17582
17643
|
sender: msg.sender,
|
|
17583
17644
|
accountName: msg.accountName || msg.sender,
|
|
17584
17645
|
groupNickname: msg.groupNickname || void 0,
|
|
17585
17646
|
timestamp: msg.timestamp,
|
|
17586
17647
|
type: msg.type,
|
|
17587
|
-
content: msg.content ?? null
|
|
17648
|
+
content: msg.content ?? null,
|
|
17649
|
+
replyToMessageId: msg.replyToMessageId || void 0
|
|
17588
17650
|
}))
|
|
17589
17651
|
};
|
|
17590
17652
|
}
|
|
17591
17653
|
function deleteTempDatabase(dbPath) {
|
|
17592
17654
|
try {
|
|
17593
|
-
if (
|
|
17655
|
+
if (fs27.existsSync(dbPath)) fs27.unlinkSync(dbPath);
|
|
17594
17656
|
const walPath = dbPath + "-wal";
|
|
17595
17657
|
const shmPath = dbPath + "-shm";
|
|
17596
|
-
if (
|
|
17597
|
-
if (
|
|
17658
|
+
if (fs27.existsSync(walPath)) fs27.unlinkSync(walPath);
|
|
17659
|
+
if (fs27.existsSync(shmPath)) fs27.unlinkSync(shmPath);
|
|
17598
17660
|
} catch {
|
|
17599
17661
|
}
|
|
17600
17662
|
}
|
|
17601
17663
|
function cleanupTempDatabases(tempDir) {
|
|
17602
17664
|
try {
|
|
17603
|
-
if (!
|
|
17604
|
-
const files =
|
|
17665
|
+
if (!fs27.existsSync(tempDir)) return;
|
|
17666
|
+
const files = fs27.readdirSync(tempDir);
|
|
17605
17667
|
for (const file of files) {
|
|
17606
17668
|
if (file.startsWith("merge_") && file.endsWith(".db")) {
|
|
17607
|
-
deleteTempDatabase(
|
|
17669
|
+
deleteTempDatabase(path22.join(tempDir, file));
|
|
17608
17670
|
}
|
|
17609
17671
|
}
|
|
17610
17672
|
} catch {
|
|
@@ -17618,8 +17680,16 @@ __export(session_service_exports, {
|
|
|
17618
17680
|
getAnalysisSession: () => getAnalysisSession,
|
|
17619
17681
|
listAnalysisSessions: () => listAnalysisSessions,
|
|
17620
17682
|
renameSession: () => renameSession2,
|
|
17683
|
+
resolveValidatedSessionOverview: () => resolveValidatedSessionOverview,
|
|
17621
17684
|
updateSessionOwnerId: () => updateSessionOwnerId2
|
|
17622
17685
|
});
|
|
17686
|
+
function resolveValidatedSessionOverview(db, sessionId, cacheDir) {
|
|
17687
|
+
try {
|
|
17688
|
+
return getValidatedOverviewCache(db, sessionId, cacheDir);
|
|
17689
|
+
} catch {
|
|
17690
|
+
return getSessionOverview(db);
|
|
17691
|
+
}
|
|
17692
|
+
}
|
|
17623
17693
|
function buildSession(db, id, dbPath, options2) {
|
|
17624
17694
|
const meta = getSessionMeta(db);
|
|
17625
17695
|
if (!meta) return null;
|
|
@@ -17690,6 +17760,10 @@ __export(member_service_exports, {
|
|
|
17690
17760
|
mergeMembers: () => mergeMembers2,
|
|
17691
17761
|
updateMemberAliases: () => updateMemberAliases2
|
|
17692
17762
|
});
|
|
17763
|
+
function invalidateMemberMutationCaches(sessionId, cacheDir) {
|
|
17764
|
+
invalidateCache(sessionId, cacheDir, CACHE_KEY_OVERVIEW);
|
|
17765
|
+
invalidateCache(sessionId, cacheDir, CACHE_KEY_MEMBERS);
|
|
17766
|
+
}
|
|
17693
17767
|
function getMembers(adapter, sessionId) {
|
|
17694
17768
|
const db = adapter.ensureReadonly(sessionId);
|
|
17695
17769
|
return getMembersWithAliases(db);
|
|
@@ -17709,17 +17783,23 @@ function updateMemberAliases2(adapter, sessionId, memberId, aliases) {
|
|
|
17709
17783
|
const db = adapter.ensureWritable(sessionId);
|
|
17710
17784
|
return updateMemberAliases(db, memberId, aliases);
|
|
17711
17785
|
}
|
|
17712
|
-
function mergeMembers2(adapter, sessionId, memberId1, memberId2) {
|
|
17786
|
+
function mergeMembers2(adapter, sessionId, memberId1, memberId2, cacheDir) {
|
|
17713
17787
|
const db = adapter.ensureWritable(sessionId);
|
|
17714
|
-
|
|
17788
|
+
const success = mergeMembers(db, memberId1, memberId2);
|
|
17789
|
+
if (success) invalidateMemberMutationCaches(sessionId, cacheDir);
|
|
17790
|
+
return success;
|
|
17715
17791
|
}
|
|
17716
|
-
function deleteMember2(adapter, sessionId, memberId) {
|
|
17792
|
+
function deleteMember2(adapter, sessionId, memberId, cacheDir) {
|
|
17717
17793
|
const db = adapter.ensureWritable(sessionId);
|
|
17718
|
-
|
|
17794
|
+
const success = deleteMember(db, memberId);
|
|
17795
|
+
if (success) invalidateMemberMutationCaches(sessionId, cacheDir);
|
|
17796
|
+
return success;
|
|
17719
17797
|
}
|
|
17720
|
-
function deleteMembers2(adapter, sessionId, memberIds) {
|
|
17798
|
+
function deleteMembers2(adapter, sessionId, memberIds, cacheDir) {
|
|
17721
17799
|
const db = adapter.ensureWritable(sessionId);
|
|
17722
|
-
|
|
17800
|
+
const success = deleteMembers(db, memberIds);
|
|
17801
|
+
if (success) invalidateMemberMutationCaches(sessionId, cacheDir);
|
|
17802
|
+
return success;
|
|
17723
17803
|
}
|
|
17724
17804
|
function getMemberNameHistory2(adapter, sessionId, memberId) {
|
|
17725
17805
|
const db = adapter.ensureReadonly(sessionId);
|
|
@@ -17732,10 +17812,7 @@ __export(session_index_service_exports, {
|
|
|
17732
17812
|
clearIndex: () => clearIndex,
|
|
17733
17813
|
generateIncrementalIndex: () => generateIncrementalIndex,
|
|
17734
17814
|
generateIndex: () => generateIndex,
|
|
17735
|
-
getAllIndexStats: () => getAllIndexStats
|
|
17736
|
-
getFtsStatus: () => getFtsStatus,
|
|
17737
|
-
rebuildFts: () => rebuildFts,
|
|
17738
|
-
searchFts: () => searchFts
|
|
17815
|
+
getAllIndexStats: () => getAllIndexStats
|
|
17739
17816
|
});
|
|
17740
17817
|
function generateIndex(adapter, sessionId, gapThreshold = 1800) {
|
|
17741
17818
|
const db = adapter.ensureWritable(sessionId);
|
|
@@ -17749,18 +17826,6 @@ function clearIndex(adapter, sessionId) {
|
|
|
17749
17826
|
const db = adapter.ensureWritable(sessionId);
|
|
17750
17827
|
clearSessionIndex(db);
|
|
17751
17828
|
}
|
|
17752
|
-
function getFtsStatus(adapter, sessionId) {
|
|
17753
|
-
const db = adapter.ensureReadonly(sessionId);
|
|
17754
|
-
return hasFtsTable(db);
|
|
17755
|
-
}
|
|
17756
|
-
function searchFts(adapter, sessionId, keywords, limit, offset) {
|
|
17757
|
-
const db = adapter.ensureReadonly(sessionId);
|
|
17758
|
-
return searchByFts(db, keywords, limit, offset);
|
|
17759
|
-
}
|
|
17760
|
-
function rebuildFts(adapter, sessionId) {
|
|
17761
|
-
const db = adapter.ensureWritable(sessionId);
|
|
17762
|
-
return rebuildFtsIndex(db);
|
|
17763
|
-
}
|
|
17764
17829
|
function getAllIndexStats(adapter) {
|
|
17765
17830
|
const ids = adapter.listSessionIds();
|
|
17766
17831
|
return ids.map((sessionId) => {
|
|
@@ -17989,8 +18054,8 @@ function clearSessionOwner(adapter, sessionId) {
|
|
|
17989
18054
|
}
|
|
17990
18055
|
|
|
17991
18056
|
// ../../packages/node-runtime/src/services/contacts/overrides.ts
|
|
17992
|
-
import
|
|
17993
|
-
import
|
|
18057
|
+
import fs28 from "fs";
|
|
18058
|
+
import path23 from "path";
|
|
17994
18059
|
var CONTACT_OVERRIDES_FILE_NAME = "contact-overrides.json";
|
|
17995
18060
|
var CONTACT_OVERRIDES_TMP_PREFIX = "contact-overrides.tmp-";
|
|
17996
18061
|
var CONTACT_OVERRIDES_VERSION = 1;
|
|
@@ -18001,23 +18066,23 @@ function createEmptyContactOverridesFile() {
|
|
|
18001
18066
|
};
|
|
18002
18067
|
}
|
|
18003
18068
|
function getContactOverridesPath(snapshotDir) {
|
|
18004
|
-
return
|
|
18069
|
+
return path23.join(snapshotDir, CONTACT_OVERRIDES_FILE_NAME);
|
|
18005
18070
|
}
|
|
18006
18071
|
function readContactOverrides(snapshotDir) {
|
|
18007
18072
|
const filePath = getContactOverridesPath(snapshotDir);
|
|
18008
|
-
if (!
|
|
18073
|
+
if (!fs28.existsSync(filePath)) return createEmptyContactOverridesFile();
|
|
18009
18074
|
try {
|
|
18010
|
-
return normalizeContactOverrides(JSON.parse(
|
|
18075
|
+
return normalizeContactOverrides(JSON.parse(fs28.readFileSync(filePath, "utf-8")));
|
|
18011
18076
|
} catch (error) {
|
|
18012
18077
|
appLogger.warn("contacts", "contact overrides file is unreadable", error);
|
|
18013
18078
|
return createEmptyContactOverridesFile();
|
|
18014
18079
|
}
|
|
18015
18080
|
}
|
|
18016
18081
|
function writeContactOverrides(snapshotDir, overrides) {
|
|
18017
|
-
if (!
|
|
18018
|
-
const tmpPath =
|
|
18019
|
-
|
|
18020
|
-
|
|
18082
|
+
if (!fs28.existsSync(snapshotDir)) fs28.mkdirSync(snapshotDir, { recursive: true });
|
|
18083
|
+
const tmpPath = path23.join(snapshotDir, `${CONTACT_OVERRIDES_TMP_PREFIX}${process.pid}-${Date.now()}`);
|
|
18084
|
+
fs28.writeFileSync(tmpPath, JSON.stringify(normalizeContactOverrides(overrides), null, 2), "utf-8");
|
|
18085
|
+
fs28.renameSync(tmpPath, getContactOverridesPath(snapshotDir));
|
|
18021
18086
|
}
|
|
18022
18087
|
function normalizeContactOverrides(value) {
|
|
18023
18088
|
const input = isRecord4(value) ? value : {};
|
|
@@ -18052,23 +18117,23 @@ function buildContactsSignature(adapter, timeRangePreset) {
|
|
|
18052
18117
|
}
|
|
18053
18118
|
|
|
18054
18119
|
// ../../packages/node-runtime/src/services/contacts/snapshot.ts
|
|
18055
|
-
import
|
|
18056
|
-
import
|
|
18120
|
+
import fs29 from "fs";
|
|
18121
|
+
import path24 from "path";
|
|
18057
18122
|
var CONTACTS_SNAPSHOT_TMP_PREFIX = "contacts-snapshot.tmp-";
|
|
18058
18123
|
function getContactsSnapshotPath(snapshotDir, timeRangePreset) {
|
|
18059
18124
|
const preset = normalizeContactsTimeRangePreset(timeRangePreset);
|
|
18060
|
-
return
|
|
18125
|
+
return path24.join(snapshotDir, `contacts-snapshot-${preset}.json`);
|
|
18061
18126
|
}
|
|
18062
18127
|
function readContactsSnapshot(snapshotDir, timeRangePreset, options2 = {}) {
|
|
18063
18128
|
const snapshotPath = getContactsSnapshotPath(snapshotDir, timeRangePreset);
|
|
18064
|
-
if (!
|
|
18129
|
+
if (!fs29.existsSync(snapshotPath)) return null;
|
|
18065
18130
|
try {
|
|
18066
|
-
return JSON.parse(
|
|
18131
|
+
return JSON.parse(fs29.readFileSync(snapshotPath, "utf-8"));
|
|
18067
18132
|
} catch (error) {
|
|
18068
18133
|
const ts = options2.now?.() ?? Date.now();
|
|
18069
|
-
const backupPath =
|
|
18134
|
+
const backupPath = path24.join(snapshotDir, `contacts-snapshot.corrupt-${ts}.json`);
|
|
18070
18135
|
try {
|
|
18071
|
-
|
|
18136
|
+
fs29.renameSync(snapshotPath, backupPath);
|
|
18072
18137
|
} catch (renameError) {
|
|
18073
18138
|
appLogger.warn("contacts", "failed to backup corrupt contacts snapshot", renameError);
|
|
18074
18139
|
}
|
|
@@ -18077,17 +18142,17 @@ function readContactsSnapshot(snapshotDir, timeRangePreset, options2 = {}) {
|
|
|
18077
18142
|
}
|
|
18078
18143
|
}
|
|
18079
18144
|
function writeContactsSnapshot(snapshotDir, snapshot) {
|
|
18080
|
-
if (!
|
|
18081
|
-
const tmpPath =
|
|
18082
|
-
|
|
18083
|
-
|
|
18145
|
+
if (!fs29.existsSync(snapshotDir)) fs29.mkdirSync(snapshotDir, { recursive: true });
|
|
18146
|
+
const tmpPath = path24.join(snapshotDir, `${CONTACTS_SNAPSHOT_TMP_PREFIX}${process.pid}-${Date.now()}`);
|
|
18147
|
+
fs29.writeFileSync(tmpPath, JSON.stringify(snapshot, null, 2), "utf-8");
|
|
18148
|
+
fs29.renameSync(tmpPath, getContactsSnapshotPath(snapshotDir, snapshot.timeRange.preset));
|
|
18084
18149
|
}
|
|
18085
18150
|
function cleanupContactsSnapshotTempFiles(snapshotDir) {
|
|
18086
|
-
if (!
|
|
18087
|
-
for (const name of
|
|
18151
|
+
if (!fs29.existsSync(snapshotDir)) return;
|
|
18152
|
+
for (const name of fs29.readdirSync(snapshotDir)) {
|
|
18088
18153
|
if (!name.startsWith(CONTACTS_SNAPSHOT_TMP_PREFIX)) continue;
|
|
18089
18154
|
try {
|
|
18090
|
-
|
|
18155
|
+
fs29.rmSync(path24.join(snapshotDir, name), { force: true });
|
|
18091
18156
|
} catch (error) {
|
|
18092
18157
|
appLogger.warn("contacts", "failed to remove contacts snapshot temp file", error);
|
|
18093
18158
|
}
|
|
@@ -18096,8 +18161,8 @@ function cleanupContactsSnapshotTempFiles(snapshotDir) {
|
|
|
18096
18161
|
|
|
18097
18162
|
// ../../packages/node-runtime/src/services/contacts/worker-runner.ts
|
|
18098
18163
|
import { Worker } from "worker_threads";
|
|
18099
|
-
import { existsSync as
|
|
18100
|
-
function resolveDefaultContactsWorkerEntryUrl(currentModuleUrl = import.meta.url, entryExists = (url) =>
|
|
18164
|
+
import { existsSync as existsSync8 } from "fs";
|
|
18165
|
+
function resolveDefaultContactsWorkerEntryUrl(currentModuleUrl = import.meta.url, entryExists = (url) => existsSync8(url)) {
|
|
18101
18166
|
const moduleUrl = typeof currentModuleUrl === "string" ? currentModuleUrl : currentModuleUrl.href;
|
|
18102
18167
|
if (moduleUrl.endsWith(".ts")) return new URL("./worker-entry.ts", moduleUrl);
|
|
18103
18168
|
if (moduleUrl.endsWith(".mjs")) return new URL("./contacts-worker.mjs", moduleUrl);
|
|
@@ -18585,23 +18650,23 @@ function buildPeopleRelationshipsSignature(adapter, timeRangePreset) {
|
|
|
18585
18650
|
}
|
|
18586
18651
|
|
|
18587
18652
|
// ../../packages/node-runtime/src/services/people/relationships/snapshot.ts
|
|
18588
|
-
import
|
|
18589
|
-
import
|
|
18653
|
+
import fs30 from "fs";
|
|
18654
|
+
import path25 from "path";
|
|
18590
18655
|
var PEOPLE_RELATIONSHIPS_SNAPSHOT_TMP_PREFIX = "people-relationships-snapshot.tmp-";
|
|
18591
18656
|
function getPeopleRelationshipsSnapshotPath(snapshotDir, timeRangePreset) {
|
|
18592
18657
|
const preset = normalizePeopleRelationshipsTimeRangePreset(timeRangePreset);
|
|
18593
|
-
return
|
|
18658
|
+
return path25.join(snapshotDir, `graph-snapshot-${preset}.json`);
|
|
18594
18659
|
}
|
|
18595
18660
|
function readPeopleRelationshipsSnapshot(snapshotDir, timeRangePreset, options2 = {}) {
|
|
18596
18661
|
const snapshotPath = getPeopleRelationshipsSnapshotPath(snapshotDir, timeRangePreset);
|
|
18597
|
-
if (!
|
|
18662
|
+
if (!fs30.existsSync(snapshotPath)) return null;
|
|
18598
18663
|
try {
|
|
18599
|
-
return JSON.parse(
|
|
18664
|
+
return JSON.parse(fs30.readFileSync(snapshotPath, "utf-8"));
|
|
18600
18665
|
} catch (error) {
|
|
18601
18666
|
const ts = options2.now?.() ?? Date.now();
|
|
18602
|
-
const backupPath =
|
|
18667
|
+
const backupPath = path25.join(snapshotDir, `graph-snapshot.corrupt-${ts}.json`);
|
|
18603
18668
|
try {
|
|
18604
|
-
|
|
18669
|
+
fs30.renameSync(snapshotPath, backupPath);
|
|
18605
18670
|
} catch (renameError) {
|
|
18606
18671
|
appLogger.warn("people-relationships", "failed to backup corrupt people relationships snapshot", renameError);
|
|
18607
18672
|
}
|
|
@@ -18610,17 +18675,17 @@ function readPeopleRelationshipsSnapshot(snapshotDir, timeRangePreset, options2
|
|
|
18610
18675
|
}
|
|
18611
18676
|
}
|
|
18612
18677
|
function writePeopleRelationshipsSnapshot(snapshotDir, snapshot) {
|
|
18613
|
-
if (!
|
|
18614
|
-
const tmpPath =
|
|
18615
|
-
|
|
18616
|
-
|
|
18678
|
+
if (!fs30.existsSync(snapshotDir)) fs30.mkdirSync(snapshotDir, { recursive: true });
|
|
18679
|
+
const tmpPath = path25.join(snapshotDir, `${PEOPLE_RELATIONSHIPS_SNAPSHOT_TMP_PREFIX}${process.pid}-${Date.now()}`);
|
|
18680
|
+
fs30.writeFileSync(tmpPath, JSON.stringify(snapshot, null, 2), "utf-8");
|
|
18681
|
+
fs30.renameSync(tmpPath, getPeopleRelationshipsSnapshotPath(snapshotDir, snapshot.timeRange.preset));
|
|
18617
18682
|
}
|
|
18618
18683
|
function cleanupPeopleRelationshipsSnapshotTempFiles(snapshotDir) {
|
|
18619
|
-
if (!
|
|
18620
|
-
for (const name of
|
|
18684
|
+
if (!fs30.existsSync(snapshotDir)) return;
|
|
18685
|
+
for (const name of fs30.readdirSync(snapshotDir)) {
|
|
18621
18686
|
if (!name.startsWith(PEOPLE_RELATIONSHIPS_SNAPSHOT_TMP_PREFIX)) continue;
|
|
18622
18687
|
try {
|
|
18623
|
-
|
|
18688
|
+
fs30.rmSync(path25.join(snapshotDir, name), { force: true });
|
|
18624
18689
|
} catch (error) {
|
|
18625
18690
|
appLogger.warn("people-relationships", "failed to remove people relationships snapshot temp file", error);
|
|
18626
18691
|
}
|
|
@@ -18628,9 +18693,9 @@ function cleanupPeopleRelationshipsSnapshotTempFiles(snapshotDir) {
|
|
|
18628
18693
|
}
|
|
18629
18694
|
|
|
18630
18695
|
// ../../packages/node-runtime/src/services/people/relationships/worker-runner.ts
|
|
18631
|
-
import { existsSync as
|
|
18696
|
+
import { existsSync as existsSync9 } from "fs";
|
|
18632
18697
|
import { Worker as Worker2 } from "worker_threads";
|
|
18633
|
-
function resolveDefaultPeopleRelationshipsWorkerEntryUrl(currentModuleUrl = import.meta.url, entryExists = (url) =>
|
|
18698
|
+
function resolveDefaultPeopleRelationshipsWorkerEntryUrl(currentModuleUrl = import.meta.url, entryExists = (url) => existsSync9(url)) {
|
|
18634
18699
|
const moduleUrl = typeof currentModuleUrl === "string" ? currentModuleUrl : currentModuleUrl.href;
|
|
18635
18700
|
if (moduleUrl.endsWith(".ts")) return new URL("./worker-entry.ts", moduleUrl);
|
|
18636
18701
|
if (moduleUrl.endsWith(".mjs")) return new URL("./people-relationships-worker.mjs", moduleUrl);
|
|
@@ -19155,7 +19220,7 @@ __export(relationships_exports, {
|
|
|
19155
19220
|
});
|
|
19156
19221
|
|
|
19157
19222
|
// ../../packages/node-runtime/src/services/global-insight/service.ts
|
|
19158
|
-
import
|
|
19223
|
+
import fs31 from "fs";
|
|
19159
19224
|
|
|
19160
19225
|
// ../../packages/node-runtime/src/services/global-insight/signature.ts
|
|
19161
19226
|
function buildAnnualSummarySignature(adapter, range) {
|
|
@@ -19172,9 +19237,9 @@ function buildAnnualSummarySignature(adapter, range) {
|
|
|
19172
19237
|
}
|
|
19173
19238
|
|
|
19174
19239
|
// ../../packages/node-runtime/src/services/global-insight/worker-runner.ts
|
|
19175
|
-
import { existsSync as
|
|
19240
|
+
import { existsSync as existsSync10 } from "fs";
|
|
19176
19241
|
import { Worker as Worker3 } from "worker_threads";
|
|
19177
|
-
function resolveDefaultAnnualSummaryWorkerEntryUrl(currentModuleUrl = import.meta.url, entryExists = (url) =>
|
|
19242
|
+
function resolveDefaultAnnualSummaryWorkerEntryUrl(currentModuleUrl = import.meta.url, entryExists = (url) => existsSync10(url)) {
|
|
19178
19243
|
const moduleUrl = typeof currentModuleUrl === "string" ? currentModuleUrl : currentModuleUrl.href;
|
|
19179
19244
|
if (moduleUrl.endsWith(".ts")) return new URL("./worker-entry.ts", moduleUrl);
|
|
19180
19245
|
if (moduleUrl.endsWith(".mjs")) return new URL("./global-insight-worker.mjs", moduleUrl);
|
|
@@ -19409,7 +19474,7 @@ var DefaultGlobalInsightService = class {
|
|
|
19409
19474
|
}
|
|
19410
19475
|
getSnapshot(range) {
|
|
19411
19476
|
const key = toAnnualSummaryRangeKey(range);
|
|
19412
|
-
if (this.snapshots.has(key) && !
|
|
19477
|
+
if (this.snapshots.has(key) && !fs31.existsSync(getAnnualSummarySnapshotPath(this.snapshotDir, range))) {
|
|
19413
19478
|
this.snapshots.delete(key);
|
|
19414
19479
|
}
|
|
19415
19480
|
if (!this.snapshots.has(key)) {
|
|
@@ -19457,8 +19522,8 @@ __export(global_insight_exports, {
|
|
|
19457
19522
|
});
|
|
19458
19523
|
|
|
19459
19524
|
// ../../packages/node-runtime/src/semantic-index/config.ts
|
|
19460
|
-
import
|
|
19461
|
-
import
|
|
19525
|
+
import fs32 from "fs";
|
|
19526
|
+
import path26 from "path";
|
|
19462
19527
|
var DEFAULT_SEMANTIC_INDEX_MODEL_DOWNLOAD_SOURCE = "huggingface";
|
|
19463
19528
|
var SEMANTIC_INDEX_CONFIG_VERSION = 1;
|
|
19464
19529
|
var SEARCH_MAX_RESULTS_DEFAULT = 10;
|
|
@@ -19547,9 +19612,9 @@ var SemanticIndexConfigStore = class {
|
|
|
19547
19612
|
return this.set({ ...this.get(), enabled });
|
|
19548
19613
|
}
|
|
19549
19614
|
get() {
|
|
19550
|
-
if (!
|
|
19615
|
+
if (!fs32.existsSync(this.filePath)) return defaultSemanticIndexConfig();
|
|
19551
19616
|
try {
|
|
19552
|
-
const raw = JSON.parse(
|
|
19617
|
+
const raw = JSON.parse(fs32.readFileSync(this.filePath, "utf-8"));
|
|
19553
19618
|
return normalizeConfig(raw);
|
|
19554
19619
|
} catch {
|
|
19555
19620
|
return defaultSemanticIndexConfig();
|
|
@@ -19557,9 +19622,9 @@ var SemanticIndexConfigStore = class {
|
|
|
19557
19622
|
}
|
|
19558
19623
|
set(config) {
|
|
19559
19624
|
const normalized = normalizeConfig(config);
|
|
19560
|
-
const dir =
|
|
19561
|
-
if (!
|
|
19562
|
-
|
|
19625
|
+
const dir = path26.dirname(this.filePath);
|
|
19626
|
+
if (!fs32.existsSync(dir)) fs32.mkdirSync(dir, { recursive: true });
|
|
19627
|
+
fs32.writeFileSync(this.filePath, JSON.stringify(normalized, null, 2), "utf-8");
|
|
19563
19628
|
return normalized;
|
|
19564
19629
|
}
|
|
19565
19630
|
resolveModelId() {
|
|
@@ -19687,10 +19752,10 @@ var LocalEmbeddingProvider = class {
|
|
|
19687
19752
|
};
|
|
19688
19753
|
|
|
19689
19754
|
// ../../packages/node-runtime/src/semantic-index/service.ts
|
|
19690
|
-
import
|
|
19755
|
+
import path27 from "path";
|
|
19691
19756
|
|
|
19692
19757
|
// ../../packages/node-runtime/src/semantic-index/chunker-config.ts
|
|
19693
|
-
import { createHash as
|
|
19758
|
+
import { createHash as createHash2 } from "crypto";
|
|
19694
19759
|
var CHUNKER_VERSION = "v1.3";
|
|
19695
19760
|
var STRATEGY_ID = "balanced";
|
|
19696
19761
|
var DEFAULT_CHUNKER_CONFIG = {
|
|
@@ -19705,7 +19770,7 @@ var DEFAULT_CHUNKER_CONFIG = {
|
|
|
19705
19770
|
semanticVoidSkipThreshold: 20
|
|
19706
19771
|
};
|
|
19707
19772
|
function sha256Hex(input) {
|
|
19708
|
-
return
|
|
19773
|
+
return createHash2("sha256").update(input, "utf-8").digest("hex");
|
|
19709
19774
|
}
|
|
19710
19775
|
function computeChunkerConfigHash(config) {
|
|
19711
19776
|
const canonical = JSON.stringify({
|
|
@@ -20097,32 +20162,6 @@ var EmbeddingIndexStore = class {
|
|
|
20097
20162
|
).all(toFloat32Buffer(params.embedding), params.dbPathHash, params.modelId, params.k);
|
|
20098
20163
|
return rows.map((row) => ({ chunkId: row.chunk_id, distance: row.distance, record: rowToRecord(row) }));
|
|
20099
20164
|
}
|
|
20100
|
-
/**
|
|
20101
|
-
* FTS message -> chunk 映射。按 (ts, id) 复合查找:取 (start_ts, start_message_id) <= (messageTs, messageId)
|
|
20102
|
-
* 的最大值,再校验 end 覆盖。
|
|
20103
|
-
* - 用 ts 而非 ID 做主键:回填旧消息后 ID 不再单调
|
|
20104
|
-
* - 用 (ts, id) 复合而非仅 ts:同秒多 chunk 时 ts 相同,需 id 打破 tie
|
|
20105
|
-
*/
|
|
20106
|
-
mapMessageToChunk(params) {
|
|
20107
|
-
const row = this.db.prepare(
|
|
20108
|
-
`SELECT ${CHUNK_COLUMNS} FROM chunk_vector_index
|
|
20109
|
-
WHERE db_path_hash = ? AND model_id = ? AND strategy_id = ?
|
|
20110
|
-
AND (start_ts < CAST(? AS INTEGER)
|
|
20111
|
-
OR (start_ts = CAST(? AS INTEGER) AND start_message_id <= CAST(? AS INTEGER)))
|
|
20112
|
-
ORDER BY start_ts DESC, start_message_id DESC LIMIT 1`
|
|
20113
|
-
).get(
|
|
20114
|
-
params.dbPathHash,
|
|
20115
|
-
params.modelId,
|
|
20116
|
-
params.strategyId,
|
|
20117
|
-
params.messageTs,
|
|
20118
|
-
params.messageTs,
|
|
20119
|
-
params.messageId
|
|
20120
|
-
);
|
|
20121
|
-
if (!row) return null;
|
|
20122
|
-
if (row.end_ts < params.messageTs) return null;
|
|
20123
|
-
if (row.end_ts === params.messageTs && row.end_message_id < params.messageId) return null;
|
|
20124
|
-
return rowToRecord(row);
|
|
20125
|
-
}
|
|
20126
20165
|
/** 删除某聊天库(全模型)的所有 chunk 元数据与向量,返回删除的元数据行数 */
|
|
20127
20166
|
deleteByDbPathHash(dbPathHash) {
|
|
20128
20167
|
return this.db.transaction(() => {
|
|
@@ -20419,7 +20458,7 @@ var SemanticIndexJobQueue = class {
|
|
|
20419
20458
|
};
|
|
20420
20459
|
|
|
20421
20460
|
// ../../packages/node-runtime/src/semantic-index/chunker.ts
|
|
20422
|
-
import { createHash as
|
|
20461
|
+
import { createHash as createHash3 } from "crypto";
|
|
20423
20462
|
var VOID_MESSAGE_TYPES = /* @__PURE__ */ new Set([
|
|
20424
20463
|
1 /* IMAGE */,
|
|
20425
20464
|
2 /* VOICE */,
|
|
@@ -20502,7 +20541,7 @@ function sumEffectiveChars(messages) {
|
|
|
20502
20541
|
return total;
|
|
20503
20542
|
}
|
|
20504
20543
|
function sha256Hex2(input) {
|
|
20505
|
-
return
|
|
20544
|
+
return createHash3("sha256").update(input, "utf-8").digest("hex");
|
|
20506
20545
|
}
|
|
20507
20546
|
var EMBEDDING_HEADER_BUDGET_RATIO = 0.25;
|
|
20508
20547
|
var SOURCE_TITLE_HEADER_SHARE = 1 / 3;
|
|
@@ -20773,21 +20812,7 @@ async function runWarmup(options2) {
|
|
|
20773
20812
|
}
|
|
20774
20813
|
}
|
|
20775
20814
|
|
|
20776
|
-
// ../../packages/node-runtime/src/semantic-index/retrieval/
|
|
20777
|
-
function reciprocalRankFusion(rankedLists, k = 60) {
|
|
20778
|
-
const scores = /* @__PURE__ */ new Map();
|
|
20779
|
-
const firstSeen = /* @__PURE__ */ new Map();
|
|
20780
|
-
let seq = 0;
|
|
20781
|
-
for (const list of rankedLists) {
|
|
20782
|
-
list.forEach((id, rank) => {
|
|
20783
|
-
scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank));
|
|
20784
|
-
if (!firstSeen.has(id)) firstSeen.set(id, seq++);
|
|
20785
|
-
});
|
|
20786
|
-
}
|
|
20787
|
-
return [...scores.entries()].map(([id, score]) => ({ id, score })).sort((a, b) => b.score - a.score || firstSeen.get(a.id) - firstSeen.get(b.id));
|
|
20788
|
-
}
|
|
20789
|
-
|
|
20790
|
-
// ../../packages/node-runtime/src/semantic-index/retrieval/hybrid-search.ts
|
|
20815
|
+
// ../../packages/node-runtime/src/semantic-index/retrieval/semantic-search.ts
|
|
20791
20816
|
function overlapsTimeRangeMs(record, filter) {
|
|
20792
20817
|
if (!filter) return true;
|
|
20793
20818
|
if (filter.startTs != null && record.endTs < filter.startTs) return false;
|
|
@@ -20795,46 +20820,22 @@ function overlapsTimeRangeMs(record, filter) {
|
|
|
20795
20820
|
return true;
|
|
20796
20821
|
}
|
|
20797
20822
|
var TIME_FILTER_POOL_MULTIPLIER = 10;
|
|
20798
|
-
async function
|
|
20799
|
-
const { embedder, store
|
|
20800
|
-
const { query, dbPathHash, modelId,
|
|
20823
|
+
async function semanticSearch(deps, params) {
|
|
20824
|
+
const { embedder, store } = deps;
|
|
20825
|
+
const { query, dbPathHash, modelId, dim, finalTopK = 5, timeRangeMs } = params;
|
|
20801
20826
|
if (!query.trim()) return [];
|
|
20802
20827
|
const poolFactor = timeRangeMs ? TIME_FILTER_POOL_MULTIPLIER : 1;
|
|
20803
20828
|
const denseTopN = Math.max((params.denseTopN ?? 40) * poolFactor, 40);
|
|
20804
|
-
const ftsTopN = Math.max((params.ftsTopN ?? 40) * poolFactor, 40);
|
|
20805
|
-
const records = /* @__PURE__ */ new Map();
|
|
20806
20829
|
const queryVector = await embedder.embedQuery(query);
|
|
20807
20830
|
const dense = store.queryDense({ dbPathHash, modelId, dim, embedding: queryVector, k: denseTopN });
|
|
20808
|
-
const
|
|
20809
|
-
const denseRankById = /* @__PURE__ */ new Map();
|
|
20831
|
+
const results = [];
|
|
20810
20832
|
for (const hit of dense) {
|
|
20811
20833
|
if (!overlapsTimeRangeMs(hit.record, timeRangeMs)) continue;
|
|
20812
|
-
denseRankById.set(hit.chunkId, denseIds.length);
|
|
20813
|
-
denseIds.push(hit.chunkId);
|
|
20814
|
-
records.set(hit.chunkId, hit.record);
|
|
20815
|
-
}
|
|
20816
|
-
const ftsIds = [];
|
|
20817
|
-
const ftsRankById = /* @__PURE__ */ new Map();
|
|
20818
|
-
for (const { id: messageId, ts: messageTs } of fts.search(query, ftsTopN)) {
|
|
20819
|
-
const record = store.mapMessageToChunk({ dbPathHash, modelId, strategyId, messageId, messageTs });
|
|
20820
|
-
if (!record || ftsRankById.has(record.chunkId)) continue;
|
|
20821
|
-
if (!overlapsTimeRangeMs(record, timeRangeMs)) continue;
|
|
20822
|
-
ftsRankById.set(record.chunkId, ftsIds.length);
|
|
20823
|
-
ftsIds.push(record.chunkId);
|
|
20824
|
-
if (!records.has(record.chunkId)) records.set(record.chunkId, record);
|
|
20825
|
-
}
|
|
20826
|
-
const fused = reciprocalRankFusion([denseIds, ftsIds], rrfK);
|
|
20827
|
-
const results = [];
|
|
20828
|
-
for (const { id, score } of fused) {
|
|
20829
|
-
const record = records.get(id) ?? store.getChunkById(id);
|
|
20830
|
-
if (!record) continue;
|
|
20831
|
-
if (!overlapsTimeRangeMs(record, timeRangeMs)) continue;
|
|
20832
20834
|
results.push({
|
|
20833
|
-
chunkId:
|
|
20834
|
-
score,
|
|
20835
|
-
record,
|
|
20836
|
-
denseRank:
|
|
20837
|
-
ftsRank: ftsRankById.get(id) ?? null
|
|
20835
|
+
chunkId: hit.chunkId,
|
|
20836
|
+
score: 1 / (1 + Math.max(0, hit.distance)),
|
|
20837
|
+
record: hit.record,
|
|
20838
|
+
denseRank: results.length
|
|
20838
20839
|
});
|
|
20839
20840
|
if (results.length >= finalTopK) break;
|
|
20840
20841
|
}
|
|
@@ -21014,35 +21015,6 @@ function createChatDbMessageRangeReader(db) {
|
|
|
21014
21015
|
};
|
|
21015
21016
|
}
|
|
21016
21017
|
|
|
21017
|
-
// ../../packages/node-runtime/src/semantic-index/chat-db/fts-searcher.ts
|
|
21018
|
-
function extractFtsKeywords(query) {
|
|
21019
|
-
const trimmed = query.trim();
|
|
21020
|
-
if (!trimmed) return [];
|
|
21021
|
-
try {
|
|
21022
|
-
const tokens = getJieba().cut(trimmed, false).map((t2) => t2.trim()).filter((t2) => t2.length > 0);
|
|
21023
|
-
return [...new Set(tokens)];
|
|
21024
|
-
} catch {
|
|
21025
|
-
return [...new Set(trimmed.split(/\s+/).filter((t2) => t2.length > 0))];
|
|
21026
|
-
}
|
|
21027
|
-
}
|
|
21028
|
-
function createChatDbFtsSearcher(db) {
|
|
21029
|
-
return {
|
|
21030
|
-
search(query, topN) {
|
|
21031
|
-
const keywords = extractFtsKeywords(query);
|
|
21032
|
-
if (keywords.length === 0) return [];
|
|
21033
|
-
const { rowids } = searchByFts(db, keywords, topN, 0);
|
|
21034
|
-
if (rowids.length === 0) return [];
|
|
21035
|
-
const placeholders = rowids.map(() => "?").join(", ");
|
|
21036
|
-
const tsRows = db.prepare(`SELECT id, ts FROM message WHERE id IN (${placeholders})`).all(...rowids);
|
|
21037
|
-
const tsById = new Map(tsRows.map((r) => [r.id, r.ts * 1e3]));
|
|
21038
|
-
return rowids.flatMap((id) => {
|
|
21039
|
-
const ts = tsById.get(id);
|
|
21040
|
-
return ts !== void 0 ? [{ id, ts }] : [];
|
|
21041
|
-
});
|
|
21042
|
-
}
|
|
21043
|
-
};
|
|
21044
|
-
}
|
|
21045
|
-
|
|
21046
21018
|
// ../../packages/node-runtime/src/semantic-index/service.ts
|
|
21047
21019
|
var SEMANTIC_INDEX_AUTH_PROFILE = "semantic-index-embedding";
|
|
21048
21020
|
function persistSemanticIndexConfig(configStore, config, options2) {
|
|
@@ -21064,16 +21036,16 @@ function resolveSearchToolEvidenceTokens(maxResultTokens) {
|
|
|
21064
21036
|
return maxResultTokens && maxResultTokens > 0 ? Math.min(SEARCH_TOOL_EVIDENCE_TOKENS, maxResultTokens) : SEARCH_TOOL_EVIDENCE_TOKENS;
|
|
21065
21037
|
}
|
|
21066
21038
|
function sessionIdFromDbPath(dbPath) {
|
|
21067
|
-
return
|
|
21039
|
+
return path27.basename(dbPath, ".db");
|
|
21068
21040
|
}
|
|
21069
21041
|
var SEMANTIC_INDEX_DB_FILE = "embedding_index.db";
|
|
21070
21042
|
var SEMANTIC_INDEX_CONFIG_FILE = "semantic-index-config.json";
|
|
21071
21043
|
function createSemanticIndexService(params) {
|
|
21072
21044
|
const { pathProvider } = params;
|
|
21073
21045
|
return new SemanticIndexService({
|
|
21074
|
-
vectorDbPath:
|
|
21075
|
-
configPath:
|
|
21076
|
-
modelsCacheDir:
|
|
21046
|
+
vectorDbPath: path27.join(pathProvider.getVectorDir(), SEMANTIC_INDEX_DB_FILE),
|
|
21047
|
+
configPath: path27.join(pathProvider.getAiDataDir(), SEMANTIC_INDEX_CONFIG_FILE),
|
|
21048
|
+
modelsCacheDir: path27.join(pathProvider.getAiDataDir(), "models", "semantic-index"),
|
|
21077
21049
|
modelDownloadProxyUrl: params.modelDownloadProxyUrl,
|
|
21078
21050
|
sessionAdapter: params.sessionAdapter,
|
|
21079
21051
|
nativeBinding: params.nativeBinding,
|
|
@@ -21363,20 +21335,17 @@ var SemanticIndexService = class {
|
|
|
21363
21335
|
const db = this.sessionAdapter.openReadonly(sessionId);
|
|
21364
21336
|
if (!db) return { available: false, reason: "not-found", blocks: [], coverage: 0, partial: false, hitCount: 0 };
|
|
21365
21337
|
const embedder = this.getEmbedder();
|
|
21366
|
-
const fts = createChatDbFtsSearcher(db);
|
|
21367
21338
|
const finalTopK = options2?.finalTopK ?? 5;
|
|
21368
21339
|
const candidateTopN = Math.max(40, finalTopK * 4);
|
|
21369
|
-
const hits = await
|
|
21370
|
-
{ embedder, store: this.store
|
|
21340
|
+
const hits = await semanticSearch(
|
|
21341
|
+
{ embedder, store: this.store },
|
|
21371
21342
|
{
|
|
21372
21343
|
query,
|
|
21373
21344
|
dbPathHash: hash,
|
|
21374
21345
|
modelId,
|
|
21375
|
-
strategyId: STRATEGY_ID,
|
|
21376
21346
|
dim,
|
|
21377
21347
|
finalTopK,
|
|
21378
21348
|
denseTopN: candidateTopN,
|
|
21379
|
-
ftsTopN: candidateTopN,
|
|
21380
21349
|
timeRangeMs: options2?.timeRangeMs
|
|
21381
21350
|
}
|
|
21382
21351
|
);
|
|
@@ -21635,9 +21604,9 @@ ${text}`;
|
|
|
21635
21604
|
|
|
21636
21605
|
// ../../packages/node-runtime/src/semantic-index/embedding/local-runtime.ts
|
|
21637
21606
|
import { execFile } from "child_process";
|
|
21638
|
-
import
|
|
21607
|
+
import fs33 from "fs";
|
|
21639
21608
|
import { createRequire as createRequire2 } from "module";
|
|
21640
|
-
import
|
|
21609
|
+
import path28 from "path";
|
|
21641
21610
|
import { pathToFileURL } from "url";
|
|
21642
21611
|
var LOCAL_EMBEDDING_RUNTIME_PACKAGE = "@huggingface/transformers";
|
|
21643
21612
|
var LOCAL_EMBEDDING_RUNTIME_VERSION = "4.2.0";
|
|
@@ -21705,7 +21674,7 @@ function verifyRuntimeInSubprocess(runtimeDir) {
|
|
|
21705
21674
|
});
|
|
21706
21675
|
}
|
|
21707
21676
|
function createRuntimeRequire(runtimeDir) {
|
|
21708
|
-
return createRequire2(
|
|
21677
|
+
return createRequire2(path28.join(runtimeDir, "package.json"));
|
|
21709
21678
|
}
|
|
21710
21679
|
async function importFromRuntime(runtimeDir, packageName) {
|
|
21711
21680
|
const resolved = createRuntimeRequire(runtimeDir).resolve(packageName);
|
|
@@ -21757,7 +21726,7 @@ var LocalEmbeddingRuntimeManager = class {
|
|
|
21757
21726
|
return this.status;
|
|
21758
21727
|
}
|
|
21759
21728
|
getRuntimeDir() {
|
|
21760
|
-
return
|
|
21729
|
+
return path28.join(this.baseDir, `transformers-${LOCAL_EMBEDDING_RUNTIME_VERSION}`, `${this.platform}-${this.arch}`);
|
|
21761
21730
|
}
|
|
21762
21731
|
async ensureInstalled() {
|
|
21763
21732
|
if (!this.installPromise) {
|
|
@@ -21792,7 +21761,7 @@ var LocalEmbeddingRuntimeManager = class {
|
|
|
21792
21761
|
arch: this.arch,
|
|
21793
21762
|
error: error instanceof Error ? error.message : String(error)
|
|
21794
21763
|
});
|
|
21795
|
-
|
|
21764
|
+
fs33.rmSync(runtimeDir, { recursive: true, force: true });
|
|
21796
21765
|
this.verifiedRuntimeDir = null;
|
|
21797
21766
|
}
|
|
21798
21767
|
}
|
|
@@ -21821,12 +21790,12 @@ var LocalEmbeddingRuntimeManager = class {
|
|
|
21821
21790
|
}
|
|
21822
21791
|
}
|
|
21823
21792
|
async installAtomically(runtimeDir) {
|
|
21824
|
-
const parentDir =
|
|
21825
|
-
|
|
21826
|
-
const stagingDir =
|
|
21793
|
+
const parentDir = path28.dirname(runtimeDir);
|
|
21794
|
+
fs33.mkdirSync(parentDir, { recursive: true });
|
|
21795
|
+
const stagingDir = fs33.mkdtempSync(path28.join(parentDir, ".installing-"));
|
|
21827
21796
|
try {
|
|
21828
|
-
|
|
21829
|
-
|
|
21797
|
+
fs33.writeFileSync(
|
|
21798
|
+
path28.join(stagingDir, "package.json"),
|
|
21830
21799
|
`${JSON.stringify({ private: true, type: "module" }, null, 2)}
|
|
21831
21800
|
`,
|
|
21832
21801
|
"utf8"
|
|
@@ -21847,19 +21816,19 @@ var LocalEmbeddingRuntimeManager = class {
|
|
|
21847
21816
|
cwd: stagingDir
|
|
21848
21817
|
});
|
|
21849
21818
|
await this.verifyStagedRuntime(stagingDir);
|
|
21850
|
-
|
|
21819
|
+
fs33.writeFileSync(path28.join(stagingDir, RUNTIME_MANIFEST_FILE), `${JSON.stringify(this.manifest(), null, 2)}
|
|
21851
21820
|
`);
|
|
21852
|
-
if (
|
|
21821
|
+
if (fs33.existsSync(runtimeDir)) {
|
|
21853
21822
|
if (this.hasCompleteLayout(runtimeDir)) return;
|
|
21854
|
-
|
|
21823
|
+
fs33.rmSync(runtimeDir, { recursive: true, force: true });
|
|
21855
21824
|
}
|
|
21856
21825
|
try {
|
|
21857
|
-
|
|
21826
|
+
fs33.renameSync(stagingDir, runtimeDir);
|
|
21858
21827
|
} catch (error) {
|
|
21859
21828
|
if (!this.hasCompleteLayout(runtimeDir)) throw error;
|
|
21860
21829
|
}
|
|
21861
21830
|
} finally {
|
|
21862
|
-
|
|
21831
|
+
fs33.rmSync(stagingDir, { recursive: true, force: true });
|
|
21863
21832
|
}
|
|
21864
21833
|
}
|
|
21865
21834
|
async verifyOnce(runtimeDir) {
|
|
@@ -21868,11 +21837,11 @@ var LocalEmbeddingRuntimeManager = class {
|
|
|
21868
21837
|
this.verifiedRuntimeDir = runtimeDir;
|
|
21869
21838
|
}
|
|
21870
21839
|
hasCompleteLayout(runtimeDir) {
|
|
21871
|
-
const manifestPath =
|
|
21872
|
-
const packagePath =
|
|
21873
|
-
if (!
|
|
21840
|
+
const manifestPath = path28.join(runtimeDir, RUNTIME_MANIFEST_FILE);
|
|
21841
|
+
const packagePath = path28.join(runtimeDir, "node_modules", "@huggingface", "transformers", "package.json");
|
|
21842
|
+
if (!fs33.existsSync(manifestPath) || !fs33.existsSync(packagePath)) return false;
|
|
21874
21843
|
try {
|
|
21875
|
-
const manifest = JSON.parse(
|
|
21844
|
+
const manifest = JSON.parse(fs33.readFileSync(manifestPath, "utf8"));
|
|
21876
21845
|
const expected = this.manifest();
|
|
21877
21846
|
return manifest.formatVersion === expected.formatVersion && manifest.packageName === expected.packageName && manifest.packageVersion === expected.packageVersion && manifest.platform === expected.platform && manifest.arch === expected.arch;
|
|
21878
21847
|
} catch {
|
|
@@ -21894,8 +21863,8 @@ function createLocalEmbeddingRuntimeManager(options2) {
|
|
|
21894
21863
|
}
|
|
21895
21864
|
|
|
21896
21865
|
// ../../packages/node-runtime/src/services/merge-cache.ts
|
|
21897
|
-
import * as
|
|
21898
|
-
import * as
|
|
21866
|
+
import * as fs34 from "fs";
|
|
21867
|
+
import * as path29 from "path";
|
|
21899
21868
|
import * as crypto from "crypto";
|
|
21900
21869
|
var MergeSessionCache = class {
|
|
21901
21870
|
cache = /* @__PURE__ */ new Map();
|
|
@@ -21904,15 +21873,15 @@ var MergeSessionCache = class {
|
|
|
21904
21873
|
constructor(pathProvider, options2) {
|
|
21905
21874
|
this.tempDir = pathProvider.getTempDir();
|
|
21906
21875
|
this.nativeBinding = options2?.nativeBinding;
|
|
21907
|
-
if (!
|
|
21908
|
-
|
|
21876
|
+
if (!fs34.existsSync(this.tempDir)) {
|
|
21877
|
+
fs34.mkdirSync(this.tempDir, { recursive: true });
|
|
21909
21878
|
}
|
|
21910
21879
|
}
|
|
21911
21880
|
generateTempDbPath(filename) {
|
|
21912
21881
|
const ts = Date.now();
|
|
21913
21882
|
const rand = Math.random().toString(36).slice(2, 8);
|
|
21914
21883
|
const safeName = filename.replace(/[/\\?%*:|"<>]/g, "_").slice(0, 40);
|
|
21915
|
-
return
|
|
21884
|
+
return path29.join(this.tempDir, `merge_${safeName}_${ts}_${rand}.db`);
|
|
21916
21885
|
}
|
|
21917
21886
|
/** Store a parsed file's temp DB and return a handle for subsequent requests. */
|
|
21918
21887
|
store(filename, tempDbPath) {
|
|
@@ -21923,7 +21892,7 @@ var MergeSessionCache = class {
|
|
|
21923
21892
|
/** Open a TempDbReader for a given handle. Caller must close the reader. */
|
|
21924
21893
|
openReader(handle) {
|
|
21925
21894
|
const entry = this.cache.get(handle);
|
|
21926
|
-
if (!entry || !
|
|
21895
|
+
if (!entry || !fs34.existsSync(entry.tempDbPath)) return null;
|
|
21927
21896
|
const adapter = openBetterSqliteDatabase(entry.tempDbPath, {
|
|
21928
21897
|
readonly: true,
|
|
21929
21898
|
nativeBinding: this.nativeBinding
|
|
@@ -21968,7 +21937,7 @@ var MergeSessionCache = class {
|
|
|
21968
21937
|
};
|
|
21969
21938
|
|
|
21970
21939
|
// ../../packages/node-runtime/src/services/push-importer.ts
|
|
21971
|
-
import * as
|
|
21940
|
+
import * as fs35 from "fs";
|
|
21972
21941
|
var SYSTEM_SENDER_ID3 = "SYSTEM";
|
|
21973
21942
|
var SYSTEM_MEMBER_NAME2 = "\u7CFB\u7EDF\u6D88\u606F";
|
|
21974
21943
|
function isRecord5(value) {
|
|
@@ -22118,7 +22087,6 @@ function writeMessages(db, messages, dedupState) {
|
|
|
22118
22087
|
let duplicateCount = 0;
|
|
22119
22088
|
let membersAdded = 0;
|
|
22120
22089
|
let minWrittenTs = Infinity;
|
|
22121
|
-
const ftsEntries = [];
|
|
22122
22090
|
const sorted = [...messages].sort((a, b) => a.timestamp - b.timestamp);
|
|
22123
22091
|
db.transaction(() => {
|
|
22124
22092
|
for (const msg of sorted) {
|
|
@@ -22141,7 +22109,7 @@ function writeMessages(db, messages, dedupState) {
|
|
|
22141
22109
|
}
|
|
22142
22110
|
}
|
|
22143
22111
|
if (!memberId) continue;
|
|
22144
|
-
|
|
22112
|
+
insertMsg.run(
|
|
22145
22113
|
memberId,
|
|
22146
22114
|
normalizeSenderAccountName(msg.sender, msg.accountName) || null,
|
|
22147
22115
|
normalizeSenderGroupNickname(msg.sender, msg.groupNickname) || null,
|
|
@@ -22151,12 +22119,11 @@ function writeMessages(db, messages, dedupState) {
|
|
|
22151
22119
|
msg.replyToMessageId || null,
|
|
22152
22120
|
msg.platformMessageId || null
|
|
22153
22121
|
);
|
|
22154
|
-
ftsEntries.push({ id: Number(result.lastInsertRowid), content: msg.content ?? null });
|
|
22155
22122
|
if (msg.timestamp < minWrittenTs) minWrittenTs = msg.timestamp;
|
|
22156
22123
|
writtenCount++;
|
|
22157
22124
|
}
|
|
22158
22125
|
});
|
|
22159
|
-
return { writtenCount, duplicateCount, membersAdded, minWrittenTs
|
|
22126
|
+
return { writtenCount, duplicateCount, membersAdded, minWrittenTs };
|
|
22160
22127
|
}
|
|
22161
22128
|
function fullImport(db, meta, members, messages) {
|
|
22162
22129
|
db.exec(CHAT_DB_SCHEMA);
|
|
@@ -22202,7 +22169,6 @@ function fullImport(db, meta, members, messages) {
|
|
|
22202
22169
|
replyToMessageId: m.replyToMessageId
|
|
22203
22170
|
}))
|
|
22204
22171
|
);
|
|
22205
|
-
buildFtsIndex(db);
|
|
22206
22172
|
try {
|
|
22207
22173
|
generateSessionIndex(db);
|
|
22208
22174
|
} catch {
|
|
@@ -22263,15 +22229,8 @@ function writeIncrementalImport(db, payload) {
|
|
|
22263
22229
|
writtenCount,
|
|
22264
22230
|
duplicateCount,
|
|
22265
22231
|
membersAdded: autoCreatedMembersAdded,
|
|
22266
|
-
minWrittenTs
|
|
22267
|
-
ftsEntries
|
|
22232
|
+
minWrittenTs
|
|
22268
22233
|
} = writeMessages(db, payload.messages, dedupState);
|
|
22269
|
-
if (ftsEntries.length > 0) {
|
|
22270
|
-
try {
|
|
22271
|
-
insertFtsEntries(db, ftsEntries);
|
|
22272
|
-
} catch {
|
|
22273
|
-
}
|
|
22274
|
-
}
|
|
22275
22234
|
if (writtenCount > 0) {
|
|
22276
22235
|
try {
|
|
22277
22236
|
const sessionGapThreshold = getSessionIndexStats(db).gapThreshold;
|
|
@@ -22336,7 +22295,7 @@ async function executePushImportUnlocked(deps, sessionId, payload) {
|
|
|
22336
22295
|
return { ok: false, reason: "invalid_payload", message: "sessionId contains invalid characters" };
|
|
22337
22296
|
}
|
|
22338
22297
|
const dbPath = deps.getDbPath(sessionId);
|
|
22339
|
-
const isNew = !
|
|
22298
|
+
const isNew = !fs35.existsSync(dbPath);
|
|
22340
22299
|
const validationError = validatePayload(payload, isNew);
|
|
22341
22300
|
if (validationError) {
|
|
22342
22301
|
return { ok: false, reason: "invalid_payload", message: validationError };
|
|
@@ -22397,10 +22356,10 @@ async function executePushImportUnlocked(deps, sessionId, payload) {
|
|
|
22397
22356
|
}
|
|
22398
22357
|
|
|
22399
22358
|
// ../../packages/node-runtime/src/services/import-idempotency.ts
|
|
22400
|
-
import { createHash as
|
|
22359
|
+
import { createHash as createHash4 } from "crypto";
|
|
22401
22360
|
var DEFAULT_IMPORT_IDEMPOTENCY_TTL_MS = 60 * 60 * 1e3;
|
|
22402
22361
|
function hashImportBody(body) {
|
|
22403
|
-
return
|
|
22362
|
+
return createHash4("sha256").update(JSON.stringify(body ?? "")).digest("hex");
|
|
22404
22363
|
}
|
|
22405
22364
|
var ImportIdempotencyCache = class {
|
|
22406
22365
|
constructor(ttlMs = DEFAULT_IMPORT_IDEMPOTENCY_TTL_MS) {
|
|
@@ -22442,6 +22401,7 @@ export {
|
|
|
22442
22401
|
getChatLabTempScopeDir,
|
|
22443
22402
|
createChatLabTempDir,
|
|
22444
22403
|
removeChatLabTempDir,
|
|
22404
|
+
PARSER_FORMAT_IDS,
|
|
22445
22405
|
validateChatLabFile,
|
|
22446
22406
|
rebaseChatLabDemoDocuments,
|
|
22447
22407
|
detectFormat,
|
|
@@ -22462,22 +22422,11 @@ export {
|
|
|
22462
22422
|
logInfo,
|
|
22463
22423
|
getErrorCount,
|
|
22464
22424
|
logSummary,
|
|
22465
|
-
computeWordFrequency,
|
|
22466
|
-
segmentText,
|
|
22467
|
-
isDictDownloaded,
|
|
22468
|
-
getDictList,
|
|
22469
|
-
loadDictBuffer,
|
|
22470
|
-
downloadDict,
|
|
22471
|
-
deleteDict,
|
|
22472
|
-
ensureDefaultDict,
|
|
22473
|
-
hasFtsTable,
|
|
22474
|
-
createFtsTable,
|
|
22475
|
-
buildFtsIndex,
|
|
22476
|
-
rebuildFtsIndex,
|
|
22477
|
-
insertFtsEntries,
|
|
22478
|
-
searchByFts,
|
|
22479
22425
|
MESSAGE_INSERT_MAX_ROWS,
|
|
22480
22426
|
MessageBatchInserter,
|
|
22427
|
+
generateFallbackMessageKey,
|
|
22428
|
+
createMessageDedupState,
|
|
22429
|
+
registerMessageAndCheckDuplicate,
|
|
22481
22430
|
streamingImport,
|
|
22482
22431
|
analyzeNewImport,
|
|
22483
22432
|
streamParseFileInfo,
|
|
@@ -22531,6 +22480,7 @@ export {
|
|
|
22531
22480
|
CHART_SCHEMA_REQUIRED_MESSAGE,
|
|
22532
22481
|
createChartSchemaGateState,
|
|
22533
22482
|
wrapWithChartSchemaGate,
|
|
22483
|
+
DEFAULT_CONTEXT_COMPRESSION_CONFIG,
|
|
22534
22484
|
checkAndCompress,
|
|
22535
22485
|
manualCompress,
|
|
22536
22486
|
createCompressionLlmAdapter,
|
|
@@ -22651,14 +22601,11 @@ export {
|
|
|
22651
22601
|
isSemanticVoid,
|
|
22652
22602
|
chunkMessages,
|
|
22653
22603
|
runWarmup,
|
|
22654
|
-
|
|
22655
|
-
hybridSearch,
|
|
22604
|
+
semanticSearch,
|
|
22656
22605
|
formatEvidenceMessages,
|
|
22657
22606
|
assembleEvidence,
|
|
22658
22607
|
createChatDbMessageSource,
|
|
22659
22608
|
createChatDbMessageRangeReader,
|
|
22660
|
-
extractFtsKeywords,
|
|
22661
|
-
createChatDbFtsSearcher,
|
|
22662
22609
|
SEMANTIC_INDEX_AUTH_PROFILE,
|
|
22663
22610
|
persistSemanticIndexConfig,
|
|
22664
22611
|
resolveSemanticIndexApiKeySet,
|
|
@@ -22676,4 +22623,4 @@ export {
|
|
|
22676
22623
|
hashImportBody,
|
|
22677
22624
|
ImportIdempotencyCache
|
|
22678
22625
|
};
|
|
22679
|
-
//# sourceMappingURL=chunk-
|
|
22626
|
+
//# sourceMappingURL=chunk-4GWWKHWA.mjs.map
|