@ppagent/memory 0.1.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1187 -0
- package/dist/index.d.ts +397 -137
- package/dist/index.js +1372 -853
- package/dist/provider.resolver-2KS2YYNV.js +616 -0
- package/llms.txt +150 -87
- package/package.json +19 -7
package/dist/index.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { v4 as uuidv44 } from "uuid";
|
|
3
3
|
|
|
4
4
|
// src/config.ts
|
|
5
|
+
import * as path from "node:path";
|
|
5
6
|
function resolveConfig(config) {
|
|
6
7
|
let embeddingBaseUrl = config.embeddingBaseUrl;
|
|
7
8
|
let embeddingApiKey = config.embeddingApiKey;
|
|
@@ -15,16 +16,46 @@ function resolveConfig(config) {
|
|
|
15
16
|
console.warn("[MemoryConfig] embeddingApiKey \u672A\u914D\u7F6E\uFF0C\u81EA\u52A8\u56DE\u9000\u4F7F\u7528 llmApiKey\u3002");
|
|
16
17
|
embeddingApiKey = config.llmApiKey;
|
|
17
18
|
}
|
|
19
|
+
const compressedContextTokenLimit = positiveInt(
|
|
20
|
+
config.compressedContextTokenLimit ?? config.historyWindowTokenLimit ?? 16 * 1024,
|
|
21
|
+
"compressedContextTokenLimit"
|
|
22
|
+
);
|
|
23
|
+
if (compressedContextTokenLimit > 32 * 1024) {
|
|
24
|
+
console.warn(
|
|
25
|
+
`[MemoryConfig] compressedContextTokenLimit=${compressedContextTokenLimit} \u8D85\u8FC7 32K\uFF0C\u4F1A\u5360\u7528\u8F83\u591A\u6A21\u578B\u4E0A\u4E0B\u6587\u5E76\u589E\u52A0\u5E38\u9A7B\u5185\u5B58\u3002`
|
|
26
|
+
);
|
|
27
|
+
}
|
|
18
28
|
return {
|
|
19
29
|
...config,
|
|
20
30
|
embeddingBaseUrl,
|
|
21
31
|
embeddingApiKey,
|
|
32
|
+
provider: config.provider ?? "auto",
|
|
33
|
+
sqlitePath: config.sqlitePath ?? path.join(path.dirname(config.lancedbPath), "memory.sqlite3"),
|
|
22
34
|
sessionTokenLimit: config.sessionTokenLimit ?? 16386,
|
|
23
|
-
historyWindowTokenLimit: config.historyWindowTokenLimit ??
|
|
35
|
+
historyWindowTokenLimit: config.historyWindowTokenLimit ?? compressedContextTokenLimit,
|
|
24
36
|
topicRatio: config.topicRatio ?? [1, 5, 20],
|
|
25
37
|
detailMaxTokens: config.detailMaxTokens ?? 2048,
|
|
26
38
|
summaryMaxTokens: config.summaryMaxTokens ?? 512,
|
|
27
39
|
conciseMaxTokens: config.conciseMaxTokens ?? 128,
|
|
40
|
+
compressedContextTokenLimit,
|
|
41
|
+
contextUsageRatio: ratio(config.contextUsageRatio ?? 0.75, "contextUsageRatio"),
|
|
42
|
+
precompressionRatio: ratio(config.precompressionRatio ?? 0.75, "precompressionRatio"),
|
|
43
|
+
compressionBatchRatio: ratio(config.compressionBatchRatio ?? 0.5, "compressionBatchRatio"),
|
|
44
|
+
compressionBatchTokenLimit: Math.max(0, Math.floor(config.compressionBatchTokenLimit ?? 0)),
|
|
45
|
+
topicSummaryMaxTokens: positiveInt(
|
|
46
|
+
config.topicSummaryMaxTokens ?? config.detailMaxTokens ?? 2048,
|
|
47
|
+
"topicSummaryMaxTokens"
|
|
48
|
+
),
|
|
49
|
+
defaultModelContextTokens: positiveInt(
|
|
50
|
+
config.defaultModelContextTokens ?? 256 * 1024,
|
|
51
|
+
"defaultModelContextTokens"
|
|
52
|
+
),
|
|
53
|
+
maxHistoryAgeMs: Math.max(0, Math.floor(config.maxHistoryAgeMs ?? 0)),
|
|
54
|
+
sessionIdleTtlMs: Math.floor(config.sessionIdleTtlMs ?? 30 * 6e4),
|
|
55
|
+
sessionSweepIntervalMs: positiveInt(
|
|
56
|
+
config.sessionSweepIntervalMs ?? 6e4,
|
|
57
|
+
"sessionSweepIntervalMs"
|
|
58
|
+
),
|
|
28
59
|
httpTimeoutMs: config.httpTimeoutMs ?? 6e4,
|
|
29
60
|
httpMaxRetries: config.httpMaxRetries ?? 2,
|
|
30
61
|
embeddingBatchSize: config.embeddingBatchSize ?? 20,
|
|
@@ -52,6 +83,18 @@ function resolveConfig(config) {
|
|
|
52
83
|
restoreConcurrency: config.restoreConcurrency ?? 8
|
|
53
84
|
};
|
|
54
85
|
}
|
|
86
|
+
function ratio(value, name) {
|
|
87
|
+
if (!Number.isFinite(value) || value <= 0 || value > 1) {
|
|
88
|
+
throw new Error(`[MemoryConfig] ${name} \u5FC5\u987B\u5728 (0, 1] \u8303\u56F4\u5185`);
|
|
89
|
+
}
|
|
90
|
+
return value;
|
|
91
|
+
}
|
|
92
|
+
function positiveInt(value, name) {
|
|
93
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
94
|
+
throw new Error(`[MemoryConfig] ${name} \u5FC5\u987B\u662F\u6B63\u6570`);
|
|
95
|
+
}
|
|
96
|
+
return Math.floor(value);
|
|
97
|
+
}
|
|
55
98
|
|
|
56
99
|
// src/constants.ts
|
|
57
100
|
var DEFAULT_NODE_TYPES = [
|
|
@@ -711,120 +754,155 @@ var GrafeoService = class _GrafeoService {
|
|
|
711
754
|
}
|
|
712
755
|
};
|
|
713
756
|
|
|
714
|
-
// src/db/
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
// src/db/sql.util.ts
|
|
719
|
-
function escLance(value) {
|
|
720
|
-
return String(value).replace(/'/g, "''");
|
|
721
|
-
}
|
|
722
|
-
function eqFilter(field, value) {
|
|
723
|
-
return `${field} = '${escLance(value)}'`;
|
|
757
|
+
// src/db/store.types.ts
|
|
758
|
+
function eq(field, value) {
|
|
759
|
+
return { op: "eq", field, value };
|
|
724
760
|
}
|
|
725
761
|
|
|
726
|
-
// src/db/
|
|
762
|
+
// src/db/memory.store.ts
|
|
727
763
|
var MESSAGES_TABLE = "messages";
|
|
728
764
|
var TOPICS_TABLE = "topics";
|
|
729
765
|
var FACTS_TABLE = "facts";
|
|
730
766
|
var SESSIONS_TABLE = "sessions";
|
|
731
767
|
var DOCUMENTS_TABLE = "documents";
|
|
732
768
|
var CHUNKS_TABLE = "chunks";
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
}
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
}
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
}
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
769
|
+
var RRF_K = 60;
|
|
770
|
+
var HYBRID_OVERFETCH = 2;
|
|
771
|
+
function tableDefs(dim) {
|
|
772
|
+
return [
|
|
773
|
+
{
|
|
774
|
+
name: MESSAGES_TABLE,
|
|
775
|
+
vectorDimension: dim,
|
|
776
|
+
columns: [
|
|
777
|
+
{ name: "message_id", type: "text" },
|
|
778
|
+
{ name: "talker_id", type: "text" },
|
|
779
|
+
{ name: "chat_id", type: "text" },
|
|
780
|
+
{ name: "user_id", type: "text" },
|
|
781
|
+
{ name: "session_id", type: "text" },
|
|
782
|
+
{ name: "type", type: "text" },
|
|
783
|
+
{ name: "content", type: "text" },
|
|
784
|
+
{ name: "parts", type: "text", nullable: true },
|
|
785
|
+
// 兼容存量数据
|
|
786
|
+
{ name: "payload", type: "text", nullable: true },
|
|
787
|
+
{ name: "vector", type: "vector" },
|
|
788
|
+
{ name: "usage", type: "int" },
|
|
789
|
+
{ name: "metadata", type: "text" },
|
|
790
|
+
{ name: "created_at", type: "long" }
|
|
791
|
+
],
|
|
792
|
+
indexes: [
|
|
793
|
+
{ column: "message_id", kind: "scalar" },
|
|
794
|
+
{ column: "session_id", kind: "scalar" },
|
|
795
|
+
{ column: "content", kind: "fts" },
|
|
796
|
+
{ column: "metadata", kind: "fts" }
|
|
797
|
+
]
|
|
798
|
+
},
|
|
799
|
+
{
|
|
800
|
+
name: TOPICS_TABLE,
|
|
801
|
+
vectorDimension: dim,
|
|
802
|
+
columns: [
|
|
803
|
+
{ name: "summary_id", type: "text" },
|
|
804
|
+
{ name: "session_id", type: "text" },
|
|
805
|
+
{ name: "user_id", type: "text" },
|
|
806
|
+
{ name: "chat_id", type: "text" },
|
|
807
|
+
{ name: "title", type: "text", nullable: true },
|
|
808
|
+
// 兼容存量数据
|
|
809
|
+
{ name: "detail", type: "text", nullable: true },
|
|
810
|
+
{ name: "summary", type: "text" },
|
|
811
|
+
{ name: "concise", type: "text", nullable: true },
|
|
812
|
+
{ name: "tokens", type: "int", nullable: true },
|
|
813
|
+
{ name: "start_message_id", type: "text", nullable: true },
|
|
814
|
+
{ name: "end_message_id", type: "text", nullable: true },
|
|
815
|
+
{ name: "vector", type: "vector" },
|
|
816
|
+
{ name: "start_time", type: "long" },
|
|
817
|
+
{ name: "end_time", type: "long" },
|
|
818
|
+
{ name: "created_at", type: "long" },
|
|
819
|
+
{ name: "updated_at", type: "long" },
|
|
820
|
+
{ name: "recall_count", type: "int" }
|
|
821
|
+
],
|
|
822
|
+
indexes: [
|
|
823
|
+
{ column: "chat_id", kind: "scalar" },
|
|
824
|
+
{ column: "summary", kind: "fts" }
|
|
825
|
+
]
|
|
826
|
+
},
|
|
827
|
+
{
|
|
828
|
+
name: FACTS_TABLE,
|
|
829
|
+
columns: [
|
|
830
|
+
{ name: "fact_id", type: "text" },
|
|
831
|
+
{ name: "level", type: "text" },
|
|
832
|
+
{ name: "chat_id", type: "text" },
|
|
833
|
+
{ name: "session_id", type: "text" },
|
|
834
|
+
{ name: "user_id", type: "text" },
|
|
835
|
+
{ name: "fact_key", type: "text", nullable: true },
|
|
836
|
+
{ name: "content", type: "text" },
|
|
837
|
+
{ name: "created_at", type: "long" },
|
|
838
|
+
{ name: "updated_at", type: "long", nullable: true }
|
|
839
|
+
],
|
|
840
|
+
indexes: []
|
|
841
|
+
},
|
|
842
|
+
{
|
|
843
|
+
name: SESSIONS_TABLE,
|
|
844
|
+
columns: [
|
|
845
|
+
{ name: "session_id", type: "text" },
|
|
846
|
+
{ name: "chat_id", type: "text" },
|
|
847
|
+
{ name: "user_id", type: "text" },
|
|
848
|
+
{ name: "title", type: "text" },
|
|
849
|
+
{ name: "metadata", type: "text" },
|
|
850
|
+
{ name: "created_at", type: "long" },
|
|
851
|
+
{ name: "updated_at", type: "long" }
|
|
852
|
+
],
|
|
853
|
+
indexes: [{ column: "session_id", kind: "scalar" }]
|
|
854
|
+
},
|
|
855
|
+
{
|
|
856
|
+
name: DOCUMENTS_TABLE,
|
|
857
|
+
vectorDimension: dim,
|
|
858
|
+
columns: [
|
|
859
|
+
{ name: "doc_id", type: "text" },
|
|
860
|
+
{ name: "user_id", type: "text" },
|
|
861
|
+
{ name: "chat_id", type: "text" },
|
|
862
|
+
{ name: "session_id", type: "text" },
|
|
863
|
+
{ name: "title", type: "text" },
|
|
864
|
+
{ name: "source_name", type: "text" },
|
|
865
|
+
{ name: "full_content", type: "text" },
|
|
866
|
+
{ name: "content_hash", type: "text" },
|
|
867
|
+
{ name: "summary", type: "text" },
|
|
868
|
+
// vector 列存储文档摘要向量(summaryVector),用于文档级粗召回
|
|
869
|
+
{ name: "vector", type: "vector" },
|
|
870
|
+
{ name: "chunk_count", type: "int" },
|
|
871
|
+
{ name: "has_graph", type: "int" },
|
|
872
|
+
// 0/1 充当布尔
|
|
873
|
+
{ name: "metadata", type: "text" },
|
|
874
|
+
{ name: "created_at", type: "long" },
|
|
875
|
+
{ name: "updated_at", type: "long" }
|
|
876
|
+
],
|
|
877
|
+
indexes: [{ column: "doc_id", kind: "scalar" }]
|
|
878
|
+
},
|
|
879
|
+
{
|
|
880
|
+
name: CHUNKS_TABLE,
|
|
881
|
+
vectorDimension: dim,
|
|
882
|
+
columns: [
|
|
883
|
+
{ name: "chunk_id", type: "text" },
|
|
884
|
+
{ name: "doc_id", type: "text" },
|
|
885
|
+
// user_id/chat_id/session_id:chunkRedundantIds=false 时存空串,仅经 documents join 过滤
|
|
886
|
+
{ name: "user_id", type: "text" },
|
|
887
|
+
{ name: "chat_id", type: "text" },
|
|
888
|
+
{ name: "session_id", type: "text" },
|
|
889
|
+
{ name: "content", type: "text" },
|
|
890
|
+
{ name: "vector", type: "vector" },
|
|
891
|
+
{ name: "heading_path", type: "text" },
|
|
892
|
+
{ name: "ordinal", type: "int" },
|
|
893
|
+
{ name: "tokens", type: "int" },
|
|
894
|
+
{ name: "metadata", type: "text" },
|
|
895
|
+
{ name: "created_at", type: "long" }
|
|
896
|
+
],
|
|
897
|
+
indexes: [
|
|
898
|
+
{ column: "doc_id", kind: "scalar" },
|
|
899
|
+
{ column: "content", kind: "fts" }
|
|
900
|
+
]
|
|
901
|
+
}
|
|
902
|
+
];
|
|
811
903
|
}
|
|
812
|
-
function
|
|
813
|
-
return
|
|
814
|
-
new Field("chunk_id", new Utf8(), false),
|
|
815
|
-
new Field("doc_id", new Utf8(), false),
|
|
816
|
-
// user_id/chat_id/session_id:chunkRedundantIds=false 时存空串,仅经 documents join 过滤
|
|
817
|
-
new Field("user_id", new Utf8(), false),
|
|
818
|
-
new Field("chat_id", new Utf8(), false),
|
|
819
|
-
new Field("session_id", new Utf8(), false),
|
|
820
|
-
new Field("content", new Utf8(), false),
|
|
821
|
-
new Field("vector", new FixedSizeList(dim, new Field("item", new Float32(), false)), false),
|
|
822
|
-
new Field("heading_path", new Utf8(), false),
|
|
823
|
-
new Field("ordinal", new Int32(), false),
|
|
824
|
-
new Field("tokens", new Int32(), false),
|
|
825
|
-
new Field("metadata", new Utf8(), false),
|
|
826
|
-
new Field("created_at", new Int64(), false)
|
|
827
|
-
]);
|
|
904
|
+
function toVector(v) {
|
|
905
|
+
return Array.isArray(v) ? v : Array.from(v);
|
|
828
906
|
}
|
|
829
907
|
function messageToRow(m) {
|
|
830
908
|
return {
|
|
@@ -836,6 +914,7 @@ function messageToRow(m) {
|
|
|
836
914
|
type: m.type,
|
|
837
915
|
content: m.content,
|
|
838
916
|
parts: m.parts ?? "[]",
|
|
917
|
+
payload: m.payload ?? null,
|
|
839
918
|
vector: m.vector,
|
|
840
919
|
usage: m.usage,
|
|
841
920
|
metadata: m.metadata,
|
|
@@ -853,8 +932,9 @@ function rowToMessage(r) {
|
|
|
853
932
|
content: r.content,
|
|
854
933
|
parts: r.parts ?? "[]",
|
|
855
934
|
// 旧数据无此列时安全降级
|
|
856
|
-
|
|
857
|
-
|
|
935
|
+
payload: r.payload ?? void 0,
|
|
936
|
+
vector: toVector(r.vector),
|
|
937
|
+
usage: positiveNumber(r.usage),
|
|
858
938
|
metadata: r.metadata,
|
|
859
939
|
createdAt: Number(r.created_at)
|
|
860
940
|
};
|
|
@@ -866,9 +946,13 @@ function topicToRow(t) {
|
|
|
866
946
|
user_id: t.userId,
|
|
867
947
|
chat_id: t.chatId,
|
|
868
948
|
title: t.title ?? "",
|
|
869
|
-
detail
|
|
949
|
+
// 旧库的 detail/concise 可能仍是 NOT NULL;兼容写入但业务只读取 summary。
|
|
950
|
+
detail: t.detail ?? t.summary,
|
|
870
951
|
summary: t.summary,
|
|
871
|
-
concise: t.concise,
|
|
952
|
+
concise: t.concise ?? t.summary,
|
|
953
|
+
tokens: t.tokens,
|
|
954
|
+
start_message_id: t.startMessageId ?? null,
|
|
955
|
+
end_message_id: t.endMessageId ?? null,
|
|
872
956
|
vector: t.vector,
|
|
873
957
|
start_time: t.startTime,
|
|
874
958
|
end_time: t.endTime,
|
|
@@ -884,10 +968,13 @@ function rowToTopic(r) {
|
|
|
884
968
|
userId: r.user_id,
|
|
885
969
|
chatId: r.chat_id,
|
|
886
970
|
title: r.title ?? "",
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
971
|
+
summary: r.summary || r.detail || r.concise || "",
|
|
972
|
+
tokens: positiveNumber(r.tokens),
|
|
973
|
+
startMessageId: r.start_message_id ?? void 0,
|
|
974
|
+
endMessageId: r.end_message_id ?? void 0,
|
|
975
|
+
detail: r.detail ?? void 0,
|
|
976
|
+
concise: r.concise ?? void 0,
|
|
977
|
+
vector: toVector(r.vector),
|
|
891
978
|
startTime: Number(r.start_time),
|
|
892
979
|
endTime: Number(r.end_time),
|
|
893
980
|
createdAt: Number(r.created_at),
|
|
@@ -902,8 +989,10 @@ function factToRow(f) {
|
|
|
902
989
|
chat_id: f.chatId,
|
|
903
990
|
session_id: f.sessionId,
|
|
904
991
|
user_id: f.userId,
|
|
992
|
+
fact_key: f.key ?? null,
|
|
905
993
|
content: f.content,
|
|
906
|
-
created_at: f.createdAt
|
|
994
|
+
created_at: f.createdAt,
|
|
995
|
+
updated_at: f.updatedAt ?? f.createdAt
|
|
907
996
|
};
|
|
908
997
|
}
|
|
909
998
|
function rowToFact(r) {
|
|
@@ -913,8 +1002,10 @@ function rowToFact(r) {
|
|
|
913
1002
|
chatId: r.chat_id,
|
|
914
1003
|
sessionId: r.session_id,
|
|
915
1004
|
userId: r.user_id,
|
|
1005
|
+
key: r.fact_key ?? void 0,
|
|
916
1006
|
content: r.content,
|
|
917
|
-
createdAt: Number(r.created_at)
|
|
1007
|
+
createdAt: Number(r.created_at),
|
|
1008
|
+
updatedAt: Number(r.updated_at ?? r.created_at)
|
|
918
1009
|
};
|
|
919
1010
|
}
|
|
920
1011
|
function sessionToRow(s) {
|
|
@@ -969,7 +1060,7 @@ function rowToDocument(r) {
|
|
|
969
1060
|
fullContent: r.full_content,
|
|
970
1061
|
contentHash: r.content_hash,
|
|
971
1062
|
summary: r.summary,
|
|
972
|
-
summaryVector:
|
|
1063
|
+
summaryVector: toVector(r.vector),
|
|
973
1064
|
chunkCount: Number(r.chunk_count),
|
|
974
1065
|
hasGraph: Number(r.has_graph) === 1,
|
|
975
1066
|
metadata: safeParseObject(r.metadata),
|
|
@@ -1001,7 +1092,7 @@ function rowToChunk(r) {
|
|
|
1001
1092
|
chatId: r.chat_id,
|
|
1002
1093
|
sessionId: r.session_id,
|
|
1003
1094
|
content: r.content,
|
|
1004
|
-
vector:
|
|
1095
|
+
vector: toVector(r.vector),
|
|
1005
1096
|
headingPath: r.heading_path,
|
|
1006
1097
|
ordinal: Number(r.ordinal),
|
|
1007
1098
|
tokens: Number(r.tokens),
|
|
@@ -1016,215 +1107,312 @@ function safeParseObject(s) {
|
|
|
1016
1107
|
return {};
|
|
1017
1108
|
}
|
|
1018
1109
|
}
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1110
|
+
function positiveNumber(value) {
|
|
1111
|
+
const parsed = Number(value);
|
|
1112
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : 0;
|
|
1113
|
+
}
|
|
1114
|
+
function migrationValueEquals(current, next) {
|
|
1115
|
+
if (current == null && next == null) return true;
|
|
1116
|
+
if (typeof next === "number") return Number(current) === next;
|
|
1117
|
+
return String(current) === String(next);
|
|
1118
|
+
}
|
|
1119
|
+
function rrfFuse(lists, keyOf) {
|
|
1120
|
+
const acc = /* @__PURE__ */ new Map();
|
|
1121
|
+
for (const list of lists) {
|
|
1122
|
+
list.forEach((row, idx) => {
|
|
1123
|
+
const id = keyOf(row);
|
|
1124
|
+
const inc = 1 / (RRF_K + idx + 1);
|
|
1125
|
+
const entry = acc.get(id);
|
|
1126
|
+
if (entry) entry.score += inc;
|
|
1127
|
+
else acc.set(id, { row, score: inc });
|
|
1128
|
+
});
|
|
1129
|
+
}
|
|
1130
|
+
return [...acc.values()].sort((a, b) => b.score - a.score).map((e) => e.row);
|
|
1131
|
+
}
|
|
1132
|
+
var MemoryStore = class {
|
|
1038
1133
|
config;
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
sessionsTable;
|
|
1044
|
-
documentsTable;
|
|
1045
|
-
chunksTable;
|
|
1046
|
-
// Scalar indexes cannot be created on empty tables (LanceDB btree limitation).
|
|
1047
|
-
// These flags defer creation to the first insert.
|
|
1048
|
-
isNewMessagesTable = false;
|
|
1049
|
-
isNewTopicsTable = false;
|
|
1050
|
-
isNewSessionsTable = false;
|
|
1051
|
-
isNewDocumentsTable = false;
|
|
1052
|
-
isNewChunksTable = false;
|
|
1053
|
-
constructor(config) {
|
|
1134
|
+
provider;
|
|
1135
|
+
providerOverride;
|
|
1136
|
+
/** provider 省略时在 init() 阶段经 provider.resolver 自动探测创建(测试可显式注入) */
|
|
1137
|
+
constructor(config, provider) {
|
|
1054
1138
|
this.config = config;
|
|
1139
|
+
this.providerOverride = provider;
|
|
1140
|
+
}
|
|
1141
|
+
/** 当前后端 provider 标识(日志/诊断用) */
|
|
1142
|
+
get providerKind() {
|
|
1143
|
+
return this.provider?.kind ?? "uninitialized";
|
|
1055
1144
|
}
|
|
1056
1145
|
async init() {
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
const existingTables = await this.conn.tableNames();
|
|
1060
|
-
if (existingTables.includes(MESSAGES_TABLE)) {
|
|
1061
|
-
this.messagesTable = await this.conn.openTable(MESSAGES_TABLE);
|
|
1062
|
-
await this._ensurePartsColumn();
|
|
1146
|
+
if (this.providerOverride) {
|
|
1147
|
+
this.provider = this.providerOverride;
|
|
1063
1148
|
} else {
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
);
|
|
1068
|
-
this.isNewMessagesTable = true;
|
|
1149
|
+
const { createProvider } = await import("./provider.resolver-2KS2YYNV.js");
|
|
1150
|
+
this.provider = await createProvider(this.config);
|
|
1151
|
+
console.info(`[memory] \u5411\u91CF\u5B58\u50A8\u540E\u7AEF\uFF1A${this.provider.kind}`);
|
|
1069
1152
|
}
|
|
1070
|
-
|
|
1071
|
-
this.topicsTable = await this.conn.openTable(TOPICS_TABLE);
|
|
1072
|
-
await this._ensureTopicTitleColumn();
|
|
1073
|
-
} else {
|
|
1074
|
-
this.topicsTable = await this.conn.createEmptyTable(
|
|
1075
|
-
TOPICS_TABLE,
|
|
1076
|
-
topicsSchema(dim)
|
|
1077
|
-
);
|
|
1078
|
-
this.isNewTopicsTable = true;
|
|
1079
|
-
}
|
|
1080
|
-
if (existingTables.includes(FACTS_TABLE)) {
|
|
1081
|
-
this.factsTable = await this.conn.openTable(FACTS_TABLE);
|
|
1082
|
-
} else {
|
|
1083
|
-
this.factsTable = await this.conn.createEmptyTable(FACTS_TABLE, factsSchema());
|
|
1084
|
-
}
|
|
1085
|
-
if (existingTables.includes(SESSIONS_TABLE)) {
|
|
1086
|
-
this.sessionsTable = await this.conn.openTable(SESSIONS_TABLE);
|
|
1087
|
-
} else {
|
|
1088
|
-
this.sessionsTable = await this.conn.createEmptyTable(SESSIONS_TABLE, sessionsSchema());
|
|
1089
|
-
this.isNewSessionsTable = true;
|
|
1090
|
-
}
|
|
1091
|
-
if (existingTables.includes(DOCUMENTS_TABLE)) {
|
|
1092
|
-
this.documentsTable = await this.conn.openTable(DOCUMENTS_TABLE);
|
|
1093
|
-
} else {
|
|
1094
|
-
this.documentsTable = await this.conn.createEmptyTable(DOCUMENTS_TABLE, documentsSchema(dim));
|
|
1095
|
-
this.isNewDocumentsTable = true;
|
|
1096
|
-
}
|
|
1097
|
-
if (existingTables.includes(CHUNKS_TABLE)) {
|
|
1098
|
-
this.chunksTable = await this.conn.openTable(CHUNKS_TABLE);
|
|
1099
|
-
} else {
|
|
1100
|
-
this.chunksTable = await this.conn.createEmptyTable(CHUNKS_TABLE, chunksSchema(dim));
|
|
1101
|
-
this.isNewChunksTable = true;
|
|
1102
|
-
}
|
|
1103
|
-
await this.ensureIndexes(this.messagesTable, MESSAGES_INDEXES);
|
|
1104
|
-
await this.ensureIndexes(this.topicsTable, TOPICS_INDEXES);
|
|
1105
|
-
await this.ensureIndexes(this.sessionsTable, SESSIONS_INDEXES);
|
|
1106
|
-
await this.ensureIndexes(this.documentsTable, DOCUMENTS_INDEXES);
|
|
1107
|
-
await this.ensureIndexes(this.chunksTable, CHUNKS_INDEXES);
|
|
1153
|
+
await this.provider.init(tableDefs(this.config.embeddingDimension));
|
|
1108
1154
|
}
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
* createIndex 默认 replace:true 会在每次启动时全量重建索引并提交新表版本——
|
|
1112
|
-
* 这正是历史上「启动越来越慢 + _versions 目录膨胀」的根源,绝不可回退到无条件 createIndex。
|
|
1113
|
-
*/
|
|
1114
|
-
async ensureIndexes(table, specs) {
|
|
1115
|
-
let indexed;
|
|
1116
|
-
try {
|
|
1117
|
-
indexed = new Set((await table.listIndices()).flatMap((i) => i.columns));
|
|
1118
|
-
} catch {
|
|
1119
|
-
indexed = /* @__PURE__ */ new Set();
|
|
1120
|
-
}
|
|
1121
|
-
for (const { column, fts } of specs) {
|
|
1122
|
-
if (indexed.has(column)) continue;
|
|
1123
|
-
try {
|
|
1124
|
-
await table.createIndex(
|
|
1125
|
-
column,
|
|
1126
|
-
fts ? { config: lancedb.Index.fts(), replace: false } : { replace: false }
|
|
1127
|
-
);
|
|
1128
|
-
} catch {
|
|
1129
|
-
}
|
|
1130
|
-
}
|
|
1155
|
+
async close() {
|
|
1156
|
+
await this.provider.close();
|
|
1131
1157
|
}
|
|
1132
1158
|
/**
|
|
1133
|
-
*
|
|
1134
|
-
*
|
|
1159
|
+
* 存储压实(碎片合并 + 历史版本清理)。
|
|
1160
|
+
* LanceDB 后端长期运行必须定期执行;SQLite 后端为可选的空间回收。
|
|
1135
1161
|
*/
|
|
1136
1162
|
async optimizeStorage(retentionMs = 0) {
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
const
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1163
|
+
return this.provider.optimize(retentionMs);
|
|
1164
|
+
}
|
|
1165
|
+
// ── messages ───────────────────────────────────────────────────────────────
|
|
1166
|
+
async addMessages(messages) {
|
|
1167
|
+
if (messages.length === 0) return;
|
|
1168
|
+
await this.provider.add(MESSAGES_TABLE, messages.map(messageToRow));
|
|
1169
|
+
}
|
|
1170
|
+
/** 0.3.x → 0.4.x 数据回填;幂等,不调用 LLM、不删除原始消息。 */
|
|
1171
|
+
async migrateLegacyData() {
|
|
1172
|
+
const report = {
|
|
1173
|
+
provider: this.providerKind,
|
|
1174
|
+
topicsScanned: 0,
|
|
1175
|
+
topicsUpdated: 0,
|
|
1176
|
+
messagesScanned: 0,
|
|
1177
|
+
messagesUpdated: 0,
|
|
1178
|
+
factsScanned: 0,
|
|
1179
|
+
factsUpdated: 0
|
|
1180
|
+
};
|
|
1181
|
+
const topicRows = await this.provider.query(TOPICS_TABLE);
|
|
1182
|
+
report.topicsScanned = topicRows.length;
|
|
1183
|
+
const messagesBySession = /* @__PURE__ */ new Map();
|
|
1184
|
+
for (const row of topicRows) {
|
|
1185
|
+
const sessionId = String(row.session_id);
|
|
1186
|
+
let messages = messagesBySession.get(sessionId);
|
|
1187
|
+
if (!messages) {
|
|
1188
|
+
messages = await this.provider.query(MESSAGES_TABLE, {
|
|
1189
|
+
filter: [eq("session_id", sessionId)],
|
|
1190
|
+
orderBy: [
|
|
1191
|
+
{ column: "created_at", ascending: true },
|
|
1192
|
+
{ column: "message_id", ascending: true }
|
|
1193
|
+
]
|
|
1157
1194
|
});
|
|
1158
|
-
|
|
1159
|
-
console.warn(`[LanceService] optimize table "${name}" failed:`, err);
|
|
1195
|
+
messagesBySession.set(sessionId, messages);
|
|
1160
1196
|
}
|
|
1197
|
+
const summary = String(row.summary || row.detail || row.concise || "");
|
|
1198
|
+
const startTime = Number(row.start_time);
|
|
1199
|
+
const endTime = Number(row.end_time);
|
|
1200
|
+
const covered = messages.filter(
|
|
1201
|
+
(message) => Number(message.created_at) >= startTime && Number(message.created_at) <= endTime
|
|
1202
|
+
);
|
|
1203
|
+
const values = {
|
|
1204
|
+
summary,
|
|
1205
|
+
tokens: positiveNumber(row.tokens) || Math.max(1, countTokens(summary)),
|
|
1206
|
+
start_message_id: row.start_message_id ?? covered.at(0)?.message_id ?? null,
|
|
1207
|
+
end_message_id: row.end_message_id ?? covered.at(-1)?.message_id ?? null,
|
|
1208
|
+
updated_at: Number(row.updated_at || Date.now())
|
|
1209
|
+
};
|
|
1210
|
+
const changed = Object.entries(values).some(
|
|
1211
|
+
([key, value]) => !migrationValueEquals(row[key], value)
|
|
1212
|
+
);
|
|
1213
|
+
if (changed) {
|
|
1214
|
+
await this.provider.update(TOPICS_TABLE, values, [eq("summary_id", String(row.summary_id))]);
|
|
1215
|
+
report.topicsUpdated++;
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
const messageRows = await this.provider.query(MESSAGES_TABLE);
|
|
1219
|
+
report.messagesScanned = messageRows.length;
|
|
1220
|
+
for (const row of messageRows) {
|
|
1221
|
+
const usage = countTokens(
|
|
1222
|
+
[row.content, row.parts ?? "[]", row.metadata ?? "{}", row.payload ?? ""].map(String).join("\n")
|
|
1223
|
+
);
|
|
1224
|
+
if (Number(row.usage) === usage) continue;
|
|
1225
|
+
await this.provider.update(MESSAGES_TABLE, { usage }, [
|
|
1226
|
+
eq("message_id", String(row.message_id))
|
|
1227
|
+
]);
|
|
1228
|
+
report.messagesUpdated++;
|
|
1229
|
+
}
|
|
1230
|
+
const factRows = await this.provider.query(FACTS_TABLE);
|
|
1231
|
+
report.factsScanned = factRows.length;
|
|
1232
|
+
for (const row of factRows) {
|
|
1233
|
+
const updatedAt = Number(row.updated_at || row.created_at);
|
|
1234
|
+
if (Number(row.updated_at) === updatedAt) continue;
|
|
1235
|
+
await this.provider.update(FACTS_TABLE, { updated_at: updatedAt }, [
|
|
1236
|
+
eq("fact_id", String(row.fact_id))
|
|
1237
|
+
]);
|
|
1238
|
+
report.factsUpdated++;
|
|
1239
|
+
}
|
|
1240
|
+
return report;
|
|
1241
|
+
}
|
|
1242
|
+
/** message_id 稳定时更新原行,否则新增;用于流式 assistant 消息最终态覆盖。 */
|
|
1243
|
+
async upsertMessages(messages) {
|
|
1244
|
+
if (messages.length === 0) return;
|
|
1245
|
+
const ids = messages.map((message) => message.messageId);
|
|
1246
|
+
const existing = await this.provider.query(MESSAGES_TABLE, {
|
|
1247
|
+
filter: [{ op: "in", field: "message_id", values: ids }],
|
|
1248
|
+
select: ["message_id"]
|
|
1249
|
+
});
|
|
1250
|
+
const existingIds = new Set(existing.map((row) => String(row.message_id)));
|
|
1251
|
+
await Promise.all(
|
|
1252
|
+
messages.filter((message) => existingIds.has(message.messageId)).map(
|
|
1253
|
+
(message) => this.provider.update(
|
|
1254
|
+
MESSAGES_TABLE,
|
|
1255
|
+
messageToRow(message),
|
|
1256
|
+
[eq("message_id", message.messageId)]
|
|
1257
|
+
)
|
|
1258
|
+
)
|
|
1259
|
+
);
|
|
1260
|
+
await this.addMessages(messages.filter((message) => !existingIds.has(message.messageId)));
|
|
1261
|
+
}
|
|
1262
|
+
async getMessagesSince(sessionId, since, limit) {
|
|
1263
|
+
try {
|
|
1264
|
+
const rows = await this.provider.query(MESSAGES_TABLE, {
|
|
1265
|
+
filter: [eq("session_id", sessionId), { op: "gt", field: "created_at", value: since }],
|
|
1266
|
+
limit
|
|
1267
|
+
});
|
|
1268
|
+
return rows.sort((a, b) => Number(a.created_at) - Number(b.created_at)).map(rowToMessage);
|
|
1269
|
+
} catch {
|
|
1270
|
+
return [];
|
|
1161
1271
|
}
|
|
1162
|
-
return results;
|
|
1163
1272
|
}
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1273
|
+
async getMessagesAfterBoundary(sessionId, endTime, endMessageId) {
|
|
1274
|
+
const messages = await this.getAllMessagesBySession(sessionId);
|
|
1275
|
+
return messages.filter(
|
|
1276
|
+
(message) => message.createdAt > endTime || message.createdAt === endTime && endMessageId != null && message.messageId.localeCompare(endMessageId) > 0
|
|
1277
|
+
);
|
|
1278
|
+
}
|
|
1279
|
+
async getLatestMessages(sessionId, limit) {
|
|
1280
|
+
if (limit <= 0) return [];
|
|
1169
1281
|
try {
|
|
1170
|
-
const
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1282
|
+
const rows = await this.provider.query(MESSAGES_TABLE, {
|
|
1283
|
+
filter: [eq("session_id", sessionId)],
|
|
1284
|
+
orderBy: [
|
|
1285
|
+
{ column: "created_at", ascending: false },
|
|
1286
|
+
{ column: "message_id", ascending: false }
|
|
1287
|
+
],
|
|
1288
|
+
limit
|
|
1289
|
+
});
|
|
1290
|
+
return rows.reverse().map(rowToMessage);
|
|
1179
1291
|
} catch {
|
|
1292
|
+
return [];
|
|
1180
1293
|
}
|
|
1181
1294
|
}
|
|
1182
|
-
/**
|
|
1183
|
-
|
|
1184
|
-
* 旧版本不支持 addColumns 时忽略,读取时 rowToTopic 的 ?? "" 兜底。
|
|
1185
|
-
*/
|
|
1186
|
-
async _ensureTopicTitleColumn() {
|
|
1295
|
+
/** 取某会话的全部消息(按 createdAt 升序),供管理面板分页切片使用 */
|
|
1296
|
+
async getAllMessagesBySession(sessionId) {
|
|
1187
1297
|
try {
|
|
1188
|
-
const
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
{ name: "title", valueSql: "CAST(NULL AS STRING)" }
|
|
1195
|
-
]);
|
|
1196
|
-
}
|
|
1298
|
+
const rows = await this.provider.query(MESSAGES_TABLE, {
|
|
1299
|
+
filter: [eq("session_id", sessionId)]
|
|
1300
|
+
});
|
|
1301
|
+
return rows.sort(
|
|
1302
|
+
(a, b) => Number(a.created_at) - Number(b.created_at) || cmpStr(a.message_id, b.message_id)
|
|
1303
|
+
).map(rowToMessage);
|
|
1197
1304
|
} catch {
|
|
1305
|
+
return [];
|
|
1198
1306
|
}
|
|
1199
1307
|
}
|
|
1200
|
-
async
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1308
|
+
async searchMessages(vector, filter, limit = 10) {
|
|
1309
|
+
const rows = await this.provider.vectorSearch(MESSAGES_TABLE, vector, { filter, limit });
|
|
1310
|
+
return rows.map((r) => ({ ...rowToMessage(r), _distance: r._distance }));
|
|
1311
|
+
}
|
|
1312
|
+
// 对 messages 表执行混合搜索(BM25 + 向量),用于 topics 搜索无结果时的回退
|
|
1313
|
+
async hybridSearchMessages(query, vector, filter, limit = 10) {
|
|
1314
|
+
const fused = await this.hybridSearch(
|
|
1315
|
+
MESSAGES_TABLE,
|
|
1316
|
+
["content", "metadata"],
|
|
1317
|
+
query,
|
|
1318
|
+
vector,
|
|
1319
|
+
filter,
|
|
1320
|
+
limit
|
|
1321
|
+
);
|
|
1322
|
+
return fused.slice(0, limit).map(rowToMessage);
|
|
1323
|
+
}
|
|
1324
|
+
async hybridSearchTopics(query, vector, filter, limit = 10) {
|
|
1325
|
+
try {
|
|
1326
|
+
const fused = (await this.hybridSearch(TOPICS_TABLE, ["summary"], query, vector, filter, limit)).slice(0, limit);
|
|
1327
|
+
if (fused.length > 0) {
|
|
1328
|
+
const total = fused.length;
|
|
1329
|
+
return fused.map((r, idx) => ({
|
|
1330
|
+
row: r,
|
|
1331
|
+
score: total - idx + Math.log1p(Number(r.recall_count))
|
|
1332
|
+
})).sort((a, b) => b.score - a.score).map((x) => rowToTopic(x.row));
|
|
1333
|
+
}
|
|
1334
|
+
return [];
|
|
1335
|
+
} catch {
|
|
1336
|
+
return [];
|
|
1206
1337
|
}
|
|
1207
1338
|
}
|
|
1339
|
+
/**
|
|
1340
|
+
* 通用混合检索:FTS(BM25)与向量两路各取 limit*HYBRID_OVERFETCH 候选,RRF 融合。
|
|
1341
|
+
* FTS 不可用(后端不支持或查询失败)时降级为纯向量。返回融合后的完整候选序列(未截断)。
|
|
1342
|
+
*/
|
|
1343
|
+
async hybridSearch(table, ftsColumns, query, vector, filter, limit) {
|
|
1344
|
+
const fetchN = limit * HYBRID_OVERFETCH;
|
|
1345
|
+
const supportsFts = this.provider.capabilities().fts;
|
|
1346
|
+
const [ftsRows, vecRows] = await Promise.all([
|
|
1347
|
+
supportsFts ? this.provider.ftsSearch(table, query, { columns: ftsColumns, filter, limit: fetchN }).catch(() => []) : Promise.resolve([]),
|
|
1348
|
+
this.provider.vectorSearch(table, vector, { filter, limit: fetchN }).catch(() => [])
|
|
1349
|
+
]);
|
|
1350
|
+
const keyColumn = this.keyColumnOf(table);
|
|
1351
|
+
return rrfFuse([ftsRows, vecRows], (r) => String(r[keyColumn]));
|
|
1352
|
+
}
|
|
1353
|
+
keyColumnOf(table) {
|
|
1354
|
+
switch (table) {
|
|
1355
|
+
case MESSAGES_TABLE:
|
|
1356
|
+
return "message_id";
|
|
1357
|
+
case TOPICS_TABLE:
|
|
1358
|
+
return "summary_id";
|
|
1359
|
+
case DOCUMENTS_TABLE:
|
|
1360
|
+
return "doc_id";
|
|
1361
|
+
case CHUNKS_TABLE:
|
|
1362
|
+
return "chunk_id";
|
|
1363
|
+
case SESSIONS_TABLE:
|
|
1364
|
+
return "session_id";
|
|
1365
|
+
case FACTS_TABLE:
|
|
1366
|
+
return "fact_id";
|
|
1367
|
+
default:
|
|
1368
|
+
throw new Error(`Unknown table: ${table}`);
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
// ── topics ─────────────────────────────────────────────────────────────────
|
|
1208
1372
|
async addTopic(topic) {
|
|
1209
|
-
await this.
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1373
|
+
await this.provider.add(TOPICS_TABLE, [topicToRow(topic)]);
|
|
1374
|
+
}
|
|
1375
|
+
async updateTopic(topic) {
|
|
1376
|
+
await this.provider.update(
|
|
1377
|
+
TOPICS_TABLE,
|
|
1378
|
+
topicToRow(topic),
|
|
1379
|
+
[eq("summary_id", topic.summaryId)]
|
|
1380
|
+
);
|
|
1214
1381
|
}
|
|
1215
1382
|
async updateTopicRecallCount(summaryId, count) {
|
|
1216
|
-
await this.
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1383
|
+
await this.provider.update(
|
|
1384
|
+
TOPICS_TABLE,
|
|
1385
|
+
{ recall_count: count, updated_at: Date.now() },
|
|
1386
|
+
[eq("summary_id", summaryId)]
|
|
1387
|
+
);
|
|
1388
|
+
}
|
|
1389
|
+
async incrementTopicRecallCounts(topics) {
|
|
1390
|
+
await Promise.all(
|
|
1391
|
+
topics.map((topic) => this.updateTopicRecallCount(topic.summaryId, topic.recallCount + 1))
|
|
1392
|
+
);
|
|
1393
|
+
}
|
|
1394
|
+
async getTopicsBySession(sessionId, since = 0) {
|
|
1395
|
+
try {
|
|
1396
|
+
const filter = [eq("session_id", sessionId)];
|
|
1397
|
+
if (since > 0) filter.push({ op: "gte", field: "end_time", value: since });
|
|
1398
|
+
const rows = await this.provider.query(TOPICS_TABLE, { filter });
|
|
1399
|
+
return rows.sort(
|
|
1400
|
+
(a, b) => Number(a.end_time) - Number(b.end_time) || cmpStr(a.summary_id, b.summary_id)
|
|
1401
|
+
).map(rowToTopic);
|
|
1402
|
+
} catch {
|
|
1403
|
+
return [];
|
|
1404
|
+
}
|
|
1220
1405
|
}
|
|
1221
1406
|
async getRecentTopics(chatId, userId, n1, n2, n3) {
|
|
1222
|
-
const filter =
|
|
1407
|
+
const filter = [eq("chat_id", chatId), eq("user_id", userId)];
|
|
1223
1408
|
const recallBoostMs = this.config.recallBoostMs;
|
|
1224
1409
|
const fetchTopics = async (count) => {
|
|
1225
1410
|
if (count <= 0) return [];
|
|
1226
1411
|
try {
|
|
1227
|
-
const rows = await this.
|
|
1412
|
+
const rows = await this.provider.query(TOPICS_TABLE, {
|
|
1413
|
+
filter,
|
|
1414
|
+
limit: count * 5
|
|
1415
|
+
});
|
|
1228
1416
|
return rows.sort((a, b) => {
|
|
1229
1417
|
const scoreA = Number(a.end_time) + Number(a.recall_count) * recallBoostMs;
|
|
1230
1418
|
const scoreB = Number(b.end_time) + Number(b.recall_count) * recallBoostMs;
|
|
@@ -1241,167 +1429,94 @@ var LanceService = class {
|
|
|
1241
1429
|
concise: allTopics.slice(n1 + n2, n1 + n2 + n3)
|
|
1242
1430
|
};
|
|
1243
1431
|
}
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
const q = this.messagesTable.query().where(`${eqFilter("session_id", sessionId)} AND created_at > ${since}`);
|
|
1247
|
-
if (limit) q.limit(limit);
|
|
1248
|
-
const rows = await q.toArray();
|
|
1249
|
-
return rows.sort((a, b) => Number(a.created_at) - Number(b.created_at)).map(rowToMessage);
|
|
1250
|
-
} catch {
|
|
1251
|
-
return [];
|
|
1252
|
-
}
|
|
1253
|
-
}
|
|
1254
|
-
async getLatestMessages(sessionId, limit) {
|
|
1255
|
-
if (limit <= 0) return [];
|
|
1256
|
-
try {
|
|
1257
|
-
const rows = await this.messagesTable.query().where(eqFilter("session_id", sessionId)).orderBy([
|
|
1258
|
-
{ columnName: "created_at", ascending: false },
|
|
1259
|
-
{ columnName: "message_id", ascending: false }
|
|
1260
|
-
]).limit(limit).toArray();
|
|
1261
|
-
return rows.reverse().map(rowToMessage);
|
|
1262
|
-
} catch {
|
|
1263
|
-
return [];
|
|
1264
|
-
}
|
|
1265
|
-
}
|
|
1266
|
-
/** 取某会话的全部消息(按 createdAt 升序),供管理面板分页切片使用 */
|
|
1267
|
-
async getAllMessagesBySession(sessionId) {
|
|
1432
|
+
/** 全量读取 topics(管理面板列表用,不做 recall 加权排序,按 endTime 倒序)*/
|
|
1433
|
+
async getAllTopics() {
|
|
1268
1434
|
try {
|
|
1269
|
-
const rows = await this.
|
|
1270
|
-
return rows.sort(
|
|
1435
|
+
const rows = await this.provider.query(TOPICS_TABLE);
|
|
1436
|
+
return rows.sort(
|
|
1437
|
+
(a, b) => Number(b.end_time) - Number(a.end_time) || cmpStr(a.summary_id, b.summary_id)
|
|
1438
|
+
).map(rowToTopic);
|
|
1271
1439
|
} catch {
|
|
1272
1440
|
return [];
|
|
1273
1441
|
}
|
|
1274
1442
|
}
|
|
1275
|
-
async
|
|
1276
|
-
|
|
1277
|
-
if (filter) q.where(filter);
|
|
1278
|
-
const rows = await q.toArray();
|
|
1279
|
-
return rows.map((r) => ({
|
|
1280
|
-
...rowToMessage(r),
|
|
1281
|
-
_distance: r._distance
|
|
1282
|
-
}));
|
|
1283
|
-
}
|
|
1284
|
-
// 对 messages 表执行混合搜索(BM25 + 向量),用于 topics 搜索无结果时的回退
|
|
1285
|
-
async hybridSearchMessages(query, vector, filter, limit = 10) {
|
|
1286
|
-
try {
|
|
1287
|
-
const q = this.messagesTable.query().fullTextSearch(query).nearestTo(vector).limit(limit);
|
|
1288
|
-
if (filter) q.where(filter);
|
|
1289
|
-
const rows = await q.toArray();
|
|
1290
|
-
return rows.map(rowToMessage);
|
|
1291
|
-
} catch {
|
|
1292
|
-
const rows = await this.searchMessages(vector, filter, limit);
|
|
1293
|
-
return rows.map(({ _distance: _d, ...r }) => r);
|
|
1294
|
-
}
|
|
1443
|
+
async deleteTopicsBySession(sessionId) {
|
|
1444
|
+
await this.provider.deleteWhere(TOPICS_TABLE, [eq("session_id", sessionId)]);
|
|
1295
1445
|
}
|
|
1296
|
-
async
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
const rows = await q.toArray();
|
|
1302
|
-
if (rows.length > 0) {
|
|
1303
|
-
const total = rows.length;
|
|
1304
|
-
return rows.map((r, idx) => ({
|
|
1305
|
-
row: r,
|
|
1306
|
-
score: total - idx + Math.log1p(Number(r.recall_count))
|
|
1307
|
-
})).sort((a, b) => b.score - a.score).map((x) => rowToTopic(x.row));
|
|
1308
|
-
}
|
|
1309
|
-
} catch {
|
|
1310
|
-
}
|
|
1311
|
-
const msgRows = await this.hybridSearchMessages(query, vector, filter, limit);
|
|
1312
|
-
return msgRows.map((r) => ({
|
|
1313
|
-
summaryId: r.messageId,
|
|
1314
|
-
sessionId: r.sessionId,
|
|
1315
|
-
userId: r.userId,
|
|
1316
|
-
chatId: r.chatId,
|
|
1317
|
-
detail: r.content,
|
|
1318
|
-
summary: r.content,
|
|
1319
|
-
concise: r.content,
|
|
1320
|
-
startTime: r.createdAt,
|
|
1321
|
-
endTime: r.createdAt,
|
|
1322
|
-
createdAt: r.createdAt,
|
|
1323
|
-
updatedAt: r.createdAt,
|
|
1324
|
-
recallCount: 0,
|
|
1325
|
-
vector: r.vector
|
|
1326
|
-
}));
|
|
1446
|
+
async deleteTopicsByIds(summaryIds) {
|
|
1447
|
+
if (summaryIds.length === 0) return;
|
|
1448
|
+
await this.provider.deleteWhere(TOPICS_TABLE, [
|
|
1449
|
+
{ op: "in", field: "summary_id", values: summaryIds }
|
|
1450
|
+
]);
|
|
1327
1451
|
}
|
|
1452
|
+
// ── facts ──────────────────────────────────────────────────────────────────
|
|
1328
1453
|
async saveFact(fact) {
|
|
1329
|
-
await this.
|
|
1454
|
+
await this.provider.add(FACTS_TABLE, [factToRow(fact)]);
|
|
1455
|
+
}
|
|
1456
|
+
async updateFact(fact) {
|
|
1457
|
+
await this.provider.update(FACTS_TABLE, factToRow(fact), [eq("fact_id", fact.factId)]);
|
|
1330
1458
|
}
|
|
1331
1459
|
async getAllFacts() {
|
|
1332
1460
|
try {
|
|
1333
|
-
const rows = await this.
|
|
1461
|
+
const rows = await this.provider.query(FACTS_TABLE);
|
|
1334
1462
|
return rows.map(rowToFact);
|
|
1335
1463
|
} catch {
|
|
1336
1464
|
return [];
|
|
1337
1465
|
}
|
|
1338
1466
|
}
|
|
1467
|
+
/** 删除单条 fact */
|
|
1468
|
+
async deleteFact(factId) {
|
|
1469
|
+
await this.provider.deleteWhere(FACTS_TABLE, [eq("fact_id", factId)]);
|
|
1470
|
+
}
|
|
1471
|
+
// ── sessions ───────────────────────────────────────────────────────────────
|
|
1339
1472
|
async getAllSessionIds() {
|
|
1340
1473
|
try {
|
|
1341
|
-
const rows = await this.
|
|
1474
|
+
const rows = await this.provider.query(MESSAGES_TABLE, {
|
|
1475
|
+
select: ["session_id"],
|
|
1476
|
+
limit: 1e4
|
|
1477
|
+
});
|
|
1342
1478
|
return [...new Set(rows.map((r) => r.session_id))];
|
|
1343
1479
|
} catch {
|
|
1344
1480
|
return [];
|
|
1345
1481
|
}
|
|
1346
1482
|
}
|
|
1347
1483
|
async insertSession(session) {
|
|
1348
|
-
await this.
|
|
1349
|
-
if (this.isNewSessionsTable) {
|
|
1350
|
-
await this.ensureIndexes(this.sessionsTable, SESSIONS_INDEXES);
|
|
1351
|
-
this.isNewSessionsTable = false;
|
|
1352
|
-
}
|
|
1484
|
+
await this.provider.add(SESSIONS_TABLE, [sessionToRow(session)]);
|
|
1353
1485
|
}
|
|
1354
1486
|
async upsertSession(session) {
|
|
1355
|
-
await this.
|
|
1356
|
-
await this.
|
|
1487
|
+
await this.provider.deleteWhere(SESSIONS_TABLE, [eq("session_id", session.sessionId)]);
|
|
1488
|
+
await this.provider.add(SESSIONS_TABLE, [sessionToRow(session)]);
|
|
1357
1489
|
}
|
|
1358
1490
|
async getAllSessions() {
|
|
1359
1491
|
try {
|
|
1360
|
-
const rows = await this.
|
|
1492
|
+
const rows = await this.provider.query(SESSIONS_TABLE);
|
|
1361
1493
|
return rows.map(rowToSession);
|
|
1362
1494
|
} catch {
|
|
1363
1495
|
return [];
|
|
1364
1496
|
}
|
|
1365
1497
|
}
|
|
1366
1498
|
async deleteSession(sessionId) {
|
|
1367
|
-
await this.
|
|
1368
|
-
}
|
|
1369
|
-
// ── 管理面板用:全量读取 / 删除 / 计数 ──────────────────────────────────────────────
|
|
1370
|
-
/** 全量读取 topics(管理面板列表用,不做 recall 加权排序,按 endTime 倒序)*/
|
|
1371
|
-
async getAllTopics() {
|
|
1372
|
-
try {
|
|
1373
|
-
const rows = await this.topicsTable.query().toArray();
|
|
1374
|
-
return rows.sort((a, b) => Number(b.end_time) - Number(a.end_time) || cmpStr(a.summary_id, b.summary_id)).map(rowToTopic);
|
|
1375
|
-
} catch {
|
|
1376
|
-
return [];
|
|
1377
|
-
}
|
|
1378
|
-
}
|
|
1379
|
-
/** 删除单条 fact */
|
|
1380
|
-
async deleteFact(factId) {
|
|
1381
|
-
await this.factsTable.delete(eqFilter("fact_id", factId));
|
|
1499
|
+
await this.provider.deleteWhere(SESSIONS_TABLE, [eq("session_id", sessionId)]);
|
|
1382
1500
|
}
|
|
1383
1501
|
/** 删除某会话下的全部消息(级联删除会话时使用)*/
|
|
1384
1502
|
async deleteMessagesBySession(sessionId) {
|
|
1385
|
-
await this.
|
|
1386
|
-
}
|
|
1387
|
-
/** 删除某会话下的全部 topics(级联删除会话时使用)*/
|
|
1388
|
-
async deleteTopicsBySession(sessionId) {
|
|
1389
|
-
await this.topicsTable.delete(eqFilter("session_id", sessionId));
|
|
1503
|
+
await this.provider.deleteWhere(MESSAGES_TABLE, [eq("session_id", sessionId)]);
|
|
1390
1504
|
}
|
|
1505
|
+
// ── 统计 ───────────────────────────────────────────────────────────────────
|
|
1391
1506
|
/** 各表行数统计(概览卡片用)*/
|
|
1392
1507
|
async countAll() {
|
|
1393
1508
|
const safeCount = async (table) => {
|
|
1394
1509
|
try {
|
|
1395
|
-
return await
|
|
1510
|
+
return await this.provider.count(table);
|
|
1396
1511
|
} catch {
|
|
1397
1512
|
return 0;
|
|
1398
1513
|
}
|
|
1399
1514
|
};
|
|
1400
1515
|
const [sessions, messages, topics, facts] = await Promise.all([
|
|
1401
|
-
safeCount(
|
|
1402
|
-
safeCount(
|
|
1403
|
-
safeCount(
|
|
1404
|
-
safeCount(
|
|
1516
|
+
safeCount(SESSIONS_TABLE),
|
|
1517
|
+
safeCount(MESSAGES_TABLE),
|
|
1518
|
+
safeCount(TOPICS_TABLE),
|
|
1519
|
+
safeCount(FACTS_TABLE)
|
|
1405
1520
|
]);
|
|
1406
1521
|
return { sessions, messages, topics, facts };
|
|
1407
1522
|
}
|
|
@@ -1422,16 +1537,20 @@ var LanceService = class {
|
|
|
1422
1537
|
};
|
|
1423
1538
|
const readCreatedAt = async (table) => {
|
|
1424
1539
|
try {
|
|
1425
|
-
const rows = await
|
|
1540
|
+
const rows = await this.provider.query(table, {
|
|
1541
|
+
select: ["created_at"],
|
|
1542
|
+
filter: [{ op: "gte", field: "created_at", value: startMs }],
|
|
1543
|
+
limit: 1e6
|
|
1544
|
+
});
|
|
1426
1545
|
return rows.map((r) => Number(r.created_at)).filter((n) => Number.isFinite(n));
|
|
1427
1546
|
} catch {
|
|
1428
1547
|
return [];
|
|
1429
1548
|
}
|
|
1430
1549
|
};
|
|
1431
1550
|
const [sessionTs, messageTs, factTs] = await Promise.all([
|
|
1432
|
-
readCreatedAt(
|
|
1433
|
-
readCreatedAt(
|
|
1434
|
-
readCreatedAt(
|
|
1551
|
+
readCreatedAt(SESSIONS_TABLE),
|
|
1552
|
+
readCreatedAt(MESSAGES_TABLE),
|
|
1553
|
+
readCreatedAt(FACTS_TABLE)
|
|
1435
1554
|
]);
|
|
1436
1555
|
const buckets = /* @__PURE__ */ new Map();
|
|
1437
1556
|
for (let i = 0; i < days; i++) {
|
|
@@ -1451,29 +1570,25 @@ var LanceService = class {
|
|
|
1451
1570
|
}
|
|
1452
1571
|
// ── 知识库:documents / chunks ─────────────────────────────────────────────
|
|
1453
1572
|
async addDocument(doc) {
|
|
1454
|
-
await this.
|
|
1455
|
-
if (this.isNewDocumentsTable) {
|
|
1456
|
-
await this.ensureIndexes(this.documentsTable, DOCUMENTS_INDEXES);
|
|
1457
|
-
this.isNewDocumentsTable = false;
|
|
1458
|
-
}
|
|
1573
|
+
await this.provider.add(DOCUMENTS_TABLE, [documentToRow(doc)]);
|
|
1459
1574
|
}
|
|
1460
1575
|
async addChunks(chunks) {
|
|
1461
1576
|
if (chunks.length === 0) return;
|
|
1462
|
-
await this.
|
|
1463
|
-
if (this.isNewChunksTable) {
|
|
1464
|
-
await this.ensureIndexes(this.chunksTable, CHUNKS_INDEXES);
|
|
1465
|
-
this.isNewChunksTable = false;
|
|
1466
|
-
}
|
|
1577
|
+
await this.provider.add(CHUNKS_TABLE, chunks.map(chunkToRow));
|
|
1467
1578
|
}
|
|
1468
1579
|
async updateDocumentGraphFlag(docId, hasGraph) {
|
|
1469
|
-
await this.
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1580
|
+
await this.provider.update(
|
|
1581
|
+
DOCUMENTS_TABLE,
|
|
1582
|
+
{ has_graph: hasGraph ? 1 : 0, updated_at: Date.now() },
|
|
1583
|
+
[eq("doc_id", docId)]
|
|
1584
|
+
);
|
|
1473
1585
|
}
|
|
1474
1586
|
async getDocument(docId) {
|
|
1475
1587
|
try {
|
|
1476
|
-
const rows = await this.
|
|
1588
|
+
const rows = await this.provider.query(DOCUMENTS_TABLE, {
|
|
1589
|
+
filter: [eq("doc_id", docId)],
|
|
1590
|
+
limit: 1
|
|
1591
|
+
});
|
|
1477
1592
|
if (rows.length === 0) return null;
|
|
1478
1593
|
return rowToDocument(rows[0]);
|
|
1479
1594
|
} catch {
|
|
@@ -1483,8 +1598,10 @@ var LanceService = class {
|
|
|
1483
1598
|
/** 查找同域同 hash 的文档(去重用)*/
|
|
1484
1599
|
async findDocumentByHash(contentHash, filter) {
|
|
1485
1600
|
try {
|
|
1486
|
-
const
|
|
1487
|
-
|
|
1601
|
+
const rows = await this.provider.query(DOCUMENTS_TABLE, {
|
|
1602
|
+
filter: [eq("content_hash", contentHash), ...filter ?? []],
|
|
1603
|
+
limit: 1
|
|
1604
|
+
});
|
|
1488
1605
|
if (rows.length === 0) return null;
|
|
1489
1606
|
return rowToDocument(rows[0]);
|
|
1490
1607
|
} catch {
|
|
@@ -1494,9 +1611,11 @@ var LanceService = class {
|
|
|
1494
1611
|
/** 按域过滤返回 docId 列表(chunkRedundantIds=false 时用于 chunk 过滤)*/
|
|
1495
1612
|
async getDocIdsByDomain(filter) {
|
|
1496
1613
|
try {
|
|
1497
|
-
const
|
|
1498
|
-
|
|
1499
|
-
|
|
1614
|
+
const rows = await this.provider.query(DOCUMENTS_TABLE, {
|
|
1615
|
+
select: ["doc_id"],
|
|
1616
|
+
filter,
|
|
1617
|
+
limit: 1e5
|
|
1618
|
+
});
|
|
1500
1619
|
return [...new Set(rows.map((r) => r.doc_id))];
|
|
1501
1620
|
} catch {
|
|
1502
1621
|
return [];
|
|
@@ -1505,13 +1624,8 @@ var LanceService = class {
|
|
|
1505
1624
|
/** 文档级粗召回:对摘要向量做向量搜索,定位候选文档 */
|
|
1506
1625
|
async searchDocuments(vector, filter, limit = 5) {
|
|
1507
1626
|
try {
|
|
1508
|
-
const
|
|
1509
|
-
|
|
1510
|
-
const rows = await q.toArray();
|
|
1511
|
-
return rows.map((r) => ({
|
|
1512
|
-
...rowToDocument(r),
|
|
1513
|
-
_distance: r._distance
|
|
1514
|
-
}));
|
|
1627
|
+
const rows = await this.provider.vectorSearch(DOCUMENTS_TABLE, vector, { filter, limit });
|
|
1628
|
+
return rows.map((r) => ({ ...rowToDocument(r), _distance: r._distance }));
|
|
1515
1629
|
} catch {
|
|
1516
1630
|
return [];
|
|
1517
1631
|
}
|
|
@@ -1519,32 +1633,30 @@ var LanceService = class {
|
|
|
1519
1633
|
/** 知识片段混合检索(BM25 + 向量),失败回退纯向量 */
|
|
1520
1634
|
async searchChunks(query, vector, filter, limit = 8) {
|
|
1521
1635
|
try {
|
|
1522
|
-
const
|
|
1523
|
-
if (
|
|
1524
|
-
|
|
1636
|
+
const fused = await this.hybridSearch(CHUNKS_TABLE, ["content"], query, vector, filter, limit);
|
|
1637
|
+
if (fused.length > 0) return fused.slice(0, limit).map(rowToChunk);
|
|
1638
|
+
} catch {
|
|
1639
|
+
}
|
|
1640
|
+
try {
|
|
1641
|
+
const rows = await this.provider.vectorSearch(CHUNKS_TABLE, vector, { filter, limit });
|
|
1525
1642
|
return rows.map(rowToChunk);
|
|
1526
1643
|
} catch {
|
|
1527
|
-
|
|
1528
|
-
const q = this.chunksTable.vectorSearch(vector).limit(limit);
|
|
1529
|
-
if (filter) q.where(filter);
|
|
1530
|
-
const rows = await q.toArray();
|
|
1531
|
-
return rows.map(rowToChunk);
|
|
1532
|
-
} catch {
|
|
1533
|
-
return [];
|
|
1534
|
-
}
|
|
1644
|
+
return [];
|
|
1535
1645
|
}
|
|
1536
1646
|
}
|
|
1537
1647
|
async deleteDocument(docId) {
|
|
1538
|
-
await this.
|
|
1648
|
+
await this.provider.deleteWhere(DOCUMENTS_TABLE, [eq("doc_id", docId)]);
|
|
1539
1649
|
}
|
|
1540
1650
|
async deleteChunksByDoc(docId) {
|
|
1541
|
-
await this.
|
|
1651
|
+
await this.provider.deleteWhere(CHUNKS_TABLE, [eq("doc_id", docId)]);
|
|
1542
1652
|
}
|
|
1543
1653
|
/** 全量读取文档(管理面板用),按更新时间倒序 */
|
|
1544
1654
|
async getAllDocuments() {
|
|
1545
1655
|
try {
|
|
1546
|
-
const rows = await this.
|
|
1547
|
-
return rows.sort(
|
|
1656
|
+
const rows = await this.provider.query(DOCUMENTS_TABLE);
|
|
1657
|
+
return rows.sort(
|
|
1658
|
+
(a, b) => Number(b.updated_at) - Number(a.updated_at) || cmpStr(a.doc_id, b.doc_id)
|
|
1659
|
+
).map(rowToDocument);
|
|
1548
1660
|
} catch {
|
|
1549
1661
|
return [];
|
|
1550
1662
|
}
|
|
@@ -1553,68 +1665,33 @@ var LanceService = class {
|
|
|
1553
1665
|
async countKnowledge() {
|
|
1554
1666
|
const safeCount = async (table) => {
|
|
1555
1667
|
try {
|
|
1556
|
-
return await
|
|
1668
|
+
return await this.provider.count(table);
|
|
1557
1669
|
} catch {
|
|
1558
1670
|
return 0;
|
|
1559
1671
|
}
|
|
1560
1672
|
};
|
|
1561
1673
|
const [documents, chunks] = await Promise.all([
|
|
1562
|
-
safeCount(
|
|
1563
|
-
safeCount(
|
|
1674
|
+
safeCount(DOCUMENTS_TABLE),
|
|
1675
|
+
safeCount(CHUNKS_TABLE)
|
|
1564
1676
|
]);
|
|
1565
1677
|
return { documents, chunks };
|
|
1566
1678
|
}
|
|
1567
1679
|
/**
|
|
1568
|
-
* 根据 metadata
|
|
1680
|
+
* 根据 metadata 字段内容构建结构化过滤条件。
|
|
1569
1681
|
*
|
|
1570
|
-
* metadata 以 JSON 字符串存储,此方法将键值对转换为
|
|
1682
|
+
* metadata 以 JSON 字符串存储,此方法将键值对转换为 jsonContains 条件,
|
|
1571
1683
|
* 可直接传给 searchMessages / hybridSearchTopics 的 filter 参数。
|
|
1572
1684
|
*
|
|
1573
|
-
* 示例:buildMetadataFilter({ env: "prod", version: 2 })
|
|
1574
|
-
* → `metadata LIKE '%"env":"prod"%' AND metadata LIKE '%"version":2%'`
|
|
1575
|
-
*
|
|
1576
1685
|
* 注意:仅适合简单标量值(字符串、数字、布尔)的精确匹配。
|
|
1577
1686
|
* 复杂嵌套对象或含空格的 JSON 值可能无法可靠匹配。
|
|
1578
1687
|
*/
|
|
1579
1688
|
static buildMetadataFilter(conditions) {
|
|
1580
|
-
return Object.entries(conditions).map(([key, value]) => {
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
};
|
|
1586
|
-
|
|
1587
|
-
// src/manager/semaphore.ts
|
|
1588
|
-
var Semaphore = class {
|
|
1589
|
-
count;
|
|
1590
|
-
queue = [];
|
|
1591
|
-
constructor(max) {
|
|
1592
|
-
this.count = max;
|
|
1593
|
-
}
|
|
1594
|
-
async run(fn) {
|
|
1595
|
-
await this.acquire();
|
|
1596
|
-
try {
|
|
1597
|
-
return await fn();
|
|
1598
|
-
} finally {
|
|
1599
|
-
this.release();
|
|
1600
|
-
}
|
|
1601
|
-
}
|
|
1602
|
-
acquire() {
|
|
1603
|
-
if (this.count > 0) {
|
|
1604
|
-
this.count--;
|
|
1605
|
-
return Promise.resolve();
|
|
1606
|
-
}
|
|
1607
|
-
return new Promise((resolve) => {
|
|
1608
|
-
this.queue.push(resolve);
|
|
1609
|
-
});
|
|
1610
|
-
}
|
|
1611
|
-
release() {
|
|
1612
|
-
const next = this.queue.shift();
|
|
1613
|
-
if (next) {
|
|
1614
|
-
next();
|
|
1615
|
-
} else {
|
|
1616
|
-
this.count++;
|
|
1617
|
-
}
|
|
1689
|
+
return Object.entries(conditions).map(([key, value]) => ({
|
|
1690
|
+
op: "jsonContains",
|
|
1691
|
+
field: "metadata",
|
|
1692
|
+
key,
|
|
1693
|
+
value
|
|
1694
|
+
}));
|
|
1618
1695
|
}
|
|
1619
1696
|
};
|
|
1620
1697
|
|
|
@@ -1690,7 +1767,10 @@ var LlmService = class {
|
|
|
1690
1767
|
* 仅在提示词明确要求 JSON 时启用;返回自然语言的调用(如 summarizeSearchResults)
|
|
1691
1768
|
* 必须传 false,否则部分端点会因 "messages 未含 json 字样" 而 400。
|
|
1692
1769
|
*/
|
|
1693
|
-
async chatCompletion(systemPrompt, userPrompt, jsonMode = true) {
|
|
1770
|
+
async chatCompletion(systemPrompt, userPrompt, jsonMode = true, maxTokens) {
|
|
1771
|
+
return (await this.chatCompletionWithUsage(systemPrompt, userPrompt, jsonMode, maxTokens)).content;
|
|
1772
|
+
}
|
|
1773
|
+
async chatCompletionWithUsage(systemPrompt, userPrompt, jsonMode = true, maxTokens) {
|
|
1694
1774
|
const url = `${this.config.llmBaseUrl.replace(/\/$/, "")}/chat/completions`;
|
|
1695
1775
|
const body = {
|
|
1696
1776
|
model: this.config.llmModel,
|
|
@@ -1701,6 +1781,7 @@ var LlmService = class {
|
|
|
1701
1781
|
]
|
|
1702
1782
|
};
|
|
1703
1783
|
if (jsonMode) body.response_format = { type: "json_object" };
|
|
1784
|
+
if (maxTokens != null) body.max_tokens = maxTokens;
|
|
1704
1785
|
const data = await postJsonWithRetry(
|
|
1705
1786
|
url,
|
|
1706
1787
|
{ Authorization: `Bearer ${this.config.llmApiKey}` },
|
|
@@ -1708,7 +1789,10 @@ var LlmService = class {
|
|
|
1708
1789
|
{ timeoutMs: this.config.httpTimeoutMs, maxRetries: this.config.httpMaxRetries },
|
|
1709
1790
|
"LLM API"
|
|
1710
1791
|
);
|
|
1711
|
-
return
|
|
1792
|
+
return {
|
|
1793
|
+
content: data.choices[0].message.content ?? "",
|
|
1794
|
+
completionTokens: data.usage?.completion_tokens
|
|
1795
|
+
};
|
|
1712
1796
|
}
|
|
1713
1797
|
// ── 带工具调用的请求 ──────────────────────────────────────────────────────────
|
|
1714
1798
|
async chatWithTools(systemPrompt, userPrompt) {
|
|
@@ -1731,27 +1815,55 @@ var LlmService = class {
|
|
|
1731
1815
|
);
|
|
1732
1816
|
return data.choices[0].message;
|
|
1733
1817
|
}
|
|
1734
|
-
// ──
|
|
1818
|
+
// ── 对话 Topic 摘要(JSON 输出)──────────────────────────────────────────────
|
|
1735
1819
|
async summarizeMessages(messages) {
|
|
1736
1820
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1737
|
-
const {
|
|
1821
|
+
const { topicSummaryMaxTokens } = this.config;
|
|
1738
1822
|
const systemPrompt = `\u4F60\u662F\u4E00\u4E2A\u5BF9\u8BDD\u8BB0\u5FC6\u7BA1\u7406\u52A9\u624B\u3002\u5F53\u524D\u7CFB\u7EDF\u65F6\u95F4\uFF1A${now}\u3002
|
|
1739
|
-
\u8BF7\
|
|
1823
|
+
\u8BF7\u628A\u7ED9\u5B9A\u5BF9\u8BDD\u538B\u7F29\u6210\u4E00\u4E2A\u53EF\u957F\u671F\u590D\u7528\u7684\u4E3B\u9898\u8BB0\u5FC6\u3002
|
|
1740
1824
|
|
|
1741
1825
|
\u8981\u6C42\uFF1A
|
|
1742
1826
|
- title\uFF1A\u4E3B\u9898\u6807\u9898\uFF0C\u4E00\u53E5\u8BDD\uFF08\u5EFA\u8BAE\u4E0D\u8D85\u8FC7 20 \u5B57\uFF09\uFF0C\u9AD8\u5EA6\u6982\u62EC\u8FD9\u6BB5\u5BF9\u8BDD\u7684\u4E3B\u9898\uFF0C\u7528\u4E8E\u5217\u8868\u5C55\u793A
|
|
1743
|
-
-
|
|
1744
|
-
-
|
|
1745
|
-
-
|
|
1827
|
+
- summary\uFF1A\u552F\u4E00\u7684\u6458\u8981\u6B63\u6587\uFF0C\u6700\u591A ${topicSummaryMaxTokens} tokens
|
|
1828
|
+
- \u6839\u636E\u91CD\u8981\u6027\u52A8\u6001\u51B3\u5B9A\u7BC7\u5E45\uFF1A\u7528\u6237\u7A33\u5B9A\u504F\u597D\u3001\u660E\u786E\u4E8B\u5B9E\u3001\u51B3\u5B9A\u4E0E\u7406\u7531\u3001\u957F\u671F\u7EA6\u675F\u3001\u672A\u89E3\u51B3\u963B\u585E\u3001\u53EF\u590D\u7528\u7ECF\u9A8C\u8981\u8BE6\u7EC6\u4FDD\u7559\uFF1B\u91CD\u590D\u8868\u8FF0\u3001\u5BD2\u6684\u548C\u4E00\u6B21\u6027\u8FC7\u7A0B\u53EF\u6781\u77ED\u6216\u7701\u7565
|
|
1829
|
+
- \u4FDD\u7559\u8C01\u5728\u4F55\u65F6\u505A\u4E86\u4EC0\u4E48\u3001\u5173\u952E\u6807\u8BC6\u7B26\u3001\u6570\u503C\u3001\u7ED3\u8BBA\u3001\u5931\u8D25\u539F\u56E0\u548C\u540E\u7EED\u52A8\u4F5C\uFF1B\u4E0D\u8981\u865A\u6784
|
|
1830
|
+
- metadata\u3001\u7ED3\u6784\u5316 parts \u548C payload \u4E2D\u5F71\u54CD\u8BED\u4E49\u7684\u4FE1\u606F\u540C\u6837\u5C5E\u4E8E\u8BB0\u5FC6\u5185\u5BB9
|
|
1746
1831
|
|
|
1747
1832
|
\u8F93\u51FA\u5FC5\u987B\u662F\u5408\u6CD5\u7684 JSON \u5BF9\u8C61\uFF0C\u4E0D\u8981\u5305\u542B\u4EFB\u4F55 markdown \u6807\u8BB0\uFF0C\u683C\u5F0F\u5982\u4E0B\uFF1A
|
|
1748
|
-
{"title": "...", "
|
|
1749
|
-
const formatted = messages.map((m) =>
|
|
1750
|
-
const
|
|
1833
|
+
{"title": "...", "summary": "..."}`;
|
|
1834
|
+
const formatted = messages.map((m) => formatStoredMessage(m)).join("\n");
|
|
1835
|
+
const response = await this.chatCompletionWithUsage(
|
|
1836
|
+
systemPrompt,
|
|
1837
|
+
`\u8BF7\u538B\u7F29\u4EE5\u4E0B\u5BF9\u8BDD\uFF1A
|
|
1751
1838
|
|
|
1752
|
-
${formatted}
|
|
1753
|
-
|
|
1754
|
-
|
|
1839
|
+
${formatted}`,
|
|
1840
|
+
true,
|
|
1841
|
+
topicSummaryMaxTokens + 128
|
|
1842
|
+
);
|
|
1843
|
+
const parsed = JSON.parse(response.content);
|
|
1844
|
+
return {
|
|
1845
|
+
title: parsed.title ?? "",
|
|
1846
|
+
summary: parsed.summary ?? "",
|
|
1847
|
+
tokens: response.completionTokens
|
|
1848
|
+
};
|
|
1849
|
+
}
|
|
1850
|
+
async summarizeTopics(topics) {
|
|
1851
|
+
const input = topics.map(
|
|
1852
|
+
(topic) => `[${new Date(topic.startTime).toISOString()} - ${new Date(topic.endTime).toISOString()}] ${topic.title}
|
|
1853
|
+
${topic.summary}`
|
|
1854
|
+
).join("\n\n");
|
|
1855
|
+
const response = await this.chatCompletionWithUsage(
|
|
1856
|
+
`\u4F60\u662F\u8BB0\u5FC6\u5F52\u5E76\u52A9\u624B\u3002\u628A\u4E00\u7EC4\u6309\u65F6\u95F4\u6392\u5217\u7684\u65E7\u4E3B\u9898\u5F52\u5E76\u6210\u4E00\u4E2A\u4E3B\u9898\u3002\u4FDD\u7559\u7A33\u5B9A\u504F\u597D\u3001\u4E8B\u5B9E\u3001\u51B3\u5B9A\u53CA\u7406\u7531\u3001\u957F\u671F\u7EA6\u675F\u3001\u963B\u585E\u4E0E\u53EF\u590D\u7528\u7ECF\u9A8C\uFF1B\u5220\u9664\u91CD\u590D\u5185\u5BB9\u3002summary \u6700\u591A ${this.config.topicSummaryMaxTokens} tokens\u3002\u53EA\u8F93\u51FA JSON\uFF1A{"title":"...","summary":"..."}`,
|
|
1857
|
+
input,
|
|
1858
|
+
true,
|
|
1859
|
+
this.config.topicSummaryMaxTokens + 128
|
|
1860
|
+
);
|
|
1861
|
+
const parsed = JSON.parse(response.content);
|
|
1862
|
+
return {
|
|
1863
|
+
title: parsed.title ?? "\u5386\u53F2\u4E3B\u9898\u5F52\u5E76",
|
|
1864
|
+
summary: parsed.summary ?? "",
|
|
1865
|
+
tokens: response.completionTokens
|
|
1866
|
+
};
|
|
1755
1867
|
}
|
|
1756
1868
|
// ── 第二步:通过工具调用提取实体和关系 ──────────────────────────────────────
|
|
1757
1869
|
async extractEntitiesFromMessages(messages) {
|
|
@@ -1794,9 +1906,8 @@ ${formatted}`
|
|
|
1794
1906
|
]);
|
|
1795
1907
|
return {
|
|
1796
1908
|
title: summary.title,
|
|
1797
|
-
detail: summary.detail,
|
|
1798
1909
|
summary: summary.summary,
|
|
1799
|
-
|
|
1910
|
+
tokens: summary.tokens,
|
|
1800
1911
|
entities: extraction.entities,
|
|
1801
1912
|
relations: extraction.relations
|
|
1802
1913
|
};
|
|
@@ -1885,146 +1996,243 @@ ${text}`
|
|
|
1885
1996
|
${context}`, false);
|
|
1886
1997
|
}
|
|
1887
1998
|
};
|
|
1999
|
+
function formatStoredMessage(message) {
|
|
2000
|
+
const extras = [message.parts, message.metadata, message.payload].filter((value) => value && value !== "[]" && value !== "{}").join(" ");
|
|
2001
|
+
return `[${new Date(message.createdAt).toISOString()}] ${message.talkerId || "user"}: ${message.content}${extras ? `
|
|
2002
|
+
meta=${extras}` : ""}`;
|
|
2003
|
+
}
|
|
1888
2004
|
|
|
1889
2005
|
// src/manager/compress.manager.ts
|
|
1890
2006
|
import { v4 as uuidv4 } from "uuid";
|
|
2007
|
+
|
|
2008
|
+
// src/manager/semaphore.ts
|
|
2009
|
+
var Semaphore = class {
|
|
2010
|
+
count;
|
|
2011
|
+
queue = [];
|
|
2012
|
+
constructor(max) {
|
|
2013
|
+
this.count = max;
|
|
2014
|
+
}
|
|
2015
|
+
async run(fn) {
|
|
2016
|
+
await this.acquire();
|
|
2017
|
+
try {
|
|
2018
|
+
return await fn();
|
|
2019
|
+
} finally {
|
|
2020
|
+
this.release();
|
|
2021
|
+
}
|
|
2022
|
+
}
|
|
2023
|
+
acquire() {
|
|
2024
|
+
if (this.count > 0) {
|
|
2025
|
+
this.count--;
|
|
2026
|
+
return Promise.resolve();
|
|
2027
|
+
}
|
|
2028
|
+
return new Promise((resolve) => {
|
|
2029
|
+
this.queue.push(resolve);
|
|
2030
|
+
});
|
|
2031
|
+
}
|
|
2032
|
+
release() {
|
|
2033
|
+
const next = this.queue.shift();
|
|
2034
|
+
if (next) {
|
|
2035
|
+
next();
|
|
2036
|
+
} else {
|
|
2037
|
+
this.count++;
|
|
2038
|
+
}
|
|
2039
|
+
}
|
|
2040
|
+
};
|
|
2041
|
+
|
|
2042
|
+
// src/manager/compress.manager.ts
|
|
1891
2043
|
var CompressManager = class {
|
|
1892
|
-
config
|
|
1893
|
-
lance;
|
|
1894
|
-
grafeo;
|
|
1895
|
-
llm;
|
|
1896
|
-
embed;
|
|
1897
|
-
sessionCache;
|
|
1898
|
-
semaphore;
|
|
1899
|
-
sessionChain = /* @__PURE__ */ new Map();
|
|
1900
|
-
constructor(config, lance, grafeo, llm, embed, sessionCache) {
|
|
2044
|
+
constructor(config, store, grafeo, llm, embed, sessionCache) {
|
|
1901
2045
|
this.config = config;
|
|
1902
|
-
this.
|
|
2046
|
+
this.store = store;
|
|
1903
2047
|
this.grafeo = grafeo;
|
|
1904
2048
|
this.llm = llm;
|
|
1905
2049
|
this.embed = embed;
|
|
1906
2050
|
this.sessionCache = sessionCache;
|
|
1907
2051
|
this.semaphore = new Semaphore(config.maxConcurrentCompressions);
|
|
1908
2052
|
}
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
2053
|
+
semaphore;
|
|
2054
|
+
sessionChain = /* @__PURE__ */ new Map();
|
|
2055
|
+
backgroundGraphs = /* @__PURE__ */ new Set();
|
|
2056
|
+
triggerCompress(sessionId, force = false, waitGraph = false, rawLimit) {
|
|
2057
|
+
const previous = this.sessionChain.get(sessionId) ?? Promise.resolve();
|
|
2058
|
+
const next = previous.then(
|
|
2059
|
+
() => this.semaphore.run(() => this.doCompress(sessionId, force, waitGraph, rawLimit))
|
|
1913
2060
|
);
|
|
1914
|
-
|
|
1915
|
-
})
|
|
2061
|
+
const settled = next.catch(() => {
|
|
2062
|
+
});
|
|
2063
|
+
this.sessionChain.set(sessionId, settled);
|
|
2064
|
+
settled.finally(() => {
|
|
2065
|
+
if (this.sessionChain.get(sessionId) === settled) this.sessionChain.delete(sessionId);
|
|
2066
|
+
});
|
|
1916
2067
|
return next;
|
|
1917
2068
|
}
|
|
1918
|
-
async
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
} catch (err) {
|
|
1931
|
-
console.error(`[CompressManager] LLM compress failed for session ${sessionId}: ${formatError(err)}`);
|
|
1932
|
-
return;
|
|
2069
|
+
async waitForIdle(sessionId) {
|
|
2070
|
+
await (this.sessionChain.get(sessionId) ?? Promise.resolve());
|
|
2071
|
+
}
|
|
2072
|
+
isBusy(sessionId) {
|
|
2073
|
+
return this.sessionChain.has(sessionId);
|
|
2074
|
+
}
|
|
2075
|
+
async waitForAllIdle() {
|
|
2076
|
+
while (this.sessionChain.size > 0 || this.backgroundGraphs.size > 0) {
|
|
2077
|
+
await Promise.all([
|
|
2078
|
+
...this.sessionChain.values(),
|
|
2079
|
+
...this.backgroundGraphs
|
|
2080
|
+
]);
|
|
1933
2081
|
}
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
2082
|
+
}
|
|
2083
|
+
async doCompress(sessionId, force, waitGraph, rawLimitOverride) {
|
|
2084
|
+
const entry = this.sessionCache.getEntry(sessionId);
|
|
2085
|
+
if (!entry || entry.messages.length === 0) return;
|
|
2086
|
+
const rawLimit = Math.max(1, rawLimitOverride ?? this.rawLimit(entry.lastModelContextTokens));
|
|
2087
|
+
const threshold = Math.max(1, Math.floor(rawLimit * this.config.precompressionRatio));
|
|
2088
|
+
if (!force && entry.totalTokens < threshold) return;
|
|
2089
|
+
let batchTarget = Math.max(1, Math.floor(rawLimit * this.config.compressionBatchRatio));
|
|
2090
|
+
if (this.config.compressionBatchTokenLimit > 0) {
|
|
2091
|
+
batchTarget = Math.min(batchTarget, this.config.compressionBatchTokenLimit);
|
|
2092
|
+
}
|
|
2093
|
+
const messages = this.sessionCache.selectOldestMessageBatch(sessionId, batchTarget);
|
|
2094
|
+
if (messages.length === 0) return;
|
|
2095
|
+
let result;
|
|
1937
2096
|
try {
|
|
1938
|
-
|
|
1939
|
-
} catch (
|
|
1940
|
-
console.error(
|
|
2097
|
+
result = await this.llm.summarizeMessages(messages);
|
|
2098
|
+
} catch (error) {
|
|
2099
|
+
console.error(
|
|
2100
|
+
`[CompressManager] LLM compress failed for session ${sessionId}: ${formatError(error)}`
|
|
2101
|
+
);
|
|
2102
|
+
throw error;
|
|
1941
2103
|
}
|
|
2104
|
+
const summary = result.summary.trim();
|
|
2105
|
+
if (!summary) throw new Error(`LLM returned an empty summary for session ${sessionId}`);
|
|
2106
|
+
const vector = await this.embed.embedOne(summary).catch((error) => {
|
|
2107
|
+
console.error(`[CompressManager] Embed topic failed for session ${sessionId}:`, error);
|
|
2108
|
+
return [];
|
|
2109
|
+
});
|
|
2110
|
+
const first = messages[0];
|
|
2111
|
+
const last = messages[messages.length - 1];
|
|
2112
|
+
const now = Date.now();
|
|
1942
2113
|
const topic = {
|
|
1943
2114
|
summaryId: uuidv4(),
|
|
1944
2115
|
sessionId,
|
|
1945
|
-
userId,
|
|
1946
|
-
chatId,
|
|
1947
|
-
title,
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
2116
|
+
userId: entry.ids.userId,
|
|
2117
|
+
chatId: entry.ids.chatId,
|
|
2118
|
+
title: result.title,
|
|
2119
|
+
summary,
|
|
2120
|
+
tokens: result.tokens != null && result.tokens > 0 ? result.tokens : Math.max(1, countTokens(summary)),
|
|
2121
|
+
startMessageId: first.messageId,
|
|
2122
|
+
endMessageId: last.messageId,
|
|
2123
|
+
startTime: first.createdAt,
|
|
2124
|
+
endTime: last.createdAt,
|
|
2125
|
+
createdAt: now,
|
|
2126
|
+
updatedAt: now,
|
|
1955
2127
|
recallCount: 0,
|
|
1956
|
-
vector
|
|
2128
|
+
vector
|
|
1957
2129
|
};
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
console.error(`[CompressManager] Rebuild history window failed:`, err);
|
|
1970
|
-
}
|
|
1971
|
-
const graphTask = this.extractAndPersistGraph(messages, sessionId, chatId, userId, endTime);
|
|
2130
|
+
await this.store.addTopic(topic);
|
|
2131
|
+
this.sessionCache.appendTopic(sessionId, topic);
|
|
2132
|
+
this.sessionCache.removeMessages(sessionId, messages.map((message) => message.messageId));
|
|
2133
|
+
await this.compactOldTopics(sessionId);
|
|
2134
|
+
const graphTask = this.extractAndPersistGraph(
|
|
2135
|
+
messages,
|
|
2136
|
+
sessionId,
|
|
2137
|
+
entry.ids.chatId,
|
|
2138
|
+
entry.ids.userId,
|
|
2139
|
+
last.createdAt
|
|
2140
|
+
);
|
|
1972
2141
|
if (waitGraph) {
|
|
1973
2142
|
await withTimeout(graphTask, this.config.graphBuildTimeoutMs, "flushChat graph build");
|
|
1974
2143
|
} else {
|
|
1975
|
-
graphTask.catch((
|
|
1976
|
-
console.error(`[CompressManager] Background graph persist failed:`,
|
|
2144
|
+
const tracked = graphTask.catch((error) => {
|
|
2145
|
+
console.error(`[CompressManager] Background graph persist failed:`, error);
|
|
1977
2146
|
});
|
|
2147
|
+
this.backgroundGraphs.add(tracked);
|
|
2148
|
+
tracked.finally(() => this.backgroundGraphs.delete(tracked));
|
|
1978
2149
|
}
|
|
1979
2150
|
}
|
|
2151
|
+
async compactOldTopics(sessionId) {
|
|
2152
|
+
while (true) {
|
|
2153
|
+
const entry = this.sessionCache.getEntry(sessionId);
|
|
2154
|
+
if (!entry || entry.topicTokens <= this.config.compressedContextTokenLimit) return;
|
|
2155
|
+
const target = Math.max(1, Math.floor(this.config.compressedContextTokenLimit / 2));
|
|
2156
|
+
const selected = [];
|
|
2157
|
+
let tokens = 0;
|
|
2158
|
+
for (const topic of entry.topics) {
|
|
2159
|
+
selected.push(topic);
|
|
2160
|
+
tokens += topic.tokens;
|
|
2161
|
+
if (tokens >= target && selected.length >= 2) break;
|
|
2162
|
+
}
|
|
2163
|
+
if (selected.length < 2) {
|
|
2164
|
+
this.sessionCache.setTopics(sessionId, entry.topics, true);
|
|
2165
|
+
return;
|
|
2166
|
+
}
|
|
2167
|
+
console.info(
|
|
2168
|
+
`[CompressManager] session ${sessionId} compressed Topic context exceeds ${this.config.compressedContextTokenLimit} tokens; compacting ${selected.length} old topics.`
|
|
2169
|
+
);
|
|
2170
|
+
const result = await this.llm.summarizeTopics(selected);
|
|
2171
|
+
const summary = result.summary.trim();
|
|
2172
|
+
if (!summary) throw new Error(`LLM returned an empty Topic rollup for session ${sessionId}`);
|
|
2173
|
+
const vector = await this.embed.embedOne(summary).catch(() => []);
|
|
2174
|
+
const first = selected[0];
|
|
2175
|
+
const last = selected[selected.length - 1];
|
|
2176
|
+
const now = Date.now();
|
|
2177
|
+
const rollup = {
|
|
2178
|
+
summaryId: uuidv4(),
|
|
2179
|
+
sessionId,
|
|
2180
|
+
userId: entry.ids.userId,
|
|
2181
|
+
chatId: entry.ids.chatId,
|
|
2182
|
+
title: result.title,
|
|
2183
|
+
summary,
|
|
2184
|
+
tokens: result.tokens != null && result.tokens > 0 ? result.tokens : Math.max(1, countTokens(summary)),
|
|
2185
|
+
startMessageId: first.startMessageId,
|
|
2186
|
+
endMessageId: last.endMessageId,
|
|
2187
|
+
startTime: first.startTime,
|
|
2188
|
+
endTime: last.endTime,
|
|
2189
|
+
createdAt: now,
|
|
2190
|
+
updatedAt: now,
|
|
2191
|
+
recallCount: 0,
|
|
2192
|
+
vector
|
|
2193
|
+
};
|
|
2194
|
+
const removedIds = selected.map((topic) => topic.summaryId);
|
|
2195
|
+
await this.store.addTopic(rollup);
|
|
2196
|
+
this.sessionCache.replaceTopics(sessionId, removedIds, rollup);
|
|
2197
|
+
await this.store.deleteTopicsByIds(removedIds);
|
|
2198
|
+
}
|
|
2199
|
+
}
|
|
2200
|
+
rawLimit(modelContextTokens = this.config.defaultModelContextTokens) {
|
|
2201
|
+
const usable = Math.floor(modelContextTokens * this.config.contextUsageRatio);
|
|
2202
|
+
return Math.max(1, usable - this.config.compressedContextTokenLimit);
|
|
2203
|
+
}
|
|
1980
2204
|
async extractAndPersistGraph(messages, sessionId, chatId, userId, endTime) {
|
|
1981
2205
|
const { entities, relations } = await this.llm.extractEntitiesFromMessages(messages);
|
|
1982
2206
|
if (entities.length === 0 && relations.length === 0) return;
|
|
1983
2207
|
await this.persistGraph(entities, relations, sessionId, chatId, userId, endTime);
|
|
1984
2208
|
}
|
|
1985
2209
|
async persistGraph(rawEntities, rawRelations, sessionId, chatId, userId, messageTime) {
|
|
1986
|
-
const entities = rawEntities.map((
|
|
1987
|
-
name:
|
|
1988
|
-
type:
|
|
1989
|
-
meta: {
|
|
1990
|
-
sessionId,
|
|
1991
|
-
chatId,
|
|
1992
|
-
userId,
|
|
1993
|
-
messageTime,
|
|
1994
|
-
...e.meta
|
|
1995
|
-
}
|
|
2210
|
+
const entities = rawEntities.map((entity) => ({
|
|
2211
|
+
name: entity.name,
|
|
2212
|
+
type: entity.type,
|
|
2213
|
+
meta: { sessionId, chatId, userId, messageTime, ...entity.meta }
|
|
1996
2214
|
}));
|
|
1997
|
-
const relations = rawRelations.map((
|
|
1998
|
-
from:
|
|
1999
|
-
to:
|
|
2000
|
-
type:
|
|
2001
|
-
happenedAt:
|
|
2002
|
-
meta: {
|
|
2003
|
-
sessionId,
|
|
2004
|
-
chatId,
|
|
2005
|
-
userId,
|
|
2006
|
-
messageTime,
|
|
2007
|
-
...r.meta
|
|
2008
|
-
}
|
|
2215
|
+
const relations = rawRelations.map((relation) => ({
|
|
2216
|
+
from: relation.from,
|
|
2217
|
+
to: relation.to,
|
|
2218
|
+
type: relation.type,
|
|
2219
|
+
happenedAt: relation.happenedAt,
|
|
2220
|
+
meta: { sessionId, chatId, userId, messageTime, ...relation.meta }
|
|
2009
2221
|
}));
|
|
2010
|
-
const
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
await this.grafeo.upsertEntitiesAndRelations(entities, relations, embeddings);
|
|
2021
|
-
} catch (err) {
|
|
2022
|
-
console.error(`[CompressManager] Upsert entities/relations failed:`, err);
|
|
2023
|
-
}
|
|
2222
|
+
const names = [
|
|
2223
|
+
.../* @__PURE__ */ new Set([
|
|
2224
|
+
...entities.map((entity) => entity.name),
|
|
2225
|
+
...relations.flatMap((relation) => [relation.from, relation.to])
|
|
2226
|
+
])
|
|
2227
|
+
];
|
|
2228
|
+
if (names.length === 0) return;
|
|
2229
|
+
const vectors = await this.embed.embed(names);
|
|
2230
|
+
const embeddings = new Map(names.map((name, index) => [name, vectors[index]]));
|
|
2231
|
+
await this.grafeo.upsertEntitiesAndRelations(entities, relations, embeddings);
|
|
2024
2232
|
}
|
|
2025
2233
|
};
|
|
2026
|
-
function formatError(
|
|
2027
|
-
return
|
|
2234
|
+
function formatError(error) {
|
|
2235
|
+
return error instanceof Error ? error.message : String(error);
|
|
2028
2236
|
}
|
|
2029
2237
|
function withTimeout(promise, timeoutMs, label) {
|
|
2030
2238
|
let timer;
|
|
@@ -2037,13 +2245,13 @@ function withTimeout(promise, timeoutMs, label) {
|
|
|
2037
2245
|
// src/manager/fact.cache.ts
|
|
2038
2246
|
import { v4 as uuidv42 } from "uuid";
|
|
2039
2247
|
var FactCache = class {
|
|
2040
|
-
|
|
2248
|
+
store;
|
|
2041
2249
|
cache = /* @__PURE__ */ new Map();
|
|
2042
|
-
constructor(
|
|
2043
|
-
this.
|
|
2250
|
+
constructor(store) {
|
|
2251
|
+
this.store = store;
|
|
2044
2252
|
}
|
|
2045
2253
|
async init() {
|
|
2046
|
-
const facts = await this.
|
|
2254
|
+
const facts = await this.store.getAllFacts();
|
|
2047
2255
|
for (const fact of facts) {
|
|
2048
2256
|
const key = this.key(fact.level, fact.level === "user" ? fact.userId : fact.chatId);
|
|
2049
2257
|
const existing = this.cache.get(key) ?? [];
|
|
@@ -2054,22 +2262,30 @@ var FactCache = class {
|
|
|
2054
2262
|
key(level, id) {
|
|
2055
2263
|
return `${level}:${id}`;
|
|
2056
2264
|
}
|
|
2057
|
-
async add(content, level, userId, chatId, sessionId = "default") {
|
|
2265
|
+
async add(content, level, userId, chatId, sessionId = "default", key) {
|
|
2266
|
+
const id = level === "user" ? userId : chatId;
|
|
2267
|
+
const cacheKey = this.key(level, id);
|
|
2268
|
+
const existing = this.cache.get(cacheKey) ?? [];
|
|
2269
|
+
const normalizedKey = key?.trim() || void 0;
|
|
2270
|
+
const existingIndex = normalizedKey ? existing.findIndex((fact2) => fact2.key === normalizedKey) : -1;
|
|
2271
|
+
const previous = existingIndex >= 0 ? existing[existingIndex] : void 0;
|
|
2272
|
+
const now = Date.now();
|
|
2058
2273
|
const fact = {
|
|
2059
|
-
factId: uuidv42(),
|
|
2274
|
+
factId: previous?.factId ?? uuidv42(),
|
|
2060
2275
|
level,
|
|
2061
2276
|
chatId,
|
|
2062
2277
|
sessionId,
|
|
2063
2278
|
userId,
|
|
2279
|
+
key: normalizedKey,
|
|
2064
2280
|
content,
|
|
2065
|
-
createdAt:
|
|
2281
|
+
createdAt: previous?.createdAt ?? now,
|
|
2282
|
+
updatedAt: now
|
|
2066
2283
|
};
|
|
2067
|
-
await this.
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
this.cache.set(k, existing);
|
|
2284
|
+
if (previous) await this.store.updateFact(fact);
|
|
2285
|
+
else await this.store.saveFact(fact);
|
|
2286
|
+
if (existingIndex >= 0) existing[existingIndex] = fact;
|
|
2287
|
+
else existing.push(fact);
|
|
2288
|
+
this.cache.set(cacheKey, existing);
|
|
2073
2289
|
}
|
|
2074
2290
|
get(level, id) {
|
|
2075
2291
|
return this.cache.get(this.key(level, id)) ?? [];
|
|
@@ -2078,7 +2294,7 @@ var FactCache = class {
|
|
|
2078
2294
|
all() {
|
|
2079
2295
|
return [...this.cache.values()].flat();
|
|
2080
2296
|
}
|
|
2081
|
-
/** 删除单条 fact
|
|
2297
|
+
/** 删除单条 fact:同步从存储与内存缓存中移除。返回是否命中。*/
|
|
2082
2298
|
async remove(factId) {
|
|
2083
2299
|
let hit = false;
|
|
2084
2300
|
for (const [k, facts] of this.cache) {
|
|
@@ -2091,13 +2307,15 @@ var FactCache = class {
|
|
|
2091
2307
|
break;
|
|
2092
2308
|
}
|
|
2093
2309
|
}
|
|
2094
|
-
await this.
|
|
2310
|
+
await this.store.deleteFact(factId);
|
|
2095
2311
|
return hit;
|
|
2096
2312
|
}
|
|
2097
2313
|
toString(level, id) {
|
|
2098
2314
|
const facts = this.get(level, id);
|
|
2099
2315
|
if (facts.length === 0) return "";
|
|
2100
|
-
return facts.sort((a, b) => a.createdAt - b.createdAt).map(
|
|
2316
|
+
return facts.sort((a, b) => a.createdAt - b.createdAt).map(
|
|
2317
|
+
(f) => `${new Date(f.updatedAt ?? f.createdAt).toISOString()}\uFF1A${f.key ? `[${f.key}] ` : ""}${f.content}`
|
|
2318
|
+
).join("\n");
|
|
2101
2319
|
}
|
|
2102
2320
|
};
|
|
2103
2321
|
|
|
@@ -2235,14 +2453,14 @@ function takeTailTokens(text, overlapTokens) {
|
|
|
2235
2453
|
// src/manager/knowledge.manager.ts
|
|
2236
2454
|
var KnowledgeManager = class {
|
|
2237
2455
|
config;
|
|
2238
|
-
|
|
2456
|
+
store;
|
|
2239
2457
|
grafeo;
|
|
2240
2458
|
llm;
|
|
2241
2459
|
embed;
|
|
2242
2460
|
semaphore;
|
|
2243
|
-
constructor(config,
|
|
2461
|
+
constructor(config, store, grafeo, llm, embed) {
|
|
2244
2462
|
this.config = config;
|
|
2245
|
-
this.
|
|
2463
|
+
this.store = store;
|
|
2246
2464
|
this.grafeo = grafeo;
|
|
2247
2465
|
this.llm = llm;
|
|
2248
2466
|
this.embed = embed;
|
|
@@ -2257,8 +2475,9 @@ var KnowledgeManager = class {
|
|
|
2257
2475
|
const now = Date.now();
|
|
2258
2476
|
const content = opts.content;
|
|
2259
2477
|
const contentHash = createHash("sha256").update(content).digest("hex");
|
|
2260
|
-
const
|
|
2261
|
-
const
|
|
2478
|
+
const dedupeScope = opts.scope ?? "session";
|
|
2479
|
+
const domainFilter = dedupeScope === "user" ? [eq("user_id", userId)] : dedupeScope === "chat" ? [eq("chat_id", chatId)] : [eq("session_id", sessionId)];
|
|
2480
|
+
const existing = await this.store.findDocumentByHash(contentHash, domainFilter);
|
|
2262
2481
|
if (existing) return { docId: existing.docId };
|
|
2263
2482
|
const pieces = chunkMarkdown(content, {
|
|
2264
2483
|
maxTokens: this.config.chunkMaxTokens,
|
|
@@ -2291,7 +2510,7 @@ var KnowledgeManager = class {
|
|
|
2291
2510
|
createdAt: now,
|
|
2292
2511
|
updatedAt: now
|
|
2293
2512
|
};
|
|
2294
|
-
await this.
|
|
2513
|
+
await this.store.addDocument(doc);
|
|
2295
2514
|
const chunkIngest = this.semaphore.run(() => this.ingestChunks(doc, pieces));
|
|
2296
2515
|
const graphBuild = chunkIngest.then((chunks) => {
|
|
2297
2516
|
if (!shouldBuildGraph) return;
|
|
@@ -2341,7 +2560,7 @@ ${p.content}` : p.content
|
|
|
2341
2560
|
metadata: {},
|
|
2342
2561
|
createdAt: doc.createdAt
|
|
2343
2562
|
}));
|
|
2344
|
-
await this.
|
|
2563
|
+
await this.store.addChunks(chunks);
|
|
2345
2564
|
return chunks;
|
|
2346
2565
|
}
|
|
2347
2566
|
async buildDocumentGraph(doc, chunks) {
|
|
@@ -2394,7 +2613,7 @@ ${p.content}` : p.content
|
|
|
2394
2613
|
embeddings,
|
|
2395
2614
|
KIND_KNOWLEDGE
|
|
2396
2615
|
);
|
|
2397
|
-
await this.
|
|
2616
|
+
await this.store.updateDocumentGraphFlag(doc.docId, true).catch((err) => {
|
|
2398
2617
|
console.error(`[KnowledgeManager] Failed to update graph flag for doc ${doc.docId}:`, err);
|
|
2399
2618
|
});
|
|
2400
2619
|
}
|
|
@@ -2407,11 +2626,11 @@ ${p.content}` : p.content
|
|
|
2407
2626
|
const mode = opts.mode ?? "auto";
|
|
2408
2627
|
const limit = opts.limit ?? this.config.knowledgeTopK;
|
|
2409
2628
|
const vector = await this.embed.embedOne(opts.query);
|
|
2410
|
-
const docFilter =
|
|
2629
|
+
const docFilter = buildDomainFilter(scope, scopeId);
|
|
2411
2630
|
let candidateDocIds;
|
|
2412
2631
|
const docTitleMap = /* @__PURE__ */ new Map();
|
|
2413
2632
|
if (this.config.docCoarseTopK > 0) {
|
|
2414
|
-
const coarse = await this.
|
|
2633
|
+
const coarse = await this.store.searchDocuments(vector, docFilter, this.config.docCoarseTopK);
|
|
2415
2634
|
if (coarse.length > 0) {
|
|
2416
2635
|
candidateDocIds = coarse.map((d) => d.docId);
|
|
2417
2636
|
for (const d of coarse) docTitleMap.set(d.docId, d.title);
|
|
@@ -2419,7 +2638,7 @@ ${p.content}` : p.content
|
|
|
2419
2638
|
}
|
|
2420
2639
|
const chunkFilter = await this.buildChunkFilter(scope, scopeId, candidateDocIds);
|
|
2421
2640
|
if (chunkFilter === NO_MATCH) return empty;
|
|
2422
|
-
const rawChunks = await this.
|
|
2641
|
+
const rawChunks = await this.store.searchChunks(opts.query, vector, chunkFilter, limit);
|
|
2423
2642
|
const scored = rawChunks.map((c) => ({
|
|
2424
2643
|
chunkId: c.chunkId,
|
|
2425
2644
|
docId: c.docId,
|
|
@@ -2433,7 +2652,7 @@ ${p.content}` : p.content
|
|
|
2433
2652
|
[...docCount.entries()].map(async ([docId, matchedChunkCount]) => {
|
|
2434
2653
|
let title = docTitleMap.get(docId);
|
|
2435
2654
|
if (title === void 0) {
|
|
2436
|
-
const d = await this.
|
|
2655
|
+
const d = await this.store.getDocument(docId);
|
|
2437
2656
|
title = d?.title ?? "";
|
|
2438
2657
|
}
|
|
2439
2658
|
return { docId, title, matchedChunkCount };
|
|
@@ -2481,32 +2700,32 @@ ${p.content}` : p.content
|
|
|
2481
2700
|
async buildChunkFilter(scope, scopeId, candidateDocIds) {
|
|
2482
2701
|
const parts = [];
|
|
2483
2702
|
if (this.config.chunkRedundantIds) {
|
|
2484
|
-
const
|
|
2485
|
-
if (
|
|
2486
|
-
if (candidateDocIds) parts.push(
|
|
2703
|
+
const domain = buildDomainFilter(scope, scopeId);
|
|
2704
|
+
if (domain) parts.push(...domain);
|
|
2705
|
+
if (candidateDocIds) parts.push({ op: "in", field: "doc_id", values: candidateDocIds });
|
|
2487
2706
|
} else {
|
|
2488
2707
|
let docIds = candidateDocIds;
|
|
2489
2708
|
if (scope !== "all") {
|
|
2490
|
-
const
|
|
2491
|
-
const domainDocIds = await this.
|
|
2709
|
+
const domain = buildDomainFilter(scope, scopeId);
|
|
2710
|
+
const domainDocIds = await this.store.getDocIdsByDomain(domain);
|
|
2492
2711
|
docIds = docIds ? domainDocIds.filter((id) => docIds.includes(id)) : domainDocIds;
|
|
2493
2712
|
}
|
|
2494
2713
|
if (docIds) {
|
|
2495
2714
|
if (docIds.length === 0) return NO_MATCH;
|
|
2496
|
-
parts.push(
|
|
2715
|
+
parts.push({ op: "in", field: "doc_id", values: docIds });
|
|
2497
2716
|
}
|
|
2498
2717
|
}
|
|
2499
|
-
return parts.length > 0 ? parts
|
|
2718
|
+
return parts.length > 0 ? parts : void 0;
|
|
2500
2719
|
}
|
|
2501
2720
|
// ── 读取 / 删除 ─────────────────────────────────────────────────────────────────
|
|
2502
2721
|
async getDocument(docId) {
|
|
2503
|
-
return this.
|
|
2722
|
+
return this.store.getDocument(docId);
|
|
2504
2723
|
}
|
|
2505
2724
|
async deleteDocument(docId) {
|
|
2506
|
-
const existing = await this.
|
|
2725
|
+
const existing = await this.store.getDocument(docId);
|
|
2507
2726
|
if (!existing) return false;
|
|
2508
|
-
await this.
|
|
2509
|
-
await this.
|
|
2727
|
+
await this.store.deleteDocument(docId);
|
|
2728
|
+
await this.store.deleteChunksByDoc(docId).catch((err) => {
|
|
2510
2729
|
console.error(`[KnowledgeManager] Failed to delete chunks for doc ${docId}:`, err);
|
|
2511
2730
|
});
|
|
2512
2731
|
await this.grafeo.deleteKnowledgeByDoc(docId).catch((err) => {
|
|
@@ -2515,7 +2734,7 @@ ${p.content}` : p.content
|
|
|
2515
2734
|
return true;
|
|
2516
2735
|
}
|
|
2517
2736
|
async listDocuments(filter, page) {
|
|
2518
|
-
let docs = await this.
|
|
2737
|
+
let docs = await this.store.getAllDocuments();
|
|
2519
2738
|
if (filter?.userId) docs = docs.filter((d) => d.userId === filter.userId);
|
|
2520
2739
|
if (filter?.chatId) docs = docs.filter((d) => d.chatId === filter.chatId);
|
|
2521
2740
|
if (filter?.sessionId) docs = docs.filter((d) => d.sessionId === filter.sessionId);
|
|
@@ -2533,19 +2752,12 @@ function dedupByContent(items) {
|
|
|
2533
2752
|
}
|
|
2534
2753
|
return out;
|
|
2535
2754
|
}
|
|
2536
|
-
function
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
if (scope === "session" && scopeId) return `session_id = '${esc(scopeId)}'`;
|
|
2541
|
-
if (scope === "chat" && scopeId) return `chat_id = '${esc(scopeId)}'`;
|
|
2542
|
-
if (scope === "user" && scopeId) return `user_id = '${esc(scopeId)}'`;
|
|
2755
|
+
function buildDomainFilter(scope, scopeId) {
|
|
2756
|
+
if (scope === "session" && scopeId) return [eq("session_id", scopeId)];
|
|
2757
|
+
if (scope === "chat" && scopeId) return [eq("chat_id", scopeId)];
|
|
2758
|
+
if (scope === "user" && scopeId) return [eq("user_id", scopeId)];
|
|
2543
2759
|
return void 0;
|
|
2544
2760
|
}
|
|
2545
|
-
function inSql(column, values) {
|
|
2546
|
-
const list = values.map((v) => `'${esc(v)}'`).join(", ");
|
|
2547
|
-
return `${column} IN (${list})`;
|
|
2548
|
-
}
|
|
2549
2761
|
function inferTitle(markdown) {
|
|
2550
2762
|
const lines = markdown.split(/\r?\n/);
|
|
2551
2763
|
for (const line of lines) {
|
|
@@ -2583,78 +2795,208 @@ function withTimeout2(promise, timeoutMs, label) {
|
|
|
2583
2795
|
var SessionCache = class {
|
|
2584
2796
|
config;
|
|
2585
2797
|
sessions = /* @__PURE__ */ new Map();
|
|
2586
|
-
|
|
2798
|
+
legacyHistoryWindows = /* @__PURE__ */ new Map();
|
|
2587
2799
|
constructor(config) {
|
|
2588
2800
|
this.config = config;
|
|
2589
2801
|
}
|
|
2590
2802
|
getEntry(sessionId) {
|
|
2803
|
+
const entry = this.sessions.get(sessionId);
|
|
2804
|
+
if (entry) entry.lastAccessAt = Date.now();
|
|
2805
|
+
return entry;
|
|
2806
|
+
}
|
|
2807
|
+
peekEntry(sessionId) {
|
|
2591
2808
|
return this.sessions.get(sessionId);
|
|
2592
2809
|
}
|
|
2593
2810
|
getOrCreateEntry(sessionId, chatId, userId) {
|
|
2594
2811
|
let entry = this.sessions.get(sessionId);
|
|
2595
2812
|
if (!entry) {
|
|
2596
|
-
entry = {
|
|
2813
|
+
entry = {
|
|
2814
|
+
messages: [],
|
|
2815
|
+
totalTokens: 0,
|
|
2816
|
+
topics: [],
|
|
2817
|
+
topicTokens: 0,
|
|
2818
|
+
ids: { chatId, userId },
|
|
2819
|
+
lastAccessAt: Date.now()
|
|
2820
|
+
};
|
|
2597
2821
|
this.sessions.set(sessionId, entry);
|
|
2822
|
+
} else {
|
|
2823
|
+
entry.ids = { chatId, userId };
|
|
2824
|
+
entry.lastAccessAt = Date.now();
|
|
2598
2825
|
}
|
|
2599
2826
|
return entry;
|
|
2600
2827
|
}
|
|
2601
|
-
|
|
2828
|
+
hydrate(sessionId, chatId, userId, messages, topics) {
|
|
2829
|
+
const entry = this.getOrCreateEntry(sessionId, chatId, userId);
|
|
2830
|
+
entry.messages = sortMessages(dedupeMessages(messages));
|
|
2831
|
+
entry.totalTokens = sumMessageTokens(entry.messages);
|
|
2832
|
+
this.setTopics(sessionId, topics);
|
|
2833
|
+
return entry;
|
|
2834
|
+
}
|
|
2835
|
+
upsertMessages(sessionId, messages, chatId, userId) {
|
|
2602
2836
|
const entry = this.getOrCreateEntry(sessionId, chatId, userId);
|
|
2603
|
-
entry.messages.
|
|
2604
|
-
|
|
2837
|
+
const byId = new Map(entry.messages.map((m) => [m.messageId, m]));
|
|
2838
|
+
for (const message of messages) byId.set(message.messageId, message);
|
|
2839
|
+
entry.messages = sortMessages([...byId.values()]);
|
|
2840
|
+
entry.totalTokens = sumMessageTokens(entry.messages);
|
|
2841
|
+
return entry;
|
|
2842
|
+
}
|
|
2843
|
+
/** @deprecated 0.3.x 单元/API 兼容;新代码使用 upsertMessages。 */
|
|
2844
|
+
addMessages(sessionId, messages, chatId, userId) {
|
|
2845
|
+
const entry = this.upsertMessages(sessionId, messages, chatId, userId);
|
|
2605
2846
|
return entry.totalTokens >= this.config.sessionTokenLimit;
|
|
2606
2847
|
}
|
|
2848
|
+
/** @deprecated 新压缩链按 messageId 精确移除。 */
|
|
2607
2849
|
clearMessages(sessionId, keepAfter) {
|
|
2608
2850
|
const entry = this.sessions.get(sessionId);
|
|
2609
2851
|
if (!entry) return;
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2852
|
+
entry.messages = keepAfter == null ? [] : entry.messages.filter((message) => message.createdAt > keepAfter);
|
|
2853
|
+
entry.totalTokens = sumMessageTokens(entry.messages);
|
|
2854
|
+
}
|
|
2855
|
+
removeMessages(sessionId, messageIds) {
|
|
2856
|
+
const entry = this.sessions.get(sessionId);
|
|
2857
|
+
if (!entry) return;
|
|
2858
|
+
const ids = new Set(messageIds);
|
|
2859
|
+
entry.messages = entry.messages.filter((m) => !ids.has(m.messageId));
|
|
2860
|
+
entry.totalTokens = sumMessageTokens(entry.messages);
|
|
2861
|
+
entry.lastAccessAt = Date.now();
|
|
2617
2862
|
}
|
|
2618
2863
|
getSessionMessages(sessionId) {
|
|
2619
|
-
return this.
|
|
2864
|
+
return this.getEntry(sessionId)?.messages ?? [];
|
|
2620
2865
|
}
|
|
2621
|
-
|
|
2622
|
-
|
|
2866
|
+
selectOldestMessageBatch(sessionId, targetTokens) {
|
|
2867
|
+
const messages = this.getSessionMessages(sessionId);
|
|
2868
|
+
if (messages.length === 0 || targetTokens <= 0) return [];
|
|
2869
|
+
const selected = [];
|
|
2870
|
+
let tokens = 0;
|
|
2871
|
+
for (const message of messages) {
|
|
2872
|
+
selected.push(message);
|
|
2873
|
+
tokens += effectiveUsage(message);
|
|
2874
|
+
if (tokens >= targetTokens) break;
|
|
2875
|
+
}
|
|
2876
|
+
return selected;
|
|
2623
2877
|
}
|
|
2624
|
-
|
|
2625
|
-
const
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
const
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2878
|
+
setTopics(sessionId, topics, truncate = true) {
|
|
2879
|
+
const entry = this.sessions.get(sessionId);
|
|
2880
|
+
if (!entry) return;
|
|
2881
|
+
const ordered = [...topics].sort(topicOrder);
|
|
2882
|
+
const kept = [];
|
|
2883
|
+
let used = 0;
|
|
2884
|
+
for (let i = ordered.length - 1; i >= 0; i--) {
|
|
2885
|
+
const topic = normalizeTopic(ordered[i]);
|
|
2886
|
+
if (truncate && used + topic.tokens > this.config.compressedContextTokenLimit && kept.length > 0) break;
|
|
2887
|
+
kept.push(topic);
|
|
2888
|
+
used += topic.tokens;
|
|
2889
|
+
}
|
|
2890
|
+
entry.topics = kept.reverse();
|
|
2891
|
+
entry.topicTokens = used;
|
|
2892
|
+
entry.lastAccessAt = Date.now();
|
|
2893
|
+
}
|
|
2894
|
+
appendTopic(sessionId, topic) {
|
|
2895
|
+
const entry = this.sessions.get(sessionId);
|
|
2896
|
+
if (!entry) return;
|
|
2897
|
+
this.setTopics(sessionId, [...entry.topics, topic], false);
|
|
2898
|
+
}
|
|
2899
|
+
replaceTopics(sessionId, removedIds, replacement) {
|
|
2900
|
+
const entry = this.sessions.get(sessionId);
|
|
2901
|
+
if (!entry) return;
|
|
2902
|
+
const ids = new Set(removedIds);
|
|
2903
|
+
this.setTopics(
|
|
2904
|
+
sessionId,
|
|
2905
|
+
[...entry.topics.filter((topic) => !ids.has(topic.summaryId)), replacement],
|
|
2906
|
+
false
|
|
2907
|
+
);
|
|
2908
|
+
}
|
|
2909
|
+
getTopics(sessionId) {
|
|
2910
|
+
return this.getEntry(sessionId)?.topics ?? [];
|
|
2911
|
+
}
|
|
2912
|
+
buildCompressedContext(sessionId) {
|
|
2913
|
+
return this.getTopics(sessionId).map((topic) => topic.title ? `## ${topic.title}
|
|
2914
|
+
${topic.summary}` : topic.summary).join("\n\n");
|
|
2915
|
+
}
|
|
2916
|
+
/** @deprecated 三级窗口兼容,仅供 0.3.x 调用方过渡。 */
|
|
2917
|
+
buildHistoryWindow(sessionId, groups) {
|
|
2918
|
+
const [detailCount, summaryCount, conciseCount] = this.config.topicRatio;
|
|
2919
|
+
const values = [
|
|
2920
|
+
...groups.detail.slice(0, detailCount).map((topic) => topic.detail ?? topic.summary),
|
|
2921
|
+
...groups.summary.slice(0, summaryCount).map((topic) => topic.summary),
|
|
2922
|
+
...groups.concise.slice(0, conciseCount).map((topic) => topic.concise ?? topic.summary)
|
|
2923
|
+
];
|
|
2924
|
+
let remaining = this.config.historyWindowTokenLimit;
|
|
2925
|
+
const kept = [];
|
|
2926
|
+
for (const value of values) {
|
|
2927
|
+
const tokens = countTokens(value);
|
|
2928
|
+
if (tokens > remaining) break;
|
|
2929
|
+
kept.push(value);
|
|
2930
|
+
remaining -= tokens;
|
|
2931
|
+
}
|
|
2932
|
+
this.legacyHistoryWindows.set(sessionId, kept.join("\n\n"));
|
|
2645
2933
|
}
|
|
2934
|
+
/** @deprecated MemoryManager.getHistoryWindow 返回结构化窗口。 */
|
|
2646
2935
|
getHistoryWindow(sessionId) {
|
|
2647
|
-
return this.
|
|
2936
|
+
return this.legacyHistoryWindows.get(sessionId) ?? this.buildCompressedContext(sessionId);
|
|
2648
2937
|
}
|
|
2938
|
+
/** @deprecated 仅为 0.3.x 兼容。 */
|
|
2649
2939
|
setHistoryWindow(sessionId, content) {
|
|
2650
|
-
this.
|
|
2940
|
+
this.legacyHistoryWindows.set(sessionId, content);
|
|
2941
|
+
}
|
|
2942
|
+
setModelContextTokens(sessionId, modelContextTokens) {
|
|
2943
|
+
const entry = this.sessions.get(sessionId);
|
|
2944
|
+
if (!entry) return;
|
|
2945
|
+
entry.lastModelContextTokens = modelContextTokens;
|
|
2946
|
+
entry.lastAccessAt = Date.now();
|
|
2947
|
+
}
|
|
2948
|
+
getAllSessionIds() {
|
|
2949
|
+
return [...this.sessions.keys()];
|
|
2950
|
+
}
|
|
2951
|
+
delete(sessionId) {
|
|
2952
|
+
this.sessions.delete(sessionId);
|
|
2953
|
+
this.legacyHistoryWindows.delete(sessionId);
|
|
2954
|
+
}
|
|
2955
|
+
evictIdle(now, ttlMs, isBusy) {
|
|
2956
|
+
if (ttlMs <= 0) return [];
|
|
2957
|
+
const evicted = [];
|
|
2958
|
+
for (const [sessionId, entry] of this.sessions) {
|
|
2959
|
+
if (now - entry.lastAccessAt < ttlMs || isBusy(sessionId)) continue;
|
|
2960
|
+
this.sessions.delete(sessionId);
|
|
2961
|
+
evicted.push(sessionId);
|
|
2962
|
+
}
|
|
2963
|
+
return evicted;
|
|
2651
2964
|
}
|
|
2652
2965
|
};
|
|
2966
|
+
function effectiveUsage(message) {
|
|
2967
|
+
if (Number.isFinite(message.usage) && message.usage > 0) return Math.floor(message.usage);
|
|
2968
|
+
return Math.max(
|
|
2969
|
+
1,
|
|
2970
|
+
(message.content.length + (message.parts?.length ?? 0) + message.metadata.length + (message.payload?.length ?? 0)) * 2
|
|
2971
|
+
);
|
|
2972
|
+
}
|
|
2973
|
+
function sumMessageTokens(messages) {
|
|
2974
|
+
return messages.reduce((sum, message) => sum + effectiveUsage(message), 0);
|
|
2975
|
+
}
|
|
2976
|
+
function dedupeMessages(messages) {
|
|
2977
|
+
return [...new Map(messages.map((message) => [message.messageId, message])).values()];
|
|
2978
|
+
}
|
|
2979
|
+
function sortMessages(messages) {
|
|
2980
|
+
return messages.sort(
|
|
2981
|
+
(a, b) => a.createdAt - b.createdAt || a.messageId.localeCompare(b.messageId)
|
|
2982
|
+
);
|
|
2983
|
+
}
|
|
2984
|
+
function topicOrder(a, b) {
|
|
2985
|
+
return a.endTime - b.endTime || a.summaryId.localeCompare(b.summaryId);
|
|
2986
|
+
}
|
|
2987
|
+
function normalizeTopic(topic) {
|
|
2988
|
+
const summary = topic.summary || topic.detail || topic.concise || "";
|
|
2989
|
+
return {
|
|
2990
|
+
...topic,
|
|
2991
|
+
summary,
|
|
2992
|
+
tokens: topic.tokens > 0 ? topic.tokens : Math.max(1, countTokens(summary))
|
|
2993
|
+
};
|
|
2994
|
+
}
|
|
2653
2995
|
|
|
2654
2996
|
// src/memory.manager.ts
|
|
2655
2997
|
var MemoryManager = class {
|
|
2656
2998
|
config;
|
|
2657
|
-
|
|
2999
|
+
store;
|
|
2658
3000
|
grafeo;
|
|
2659
3001
|
embed;
|
|
2660
3002
|
llm;
|
|
@@ -2664,18 +3006,24 @@ var MemoryManager = class {
|
|
|
2664
3006
|
knowledgeManager;
|
|
2665
3007
|
sessionMap = /* @__PURE__ */ new Map();
|
|
2666
3008
|
optimizeTimer;
|
|
3009
|
+
sessionSweepTimer;
|
|
2667
3010
|
optimizeRunning = false;
|
|
3011
|
+
optimizeTask;
|
|
3012
|
+
destroyTask;
|
|
3013
|
+
hydration = /* @__PURE__ */ new Map();
|
|
3014
|
+
pendingWrites = /* @__PURE__ */ new Map();
|
|
3015
|
+
warnedDefaultModelContext = false;
|
|
2668
3016
|
constructor(config) {
|
|
2669
3017
|
this.config = resolveConfig(config);
|
|
2670
|
-
this.
|
|
3018
|
+
this.store = new MemoryStore(this.config);
|
|
2671
3019
|
this.grafeo = new GrafeoService(this.config);
|
|
2672
3020
|
this.embed = new EmbedService(this.config);
|
|
2673
3021
|
this.llm = new LlmService(this.config);
|
|
2674
3022
|
this.sessionCache = new SessionCache(this.config);
|
|
2675
|
-
this.factCache = new FactCache(this.
|
|
3023
|
+
this.factCache = new FactCache(this.store);
|
|
2676
3024
|
this.compressManager = new CompressManager(
|
|
2677
3025
|
this.config,
|
|
2678
|
-
this.
|
|
3026
|
+
this.store,
|
|
2679
3027
|
this.grafeo,
|
|
2680
3028
|
this.llm,
|
|
2681
3029
|
this.embed,
|
|
@@ -2683,7 +3031,7 @@ var MemoryManager = class {
|
|
|
2683
3031
|
);
|
|
2684
3032
|
this.knowledgeManager = new KnowledgeManager(
|
|
2685
3033
|
this.config,
|
|
2686
|
-
this.
|
|
3034
|
+
this.store,
|
|
2687
3035
|
this.grafeo,
|
|
2688
3036
|
this.llm,
|
|
2689
3037
|
this.embed
|
|
@@ -2691,24 +3039,33 @@ var MemoryManager = class {
|
|
|
2691
3039
|
}
|
|
2692
3040
|
async init() {
|
|
2693
3041
|
initEncoder();
|
|
2694
|
-
await this.
|
|
3042
|
+
await this.store.init();
|
|
2695
3043
|
await this.grafeo.init();
|
|
2696
3044
|
await this.factCache.init();
|
|
2697
|
-
const allSessions = await this.
|
|
3045
|
+
const allSessions = await this.store.getAllSessions();
|
|
2698
3046
|
for (const s of allSessions) {
|
|
2699
3047
|
this.sessionMap.set(s.sessionId, this.deserializeSession(s));
|
|
2700
3048
|
}
|
|
2701
|
-
await this.restoreFromStorage();
|
|
2702
3049
|
if (this.config.autoOptimizeOnInit) {
|
|
2703
|
-
|
|
3050
|
+
this.optimizeTask = this.runBackgroundOptimize(this.config.optimizeVersionRetentionMs);
|
|
2704
3051
|
}
|
|
2705
3052
|
if (this.config.autoOptimizeIntervalMs > 0) {
|
|
2706
3053
|
const retention = Math.max(this.config.optimizeVersionRetentionMs, 6e4);
|
|
2707
3054
|
this.optimizeTimer = setInterval(() => {
|
|
2708
|
-
|
|
3055
|
+
this.optimizeTask = this.runBackgroundOptimize(retention);
|
|
2709
3056
|
}, this.config.autoOptimizeIntervalMs);
|
|
2710
3057
|
this.optimizeTimer.unref?.();
|
|
2711
3058
|
}
|
|
3059
|
+
if (this.config.sessionIdleTtlMs > 0) {
|
|
3060
|
+
this.sessionSweepTimer = setInterval(() => {
|
|
3061
|
+
this.sessionCache.evictIdle(
|
|
3062
|
+
Date.now(),
|
|
3063
|
+
this.config.sessionIdleTtlMs,
|
|
3064
|
+
(sessionId) => this.isSessionBusy(sessionId)
|
|
3065
|
+
);
|
|
3066
|
+
}, this.config.sessionSweepIntervalMs);
|
|
3067
|
+
this.sessionSweepTimer.unref?.();
|
|
3068
|
+
}
|
|
2712
3069
|
}
|
|
2713
3070
|
/** 后台压实的统一入口:防重入(上一轮未结束则跳过),失败仅告警不影响服务。*/
|
|
2714
3071
|
async runBackgroundOptimize(retentionMs) {
|
|
@@ -2734,122 +3091,143 @@ var MemoryManager = class {
|
|
|
2734
3091
|
* @param retentionMs 保留多久内的历史版本,默认取配置 optimizeVersionRetentionMs
|
|
2735
3092
|
*/
|
|
2736
3093
|
async optimizeStorage(retentionMs) {
|
|
2737
|
-
return this.
|
|
2738
|
-
}
|
|
2739
|
-
async
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
if (
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
}
|
|
2771
|
-
})
|
|
2772
|
-
)
|
|
2773
|
-
);
|
|
3094
|
+
return this.store.optimizeStorage(retentionMs ?? this.config.optimizeVersionRetentionMs);
|
|
3095
|
+
}
|
|
3096
|
+
async ensureSessionHydrated(sessionId, fallbackIds) {
|
|
3097
|
+
if (this.sessionCache.peekEntry(sessionId)) return;
|
|
3098
|
+
let task = this.hydration.get(sessionId);
|
|
3099
|
+
if (!task) {
|
|
3100
|
+
task = (async () => {
|
|
3101
|
+
const allTopics = await this.store.getTopicsBySession(sessionId);
|
|
3102
|
+
const latestTopic = allTopics.at(-1);
|
|
3103
|
+
const allRawAfterBoundary = latestTopic ? await this.store.getMessagesAfterBoundary(
|
|
3104
|
+
sessionId,
|
|
3105
|
+
latestTopic.endTime,
|
|
3106
|
+
latestTopic.endMessageId
|
|
3107
|
+
) : await this.store.getAllMessagesBySession(sessionId);
|
|
3108
|
+
const since = this.config.maxHistoryAgeMs > 0 ? Date.now() - this.config.maxHistoryAgeMs : 0;
|
|
3109
|
+
const rawMessages = since > 0 ? allRawAfterBoundary.filter((message) => message.createdAt >= since) : allRawAfterBoundary;
|
|
3110
|
+
const session = this.sessionMap.get(sessionId);
|
|
3111
|
+
const ids = allRawAfterBoundary.at(-1) ?? latestTopic ?? session ?? fallbackIds ?? { chatId: DEFAULT_CHAT_ID, userId: DEFAULT_USER_ID };
|
|
3112
|
+
const visibleTopics = since > 0 ? allTopics.filter((topic) => topic.endTime >= since) : allTopics;
|
|
3113
|
+
this.sessionCache.hydrate(
|
|
3114
|
+
sessionId,
|
|
3115
|
+
ids.chatId,
|
|
3116
|
+
ids.userId,
|
|
3117
|
+
rawMessages,
|
|
3118
|
+
visibleTopics
|
|
3119
|
+
);
|
|
3120
|
+
})();
|
|
3121
|
+
this.hydration.set(sessionId, task);
|
|
3122
|
+
task.finally(() => {
|
|
3123
|
+
if (this.hydration.get(sessionId) === task) this.hydration.delete(sessionId);
|
|
3124
|
+
});
|
|
3125
|
+
}
|
|
3126
|
+
await task;
|
|
2774
3127
|
}
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
3128
|
+
isSessionBusy(sessionId) {
|
|
3129
|
+
return this.hydration.has(sessionId) || this.pendingWrites.has(sessionId) || this.compressManager.isBusy(sessionId);
|
|
3130
|
+
}
|
|
3131
|
+
waitForPendingWrites(sessionId) {
|
|
3132
|
+
return this.pendingWrites.get(sessionId) ?? Promise.resolve();
|
|
3133
|
+
}
|
|
3134
|
+
queueSessionWrite(sessionId, write) {
|
|
3135
|
+
const previous = this.pendingWrites.get(sessionId) ?? Promise.resolve();
|
|
3136
|
+
const task = previous.catch(() => {
|
|
3137
|
+
}).then(write);
|
|
3138
|
+
const settled = task.catch(() => {
|
|
3139
|
+
});
|
|
3140
|
+
this.pendingWrites.set(sessionId, settled);
|
|
3141
|
+
settled.finally(() => {
|
|
3142
|
+
if (this.pendingWrites.get(sessionId) === settled) this.pendingWrites.delete(sessionId);
|
|
3143
|
+
});
|
|
3144
|
+
return task;
|
|
3145
|
+
}
|
|
3146
|
+
updateChat(messages, opts) {
|
|
3147
|
+
if (messages.length === 0) return Promise.resolve();
|
|
3148
|
+
const sessionId = opts?.sessionId ?? messages[0]?.sessionId ?? DEFAULT_SESSION_ID;
|
|
3149
|
+
return this.queueSessionWrite(sessionId, () => this.doUpdateChat(messages, opts, sessionId));
|
|
3150
|
+
}
|
|
3151
|
+
async doUpdateChat(messages, opts, sessionId) {
|
|
3152
|
+
const userId = opts?.userId ?? messages[0]?.userId ?? DEFAULT_USER_ID;
|
|
3153
|
+
const chatId = opts?.chatId ?? messages[0]?.chatId ?? DEFAULT_CHAT_ID;
|
|
2780
3154
|
const now = Date.now();
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
}
|
|
2801
|
-
const stored = withUsage.map((m, i) => ({
|
|
2802
|
-
messageId: m.messageId,
|
|
2803
|
-
talkerId: m.talkerId,
|
|
2804
|
-
chatId: m.chatId,
|
|
2805
|
-
userId: m.userId,
|
|
2806
|
-
sessionId: m.sessionId,
|
|
2807
|
-
type: m.type,
|
|
2808
|
-
content: m.content,
|
|
2809
|
-
parts: m.parts,
|
|
2810
|
-
usage: m.usage,
|
|
2811
|
-
metadata: m.metadata,
|
|
2812
|
-
vector: vectors[i],
|
|
2813
|
-
createdAt: m.createdAt
|
|
2814
|
-
}));
|
|
2815
|
-
await this.lance.addMessages(stored);
|
|
2816
|
-
if (!this.sessionMap.has(sessionId)) {
|
|
2817
|
-
const session = {
|
|
2818
|
-
sessionId,
|
|
2819
|
-
chatId,
|
|
2820
|
-
userId,
|
|
2821
|
-
title: opts?.sessionTitle ?? "",
|
|
2822
|
-
metadata: JSON.stringify(opts?.sessionMetadata ?? {}),
|
|
2823
|
-
createdAt: now,
|
|
2824
|
-
updatedAt: now
|
|
3155
|
+
await this.ensureSessionHydrated(sessionId, { chatId, userId });
|
|
3156
|
+
const normalized = messages.map((message) => {
|
|
3157
|
+
const parts = JSON.stringify(message.parts ?? []);
|
|
3158
|
+
const metadata = JSON.stringify(message.metadata ?? {});
|
|
3159
|
+
const payload = message.payload === void 0 ? void 0 : JSON.stringify(message.payload);
|
|
3160
|
+
const tokenInput = [message.content, parts, metadata, payload ?? ""].join("\n");
|
|
3161
|
+
return {
|
|
3162
|
+
messageId: message.messageId ?? uuidv44(),
|
|
3163
|
+
talkerId: message.talkerId ?? "user",
|
|
3164
|
+
chatId: message.chatId ?? chatId,
|
|
3165
|
+
userId: message.userId ?? userId,
|
|
3166
|
+
sessionId: message.sessionId ?? sessionId,
|
|
3167
|
+
type: message.type ?? "text",
|
|
3168
|
+
content: message.content,
|
|
3169
|
+
parts,
|
|
3170
|
+
payload,
|
|
3171
|
+
usage: message.usage ?? countTokens(tokenInput),
|
|
3172
|
+
metadata,
|
|
3173
|
+
createdAt: message.createdAt ?? now
|
|
2825
3174
|
};
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
3175
|
+
});
|
|
3176
|
+
const vectors = await this.embed.embed(normalized.map((message) => message.content));
|
|
3177
|
+
const stored = normalized.map((message, index) => ({
|
|
3178
|
+
...message,
|
|
3179
|
+
vector: vectors[index]
|
|
3180
|
+
}));
|
|
3181
|
+
await this.store.upsertMessages(stored);
|
|
3182
|
+
this.sessionCache.upsertMessages(sessionId, stored, chatId, userId);
|
|
3183
|
+
const existing = this.sessionMap.get(sessionId);
|
|
3184
|
+
const session = existing ? {
|
|
3185
|
+
...this.serializeSession(existing),
|
|
3186
|
+
updatedAt: now,
|
|
3187
|
+
...opts?.sessionTitle !== void 0 && { title: opts.sessionTitle },
|
|
3188
|
+
...opts?.sessionMetadata !== void 0 && {
|
|
3189
|
+
metadata: JSON.stringify(opts.sessionMetadata)
|
|
3190
|
+
}
|
|
3191
|
+
} : {
|
|
3192
|
+
sessionId,
|
|
3193
|
+
chatId,
|
|
3194
|
+
userId,
|
|
3195
|
+
title: opts?.sessionTitle ?? "",
|
|
3196
|
+
metadata: JSON.stringify(opts?.sessionMetadata ?? {}),
|
|
3197
|
+
createdAt: now,
|
|
3198
|
+
updatedAt: now
|
|
3199
|
+
};
|
|
3200
|
+
this.sessionMap.set(sessionId, this.deserializeSession(session));
|
|
3201
|
+
if (existing) await this.store.upsertSession(session);
|
|
3202
|
+
else await this.store.insertSession(session);
|
|
3203
|
+
const entry = this.sessionCache.getEntry(sessionId);
|
|
3204
|
+
const rawLimit = this.calculateWindowUsage(
|
|
3205
|
+
entry.lastModelContextTokens ?? this.config.defaultModelContextTokens
|
|
3206
|
+
).rawTokenLimit;
|
|
3207
|
+
if (entry.totalTokens >= Math.floor(rawLimit * this.config.precompressionRatio)) {
|
|
3208
|
+
void this.compressManager.triggerCompress(sessionId, false, false, rawLimit).catch((error) => {
|
|
3209
|
+
console.error(`[MemoryManager] Background compress failed for ${sessionId}:`, error);
|
|
2835
3210
|
});
|
|
2836
3211
|
}
|
|
2837
|
-
(async () => {
|
|
2838
|
-
const overLimit = this.sessionCache.addMessages(sessionId, stored, chatId, userId);
|
|
2839
|
-
if (overLimit) {
|
|
2840
|
-
await this.compressManager.triggerCompress(sessionId).catch((err) => {
|
|
2841
|
-
console.error("[MemoryManager] Background compress failed:", err);
|
|
2842
|
-
});
|
|
2843
|
-
}
|
|
2844
|
-
})();
|
|
2845
3212
|
}
|
|
2846
3213
|
async flushChat(sessionId, opts) {
|
|
2847
3214
|
const sid = sessionId ?? DEFAULT_SESSION_ID;
|
|
2848
|
-
|
|
3215
|
+
await this.waitForPendingWrites(sid);
|
|
3216
|
+
await this.ensureSessionHydrated(sid);
|
|
3217
|
+
const entry = this.sessionCache.getEntry(sid);
|
|
3218
|
+
const rawLimit = this.calculateWindowUsage(
|
|
3219
|
+
entry?.lastModelContextTokens ?? this.config.defaultModelContextTokens
|
|
3220
|
+
).rawTokenLimit;
|
|
3221
|
+
const promise = this.compressManager.triggerCompress(
|
|
3222
|
+
sid,
|
|
3223
|
+
true,
|
|
3224
|
+
opts?.waitGraph === true,
|
|
3225
|
+
rawLimit
|
|
3226
|
+
);
|
|
2849
3227
|
if (opts?.wait) await promise;
|
|
2850
3228
|
}
|
|
2851
|
-
async updateFacts(content, level, userId, chatId, sessionId) {
|
|
2852
|
-
await this.factCache.add(content, level, userId, chatId, sessionId);
|
|
3229
|
+
async updateFacts(content, level, userId, chatId, sessionId, key) {
|
|
3230
|
+
await this.factCache.add(content, level, userId, chatId, sessionId, key);
|
|
2853
3231
|
}
|
|
2854
3232
|
async updateEntity(entities, relations, context) {
|
|
2855
3233
|
if (entities.length === 0 && relations.length === 0) return;
|
|
@@ -2885,7 +3263,7 @@ var MemoryManager = class {
|
|
|
2885
3263
|
} = opts;
|
|
2886
3264
|
if (!query.trim()) return [];
|
|
2887
3265
|
const vector = await this.embed.embedOne(query);
|
|
2888
|
-
const filter = this.
|
|
3266
|
+
const filter = this.buildScopeFilter(scope, scopeId);
|
|
2889
3267
|
let useGraph = false;
|
|
2890
3268
|
if (mode === "all") {
|
|
2891
3269
|
useGraph = true;
|
|
@@ -2893,16 +3271,32 @@ var MemoryManager = class {
|
|
|
2893
3271
|
useGraph = await this.llm.judgeNeedsGraphSearch(query).catch(() => false);
|
|
2894
3272
|
}
|
|
2895
3273
|
const tasks = [
|
|
2896
|
-
this.
|
|
2897
|
-
(topics)
|
|
3274
|
+
this.store.hybridSearchTopics(query, vector, filter, limit).then((topics) => {
|
|
3275
|
+
void this.store.incrementTopicRecallCounts(topics).catch(() => {
|
|
3276
|
+
});
|
|
3277
|
+
return topics.map((topic, index) => ({
|
|
2898
3278
|
type: "topic",
|
|
2899
|
-
content:
|
|
2900
|
-
score: 1,
|
|
3279
|
+
content: topic.summary,
|
|
3280
|
+
score: 1 / (index + 1),
|
|
2901
3281
|
meta: {
|
|
2902
|
-
summaryId:
|
|
2903
|
-
sessionId:
|
|
2904
|
-
chatId:
|
|
2905
|
-
userId:
|
|
3282
|
+
summaryId: topic.summaryId,
|
|
3283
|
+
sessionId: topic.sessionId,
|
|
3284
|
+
chatId: topic.chatId,
|
|
3285
|
+
userId: topic.userId
|
|
3286
|
+
}
|
|
3287
|
+
}));
|
|
3288
|
+
}).catch(() => []),
|
|
3289
|
+
this.store.hybridSearchMessages(query, vector, filter, limit).then(
|
|
3290
|
+
(messages) => messages.map((message, index) => ({
|
|
3291
|
+
type: "message",
|
|
3292
|
+
content: message.content,
|
|
3293
|
+
score: 1 / (index + 1),
|
|
3294
|
+
meta: {
|
|
3295
|
+
messageId: message.messageId,
|
|
3296
|
+
sessionId: message.sessionId,
|
|
3297
|
+
chatId: message.chatId,
|
|
3298
|
+
userId: message.userId,
|
|
3299
|
+
metadata: safeParseObject2(message.metadata)
|
|
2906
3300
|
}
|
|
2907
3301
|
}))
|
|
2908
3302
|
).catch(() => [])
|
|
@@ -2961,15 +3355,84 @@ var MemoryManager = class {
|
|
|
2961
3355
|
...this.factCache.get("chat", chatId)
|
|
2962
3356
|
].sort((a, b) => a.createdAt - b.createdAt);
|
|
2963
3357
|
if (merged.length === 0) return "";
|
|
2964
|
-
return merged.map(
|
|
3358
|
+
return merged.map(
|
|
3359
|
+
(f) => `${new Date(f.updatedAt ?? f.createdAt).toISOString()}\uFF1A${f.key ? `[${f.key}] ` : ""}${f.content}`
|
|
3360
|
+
).join("\n");
|
|
3361
|
+
}
|
|
3362
|
+
async getHistoryWindow(sessionId, modelContextTokens) {
|
|
3363
|
+
let modelTokens = modelContextTokens;
|
|
3364
|
+
if (modelTokens == null) {
|
|
3365
|
+
modelTokens = this.config.defaultModelContextTokens;
|
|
3366
|
+
if (!this.warnedDefaultModelContext) {
|
|
3367
|
+
this.warnedDefaultModelContext = true;
|
|
3368
|
+
console.warn(
|
|
3369
|
+
`[MemoryManager] getHistoryWindow \u672A\u4F20 modelContextTokens\uFF0C\u4F7F\u7528\u9ED8\u8BA4 ${modelTokens} tokens\u3002`
|
|
3370
|
+
);
|
|
3371
|
+
}
|
|
3372
|
+
}
|
|
3373
|
+
if (!Number.isFinite(modelTokens) || modelTokens <= 0) {
|
|
3374
|
+
throw new Error("modelContextTokens \u5FC5\u987B\u662F\u6B63\u6570");
|
|
3375
|
+
}
|
|
3376
|
+
modelTokens = Math.floor(modelTokens);
|
|
3377
|
+
await this.waitForPendingWrites(sessionId);
|
|
3378
|
+
await this.ensureSessionHydrated(sessionId);
|
|
3379
|
+
this.sessionCache.setModelContextTokens(sessionId, modelTokens);
|
|
3380
|
+
const usageLimits = this.calculateWindowUsage(modelTokens);
|
|
3381
|
+
for (let attempt = 0; attempt < 100; attempt++) {
|
|
3382
|
+
let entry2 = this.sessionCache.getEntry(sessionId);
|
|
3383
|
+
if (entry2.totalTokens <= usageLimits.rawTokenLimit || entry2.messages.length === 0) break;
|
|
3384
|
+
await this.compressManager.waitForIdle(sessionId);
|
|
3385
|
+
entry2 = this.sessionCache.getEntry(sessionId);
|
|
3386
|
+
if (entry2.totalTokens <= usageLimits.rawTokenLimit || entry2.messages.length === 0) break;
|
|
3387
|
+
console.warn(
|
|
3388
|
+
`[MemoryManager] session ${sessionId} raw history (${entry2.totalTokens}) exceeds current model budget (${usageLimits.rawTokenLimit}); waiting for immediate compression.`
|
|
3389
|
+
);
|
|
3390
|
+
const beforeTokens = entry2.totalTokens;
|
|
3391
|
+
await this.compressManager.triggerCompress(
|
|
3392
|
+
sessionId,
|
|
3393
|
+
true,
|
|
3394
|
+
false,
|
|
3395
|
+
usageLimits.rawTokenLimit
|
|
3396
|
+
);
|
|
3397
|
+
const after = this.sessionCache.getEntry(sessionId);
|
|
3398
|
+
if (after.messages.length > 0 && after.totalTokens >= beforeTokens) {
|
|
3399
|
+
throw new Error(`Compression made no progress for session ${sessionId}`);
|
|
3400
|
+
}
|
|
3401
|
+
}
|
|
3402
|
+
const entry = this.sessionCache.getEntry(sessionId);
|
|
3403
|
+
if (entry.totalTokens > usageLimits.rawTokenLimit) {
|
|
3404
|
+
throw new Error(`Unable to fit session ${sessionId} into the requested model context`);
|
|
3405
|
+
}
|
|
3406
|
+
return {
|
|
3407
|
+
sessionId,
|
|
3408
|
+
compressedContext: this.sessionCache.buildCompressedContext(sessionId),
|
|
3409
|
+
recentMessages: entry.messages.map(toMemoryRawMessage),
|
|
3410
|
+
usage: {
|
|
3411
|
+
...usageLimits,
|
|
3412
|
+
compressedTokens: entry.topicTokens,
|
|
3413
|
+
rawTokens: entry.totalTokens
|
|
3414
|
+
}
|
|
3415
|
+
};
|
|
2965
3416
|
}
|
|
2966
|
-
|
|
2967
|
-
|
|
3417
|
+
calculateWindowUsage(modelContextTokens) {
|
|
3418
|
+
const usableContextTokens = Math.floor(modelContextTokens * this.config.contextUsageRatio);
|
|
3419
|
+
const rawTokenLimit = usableContextTokens - this.config.compressedContextTokenLimit;
|
|
3420
|
+
if (rawTokenLimit <= 0) {
|
|
3421
|
+
throw new Error(
|
|
3422
|
+
`modelContextTokens=${modelContextTokens} \u5728 contextUsageRatio=${this.config.contextUsageRatio} \u4E0B\u4E0D\u8DB3\u4EE5\u5BB9\u7EB3 compressedContextTokenLimit=${this.config.compressedContextTokenLimit}`
|
|
3423
|
+
);
|
|
3424
|
+
}
|
|
3425
|
+
return {
|
|
3426
|
+
modelContextTokens,
|
|
3427
|
+
usableContextTokens,
|
|
3428
|
+
compressedTokenLimit: this.config.compressedContextTokenLimit,
|
|
3429
|
+
rawTokenLimit
|
|
3430
|
+
};
|
|
2968
3431
|
}
|
|
2969
|
-
|
|
2970
|
-
if (scope === "session" && scopeId) return
|
|
2971
|
-
if (scope === "chat" && scopeId) return
|
|
2972
|
-
if (scope === "user" && scopeId) return
|
|
3432
|
+
buildScopeFilter(scope, scopeId) {
|
|
3433
|
+
if (scope === "session" && scopeId) return [eq("session_id", scopeId)];
|
|
3434
|
+
if (scope === "chat" && scopeId) return [eq("chat_id", scopeId)];
|
|
3435
|
+
if (scope === "user" && scopeId) return [eq("user_id", scopeId)];
|
|
2973
3436
|
return void 0;
|
|
2974
3437
|
}
|
|
2975
3438
|
deserializeSession(s) {
|
|
@@ -2980,7 +3443,7 @@ var MemoryManager = class {
|
|
|
2980
3443
|
}
|
|
2981
3444
|
// ── Message queries ──────────────────────────────────────────────────────────
|
|
2982
3445
|
async getRecentMessages(sessionId, limit) {
|
|
2983
|
-
return this.
|
|
3446
|
+
return this.store.getLatestMessages(sessionId, limit);
|
|
2984
3447
|
}
|
|
2985
3448
|
// ── Session queries (in-memory) ──────────────────────────────────────────────
|
|
2986
3449
|
getSession(sessionId) {
|
|
@@ -3016,17 +3479,18 @@ var MemoryManager = class {
|
|
|
3016
3479
|
updatedAt: Date.now()
|
|
3017
3480
|
};
|
|
3018
3481
|
this.sessionMap.set(sessionId, updated);
|
|
3019
|
-
await this.
|
|
3482
|
+
await this.store.upsertSession(this.serializeSession(updated));
|
|
3020
3483
|
return updated;
|
|
3021
3484
|
}
|
|
3022
3485
|
async deleteSession(sessionId) {
|
|
3023
3486
|
if (!this.sessionMap.has(sessionId)) return false;
|
|
3024
3487
|
this.sessionMap.delete(sessionId);
|
|
3025
|
-
|
|
3026
|
-
await this.
|
|
3488
|
+
this.sessionCache.delete(sessionId);
|
|
3489
|
+
await this.store.deleteSession(sessionId);
|
|
3490
|
+
await this.store.deleteMessagesBySession(sessionId).catch((err) => {
|
|
3027
3491
|
console.error(`[MemoryManager] Failed to delete messages for session ${sessionId}:`, err);
|
|
3028
3492
|
});
|
|
3029
|
-
await this.
|
|
3493
|
+
await this.store.deleteTopicsBySession(sessionId).catch((err) => {
|
|
3030
3494
|
console.error(`[MemoryManager] Failed to delete topics for session ${sessionId}:`, err);
|
|
3031
3495
|
});
|
|
3032
3496
|
return true;
|
|
@@ -3035,8 +3499,8 @@ var MemoryManager = class {
|
|
|
3035
3499
|
/** 概览统计:各类记忆数据的总量 */
|
|
3036
3500
|
async stats() {
|
|
3037
3501
|
const [counts, knowledge, entities, relations] = await Promise.all([
|
|
3038
|
-
this.
|
|
3039
|
-
this.
|
|
3502
|
+
this.store.countAll(),
|
|
3503
|
+
this.store.countKnowledge(),
|
|
3040
3504
|
this.grafeo.getAllEntities(),
|
|
3041
3505
|
this.grafeo.getAllRelations()
|
|
3042
3506
|
]);
|
|
@@ -3050,7 +3514,7 @@ var MemoryManager = class {
|
|
|
3050
3514
|
/** 按天聚合最近 days 天的活跃趋势(概览图表用,含零值天)。days 默认 30,范围 1~365。 */
|
|
3051
3515
|
async trend(days = 30) {
|
|
3052
3516
|
const n = Math.min(365, Math.max(1, Math.floor(days)));
|
|
3053
|
-
return this.
|
|
3517
|
+
return this.store.trendDaily(n);
|
|
3054
3518
|
}
|
|
3055
3519
|
listSessions(filter, page) {
|
|
3056
3520
|
let results = [...this.sessionMap.values()];
|
|
@@ -3061,13 +3525,13 @@ var MemoryManager = class {
|
|
|
3061
3525
|
}
|
|
3062
3526
|
async listMessages(sessionId, limitOrPage) {
|
|
3063
3527
|
if (typeof limitOrPage === "object") {
|
|
3064
|
-
const all = await this.
|
|
3528
|
+
const all = await this.store.getAllMessagesBySession(sessionId);
|
|
3065
3529
|
return paginate(all, limitOrPage);
|
|
3066
3530
|
}
|
|
3067
|
-
return this.
|
|
3531
|
+
return this.store.getLatestMessages(sessionId, limitOrPage ?? 100);
|
|
3068
3532
|
}
|
|
3069
3533
|
async listTopics(filter, page) {
|
|
3070
|
-
let topics = await this.
|
|
3534
|
+
let topics = await this.store.getAllTopics();
|
|
3071
3535
|
if (filter?.sessionId) topics = topics.filter((t) => t.sessionId === filter.sessionId);
|
|
3072
3536
|
if (filter?.userId) topics = topics.filter((t) => t.userId === filter.userId);
|
|
3073
3537
|
if (filter?.chatId) topics = topics.filter((t) => t.chatId === filter.chatId);
|
|
@@ -3096,8 +3560,8 @@ var MemoryManager = class {
|
|
|
3096
3560
|
return this.factCache.all().find((f) => f.chatId === chatId)?.userId;
|
|
3097
3561
|
}
|
|
3098
3562
|
/** 手动新增一条事实 */
|
|
3099
|
-
async addFact(content, level, userId, chatId, sessionId) {
|
|
3100
|
-
await this.factCache.add(content, level, userId, chatId, sessionId);
|
|
3563
|
+
async addFact(content, level, userId, chatId, sessionId, key) {
|
|
3564
|
+
await this.factCache.add(content, level, userId, chatId, sessionId, key);
|
|
3101
3565
|
}
|
|
3102
3566
|
/** 删除单条事实,返回是否命中 */
|
|
3103
3567
|
async deleteFact(factId) {
|
|
@@ -3136,18 +3600,73 @@ var MemoryManager = class {
|
|
|
3136
3600
|
return page ? this.knowledgeManager.listDocuments(filter, page) : this.knowledgeManager.listDocuments(filter);
|
|
3137
3601
|
}
|
|
3138
3602
|
destroy() {
|
|
3603
|
+
if (this.destroyTask) return this.destroyTask;
|
|
3139
3604
|
if (this.optimizeTimer) {
|
|
3140
3605
|
clearInterval(this.optimizeTimer);
|
|
3141
3606
|
this.optimizeTimer = void 0;
|
|
3142
3607
|
}
|
|
3143
|
-
this.
|
|
3608
|
+
if (this.sessionSweepTimer) {
|
|
3609
|
+
clearInterval(this.sessionSweepTimer);
|
|
3610
|
+
this.sessionSweepTimer = void 0;
|
|
3611
|
+
}
|
|
3612
|
+
const writes = [...this.pendingWrites.values(), ...this.hydration.values()];
|
|
3613
|
+
this.destroyTask = Promise.all(writes).catch(() => {
|
|
3614
|
+
}).then(() => this.compressManager.waitForAllIdle()).catch(() => {
|
|
3615
|
+
}).then(() => this.optimizeTask).catch(() => {
|
|
3616
|
+
}).then(async () => {
|
|
3617
|
+
this.grafeo.close();
|
|
3618
|
+
await this.store.close().catch(() => {
|
|
3619
|
+
});
|
|
3620
|
+
});
|
|
3621
|
+
return this.destroyTask;
|
|
3144
3622
|
}
|
|
3145
3623
|
};
|
|
3624
|
+
function toMemoryRawMessage(message) {
|
|
3625
|
+
const parts = safeParseArray(message.parts);
|
|
3626
|
+
const payload = message.payload === void 0 ? void 0 : safeParseValue(message.payload);
|
|
3627
|
+
return {
|
|
3628
|
+
messageId: message.messageId,
|
|
3629
|
+
talkerId: message.talkerId,
|
|
3630
|
+
chatId: message.chatId,
|
|
3631
|
+
userId: message.userId,
|
|
3632
|
+
sessionId: message.sessionId,
|
|
3633
|
+
type: message.type,
|
|
3634
|
+
content: message.content,
|
|
3635
|
+
...parts.length > 0 && { parts },
|
|
3636
|
+
...payload !== void 0 && { payload },
|
|
3637
|
+
usage: message.usage,
|
|
3638
|
+
metadata: safeParseObject2(message.metadata),
|
|
3639
|
+
createdAt: message.createdAt
|
|
3640
|
+
};
|
|
3641
|
+
}
|
|
3642
|
+
function safeParseArray(value) {
|
|
3643
|
+
try {
|
|
3644
|
+
const parsed = JSON.parse(value);
|
|
3645
|
+
return Array.isArray(parsed) ? parsed : [];
|
|
3646
|
+
} catch {
|
|
3647
|
+
return [];
|
|
3648
|
+
}
|
|
3649
|
+
}
|
|
3650
|
+
function safeParseObject2(value) {
|
|
3651
|
+
try {
|
|
3652
|
+
const parsed = JSON.parse(value);
|
|
3653
|
+
return parsed != null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
|
3654
|
+
} catch {
|
|
3655
|
+
return {};
|
|
3656
|
+
}
|
|
3657
|
+
}
|
|
3658
|
+
function safeParseValue(value) {
|
|
3659
|
+
try {
|
|
3660
|
+
return JSON.parse(value);
|
|
3661
|
+
} catch {
|
|
3662
|
+
return value;
|
|
3663
|
+
}
|
|
3664
|
+
}
|
|
3146
3665
|
export {
|
|
3147
3666
|
DEFAULT_NODE_TYPES,
|
|
3148
3667
|
DEFAULT_RELATION_TYPES,
|
|
3149
3668
|
KIND_CONVERSATION,
|
|
3150
3669
|
KIND_KNOWLEDGE,
|
|
3151
|
-
|
|
3152
|
-
|
|
3670
|
+
MemoryManager,
|
|
3671
|
+
MemoryStore
|
|
3153
3672
|
};
|