@ppagent/memory 0.1.2 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +291 -127
- package/dist/index.js +490 -537
- package/dist/provider.resolver-ZLQ766IO.js +585 -0
- package/llms.txt +862 -845
- package/package.json +16 -7
package/dist/index.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { v4 as uuidv44 } from "uuid";
|
|
3
3
|
|
|
4
4
|
// src/config.ts
|
|
5
|
+
import * as path from "node:path";
|
|
5
6
|
function resolveConfig(config) {
|
|
6
7
|
let embeddingBaseUrl = config.embeddingBaseUrl;
|
|
7
8
|
let embeddingApiKey = config.embeddingApiKey;
|
|
@@ -19,6 +20,8 @@ function resolveConfig(config) {
|
|
|
19
20
|
...config,
|
|
20
21
|
embeddingBaseUrl,
|
|
21
22
|
embeddingApiKey,
|
|
23
|
+
provider: config.provider ?? "auto",
|
|
24
|
+
sqlitePath: config.sqlitePath ?? path.join(path.dirname(config.lancedbPath), "memory.sqlite3"),
|
|
22
25
|
sessionTokenLimit: config.sessionTokenLimit ?? 16386,
|
|
23
26
|
historyWindowTokenLimit: config.historyWindowTokenLimit ?? 10240,
|
|
24
27
|
topicRatio: config.topicRatio ?? [1, 5, 20],
|
|
@@ -47,6 +50,7 @@ function resolveConfig(config) {
|
|
|
47
50
|
graphExtractConcurrency: config.graphExtractConcurrency ?? 2,
|
|
48
51
|
graphBuildTimeoutMs: config.graphBuildTimeoutMs ?? 12e4,
|
|
49
52
|
autoOptimizeOnInit: config.autoOptimizeOnInit ?? true,
|
|
53
|
+
autoOptimizeIntervalMs: config.autoOptimizeIntervalMs ?? 6 * 36e5,
|
|
50
54
|
optimizeVersionRetentionMs: config.optimizeVersionRetentionMs ?? 0,
|
|
51
55
|
restoreConcurrency: config.restoreConcurrency ?? 8
|
|
52
56
|
};
|
|
@@ -710,120 +714,149 @@ var GrafeoService = class _GrafeoService {
|
|
|
710
714
|
}
|
|
711
715
|
};
|
|
712
716
|
|
|
713
|
-
// src/db/
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
// src/db/sql.util.ts
|
|
718
|
-
function escLance(value) {
|
|
719
|
-
return String(value).replace(/'/g, "''");
|
|
720
|
-
}
|
|
721
|
-
function eqFilter(field, value) {
|
|
722
|
-
return `${field} = '${escLance(value)}'`;
|
|
717
|
+
// src/db/store.types.ts
|
|
718
|
+
function eq(field, value) {
|
|
719
|
+
return { op: "eq", field, value };
|
|
723
720
|
}
|
|
724
721
|
|
|
725
|
-
// src/db/
|
|
722
|
+
// src/db/memory.store.ts
|
|
726
723
|
var MESSAGES_TABLE = "messages";
|
|
727
724
|
var TOPICS_TABLE = "topics";
|
|
728
725
|
var FACTS_TABLE = "facts";
|
|
729
726
|
var SESSIONS_TABLE = "sessions";
|
|
730
727
|
var DOCUMENTS_TABLE = "documents";
|
|
731
728
|
var CHUNKS_TABLE = "chunks";
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
}
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
}
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
}
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
729
|
+
var RRF_K = 60;
|
|
730
|
+
var HYBRID_OVERFETCH = 2;
|
|
731
|
+
function tableDefs(dim) {
|
|
732
|
+
return [
|
|
733
|
+
{
|
|
734
|
+
name: MESSAGES_TABLE,
|
|
735
|
+
vectorDimension: dim,
|
|
736
|
+
columns: [
|
|
737
|
+
{ name: "message_id", type: "text" },
|
|
738
|
+
{ name: "talker_id", type: "text" },
|
|
739
|
+
{ name: "chat_id", type: "text" },
|
|
740
|
+
{ name: "user_id", type: "text" },
|
|
741
|
+
{ name: "session_id", type: "text" },
|
|
742
|
+
{ name: "type", type: "text" },
|
|
743
|
+
{ name: "content", type: "text" },
|
|
744
|
+
{ name: "parts", type: "text", nullable: true },
|
|
745
|
+
// 兼容存量数据
|
|
746
|
+
{ name: "vector", type: "vector" },
|
|
747
|
+
{ name: "usage", type: "int" },
|
|
748
|
+
{ name: "metadata", type: "text" },
|
|
749
|
+
{ name: "created_at", type: "long" }
|
|
750
|
+
],
|
|
751
|
+
indexes: [
|
|
752
|
+
{ column: "message_id", kind: "scalar" },
|
|
753
|
+
{ column: "session_id", kind: "scalar" },
|
|
754
|
+
{ column: "content", kind: "fts" },
|
|
755
|
+
{ column: "metadata", kind: "fts" }
|
|
756
|
+
]
|
|
757
|
+
},
|
|
758
|
+
{
|
|
759
|
+
name: TOPICS_TABLE,
|
|
760
|
+
vectorDimension: dim,
|
|
761
|
+
columns: [
|
|
762
|
+
{ name: "summary_id", type: "text" },
|
|
763
|
+
{ name: "session_id", type: "text" },
|
|
764
|
+
{ name: "user_id", type: "text" },
|
|
765
|
+
{ name: "chat_id", type: "text" },
|
|
766
|
+
{ name: "title", type: "text", nullable: true },
|
|
767
|
+
// 兼容存量数据
|
|
768
|
+
{ name: "detail", type: "text" },
|
|
769
|
+
{ name: "summary", type: "text" },
|
|
770
|
+
{ name: "concise", type: "text" },
|
|
771
|
+
{ name: "vector", type: "vector" },
|
|
772
|
+
{ name: "start_time", type: "long" },
|
|
773
|
+
{ name: "end_time", type: "long" },
|
|
774
|
+
{ name: "created_at", type: "long" },
|
|
775
|
+
{ name: "updated_at", type: "long" },
|
|
776
|
+
{ name: "recall_count", type: "int" }
|
|
777
|
+
],
|
|
778
|
+
indexes: [
|
|
779
|
+
{ column: "chat_id", kind: "scalar" },
|
|
780
|
+
{ column: "detail", kind: "fts" }
|
|
781
|
+
]
|
|
782
|
+
},
|
|
783
|
+
{
|
|
784
|
+
name: FACTS_TABLE,
|
|
785
|
+
columns: [
|
|
786
|
+
{ name: "fact_id", type: "text" },
|
|
787
|
+
{ name: "level", type: "text" },
|
|
788
|
+
{ name: "chat_id", type: "text" },
|
|
789
|
+
{ name: "session_id", type: "text" },
|
|
790
|
+
{ name: "user_id", type: "text" },
|
|
791
|
+
{ name: "content", type: "text" },
|
|
792
|
+
{ name: "created_at", type: "long" }
|
|
793
|
+
],
|
|
794
|
+
indexes: []
|
|
795
|
+
},
|
|
796
|
+
{
|
|
797
|
+
name: SESSIONS_TABLE,
|
|
798
|
+
columns: [
|
|
799
|
+
{ name: "session_id", type: "text" },
|
|
800
|
+
{ name: "chat_id", type: "text" },
|
|
801
|
+
{ name: "user_id", type: "text" },
|
|
802
|
+
{ name: "title", type: "text" },
|
|
803
|
+
{ name: "metadata", type: "text" },
|
|
804
|
+
{ name: "created_at", type: "long" },
|
|
805
|
+
{ name: "updated_at", type: "long" }
|
|
806
|
+
],
|
|
807
|
+
indexes: [{ column: "session_id", kind: "scalar" }]
|
|
808
|
+
},
|
|
809
|
+
{
|
|
810
|
+
name: DOCUMENTS_TABLE,
|
|
811
|
+
vectorDimension: dim,
|
|
812
|
+
columns: [
|
|
813
|
+
{ name: "doc_id", type: "text" },
|
|
814
|
+
{ name: "user_id", type: "text" },
|
|
815
|
+
{ name: "chat_id", type: "text" },
|
|
816
|
+
{ name: "session_id", type: "text" },
|
|
817
|
+
{ name: "title", type: "text" },
|
|
818
|
+
{ name: "source_name", type: "text" },
|
|
819
|
+
{ name: "full_content", type: "text" },
|
|
820
|
+
{ name: "content_hash", type: "text" },
|
|
821
|
+
{ name: "summary", type: "text" },
|
|
822
|
+
// vector 列存储文档摘要向量(summaryVector),用于文档级粗召回
|
|
823
|
+
{ name: "vector", type: "vector" },
|
|
824
|
+
{ name: "chunk_count", type: "int" },
|
|
825
|
+
{ name: "has_graph", type: "int" },
|
|
826
|
+
// 0/1 充当布尔
|
|
827
|
+
{ name: "metadata", type: "text" },
|
|
828
|
+
{ name: "created_at", type: "long" },
|
|
829
|
+
{ name: "updated_at", type: "long" }
|
|
830
|
+
],
|
|
831
|
+
indexes: [{ column: "doc_id", kind: "scalar" }]
|
|
832
|
+
},
|
|
833
|
+
{
|
|
834
|
+
name: CHUNKS_TABLE,
|
|
835
|
+
vectorDimension: dim,
|
|
836
|
+
columns: [
|
|
837
|
+
{ name: "chunk_id", type: "text" },
|
|
838
|
+
{ name: "doc_id", type: "text" },
|
|
839
|
+
// user_id/chat_id/session_id:chunkRedundantIds=false 时存空串,仅经 documents join 过滤
|
|
840
|
+
{ name: "user_id", type: "text" },
|
|
841
|
+
{ name: "chat_id", type: "text" },
|
|
842
|
+
{ name: "session_id", type: "text" },
|
|
843
|
+
{ name: "content", type: "text" },
|
|
844
|
+
{ name: "vector", type: "vector" },
|
|
845
|
+
{ name: "heading_path", type: "text" },
|
|
846
|
+
{ name: "ordinal", type: "int" },
|
|
847
|
+
{ name: "tokens", type: "int" },
|
|
848
|
+
{ name: "metadata", type: "text" },
|
|
849
|
+
{ name: "created_at", type: "long" }
|
|
850
|
+
],
|
|
851
|
+
indexes: [
|
|
852
|
+
{ column: "doc_id", kind: "scalar" },
|
|
853
|
+
{ column: "content", kind: "fts" }
|
|
854
|
+
]
|
|
855
|
+
}
|
|
856
|
+
];
|
|
810
857
|
}
|
|
811
|
-
function
|
|
812
|
-
return
|
|
813
|
-
new Field("chunk_id", new Utf8(), false),
|
|
814
|
-
new Field("doc_id", new Utf8(), false),
|
|
815
|
-
// user_id/chat_id/session_id:chunkRedundantIds=false 时存空串,仅经 documents join 过滤
|
|
816
|
-
new Field("user_id", new Utf8(), false),
|
|
817
|
-
new Field("chat_id", new Utf8(), false),
|
|
818
|
-
new Field("session_id", new Utf8(), false),
|
|
819
|
-
new Field("content", new Utf8(), false),
|
|
820
|
-
new Field("vector", new FixedSizeList(dim, new Field("item", new Float32(), false)), false),
|
|
821
|
-
new Field("heading_path", new Utf8(), false),
|
|
822
|
-
new Field("ordinal", new Int32(), false),
|
|
823
|
-
new Field("tokens", new Int32(), false),
|
|
824
|
-
new Field("metadata", new Utf8(), false),
|
|
825
|
-
new Field("created_at", new Int64(), false)
|
|
826
|
-
]);
|
|
858
|
+
function toVector(v) {
|
|
859
|
+
return Array.isArray(v) ? v : Array.from(v);
|
|
827
860
|
}
|
|
828
861
|
function messageToRow(m) {
|
|
829
862
|
return {
|
|
@@ -852,7 +885,7 @@ function rowToMessage(r) {
|
|
|
852
885
|
content: r.content,
|
|
853
886
|
parts: r.parts ?? "[]",
|
|
854
887
|
// 旧数据无此列时安全降级
|
|
855
|
-
vector:
|
|
888
|
+
vector: toVector(r.vector),
|
|
856
889
|
usage: Number(r.usage),
|
|
857
890
|
metadata: r.metadata,
|
|
858
891
|
createdAt: Number(r.created_at)
|
|
@@ -886,7 +919,7 @@ function rowToTopic(r) {
|
|
|
886
919
|
detail: r.detail,
|
|
887
920
|
summary: r.summary,
|
|
888
921
|
concise: r.concise,
|
|
889
|
-
vector:
|
|
922
|
+
vector: toVector(r.vector),
|
|
890
923
|
startTime: Number(r.start_time),
|
|
891
924
|
endTime: Number(r.end_time),
|
|
892
925
|
createdAt: Number(r.created_at),
|
|
@@ -968,7 +1001,7 @@ function rowToDocument(r) {
|
|
|
968
1001
|
fullContent: r.full_content,
|
|
969
1002
|
contentHash: r.content_hash,
|
|
970
1003
|
summary: r.summary,
|
|
971
|
-
summaryVector:
|
|
1004
|
+
summaryVector: toVector(r.vector),
|
|
972
1005
|
chunkCount: Number(r.chunk_count),
|
|
973
1006
|
hasGraph: Number(r.has_graph) === 1,
|
|
974
1007
|
metadata: safeParseObject(r.metadata),
|
|
@@ -1000,7 +1033,7 @@ function rowToChunk(r) {
|
|
|
1000
1033
|
chatId: r.chat_id,
|
|
1001
1034
|
sessionId: r.session_id,
|
|
1002
1035
|
content: r.content,
|
|
1003
|
-
vector:
|
|
1036
|
+
vector: toVector(r.vector),
|
|
1004
1037
|
headingPath: r.heading_path,
|
|
1005
1038
|
ordinal: Number(r.ordinal),
|
|
1006
1039
|
tokens: Number(r.tokens),
|
|
@@ -1015,245 +1048,80 @@ function safeParseObject(s) {
|
|
|
1015
1048
|
return {};
|
|
1016
1049
|
}
|
|
1017
1050
|
}
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
var
|
|
1032
|
-
{ column: "doc_id" },
|
|
1033
|
-
{ column: "content", fts: true }
|
|
1034
|
-
// 知识片段混合检索
|
|
1035
|
-
];
|
|
1036
|
-
var LanceService = class {
|
|
1051
|
+
function rrfFuse(lists, keyOf) {
|
|
1052
|
+
const acc = /* @__PURE__ */ new Map();
|
|
1053
|
+
for (const list of lists) {
|
|
1054
|
+
list.forEach((row, idx) => {
|
|
1055
|
+
const id = keyOf(row);
|
|
1056
|
+
const inc = 1 / (RRF_K + idx + 1);
|
|
1057
|
+
const entry = acc.get(id);
|
|
1058
|
+
if (entry) entry.score += inc;
|
|
1059
|
+
else acc.set(id, { row, score: inc });
|
|
1060
|
+
});
|
|
1061
|
+
}
|
|
1062
|
+
return [...acc.values()].sort((a, b) => b.score - a.score).map((e) => e.row);
|
|
1063
|
+
}
|
|
1064
|
+
var MemoryStore = class {
|
|
1037
1065
|
config;
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
sessionsTable;
|
|
1043
|
-
documentsTable;
|
|
1044
|
-
chunksTable;
|
|
1045
|
-
// Scalar indexes cannot be created on empty tables (LanceDB btree limitation).
|
|
1046
|
-
// These flags defer creation to the first insert.
|
|
1047
|
-
isNewMessagesTable = false;
|
|
1048
|
-
isNewTopicsTable = false;
|
|
1049
|
-
isNewSessionsTable = false;
|
|
1050
|
-
isNewDocumentsTable = false;
|
|
1051
|
-
isNewChunksTable = false;
|
|
1052
|
-
constructor(config) {
|
|
1066
|
+
provider;
|
|
1067
|
+
providerOverride;
|
|
1068
|
+
/** provider 省略时在 init() 阶段经 provider.resolver 自动探测创建(测试可显式注入) */
|
|
1069
|
+
constructor(config, provider) {
|
|
1053
1070
|
this.config = config;
|
|
1071
|
+
this.providerOverride = provider;
|
|
1072
|
+
}
|
|
1073
|
+
/** 当前后端 provider 标识(日志/诊断用) */
|
|
1074
|
+
get providerKind() {
|
|
1075
|
+
return this.provider?.kind ?? "uninitialized";
|
|
1054
1076
|
}
|
|
1055
1077
|
async init() {
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
const existingTables = await this.conn.tableNames();
|
|
1059
|
-
if (existingTables.includes(MESSAGES_TABLE)) {
|
|
1060
|
-
this.messagesTable = await this.conn.openTable(MESSAGES_TABLE);
|
|
1061
|
-
await this._ensurePartsColumn();
|
|
1062
|
-
} else {
|
|
1063
|
-
this.messagesTable = await this.conn.createEmptyTable(
|
|
1064
|
-
MESSAGES_TABLE,
|
|
1065
|
-
messagesSchema(dim)
|
|
1066
|
-
);
|
|
1067
|
-
this.isNewMessagesTable = true;
|
|
1068
|
-
}
|
|
1069
|
-
if (existingTables.includes(TOPICS_TABLE)) {
|
|
1070
|
-
this.topicsTable = await this.conn.openTable(TOPICS_TABLE);
|
|
1071
|
-
await this._ensureTopicTitleColumn();
|
|
1072
|
-
} else {
|
|
1073
|
-
this.topicsTable = await this.conn.createEmptyTable(
|
|
1074
|
-
TOPICS_TABLE,
|
|
1075
|
-
topicsSchema(dim)
|
|
1076
|
-
);
|
|
1077
|
-
this.isNewTopicsTable = true;
|
|
1078
|
-
}
|
|
1079
|
-
if (existingTables.includes(FACTS_TABLE)) {
|
|
1080
|
-
this.factsTable = await this.conn.openTable(FACTS_TABLE);
|
|
1078
|
+
if (this.providerOverride) {
|
|
1079
|
+
this.provider = this.providerOverride;
|
|
1081
1080
|
} else {
|
|
1082
|
-
|
|
1081
|
+
const { createProvider } = await import("./provider.resolver-ZLQ766IO.js");
|
|
1082
|
+
this.provider = await createProvider(this.config);
|
|
1083
|
+
console.info(`[memory] \u5411\u91CF\u5B58\u50A8\u540E\u7AEF\uFF1A${this.provider.kind}`);
|
|
1083
1084
|
}
|
|
1084
|
-
|
|
1085
|
-
this.sessionsTable = await this.conn.openTable(SESSIONS_TABLE);
|
|
1086
|
-
} else {
|
|
1087
|
-
this.sessionsTable = await this.conn.createEmptyTable(SESSIONS_TABLE, sessionsSchema());
|
|
1088
|
-
this.isNewSessionsTable = true;
|
|
1089
|
-
}
|
|
1090
|
-
if (existingTables.includes(DOCUMENTS_TABLE)) {
|
|
1091
|
-
this.documentsTable = await this.conn.openTable(DOCUMENTS_TABLE);
|
|
1092
|
-
} else {
|
|
1093
|
-
this.documentsTable = await this.conn.createEmptyTable(DOCUMENTS_TABLE, documentsSchema(dim));
|
|
1094
|
-
this.isNewDocumentsTable = true;
|
|
1095
|
-
}
|
|
1096
|
-
if (existingTables.includes(CHUNKS_TABLE)) {
|
|
1097
|
-
this.chunksTable = await this.conn.openTable(CHUNKS_TABLE);
|
|
1098
|
-
} else {
|
|
1099
|
-
this.chunksTable = await this.conn.createEmptyTable(CHUNKS_TABLE, chunksSchema(dim));
|
|
1100
|
-
this.isNewChunksTable = true;
|
|
1101
|
-
}
|
|
1102
|
-
await this.ensureIndexes(this.messagesTable, MESSAGES_INDEXES);
|
|
1103
|
-
await this.ensureIndexes(this.topicsTable, TOPICS_INDEXES);
|
|
1104
|
-
await this.ensureIndexes(this.sessionsTable, SESSIONS_INDEXES);
|
|
1105
|
-
await this.ensureIndexes(this.documentsTable, DOCUMENTS_INDEXES);
|
|
1106
|
-
await this.ensureIndexes(this.chunksTable, CHUNKS_INDEXES);
|
|
1085
|
+
await this.provider.init(tableDefs(this.config.embeddingDimension));
|
|
1107
1086
|
}
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
* createIndex 默认 replace:true 会在每次启动时全量重建索引并提交新表版本——
|
|
1111
|
-
* 这正是历史上「启动越来越慢 + _versions 目录膨胀」的根源,绝不可回退到无条件 createIndex。
|
|
1112
|
-
*/
|
|
1113
|
-
async ensureIndexes(table, specs) {
|
|
1114
|
-
let indexed;
|
|
1115
|
-
try {
|
|
1116
|
-
indexed = new Set((await table.listIndices()).flatMap((i) => i.columns));
|
|
1117
|
-
} catch {
|
|
1118
|
-
indexed = /* @__PURE__ */ new Set();
|
|
1119
|
-
}
|
|
1120
|
-
for (const { column, fts } of specs) {
|
|
1121
|
-
if (indexed.has(column)) continue;
|
|
1122
|
-
try {
|
|
1123
|
-
await table.createIndex(
|
|
1124
|
-
column,
|
|
1125
|
-
fts ? { config: lancedb.Index.fts(), replace: false } : { replace: false }
|
|
1126
|
-
);
|
|
1127
|
-
} catch {
|
|
1128
|
-
}
|
|
1129
|
-
}
|
|
1087
|
+
async close() {
|
|
1088
|
+
await this.provider.close();
|
|
1130
1089
|
}
|
|
1131
1090
|
/**
|
|
1132
|
-
*
|
|
1133
|
-
*
|
|
1091
|
+
* 存储压实(碎片合并 + 历史版本清理)。
|
|
1092
|
+
* LanceDB 后端长期运行必须定期执行;SQLite 后端为可选的空间回收。
|
|
1134
1093
|
*/
|
|
1135
1094
|
async optimizeStorage(retentionMs = 0) {
|
|
1136
|
-
|
|
1137
|
-
const targets = [
|
|
1138
|
-
[MESSAGES_TABLE, this.messagesTable],
|
|
1139
|
-
[TOPICS_TABLE, this.topicsTable],
|
|
1140
|
-
[FACTS_TABLE, this.factsTable],
|
|
1141
|
-
[SESSIONS_TABLE, this.sessionsTable],
|
|
1142
|
-
[DOCUMENTS_TABLE, this.documentsTable],
|
|
1143
|
-
[CHUNKS_TABLE, this.chunksTable]
|
|
1144
|
-
];
|
|
1145
|
-
const results = [];
|
|
1146
|
-
for (const [name, table] of targets) {
|
|
1147
|
-
try {
|
|
1148
|
-
const stats = await table.optimize({ cleanupOlderThan: cutoff });
|
|
1149
|
-
results.push({
|
|
1150
|
-
table: name,
|
|
1151
|
-
fragmentsRemoved: stats.compaction.fragmentsRemoved,
|
|
1152
|
-
fragmentsAdded: stats.compaction.fragmentsAdded,
|
|
1153
|
-
filesRemoved: stats.compaction.filesRemoved,
|
|
1154
|
-
oldVersionsRemoved: stats.prune.oldVersionsRemoved,
|
|
1155
|
-
bytesRemoved: Number(stats.prune.bytesRemoved)
|
|
1156
|
-
});
|
|
1157
|
-
} catch (err) {
|
|
1158
|
-
console.warn(`[LanceService] optimize table "${name}" failed:`, err);
|
|
1159
|
-
}
|
|
1160
|
-
}
|
|
1161
|
-
return results;
|
|
1162
|
-
}
|
|
1163
|
-
/**
|
|
1164
|
-
* 为存量 messages 表添加 parts 列(如果缺失)。
|
|
1165
|
-
* LanceDB 0.14+ 支持 addColumns;旧版本会 throw,此时 rowToMessage 的 ?? "[]" 兜底。
|
|
1166
|
-
*/
|
|
1167
|
-
async _ensurePartsColumn() {
|
|
1168
|
-
try {
|
|
1169
|
-
const schema = await this.messagesTable.schema();
|
|
1170
|
-
const hasPartsCol = schema.fields?.some(
|
|
1171
|
-
(f) => f.name === "parts"
|
|
1172
|
-
);
|
|
1173
|
-
if (!hasPartsCol) {
|
|
1174
|
-
await this.messagesTable.addColumns([
|
|
1175
|
-
{ name: "parts", valueSql: "CAST(NULL AS STRING)" }
|
|
1176
|
-
]);
|
|
1177
|
-
}
|
|
1178
|
-
} catch {
|
|
1179
|
-
}
|
|
1180
|
-
}
|
|
1181
|
-
/**
|
|
1182
|
-
* 为存量 topics 表添加 title 列(如果缺失)。
|
|
1183
|
-
* 旧版本不支持 addColumns 时忽略,读取时 rowToTopic 的 ?? "" 兜底。
|
|
1184
|
-
*/
|
|
1185
|
-
async _ensureTopicTitleColumn() {
|
|
1186
|
-
try {
|
|
1187
|
-
const schema = await this.topicsTable.schema();
|
|
1188
|
-
const hasTitleCol = schema.fields?.some(
|
|
1189
|
-
(f) => f.name === "title"
|
|
1190
|
-
);
|
|
1191
|
-
if (!hasTitleCol) {
|
|
1192
|
-
await this.topicsTable.addColumns([
|
|
1193
|
-
{ name: "title", valueSql: "CAST(NULL AS STRING)" }
|
|
1194
|
-
]);
|
|
1195
|
-
}
|
|
1196
|
-
} catch {
|
|
1197
|
-
}
|
|
1095
|
+
return this.provider.optimize(retentionMs);
|
|
1198
1096
|
}
|
|
1097
|
+
// ── messages ───────────────────────────────────────────────────────────────
|
|
1199
1098
|
async addMessages(messages) {
|
|
1200
1099
|
if (messages.length === 0) return;
|
|
1201
|
-
await this.
|
|
1202
|
-
if (this.isNewMessagesTable) {
|
|
1203
|
-
await this.ensureIndexes(this.messagesTable, MESSAGES_INDEXES);
|
|
1204
|
-
this.isNewMessagesTable = false;
|
|
1205
|
-
}
|
|
1206
|
-
}
|
|
1207
|
-
async addTopic(topic) {
|
|
1208
|
-
await this.topicsTable.add([topicToRow(topic)]);
|
|
1209
|
-
if (this.isNewTopicsTable) {
|
|
1210
|
-
await this.ensureIndexes(this.topicsTable, TOPICS_INDEXES);
|
|
1211
|
-
this.isNewTopicsTable = false;
|
|
1212
|
-
}
|
|
1213
|
-
}
|
|
1214
|
-
async updateTopicRecallCount(summaryId, count) {
|
|
1215
|
-
await this.topicsTable.update({
|
|
1216
|
-
values: { recall_count: count, updated_at: Date.now() },
|
|
1217
|
-
where: eqFilter("summary_id", summaryId)
|
|
1218
|
-
});
|
|
1219
|
-
}
|
|
1220
|
-
async getRecentTopics(chatId, userId, n1, n2, n3) {
|
|
1221
|
-
const filter = `${eqFilter("chat_id", chatId)} AND ${eqFilter("user_id", userId)}`;
|
|
1222
|
-
const recallBoostMs = this.config.recallBoostMs;
|
|
1223
|
-
const fetchTopics = async (count) => {
|
|
1224
|
-
if (count <= 0) return [];
|
|
1225
|
-
try {
|
|
1226
|
-
const rows = await this.topicsTable.query().where(filter).limit(count * 5).toArray();
|
|
1227
|
-
return rows.sort((a, b) => {
|
|
1228
|
-
const scoreA = Number(a.end_time) + Number(a.recall_count) * recallBoostMs;
|
|
1229
|
-
const scoreB = Number(b.end_time) + Number(b.recall_count) * recallBoostMs;
|
|
1230
|
-
return scoreB - scoreA;
|
|
1231
|
-
}).slice(0, count).map(rowToTopic);
|
|
1232
|
-
} catch {
|
|
1233
|
-
return [];
|
|
1234
|
-
}
|
|
1235
|
-
};
|
|
1236
|
-
const allTopics = await fetchTopics(n1 + n2 + n3);
|
|
1237
|
-
return {
|
|
1238
|
-
detail: allTopics.slice(0, n1),
|
|
1239
|
-
summary: allTopics.slice(n1, n1 + n2),
|
|
1240
|
-
concise: allTopics.slice(n1 + n2, n1 + n2 + n3)
|
|
1241
|
-
};
|
|
1100
|
+
await this.provider.add(MESSAGES_TABLE, messages.map(messageToRow));
|
|
1242
1101
|
}
|
|
1243
1102
|
async getMessagesSince(sessionId, since, limit) {
|
|
1244
1103
|
try {
|
|
1245
|
-
const
|
|
1246
|
-
|
|
1247
|
-
|
|
1104
|
+
const rows = await this.provider.query(MESSAGES_TABLE, {
|
|
1105
|
+
filter: [eq("session_id", sessionId), { op: "gt", field: "created_at", value: since }],
|
|
1106
|
+
limit
|
|
1107
|
+
});
|
|
1248
1108
|
return rows.sort((a, b) => Number(a.created_at) - Number(b.created_at)).map(rowToMessage);
|
|
1249
1109
|
} catch {
|
|
1250
1110
|
return [];
|
|
1251
1111
|
}
|
|
1252
1112
|
}
|
|
1253
1113
|
async getLatestMessages(sessionId, limit) {
|
|
1114
|
+
if (limit <= 0) return [];
|
|
1254
1115
|
try {
|
|
1255
|
-
const rows = await this.
|
|
1256
|
-
|
|
1116
|
+
const rows = await this.provider.query(MESSAGES_TABLE, {
|
|
1117
|
+
filter: [eq("session_id", sessionId)],
|
|
1118
|
+
orderBy: [
|
|
1119
|
+
{ column: "created_at", ascending: false },
|
|
1120
|
+
{ column: "message_id", ascending: false }
|
|
1121
|
+
],
|
|
1122
|
+
limit
|
|
1123
|
+
});
|
|
1124
|
+
return rows.reverse().map(rowToMessage);
|
|
1257
1125
|
} catch {
|
|
1258
1126
|
return [];
|
|
1259
1127
|
}
|
|
@@ -1261,54 +1129,51 @@ var LanceService = class {
|
|
|
1261
1129
|
/** 取某会话的全部消息(按 createdAt 升序),供管理面板分页切片使用 */
|
|
1262
1130
|
async getAllMessagesBySession(sessionId) {
|
|
1263
1131
|
try {
|
|
1264
|
-
const rows = await this.
|
|
1265
|
-
|
|
1132
|
+
const rows = await this.provider.query(MESSAGES_TABLE, {
|
|
1133
|
+
filter: [eq("session_id", sessionId)]
|
|
1134
|
+
});
|
|
1135
|
+
return rows.sort(
|
|
1136
|
+
(a, b) => Number(a.created_at) - Number(b.created_at) || cmpStr(a.message_id, b.message_id)
|
|
1137
|
+
).map(rowToMessage);
|
|
1266
1138
|
} catch {
|
|
1267
1139
|
return [];
|
|
1268
1140
|
}
|
|
1269
1141
|
}
|
|
1270
1142
|
async searchMessages(vector, filter, limit = 10) {
|
|
1271
|
-
const
|
|
1272
|
-
|
|
1273
|
-
const rows = await q.toArray();
|
|
1274
|
-
return rows.map((r) => ({
|
|
1275
|
-
...rowToMessage(r),
|
|
1276
|
-
_distance: r._distance
|
|
1277
|
-
}));
|
|
1143
|
+
const rows = await this.provider.vectorSearch(MESSAGES_TABLE, vector, { filter, limit });
|
|
1144
|
+
return rows.map((r) => ({ ...rowToMessage(r), _distance: r._distance }));
|
|
1278
1145
|
}
|
|
1279
1146
|
// 对 messages 表执行混合搜索(BM25 + 向量),用于 topics 搜索无结果时的回退
|
|
1280
1147
|
async hybridSearchMessages(query, vector, filter, limit = 10) {
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1148
|
+
const fused = await this.hybridSearch(
|
|
1149
|
+
MESSAGES_TABLE,
|
|
1150
|
+
["content", "metadata"],
|
|
1151
|
+
query,
|
|
1152
|
+
vector,
|
|
1153
|
+
filter,
|
|
1154
|
+
limit
|
|
1155
|
+
);
|
|
1156
|
+
return fused.slice(0, limit).map(rowToMessage);
|
|
1290
1157
|
}
|
|
1291
1158
|
async hybridSearchTopics(query, vector, filter, limit = 10) {
|
|
1292
|
-
const recallBoostMs = this.config.recallBoostMs;
|
|
1293
1159
|
try {
|
|
1294
|
-
const
|
|
1295
|
-
if (
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
const total = rows.length;
|
|
1299
|
-
return rows.map((r, idx) => ({
|
|
1160
|
+
const fused = (await this.hybridSearch(TOPICS_TABLE, ["detail"], query, vector, filter, limit)).slice(0, limit);
|
|
1161
|
+
if (fused.length > 0) {
|
|
1162
|
+
const total = fused.length;
|
|
1163
|
+
return fused.map((r, idx) => ({
|
|
1300
1164
|
row: r,
|
|
1301
1165
|
score: total - idx + Math.log1p(Number(r.recall_count))
|
|
1302
1166
|
})).sort((a, b) => b.score - a.score).map((x) => rowToTopic(x.row));
|
|
1303
1167
|
}
|
|
1304
1168
|
} catch {
|
|
1305
1169
|
}
|
|
1306
|
-
const
|
|
1307
|
-
return
|
|
1170
|
+
const msgs = await this.hybridSearchMessages(query, vector, filter, limit);
|
|
1171
|
+
return msgs.map((r) => ({
|
|
1308
1172
|
summaryId: r.messageId,
|
|
1309
1173
|
sessionId: r.sessionId,
|
|
1310
1174
|
userId: r.userId,
|
|
1311
1175
|
chatId: r.chatId,
|
|
1176
|
+
title: "",
|
|
1312
1177
|
detail: r.content,
|
|
1313
1178
|
summary: r.content,
|
|
1314
1179
|
concise: r.content,
|
|
@@ -1320,83 +1185,154 @@ var LanceService = class {
|
|
|
1320
1185
|
vector: r.vector
|
|
1321
1186
|
}));
|
|
1322
1187
|
}
|
|
1188
|
+
/**
|
|
1189
|
+
* 通用混合检索:FTS(BM25)与向量两路各取 limit*HYBRID_OVERFETCH 候选,RRF 融合。
|
|
1190
|
+
* FTS 不可用(后端不支持或查询失败)时降级为纯向量。返回融合后的完整候选序列(未截断)。
|
|
1191
|
+
*/
|
|
1192
|
+
async hybridSearch(table, ftsColumns, query, vector, filter, limit) {
|
|
1193
|
+
const fetchN = limit * HYBRID_OVERFETCH;
|
|
1194
|
+
const supportsFts = this.provider.capabilities().fts;
|
|
1195
|
+
const [ftsRows, vecRows] = await Promise.all([
|
|
1196
|
+
supportsFts ? this.provider.ftsSearch(table, query, { columns: ftsColumns, filter, limit: fetchN }).catch(() => []) : Promise.resolve([]),
|
|
1197
|
+
this.provider.vectorSearch(table, vector, { filter, limit: fetchN }).catch(() => [])
|
|
1198
|
+
]);
|
|
1199
|
+
const keyColumn = this.keyColumnOf(table);
|
|
1200
|
+
return rrfFuse([ftsRows, vecRows], (r) => String(r[keyColumn]));
|
|
1201
|
+
}
|
|
1202
|
+
keyColumnOf(table) {
|
|
1203
|
+
switch (table) {
|
|
1204
|
+
case MESSAGES_TABLE:
|
|
1205
|
+
return "message_id";
|
|
1206
|
+
case TOPICS_TABLE:
|
|
1207
|
+
return "summary_id";
|
|
1208
|
+
case DOCUMENTS_TABLE:
|
|
1209
|
+
return "doc_id";
|
|
1210
|
+
case CHUNKS_TABLE:
|
|
1211
|
+
return "chunk_id";
|
|
1212
|
+
case SESSIONS_TABLE:
|
|
1213
|
+
return "session_id";
|
|
1214
|
+
case FACTS_TABLE:
|
|
1215
|
+
return "fact_id";
|
|
1216
|
+
default:
|
|
1217
|
+
throw new Error(`Unknown table: ${table}`);
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
// ── topics ─────────────────────────────────────────────────────────────────
|
|
1221
|
+
async addTopic(topic) {
|
|
1222
|
+
await this.provider.add(TOPICS_TABLE, [topicToRow(topic)]);
|
|
1223
|
+
}
|
|
1224
|
+
async updateTopicRecallCount(summaryId, count) {
|
|
1225
|
+
await this.provider.update(
|
|
1226
|
+
TOPICS_TABLE,
|
|
1227
|
+
{ recall_count: count, updated_at: Date.now() },
|
|
1228
|
+
[eq("summary_id", summaryId)]
|
|
1229
|
+
);
|
|
1230
|
+
}
|
|
1231
|
+
async getRecentTopics(chatId, userId, n1, n2, n3) {
|
|
1232
|
+
const filter = [eq("chat_id", chatId), eq("user_id", userId)];
|
|
1233
|
+
const recallBoostMs = this.config.recallBoostMs;
|
|
1234
|
+
const fetchTopics = async (count) => {
|
|
1235
|
+
if (count <= 0) return [];
|
|
1236
|
+
try {
|
|
1237
|
+
const rows = await this.provider.query(TOPICS_TABLE, {
|
|
1238
|
+
filter,
|
|
1239
|
+
limit: count * 5
|
|
1240
|
+
});
|
|
1241
|
+
return rows.sort((a, b) => {
|
|
1242
|
+
const scoreA = Number(a.end_time) + Number(a.recall_count) * recallBoostMs;
|
|
1243
|
+
const scoreB = Number(b.end_time) + Number(b.recall_count) * recallBoostMs;
|
|
1244
|
+
return scoreB - scoreA;
|
|
1245
|
+
}).slice(0, count).map(rowToTopic);
|
|
1246
|
+
} catch {
|
|
1247
|
+
return [];
|
|
1248
|
+
}
|
|
1249
|
+
};
|
|
1250
|
+
const allTopics = await fetchTopics(n1 + n2 + n3);
|
|
1251
|
+
return {
|
|
1252
|
+
detail: allTopics.slice(0, n1),
|
|
1253
|
+
summary: allTopics.slice(n1, n1 + n2),
|
|
1254
|
+
concise: allTopics.slice(n1 + n2, n1 + n2 + n3)
|
|
1255
|
+
};
|
|
1256
|
+
}
|
|
1257
|
+
/** 全量读取 topics(管理面板列表用,不做 recall 加权排序,按 endTime 倒序)*/
|
|
1258
|
+
async getAllTopics() {
|
|
1259
|
+
try {
|
|
1260
|
+
const rows = await this.provider.query(TOPICS_TABLE);
|
|
1261
|
+
return rows.sort(
|
|
1262
|
+
(a, b) => Number(b.end_time) - Number(a.end_time) || cmpStr(a.summary_id, b.summary_id)
|
|
1263
|
+
).map(rowToTopic);
|
|
1264
|
+
} catch {
|
|
1265
|
+
return [];
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
async deleteTopicsBySession(sessionId) {
|
|
1269
|
+
await this.provider.deleteWhere(TOPICS_TABLE, [eq("session_id", sessionId)]);
|
|
1270
|
+
}
|
|
1271
|
+
// ── facts ──────────────────────────────────────────────────────────────────
|
|
1323
1272
|
async saveFact(fact) {
|
|
1324
|
-
await this.
|
|
1273
|
+
await this.provider.add(FACTS_TABLE, [factToRow(fact)]);
|
|
1325
1274
|
}
|
|
1326
1275
|
async getAllFacts() {
|
|
1327
1276
|
try {
|
|
1328
|
-
const rows = await this.
|
|
1277
|
+
const rows = await this.provider.query(FACTS_TABLE);
|
|
1329
1278
|
return rows.map(rowToFact);
|
|
1330
1279
|
} catch {
|
|
1331
1280
|
return [];
|
|
1332
1281
|
}
|
|
1333
1282
|
}
|
|
1283
|
+
/** 删除单条 fact */
|
|
1284
|
+
async deleteFact(factId) {
|
|
1285
|
+
await this.provider.deleteWhere(FACTS_TABLE, [eq("fact_id", factId)]);
|
|
1286
|
+
}
|
|
1287
|
+
// ── sessions ───────────────────────────────────────────────────────────────
|
|
1334
1288
|
async getAllSessionIds() {
|
|
1335
1289
|
try {
|
|
1336
|
-
const rows = await this.
|
|
1290
|
+
const rows = await this.provider.query(MESSAGES_TABLE, {
|
|
1291
|
+
select: ["session_id"],
|
|
1292
|
+
limit: 1e4
|
|
1293
|
+
});
|
|
1337
1294
|
return [...new Set(rows.map((r) => r.session_id))];
|
|
1338
1295
|
} catch {
|
|
1339
1296
|
return [];
|
|
1340
1297
|
}
|
|
1341
1298
|
}
|
|
1342
1299
|
async insertSession(session) {
|
|
1343
|
-
await this.
|
|
1344
|
-
if (this.isNewSessionsTable) {
|
|
1345
|
-
await this.ensureIndexes(this.sessionsTable, SESSIONS_INDEXES);
|
|
1346
|
-
this.isNewSessionsTable = false;
|
|
1347
|
-
}
|
|
1300
|
+
await this.provider.add(SESSIONS_TABLE, [sessionToRow(session)]);
|
|
1348
1301
|
}
|
|
1349
1302
|
async upsertSession(session) {
|
|
1350
|
-
await this.
|
|
1351
|
-
await this.
|
|
1303
|
+
await this.provider.deleteWhere(SESSIONS_TABLE, [eq("session_id", session.sessionId)]);
|
|
1304
|
+
await this.provider.add(SESSIONS_TABLE, [sessionToRow(session)]);
|
|
1352
1305
|
}
|
|
1353
1306
|
async getAllSessions() {
|
|
1354
1307
|
try {
|
|
1355
|
-
const rows = await this.
|
|
1308
|
+
const rows = await this.provider.query(SESSIONS_TABLE);
|
|
1356
1309
|
return rows.map(rowToSession);
|
|
1357
1310
|
} catch {
|
|
1358
1311
|
return [];
|
|
1359
1312
|
}
|
|
1360
1313
|
}
|
|
1361
1314
|
async deleteSession(sessionId) {
|
|
1362
|
-
await this.
|
|
1363
|
-
}
|
|
1364
|
-
// ── 管理面板用:全量读取 / 删除 / 计数 ──────────────────────────────────────────────
|
|
1365
|
-
/** 全量读取 topics(管理面板列表用,不做 recall 加权排序,按 endTime 倒序)*/
|
|
1366
|
-
async getAllTopics() {
|
|
1367
|
-
try {
|
|
1368
|
-
const rows = await this.topicsTable.query().toArray();
|
|
1369
|
-
return rows.sort((a, b) => Number(b.end_time) - Number(a.end_time) || cmpStr(a.summary_id, b.summary_id)).map(rowToTopic);
|
|
1370
|
-
} catch {
|
|
1371
|
-
return [];
|
|
1372
|
-
}
|
|
1373
|
-
}
|
|
1374
|
-
/** 删除单条 fact */
|
|
1375
|
-
async deleteFact(factId) {
|
|
1376
|
-
await this.factsTable.delete(eqFilter("fact_id", factId));
|
|
1315
|
+
await this.provider.deleteWhere(SESSIONS_TABLE, [eq("session_id", sessionId)]);
|
|
1377
1316
|
}
|
|
1378
1317
|
/** 删除某会话下的全部消息(级联删除会话时使用)*/
|
|
1379
1318
|
async deleteMessagesBySession(sessionId) {
|
|
1380
|
-
await this.
|
|
1381
|
-
}
|
|
1382
|
-
/** 删除某会话下的全部 topics(级联删除会话时使用)*/
|
|
1383
|
-
async deleteTopicsBySession(sessionId) {
|
|
1384
|
-
await this.topicsTable.delete(eqFilter("session_id", sessionId));
|
|
1319
|
+
await this.provider.deleteWhere(MESSAGES_TABLE, [eq("session_id", sessionId)]);
|
|
1385
1320
|
}
|
|
1321
|
+
// ── 统计 ───────────────────────────────────────────────────────────────────
|
|
1386
1322
|
/** 各表行数统计(概览卡片用)*/
|
|
1387
1323
|
async countAll() {
|
|
1388
1324
|
const safeCount = async (table) => {
|
|
1389
1325
|
try {
|
|
1390
|
-
return await
|
|
1326
|
+
return await this.provider.count(table);
|
|
1391
1327
|
} catch {
|
|
1392
1328
|
return 0;
|
|
1393
1329
|
}
|
|
1394
1330
|
};
|
|
1395
1331
|
const [sessions, messages, topics, facts] = await Promise.all([
|
|
1396
|
-
safeCount(
|
|
1397
|
-
safeCount(
|
|
1398
|
-
safeCount(
|
|
1399
|
-
safeCount(
|
|
1332
|
+
safeCount(SESSIONS_TABLE),
|
|
1333
|
+
safeCount(MESSAGES_TABLE),
|
|
1334
|
+
safeCount(TOPICS_TABLE),
|
|
1335
|
+
safeCount(FACTS_TABLE)
|
|
1400
1336
|
]);
|
|
1401
1337
|
return { sessions, messages, topics, facts };
|
|
1402
1338
|
}
|
|
@@ -1417,16 +1353,20 @@ var LanceService = class {
|
|
|
1417
1353
|
};
|
|
1418
1354
|
const readCreatedAt = async (table) => {
|
|
1419
1355
|
try {
|
|
1420
|
-
const rows = await
|
|
1356
|
+
const rows = await this.provider.query(table, {
|
|
1357
|
+
select: ["created_at"],
|
|
1358
|
+
filter: [{ op: "gte", field: "created_at", value: startMs }],
|
|
1359
|
+
limit: 1e6
|
|
1360
|
+
});
|
|
1421
1361
|
return rows.map((r) => Number(r.created_at)).filter((n) => Number.isFinite(n));
|
|
1422
1362
|
} catch {
|
|
1423
1363
|
return [];
|
|
1424
1364
|
}
|
|
1425
1365
|
};
|
|
1426
1366
|
const [sessionTs, messageTs, factTs] = await Promise.all([
|
|
1427
|
-
readCreatedAt(
|
|
1428
|
-
readCreatedAt(
|
|
1429
|
-
readCreatedAt(
|
|
1367
|
+
readCreatedAt(SESSIONS_TABLE),
|
|
1368
|
+
readCreatedAt(MESSAGES_TABLE),
|
|
1369
|
+
readCreatedAt(FACTS_TABLE)
|
|
1430
1370
|
]);
|
|
1431
1371
|
const buckets = /* @__PURE__ */ new Map();
|
|
1432
1372
|
for (let i = 0; i < days; i++) {
|
|
@@ -1446,29 +1386,25 @@ var LanceService = class {
|
|
|
1446
1386
|
}
|
|
1447
1387
|
// ── 知识库:documents / chunks ─────────────────────────────────────────────
|
|
1448
1388
|
async addDocument(doc) {
|
|
1449
|
-
await this.
|
|
1450
|
-
if (this.isNewDocumentsTable) {
|
|
1451
|
-
await this.ensureIndexes(this.documentsTable, DOCUMENTS_INDEXES);
|
|
1452
|
-
this.isNewDocumentsTable = false;
|
|
1453
|
-
}
|
|
1389
|
+
await this.provider.add(DOCUMENTS_TABLE, [documentToRow(doc)]);
|
|
1454
1390
|
}
|
|
1455
1391
|
async addChunks(chunks) {
|
|
1456
1392
|
if (chunks.length === 0) return;
|
|
1457
|
-
await this.
|
|
1458
|
-
if (this.isNewChunksTable) {
|
|
1459
|
-
await this.ensureIndexes(this.chunksTable, CHUNKS_INDEXES);
|
|
1460
|
-
this.isNewChunksTable = false;
|
|
1461
|
-
}
|
|
1393
|
+
await this.provider.add(CHUNKS_TABLE, chunks.map(chunkToRow));
|
|
1462
1394
|
}
|
|
1463
1395
|
async updateDocumentGraphFlag(docId, hasGraph) {
|
|
1464
|
-
await this.
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1396
|
+
await this.provider.update(
|
|
1397
|
+
DOCUMENTS_TABLE,
|
|
1398
|
+
{ has_graph: hasGraph ? 1 : 0, updated_at: Date.now() },
|
|
1399
|
+
[eq("doc_id", docId)]
|
|
1400
|
+
);
|
|
1468
1401
|
}
|
|
1469
1402
|
async getDocument(docId) {
|
|
1470
1403
|
try {
|
|
1471
|
-
const rows = await this.
|
|
1404
|
+
const rows = await this.provider.query(DOCUMENTS_TABLE, {
|
|
1405
|
+
filter: [eq("doc_id", docId)],
|
|
1406
|
+
limit: 1
|
|
1407
|
+
});
|
|
1472
1408
|
if (rows.length === 0) return null;
|
|
1473
1409
|
return rowToDocument(rows[0]);
|
|
1474
1410
|
} catch {
|
|
@@ -1478,8 +1414,10 @@ var LanceService = class {
|
|
|
1478
1414
|
/** 查找同域同 hash 的文档(去重用)*/
|
|
1479
1415
|
async findDocumentByHash(contentHash, filter) {
|
|
1480
1416
|
try {
|
|
1481
|
-
const
|
|
1482
|
-
|
|
1417
|
+
const rows = await this.provider.query(DOCUMENTS_TABLE, {
|
|
1418
|
+
filter: [eq("content_hash", contentHash), ...filter ?? []],
|
|
1419
|
+
limit: 1
|
|
1420
|
+
});
|
|
1483
1421
|
if (rows.length === 0) return null;
|
|
1484
1422
|
return rowToDocument(rows[0]);
|
|
1485
1423
|
} catch {
|
|
@@ -1489,9 +1427,11 @@ var LanceService = class {
|
|
|
1489
1427
|
/** 按域过滤返回 docId 列表(chunkRedundantIds=false 时用于 chunk 过滤)*/
|
|
1490
1428
|
async getDocIdsByDomain(filter) {
|
|
1491
1429
|
try {
|
|
1492
|
-
const
|
|
1493
|
-
|
|
1494
|
-
|
|
1430
|
+
const rows = await this.provider.query(DOCUMENTS_TABLE, {
|
|
1431
|
+
select: ["doc_id"],
|
|
1432
|
+
filter,
|
|
1433
|
+
limit: 1e5
|
|
1434
|
+
});
|
|
1495
1435
|
return [...new Set(rows.map((r) => r.doc_id))];
|
|
1496
1436
|
} catch {
|
|
1497
1437
|
return [];
|
|
@@ -1500,13 +1440,8 @@ var LanceService = class {
|
|
|
1500
1440
|
/** 文档级粗召回:对摘要向量做向量搜索,定位候选文档 */
|
|
1501
1441
|
async searchDocuments(vector, filter, limit = 5) {
|
|
1502
1442
|
try {
|
|
1503
|
-
const
|
|
1504
|
-
|
|
1505
|
-
const rows = await q.toArray();
|
|
1506
|
-
return rows.map((r) => ({
|
|
1507
|
-
...rowToDocument(r),
|
|
1508
|
-
_distance: r._distance
|
|
1509
|
-
}));
|
|
1443
|
+
const rows = await this.provider.vectorSearch(DOCUMENTS_TABLE, vector, { filter, limit });
|
|
1444
|
+
return rows.map((r) => ({ ...rowToDocument(r), _distance: r._distance }));
|
|
1510
1445
|
} catch {
|
|
1511
1446
|
return [];
|
|
1512
1447
|
}
|
|
@@ -1514,32 +1449,30 @@ var LanceService = class {
|
|
|
1514
1449
|
/** 知识片段混合检索(BM25 + 向量),失败回退纯向量 */
|
|
1515
1450
|
async searchChunks(query, vector, filter, limit = 8) {
|
|
1516
1451
|
try {
|
|
1517
|
-
const
|
|
1518
|
-
if (
|
|
1519
|
-
|
|
1452
|
+
const fused = await this.hybridSearch(CHUNKS_TABLE, ["content"], query, vector, filter, limit);
|
|
1453
|
+
if (fused.length > 0) return fused.slice(0, limit).map(rowToChunk);
|
|
1454
|
+
} catch {
|
|
1455
|
+
}
|
|
1456
|
+
try {
|
|
1457
|
+
const rows = await this.provider.vectorSearch(CHUNKS_TABLE, vector, { filter, limit });
|
|
1520
1458
|
return rows.map(rowToChunk);
|
|
1521
1459
|
} catch {
|
|
1522
|
-
|
|
1523
|
-
const q = this.chunksTable.vectorSearch(vector).limit(limit);
|
|
1524
|
-
if (filter) q.where(filter);
|
|
1525
|
-
const rows = await q.toArray();
|
|
1526
|
-
return rows.map(rowToChunk);
|
|
1527
|
-
} catch {
|
|
1528
|
-
return [];
|
|
1529
|
-
}
|
|
1460
|
+
return [];
|
|
1530
1461
|
}
|
|
1531
1462
|
}
|
|
1532
1463
|
async deleteDocument(docId) {
|
|
1533
|
-
await this.
|
|
1464
|
+
await this.provider.deleteWhere(DOCUMENTS_TABLE, [eq("doc_id", docId)]);
|
|
1534
1465
|
}
|
|
1535
1466
|
async deleteChunksByDoc(docId) {
|
|
1536
|
-
await this.
|
|
1467
|
+
await this.provider.deleteWhere(CHUNKS_TABLE, [eq("doc_id", docId)]);
|
|
1537
1468
|
}
|
|
1538
1469
|
/** 全量读取文档(管理面板用),按更新时间倒序 */
|
|
1539
1470
|
async getAllDocuments() {
|
|
1540
1471
|
try {
|
|
1541
|
-
const rows = await this.
|
|
1542
|
-
return rows.sort(
|
|
1472
|
+
const rows = await this.provider.query(DOCUMENTS_TABLE);
|
|
1473
|
+
return rows.sort(
|
|
1474
|
+
(a, b) => Number(b.updated_at) - Number(a.updated_at) || cmpStr(a.doc_id, b.doc_id)
|
|
1475
|
+
).map(rowToDocument);
|
|
1543
1476
|
} catch {
|
|
1544
1477
|
return [];
|
|
1545
1478
|
}
|
|
@@ -1548,34 +1481,33 @@ var LanceService = class {
|
|
|
1548
1481
|
async countKnowledge() {
|
|
1549
1482
|
const safeCount = async (table) => {
|
|
1550
1483
|
try {
|
|
1551
|
-
return await
|
|
1484
|
+
return await this.provider.count(table);
|
|
1552
1485
|
} catch {
|
|
1553
1486
|
return 0;
|
|
1554
1487
|
}
|
|
1555
1488
|
};
|
|
1556
1489
|
const [documents, chunks] = await Promise.all([
|
|
1557
|
-
safeCount(
|
|
1558
|
-
safeCount(
|
|
1490
|
+
safeCount(DOCUMENTS_TABLE),
|
|
1491
|
+
safeCount(CHUNKS_TABLE)
|
|
1559
1492
|
]);
|
|
1560
1493
|
return { documents, chunks };
|
|
1561
1494
|
}
|
|
1562
1495
|
/**
|
|
1563
|
-
* 根据 metadata
|
|
1496
|
+
* 根据 metadata 字段内容构建结构化过滤条件。
|
|
1564
1497
|
*
|
|
1565
|
-
* metadata 以 JSON 字符串存储,此方法将键值对转换为
|
|
1498
|
+
* metadata 以 JSON 字符串存储,此方法将键值对转换为 jsonContains 条件,
|
|
1566
1499
|
* 可直接传给 searchMessages / hybridSearchTopics 的 filter 参数。
|
|
1567
1500
|
*
|
|
1568
|
-
* 示例:buildMetadataFilter({ env: "prod", version: 2 })
|
|
1569
|
-
* → `metadata LIKE '%"env":"prod"%' AND metadata LIKE '%"version":2%'`
|
|
1570
|
-
*
|
|
1571
1501
|
* 注意:仅适合简单标量值(字符串、数字、布尔)的精确匹配。
|
|
1572
1502
|
* 复杂嵌套对象或含空格的 JSON 值可能无法可靠匹配。
|
|
1573
1503
|
*/
|
|
1574
1504
|
static buildMetadataFilter(conditions) {
|
|
1575
|
-
return Object.entries(conditions).map(([key, value]) => {
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1505
|
+
return Object.entries(conditions).map(([key, value]) => ({
|
|
1506
|
+
op: "jsonContains",
|
|
1507
|
+
field: "metadata",
|
|
1508
|
+
key,
|
|
1509
|
+
value
|
|
1510
|
+
}));
|
|
1579
1511
|
}
|
|
1580
1512
|
};
|
|
1581
1513
|
|
|
@@ -1885,16 +1817,16 @@ ${context}`, false);
|
|
|
1885
1817
|
import { v4 as uuidv4 } from "uuid";
|
|
1886
1818
|
var CompressManager = class {
|
|
1887
1819
|
config;
|
|
1888
|
-
|
|
1820
|
+
store;
|
|
1889
1821
|
grafeo;
|
|
1890
1822
|
llm;
|
|
1891
1823
|
embed;
|
|
1892
1824
|
sessionCache;
|
|
1893
1825
|
semaphore;
|
|
1894
1826
|
sessionChain = /* @__PURE__ */ new Map();
|
|
1895
|
-
constructor(config,
|
|
1827
|
+
constructor(config, store, grafeo, llm, embed, sessionCache) {
|
|
1896
1828
|
this.config = config;
|
|
1897
|
-
this.
|
|
1829
|
+
this.store = store;
|
|
1898
1830
|
this.grafeo = grafeo;
|
|
1899
1831
|
this.llm = llm;
|
|
1900
1832
|
this.embed = embed;
|
|
@@ -1951,14 +1883,14 @@ var CompressManager = class {
|
|
|
1951
1883
|
vector: topicVector
|
|
1952
1884
|
};
|
|
1953
1885
|
try {
|
|
1954
|
-
await this.
|
|
1886
|
+
await this.store.addTopic(topic);
|
|
1955
1887
|
} catch (err) {
|
|
1956
1888
|
console.error(`[CompressManager] Save topic failed:`, err);
|
|
1957
1889
|
}
|
|
1958
1890
|
this.sessionCache.clearMessages(sessionId, endTime);
|
|
1959
1891
|
const [n1, n2, n3] = this.config.topicRatio;
|
|
1960
1892
|
try {
|
|
1961
|
-
const topicGroups = await this.
|
|
1893
|
+
const topicGroups = await this.store.getRecentTopics(chatId, userId, n1, n2, n3);
|
|
1962
1894
|
this.sessionCache.buildHistoryWindow(sessionId, topicGroups);
|
|
1963
1895
|
} catch (err) {
|
|
1964
1896
|
console.error(`[CompressManager] Rebuild history window failed:`, err);
|
|
@@ -2032,13 +1964,13 @@ function withTimeout(promise, timeoutMs, label) {
|
|
|
2032
1964
|
// src/manager/fact.cache.ts
|
|
2033
1965
|
import { v4 as uuidv42 } from "uuid";
|
|
2034
1966
|
var FactCache = class {
|
|
2035
|
-
|
|
1967
|
+
store;
|
|
2036
1968
|
cache = /* @__PURE__ */ new Map();
|
|
2037
|
-
constructor(
|
|
2038
|
-
this.
|
|
1969
|
+
constructor(store) {
|
|
1970
|
+
this.store = store;
|
|
2039
1971
|
}
|
|
2040
1972
|
async init() {
|
|
2041
|
-
const facts = await this.
|
|
1973
|
+
const facts = await this.store.getAllFacts();
|
|
2042
1974
|
for (const fact of facts) {
|
|
2043
1975
|
const key = this.key(fact.level, fact.level === "user" ? fact.userId : fact.chatId);
|
|
2044
1976
|
const existing = this.cache.get(key) ?? [];
|
|
@@ -2059,7 +1991,7 @@ var FactCache = class {
|
|
|
2059
1991
|
content,
|
|
2060
1992
|
createdAt: Date.now()
|
|
2061
1993
|
};
|
|
2062
|
-
await this.
|
|
1994
|
+
await this.store.saveFact(fact);
|
|
2063
1995
|
const id = level === "user" ? userId : chatId;
|
|
2064
1996
|
const k = this.key(level, id);
|
|
2065
1997
|
const existing = this.cache.get(k) ?? [];
|
|
@@ -2073,7 +2005,7 @@ var FactCache = class {
|
|
|
2073
2005
|
all() {
|
|
2074
2006
|
return [...this.cache.values()].flat();
|
|
2075
2007
|
}
|
|
2076
|
-
/** 删除单条 fact
|
|
2008
|
+
/** 删除单条 fact:同步从存储与内存缓存中移除。返回是否命中。*/
|
|
2077
2009
|
async remove(factId) {
|
|
2078
2010
|
let hit = false;
|
|
2079
2011
|
for (const [k, facts] of this.cache) {
|
|
@@ -2086,7 +2018,7 @@ var FactCache = class {
|
|
|
2086
2018
|
break;
|
|
2087
2019
|
}
|
|
2088
2020
|
}
|
|
2089
|
-
await this.
|
|
2021
|
+
await this.store.deleteFact(factId);
|
|
2090
2022
|
return hit;
|
|
2091
2023
|
}
|
|
2092
2024
|
toString(level, id) {
|
|
@@ -2230,14 +2162,14 @@ function takeTailTokens(text, overlapTokens) {
|
|
|
2230
2162
|
// src/manager/knowledge.manager.ts
|
|
2231
2163
|
var KnowledgeManager = class {
|
|
2232
2164
|
config;
|
|
2233
|
-
|
|
2165
|
+
store;
|
|
2234
2166
|
grafeo;
|
|
2235
2167
|
llm;
|
|
2236
2168
|
embed;
|
|
2237
2169
|
semaphore;
|
|
2238
|
-
constructor(config,
|
|
2170
|
+
constructor(config, store, grafeo, llm, embed) {
|
|
2239
2171
|
this.config = config;
|
|
2240
|
-
this.
|
|
2172
|
+
this.store = store;
|
|
2241
2173
|
this.grafeo = grafeo;
|
|
2242
2174
|
this.llm = llm;
|
|
2243
2175
|
this.embed = embed;
|
|
@@ -2252,8 +2184,12 @@ var KnowledgeManager = class {
|
|
|
2252
2184
|
const now = Date.now();
|
|
2253
2185
|
const content = opts.content;
|
|
2254
2186
|
const contentHash = createHash("sha256").update(content).digest("hex");
|
|
2255
|
-
const domainFilter =
|
|
2256
|
-
|
|
2187
|
+
const domainFilter = [
|
|
2188
|
+
eq("user_id", userId),
|
|
2189
|
+
eq("chat_id", chatId),
|
|
2190
|
+
eq("session_id", sessionId)
|
|
2191
|
+
];
|
|
2192
|
+
const existing = await this.store.findDocumentByHash(contentHash, domainFilter);
|
|
2257
2193
|
if (existing) return { docId: existing.docId };
|
|
2258
2194
|
const pieces = chunkMarkdown(content, {
|
|
2259
2195
|
maxTokens: this.config.chunkMaxTokens,
|
|
@@ -2286,7 +2222,7 @@ var KnowledgeManager = class {
|
|
|
2286
2222
|
createdAt: now,
|
|
2287
2223
|
updatedAt: now
|
|
2288
2224
|
};
|
|
2289
|
-
await this.
|
|
2225
|
+
await this.store.addDocument(doc);
|
|
2290
2226
|
const chunkIngest = this.semaphore.run(() => this.ingestChunks(doc, pieces));
|
|
2291
2227
|
const graphBuild = chunkIngest.then((chunks) => {
|
|
2292
2228
|
if (!shouldBuildGraph) return;
|
|
@@ -2336,7 +2272,7 @@ ${p.content}` : p.content
|
|
|
2336
2272
|
metadata: {},
|
|
2337
2273
|
createdAt: doc.createdAt
|
|
2338
2274
|
}));
|
|
2339
|
-
await this.
|
|
2275
|
+
await this.store.addChunks(chunks);
|
|
2340
2276
|
return chunks;
|
|
2341
2277
|
}
|
|
2342
2278
|
async buildDocumentGraph(doc, chunks) {
|
|
@@ -2389,7 +2325,7 @@ ${p.content}` : p.content
|
|
|
2389
2325
|
embeddings,
|
|
2390
2326
|
KIND_KNOWLEDGE
|
|
2391
2327
|
);
|
|
2392
|
-
await this.
|
|
2328
|
+
await this.store.updateDocumentGraphFlag(doc.docId, true).catch((err) => {
|
|
2393
2329
|
console.error(`[KnowledgeManager] Failed to update graph flag for doc ${doc.docId}:`, err);
|
|
2394
2330
|
});
|
|
2395
2331
|
}
|
|
@@ -2402,11 +2338,11 @@ ${p.content}` : p.content
|
|
|
2402
2338
|
const mode = opts.mode ?? "auto";
|
|
2403
2339
|
const limit = opts.limit ?? this.config.knowledgeTopK;
|
|
2404
2340
|
const vector = await this.embed.embedOne(opts.query);
|
|
2405
|
-
const docFilter =
|
|
2341
|
+
const docFilter = buildDomainFilter(scope, scopeId);
|
|
2406
2342
|
let candidateDocIds;
|
|
2407
2343
|
const docTitleMap = /* @__PURE__ */ new Map();
|
|
2408
2344
|
if (this.config.docCoarseTopK > 0) {
|
|
2409
|
-
const coarse = await this.
|
|
2345
|
+
const coarse = await this.store.searchDocuments(vector, docFilter, this.config.docCoarseTopK);
|
|
2410
2346
|
if (coarse.length > 0) {
|
|
2411
2347
|
candidateDocIds = coarse.map((d) => d.docId);
|
|
2412
2348
|
for (const d of coarse) docTitleMap.set(d.docId, d.title);
|
|
@@ -2414,7 +2350,7 @@ ${p.content}` : p.content
|
|
|
2414
2350
|
}
|
|
2415
2351
|
const chunkFilter = await this.buildChunkFilter(scope, scopeId, candidateDocIds);
|
|
2416
2352
|
if (chunkFilter === NO_MATCH) return empty;
|
|
2417
|
-
const rawChunks = await this.
|
|
2353
|
+
const rawChunks = await this.store.searchChunks(opts.query, vector, chunkFilter, limit);
|
|
2418
2354
|
const scored = rawChunks.map((c) => ({
|
|
2419
2355
|
chunkId: c.chunkId,
|
|
2420
2356
|
docId: c.docId,
|
|
@@ -2428,7 +2364,7 @@ ${p.content}` : p.content
|
|
|
2428
2364
|
[...docCount.entries()].map(async ([docId, matchedChunkCount]) => {
|
|
2429
2365
|
let title = docTitleMap.get(docId);
|
|
2430
2366
|
if (title === void 0) {
|
|
2431
|
-
const d = await this.
|
|
2367
|
+
const d = await this.store.getDocument(docId);
|
|
2432
2368
|
title = d?.title ?? "";
|
|
2433
2369
|
}
|
|
2434
2370
|
return { docId, title, matchedChunkCount };
|
|
@@ -2476,32 +2412,32 @@ ${p.content}` : p.content
|
|
|
2476
2412
|
async buildChunkFilter(scope, scopeId, candidateDocIds) {
|
|
2477
2413
|
const parts = [];
|
|
2478
2414
|
if (this.config.chunkRedundantIds) {
|
|
2479
|
-
const
|
|
2480
|
-
if (
|
|
2481
|
-
if (candidateDocIds) parts.push(
|
|
2415
|
+
const domain = buildDomainFilter(scope, scopeId);
|
|
2416
|
+
if (domain) parts.push(...domain);
|
|
2417
|
+
if (candidateDocIds) parts.push({ op: "in", field: "doc_id", values: candidateDocIds });
|
|
2482
2418
|
} else {
|
|
2483
2419
|
let docIds = candidateDocIds;
|
|
2484
2420
|
if (scope !== "all") {
|
|
2485
|
-
const
|
|
2486
|
-
const domainDocIds = await this.
|
|
2421
|
+
const domain = buildDomainFilter(scope, scopeId);
|
|
2422
|
+
const domainDocIds = await this.store.getDocIdsByDomain(domain);
|
|
2487
2423
|
docIds = docIds ? domainDocIds.filter((id) => docIds.includes(id)) : domainDocIds;
|
|
2488
2424
|
}
|
|
2489
2425
|
if (docIds) {
|
|
2490
2426
|
if (docIds.length === 0) return NO_MATCH;
|
|
2491
|
-
parts.push(
|
|
2427
|
+
parts.push({ op: "in", field: "doc_id", values: docIds });
|
|
2492
2428
|
}
|
|
2493
2429
|
}
|
|
2494
|
-
return parts.length > 0 ? parts
|
|
2430
|
+
return parts.length > 0 ? parts : void 0;
|
|
2495
2431
|
}
|
|
2496
2432
|
// ── 读取 / 删除 ─────────────────────────────────────────────────────────────────
|
|
2497
2433
|
async getDocument(docId) {
|
|
2498
|
-
return this.
|
|
2434
|
+
return this.store.getDocument(docId);
|
|
2499
2435
|
}
|
|
2500
2436
|
async deleteDocument(docId) {
|
|
2501
|
-
const existing = await this.
|
|
2437
|
+
const existing = await this.store.getDocument(docId);
|
|
2502
2438
|
if (!existing) return false;
|
|
2503
|
-
await this.
|
|
2504
|
-
await this.
|
|
2439
|
+
await this.store.deleteDocument(docId);
|
|
2440
|
+
await this.store.deleteChunksByDoc(docId).catch((err) => {
|
|
2505
2441
|
console.error(`[KnowledgeManager] Failed to delete chunks for doc ${docId}:`, err);
|
|
2506
2442
|
});
|
|
2507
2443
|
await this.grafeo.deleteKnowledgeByDoc(docId).catch((err) => {
|
|
@@ -2510,7 +2446,7 @@ ${p.content}` : p.content
|
|
|
2510
2446
|
return true;
|
|
2511
2447
|
}
|
|
2512
2448
|
async listDocuments(filter, page) {
|
|
2513
|
-
let docs = await this.
|
|
2449
|
+
let docs = await this.store.getAllDocuments();
|
|
2514
2450
|
if (filter?.userId) docs = docs.filter((d) => d.userId === filter.userId);
|
|
2515
2451
|
if (filter?.chatId) docs = docs.filter((d) => d.chatId === filter.chatId);
|
|
2516
2452
|
if (filter?.sessionId) docs = docs.filter((d) => d.sessionId === filter.sessionId);
|
|
@@ -2528,19 +2464,12 @@ function dedupByContent(items) {
|
|
|
2528
2464
|
}
|
|
2529
2465
|
return out;
|
|
2530
2466
|
}
|
|
2531
|
-
function
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
if (scope === "session" && scopeId) return `session_id = '${esc(scopeId)}'`;
|
|
2536
|
-
if (scope === "chat" && scopeId) return `chat_id = '${esc(scopeId)}'`;
|
|
2537
|
-
if (scope === "user" && scopeId) return `user_id = '${esc(scopeId)}'`;
|
|
2467
|
+
function buildDomainFilter(scope, scopeId) {
|
|
2468
|
+
if (scope === "session" && scopeId) return [eq("session_id", scopeId)];
|
|
2469
|
+
if (scope === "chat" && scopeId) return [eq("chat_id", scopeId)];
|
|
2470
|
+
if (scope === "user" && scopeId) return [eq("user_id", scopeId)];
|
|
2538
2471
|
return void 0;
|
|
2539
2472
|
}
|
|
2540
|
-
function inSql(column, values) {
|
|
2541
|
-
const list = values.map((v) => `'${esc(v)}'`).join(", ");
|
|
2542
|
-
return `${column} IN (${list})`;
|
|
2543
|
-
}
|
|
2544
2473
|
function inferTitle(markdown) {
|
|
2545
2474
|
const lines = markdown.split(/\r?\n/);
|
|
2546
2475
|
for (const line of lines) {
|
|
@@ -2649,7 +2578,7 @@ var SessionCache = class {
|
|
|
2649
2578
|
// src/memory.manager.ts
|
|
2650
2579
|
var MemoryManager = class {
|
|
2651
2580
|
config;
|
|
2652
|
-
|
|
2581
|
+
store;
|
|
2653
2582
|
grafeo;
|
|
2654
2583
|
embed;
|
|
2655
2584
|
llm;
|
|
@@ -2658,17 +2587,19 @@ var MemoryManager = class {
|
|
|
2658
2587
|
compressManager;
|
|
2659
2588
|
knowledgeManager;
|
|
2660
2589
|
sessionMap = /* @__PURE__ */ new Map();
|
|
2590
|
+
optimizeTimer;
|
|
2591
|
+
optimizeRunning = false;
|
|
2661
2592
|
constructor(config) {
|
|
2662
2593
|
this.config = resolveConfig(config);
|
|
2663
|
-
this.
|
|
2594
|
+
this.store = new MemoryStore(this.config);
|
|
2664
2595
|
this.grafeo = new GrafeoService(this.config);
|
|
2665
2596
|
this.embed = new EmbedService(this.config);
|
|
2666
2597
|
this.llm = new LlmService(this.config);
|
|
2667
2598
|
this.sessionCache = new SessionCache(this.config);
|
|
2668
|
-
this.factCache = new FactCache(this.
|
|
2599
|
+
this.factCache = new FactCache(this.store);
|
|
2669
2600
|
this.compressManager = new CompressManager(
|
|
2670
2601
|
this.config,
|
|
2671
|
-
this.
|
|
2602
|
+
this.store,
|
|
2672
2603
|
this.grafeo,
|
|
2673
2604
|
this.llm,
|
|
2674
2605
|
this.embed,
|
|
@@ -2676,7 +2607,7 @@ var MemoryManager = class {
|
|
|
2676
2607
|
);
|
|
2677
2608
|
this.knowledgeManager = new KnowledgeManager(
|
|
2678
2609
|
this.config,
|
|
2679
|
-
this.
|
|
2610
|
+
this.store,
|
|
2680
2611
|
this.grafeo,
|
|
2681
2612
|
this.llm,
|
|
2682
2613
|
this.embed
|
|
@@ -2684,26 +2615,42 @@ var MemoryManager = class {
|
|
|
2684
2615
|
}
|
|
2685
2616
|
async init() {
|
|
2686
2617
|
initEncoder();
|
|
2687
|
-
await this.
|
|
2618
|
+
await this.store.init();
|
|
2688
2619
|
await this.grafeo.init();
|
|
2689
2620
|
await this.factCache.init();
|
|
2690
|
-
const allSessions = await this.
|
|
2621
|
+
const allSessions = await this.store.getAllSessions();
|
|
2691
2622
|
for (const s of allSessions) {
|
|
2692
2623
|
this.sessionMap.set(s.sessionId, this.deserializeSession(s));
|
|
2693
2624
|
}
|
|
2694
2625
|
await this.restoreFromStorage();
|
|
2695
2626
|
if (this.config.autoOptimizeOnInit) {
|
|
2696
|
-
void this.
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2627
|
+
void this.runBackgroundOptimize(this.config.optimizeVersionRetentionMs);
|
|
2628
|
+
}
|
|
2629
|
+
if (this.config.autoOptimizeIntervalMs > 0) {
|
|
2630
|
+
const retention = Math.max(this.config.optimizeVersionRetentionMs, 6e4);
|
|
2631
|
+
this.optimizeTimer = setInterval(() => {
|
|
2632
|
+
void this.runBackgroundOptimize(retention);
|
|
2633
|
+
}, this.config.autoOptimizeIntervalMs);
|
|
2634
|
+
this.optimizeTimer.unref?.();
|
|
2635
|
+
}
|
|
2636
|
+
}
|
|
2637
|
+
/** 后台压实的统一入口:防重入(上一轮未结束则跳过),失败仅告警不影响服务。*/
|
|
2638
|
+
async runBackgroundOptimize(retentionMs) {
|
|
2639
|
+
if (this.optimizeRunning) return;
|
|
2640
|
+
this.optimizeRunning = true;
|
|
2641
|
+
try {
|
|
2642
|
+
const results = await this.optimizeStorage(retentionMs);
|
|
2643
|
+
const versions = results.reduce((s, r) => s + r.oldVersionsRemoved, 0);
|
|
2644
|
+
const fragments = results.reduce((s, r) => s + r.fragmentsRemoved, 0);
|
|
2645
|
+
if (versions > 0 || fragments > 0) {
|
|
2646
|
+
console.info(
|
|
2647
|
+
`[MemoryManager] \u540E\u53F0\u5B58\u50A8\u538B\u5B9E\u5B8C\u6210\uFF1A\u6E05\u7406\u5386\u53F2\u7248\u672C ${versions} \u4E2A\uFF0C\u5408\u5E76\u788E\u7247 ${fragments} \u4E2A`
|
|
2648
|
+
);
|
|
2649
|
+
}
|
|
2650
|
+
} catch (err) {
|
|
2651
|
+
console.warn("[MemoryManager] \u540E\u53F0\u5B58\u50A8\u538B\u5B9E\u5931\u8D25\uFF08\u4E0D\u5F71\u54CD\u670D\u52A1\uFF09:", err);
|
|
2652
|
+
} finally {
|
|
2653
|
+
this.optimizeRunning = false;
|
|
2707
2654
|
}
|
|
2708
2655
|
}
|
|
2709
2656
|
/**
|
|
@@ -2711,18 +2658,18 @@ var MemoryManager = class {
|
|
|
2711
2658
|
* @param retentionMs 保留多久内的历史版本,默认取配置 optimizeVersionRetentionMs
|
|
2712
2659
|
*/
|
|
2713
2660
|
async optimizeStorage(retentionMs) {
|
|
2714
|
-
return this.
|
|
2661
|
+
return this.store.optimizeStorage(retentionMs ?? this.config.optimizeVersionRetentionMs);
|
|
2715
2662
|
}
|
|
2716
2663
|
async restoreFromStorage() {
|
|
2717
2664
|
const [n1, n2, n3] = this.config.topicRatio;
|
|
2718
|
-
const sessionIds = await this.
|
|
2665
|
+
const sessionIds = await this.store.getAllSessionIds();
|
|
2719
2666
|
if (sessionIds.length === 0) return;
|
|
2720
2667
|
const topicsCache = /* @__PURE__ */ new Map();
|
|
2721
2668
|
const getTopics = (chatId, userId) => {
|
|
2722
2669
|
const key = `${chatId}\0${userId}`;
|
|
2723
2670
|
let cached = topicsCache.get(key);
|
|
2724
2671
|
if (!cached) {
|
|
2725
|
-
cached = this.
|
|
2672
|
+
cached = this.store.getRecentTopics(chatId, userId, n1, n2, n3);
|
|
2726
2673
|
topicsCache.set(key, cached);
|
|
2727
2674
|
}
|
|
2728
2675
|
return cached;
|
|
@@ -2731,12 +2678,12 @@ var MemoryManager = class {
|
|
|
2731
2678
|
await Promise.all(
|
|
2732
2679
|
sessionIds.map(
|
|
2733
2680
|
(sessionId) => sem.run(async () => {
|
|
2734
|
-
const recentMessages = await this.
|
|
2681
|
+
const recentMessages = await this.store.getLatestMessages(sessionId, 1);
|
|
2735
2682
|
if (recentMessages.length === 0) return;
|
|
2736
2683
|
const { chatId, userId } = recentMessages[0];
|
|
2737
2684
|
const topicGroups = await getTopics(chatId, userId);
|
|
2738
2685
|
const n1EndTime = topicGroups.detail.length > 0 ? topicGroups.detail[topicGroups.detail.length - 1].endTime : 0;
|
|
2739
|
-
const rawMessages = n1EndTime > 0 ? await this.
|
|
2686
|
+
const rawMessages = n1EndTime > 0 ? await this.store.getMessagesSince(sessionId, n1EndTime) : await this.store.getLatestMessages(sessionId, 100);
|
|
2740
2687
|
if (rawMessages.length > 0) {
|
|
2741
2688
|
this.sessionCache.addMessages(sessionId, rawMessages, chatId, userId);
|
|
2742
2689
|
}
|
|
@@ -2789,7 +2736,7 @@ var MemoryManager = class {
|
|
|
2789
2736
|
vector: vectors[i],
|
|
2790
2737
|
createdAt: m.createdAt
|
|
2791
2738
|
}));
|
|
2792
|
-
await this.
|
|
2739
|
+
await this.store.addMessages(stored);
|
|
2793
2740
|
if (!this.sessionMap.has(sessionId)) {
|
|
2794
2741
|
const session = {
|
|
2795
2742
|
sessionId,
|
|
@@ -2801,13 +2748,13 @@ var MemoryManager = class {
|
|
|
2801
2748
|
updatedAt: now
|
|
2802
2749
|
};
|
|
2803
2750
|
this.sessionMap.set(sessionId, this.deserializeSession(session));
|
|
2804
|
-
this.
|
|
2751
|
+
this.store.insertSession(session).catch((err) => {
|
|
2805
2752
|
console.error("[MemoryManager] Failed to insert session:", err);
|
|
2806
2753
|
});
|
|
2807
2754
|
} else {
|
|
2808
2755
|
const view = this.sessionMap.get(sessionId);
|
|
2809
2756
|
view.updatedAt = now;
|
|
2810
|
-
this.
|
|
2757
|
+
this.store.upsertSession(this.serializeSession(view)).catch((err) => {
|
|
2811
2758
|
console.error(`[MemoryManager] Failed to upsert session ${sessionId}:`, err);
|
|
2812
2759
|
});
|
|
2813
2760
|
}
|
|
@@ -2862,7 +2809,7 @@ var MemoryManager = class {
|
|
|
2862
2809
|
} = opts;
|
|
2863
2810
|
if (!query.trim()) return [];
|
|
2864
2811
|
const vector = await this.embed.embedOne(query);
|
|
2865
|
-
const filter = this.
|
|
2812
|
+
const filter = this.buildScopeFilter(scope, scopeId);
|
|
2866
2813
|
let useGraph = false;
|
|
2867
2814
|
if (mode === "all") {
|
|
2868
2815
|
useGraph = true;
|
|
@@ -2870,7 +2817,7 @@ var MemoryManager = class {
|
|
|
2870
2817
|
useGraph = await this.llm.judgeNeedsGraphSearch(query).catch(() => false);
|
|
2871
2818
|
}
|
|
2872
2819
|
const tasks = [
|
|
2873
|
-
this.
|
|
2820
|
+
this.store.hybridSearchTopics(query, vector, filter, limit).then(
|
|
2874
2821
|
(topics) => topics.map((t) => ({
|
|
2875
2822
|
type: "topic",
|
|
2876
2823
|
content: t.detail,
|
|
@@ -2943,10 +2890,10 @@ var MemoryManager = class {
|
|
|
2943
2890
|
getHistoryWindow(sessionId) {
|
|
2944
2891
|
return this.sessionCache.getHistoryWindow(sessionId);
|
|
2945
2892
|
}
|
|
2946
|
-
|
|
2947
|
-
if (scope === "session" && scopeId) return
|
|
2948
|
-
if (scope === "chat" && scopeId) return
|
|
2949
|
-
if (scope === "user" && scopeId) return
|
|
2893
|
+
buildScopeFilter(scope, scopeId) {
|
|
2894
|
+
if (scope === "session" && scopeId) return [eq("session_id", scopeId)];
|
|
2895
|
+
if (scope === "chat" && scopeId) return [eq("chat_id", scopeId)];
|
|
2896
|
+
if (scope === "user" && scopeId) return [eq("user_id", scopeId)];
|
|
2950
2897
|
return void 0;
|
|
2951
2898
|
}
|
|
2952
2899
|
deserializeSession(s) {
|
|
@@ -2957,7 +2904,7 @@ var MemoryManager = class {
|
|
|
2957
2904
|
}
|
|
2958
2905
|
// ── Message queries ──────────────────────────────────────────────────────────
|
|
2959
2906
|
async getRecentMessages(sessionId, limit) {
|
|
2960
|
-
return this.
|
|
2907
|
+
return this.store.getLatestMessages(sessionId, limit);
|
|
2961
2908
|
}
|
|
2962
2909
|
// ── Session queries (in-memory) ──────────────────────────────────────────────
|
|
2963
2910
|
getSession(sessionId) {
|
|
@@ -2993,17 +2940,17 @@ var MemoryManager = class {
|
|
|
2993
2940
|
updatedAt: Date.now()
|
|
2994
2941
|
};
|
|
2995
2942
|
this.sessionMap.set(sessionId, updated);
|
|
2996
|
-
await this.
|
|
2943
|
+
await this.store.upsertSession(this.serializeSession(updated));
|
|
2997
2944
|
return updated;
|
|
2998
2945
|
}
|
|
2999
2946
|
async deleteSession(sessionId) {
|
|
3000
2947
|
if (!this.sessionMap.has(sessionId)) return false;
|
|
3001
2948
|
this.sessionMap.delete(sessionId);
|
|
3002
|
-
await this.
|
|
3003
|
-
await this.
|
|
2949
|
+
await this.store.deleteSession(sessionId);
|
|
2950
|
+
await this.store.deleteMessagesBySession(sessionId).catch((err) => {
|
|
3004
2951
|
console.error(`[MemoryManager] Failed to delete messages for session ${sessionId}:`, err);
|
|
3005
2952
|
});
|
|
3006
|
-
await this.
|
|
2953
|
+
await this.store.deleteTopicsBySession(sessionId).catch((err) => {
|
|
3007
2954
|
console.error(`[MemoryManager] Failed to delete topics for session ${sessionId}:`, err);
|
|
3008
2955
|
});
|
|
3009
2956
|
return true;
|
|
@@ -3012,8 +2959,8 @@ var MemoryManager = class {
|
|
|
3012
2959
|
/** 概览统计:各类记忆数据的总量 */
|
|
3013
2960
|
async stats() {
|
|
3014
2961
|
const [counts, knowledge, entities, relations] = await Promise.all([
|
|
3015
|
-
this.
|
|
3016
|
-
this.
|
|
2962
|
+
this.store.countAll(),
|
|
2963
|
+
this.store.countKnowledge(),
|
|
3017
2964
|
this.grafeo.getAllEntities(),
|
|
3018
2965
|
this.grafeo.getAllRelations()
|
|
3019
2966
|
]);
|
|
@@ -3027,7 +2974,7 @@ var MemoryManager = class {
|
|
|
3027
2974
|
/** 按天聚合最近 days 天的活跃趋势(概览图表用,含零值天)。days 默认 30,范围 1~365。 */
|
|
3028
2975
|
async trend(days = 30) {
|
|
3029
2976
|
const n = Math.min(365, Math.max(1, Math.floor(days)));
|
|
3030
|
-
return this.
|
|
2977
|
+
return this.store.trendDaily(n);
|
|
3031
2978
|
}
|
|
3032
2979
|
listSessions(filter, page) {
|
|
3033
2980
|
let results = [...this.sessionMap.values()];
|
|
@@ -3038,13 +2985,13 @@ var MemoryManager = class {
|
|
|
3038
2985
|
}
|
|
3039
2986
|
async listMessages(sessionId, limitOrPage) {
|
|
3040
2987
|
if (typeof limitOrPage === "object") {
|
|
3041
|
-
const all = await this.
|
|
2988
|
+
const all = await this.store.getAllMessagesBySession(sessionId);
|
|
3042
2989
|
return paginate(all, limitOrPage);
|
|
3043
2990
|
}
|
|
3044
|
-
return this.
|
|
2991
|
+
return this.store.getLatestMessages(sessionId, limitOrPage ?? 100);
|
|
3045
2992
|
}
|
|
3046
2993
|
async listTopics(filter, page) {
|
|
3047
|
-
let topics = await this.
|
|
2994
|
+
let topics = await this.store.getAllTopics();
|
|
3048
2995
|
if (filter?.sessionId) topics = topics.filter((t) => t.sessionId === filter.sessionId);
|
|
3049
2996
|
if (filter?.userId) topics = topics.filter((t) => t.userId === filter.userId);
|
|
3050
2997
|
if (filter?.chatId) topics = topics.filter((t) => t.chatId === filter.chatId);
|
|
@@ -3113,7 +3060,13 @@ var MemoryManager = class {
|
|
|
3113
3060
|
return page ? this.knowledgeManager.listDocuments(filter, page) : this.knowledgeManager.listDocuments(filter);
|
|
3114
3061
|
}
|
|
3115
3062
|
destroy() {
|
|
3063
|
+
if (this.optimizeTimer) {
|
|
3064
|
+
clearInterval(this.optimizeTimer);
|
|
3065
|
+
this.optimizeTimer = void 0;
|
|
3066
|
+
}
|
|
3116
3067
|
this.grafeo.close();
|
|
3068
|
+
void this.store.close().catch(() => {
|
|
3069
|
+
});
|
|
3117
3070
|
}
|
|
3118
3071
|
};
|
|
3119
3072
|
export {
|
|
@@ -3121,6 +3074,6 @@ export {
|
|
|
3121
3074
|
DEFAULT_RELATION_TYPES,
|
|
3122
3075
|
KIND_CONVERSATION,
|
|
3123
3076
|
KIND_KNOWLEDGE,
|
|
3124
|
-
|
|
3125
|
-
|
|
3077
|
+
MemoryManager,
|
|
3078
|
+
MemoryStore
|
|
3126
3079
|
};
|