@ppagent/memory 0.1.4 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +276 -126
- package/dist/index.js +454 -528
- package/dist/provider.resolver-ZLQ766IO.js +585 -0
- package/llms.txt +859 -842
- package/package.json +16 -7
package/dist/index.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { v4 as uuidv44 } from "uuid";
|
|
3
3
|
|
|
4
4
|
// src/config.ts
|
|
5
|
+
import * as path from "node:path";
|
|
5
6
|
function resolveConfig(config) {
|
|
6
7
|
let embeddingBaseUrl = config.embeddingBaseUrl;
|
|
7
8
|
let embeddingApiKey = config.embeddingApiKey;
|
|
@@ -19,6 +20,8 @@ function resolveConfig(config) {
|
|
|
19
20
|
...config,
|
|
20
21
|
embeddingBaseUrl,
|
|
21
22
|
embeddingApiKey,
|
|
23
|
+
provider: config.provider ?? "auto",
|
|
24
|
+
sqlitePath: config.sqlitePath ?? path.join(path.dirname(config.lancedbPath), "memory.sqlite3"),
|
|
22
25
|
sessionTokenLimit: config.sessionTokenLimit ?? 16386,
|
|
23
26
|
historyWindowTokenLimit: config.historyWindowTokenLimit ?? 10240,
|
|
24
27
|
topicRatio: config.topicRatio ?? [1, 5, 20],
|
|
@@ -711,120 +714,149 @@ var GrafeoService = class _GrafeoService {
|
|
|
711
714
|
}
|
|
712
715
|
};
|
|
713
716
|
|
|
714
|
-
// src/db/
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
// src/db/sql.util.ts
|
|
719
|
-
function escLance(value) {
|
|
720
|
-
return String(value).replace(/'/g, "''");
|
|
721
|
-
}
|
|
722
|
-
function eqFilter(field, value) {
|
|
723
|
-
return `${field} = '${escLance(value)}'`;
|
|
717
|
+
// src/db/store.types.ts
|
|
718
|
+
function eq(field, value) {
|
|
719
|
+
return { op: "eq", field, value };
|
|
724
720
|
}
|
|
725
721
|
|
|
726
|
-
// src/db/
|
|
722
|
+
// src/db/memory.store.ts
|
|
727
723
|
var MESSAGES_TABLE = "messages";
|
|
728
724
|
var TOPICS_TABLE = "topics";
|
|
729
725
|
var FACTS_TABLE = "facts";
|
|
730
726
|
var SESSIONS_TABLE = "sessions";
|
|
731
727
|
var DOCUMENTS_TABLE = "documents";
|
|
732
728
|
var CHUNKS_TABLE = "chunks";
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
}
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
}
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
}
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
729
|
+
var RRF_K = 60;
|
|
730
|
+
var HYBRID_OVERFETCH = 2;
|
|
731
|
+
function tableDefs(dim) {
|
|
732
|
+
return [
|
|
733
|
+
{
|
|
734
|
+
name: MESSAGES_TABLE,
|
|
735
|
+
vectorDimension: dim,
|
|
736
|
+
columns: [
|
|
737
|
+
{ name: "message_id", type: "text" },
|
|
738
|
+
{ name: "talker_id", type: "text" },
|
|
739
|
+
{ name: "chat_id", type: "text" },
|
|
740
|
+
{ name: "user_id", type: "text" },
|
|
741
|
+
{ name: "session_id", type: "text" },
|
|
742
|
+
{ name: "type", type: "text" },
|
|
743
|
+
{ name: "content", type: "text" },
|
|
744
|
+
{ name: "parts", type: "text", nullable: true },
|
|
745
|
+
// 兼容存量数据
|
|
746
|
+
{ name: "vector", type: "vector" },
|
|
747
|
+
{ name: "usage", type: "int" },
|
|
748
|
+
{ name: "metadata", type: "text" },
|
|
749
|
+
{ name: "created_at", type: "long" }
|
|
750
|
+
],
|
|
751
|
+
indexes: [
|
|
752
|
+
{ column: "message_id", kind: "scalar" },
|
|
753
|
+
{ column: "session_id", kind: "scalar" },
|
|
754
|
+
{ column: "content", kind: "fts" },
|
|
755
|
+
{ column: "metadata", kind: "fts" }
|
|
756
|
+
]
|
|
757
|
+
},
|
|
758
|
+
{
|
|
759
|
+
name: TOPICS_TABLE,
|
|
760
|
+
vectorDimension: dim,
|
|
761
|
+
columns: [
|
|
762
|
+
{ name: "summary_id", type: "text" },
|
|
763
|
+
{ name: "session_id", type: "text" },
|
|
764
|
+
{ name: "user_id", type: "text" },
|
|
765
|
+
{ name: "chat_id", type: "text" },
|
|
766
|
+
{ name: "title", type: "text", nullable: true },
|
|
767
|
+
// 兼容存量数据
|
|
768
|
+
{ name: "detail", type: "text" },
|
|
769
|
+
{ name: "summary", type: "text" },
|
|
770
|
+
{ name: "concise", type: "text" },
|
|
771
|
+
{ name: "vector", type: "vector" },
|
|
772
|
+
{ name: "start_time", type: "long" },
|
|
773
|
+
{ name: "end_time", type: "long" },
|
|
774
|
+
{ name: "created_at", type: "long" },
|
|
775
|
+
{ name: "updated_at", type: "long" },
|
|
776
|
+
{ name: "recall_count", type: "int" }
|
|
777
|
+
],
|
|
778
|
+
indexes: [
|
|
779
|
+
{ column: "chat_id", kind: "scalar" },
|
|
780
|
+
{ column: "detail", kind: "fts" }
|
|
781
|
+
]
|
|
782
|
+
},
|
|
783
|
+
{
|
|
784
|
+
name: FACTS_TABLE,
|
|
785
|
+
columns: [
|
|
786
|
+
{ name: "fact_id", type: "text" },
|
|
787
|
+
{ name: "level", type: "text" },
|
|
788
|
+
{ name: "chat_id", type: "text" },
|
|
789
|
+
{ name: "session_id", type: "text" },
|
|
790
|
+
{ name: "user_id", type: "text" },
|
|
791
|
+
{ name: "content", type: "text" },
|
|
792
|
+
{ name: "created_at", type: "long" }
|
|
793
|
+
],
|
|
794
|
+
indexes: []
|
|
795
|
+
},
|
|
796
|
+
{
|
|
797
|
+
name: SESSIONS_TABLE,
|
|
798
|
+
columns: [
|
|
799
|
+
{ name: "session_id", type: "text" },
|
|
800
|
+
{ name: "chat_id", type: "text" },
|
|
801
|
+
{ name: "user_id", type: "text" },
|
|
802
|
+
{ name: "title", type: "text" },
|
|
803
|
+
{ name: "metadata", type: "text" },
|
|
804
|
+
{ name: "created_at", type: "long" },
|
|
805
|
+
{ name: "updated_at", type: "long" }
|
|
806
|
+
],
|
|
807
|
+
indexes: [{ column: "session_id", kind: "scalar" }]
|
|
808
|
+
},
|
|
809
|
+
{
|
|
810
|
+
name: DOCUMENTS_TABLE,
|
|
811
|
+
vectorDimension: dim,
|
|
812
|
+
columns: [
|
|
813
|
+
{ name: "doc_id", type: "text" },
|
|
814
|
+
{ name: "user_id", type: "text" },
|
|
815
|
+
{ name: "chat_id", type: "text" },
|
|
816
|
+
{ name: "session_id", type: "text" },
|
|
817
|
+
{ name: "title", type: "text" },
|
|
818
|
+
{ name: "source_name", type: "text" },
|
|
819
|
+
{ name: "full_content", type: "text" },
|
|
820
|
+
{ name: "content_hash", type: "text" },
|
|
821
|
+
{ name: "summary", type: "text" },
|
|
822
|
+
// vector 列存储文档摘要向量(summaryVector),用于文档级粗召回
|
|
823
|
+
{ name: "vector", type: "vector" },
|
|
824
|
+
{ name: "chunk_count", type: "int" },
|
|
825
|
+
{ name: "has_graph", type: "int" },
|
|
826
|
+
// 0/1 充当布尔
|
|
827
|
+
{ name: "metadata", type: "text" },
|
|
828
|
+
{ name: "created_at", type: "long" },
|
|
829
|
+
{ name: "updated_at", type: "long" }
|
|
830
|
+
],
|
|
831
|
+
indexes: [{ column: "doc_id", kind: "scalar" }]
|
|
832
|
+
},
|
|
833
|
+
{
|
|
834
|
+
name: CHUNKS_TABLE,
|
|
835
|
+
vectorDimension: dim,
|
|
836
|
+
columns: [
|
|
837
|
+
{ name: "chunk_id", type: "text" },
|
|
838
|
+
{ name: "doc_id", type: "text" },
|
|
839
|
+
// user_id/chat_id/session_id:chunkRedundantIds=false 时存空串,仅经 documents join 过滤
|
|
840
|
+
{ name: "user_id", type: "text" },
|
|
841
|
+
{ name: "chat_id", type: "text" },
|
|
842
|
+
{ name: "session_id", type: "text" },
|
|
843
|
+
{ name: "content", type: "text" },
|
|
844
|
+
{ name: "vector", type: "vector" },
|
|
845
|
+
{ name: "heading_path", type: "text" },
|
|
846
|
+
{ name: "ordinal", type: "int" },
|
|
847
|
+
{ name: "tokens", type: "int" },
|
|
848
|
+
{ name: "metadata", type: "text" },
|
|
849
|
+
{ name: "created_at", type: "long" }
|
|
850
|
+
],
|
|
851
|
+
indexes: [
|
|
852
|
+
{ column: "doc_id", kind: "scalar" },
|
|
853
|
+
{ column: "content", kind: "fts" }
|
|
854
|
+
]
|
|
855
|
+
}
|
|
856
|
+
];
|
|
811
857
|
}
|
|
812
|
-
function
|
|
813
|
-
return
|
|
814
|
-
new Field("chunk_id", new Utf8(), false),
|
|
815
|
-
new Field("doc_id", new Utf8(), false),
|
|
816
|
-
// user_id/chat_id/session_id:chunkRedundantIds=false 时存空串,仅经 documents join 过滤
|
|
817
|
-
new Field("user_id", new Utf8(), false),
|
|
818
|
-
new Field("chat_id", new Utf8(), false),
|
|
819
|
-
new Field("session_id", new Utf8(), false),
|
|
820
|
-
new Field("content", new Utf8(), false),
|
|
821
|
-
new Field("vector", new FixedSizeList(dim, new Field("item", new Float32(), false)), false),
|
|
822
|
-
new Field("heading_path", new Utf8(), false),
|
|
823
|
-
new Field("ordinal", new Int32(), false),
|
|
824
|
-
new Field("tokens", new Int32(), false),
|
|
825
|
-
new Field("metadata", new Utf8(), false),
|
|
826
|
-
new Field("created_at", new Int64(), false)
|
|
827
|
-
]);
|
|
858
|
+
function toVector(v) {
|
|
859
|
+
return Array.isArray(v) ? v : Array.from(v);
|
|
828
860
|
}
|
|
829
861
|
function messageToRow(m) {
|
|
830
862
|
return {
|
|
@@ -853,7 +885,7 @@ function rowToMessage(r) {
|
|
|
853
885
|
content: r.content,
|
|
854
886
|
parts: r.parts ?? "[]",
|
|
855
887
|
// 旧数据无此列时安全降级
|
|
856
|
-
vector:
|
|
888
|
+
vector: toVector(r.vector),
|
|
857
889
|
usage: Number(r.usage),
|
|
858
890
|
metadata: r.metadata,
|
|
859
891
|
createdAt: Number(r.created_at)
|
|
@@ -887,7 +919,7 @@ function rowToTopic(r) {
|
|
|
887
919
|
detail: r.detail,
|
|
888
920
|
summary: r.summary,
|
|
889
921
|
concise: r.concise,
|
|
890
|
-
vector:
|
|
922
|
+
vector: toVector(r.vector),
|
|
891
923
|
startTime: Number(r.start_time),
|
|
892
924
|
endTime: Number(r.end_time),
|
|
893
925
|
createdAt: Number(r.created_at),
|
|
@@ -969,7 +1001,7 @@ function rowToDocument(r) {
|
|
|
969
1001
|
fullContent: r.full_content,
|
|
970
1002
|
contentHash: r.content_hash,
|
|
971
1003
|
summary: r.summary,
|
|
972
|
-
summaryVector:
|
|
1004
|
+
summaryVector: toVector(r.vector),
|
|
973
1005
|
chunkCount: Number(r.chunk_count),
|
|
974
1006
|
hasGraph: Number(r.has_graph) === 1,
|
|
975
1007
|
metadata: safeParseObject(r.metadata),
|
|
@@ -1001,7 +1033,7 @@ function rowToChunk(r) {
|
|
|
1001
1033
|
chatId: r.chat_id,
|
|
1002
1034
|
sessionId: r.session_id,
|
|
1003
1035
|
content: r.content,
|
|
1004
|
-
vector:
|
|
1036
|
+
vector: toVector(r.vector),
|
|
1005
1037
|
headingPath: r.heading_path,
|
|
1006
1038
|
ordinal: Number(r.ordinal),
|
|
1007
1039
|
tokens: Number(r.tokens),
|
|
@@ -1016,236 +1048,63 @@ function safeParseObject(s) {
|
|
|
1016
1048
|
return {};
|
|
1017
1049
|
}
|
|
1018
1050
|
}
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
var
|
|
1033
|
-
{ column: "doc_id" },
|
|
1034
|
-
{ column: "content", fts: true }
|
|
1035
|
-
// 知识片段混合检索
|
|
1036
|
-
];
|
|
1037
|
-
var LanceService = class {
|
|
1051
|
+
function rrfFuse(lists, keyOf) {
|
|
1052
|
+
const acc = /* @__PURE__ */ new Map();
|
|
1053
|
+
for (const list of lists) {
|
|
1054
|
+
list.forEach((row, idx) => {
|
|
1055
|
+
const id = keyOf(row);
|
|
1056
|
+
const inc = 1 / (RRF_K + idx + 1);
|
|
1057
|
+
const entry = acc.get(id);
|
|
1058
|
+
if (entry) entry.score += inc;
|
|
1059
|
+
else acc.set(id, { row, score: inc });
|
|
1060
|
+
});
|
|
1061
|
+
}
|
|
1062
|
+
return [...acc.values()].sort((a, b) => b.score - a.score).map((e) => e.row);
|
|
1063
|
+
}
|
|
1064
|
+
var MemoryStore = class {
|
|
1038
1065
|
config;
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
sessionsTable;
|
|
1044
|
-
documentsTable;
|
|
1045
|
-
chunksTable;
|
|
1046
|
-
// Scalar indexes cannot be created on empty tables (LanceDB btree limitation).
|
|
1047
|
-
// These flags defer creation to the first insert.
|
|
1048
|
-
isNewMessagesTable = false;
|
|
1049
|
-
isNewTopicsTable = false;
|
|
1050
|
-
isNewSessionsTable = false;
|
|
1051
|
-
isNewDocumentsTable = false;
|
|
1052
|
-
isNewChunksTable = false;
|
|
1053
|
-
constructor(config) {
|
|
1066
|
+
provider;
|
|
1067
|
+
providerOverride;
|
|
1068
|
+
/** provider 省略时在 init() 阶段经 provider.resolver 自动探测创建(测试可显式注入) */
|
|
1069
|
+
constructor(config, provider) {
|
|
1054
1070
|
this.config = config;
|
|
1071
|
+
this.providerOverride = provider;
|
|
1072
|
+
}
|
|
1073
|
+
/** 当前后端 provider 标识(日志/诊断用) */
|
|
1074
|
+
get providerKind() {
|
|
1075
|
+
return this.provider?.kind ?? "uninitialized";
|
|
1055
1076
|
}
|
|
1056
1077
|
async init() {
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
const existingTables = await this.conn.tableNames();
|
|
1060
|
-
if (existingTables.includes(MESSAGES_TABLE)) {
|
|
1061
|
-
this.messagesTable = await this.conn.openTable(MESSAGES_TABLE);
|
|
1062
|
-
await this._ensurePartsColumn();
|
|
1078
|
+
if (this.providerOverride) {
|
|
1079
|
+
this.provider = this.providerOverride;
|
|
1063
1080
|
} else {
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
);
|
|
1068
|
-
this.isNewMessagesTable = true;
|
|
1081
|
+
const { createProvider } = await import("./provider.resolver-ZLQ766IO.js");
|
|
1082
|
+
this.provider = await createProvider(this.config);
|
|
1083
|
+
console.info(`[memory] \u5411\u91CF\u5B58\u50A8\u540E\u7AEF\uFF1A${this.provider.kind}`);
|
|
1069
1084
|
}
|
|
1070
|
-
|
|
1071
|
-
this.topicsTable = await this.conn.openTable(TOPICS_TABLE);
|
|
1072
|
-
await this._ensureTopicTitleColumn();
|
|
1073
|
-
} else {
|
|
1074
|
-
this.topicsTable = await this.conn.createEmptyTable(
|
|
1075
|
-
TOPICS_TABLE,
|
|
1076
|
-
topicsSchema(dim)
|
|
1077
|
-
);
|
|
1078
|
-
this.isNewTopicsTable = true;
|
|
1079
|
-
}
|
|
1080
|
-
if (existingTables.includes(FACTS_TABLE)) {
|
|
1081
|
-
this.factsTable = await this.conn.openTable(FACTS_TABLE);
|
|
1082
|
-
} else {
|
|
1083
|
-
this.factsTable = await this.conn.createEmptyTable(FACTS_TABLE, factsSchema());
|
|
1084
|
-
}
|
|
1085
|
-
if (existingTables.includes(SESSIONS_TABLE)) {
|
|
1086
|
-
this.sessionsTable = await this.conn.openTable(SESSIONS_TABLE);
|
|
1087
|
-
} else {
|
|
1088
|
-
this.sessionsTable = await this.conn.createEmptyTable(SESSIONS_TABLE, sessionsSchema());
|
|
1089
|
-
this.isNewSessionsTable = true;
|
|
1090
|
-
}
|
|
1091
|
-
if (existingTables.includes(DOCUMENTS_TABLE)) {
|
|
1092
|
-
this.documentsTable = await this.conn.openTable(DOCUMENTS_TABLE);
|
|
1093
|
-
} else {
|
|
1094
|
-
this.documentsTable = await this.conn.createEmptyTable(DOCUMENTS_TABLE, documentsSchema(dim));
|
|
1095
|
-
this.isNewDocumentsTable = true;
|
|
1096
|
-
}
|
|
1097
|
-
if (existingTables.includes(CHUNKS_TABLE)) {
|
|
1098
|
-
this.chunksTable = await this.conn.openTable(CHUNKS_TABLE);
|
|
1099
|
-
} else {
|
|
1100
|
-
this.chunksTable = await this.conn.createEmptyTable(CHUNKS_TABLE, chunksSchema(dim));
|
|
1101
|
-
this.isNewChunksTable = true;
|
|
1102
|
-
}
|
|
1103
|
-
await this.ensureIndexes(this.messagesTable, MESSAGES_INDEXES);
|
|
1104
|
-
await this.ensureIndexes(this.topicsTable, TOPICS_INDEXES);
|
|
1105
|
-
await this.ensureIndexes(this.sessionsTable, SESSIONS_INDEXES);
|
|
1106
|
-
await this.ensureIndexes(this.documentsTable, DOCUMENTS_INDEXES);
|
|
1107
|
-
await this.ensureIndexes(this.chunksTable, CHUNKS_INDEXES);
|
|
1085
|
+
await this.provider.init(tableDefs(this.config.embeddingDimension));
|
|
1108
1086
|
}
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
* createIndex 默认 replace:true 会在每次启动时全量重建索引并提交新表版本——
|
|
1112
|
-
* 这正是历史上「启动越来越慢 + _versions 目录膨胀」的根源,绝不可回退到无条件 createIndex。
|
|
1113
|
-
*/
|
|
1114
|
-
async ensureIndexes(table, specs) {
|
|
1115
|
-
let indexed;
|
|
1116
|
-
try {
|
|
1117
|
-
indexed = new Set((await table.listIndices()).flatMap((i) => i.columns));
|
|
1118
|
-
} catch {
|
|
1119
|
-
indexed = /* @__PURE__ */ new Set();
|
|
1120
|
-
}
|
|
1121
|
-
for (const { column, fts } of specs) {
|
|
1122
|
-
if (indexed.has(column)) continue;
|
|
1123
|
-
try {
|
|
1124
|
-
await table.createIndex(
|
|
1125
|
-
column,
|
|
1126
|
-
fts ? { config: lancedb.Index.fts(), replace: false } : { replace: false }
|
|
1127
|
-
);
|
|
1128
|
-
} catch {
|
|
1129
|
-
}
|
|
1130
|
-
}
|
|
1087
|
+
async close() {
|
|
1088
|
+
await this.provider.close();
|
|
1131
1089
|
}
|
|
1132
1090
|
/**
|
|
1133
|
-
*
|
|
1134
|
-
*
|
|
1091
|
+
* 存储压实(碎片合并 + 历史版本清理)。
|
|
1092
|
+
* LanceDB 后端长期运行必须定期执行;SQLite 后端为可选的空间回收。
|
|
1135
1093
|
*/
|
|
1136
1094
|
async optimizeStorage(retentionMs = 0) {
|
|
1137
|
-
|
|
1138
|
-
const targets = [
|
|
1139
|
-
[MESSAGES_TABLE, this.messagesTable],
|
|
1140
|
-
[TOPICS_TABLE, this.topicsTable],
|
|
1141
|
-
[FACTS_TABLE, this.factsTable],
|
|
1142
|
-
[SESSIONS_TABLE, this.sessionsTable],
|
|
1143
|
-
[DOCUMENTS_TABLE, this.documentsTable],
|
|
1144
|
-
[CHUNKS_TABLE, this.chunksTable]
|
|
1145
|
-
];
|
|
1146
|
-
const results = [];
|
|
1147
|
-
for (const [name, table] of targets) {
|
|
1148
|
-
try {
|
|
1149
|
-
const stats = await table.optimize({ cleanupOlderThan: cutoff });
|
|
1150
|
-
results.push({
|
|
1151
|
-
table: name,
|
|
1152
|
-
fragmentsRemoved: stats.compaction.fragmentsRemoved,
|
|
1153
|
-
fragmentsAdded: stats.compaction.fragmentsAdded,
|
|
1154
|
-
filesRemoved: stats.compaction.filesRemoved,
|
|
1155
|
-
oldVersionsRemoved: stats.prune.oldVersionsRemoved,
|
|
1156
|
-
bytesRemoved: Number(stats.prune.bytesRemoved)
|
|
1157
|
-
});
|
|
1158
|
-
} catch (err) {
|
|
1159
|
-
console.warn(`[LanceService] optimize table "${name}" failed:`, err);
|
|
1160
|
-
}
|
|
1161
|
-
}
|
|
1162
|
-
return results;
|
|
1163
|
-
}
|
|
1164
|
-
/**
|
|
1165
|
-
* 为存量 messages 表添加 parts 列(如果缺失)。
|
|
1166
|
-
* LanceDB 0.14+ 支持 addColumns;旧版本会 throw,此时 rowToMessage 的 ?? "[]" 兜底。
|
|
1167
|
-
*/
|
|
1168
|
-
async _ensurePartsColumn() {
|
|
1169
|
-
try {
|
|
1170
|
-
const schema = await this.messagesTable.schema();
|
|
1171
|
-
const hasPartsCol = schema.fields?.some(
|
|
1172
|
-
(f) => f.name === "parts"
|
|
1173
|
-
);
|
|
1174
|
-
if (!hasPartsCol) {
|
|
1175
|
-
await this.messagesTable.addColumns([
|
|
1176
|
-
{ name: "parts", valueSql: "CAST(NULL AS STRING)" }
|
|
1177
|
-
]);
|
|
1178
|
-
}
|
|
1179
|
-
} catch {
|
|
1180
|
-
}
|
|
1181
|
-
}
|
|
1182
|
-
/**
|
|
1183
|
-
* 为存量 topics 表添加 title 列(如果缺失)。
|
|
1184
|
-
* 旧版本不支持 addColumns 时忽略,读取时 rowToTopic 的 ?? "" 兜底。
|
|
1185
|
-
*/
|
|
1186
|
-
async _ensureTopicTitleColumn() {
|
|
1187
|
-
try {
|
|
1188
|
-
const schema = await this.topicsTable.schema();
|
|
1189
|
-
const hasTitleCol = schema.fields?.some(
|
|
1190
|
-
(f) => f.name === "title"
|
|
1191
|
-
);
|
|
1192
|
-
if (!hasTitleCol) {
|
|
1193
|
-
await this.topicsTable.addColumns([
|
|
1194
|
-
{ name: "title", valueSql: "CAST(NULL AS STRING)" }
|
|
1195
|
-
]);
|
|
1196
|
-
}
|
|
1197
|
-
} catch {
|
|
1198
|
-
}
|
|
1095
|
+
return this.provider.optimize(retentionMs);
|
|
1199
1096
|
}
|
|
1097
|
+
// ── messages ───────────────────────────────────────────────────────────────
|
|
1200
1098
|
async addMessages(messages) {
|
|
1201
1099
|
if (messages.length === 0) return;
|
|
1202
|
-
await this.
|
|
1203
|
-
if (this.isNewMessagesTable) {
|
|
1204
|
-
await this.ensureIndexes(this.messagesTable, MESSAGES_INDEXES);
|
|
1205
|
-
this.isNewMessagesTable = false;
|
|
1206
|
-
}
|
|
1207
|
-
}
|
|
1208
|
-
async addTopic(topic) {
|
|
1209
|
-
await this.topicsTable.add([topicToRow(topic)]);
|
|
1210
|
-
if (this.isNewTopicsTable) {
|
|
1211
|
-
await this.ensureIndexes(this.topicsTable, TOPICS_INDEXES);
|
|
1212
|
-
this.isNewTopicsTable = false;
|
|
1213
|
-
}
|
|
1214
|
-
}
|
|
1215
|
-
async updateTopicRecallCount(summaryId, count) {
|
|
1216
|
-
await this.topicsTable.update({
|
|
1217
|
-
values: { recall_count: count, updated_at: Date.now() },
|
|
1218
|
-
where: eqFilter("summary_id", summaryId)
|
|
1219
|
-
});
|
|
1220
|
-
}
|
|
1221
|
-
async getRecentTopics(chatId, userId, n1, n2, n3) {
|
|
1222
|
-
const filter = `${eqFilter("chat_id", chatId)} AND ${eqFilter("user_id", userId)}`;
|
|
1223
|
-
const recallBoostMs = this.config.recallBoostMs;
|
|
1224
|
-
const fetchTopics = async (count) => {
|
|
1225
|
-
if (count <= 0) return [];
|
|
1226
|
-
try {
|
|
1227
|
-
const rows = await this.topicsTable.query().where(filter).limit(count * 5).toArray();
|
|
1228
|
-
return rows.sort((a, b) => {
|
|
1229
|
-
const scoreA = Number(a.end_time) + Number(a.recall_count) * recallBoostMs;
|
|
1230
|
-
const scoreB = Number(b.end_time) + Number(b.recall_count) * recallBoostMs;
|
|
1231
|
-
return scoreB - scoreA;
|
|
1232
|
-
}).slice(0, count).map(rowToTopic);
|
|
1233
|
-
} catch {
|
|
1234
|
-
return [];
|
|
1235
|
-
}
|
|
1236
|
-
};
|
|
1237
|
-
const allTopics = await fetchTopics(n1 + n2 + n3);
|
|
1238
|
-
return {
|
|
1239
|
-
detail: allTopics.slice(0, n1),
|
|
1240
|
-
summary: allTopics.slice(n1, n1 + n2),
|
|
1241
|
-
concise: allTopics.slice(n1 + n2, n1 + n2 + n3)
|
|
1242
|
-
};
|
|
1100
|
+
await this.provider.add(MESSAGES_TABLE, messages.map(messageToRow));
|
|
1243
1101
|
}
|
|
1244
1102
|
async getMessagesSince(sessionId, since, limit) {
|
|
1245
1103
|
try {
|
|
1246
|
-
const
|
|
1247
|
-
|
|
1248
|
-
|
|
1104
|
+
const rows = await this.provider.query(MESSAGES_TABLE, {
|
|
1105
|
+
filter: [eq("session_id", sessionId), { op: "gt", field: "created_at", value: since }],
|
|
1106
|
+
limit
|
|
1107
|
+
});
|
|
1249
1108
|
return rows.sort((a, b) => Number(a.created_at) - Number(b.created_at)).map(rowToMessage);
|
|
1250
1109
|
} catch {
|
|
1251
1110
|
return [];
|
|
@@ -1254,10 +1113,14 @@ var LanceService = class {
|
|
|
1254
1113
|
async getLatestMessages(sessionId, limit) {
|
|
1255
1114
|
if (limit <= 0) return [];
|
|
1256
1115
|
try {
|
|
1257
|
-
const rows = await this.
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1116
|
+
const rows = await this.provider.query(MESSAGES_TABLE, {
|
|
1117
|
+
filter: [eq("session_id", sessionId)],
|
|
1118
|
+
orderBy: [
|
|
1119
|
+
{ column: "created_at", ascending: false },
|
|
1120
|
+
{ column: "message_id", ascending: false }
|
|
1121
|
+
],
|
|
1122
|
+
limit
|
|
1123
|
+
});
|
|
1261
1124
|
return rows.reverse().map(rowToMessage);
|
|
1262
1125
|
} catch {
|
|
1263
1126
|
return [];
|
|
@@ -1266,54 +1129,51 @@ var LanceService = class {
|
|
|
1266
1129
|
/** 取某会话的全部消息(按 createdAt 升序),供管理面板分页切片使用 */
|
|
1267
1130
|
async getAllMessagesBySession(sessionId) {
|
|
1268
1131
|
try {
|
|
1269
|
-
const rows = await this.
|
|
1270
|
-
|
|
1132
|
+
const rows = await this.provider.query(MESSAGES_TABLE, {
|
|
1133
|
+
filter: [eq("session_id", sessionId)]
|
|
1134
|
+
});
|
|
1135
|
+
return rows.sort(
|
|
1136
|
+
(a, b) => Number(a.created_at) - Number(b.created_at) || cmpStr(a.message_id, b.message_id)
|
|
1137
|
+
).map(rowToMessage);
|
|
1271
1138
|
} catch {
|
|
1272
1139
|
return [];
|
|
1273
1140
|
}
|
|
1274
1141
|
}
|
|
1275
1142
|
async searchMessages(vector, filter, limit = 10) {
|
|
1276
|
-
const
|
|
1277
|
-
|
|
1278
|
-
const rows = await q.toArray();
|
|
1279
|
-
return rows.map((r) => ({
|
|
1280
|
-
...rowToMessage(r),
|
|
1281
|
-
_distance: r._distance
|
|
1282
|
-
}));
|
|
1143
|
+
const rows = await this.provider.vectorSearch(MESSAGES_TABLE, vector, { filter, limit });
|
|
1144
|
+
return rows.map((r) => ({ ...rowToMessage(r), _distance: r._distance }));
|
|
1283
1145
|
}
|
|
1284
1146
|
// 对 messages 表执行混合搜索(BM25 + 向量),用于 topics 搜索无结果时的回退
|
|
1285
1147
|
async hybridSearchMessages(query, vector, filter, limit = 10) {
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1148
|
+
const fused = await this.hybridSearch(
|
|
1149
|
+
MESSAGES_TABLE,
|
|
1150
|
+
["content", "metadata"],
|
|
1151
|
+
query,
|
|
1152
|
+
vector,
|
|
1153
|
+
filter,
|
|
1154
|
+
limit
|
|
1155
|
+
);
|
|
1156
|
+
return fused.slice(0, limit).map(rowToMessage);
|
|
1295
1157
|
}
|
|
1296
1158
|
async hybridSearchTopics(query, vector, filter, limit = 10) {
|
|
1297
|
-
const recallBoostMs = this.config.recallBoostMs;
|
|
1298
1159
|
try {
|
|
1299
|
-
const
|
|
1300
|
-
if (
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
const total = rows.length;
|
|
1304
|
-
return rows.map((r, idx) => ({
|
|
1160
|
+
const fused = (await this.hybridSearch(TOPICS_TABLE, ["detail"], query, vector, filter, limit)).slice(0, limit);
|
|
1161
|
+
if (fused.length > 0) {
|
|
1162
|
+
const total = fused.length;
|
|
1163
|
+
return fused.map((r, idx) => ({
|
|
1305
1164
|
row: r,
|
|
1306
1165
|
score: total - idx + Math.log1p(Number(r.recall_count))
|
|
1307
1166
|
})).sort((a, b) => b.score - a.score).map((x) => rowToTopic(x.row));
|
|
1308
1167
|
}
|
|
1309
1168
|
} catch {
|
|
1310
1169
|
}
|
|
1311
|
-
const
|
|
1312
|
-
return
|
|
1170
|
+
const msgs = await this.hybridSearchMessages(query, vector, filter, limit);
|
|
1171
|
+
return msgs.map((r) => ({
|
|
1313
1172
|
summaryId: r.messageId,
|
|
1314
1173
|
sessionId: r.sessionId,
|
|
1315
1174
|
userId: r.userId,
|
|
1316
1175
|
chatId: r.chatId,
|
|
1176
|
+
title: "",
|
|
1317
1177
|
detail: r.content,
|
|
1318
1178
|
summary: r.content,
|
|
1319
1179
|
concise: r.content,
|
|
@@ -1325,83 +1185,154 @@ var LanceService = class {
|
|
|
1325
1185
|
vector: r.vector
|
|
1326
1186
|
}));
|
|
1327
1187
|
}
|
|
1188
|
+
/**
|
|
1189
|
+
* 通用混合检索:FTS(BM25)与向量两路各取 limit*HYBRID_OVERFETCH 候选,RRF 融合。
|
|
1190
|
+
* FTS 不可用(后端不支持或查询失败)时降级为纯向量。返回融合后的完整候选序列(未截断)。
|
|
1191
|
+
*/
|
|
1192
|
+
async hybridSearch(table, ftsColumns, query, vector, filter, limit) {
|
|
1193
|
+
const fetchN = limit * HYBRID_OVERFETCH;
|
|
1194
|
+
const supportsFts = this.provider.capabilities().fts;
|
|
1195
|
+
const [ftsRows, vecRows] = await Promise.all([
|
|
1196
|
+
supportsFts ? this.provider.ftsSearch(table, query, { columns: ftsColumns, filter, limit: fetchN }).catch(() => []) : Promise.resolve([]),
|
|
1197
|
+
this.provider.vectorSearch(table, vector, { filter, limit: fetchN }).catch(() => [])
|
|
1198
|
+
]);
|
|
1199
|
+
const keyColumn = this.keyColumnOf(table);
|
|
1200
|
+
return rrfFuse([ftsRows, vecRows], (r) => String(r[keyColumn]));
|
|
1201
|
+
}
|
|
1202
|
+
keyColumnOf(table) {
|
|
1203
|
+
switch (table) {
|
|
1204
|
+
case MESSAGES_TABLE:
|
|
1205
|
+
return "message_id";
|
|
1206
|
+
case TOPICS_TABLE:
|
|
1207
|
+
return "summary_id";
|
|
1208
|
+
case DOCUMENTS_TABLE:
|
|
1209
|
+
return "doc_id";
|
|
1210
|
+
case CHUNKS_TABLE:
|
|
1211
|
+
return "chunk_id";
|
|
1212
|
+
case SESSIONS_TABLE:
|
|
1213
|
+
return "session_id";
|
|
1214
|
+
case FACTS_TABLE:
|
|
1215
|
+
return "fact_id";
|
|
1216
|
+
default:
|
|
1217
|
+
throw new Error(`Unknown table: ${table}`);
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
// ── topics ─────────────────────────────────────────────────────────────────
|
|
1221
|
+
async addTopic(topic) {
|
|
1222
|
+
await this.provider.add(TOPICS_TABLE, [topicToRow(topic)]);
|
|
1223
|
+
}
|
|
1224
|
+
async updateTopicRecallCount(summaryId, count) {
|
|
1225
|
+
await this.provider.update(
|
|
1226
|
+
TOPICS_TABLE,
|
|
1227
|
+
{ recall_count: count, updated_at: Date.now() },
|
|
1228
|
+
[eq("summary_id", summaryId)]
|
|
1229
|
+
);
|
|
1230
|
+
}
|
|
1231
|
+
async getRecentTopics(chatId, userId, n1, n2, n3) {
|
|
1232
|
+
const filter = [eq("chat_id", chatId), eq("user_id", userId)];
|
|
1233
|
+
const recallBoostMs = this.config.recallBoostMs;
|
|
1234
|
+
const fetchTopics = async (count) => {
|
|
1235
|
+
if (count <= 0) return [];
|
|
1236
|
+
try {
|
|
1237
|
+
const rows = await this.provider.query(TOPICS_TABLE, {
|
|
1238
|
+
filter,
|
|
1239
|
+
limit: count * 5
|
|
1240
|
+
});
|
|
1241
|
+
return rows.sort((a, b) => {
|
|
1242
|
+
const scoreA = Number(a.end_time) + Number(a.recall_count) * recallBoostMs;
|
|
1243
|
+
const scoreB = Number(b.end_time) + Number(b.recall_count) * recallBoostMs;
|
|
1244
|
+
return scoreB - scoreA;
|
|
1245
|
+
}).slice(0, count).map(rowToTopic);
|
|
1246
|
+
} catch {
|
|
1247
|
+
return [];
|
|
1248
|
+
}
|
|
1249
|
+
};
|
|
1250
|
+
const allTopics = await fetchTopics(n1 + n2 + n3);
|
|
1251
|
+
return {
|
|
1252
|
+
detail: allTopics.slice(0, n1),
|
|
1253
|
+
summary: allTopics.slice(n1, n1 + n2),
|
|
1254
|
+
concise: allTopics.slice(n1 + n2, n1 + n2 + n3)
|
|
1255
|
+
};
|
|
1256
|
+
}
|
|
1257
|
+
/** 全量读取 topics(管理面板列表用,不做 recall 加权排序,按 endTime 倒序)*/
|
|
1258
|
+
async getAllTopics() {
|
|
1259
|
+
try {
|
|
1260
|
+
const rows = await this.provider.query(TOPICS_TABLE);
|
|
1261
|
+
return rows.sort(
|
|
1262
|
+
(a, b) => Number(b.end_time) - Number(a.end_time) || cmpStr(a.summary_id, b.summary_id)
|
|
1263
|
+
).map(rowToTopic);
|
|
1264
|
+
} catch {
|
|
1265
|
+
return [];
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
async deleteTopicsBySession(sessionId) {
|
|
1269
|
+
await this.provider.deleteWhere(TOPICS_TABLE, [eq("session_id", sessionId)]);
|
|
1270
|
+
}
|
|
1271
|
+
// ── facts ──────────────────────────────────────────────────────────────────
|
|
1328
1272
|
async saveFact(fact) {
|
|
1329
|
-
await this.
|
|
1273
|
+
await this.provider.add(FACTS_TABLE, [factToRow(fact)]);
|
|
1330
1274
|
}
|
|
1331
1275
|
async getAllFacts() {
|
|
1332
1276
|
try {
|
|
1333
|
-
const rows = await this.
|
|
1277
|
+
const rows = await this.provider.query(FACTS_TABLE);
|
|
1334
1278
|
return rows.map(rowToFact);
|
|
1335
1279
|
} catch {
|
|
1336
1280
|
return [];
|
|
1337
1281
|
}
|
|
1338
1282
|
}
|
|
1283
|
+
/** 删除单条 fact */
|
|
1284
|
+
async deleteFact(factId) {
|
|
1285
|
+
await this.provider.deleteWhere(FACTS_TABLE, [eq("fact_id", factId)]);
|
|
1286
|
+
}
|
|
1287
|
+
// ── sessions ───────────────────────────────────────────────────────────────
|
|
1339
1288
|
async getAllSessionIds() {
|
|
1340
1289
|
try {
|
|
1341
|
-
const rows = await this.
|
|
1290
|
+
const rows = await this.provider.query(MESSAGES_TABLE, {
|
|
1291
|
+
select: ["session_id"],
|
|
1292
|
+
limit: 1e4
|
|
1293
|
+
});
|
|
1342
1294
|
return [...new Set(rows.map((r) => r.session_id))];
|
|
1343
1295
|
} catch {
|
|
1344
1296
|
return [];
|
|
1345
1297
|
}
|
|
1346
1298
|
}
|
|
1347
1299
|
async insertSession(session) {
|
|
1348
|
-
await this.
|
|
1349
|
-
if (this.isNewSessionsTable) {
|
|
1350
|
-
await this.ensureIndexes(this.sessionsTable, SESSIONS_INDEXES);
|
|
1351
|
-
this.isNewSessionsTable = false;
|
|
1352
|
-
}
|
|
1300
|
+
await this.provider.add(SESSIONS_TABLE, [sessionToRow(session)]);
|
|
1353
1301
|
}
|
|
1354
1302
|
async upsertSession(session) {
|
|
1355
|
-
await this.
|
|
1356
|
-
await this.
|
|
1303
|
+
await this.provider.deleteWhere(SESSIONS_TABLE, [eq("session_id", session.sessionId)]);
|
|
1304
|
+
await this.provider.add(SESSIONS_TABLE, [sessionToRow(session)]);
|
|
1357
1305
|
}
|
|
1358
1306
|
async getAllSessions() {
|
|
1359
1307
|
try {
|
|
1360
|
-
const rows = await this.
|
|
1308
|
+
const rows = await this.provider.query(SESSIONS_TABLE);
|
|
1361
1309
|
return rows.map(rowToSession);
|
|
1362
1310
|
} catch {
|
|
1363
1311
|
return [];
|
|
1364
1312
|
}
|
|
1365
1313
|
}
|
|
1366
1314
|
async deleteSession(sessionId) {
|
|
1367
|
-
await this.
|
|
1368
|
-
}
|
|
1369
|
-
// ── 管理面板用:全量读取 / 删除 / 计数 ──────────────────────────────────────────────
|
|
1370
|
-
/** 全量读取 topics(管理面板列表用,不做 recall 加权排序,按 endTime 倒序)*/
|
|
1371
|
-
async getAllTopics() {
|
|
1372
|
-
try {
|
|
1373
|
-
const rows = await this.topicsTable.query().toArray();
|
|
1374
|
-
return rows.sort((a, b) => Number(b.end_time) - Number(a.end_time) || cmpStr(a.summary_id, b.summary_id)).map(rowToTopic);
|
|
1375
|
-
} catch {
|
|
1376
|
-
return [];
|
|
1377
|
-
}
|
|
1378
|
-
}
|
|
1379
|
-
/** 删除单条 fact */
|
|
1380
|
-
async deleteFact(factId) {
|
|
1381
|
-
await this.factsTable.delete(eqFilter("fact_id", factId));
|
|
1315
|
+
await this.provider.deleteWhere(SESSIONS_TABLE, [eq("session_id", sessionId)]);
|
|
1382
1316
|
}
|
|
1383
1317
|
/** 删除某会话下的全部消息(级联删除会话时使用)*/
|
|
1384
1318
|
async deleteMessagesBySession(sessionId) {
|
|
1385
|
-
await this.
|
|
1386
|
-
}
|
|
1387
|
-
/** 删除某会话下的全部 topics(级联删除会话时使用)*/
|
|
1388
|
-
async deleteTopicsBySession(sessionId) {
|
|
1389
|
-
await this.topicsTable.delete(eqFilter("session_id", sessionId));
|
|
1319
|
+
await this.provider.deleteWhere(MESSAGES_TABLE, [eq("session_id", sessionId)]);
|
|
1390
1320
|
}
|
|
1321
|
+
// ── 统计 ───────────────────────────────────────────────────────────────────
|
|
1391
1322
|
/** 各表行数统计(概览卡片用)*/
|
|
1392
1323
|
async countAll() {
|
|
1393
1324
|
const safeCount = async (table) => {
|
|
1394
1325
|
try {
|
|
1395
|
-
return await
|
|
1326
|
+
return await this.provider.count(table);
|
|
1396
1327
|
} catch {
|
|
1397
1328
|
return 0;
|
|
1398
1329
|
}
|
|
1399
1330
|
};
|
|
1400
1331
|
const [sessions, messages, topics, facts] = await Promise.all([
|
|
1401
|
-
safeCount(
|
|
1402
|
-
safeCount(
|
|
1403
|
-
safeCount(
|
|
1404
|
-
safeCount(
|
|
1332
|
+
safeCount(SESSIONS_TABLE),
|
|
1333
|
+
safeCount(MESSAGES_TABLE),
|
|
1334
|
+
safeCount(TOPICS_TABLE),
|
|
1335
|
+
safeCount(FACTS_TABLE)
|
|
1405
1336
|
]);
|
|
1406
1337
|
return { sessions, messages, topics, facts };
|
|
1407
1338
|
}
|
|
@@ -1422,16 +1353,20 @@ var LanceService = class {
|
|
|
1422
1353
|
};
|
|
1423
1354
|
const readCreatedAt = async (table) => {
|
|
1424
1355
|
try {
|
|
1425
|
-
const rows = await
|
|
1356
|
+
const rows = await this.provider.query(table, {
|
|
1357
|
+
select: ["created_at"],
|
|
1358
|
+
filter: [{ op: "gte", field: "created_at", value: startMs }],
|
|
1359
|
+
limit: 1e6
|
|
1360
|
+
});
|
|
1426
1361
|
return rows.map((r) => Number(r.created_at)).filter((n) => Number.isFinite(n));
|
|
1427
1362
|
} catch {
|
|
1428
1363
|
return [];
|
|
1429
1364
|
}
|
|
1430
1365
|
};
|
|
1431
1366
|
const [sessionTs, messageTs, factTs] = await Promise.all([
|
|
1432
|
-
readCreatedAt(
|
|
1433
|
-
readCreatedAt(
|
|
1434
|
-
readCreatedAt(
|
|
1367
|
+
readCreatedAt(SESSIONS_TABLE),
|
|
1368
|
+
readCreatedAt(MESSAGES_TABLE),
|
|
1369
|
+
readCreatedAt(FACTS_TABLE)
|
|
1435
1370
|
]);
|
|
1436
1371
|
const buckets = /* @__PURE__ */ new Map();
|
|
1437
1372
|
for (let i = 0; i < days; i++) {
|
|
@@ -1451,29 +1386,25 @@ var LanceService = class {
|
|
|
1451
1386
|
}
|
|
1452
1387
|
// ── 知识库:documents / chunks ─────────────────────────────────────────────
|
|
1453
1388
|
async addDocument(doc) {
|
|
1454
|
-
await this.
|
|
1455
|
-
if (this.isNewDocumentsTable) {
|
|
1456
|
-
await this.ensureIndexes(this.documentsTable, DOCUMENTS_INDEXES);
|
|
1457
|
-
this.isNewDocumentsTable = false;
|
|
1458
|
-
}
|
|
1389
|
+
await this.provider.add(DOCUMENTS_TABLE, [documentToRow(doc)]);
|
|
1459
1390
|
}
|
|
1460
1391
|
async addChunks(chunks) {
|
|
1461
1392
|
if (chunks.length === 0) return;
|
|
1462
|
-
await this.
|
|
1463
|
-
if (this.isNewChunksTable) {
|
|
1464
|
-
await this.ensureIndexes(this.chunksTable, CHUNKS_INDEXES);
|
|
1465
|
-
this.isNewChunksTable = false;
|
|
1466
|
-
}
|
|
1393
|
+
await this.provider.add(CHUNKS_TABLE, chunks.map(chunkToRow));
|
|
1467
1394
|
}
|
|
1468
1395
|
async updateDocumentGraphFlag(docId, hasGraph) {
|
|
1469
|
-
await this.
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1396
|
+
await this.provider.update(
|
|
1397
|
+
DOCUMENTS_TABLE,
|
|
1398
|
+
{ has_graph: hasGraph ? 1 : 0, updated_at: Date.now() },
|
|
1399
|
+
[eq("doc_id", docId)]
|
|
1400
|
+
);
|
|
1473
1401
|
}
|
|
1474
1402
|
async getDocument(docId) {
|
|
1475
1403
|
try {
|
|
1476
|
-
const rows = await this.
|
|
1404
|
+
const rows = await this.provider.query(DOCUMENTS_TABLE, {
|
|
1405
|
+
filter: [eq("doc_id", docId)],
|
|
1406
|
+
limit: 1
|
|
1407
|
+
});
|
|
1477
1408
|
if (rows.length === 0) return null;
|
|
1478
1409
|
return rowToDocument(rows[0]);
|
|
1479
1410
|
} catch {
|
|
@@ -1483,8 +1414,10 @@ var LanceService = class {
|
|
|
1483
1414
|
/** 查找同域同 hash 的文档(去重用)*/
|
|
1484
1415
|
async findDocumentByHash(contentHash, filter) {
|
|
1485
1416
|
try {
|
|
1486
|
-
const
|
|
1487
|
-
|
|
1417
|
+
const rows = await this.provider.query(DOCUMENTS_TABLE, {
|
|
1418
|
+
filter: [eq("content_hash", contentHash), ...filter ?? []],
|
|
1419
|
+
limit: 1
|
|
1420
|
+
});
|
|
1488
1421
|
if (rows.length === 0) return null;
|
|
1489
1422
|
return rowToDocument(rows[0]);
|
|
1490
1423
|
} catch {
|
|
@@ -1494,9 +1427,11 @@ var LanceService = class {
|
|
|
1494
1427
|
/** 按域过滤返回 docId 列表(chunkRedundantIds=false 时用于 chunk 过滤)*/
|
|
1495
1428
|
async getDocIdsByDomain(filter) {
|
|
1496
1429
|
try {
|
|
1497
|
-
const
|
|
1498
|
-
|
|
1499
|
-
|
|
1430
|
+
const rows = await this.provider.query(DOCUMENTS_TABLE, {
|
|
1431
|
+
select: ["doc_id"],
|
|
1432
|
+
filter,
|
|
1433
|
+
limit: 1e5
|
|
1434
|
+
});
|
|
1500
1435
|
return [...new Set(rows.map((r) => r.doc_id))];
|
|
1501
1436
|
} catch {
|
|
1502
1437
|
return [];
|
|
@@ -1505,13 +1440,8 @@ var LanceService = class {
|
|
|
1505
1440
|
/** 文档级粗召回:对摘要向量做向量搜索,定位候选文档 */
|
|
1506
1441
|
async searchDocuments(vector, filter, limit = 5) {
|
|
1507
1442
|
try {
|
|
1508
|
-
const
|
|
1509
|
-
|
|
1510
|
-
const rows = await q.toArray();
|
|
1511
|
-
return rows.map((r) => ({
|
|
1512
|
-
...rowToDocument(r),
|
|
1513
|
-
_distance: r._distance
|
|
1514
|
-
}));
|
|
1443
|
+
const rows = await this.provider.vectorSearch(DOCUMENTS_TABLE, vector, { filter, limit });
|
|
1444
|
+
return rows.map((r) => ({ ...rowToDocument(r), _distance: r._distance }));
|
|
1515
1445
|
} catch {
|
|
1516
1446
|
return [];
|
|
1517
1447
|
}
|
|
@@ -1519,32 +1449,30 @@ var LanceService = class {
|
|
|
1519
1449
|
/** 知识片段混合检索(BM25 + 向量),失败回退纯向量 */
|
|
1520
1450
|
async searchChunks(query, vector, filter, limit = 8) {
|
|
1521
1451
|
try {
|
|
1522
|
-
const
|
|
1523
|
-
if (
|
|
1524
|
-
|
|
1452
|
+
const fused = await this.hybridSearch(CHUNKS_TABLE, ["content"], query, vector, filter, limit);
|
|
1453
|
+
if (fused.length > 0) return fused.slice(0, limit).map(rowToChunk);
|
|
1454
|
+
} catch {
|
|
1455
|
+
}
|
|
1456
|
+
try {
|
|
1457
|
+
const rows = await this.provider.vectorSearch(CHUNKS_TABLE, vector, { filter, limit });
|
|
1525
1458
|
return rows.map(rowToChunk);
|
|
1526
1459
|
} catch {
|
|
1527
|
-
|
|
1528
|
-
const q = this.chunksTable.vectorSearch(vector).limit(limit);
|
|
1529
|
-
if (filter) q.where(filter);
|
|
1530
|
-
const rows = await q.toArray();
|
|
1531
|
-
return rows.map(rowToChunk);
|
|
1532
|
-
} catch {
|
|
1533
|
-
return [];
|
|
1534
|
-
}
|
|
1460
|
+
return [];
|
|
1535
1461
|
}
|
|
1536
1462
|
}
|
|
1537
1463
|
async deleteDocument(docId) {
|
|
1538
|
-
await this.
|
|
1464
|
+
await this.provider.deleteWhere(DOCUMENTS_TABLE, [eq("doc_id", docId)]);
|
|
1539
1465
|
}
|
|
1540
1466
|
async deleteChunksByDoc(docId) {
|
|
1541
|
-
await this.
|
|
1467
|
+
await this.provider.deleteWhere(CHUNKS_TABLE, [eq("doc_id", docId)]);
|
|
1542
1468
|
}
|
|
1543
1469
|
/** 全量读取文档(管理面板用),按更新时间倒序 */
|
|
1544
1470
|
async getAllDocuments() {
|
|
1545
1471
|
try {
|
|
1546
|
-
const rows = await this.
|
|
1547
|
-
return rows.sort(
|
|
1472
|
+
const rows = await this.provider.query(DOCUMENTS_TABLE);
|
|
1473
|
+
return rows.sort(
|
|
1474
|
+
(a, b) => Number(b.updated_at) - Number(a.updated_at) || cmpStr(a.doc_id, b.doc_id)
|
|
1475
|
+
).map(rowToDocument);
|
|
1548
1476
|
} catch {
|
|
1549
1477
|
return [];
|
|
1550
1478
|
}
|
|
@@ -1553,34 +1481,33 @@ var LanceService = class {
|
|
|
1553
1481
|
async countKnowledge() {
|
|
1554
1482
|
const safeCount = async (table) => {
|
|
1555
1483
|
try {
|
|
1556
|
-
return await
|
|
1484
|
+
return await this.provider.count(table);
|
|
1557
1485
|
} catch {
|
|
1558
1486
|
return 0;
|
|
1559
1487
|
}
|
|
1560
1488
|
};
|
|
1561
1489
|
const [documents, chunks] = await Promise.all([
|
|
1562
|
-
safeCount(
|
|
1563
|
-
safeCount(
|
|
1490
|
+
safeCount(DOCUMENTS_TABLE),
|
|
1491
|
+
safeCount(CHUNKS_TABLE)
|
|
1564
1492
|
]);
|
|
1565
1493
|
return { documents, chunks };
|
|
1566
1494
|
}
|
|
1567
1495
|
/**
|
|
1568
|
-
* 根据 metadata
|
|
1496
|
+
* 根据 metadata 字段内容构建结构化过滤条件。
|
|
1569
1497
|
*
|
|
1570
|
-
* metadata 以 JSON 字符串存储,此方法将键值对转换为
|
|
1498
|
+
* metadata 以 JSON 字符串存储,此方法将键值对转换为 jsonContains 条件,
|
|
1571
1499
|
* 可直接传给 searchMessages / hybridSearchTopics 的 filter 参数。
|
|
1572
1500
|
*
|
|
1573
|
-
* 示例:buildMetadataFilter({ env: "prod", version: 2 })
|
|
1574
|
-
* → `metadata LIKE '%"env":"prod"%' AND metadata LIKE '%"version":2%'`
|
|
1575
|
-
*
|
|
1576
1501
|
* 注意:仅适合简单标量值(字符串、数字、布尔)的精确匹配。
|
|
1577
1502
|
* 复杂嵌套对象或含空格的 JSON 值可能无法可靠匹配。
|
|
1578
1503
|
*/
|
|
1579
1504
|
static buildMetadataFilter(conditions) {
|
|
1580
|
-
return Object.entries(conditions).map(([key, value]) => {
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1505
|
+
return Object.entries(conditions).map(([key, value]) => ({
|
|
1506
|
+
op: "jsonContains",
|
|
1507
|
+
field: "metadata",
|
|
1508
|
+
key,
|
|
1509
|
+
value
|
|
1510
|
+
}));
|
|
1584
1511
|
}
|
|
1585
1512
|
};
|
|
1586
1513
|
|
|
@@ -1890,16 +1817,16 @@ ${context}`, false);
|
|
|
1890
1817
|
import { v4 as uuidv4 } from "uuid";
|
|
1891
1818
|
var CompressManager = class {
|
|
1892
1819
|
config;
|
|
1893
|
-
|
|
1820
|
+
store;
|
|
1894
1821
|
grafeo;
|
|
1895
1822
|
llm;
|
|
1896
1823
|
embed;
|
|
1897
1824
|
sessionCache;
|
|
1898
1825
|
semaphore;
|
|
1899
1826
|
sessionChain = /* @__PURE__ */ new Map();
|
|
1900
|
-
constructor(config,
|
|
1827
|
+
constructor(config, store, grafeo, llm, embed, sessionCache) {
|
|
1901
1828
|
this.config = config;
|
|
1902
|
-
this.
|
|
1829
|
+
this.store = store;
|
|
1903
1830
|
this.grafeo = grafeo;
|
|
1904
1831
|
this.llm = llm;
|
|
1905
1832
|
this.embed = embed;
|
|
@@ -1956,14 +1883,14 @@ var CompressManager = class {
|
|
|
1956
1883
|
vector: topicVector
|
|
1957
1884
|
};
|
|
1958
1885
|
try {
|
|
1959
|
-
await this.
|
|
1886
|
+
await this.store.addTopic(topic);
|
|
1960
1887
|
} catch (err) {
|
|
1961
1888
|
console.error(`[CompressManager] Save topic failed:`, err);
|
|
1962
1889
|
}
|
|
1963
1890
|
this.sessionCache.clearMessages(sessionId, endTime);
|
|
1964
1891
|
const [n1, n2, n3] = this.config.topicRatio;
|
|
1965
1892
|
try {
|
|
1966
|
-
const topicGroups = await this.
|
|
1893
|
+
const topicGroups = await this.store.getRecentTopics(chatId, userId, n1, n2, n3);
|
|
1967
1894
|
this.sessionCache.buildHistoryWindow(sessionId, topicGroups);
|
|
1968
1895
|
} catch (err) {
|
|
1969
1896
|
console.error(`[CompressManager] Rebuild history window failed:`, err);
|
|
@@ -2037,13 +1964,13 @@ function withTimeout(promise, timeoutMs, label) {
|
|
|
2037
1964
|
// src/manager/fact.cache.ts
|
|
2038
1965
|
import { v4 as uuidv42 } from "uuid";
|
|
2039
1966
|
var FactCache = class {
|
|
2040
|
-
|
|
1967
|
+
store;
|
|
2041
1968
|
cache = /* @__PURE__ */ new Map();
|
|
2042
|
-
constructor(
|
|
2043
|
-
this.
|
|
1969
|
+
constructor(store) {
|
|
1970
|
+
this.store = store;
|
|
2044
1971
|
}
|
|
2045
1972
|
async init() {
|
|
2046
|
-
const facts = await this.
|
|
1973
|
+
const facts = await this.store.getAllFacts();
|
|
2047
1974
|
for (const fact of facts) {
|
|
2048
1975
|
const key = this.key(fact.level, fact.level === "user" ? fact.userId : fact.chatId);
|
|
2049
1976
|
const existing = this.cache.get(key) ?? [];
|
|
@@ -2064,7 +1991,7 @@ var FactCache = class {
|
|
|
2064
1991
|
content,
|
|
2065
1992
|
createdAt: Date.now()
|
|
2066
1993
|
};
|
|
2067
|
-
await this.
|
|
1994
|
+
await this.store.saveFact(fact);
|
|
2068
1995
|
const id = level === "user" ? userId : chatId;
|
|
2069
1996
|
const k = this.key(level, id);
|
|
2070
1997
|
const existing = this.cache.get(k) ?? [];
|
|
@@ -2078,7 +2005,7 @@ var FactCache = class {
|
|
|
2078
2005
|
all() {
|
|
2079
2006
|
return [...this.cache.values()].flat();
|
|
2080
2007
|
}
|
|
2081
|
-
/** 删除单条 fact
|
|
2008
|
+
/** 删除单条 fact:同步从存储与内存缓存中移除。返回是否命中。*/
|
|
2082
2009
|
async remove(factId) {
|
|
2083
2010
|
let hit = false;
|
|
2084
2011
|
for (const [k, facts] of this.cache) {
|
|
@@ -2091,7 +2018,7 @@ var FactCache = class {
|
|
|
2091
2018
|
break;
|
|
2092
2019
|
}
|
|
2093
2020
|
}
|
|
2094
|
-
await this.
|
|
2021
|
+
await this.store.deleteFact(factId);
|
|
2095
2022
|
return hit;
|
|
2096
2023
|
}
|
|
2097
2024
|
toString(level, id) {
|
|
@@ -2235,14 +2162,14 @@ function takeTailTokens(text, overlapTokens) {
|
|
|
2235
2162
|
// src/manager/knowledge.manager.ts
|
|
2236
2163
|
var KnowledgeManager = class {
|
|
2237
2164
|
config;
|
|
2238
|
-
|
|
2165
|
+
store;
|
|
2239
2166
|
grafeo;
|
|
2240
2167
|
llm;
|
|
2241
2168
|
embed;
|
|
2242
2169
|
semaphore;
|
|
2243
|
-
constructor(config,
|
|
2170
|
+
constructor(config, store, grafeo, llm, embed) {
|
|
2244
2171
|
this.config = config;
|
|
2245
|
-
this.
|
|
2172
|
+
this.store = store;
|
|
2246
2173
|
this.grafeo = grafeo;
|
|
2247
2174
|
this.llm = llm;
|
|
2248
2175
|
this.embed = embed;
|
|
@@ -2257,8 +2184,12 @@ var KnowledgeManager = class {
|
|
|
2257
2184
|
const now = Date.now();
|
|
2258
2185
|
const content = opts.content;
|
|
2259
2186
|
const contentHash = createHash("sha256").update(content).digest("hex");
|
|
2260
|
-
const domainFilter =
|
|
2261
|
-
|
|
2187
|
+
const domainFilter = [
|
|
2188
|
+
eq("user_id", userId),
|
|
2189
|
+
eq("chat_id", chatId),
|
|
2190
|
+
eq("session_id", sessionId)
|
|
2191
|
+
];
|
|
2192
|
+
const existing = await this.store.findDocumentByHash(contentHash, domainFilter);
|
|
2262
2193
|
if (existing) return { docId: existing.docId };
|
|
2263
2194
|
const pieces = chunkMarkdown(content, {
|
|
2264
2195
|
maxTokens: this.config.chunkMaxTokens,
|
|
@@ -2291,7 +2222,7 @@ var KnowledgeManager = class {
|
|
|
2291
2222
|
createdAt: now,
|
|
2292
2223
|
updatedAt: now
|
|
2293
2224
|
};
|
|
2294
|
-
await this.
|
|
2225
|
+
await this.store.addDocument(doc);
|
|
2295
2226
|
const chunkIngest = this.semaphore.run(() => this.ingestChunks(doc, pieces));
|
|
2296
2227
|
const graphBuild = chunkIngest.then((chunks) => {
|
|
2297
2228
|
if (!shouldBuildGraph) return;
|
|
@@ -2341,7 +2272,7 @@ ${p.content}` : p.content
|
|
|
2341
2272
|
metadata: {},
|
|
2342
2273
|
createdAt: doc.createdAt
|
|
2343
2274
|
}));
|
|
2344
|
-
await this.
|
|
2275
|
+
await this.store.addChunks(chunks);
|
|
2345
2276
|
return chunks;
|
|
2346
2277
|
}
|
|
2347
2278
|
async buildDocumentGraph(doc, chunks) {
|
|
@@ -2394,7 +2325,7 @@ ${p.content}` : p.content
|
|
|
2394
2325
|
embeddings,
|
|
2395
2326
|
KIND_KNOWLEDGE
|
|
2396
2327
|
);
|
|
2397
|
-
await this.
|
|
2328
|
+
await this.store.updateDocumentGraphFlag(doc.docId, true).catch((err) => {
|
|
2398
2329
|
console.error(`[KnowledgeManager] Failed to update graph flag for doc ${doc.docId}:`, err);
|
|
2399
2330
|
});
|
|
2400
2331
|
}
|
|
@@ -2407,11 +2338,11 @@ ${p.content}` : p.content
|
|
|
2407
2338
|
const mode = opts.mode ?? "auto";
|
|
2408
2339
|
const limit = opts.limit ?? this.config.knowledgeTopK;
|
|
2409
2340
|
const vector = await this.embed.embedOne(opts.query);
|
|
2410
|
-
const docFilter =
|
|
2341
|
+
const docFilter = buildDomainFilter(scope, scopeId);
|
|
2411
2342
|
let candidateDocIds;
|
|
2412
2343
|
const docTitleMap = /* @__PURE__ */ new Map();
|
|
2413
2344
|
if (this.config.docCoarseTopK > 0) {
|
|
2414
|
-
const coarse = await this.
|
|
2345
|
+
const coarse = await this.store.searchDocuments(vector, docFilter, this.config.docCoarseTopK);
|
|
2415
2346
|
if (coarse.length > 0) {
|
|
2416
2347
|
candidateDocIds = coarse.map((d) => d.docId);
|
|
2417
2348
|
for (const d of coarse) docTitleMap.set(d.docId, d.title);
|
|
@@ -2419,7 +2350,7 @@ ${p.content}` : p.content
|
|
|
2419
2350
|
}
|
|
2420
2351
|
const chunkFilter = await this.buildChunkFilter(scope, scopeId, candidateDocIds);
|
|
2421
2352
|
if (chunkFilter === NO_MATCH) return empty;
|
|
2422
|
-
const rawChunks = await this.
|
|
2353
|
+
const rawChunks = await this.store.searchChunks(opts.query, vector, chunkFilter, limit);
|
|
2423
2354
|
const scored = rawChunks.map((c) => ({
|
|
2424
2355
|
chunkId: c.chunkId,
|
|
2425
2356
|
docId: c.docId,
|
|
@@ -2433,7 +2364,7 @@ ${p.content}` : p.content
|
|
|
2433
2364
|
[...docCount.entries()].map(async ([docId, matchedChunkCount]) => {
|
|
2434
2365
|
let title = docTitleMap.get(docId);
|
|
2435
2366
|
if (title === void 0) {
|
|
2436
|
-
const d = await this.
|
|
2367
|
+
const d = await this.store.getDocument(docId);
|
|
2437
2368
|
title = d?.title ?? "";
|
|
2438
2369
|
}
|
|
2439
2370
|
return { docId, title, matchedChunkCount };
|
|
@@ -2481,32 +2412,32 @@ ${p.content}` : p.content
|
|
|
2481
2412
|
async buildChunkFilter(scope, scopeId, candidateDocIds) {
|
|
2482
2413
|
const parts = [];
|
|
2483
2414
|
if (this.config.chunkRedundantIds) {
|
|
2484
|
-
const
|
|
2485
|
-
if (
|
|
2486
|
-
if (candidateDocIds) parts.push(
|
|
2415
|
+
const domain = buildDomainFilter(scope, scopeId);
|
|
2416
|
+
if (domain) parts.push(...domain);
|
|
2417
|
+
if (candidateDocIds) parts.push({ op: "in", field: "doc_id", values: candidateDocIds });
|
|
2487
2418
|
} else {
|
|
2488
2419
|
let docIds = candidateDocIds;
|
|
2489
2420
|
if (scope !== "all") {
|
|
2490
|
-
const
|
|
2491
|
-
const domainDocIds = await this.
|
|
2421
|
+
const domain = buildDomainFilter(scope, scopeId);
|
|
2422
|
+
const domainDocIds = await this.store.getDocIdsByDomain(domain);
|
|
2492
2423
|
docIds = docIds ? domainDocIds.filter((id) => docIds.includes(id)) : domainDocIds;
|
|
2493
2424
|
}
|
|
2494
2425
|
if (docIds) {
|
|
2495
2426
|
if (docIds.length === 0) return NO_MATCH;
|
|
2496
|
-
parts.push(
|
|
2427
|
+
parts.push({ op: "in", field: "doc_id", values: docIds });
|
|
2497
2428
|
}
|
|
2498
2429
|
}
|
|
2499
|
-
return parts.length > 0 ? parts
|
|
2430
|
+
return parts.length > 0 ? parts : void 0;
|
|
2500
2431
|
}
|
|
2501
2432
|
// ── 读取 / 删除 ─────────────────────────────────────────────────────────────────
|
|
2502
2433
|
async getDocument(docId) {
|
|
2503
|
-
return this.
|
|
2434
|
+
return this.store.getDocument(docId);
|
|
2504
2435
|
}
|
|
2505
2436
|
async deleteDocument(docId) {
|
|
2506
|
-
const existing = await this.
|
|
2437
|
+
const existing = await this.store.getDocument(docId);
|
|
2507
2438
|
if (!existing) return false;
|
|
2508
|
-
await this.
|
|
2509
|
-
await this.
|
|
2439
|
+
await this.store.deleteDocument(docId);
|
|
2440
|
+
await this.store.deleteChunksByDoc(docId).catch((err) => {
|
|
2510
2441
|
console.error(`[KnowledgeManager] Failed to delete chunks for doc ${docId}:`, err);
|
|
2511
2442
|
});
|
|
2512
2443
|
await this.grafeo.deleteKnowledgeByDoc(docId).catch((err) => {
|
|
@@ -2515,7 +2446,7 @@ ${p.content}` : p.content
|
|
|
2515
2446
|
return true;
|
|
2516
2447
|
}
|
|
2517
2448
|
async listDocuments(filter, page) {
|
|
2518
|
-
let docs = await this.
|
|
2449
|
+
let docs = await this.store.getAllDocuments();
|
|
2519
2450
|
if (filter?.userId) docs = docs.filter((d) => d.userId === filter.userId);
|
|
2520
2451
|
if (filter?.chatId) docs = docs.filter((d) => d.chatId === filter.chatId);
|
|
2521
2452
|
if (filter?.sessionId) docs = docs.filter((d) => d.sessionId === filter.sessionId);
|
|
@@ -2533,19 +2464,12 @@ function dedupByContent(items) {
|
|
|
2533
2464
|
}
|
|
2534
2465
|
return out;
|
|
2535
2466
|
}
|
|
2536
|
-
function
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
if (scope === "session" && scopeId) return `session_id = '${esc(scopeId)}'`;
|
|
2541
|
-
if (scope === "chat" && scopeId) return `chat_id = '${esc(scopeId)}'`;
|
|
2542
|
-
if (scope === "user" && scopeId) return `user_id = '${esc(scopeId)}'`;
|
|
2467
|
+
function buildDomainFilter(scope, scopeId) {
|
|
2468
|
+
if (scope === "session" && scopeId) return [eq("session_id", scopeId)];
|
|
2469
|
+
if (scope === "chat" && scopeId) return [eq("chat_id", scopeId)];
|
|
2470
|
+
if (scope === "user" && scopeId) return [eq("user_id", scopeId)];
|
|
2543
2471
|
return void 0;
|
|
2544
2472
|
}
|
|
2545
|
-
function inSql(column, values) {
|
|
2546
|
-
const list = values.map((v) => `'${esc(v)}'`).join(", ");
|
|
2547
|
-
return `${column} IN (${list})`;
|
|
2548
|
-
}
|
|
2549
2473
|
function inferTitle(markdown) {
|
|
2550
2474
|
const lines = markdown.split(/\r?\n/);
|
|
2551
2475
|
for (const line of lines) {
|
|
@@ -2654,7 +2578,7 @@ var SessionCache = class {
|
|
|
2654
2578
|
// src/memory.manager.ts
|
|
2655
2579
|
var MemoryManager = class {
|
|
2656
2580
|
config;
|
|
2657
|
-
|
|
2581
|
+
store;
|
|
2658
2582
|
grafeo;
|
|
2659
2583
|
embed;
|
|
2660
2584
|
llm;
|
|
@@ -2667,15 +2591,15 @@ var MemoryManager = class {
|
|
|
2667
2591
|
optimizeRunning = false;
|
|
2668
2592
|
constructor(config) {
|
|
2669
2593
|
this.config = resolveConfig(config);
|
|
2670
|
-
this.
|
|
2594
|
+
this.store = new MemoryStore(this.config);
|
|
2671
2595
|
this.grafeo = new GrafeoService(this.config);
|
|
2672
2596
|
this.embed = new EmbedService(this.config);
|
|
2673
2597
|
this.llm = new LlmService(this.config);
|
|
2674
2598
|
this.sessionCache = new SessionCache(this.config);
|
|
2675
|
-
this.factCache = new FactCache(this.
|
|
2599
|
+
this.factCache = new FactCache(this.store);
|
|
2676
2600
|
this.compressManager = new CompressManager(
|
|
2677
2601
|
this.config,
|
|
2678
|
-
this.
|
|
2602
|
+
this.store,
|
|
2679
2603
|
this.grafeo,
|
|
2680
2604
|
this.llm,
|
|
2681
2605
|
this.embed,
|
|
@@ -2683,7 +2607,7 @@ var MemoryManager = class {
|
|
|
2683
2607
|
);
|
|
2684
2608
|
this.knowledgeManager = new KnowledgeManager(
|
|
2685
2609
|
this.config,
|
|
2686
|
-
this.
|
|
2610
|
+
this.store,
|
|
2687
2611
|
this.grafeo,
|
|
2688
2612
|
this.llm,
|
|
2689
2613
|
this.embed
|
|
@@ -2691,10 +2615,10 @@ var MemoryManager = class {
|
|
|
2691
2615
|
}
|
|
2692
2616
|
async init() {
|
|
2693
2617
|
initEncoder();
|
|
2694
|
-
await this.
|
|
2618
|
+
await this.store.init();
|
|
2695
2619
|
await this.grafeo.init();
|
|
2696
2620
|
await this.factCache.init();
|
|
2697
|
-
const allSessions = await this.
|
|
2621
|
+
const allSessions = await this.store.getAllSessions();
|
|
2698
2622
|
for (const s of allSessions) {
|
|
2699
2623
|
this.sessionMap.set(s.sessionId, this.deserializeSession(s));
|
|
2700
2624
|
}
|
|
@@ -2734,18 +2658,18 @@ var MemoryManager = class {
|
|
|
2734
2658
|
* @param retentionMs 保留多久内的历史版本,默认取配置 optimizeVersionRetentionMs
|
|
2735
2659
|
*/
|
|
2736
2660
|
async optimizeStorage(retentionMs) {
|
|
2737
|
-
return this.
|
|
2661
|
+
return this.store.optimizeStorage(retentionMs ?? this.config.optimizeVersionRetentionMs);
|
|
2738
2662
|
}
|
|
2739
2663
|
async restoreFromStorage() {
|
|
2740
2664
|
const [n1, n2, n3] = this.config.topicRatio;
|
|
2741
|
-
const sessionIds = await this.
|
|
2665
|
+
const sessionIds = await this.store.getAllSessionIds();
|
|
2742
2666
|
if (sessionIds.length === 0) return;
|
|
2743
2667
|
const topicsCache = /* @__PURE__ */ new Map();
|
|
2744
2668
|
const getTopics = (chatId, userId) => {
|
|
2745
2669
|
const key = `${chatId}\0${userId}`;
|
|
2746
2670
|
let cached = topicsCache.get(key);
|
|
2747
2671
|
if (!cached) {
|
|
2748
|
-
cached = this.
|
|
2672
|
+
cached = this.store.getRecentTopics(chatId, userId, n1, n2, n3);
|
|
2749
2673
|
topicsCache.set(key, cached);
|
|
2750
2674
|
}
|
|
2751
2675
|
return cached;
|
|
@@ -2754,12 +2678,12 @@ var MemoryManager = class {
|
|
|
2754
2678
|
await Promise.all(
|
|
2755
2679
|
sessionIds.map(
|
|
2756
2680
|
(sessionId) => sem.run(async () => {
|
|
2757
|
-
const recentMessages = await this.
|
|
2681
|
+
const recentMessages = await this.store.getLatestMessages(sessionId, 1);
|
|
2758
2682
|
if (recentMessages.length === 0) return;
|
|
2759
2683
|
const { chatId, userId } = recentMessages[0];
|
|
2760
2684
|
const topicGroups = await getTopics(chatId, userId);
|
|
2761
2685
|
const n1EndTime = topicGroups.detail.length > 0 ? topicGroups.detail[topicGroups.detail.length - 1].endTime : 0;
|
|
2762
|
-
const rawMessages = n1EndTime > 0 ? await this.
|
|
2686
|
+
const rawMessages = n1EndTime > 0 ? await this.store.getMessagesSince(sessionId, n1EndTime) : await this.store.getLatestMessages(sessionId, 100);
|
|
2763
2687
|
if (rawMessages.length > 0) {
|
|
2764
2688
|
this.sessionCache.addMessages(sessionId, rawMessages, chatId, userId);
|
|
2765
2689
|
}
|
|
@@ -2812,7 +2736,7 @@ var MemoryManager = class {
|
|
|
2812
2736
|
vector: vectors[i],
|
|
2813
2737
|
createdAt: m.createdAt
|
|
2814
2738
|
}));
|
|
2815
|
-
await this.
|
|
2739
|
+
await this.store.addMessages(stored);
|
|
2816
2740
|
if (!this.sessionMap.has(sessionId)) {
|
|
2817
2741
|
const session = {
|
|
2818
2742
|
sessionId,
|
|
@@ -2824,13 +2748,13 @@ var MemoryManager = class {
|
|
|
2824
2748
|
updatedAt: now
|
|
2825
2749
|
};
|
|
2826
2750
|
this.sessionMap.set(sessionId, this.deserializeSession(session));
|
|
2827
|
-
this.
|
|
2751
|
+
this.store.insertSession(session).catch((err) => {
|
|
2828
2752
|
console.error("[MemoryManager] Failed to insert session:", err);
|
|
2829
2753
|
});
|
|
2830
2754
|
} else {
|
|
2831
2755
|
const view = this.sessionMap.get(sessionId);
|
|
2832
2756
|
view.updatedAt = now;
|
|
2833
|
-
this.
|
|
2757
|
+
this.store.upsertSession(this.serializeSession(view)).catch((err) => {
|
|
2834
2758
|
console.error(`[MemoryManager] Failed to upsert session ${sessionId}:`, err);
|
|
2835
2759
|
});
|
|
2836
2760
|
}
|
|
@@ -2885,7 +2809,7 @@ var MemoryManager = class {
|
|
|
2885
2809
|
} = opts;
|
|
2886
2810
|
if (!query.trim()) return [];
|
|
2887
2811
|
const vector = await this.embed.embedOne(query);
|
|
2888
|
-
const filter = this.
|
|
2812
|
+
const filter = this.buildScopeFilter(scope, scopeId);
|
|
2889
2813
|
let useGraph = false;
|
|
2890
2814
|
if (mode === "all") {
|
|
2891
2815
|
useGraph = true;
|
|
@@ -2893,7 +2817,7 @@ var MemoryManager = class {
|
|
|
2893
2817
|
useGraph = await this.llm.judgeNeedsGraphSearch(query).catch(() => false);
|
|
2894
2818
|
}
|
|
2895
2819
|
const tasks = [
|
|
2896
|
-
this.
|
|
2820
|
+
this.store.hybridSearchTopics(query, vector, filter, limit).then(
|
|
2897
2821
|
(topics) => topics.map((t) => ({
|
|
2898
2822
|
type: "topic",
|
|
2899
2823
|
content: t.detail,
|
|
@@ -2966,10 +2890,10 @@ var MemoryManager = class {
|
|
|
2966
2890
|
getHistoryWindow(sessionId) {
|
|
2967
2891
|
return this.sessionCache.getHistoryWindow(sessionId);
|
|
2968
2892
|
}
|
|
2969
|
-
|
|
2970
|
-
if (scope === "session" && scopeId) return
|
|
2971
|
-
if (scope === "chat" && scopeId) return
|
|
2972
|
-
if (scope === "user" && scopeId) return
|
|
2893
|
+
buildScopeFilter(scope, scopeId) {
|
|
2894
|
+
if (scope === "session" && scopeId) return [eq("session_id", scopeId)];
|
|
2895
|
+
if (scope === "chat" && scopeId) return [eq("chat_id", scopeId)];
|
|
2896
|
+
if (scope === "user" && scopeId) return [eq("user_id", scopeId)];
|
|
2973
2897
|
return void 0;
|
|
2974
2898
|
}
|
|
2975
2899
|
deserializeSession(s) {
|
|
@@ -2980,7 +2904,7 @@ var MemoryManager = class {
|
|
|
2980
2904
|
}
|
|
2981
2905
|
// ── Message queries ──────────────────────────────────────────────────────────
|
|
2982
2906
|
async getRecentMessages(sessionId, limit) {
|
|
2983
|
-
return this.
|
|
2907
|
+
return this.store.getLatestMessages(sessionId, limit);
|
|
2984
2908
|
}
|
|
2985
2909
|
// ── Session queries (in-memory) ──────────────────────────────────────────────
|
|
2986
2910
|
getSession(sessionId) {
|
|
@@ -3016,17 +2940,17 @@ var MemoryManager = class {
|
|
|
3016
2940
|
updatedAt: Date.now()
|
|
3017
2941
|
};
|
|
3018
2942
|
this.sessionMap.set(sessionId, updated);
|
|
3019
|
-
await this.
|
|
2943
|
+
await this.store.upsertSession(this.serializeSession(updated));
|
|
3020
2944
|
return updated;
|
|
3021
2945
|
}
|
|
3022
2946
|
async deleteSession(sessionId) {
|
|
3023
2947
|
if (!this.sessionMap.has(sessionId)) return false;
|
|
3024
2948
|
this.sessionMap.delete(sessionId);
|
|
3025
|
-
await this.
|
|
3026
|
-
await this.
|
|
2949
|
+
await this.store.deleteSession(sessionId);
|
|
2950
|
+
await this.store.deleteMessagesBySession(sessionId).catch((err) => {
|
|
3027
2951
|
console.error(`[MemoryManager] Failed to delete messages for session ${sessionId}:`, err);
|
|
3028
2952
|
});
|
|
3029
|
-
await this.
|
|
2953
|
+
await this.store.deleteTopicsBySession(sessionId).catch((err) => {
|
|
3030
2954
|
console.error(`[MemoryManager] Failed to delete topics for session ${sessionId}:`, err);
|
|
3031
2955
|
});
|
|
3032
2956
|
return true;
|
|
@@ -3035,8 +2959,8 @@ var MemoryManager = class {
|
|
|
3035
2959
|
/** 概览统计:各类记忆数据的总量 */
|
|
3036
2960
|
async stats() {
|
|
3037
2961
|
const [counts, knowledge, entities, relations] = await Promise.all([
|
|
3038
|
-
this.
|
|
3039
|
-
this.
|
|
2962
|
+
this.store.countAll(),
|
|
2963
|
+
this.store.countKnowledge(),
|
|
3040
2964
|
this.grafeo.getAllEntities(),
|
|
3041
2965
|
this.grafeo.getAllRelations()
|
|
3042
2966
|
]);
|
|
@@ -3050,7 +2974,7 @@ var MemoryManager = class {
|
|
|
3050
2974
|
/** 按天聚合最近 days 天的活跃趋势(概览图表用,含零值天)。days 默认 30,范围 1~365。 */
|
|
3051
2975
|
async trend(days = 30) {
|
|
3052
2976
|
const n = Math.min(365, Math.max(1, Math.floor(days)));
|
|
3053
|
-
return this.
|
|
2977
|
+
return this.store.trendDaily(n);
|
|
3054
2978
|
}
|
|
3055
2979
|
listSessions(filter, page) {
|
|
3056
2980
|
let results = [...this.sessionMap.values()];
|
|
@@ -3061,13 +2985,13 @@ var MemoryManager = class {
|
|
|
3061
2985
|
}
|
|
3062
2986
|
async listMessages(sessionId, limitOrPage) {
|
|
3063
2987
|
if (typeof limitOrPage === "object") {
|
|
3064
|
-
const all = await this.
|
|
2988
|
+
const all = await this.store.getAllMessagesBySession(sessionId);
|
|
3065
2989
|
return paginate(all, limitOrPage);
|
|
3066
2990
|
}
|
|
3067
|
-
return this.
|
|
2991
|
+
return this.store.getLatestMessages(sessionId, limitOrPage ?? 100);
|
|
3068
2992
|
}
|
|
3069
2993
|
async listTopics(filter, page) {
|
|
3070
|
-
let topics = await this.
|
|
2994
|
+
let topics = await this.store.getAllTopics();
|
|
3071
2995
|
if (filter?.sessionId) topics = topics.filter((t) => t.sessionId === filter.sessionId);
|
|
3072
2996
|
if (filter?.userId) topics = topics.filter((t) => t.userId === filter.userId);
|
|
3073
2997
|
if (filter?.chatId) topics = topics.filter((t) => t.chatId === filter.chatId);
|
|
@@ -3141,6 +3065,8 @@ var MemoryManager = class {
|
|
|
3141
3065
|
this.optimizeTimer = void 0;
|
|
3142
3066
|
}
|
|
3143
3067
|
this.grafeo.close();
|
|
3068
|
+
void this.store.close().catch(() => {
|
|
3069
|
+
});
|
|
3144
3070
|
}
|
|
3145
3071
|
};
|
|
3146
3072
|
export {
|
|
@@ -3148,6 +3074,6 @@ export {
|
|
|
3148
3074
|
DEFAULT_RELATION_TYPES,
|
|
3149
3075
|
KIND_CONVERSATION,
|
|
3150
3076
|
KIND_KNOWLEDGE,
|
|
3151
|
-
|
|
3152
|
-
|
|
3077
|
+
MemoryManager,
|
|
3078
|
+
MemoryStore
|
|
3153
3079
|
};
|