opcode-pg-memory 2.2.8 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +232 -214
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +30 -21006
- package/dist/index.js.map +1 -0
- package/dist/mcp-server.js +319 -302
- package/dist/mcp-server.js.map +1 -0
- package/dist/src/cache/semantic-cache.js +399 -0
- package/dist/src/cache/semantic-cache.js.map +1 -0
- package/dist/src/cli.js +404 -0
- package/dist/src/cli.js.map +1 -0
- package/dist/src/config.d.ts +5 -0
- package/dist/src/config.d.ts.map +1 -1
- package/dist/src/config.js +89 -0
- package/dist/src/config.js.map +1 -0
- package/dist/src/db/init-db.js +545 -0
- package/dist/src/db/init-db.js.map +1 -0
- package/dist/src/hooks/message-part-updated.js +203 -0
- package/dist/src/hooks/message-part-updated.js.map +1 -0
- package/dist/src/hooks/message-updated.js +347 -0
- package/dist/src/hooks/message-updated.js.map +1 -0
- package/dist/src/hooks/session-compacting.js +179 -0
- package/dist/src/hooks/session-compacting.js.map +1 -0
- package/dist/src/hooks/session-completed.js +337 -0
- package/dist/src/hooks/session-completed.js.map +1 -0
- package/dist/src/hooks/session-created.js +206 -0
- package/dist/src/hooks/session-created.js.map +1 -0
- package/dist/src/hooks/tool-execute.js +267 -0
- package/dist/src/hooks/tool-execute.js.map +1 -0
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +643 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/mcp/hindsight-reflect-omo.js +318 -0
- package/dist/src/mcp/hindsight-reflect-omo.js.map +1 -0
- package/dist/src/mcp/hindsight-reflect.js +838 -0
- package/dist/src/mcp/hindsight-reflect.js.map +1 -0
- package/dist/src/mcp/recall-memory-omo.js +263 -0
- package/dist/src/mcp/recall-memory-omo.js.map +1 -0
- package/dist/src/mcp/recall-memory.d.ts +6 -0
- package/dist/src/mcp/recall-memory.d.ts.map +1 -1
- package/dist/src/mcp/recall-memory.js +900 -0
- package/dist/src/mcp/recall-memory.js.map +1 -0
- package/dist/src/omo/adapter.js +583 -0
- package/dist/src/omo/adapter.js.map +1 -0
- package/dist/src/omo/types.js +44 -0
- package/dist/src/omo/types.js.map +1 -0
- package/dist/src/services/db-polling.d.ts +30 -0
- package/dist/src/services/db-polling.d.ts.map +1 -0
- package/dist/src/services/db-polling.js +97 -0
- package/dist/src/services/db-polling.js.map +1 -0
- package/dist/src/services/event-synchronizer.d.ts +15 -0
- package/dist/src/services/event-synchronizer.d.ts.map +1 -0
- package/dist/src/services/event-synchronizer.js +119 -0
- package/dist/src/services/event-synchronizer.js.map +1 -0
- package/dist/src/services/keyword.js +29 -0
- package/dist/src/services/keyword.js.map +1 -0
- package/dist/src/services/logger.js +42 -0
- package/dist/src/services/logger.js.map +1 -0
- package/dist/src/services/opencode-schema-adapter.d.ts +34 -0
- package/dist/src/services/opencode-schema-adapter.d.ts.map +1 -0
- package/dist/src/services/opencode-schema-adapter.js +96 -0
- package/dist/src/services/opencode-schema-adapter.js.map +1 -0
- package/dist/src/services/privacy.js +23 -0
- package/dist/src/services/privacy.js.map +1 -0
- package/dist/src/topic/segment-manager.js +447 -0
- package/dist/src/topic/segment-manager.js.map +1 -0
- package/dist/src/types.d.ts +20 -2
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/types.js +8 -0
- package/dist/src/types.js.map +1 -0
- package/dist/src/utils/embedding.js +180 -0
- package/dist/src/utils/embedding.js.map +1 -0
- package/dist/src/utils/token-budget.js +152 -0
- package/dist/src/utils/token-budget.js.map +1 -0
- package/package.json +6 -5
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.EmbeddingService = void 0;
|
|
7
|
+
exports.createEmbeddingService = createEmbeddingService;
|
|
8
|
+
exports.getEmbeddingService = getEmbeddingService;
|
|
9
|
+
const openai_1 = __importDefault(require("openai"));
|
|
10
|
+
// Ollama 默认参数
|
|
11
|
+
const DEFAULT_OLLAMA_MAX_TOKENS = 30000; // 留 2K 安全余量
|
|
12
|
+
const AVG_CHARS_PER_TOKEN = 4; // 平均每个 token 约 4 字符
|
|
13
|
+
const CHUNK_OVERLAP = 2000; // 分段重叠 2K 字符,保证上下文连贯
|
|
14
|
+
class EmbeddingService {
|
|
15
|
+
client;
|
|
16
|
+
config;
|
|
17
|
+
constructor(config) {
|
|
18
|
+
this.config = config;
|
|
19
|
+
this.client = null;
|
|
20
|
+
if (config.provider === 'deepseek') {
|
|
21
|
+
this.client = new openai_1.default({
|
|
22
|
+
apiKey: process.env.DEEPSEEK_API_KEY,
|
|
23
|
+
baseURL: process.env.DEEPSEEK_BASE_URL || 'https://api.deepseek.com',
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
else if (config.provider === 'openai') {
|
|
27
|
+
this.client = new openai_1.default({
|
|
28
|
+
apiKey: process.env.OPENAI_API_KEY,
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
// ollama 不使用 OpenAI 客户端,直接调用本地 API
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* 生成单个文本的 embedding
|
|
35
|
+
* 对于长文本,自动分段嵌入后合并
|
|
36
|
+
*/
|
|
37
|
+
async generateEmbedding(text) {
|
|
38
|
+
try {
|
|
39
|
+
// 1. 估算 token 数
|
|
40
|
+
const estimatedTokens = Math.ceil(text.length / AVG_CHARS_PER_TOKEN);
|
|
41
|
+
if (estimatedTokens <= DEFAULT_OLLAMA_MAX_TOKENS) {
|
|
42
|
+
// 短文本:直接嵌入
|
|
43
|
+
return this.embedSingleChunk(text);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
// 长文本:分段嵌入 + 平均合并
|
|
47
|
+
const maxChunkLength = DEFAULT_OLLAMA_MAX_TOKENS * AVG_CHARS_PER_TOKEN;
|
|
48
|
+
const chunks = this.chunkText(text, maxChunkLength, CHUNK_OVERLAP);
|
|
49
|
+
console.log(`[Embedding] Long text detected (${estimatedTokens} tokens), splitting into ${chunks.length} chunks`);
|
|
50
|
+
const embeddings = await Promise.all(chunks.map(chunk => this.embedSingleChunk(chunk)));
|
|
51
|
+
return this.averageEmbeddings(embeddings);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
console.error(`[Embedding] Failed to generate embedding:`, error);
|
|
56
|
+
throw error;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* 批量生成 embeddings
|
|
61
|
+
*/
|
|
62
|
+
async generateEmbeddings(texts) {
|
|
63
|
+
const results = [];
|
|
64
|
+
// 对于批量处理,我们仍然需要对每个文本单独处理(因为长度可能不同)
|
|
65
|
+
for (const text of texts) {
|
|
66
|
+
results.push(await this.generateEmbedding(text));
|
|
67
|
+
}
|
|
68
|
+
return results;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* 嵌入单个文本块
|
|
72
|
+
*/
|
|
73
|
+
async embedSingleChunk(text) {
|
|
74
|
+
if (this.config.provider === 'ollama') {
|
|
75
|
+
return await this.embedWithOllama(text);
|
|
76
|
+
}
|
|
77
|
+
else if (this.client) {
|
|
78
|
+
const response = await this.client.embeddings.create({
|
|
79
|
+
model: this.config.model,
|
|
80
|
+
input: text,
|
|
81
|
+
dimensions: this.config.dimensions,
|
|
82
|
+
});
|
|
83
|
+
return response.data[0].embedding;
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
throw new Error('No embedding client available');
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* 调用本地 Ollama API 生成 embedding
|
|
91
|
+
*/
|
|
92
|
+
async embedWithOllama(text) {
|
|
93
|
+
const response = await fetch('http://localhost:11434/api/embeddings', {
|
|
94
|
+
method: 'POST',
|
|
95
|
+
headers: { 'Content-Type': 'application/json' },
|
|
96
|
+
body: JSON.stringify({
|
|
97
|
+
model: this.config.model,
|
|
98
|
+
prompt: text,
|
|
99
|
+
// qwen3-embedding 支持自定义维度 (32-1024)
|
|
100
|
+
// 如果配置的维度在范围内,请求自定义;否则让模型使用默认维度
|
|
101
|
+
...(this.config.dimensions >= 32 && this.config.dimensions <= 1024 && {
|
|
102
|
+
parameters: { output_dimension: this.config.dimensions }
|
|
103
|
+
})
|
|
104
|
+
}),
|
|
105
|
+
});
|
|
106
|
+
if (!response.ok) {
|
|
107
|
+
const error = await response.text();
|
|
108
|
+
throw new Error(`Ollama API error: ${error}`);
|
|
109
|
+
}
|
|
110
|
+
const data = await response.json();
|
|
111
|
+
return data.embedding;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* 将长文本切分成重叠的块
|
|
115
|
+
*/
|
|
116
|
+
chunkText(text, maxLength, overlap) {
|
|
117
|
+
const chunks = [];
|
|
118
|
+
let start = 0;
|
|
119
|
+
while (start < text.length) {
|
|
120
|
+
const end = Math.min(start + maxLength, text.length);
|
|
121
|
+
chunks.push(text.substring(start, end));
|
|
122
|
+
start += maxLength - overlap;
|
|
123
|
+
}
|
|
124
|
+
return chunks;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* 平均合并多个 embeddings(向量的逐元素平均)
|
|
128
|
+
*/
|
|
129
|
+
averageEmbeddings(embeddings) {
|
|
130
|
+
if (embeddings.length === 0) {
|
|
131
|
+
return [];
|
|
132
|
+
}
|
|
133
|
+
const dim = embeddings[0].length;
|
|
134
|
+
const avg = new Array(dim).fill(0);
|
|
135
|
+
for (const emb of embeddings) {
|
|
136
|
+
for (let i = 0; i < dim; i++) {
|
|
137
|
+
avg[i] += emb[i];
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
for (let i = 0; i < dim; i++) {
|
|
141
|
+
avg[i] /= embeddings.length;
|
|
142
|
+
}
|
|
143
|
+
return avg;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
exports.EmbeddingService = EmbeddingService;
|
|
147
|
+
// 创建默认实例
|
|
148
|
+
function createEmbeddingService() {
|
|
149
|
+
const provider = process.env.EMBEDDING_PROVIDER || 'deepseek';
|
|
150
|
+
const model = process.env.EMBEDDING_MODEL || 'deepseek-embedding';
|
|
151
|
+
const dimensions = parseInt(process.env.EMBEDDING_DIMENSIONS || '1536', 10);
|
|
152
|
+
const batchSize = parseInt(process.env.EMBEDDING_BATCH_SIZE || '100', 10);
|
|
153
|
+
return new EmbeddingService({
|
|
154
|
+
provider,
|
|
155
|
+
model,
|
|
156
|
+
dimensions,
|
|
157
|
+
batchSize,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
// 共享单例实例(惰性初始化)
|
|
161
|
+
let _sharedService = null;
|
|
162
|
+
/**
|
|
163
|
+
* 获取共享的 EmbeddingService 单例
|
|
164
|
+
* 惰性初始化:首次调用时创建,后续调用返回同一实例。
|
|
165
|
+
* 如果创建失败(如缺少 API Key),返回 null 而非抛出异常。
|
|
166
|
+
*/
|
|
167
|
+
function getEmbeddingService() {
|
|
168
|
+
if (!_sharedService) {
|
|
169
|
+
try {
|
|
170
|
+
_sharedService = createEmbeddingService();
|
|
171
|
+
console.log(`[Embedding] Service initialized: provider=${process.env.EMBEDDING_PROVIDER || 'deepseek'}, model=${process.env.EMBEDDING_MODEL || 'deepseek-embedding'}, dimensions=${process.env.EMBEDDING_DIMENSIONS || '1536'}`);
|
|
172
|
+
}
|
|
173
|
+
catch (error) {
|
|
174
|
+
console.warn('[Embedding] Failed to create embedding service:', error);
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return _sharedService;
|
|
179
|
+
}
|
|
180
|
+
//# sourceMappingURL=embedding.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding.js","sourceRoot":"","sources":["../../../src/utils/embedding.ts"],"names":[],"mappings":";;;;;;AAuKA,wDAYC;AAUD,kDAWC;AAxMD,oDAA4B;AAS5B,cAAc;AACd,MAAM,yBAAyB,GAAG,KAAK,CAAC,CAAE,YAAY;AACtD,MAAM,mBAAmB,GAAG,CAAC,CAAC,CAAa,oBAAoB;AAC/D,MAAM,aAAa,GAAG,IAAI,CAAC,CAAgB,qBAAqB;AAEhE,MAAa,gBAAgB;IACnB,MAAM,CAAgB;IACtB,MAAM,CAAkB;IAEhC,YAAY,MAAuB;QACjC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAEnB,IAAI,MAAM,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,MAAM,GAAG,IAAI,gBAAM,CAAC;gBACvB,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB;gBACpC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,0BAA0B;aACrE,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACxC,IAAI,CAAC,MAAM,GAAG,IAAI,gBAAM,CAAC;gBACvB,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;aACnC,CAAC,CAAC;QACL,CAAC;QACD,mCAAmC;IACrC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,iBAAiB,CAAC,IAAY;QAClC,IAAI,CAAC;YACH,gBAAgB;YAChB,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,mBAAmB,CAAC,CAAC;YAErE,IAAI,eAAe,IAAI,yBAAyB,EAAE,CAAC;gBACjD,WAAW;gBACX,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YACrC,CAAC;iBAAM,CAAC;gBACN,kBAAkB;gBAClB,MAAM,cAAc,GAAG,yBAAyB,GAAG,mBAAmB,CAAC;gBACvE,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,EAAE,aAAa,CAAC,CAAC;gBAEnE,OAAO,CAAC,GAAG,CAAC,mCAAmC,eAAe,4BAA4B,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;gBAElH,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAClD,CAAC;gBAEF,OAAO,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,KAAK,CAAC,CAAC;YAClE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,KAAe;QACtC,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,mCAAmC;QACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,IAAY;QACzC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACtC,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACvB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACnD,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;gBACxB,KAAK,EAAE,IAAI;gBACX,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;aACnC,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAAC,IAAY;QACxC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;YACpE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;gBACxB,MAAM,EAAE,IAAI;gBACZ,oCAAoC;gBACpC,gCAAgC;gBAChC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,IAAI,IAAI;oBACpE,UAAU,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE;iBACzD,CAAC;aACH,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAC;QAChD,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAA6B,CAAC;QAC9D,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,IAAY,EAAE,SAAiB,EAAE,OAAe;QAChE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACxC,KAAK,IAAI,SAAS,GAAG,OAAO,CAAC;QAC/B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,UAAsB;QAC9C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAEnC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7B,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,GAAG,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC;QAC9B,CAAC;QAED,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAtJD,4CAsJC;AAED,SAAS;AACT,SAAgB,sBAAsB;IACpC,MAAM,QAAQ,GAAI,OAAO,CAAC,GAAG,CAAC,kBAAuD,IAAI,UAAU,CAAC;IACpG,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,oBAAoB,CAAC;IAClE,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,MAAM,EAAE,EAAE,CAAC,CAAC;IAC5E,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;IAE1E,OAAO,IAAI,gBAAgB,CAAC;QAC1B,QAAQ;QACR,KAAK;QACL,UAAU;QACV,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED,gBAAgB;AAChB,IAAI,cAAc,GAA4B,IAAI,CAAC;AAEnD;;;;GAIG;AACH,SAAgB,mBAAmB;IACjC,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,cAAc,GAAG,sBAAsB,EAAE,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,6CAA6C,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,UAAU,WAAW,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,oBAAoB,gBAAgB,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,MAAM,EAAE,CAAC,CAAC;QACnO,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;YACvE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,cAAc,CAAC;AACxB,CAAC"}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_TOKEN_BUDGET_CONFIG = void 0;
|
|
4
|
+
exports.calculateTokenBudget = calculateTokenBudget;
|
|
5
|
+
exports.estimateTokens = estimateTokens;
|
|
6
|
+
exports.formatEntity = formatEntity;
|
|
7
|
+
exports.formatReflection = formatReflection;
|
|
8
|
+
exports.retrieveFactsWithBudget = retrieveFactsWithBudget;
|
|
9
|
+
exports.truncateToTokenLimit = truncateToTokenLimit;
|
|
10
|
+
exports.calculateTotalTokens = calculateTotalTokens;
|
|
11
|
+
exports.checkBudgetOverflow = checkBudgetOverflow;
|
|
12
|
+
exports.DEFAULT_TOKEN_BUDGET_CONFIG = {
|
|
13
|
+
contextLimitRatio: 0.05,
|
|
14
|
+
minTokens: 500,
|
|
15
|
+
maxTokens: 4000,
|
|
16
|
+
tierWeights: {
|
|
17
|
+
permanent: 0.5,
|
|
18
|
+
project: 0.3,
|
|
19
|
+
session: 0.2
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* 计算 Token 预算
|
|
24
|
+
* Formula: MAX_INJECT_TOKENS = clamp(modelContextLimit * 0.05, 500, 4000)
|
|
25
|
+
*/
|
|
26
|
+
function calculateTokenBudget(modelContextLimit, config = {}) {
|
|
27
|
+
const mergedConfig = { ...exports.DEFAULT_TOKEN_BUDGET_CONFIG, ...config };
|
|
28
|
+
const rawBudget = Math.floor(modelContextLimit * mergedConfig.contextLimitRatio);
|
|
29
|
+
return Math.max(mergedConfig.minTokens, Math.min(mergedConfig.maxTokens, rawBudget));
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* 估算文本的 token 数量
|
|
33
|
+
* 简化版:假设平均每个 token 约 4 个字符(英文)或 2 个字符(中文)
|
|
34
|
+
*/
|
|
35
|
+
function estimateTokens(text) {
|
|
36
|
+
if (!text)
|
|
37
|
+
return 0;
|
|
38
|
+
// 检测中文字符比例
|
|
39
|
+
const chineseChars = (text.match(/[\u4e00-\u9fa5]/g) || []).length;
|
|
40
|
+
const totalChars = text.length;
|
|
41
|
+
const chineseRatio = chineseChars / totalChars;
|
|
42
|
+
// 混合计算
|
|
43
|
+
if (chineseRatio > 0.5) {
|
|
44
|
+
// 主要中文:每个汉字约 1-2 tokens
|
|
45
|
+
return Math.ceil(totalChars * 0.8);
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
// 主要英文:每个 token 约 4 个字符
|
|
49
|
+
return Math.ceil(totalChars / 4);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* 格式化实体为注入文本
|
|
54
|
+
*/
|
|
55
|
+
function formatEntity(entity) {
|
|
56
|
+
let formatted = `[${entity.type.toUpperCase()}] ${entity.name}`;
|
|
57
|
+
if (entity.description) {
|
|
58
|
+
formatted += `: ${entity.description}`;
|
|
59
|
+
}
|
|
60
|
+
formatted += ` (tier: ${entity.tier}, weight: ${entity.weight.toFixed(2)})`;
|
|
61
|
+
return formatted;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* 格式化反思为注入文本
|
|
65
|
+
*/
|
|
66
|
+
function formatReflection(reflection) {
|
|
67
|
+
let formatted = '[REFLECTION]';
|
|
68
|
+
if (reflection.pattern_type) {
|
|
69
|
+
formatted += ` (${reflection.pattern_type})`;
|
|
70
|
+
}
|
|
71
|
+
formatted += `: ${reflection.summary}`;
|
|
72
|
+
formatted += ` (confidence: ${reflection.confidence.toFixed(2)})`;
|
|
73
|
+
return formatted;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* 按 token 预算检索事实
|
|
77
|
+
* 优先顺序:permanent > project > session
|
|
78
|
+
*/
|
|
79
|
+
async function retrieveFactsWithBudget(sessionId, modelContextLimit, fetchers, config = {}) {
|
|
80
|
+
const mergedConfig = { ...exports.DEFAULT_TOKEN_BUDGET_CONFIG, ...config };
|
|
81
|
+
const budget = calculateTokenBudget(modelContextLimit, mergedConfig);
|
|
82
|
+
const facts = [];
|
|
83
|
+
let usedTokens = 0;
|
|
84
|
+
// 按 tier 优先级检索
|
|
85
|
+
const tierOrder = ['permanent', 'project', 'session'];
|
|
86
|
+
const tierAllocation = {
|
|
87
|
+
permanent: Math.floor(budget * mergedConfig.tierWeights.permanent),
|
|
88
|
+
project: Math.floor(budget * mergedConfig.tierWeights.project),
|
|
89
|
+
session: Math.floor(budget * mergedConfig.tierWeights.session)
|
|
90
|
+
};
|
|
91
|
+
for (const tier of tierOrder) {
|
|
92
|
+
const tierBudget = tierAllocation[tier];
|
|
93
|
+
let tierUsed = 0;
|
|
94
|
+
// 获取候选列表(足够大的限制以确保我们有足够的选择)
|
|
95
|
+
const candidates = await fetchers.fetchByTier(sessionId, tier, 100);
|
|
96
|
+
for (const candidate of candidates) {
|
|
97
|
+
const { text, tokens } = fetchers.extractContent(candidate);
|
|
98
|
+
// 检查是否超出预算
|
|
99
|
+
if (tierUsed + tokens <= tierBudget && usedTokens + tokens <= budget) {
|
|
100
|
+
facts.push({
|
|
101
|
+
id: `temp-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
102
|
+
type: 'entity',
|
|
103
|
+
content: text,
|
|
104
|
+
tier,
|
|
105
|
+
tokens,
|
|
106
|
+
relevanceScore: 1.0, // 初始分数,后续可调整
|
|
107
|
+
metadata: { source: candidate }
|
|
108
|
+
});
|
|
109
|
+
tierUsed += tokens;
|
|
110
|
+
usedTokens += tokens;
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return facts;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* 截断文本以适应 token 预算
|
|
121
|
+
*/
|
|
122
|
+
function truncateToTokenLimit(text, maxTokens) {
|
|
123
|
+
const estimatedTokens = estimateTokens(text);
|
|
124
|
+
if (estimatedTokens <= maxTokens) {
|
|
125
|
+
return text;
|
|
126
|
+
}
|
|
127
|
+
// 按比例截断
|
|
128
|
+
const ratio = maxTokens / estimatedTokens;
|
|
129
|
+
const targetLength = Math.floor(text.length * ratio);
|
|
130
|
+
return text.substring(0, targetLength) + '... [truncated]';
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* 计算注入内容的总 token 数
|
|
134
|
+
*/
|
|
135
|
+
function calculateTotalTokens(facts) {
|
|
136
|
+
return facts.reduce((sum, fact) => sum + fact.tokens, 0);
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* 检查是否超出预算并提供警告
|
|
140
|
+
*/
|
|
141
|
+
function checkBudgetOverflow(facts, budget) {
|
|
142
|
+
const totalTokens = calculateTotalTokens(facts);
|
|
143
|
+
const overflowTokens = Math.max(0, totalTokens - budget);
|
|
144
|
+
return {
|
|
145
|
+
withinBudget: overflowTokens === 0,
|
|
146
|
+
overflowTokens,
|
|
147
|
+
warning: overflowTokens > 0
|
|
148
|
+
? `Token budget exceeded by ${overflowTokens} tokens (${totalTokens}/${budget})`
|
|
149
|
+
: undefined
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
//# sourceMappingURL=token-budget.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token-budget.js","sourceRoot":"","sources":["../../../src/utils/token-budget.ts"],"names":[],"mappings":";;;AA4BA,oDAUC;AAMD,wCAgBC;AAKD,oCAgBC;AAKD,4CAeC;AAMD,0DAoDC;AAKD,oDAYC;AAKD,oDAEC;AAKD,kDAcC;AA7LY,QAAA,2BAA2B,GAAsB;IAC5D,iBAAiB,EAAE,IAAI;IACvB,SAAS,EAAE,GAAG;IACd,SAAS,EAAE,IAAI;IACf,WAAW,EAAE;QACX,SAAS,EAAE,GAAG;QACd,OAAO,EAAE,GAAG;QACZ,OAAO,EAAE,GAAG;KACb;CACF,CAAC;AAEF;;;GAGG;AACH,SAAgB,oBAAoB,CAClC,iBAAyB,EACzB,SAAqC,EAAE;IAEvC,MAAM,YAAY,GAAG,EAAE,GAAG,mCAA2B,EAAE,GAAG,MAAM,EAAE,CAAC;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,YAAY,CAAC,iBAAiB,CAAC,CAAC;IACjF,OAAO,IAAI,CAAC,GAAG,CACb,YAAY,CAAC,SAAS,EACtB,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS,EAAE,SAAS,CAAC,CAC5C,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAgB,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,WAAW;IACX,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACnE,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;IAC/B,MAAM,YAAY,GAAG,YAAY,GAAG,UAAU,CAAC;IAE/C,OAAO;IACP,IAAI,YAAY,GAAG,GAAG,EAAE,CAAC;QACvB,wBAAwB;QACxB,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC;IACrC,CAAC;SAAM,CAAC;QACN,wBAAwB;QACxB,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,YAAY,CAAC,MAM5B;IACC,IAAI,SAAS,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC;IAEhE,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,SAAS,IAAI,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC;IACzC,CAAC;IAED,SAAS,IAAI,WAAW,MAAM,CAAC,IAAI,aAAa,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAE5E,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,SAAgB,gBAAgB,CAAC,UAIhC;IACC,IAAI,SAAS,GAAG,cAAc,CAAC;IAE/B,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;QAC5B,SAAS,IAAI,KAAK,UAAU,CAAC,YAAY,GAAG,CAAC;IAC/C,CAAC;IAED,SAAS,IAAI,KAAK,UAAU,CAAC,OAAO,EAAE,CAAC;IACvC,SAAS,IAAI,iBAAiB,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAElE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,uBAAuB,CAC3C,SAAiB,EACjB,iBAAyB,EACzB,QAGC,EACD,SAAqC,EAAE;IAEvC,MAAM,YAAY,GAAG,EAAE,GAAG,mCAA2B,EAAE,GAAG,MAAM,EAAE,CAAC;IACnE,MAAM,MAAM,GAAG,oBAAoB,CAAC,iBAAiB,EAAE,YAAY,CAAC,CAAC;IACrE,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,eAAe;IACf,MAAM,SAAS,GAAiB,CAAC,WAAW,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACpE,MAAM,cAAc,GAAG;QACrB,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC,SAAS,CAAC;QAClE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC,OAAO,CAAC;QAC9D,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC,OAAO,CAAC;KAC/D,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,4BAA4B;QAC5B,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QAEpE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAE5D,WAAW;YACX,IAAI,QAAQ,GAAG,MAAM,IAAI,UAAU,IAAI,UAAU,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;gBACrE,KAAK,CAAC,IAAI,CAAC;oBACT,EAAE,EAAE,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;oBACnE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,IAAI;oBACb,IAAI;oBACJ,MAAM;oBACN,cAAc,EAAE,GAAG,EAAE,aAAa;oBAClC,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE;iBAChC,CAAC,CAAC;gBACH,QAAQ,IAAI,MAAM,CAAC;gBACnB,UAAU,IAAI,MAAM,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAgB,oBAAoB,CAAC,IAAY,EAAE,SAAiB;IAClE,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAE7C,IAAI,eAAe,IAAI,SAAS,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,QAAQ;IACR,MAAM,KAAK,GAAG,SAAS,GAAG,eAAe,CAAC;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;IAErD,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,YAAY,CAAC,GAAG,iBAAiB,CAAC;AAC7D,CAAC;AAED;;GAEG;AACH,SAAgB,oBAAoB,CAAC,KAAsB;IACzD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,SAAgB,mBAAmB,CACjC,KAAsB,EACtB,MAAc;IAEd,MAAM,WAAW,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC,CAAC;IAEzD,OAAO;QACL,YAAY,EAAE,cAAc,KAAK,CAAC;QAClC,cAAc;QACd,OAAO,EAAE,cAAc,GAAG,CAAC;YACzB,CAAC,CAAC,4BAA4B,cAAc,YAAY,WAAW,IAAI,MAAM,GAAG;YAChF,CAAC,CAAC,SAAS;KACd,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opcode-pg-memory",
|
|
3
|
-
"version": "2.
|
|
4
|
-
"
|
|
5
|
-
"main": "dist/index.js",
|
|
3
|
+
"version": "2.3.0",
|
|
4
|
+
"main": "dist/index.js",
|
|
6
5
|
"types": "dist/index.d.ts",
|
|
7
6
|
"bin": {
|
|
8
7
|
"opcode-pg-memory": "dist/cli.js"
|
|
9
8
|
},
|
|
10
9
|
"scripts": {
|
|
11
|
-
"build": "
|
|
10
|
+
"build": "tsc && bun build ./src/cli.ts --outfile ./dist/cli.js --target node && bun build ./mcp-server.ts --outfile ./dist/mcp-server.js --target node",
|
|
12
11
|
"typecheck": "tsc --noEmit",
|
|
13
12
|
"clean": "rimraf dist"
|
|
14
13
|
},
|
|
@@ -61,9 +60,11 @@
|
|
|
61
60
|
"rimraf": "^5.0.5",
|
|
62
61
|
"jest": "^29.7.0",
|
|
63
62
|
"ts-jest": "^29.1.1",
|
|
64
|
-
"@types/jest": "^29.5.10"
|
|
63
|
+
"@types/jest": "^29.5.10",
|
|
64
|
+
"bun-types": "latest"
|
|
65
65
|
},
|
|
66
66
|
"engines": {
|
|
67
67
|
"node": ">=18.0.0"
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
|
+
|