@modusensus/dsh-mneme 0.2.11 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -4
- package/lib/config.js +14 -0
- package/lib/dream/decisions.js +39 -7
- package/lib/dream.js +1 -1
- package/lib/entities/extractor.js +242 -0
- package/lib/index.js +41 -1
- package/lib/service.js +96 -5
- package/lib/store.js +292 -0
- package/package.json +1 -1
- package/src/config.js +14 -0
- package/src/dream/decisions.js +39 -7
- package/src/dream.js +1 -1
- package/src/entities/extractor.js +242 -0
- package/src/index.js +41 -1
- package/src/service.js +96 -5
- package/src/store.js +292 -0
- package/test/entities.test.js +522 -0
- package/test/fnew-0112.test.js +13 -18
package/README.md
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
[](https://www.npmjs.com/package/@modusensus/dsh-mneme)
|
|
6
6
|
[](LICENSE)
|
|
7
7
|
[](https://github.com/awesome-dsh-plugin/awesome-dsh-plugin)
|
|
8
|
-
[](https://github.com/modusensus/dsh-mneme)
|
|
9
9
|
|
|
10
10
|
> 给 DeepSeek Harness 的跨会话记忆插件:让 Agent 记住你、记住项目、自动整理记忆。**Mneme**(Μνήμη)——希腊记忆女神 Mnemosyne 之名,掌管记忆与梦境,正如 autoDream 在后台巩固记忆。
|
|
11
11
|
|
|
@@ -92,7 +92,19 @@ v0.2 起新增**完全离线的语义记忆引擎**(本地模型 + 精排 +
|
|
|
92
92
|
|
|
93
93
|
配置只需在 `cordis.patch.yml` 里设置 `embedProvider`(默认 `openai`,保持 v0.1 行为;改为 `local` 即离线)。升级无需迁移数据。
|
|
94
94
|
|
|
95
|
-
|
|
95
|
+
### 实体结构化记忆(Entity Gene)🧬
|
|
96
|
+
|
|
97
|
+
v0.3.0 起新增**记忆基因**层:从记忆里抽取**命名实体**、**带时间轴的属性**、**实体间关系**,让搜索从"字面关键词"升级为"按实体/属性精确召回"。
|
|
98
|
+
|
|
99
|
+
- **三表**:`entities` / `entity_attrs`(`valid_until` 快照式时间轴)/ `entity_relations`,旧库打开自动建表,幂等无迁移成本
|
|
100
|
+
- **自动抽取**:`entityExtractionEnabled=true` 后,新写入的记忆 fire-and-forget 触发 LLM 抽取(同名实体去重、属性存时间轴、关系追加;失败绝不阻塞写入)
|
|
101
|
+
- **实体搜索**(`searchMemories` 前缀路由,`entitySearchEnabled` 默认开):
|
|
102
|
+
- `entity:阿尔托` → 属性精确关联的记忆(`_score 1.0`)排在关键词提及(`_score 0.7`)之前
|
|
103
|
+
- `attr:国籍=芬兰` → 精确匹配该属性值的记忆
|
|
104
|
+
- `attr:国籍` → 该属性键的**全部**当前有效记忆(value 为空契约)
|
|
105
|
+
- **autoDream 联动**:update 决策写 `supersedes` 自引用(属性版本被替代);merge 决策把 loser 的属性归属迁移到 keeper(keeper 已有同键当前值则失效)
|
|
106
|
+
|
|
107
|
+
> 📖 详见 [实体结构化记忆设计](docs/ENTITIES.md) · [语义增强架构](docs/SEMANTIC.md) · [本地模型部署指南](docs/LOCAL_MODEL.md) · [从 v0.1 升级说明](docs/MIGRATION.md)
|
|
96
108
|
|
|
97
109
|
## 📦 安装
|
|
98
110
|
|
|
@@ -184,6 +196,11 @@ dsh web
|
|
|
184
196
|
| `reflectionFailureTracking` | `true` | 失败追踪总开关 |
|
|
185
197
|
| `reflectionUpdateMaxPerRun` | `2` | 每次整理最多 update 数 |
|
|
186
198
|
| `reflectionUpdateMinAgeHours` | `24` | 新建记忆保护期(小时) |
|
|
199
|
+
| `entityExtractionEnabled` | `false` | 实体抽取总开关(v0.3.0;存储层恒可用) |
|
|
200
|
+
| `entityExtractionModel` | 空 | 抽取专用模型(空 = 用 agent 默认模型) |
|
|
201
|
+
| `entityExtractionMaxEntities` | `10` | 每次抽取实体数上限 |
|
|
202
|
+
| `entityExtractionMaxAttrs` | `20` | 每实体属性数上限 |
|
|
203
|
+
| `entitySearchEnabled` | `true` | `entity:` / `attr:` 前缀搜索开关 |
|
|
187
204
|
|
|
188
205
|
> 🔐 **API 安全**:DSH 无内置鉴权且默认仅监听 `127.0.0.1`。插件 API 默认开放(便于 Web 面板即装即用)。如需防护(如局域网暴露),在配置中设置 `apiToken`:写操作(画像/规则/命令)与密钥端点(`vector-config`、`vector-reindex`)需携带 `Authorization: Bearer <token>`(前端设置面板可填入同一 token),只读的 `list` / `search` / `semantic` 保持开放。`/api/dsh-mneme/vector-config` 返回的 `apiKey` 已掩码(`sk-***…`),存储仍保留明文供调用;前端回传空或掩码值表示"不改 key"。
|
|
189
206
|
|
|
@@ -219,13 +236,14 @@ src/
|
|
|
219
236
|
├── summarize.js # 会话结束 LLM 摘要
|
|
220
237
|
├── dream.js # autoDream 调度 + runDream(LLM 决策 + 摘要)
|
|
221
238
|
├── dream/decisions.js# 决策校验(fail-safe)+ 决策应用
|
|
239
|
+
├── entities/extractor.js # 实体抽取器(v0.3.0:LLM JSON 抽取 + 去重 + fail-safe)
|
|
222
240
|
├── embedding.js # OpenAI 兼容 embeddings 客户端 + 向量检索
|
|
223
241
|
├── api.js # HTTP 路由(Web 面板数据通道)
|
|
224
242
|
└── index.js # 插件接线
|
|
225
243
|
lib/
|
|
226
244
|
├── client.js # Web 面板(手写 ModuleLoader bundle)
|
|
227
245
|
└── *.js # src 的同步分发产物
|
|
228
|
-
test/ #
|
|
246
|
+
test/ # 404 个 node:test 测试(含审计与三轴线压测不变量)
|
|
229
247
|
scripts/ # e2e-dsh.js 端到端演示 · stress-dsh.js 三轴线压测 · sync-lib.js 同步
|
|
230
248
|
```
|
|
231
249
|
|
|
@@ -234,7 +252,7 @@ scripts/ # e2e-dsh.js 端到端演示 · stress-dsh.js 三轴线压
|
|
|
234
252
|
```bash
|
|
235
253
|
cd dsh-mneme
|
|
236
254
|
npm install # 安装 peer 依赖(以 devDependencies 形式,用于本地测试)
|
|
237
|
-
npm test # 运行
|
|
255
|
+
npm test # 运行 404 个测试
|
|
238
256
|
npm run stress # 三轴线压测:长会话检索 / 冲突仲裁 / 多 Agent 并发(离线 mock LLM)
|
|
239
257
|
npm run sync # 把 src/ 同步到 lib/(发布时由 prepack 钩子自动执行)
|
|
240
258
|
```
|
|
@@ -250,6 +268,7 @@ npm run sync # 把 src/ 同步到 lib/(发布时由 prepack 钩子自动
|
|
|
250
268
|
- [记忆库设计](../docs/superpowers/specs/2026-08-13-dsh-mneme-design.md)
|
|
251
269
|
- [autoDream 设计](../docs/superpowers/specs/2026-08-13-dsh-mneme-autodream-design.md)
|
|
252
270
|
- [实施计划](../docs/superpowers/plans/2026-08-13-dsh-memory-autodream.md)
|
|
271
|
+
- [实体结构化记忆设计](docs/ENTITIES.md)
|
|
253
272
|
- [语义增强架构](docs/SEMANTIC.md)
|
|
254
273
|
- [本地模型部署指南](docs/LOCAL_MODEL.md)
|
|
255
274
|
- [从 v0.1 升级说明](docs/MIGRATION.md)
|
package/lib/config.js
CHANGED
|
@@ -75,4 +75,18 @@ export const Config = z.object({
|
|
|
75
75
|
conflictFreezeEnabled: z.boolean().default(false),
|
|
76
76
|
// Maximum number of frozen conflicts to keep pending for manual review.
|
|
77
77
|
conflictFreezeMaxPending: z.natural().min(1).max(1000).default(100),
|
|
78
|
+
|
|
79
|
+
// --- entity gene (v0.3.0) -----------------------------------------------
|
|
80
|
+
// Opt-in: when false (default) nothing in the pipeline extracts entities.
|
|
81
|
+
// The storage layer (entities/entity_attrs/entity_relations tables + CRUD)
|
|
82
|
+
// is always available regardless of this flag.
|
|
83
|
+
entityExtractionEnabled: z.boolean().default(false),
|
|
84
|
+
// Optional model override for entity extraction; empty = use the caller's
|
|
85
|
+
// default provider/model.
|
|
86
|
+
entityExtractionModel: z.string().default(""),
|
|
87
|
+
// Cap on entities per extraction pass and attributes per entity.
|
|
88
|
+
entityExtractionMaxEntities: z.natural().min(1).max(20).default(10),
|
|
89
|
+
entityExtractionMaxAttrs: z.natural().min(1).max(50).default(20),
|
|
90
|
+
// Prefix/semantic search over entity names (used by recall).
|
|
91
|
+
entitySearchEnabled: z.boolean().default(true),
|
|
78
92
|
});
|
package/lib/dream/decisions.js
CHANGED
|
@@ -167,7 +167,7 @@ function casGuard(service, snapshot, ids) {
|
|
|
167
167
|
* committed - the decisions that actually landed, for outcome/receipt based
|
|
168
168
|
* on real committed sub-steps rather than the raw LLM list.
|
|
169
169
|
*/
|
|
170
|
-
export function applyDecisions(decisions, service, logger = null, snapshot = null) {
|
|
170
|
+
export function applyDecisions(decisions, service, logger = null, snapshot = null, config = {}) {
|
|
171
171
|
let applied = 0;
|
|
172
172
|
const conflicts = [];
|
|
173
173
|
const failures = [];
|
|
@@ -180,7 +180,7 @@ export function applyDecisions(decisions, service, logger = null, snapshot = nul
|
|
|
180
180
|
committed.push({ action: "keep", ids: d.ids });
|
|
181
181
|
continue;
|
|
182
182
|
}
|
|
183
|
-
const outcome = applyOne(d, service, snapshot);
|
|
183
|
+
const outcome = applyOne(d, service, snapshot, config);
|
|
184
184
|
if (outcome === "skipped") continue;
|
|
185
185
|
applied += outcome.applied;
|
|
186
186
|
committed.push(outcome.committed);
|
|
@@ -197,12 +197,12 @@ export function applyDecisions(decisions, service, logger = null, snapshot = nul
|
|
|
197
197
|
return { applied, conflicts, failures, committed };
|
|
198
198
|
}
|
|
199
199
|
|
|
200
|
-
function applyOne(d, service, snapshot) {
|
|
200
|
+
function applyOne(d, service, snapshot, config = {}) {
|
|
201
201
|
switch (d.action) {
|
|
202
202
|
case "archive": return applyArchive(d, service, snapshot);
|
|
203
|
-
case "merge": return applyMerge(d, service, snapshot);
|
|
203
|
+
case "merge": return applyMerge(d, service, snapshot, config);
|
|
204
204
|
case "conflict": return applyConflict(d, service, snapshot);
|
|
205
|
-
default: return applyUpdate(d, service, snapshot);
|
|
205
|
+
default: return applyUpdate(d, service, snapshot, config);
|
|
206
206
|
}
|
|
207
207
|
}
|
|
208
208
|
|
|
@@ -222,7 +222,7 @@ function applyArchive(d, service, snapshot) {
|
|
|
222
222
|
return { applied: targets.length, committed: { action: "archive", ids: targets } };
|
|
223
223
|
}
|
|
224
224
|
|
|
225
|
-
function applyMerge(d, service, snapshot) {
|
|
225
|
+
function applyMerge(d, service, snapshot, config = {}) {
|
|
226
226
|
const sources = d.ids.filter((id) => id !== d.keepSource);
|
|
227
227
|
// Idempotent replay: if every other source is already archived, this merge
|
|
228
228
|
// already landed — skip so a replayed/concurrent decision never double-counts
|
|
@@ -241,6 +241,18 @@ function applyMerge(d, service, snapshot) {
|
|
|
241
241
|
const mem = service.getById(id);
|
|
242
242
|
if (mem && !mem.archived) service.setArchived(id, true);
|
|
243
243
|
}
|
|
244
|
+
// 4.3.2 迁移实体关联(opt-in):将 loser(source)记忆关联的 entity_attrs 的
|
|
245
|
+
// memory_id 迁移到 keeper;keeper 已有同 entity+key 的当前属性时 loser 行被
|
|
246
|
+
// 失效。单个 source 迁移失败只告警,绝不能导致整个 merge 事务回滚(fail-safe)。
|
|
247
|
+
if (config.entityExtractionEnabled && typeof service.migrateAttrsToMemory === "function") {
|
|
248
|
+
for (const id of sources) {
|
|
249
|
+
try {
|
|
250
|
+
service.migrateAttrsToMemory(id, d.keepSource, new Date().toISOString());
|
|
251
|
+
} catch (error) {
|
|
252
|
+
logger?.warn?.(`dsh-mneme dream: failed to migrate attrs from ${id} to ${d.keepSource}: ${error.message}`);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
244
256
|
});
|
|
245
257
|
return {
|
|
246
258
|
applied: 1,
|
|
@@ -269,7 +281,7 @@ function applyConflict(d, service, snapshot) {
|
|
|
269
281
|
return { applied: 1, committed: { action: "conflict", winner: d.winner, loser: d.loser, count_before: 2, count_after: 1 } };
|
|
270
282
|
}
|
|
271
283
|
|
|
272
|
-
function applyUpdate(d, service, snapshot) {
|
|
284
|
+
function applyUpdate(d, service, snapshot, config = {}) {
|
|
273
285
|
const id = d.ids[0];
|
|
274
286
|
const mem = service.getById(id);
|
|
275
287
|
if (!mem || mem.archived) return "skipped";
|
|
@@ -288,5 +300,25 @@ function applyUpdate(d, service, snapshot) {
|
|
|
288
300
|
importance: d.importance ?? cur.importance
|
|
289
301
|
});
|
|
290
302
|
});
|
|
303
|
+
// 4.3.1 supersedes 关系(opt-in):事务提交成功后,为该记忆关联的每条实体属性
|
|
304
|
+
// 建立自引用 supersedes 关系,表示"此属性版本已被替代"。仅记录、绝不阻断主流程
|
|
305
|
+
// (fail-safe):记录失败只告警,update 本身照常生效。
|
|
306
|
+
if (config.entityExtractionEnabled && typeof service.saveRelation === "function" && typeof service.getAttrsByMemory === "function") {
|
|
307
|
+
try {
|
|
308
|
+
const oldAttrs = service.getAttrsByMemory(id);
|
|
309
|
+
for (const attr of oldAttrs) {
|
|
310
|
+
if (!attr.entity_id) continue;
|
|
311
|
+
service.saveRelation({
|
|
312
|
+
from_entity: attr.entity_id,
|
|
313
|
+
to_entity: attr.entity_id,
|
|
314
|
+
relation_type: "supersedes",
|
|
315
|
+
memory_id: id,
|
|
316
|
+
metadata: JSON.stringify({ attr_key: attr.attr_key, old_value: attr.attr_value })
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
} catch (error) {
|
|
320
|
+
logger?.warn?.(`dsh-mneme dream: failed to record supersedes relations for ${id}: ${error.message}`);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
291
323
|
return { applied: 1, committed: { action: "update", ids: [id], title: d.title, content: d.content, importance: d.importance, count_before: 1, count_after: 1 } };
|
|
292
324
|
}
|
package/lib/dream.js
CHANGED
|
@@ -541,7 +541,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
541
541
|
// a target changed during the LLM call is skipped and reported as a
|
|
542
542
|
// conflict instead of being overwritten (item ①). Frozen conflicts are
|
|
543
543
|
// excluded from this list (they are parked, not applied).
|
|
544
|
-
const { applied, conflicts, failures, committed } = applyDecisions(applyList, service, logger, snapshot);
|
|
544
|
+
const { applied, conflicts, failures, committed } = applyDecisions(applyList, service, logger, snapshot, config);
|
|
545
545
|
// Per-record receipt chain: one row per actually-committed merge/conflict/
|
|
546
546
|
// update verdict, stamped with the decision-basis digest + idempotency
|
|
547
547
|
// counters (count_before → count_after). Written here, before the run audit
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
const VALID_TYPES = new Set(["person", "project", "concept", "technology", "organization"]);
|
|
2
|
+
const VALID_RELATIONS = new Set(["uses", "depends_on", "part_of", "related_to"]);
|
|
3
|
+
const MAX_MEMORY_CHARS = 4000;
|
|
4
|
+
|
|
5
|
+
function buildSystemPrompt(config) {
|
|
6
|
+
const maxEntities = config.entityExtractionMaxEntities ?? 10;
|
|
7
|
+
const maxAttrs = config.entityExtractionMaxAttrs ?? 20;
|
|
8
|
+
|
|
9
|
+
return `You are an entity extraction engine. Extract named entities, their attributes, and relations from the given text.
|
|
10
|
+
|
|
11
|
+
RULES:
|
|
12
|
+
- Entities are concrete people, projects, technologies, concepts, or organizations.
|
|
13
|
+
- Attributes are characteristic states of an entity. Only extract attributes explicitly mentioned in the text.
|
|
14
|
+
- Do NOT infer, guess, or hallucinate. Only extract what is clearly stated.
|
|
15
|
+
- Use canonical names (full name / primary name). Do NOT merge synonyms — different spellings of the same person are different entities.
|
|
16
|
+
- Output MUST be strict JSON with this exact structure:
|
|
17
|
+
{
|
|
18
|
+
"entities": [
|
|
19
|
+
{"name": "string", "type": "person|project|concept|technology|organization", "attrs": [{"key": "string", "value": "string", "confidence": 0.9}]}
|
|
20
|
+
],
|
|
21
|
+
"relations": [
|
|
22
|
+
{"from": "entityName", "to": "entityName", "type": "uses|depends_on|part_of|related_to"}
|
|
23
|
+
]
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
CONSTRAINTS:
|
|
27
|
+
- "type" must be one of: person, project, concept, technology, organization. If unsure, use "concept".
|
|
28
|
+
- "relations" from/to must reference entity names from the "entities" list.
|
|
29
|
+
- Maximum ${maxEntities} entities. Maximum ${maxAttrs} attributes per entity. Truncate if exceeded.
|
|
30
|
+
- Only include entity-related attributes. Ignore irrelevant miscellaneous details.
|
|
31
|
+
- Return ONLY the JSON object. No explanations, no markdown, no code fences.`;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function buildUserMessage(memoryText) {
|
|
35
|
+
const truncated = memoryText.length > MAX_MEMORY_CHARS
|
|
36
|
+
? memoryText.slice(0, MAX_MEMORY_CHARS) + "..."
|
|
37
|
+
: memoryText;
|
|
38
|
+
return truncated;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function extractJsonFromText(text) {
|
|
42
|
+
if (!text || typeof text !== "string") return null;
|
|
43
|
+
|
|
44
|
+
// Try direct parse first
|
|
45
|
+
try {
|
|
46
|
+
return JSON.parse(text);
|
|
47
|
+
} catch {
|
|
48
|
+
// fall through
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Try to find first {...} block
|
|
52
|
+
const match = text.match(/\{[\s\S]*\}/);
|
|
53
|
+
if (match) {
|
|
54
|
+
try {
|
|
55
|
+
return JSON.parse(match[0]);
|
|
56
|
+
} catch {
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function sanitizeType(type) {
|
|
65
|
+
if (VALID_TYPES.has(type)) return type;
|
|
66
|
+
return "concept";
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function sanitizeConfidence(conf) {
|
|
70
|
+
const num = Number(conf);
|
|
71
|
+
if (Number.isNaN(num)) return 0.9;
|
|
72
|
+
return Math.min(1, Math.max(0, num));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function sanitizeRelationType(type) {
|
|
76
|
+
if (VALID_RELATIONS.has(type)) return type;
|
|
77
|
+
return "related_to";
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function sanitizeExtractedData(data, config) {
|
|
81
|
+
const maxEntities = config.entityExtractionMaxEntities ?? 10;
|
|
82
|
+
const maxAttrs = config.entityExtractionMaxAttrs ?? 20;
|
|
83
|
+
|
|
84
|
+
if (!data || !Array.isArray(data.entities)) {
|
|
85
|
+
throw new Error("Invalid extraction: missing entities array");
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const entities = [];
|
|
89
|
+
const seenNames = new Set();
|
|
90
|
+
|
|
91
|
+
for (const rawEntity of data.entities.slice(0, maxEntities)) {
|
|
92
|
+
if (!rawEntity || typeof rawEntity.name !== "string" || !rawEntity.name.trim()) continue;
|
|
93
|
+
|
|
94
|
+
const name = rawEntity.name.trim();
|
|
95
|
+
if (seenNames.has(name)) continue;
|
|
96
|
+
seenNames.add(name);
|
|
97
|
+
|
|
98
|
+
const attrs = [];
|
|
99
|
+
if (Array.isArray(rawEntity.attrs)) {
|
|
100
|
+
for (const rawAttr of rawEntity.attrs.slice(0, maxAttrs)) {
|
|
101
|
+
if (!rawAttr || typeof rawAttr.key !== "string" || typeof rawAttr.value !== "string") continue;
|
|
102
|
+
attrs.push({
|
|
103
|
+
key: rawAttr.key.trim(),
|
|
104
|
+
value: rawAttr.value.trim(),
|
|
105
|
+
confidence: sanitizeConfidence(rawAttr.confidence)
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
entities.push({
|
|
111
|
+
name,
|
|
112
|
+
type: sanitizeType(rawEntity.type),
|
|
113
|
+
attrs
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const relations = [];
|
|
118
|
+
if (Array.isArray(data.relations)) {
|
|
119
|
+
for (const rawRel of data.relations) {
|
|
120
|
+
if (!rawRel || typeof rawRel.from !== "string" || typeof rawRel.to !== "string") continue;
|
|
121
|
+
if (!seenNames.has(rawRel.from) || !seenNames.has(rawRel.to)) continue;
|
|
122
|
+
relations.push({
|
|
123
|
+
from: rawRel.from,
|
|
124
|
+
to: rawRel.to,
|
|
125
|
+
type: sanitizeRelationType(rawRel.type)
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return { entities, relations };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function resolveEntity(entity, store) {
|
|
134
|
+
const existing = store.findEntityByName(entity.name);
|
|
135
|
+
if (existing) {
|
|
136
|
+
store.updateEntity(existing.id, {});
|
|
137
|
+
return existing.id;
|
|
138
|
+
}
|
|
139
|
+
const created = store.createEntity({ name: entity.name, type: entity.type });
|
|
140
|
+
return created.id;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export async function extractEntities(memory, { store, config, callLLM, logger }) {
|
|
144
|
+
try {
|
|
145
|
+
if (!memory || !memory.content) {
|
|
146
|
+
return { ok: false, error: "Invalid memory: missing content" };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const model = config.entityExtractionModel || null;
|
|
150
|
+
const systemPrompt = buildSystemPrompt(config);
|
|
151
|
+
const userText = buildUserMessage(memory.content);
|
|
152
|
+
|
|
153
|
+
const messages = [
|
|
154
|
+
{ role: "system", content: [{ type: "text", text: systemPrompt }] },
|
|
155
|
+
{ role: "user", content: [{ type: "text", text: userText }] }
|
|
156
|
+
];
|
|
157
|
+
|
|
158
|
+
const options = model ? { model } : {};
|
|
159
|
+
const llmResponse = await callLLM(messages, options);
|
|
160
|
+
|
|
161
|
+
if (!llmResponse) {
|
|
162
|
+
return { ok: false, error: "LLM returned empty response" };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const rawData = extractJsonFromText(llmResponse);
|
|
166
|
+
if (!rawData) {
|
|
167
|
+
return { ok: false, error: "Failed to parse JSON from LLM response" };
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const { entities, relations } = sanitizeExtractedData(rawData, config);
|
|
171
|
+
|
|
172
|
+
const resolvedEntities = [];
|
|
173
|
+
const entityIdMap = new Map();
|
|
174
|
+
let skipCount = 0;
|
|
175
|
+
|
|
176
|
+
// Resolve entities
|
|
177
|
+
for (const entity of entities) {
|
|
178
|
+
try {
|
|
179
|
+
const entityId = await resolveEntity(entity, store);
|
|
180
|
+
entityIdMap.set(entity.name, entityId);
|
|
181
|
+
resolvedEntities.push({ ...entity, entity_id: entityId });
|
|
182
|
+
} catch (err) {
|
|
183
|
+
skipCount++;
|
|
184
|
+
logger?.warn?.(`[extractor] Failed to resolve entity "${entity.name}":`, err.message);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Record attributes
|
|
189
|
+
const attrs = [];
|
|
190
|
+
for (const entity of resolvedEntities) {
|
|
191
|
+
for (const attr of entity.attrs) {
|
|
192
|
+
try {
|
|
193
|
+
const saved = store.saveAttr({
|
|
194
|
+
entity_id: entity.entity_id,
|
|
195
|
+
attr_key: attr.key,
|
|
196
|
+
attr_value: String(attr.value),
|
|
197
|
+
memory_id: memory.id,
|
|
198
|
+
confidence: attr.confidence,
|
|
199
|
+
source: "llm_extract"
|
|
200
|
+
});
|
|
201
|
+
attrs.push(saved);
|
|
202
|
+
} catch (err) {
|
|
203
|
+
skipCount++;
|
|
204
|
+
logger?.warn?.(`[extractor] Failed to save attr "${attr.key}" for entity "${entity.name}":`, err.message);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Record relations
|
|
210
|
+
const savedRelations = [];
|
|
211
|
+
for (const rel of relations) {
|
|
212
|
+
const fromId = entityIdMap.get(rel.from);
|
|
213
|
+
const toId = entityIdMap.get(rel.to);
|
|
214
|
+
if (!fromId || !toId) continue;
|
|
215
|
+
|
|
216
|
+
try {
|
|
217
|
+
const saved = store.saveRelation({
|
|
218
|
+
from_entity: fromId,
|
|
219
|
+
to_entity: toId,
|
|
220
|
+
relation_type: rel.type,
|
|
221
|
+
memory_id: memory.id,
|
|
222
|
+
metadata: { model: model || "default" }
|
|
223
|
+
});
|
|
224
|
+
savedRelations.push(saved);
|
|
225
|
+
} catch (err) {
|
|
226
|
+
skipCount++;
|
|
227
|
+
logger?.warn?.(`[extractor] Failed to save relation "${rel.from} -> ${rel.to}":`, err.message);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
return {
|
|
232
|
+
ok: true,
|
|
233
|
+
entities: resolvedEntities,
|
|
234
|
+
attrs,
|
|
235
|
+
relations: savedRelations,
|
|
236
|
+
skipped: skipCount
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
} catch (err) {
|
|
240
|
+
return { ok: false, error: String(err) };
|
|
241
|
+
}
|
|
242
|
+
}
|
package/lib/index.js
CHANGED
|
@@ -13,6 +13,7 @@ import { createEmbedderByProvider } from "./local-embedder.js";
|
|
|
13
13
|
import { LocalReranker } from "./reranker.js";
|
|
14
14
|
import { createVectorIndex } from "./vector-index.js";
|
|
15
15
|
import { Config } from "./config.js";
|
|
16
|
+
import { extractEntities } from "./entities/extractor.js";
|
|
16
17
|
import { mkdirSync } from "node:fs";
|
|
17
18
|
import { join } from "node:path";
|
|
18
19
|
import { homedir } from "node:os";
|
|
@@ -42,7 +43,7 @@ export const apply = (ctx, config) => {
|
|
|
42
43
|
store.deleteOldFailures(new Date(Date.now() - 90 * 86400000).toISOString());
|
|
43
44
|
} catch { /* non-fatal */ }
|
|
44
45
|
const mirror = createMirror(memoryDir);
|
|
45
|
-
const service = createService({ store, mirror, config: cfg });
|
|
46
|
+
const service = createService({ store, mirror, config: cfg, logger: ctx.logger });
|
|
46
47
|
|
|
47
48
|
// Recall-layer receipt: when searchMemories runs with recordRecall=true, the
|
|
48
49
|
// retrieval scene (query/mode/topK/threshold + candidates) is persisted to
|
|
@@ -164,6 +165,45 @@ export const apply = (ctx, config) => {
|
|
|
164
165
|
service.setDreamHook(() => dream.maybeSchedule(service));
|
|
165
166
|
}
|
|
166
167
|
|
|
168
|
+
// Entity gene extraction (v0.3.0): wire the extractor into the service as a
|
|
169
|
+
// hook so saveWithDedupe can fire-and-forget an extraction pass on fresh
|
|
170
|
+
// writes. The service never sees ctx.llm — index.js adapts it here into the
|
|
171
|
+
// callLLM(messages, options) => Promise<string> contract the extractor
|
|
172
|
+
// expects, reusing the same ctx.llm.stream consumption pattern as dream.js.
|
|
173
|
+
// Explicit opt-in only (entityExtractionEnabled defaults to false); any LLM
|
|
174
|
+
// failure degrades inside the extractor to { ok:false }, never a write error.
|
|
175
|
+
if (cfg.entityExtractionEnabled && ctx.llm) {
|
|
176
|
+
const streamEntityText = async (messages, options = {}) => {
|
|
177
|
+
let route = null;
|
|
178
|
+
if (options.model) {
|
|
179
|
+
route = { model: options.model };
|
|
180
|
+
} else {
|
|
181
|
+
try {
|
|
182
|
+
const sel = ctx.agentDefaultModel?.currentSelection?.();
|
|
183
|
+
if (sel?.provider && sel?.model) route = sel;
|
|
184
|
+
} catch { /* fall through to no route */ }
|
|
185
|
+
}
|
|
186
|
+
let text = "";
|
|
187
|
+
for await (const chunk of ctx.llm.stream({
|
|
188
|
+
...(route ?? {}),
|
|
189
|
+
purpose: "entity-extract",
|
|
190
|
+
maxTokens: 4096,
|
|
191
|
+
messages
|
|
192
|
+
})) {
|
|
193
|
+
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
194
|
+
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) return undefined;
|
|
195
|
+
}
|
|
196
|
+
return text;
|
|
197
|
+
};
|
|
198
|
+
service.setEntityExtractor((memory) =>
|
|
199
|
+
extractEntities(memory, { store, config: cfg, callLLM: streamEntityText, logger: ctx.logger })
|
|
200
|
+
.catch((err) => {
|
|
201
|
+
ctx.logger?.warn?.(`[dsh-mneme] entity extraction failed: ${String(err)}`);
|
|
202
|
+
return { ok: false, error: String(err) };
|
|
203
|
+
})
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
|
|
167
207
|
const disposers = [];
|
|
168
208
|
|
|
169
209
|
ctx.inject(["systemPrompt"], (promptCtx) => {
|
package/lib/service.js
CHANGED
|
@@ -3,7 +3,7 @@ import { TYPE_FILE } from "./mirror.js";
|
|
|
3
3
|
|
|
4
4
|
const INJECT_TYPES = new Set(["preference", "project", "decision", "summary"]);
|
|
5
5
|
|
|
6
|
-
export function createService({ store, mirror, config, onWrite }) {
|
|
6
|
+
export function createService({ store, mirror, config, onWrite, logger }) {
|
|
7
7
|
// Optional dream scheduler hook, installed via setDreamHook after creation
|
|
8
8
|
// (the scheduler holds a reference back to the service, so it cannot be
|
|
9
9
|
// passed in the constructor). Fired on the same write events as onWrite.
|
|
@@ -13,6 +13,14 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
13
13
|
// any content write it fire-and-forgets a re-embed of the row so vector
|
|
14
14
|
// search stays in sync; failures are swallowed inside the embedder.
|
|
15
15
|
let embedder = null;
|
|
16
|
+
|
|
17
|
+
// Optional entity extractor, installed via setEntityExtractor after creation
|
|
18
|
+
// (index.js injects it so the service never depends on the LLM directly).
|
|
19
|
+
// After a new memory is saved it fire-and-forgets an extraction pass for the
|
|
20
|
+
// entity gene (v0.3.0); failures are swallowed so a broken extraction never
|
|
21
|
+
// surfaces as a write failure. Extraction only runs when
|
|
22
|
+
// config.entityExtractionEnabled is true.
|
|
23
|
+
let entityExtractor = null;
|
|
16
24
|
let vectorIndex = null;
|
|
17
25
|
let reranker = null;
|
|
18
26
|
|
|
@@ -36,6 +44,25 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
36
44
|
}
|
|
37
45
|
}
|
|
38
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Fire-and-forget entity extraction for a freshly saved memory (entity gene
|
|
49
|
+
* v0.3.0). Opt-in via config.entityExtractionEnabled; the extractor is
|
|
50
|
+
* injected as a hook so the service never needs a direct LLM reference.
|
|
51
|
+
* The hook itself is expected to resolve to { ok:boolean } and never throw;
|
|
52
|
+
* a thrown rejection is swallowed here as a final fail-safe.
|
|
53
|
+
*/
|
|
54
|
+
function scheduleEntityExtraction(memory) {
|
|
55
|
+
if (txDepth > 0) return; // deferred to the transaction's commit
|
|
56
|
+
if (!config.entityExtractionEnabled || !entityExtractor) return;
|
|
57
|
+
try {
|
|
58
|
+
entityExtractor(memory).catch((err) => {
|
|
59
|
+
logger?.warn?.("entity extraction failed:", err);
|
|
60
|
+
});
|
|
61
|
+
} catch (err) {
|
|
62
|
+
logger?.warn?.("entity extraction failed:", err);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
39
66
|
/**
|
|
40
67
|
* Cross-encoder rerank over a candidate list (best effort). Reranker
|
|
41
68
|
* failures degrade to the original candidate order — reranking is an
|
|
@@ -58,6 +85,48 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
58
85
|
}
|
|
59
86
|
}
|
|
60
87
|
|
|
88
|
+
/**
|
|
89
|
+
* Search for memories attached to a named entity (v0.3.0 Phase 3).
|
|
90
|
+
* 合并优先级(桉桉确认):entity_attrs.memory_id 精确关联 = 1.0 > 关键词提及 = 0.7;
|
|
91
|
+
* attr 命中不覆盖,keyword 只补充召回,最后按 _score 降序取 topK。
|
|
92
|
+
* @param {string} entityName
|
|
93
|
+
* @param {object} [options]
|
|
94
|
+
* @param {number} [options.topK=20]
|
|
95
|
+
* @returns {any[]}
|
|
96
|
+
*/
|
|
97
|
+
function searchByEntity(entityName, { topK = 20 } = {}) {
|
|
98
|
+
const entity = store.findEntityByName(entityName);
|
|
99
|
+
if (!entity) return [];
|
|
100
|
+
const attrs = store.getCurrentAttrs(entity.id);
|
|
101
|
+
const memoryIds = [...new Set(attrs.map((a) => a.memory_id).filter(Boolean))];
|
|
102
|
+
const attrHits = memoryIds.map((id) => store.getById(id)).filter(Boolean);
|
|
103
|
+
const keywordHits = store.search(entityName, { limit: topK });
|
|
104
|
+
const merged = new Map();
|
|
105
|
+
for (const mem of attrHits) merged.set(mem.id, { ...mem, _source: "entity_attr", _score: 1.0 });
|
|
106
|
+
for (const mem of keywordHits) {
|
|
107
|
+
if (!merged.has(mem.id)) merged.set(mem.id, { ...mem, _source: "keyword", _score: 0.7 });
|
|
108
|
+
}
|
|
109
|
+
return Array.from(merged.values()).sort((a, b) => b._score - a._score).slice(0, topK);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Search for memories by attribute key/value (v0.3.0 Phase 3).
|
|
114
|
+
* value 为空时由 store.findMemoriesByAttr 返回该 key 的全部有效记忆。
|
|
115
|
+
* @param {string} key
|
|
116
|
+
* @param {string | undefined} value
|
|
117
|
+
* @param {object} [options]
|
|
118
|
+
* @param {number} [options.topK=20]
|
|
119
|
+
* @returns {any[]}
|
|
120
|
+
*/
|
|
121
|
+
function searchByAttr(key, value, { topK = 20 } = {}) {
|
|
122
|
+
if (!key) return [];
|
|
123
|
+
// value 可能为 undefined(attr:key 无 = 值):归一为空串后交给
|
|
124
|
+
// store.findMemoriesByAttr —— 空 value 契约 = 返回该 attr_key 的全部
|
|
125
|
+
// 当前有效记忆(v0.3.0,store.js 已实现)。
|
|
126
|
+
const rows = store.findMemoriesByAttr(key, value ?? "");
|
|
127
|
+
return rows.slice(0, topK);
|
|
128
|
+
}
|
|
129
|
+
|
|
61
130
|
/**
|
|
62
131
|
* Semantic-aware memory search: keyword recall (store.search) plus optional
|
|
63
132
|
* vector recall + rerank. mode:
|
|
@@ -85,9 +154,22 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
85
154
|
return base * (0.5 + (row.importance ?? 3) / 10);
|
|
86
155
|
}
|
|
87
156
|
|
|
88
|
-
async function searchMemories(query,
|
|
157
|
+
async function searchMemories(query, options = {}) {
|
|
158
|
+
const { mode = "auto", topK = 20, threshold, useRerank = true, recordRecall = false } = options;
|
|
89
159
|
const q = String(query ?? "").trim();
|
|
90
160
|
if (!q) return [];
|
|
161
|
+
|
|
162
|
+
// entity:/attr: 前缀路由(v0.3.0 Phase 3)。entitySearchEnabled 关闭时走原逻辑。
|
|
163
|
+
if (config?.entitySearchEnabled) {
|
|
164
|
+
if (q.startsWith("entity:")) {
|
|
165
|
+
return searchByEntity(q.slice(7).trim(), options);
|
|
166
|
+
}
|
|
167
|
+
if (q.startsWith("attr:")) {
|
|
168
|
+
const [key, value] = q.slice(5).split("=");
|
|
169
|
+
return searchByAttr(key, value, options);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
91
173
|
const lim = topK > 0 ? topK : 20;
|
|
92
174
|
|
|
93
175
|
// Keyword results, decorated with a score so they can be weight-blended
|
|
@@ -224,7 +306,7 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
224
306
|
try {
|
|
225
307
|
syncMirror();
|
|
226
308
|
} catch (error) {
|
|
227
|
-
|
|
309
|
+
logger?.warn?.("syncMirror failed after transaction:", error);
|
|
228
310
|
}
|
|
229
311
|
notifyWrite();
|
|
230
312
|
}
|
|
@@ -261,6 +343,7 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
261
343
|
syncMirror();
|
|
262
344
|
notifyWrite();
|
|
263
345
|
scheduleEmbed(created);
|
|
346
|
+
scheduleEntityExtraction(created);
|
|
264
347
|
return { action: "created", memory: created };
|
|
265
348
|
}
|
|
266
349
|
|
|
@@ -405,7 +488,7 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
405
488
|
try {
|
|
406
489
|
mirror.sync(reconcileHumanEdits(store.list({ limit: 500, includeForgotten: false })));
|
|
407
490
|
} catch (error) {
|
|
408
|
-
|
|
491
|
+
logger?.warn?.("syncMirror failed:", error);
|
|
409
492
|
}
|
|
410
493
|
}
|
|
411
494
|
|
|
@@ -417,6 +500,7 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
417
500
|
transaction,
|
|
418
501
|
setDreamHook(fn) { dreamHook = fn; },
|
|
419
502
|
setEmbedder(emb) { embedder = emb; },
|
|
503
|
+
setEntityExtractor(fn) { entityExtractor = fn; },
|
|
420
504
|
setVectorIndex(vi) { vectorIndex = vi; },
|
|
421
505
|
setReranker(rn) { reranker = rn; },
|
|
422
506
|
setRecallRecorder(fn) { recallRecorder = fn; },
|
|
@@ -517,6 +601,13 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
517
601
|
saveConflictPending: (r) => store.saveConflictPending(r),
|
|
518
602
|
listConflictPending: (opts) => store.listConflictPending(opts),
|
|
519
603
|
resolveConflictPending: (id, o) => store.resolveConflictPending(id, o),
|
|
520
|
-
countConflictPending: () => store.countConflictPending()
|
|
604
|
+
countConflictPending: () => store.countConflictPending(),
|
|
605
|
+
// Entity gene (v0.3.0) passthroughs for the autoDream apply path
|
|
606
|
+
// (applyDecisions): records supersedes relations after an update and
|
|
607
|
+
// migrates entity_attrs on merge. Bookkeeping writes like the audit
|
|
608
|
+
// passthroughs above — never write-hook-triggering memory mutations.
|
|
609
|
+
saveRelation: (r) => store.saveRelation(r),
|
|
610
|
+
getAttrsByMemory: (id) => store.getAttrsByMemory(id),
|
|
611
|
+
migrateAttrsToMemory: (fromId, toId, now) => store.migrateAttrsToMemory(fromId, toId, now)
|
|
521
612
|
};
|
|
522
613
|
}
|