dsh-prime-memory 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.en.md +28 -0
- package/CHANGELOG.ja.md +30 -0
- package/CHANGELOG.ko.md +30 -0
- package/CHANGELOG.md +1220 -0
- package/ENGINEERING-NOTES.md +452 -0
- package/INSTALL.en.md +92 -0
- package/INSTALL.ja.md +92 -0
- package/INSTALL.ko.md +92 -0
- package/INSTALL.md +92 -0
- package/LICENSE +21 -0
- package/README.en.md +458 -0
- package/README.ja.md +306 -0
- package/README.ko.md +306 -0
- package/README.md +425 -0
- package/assets/changelog/0.8.10/01-write-only-pill.png +0 -0
- package/assets/changelog/0.8.9/01-panel.png +0 -0
- package/assets/changelog/0.8.9/02-halo.png +0 -0
- package/assets/changelog/0.8.9/03-layer-segmented-panel.png +0 -0
- package/assets/changelog/0.8.9/04-layer-l1-panel.png +0 -0
- package/assets/img/EmbeddingSource.png +0 -0
- package/assets/img/Hero.png +0 -0
- package/assets/img/Layers.png +0 -0
- package/assets/img/MemoryTools.png +0 -0
- package/assets/img/Modes.png +0 -0
- package/assets/img/ToolTrajectory.png +0 -0
- package/assets/img/ui-dark.jpg +0 -0
- package/assets/img/ui-light.jpg +0 -0
- package/assets/readme/bench-dialog.svg +70 -0
- package/assets/readme/bench-workflow.svg +79 -0
- package/assets/readme/flow.svg +189 -0
- package/assets/readme/storage.svg +115 -0
- package/cordis.patch.yml +16 -0
- package/dist/bench-control.d.ts +34 -0
- package/dist/bench-control.js +16 -0
- package/dist/client.js +4293 -0
- package/dist/config.d.ts +683 -0
- package/dist/config.js +129 -0
- package/dist/contract.d.ts +820 -0
- package/dist/contract.js +1 -0
- package/dist/embedding-worker.cjs +176 -0
- package/dist/graph/apply.d.ts +37 -0
- package/dist/graph/apply.js +270 -0
- package/dist/graph/constraints.d.ts +47 -0
- package/dist/graph/constraints.js +38 -0
- package/dist/graph/search.d.ts +16 -0
- package/dist/graph/search.js +115 -0
- package/dist/graph/types.d.ts +142 -0
- package/dist/graph/types.js +14 -0
- package/dist/hooks/capture.d.ts +32 -0
- package/dist/hooks/capture.js +194 -0
- package/dist/hooks/recall.d.ts +63 -0
- package/dist/hooks/recall.js +429 -0
- package/dist/index.d.ts +534 -0
- package/dist/index.js +344 -0
- package/dist/llm-usage.d.ts +26 -0
- package/dist/llm-usage.js +37 -0
- package/dist/llm.d.ts +153 -0
- package/dist/llm.js +530 -0
- package/dist/pipeline/graph.d.ts +35 -0
- package/dist/pipeline/graph.js +104 -0
- package/dist/pipeline/l1.d.ts +19 -0
- package/dist/pipeline/l1.js +271 -0
- package/dist/pipeline/l2.d.ts +13 -0
- package/dist/pipeline/l2.js +83 -0
- package/dist/pipeline/l3.d.ts +15 -0
- package/dist/pipeline/l3.js +78 -0
- package/dist/pipeline/rebuild.d.ts +61 -0
- package/dist/pipeline/rebuild.js +307 -0
- package/dist/pipeline/ruminate.d.ts +89 -0
- package/dist/pipeline/ruminate.js +298 -0
- package/dist/pipeline/runner.d.ts +167 -0
- package/dist/pipeline/runner.js +638 -0
- package/dist/pipeline/trigger.d.ts +40 -0
- package/dist/pipeline/trigger.js +75 -0
- package/dist/prompts/graph-projection.d.ts +70 -0
- package/dist/prompts/graph-projection.js +167 -0
- package/dist/prompts/l1-dedup.d.ts +22 -0
- package/dist/prompts/l1-dedup.js +251 -0
- package/dist/prompts/l1-extraction.d.ts +22 -0
- package/dist/prompts/l1-extraction.js +457 -0
- package/dist/prompts/persona.d.ts +23 -0
- package/dist/prompts/persona.js +240 -0
- package/dist/prompts/scene.d.ts +32 -0
- package/dist/prompts/scene.js +414 -0
- package/dist/runtime-package-lock.json +982 -0
- package/dist/settings.d.ts +50 -0
- package/dist/settings.js +355 -0
- package/dist/stats.d.ts +109 -0
- package/dist/stats.js +929 -0
- package/dist/store/bm25.d.ts +19 -0
- package/dist/store/bm25.js +63 -0
- package/dist/store/cost-ledger.d.ts +75 -0
- package/dist/store/cost-ledger.js +171 -0
- package/dist/store/download-queue.d.ts +79 -0
- package/dist/store/download-queue.js +424 -0
- package/dist/store/embedding-source.d.ts +118 -0
- package/dist/store/embedding-source.js +443 -0
- package/dist/store/embedding.d.ts +90 -0
- package/dist/store/embedding.js +206 -0
- package/dist/store/graph-store.d.ts +94 -0
- package/dist/store/graph-store.js +641 -0
- package/dist/store/l0.d.ts +40 -0
- package/dist/store/l0.js +197 -0
- package/dist/store/l1.d.ts +93 -0
- package/dist/store/l1.js +297 -0
- package/dist/store/local-embedding.d.ts +89 -0
- package/dist/store/local-embedding.js +227 -0
- package/dist/store/model-catalog.d.ts +48 -0
- package/dist/store/model-catalog.js +81 -0
- package/dist/store/occupancy.d.ts +30 -0
- package/dist/store/occupancy.js +134 -0
- package/dist/store/pending.d.ts +36 -0
- package/dist/store/pending.js +103 -0
- package/dist/store/persona.d.ts +15 -0
- package/dist/store/persona.js +60 -0
- package/dist/store/recall-dedupe.d.ts +26 -0
- package/dist/store/recall-dedupe.js +138 -0
- package/dist/store/runtime-installer.d.ts +59 -0
- package/dist/store/runtime-installer.js +243 -0
- package/dist/store/scenes.d.ts +24 -0
- package/dist/store/scenes.js +160 -0
- package/dist/store/search-utils.d.ts +38 -0
- package/dist/store/search-utils.js +100 -0
- package/dist/store/session-modes.d.ts +35 -0
- package/dist/store/session-modes.js +144 -0
- package/dist/store/sqlite.d.ts +246 -0
- package/dist/store/sqlite.js +1491 -0
- package/dist/store/state.d.ts +41 -0
- package/dist/store/state.js +72 -0
- package/dist/token-cost.d.ts +23 -0
- package/dist/token-cost.js +185 -0
- package/dist/tools/index.d.ts +34 -0
- package/dist/tools/index.js +758 -0
- package/dist/types.d.ts +139 -0
- package/dist/types.js +38 -0
- package/dist/util/context-occupancy.d.ts +68 -0
- package/dist/util/context-occupancy.js +92 -0
- package/dist/util/filelog.d.ts +6 -0
- package/dist/util/filelog.js +108 -0
- package/dist/util/io.d.ts +18 -0
- package/dist/util/io.js +97 -0
- package/dist/util/recall-budget.d.ts +31 -0
- package/dist/util/recall-budget.js +84 -0
- package/dist/util/sanitize.d.ts +11 -0
- package/dist/util/sanitize.js +67 -0
- package/dist/util/text.d.ts +16 -0
- package/dist/util/text.js +61 -0
- package/dist/util/tokenizer.d.ts +9 -0
- package/dist/util/tokenizer.js +50 -0
- package/dsh.plugin.json +22 -0
- package/package.json +118 -0
package/dist/contract.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 本地嵌入 worker:transformers.js 的模型加载与推理在本线程执行,主线程只
|
|
3
|
+
* 投递文本、收回向量。onnxruntime-node 的 run/loadModel 是 setImmediate 回调里
|
|
4
|
+
* 的同步调用(Promise 包装不卸载计算),留在主线程会冻结宿主事件循环——
|
|
5
|
+
* dsh 页面一切交互无响应数秒的根因。
|
|
6
|
+
*
|
|
7
|
+
* 协议(与 src/store/local-embedding.ts 的 RealWorkerChannel 配对;自增 id):
|
|
8
|
+
* - 入向:{id,type:'ping'} | {id,type:'warmup'} | {id,type:'embed',texts,priority}
|
|
9
|
+
* - 出向:{id,ok:true,type:'pong'|'ready'|'embedded',vectors?}(向量经
|
|
10
|
+
* transfer 列表零拷贝回传)| {id,ok:false,stage:'load'|'infer',error}
|
|
11
|
+
* | {type:'fatal',error}
|
|
12
|
+
*
|
|
13
|
+
* 调度:单线程串行。embedBatch 拆成逐条 job、条间 setImmediate 让路——
|
|
14
|
+
* 单条请求(priority,即召回 query)unshift 插队,最坏只等"正在算的那一条",
|
|
15
|
+
* 不会被 reindex 批次(16/32 条 ≈5-10s)堵在队尾。
|
|
16
|
+
*
|
|
17
|
+
* 信任边界(沿 AGENTS.md 审查记录):runtime/ 目录在数据目录信任边界内,
|
|
18
|
+
* 本线程按其 package.json createRequire 加载 transformers——对数据目录有写
|
|
19
|
+
* 权限的进程理论上可植入恶意模块;模型文件逐文件 sha256 锁定,运行时代码
|
|
20
|
+
* 无同级防护,完整修复需安装指纹设计,暂以文档化假设为准。
|
|
21
|
+
*/
|
|
22
|
+
'use strict';
|
|
23
|
+
|
|
24
|
+
const { parentPort, workerData } = require('node:worker_threads');
|
|
25
|
+
const path = require('node:path');
|
|
26
|
+
const { createRequire } = require('node:module');
|
|
27
|
+
|
|
28
|
+
/** 错误消息归一化(Error 取 message,其余 String 化)。 */
|
|
29
|
+
function errMsg(err) {
|
|
30
|
+
return err && err.message ? err.message : String(err);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const cfg = workerData || {};
|
|
34
|
+
const RUNTIME_DIR = typeof cfg.runtimeDir === 'string' ? cfg.runtimeDir : '';
|
|
35
|
+
const MODEL_DIR = typeof cfg.modelDir === 'string' ? cfg.modelDir : '';
|
|
36
|
+
const POOLING = cfg.pooling === 'cls' ? 'cls' : 'mean';
|
|
37
|
+
const DTYPE = typeof cfg.dtype === 'string' ? cfg.dtype : 'q8';
|
|
38
|
+
const MAX_INPUT_CHARS = cfg.maxInputChars > 0 ? cfg.maxInputChars : 5000;
|
|
39
|
+
|
|
40
|
+
let extractor = null; // pipeline('feature-extraction', …)
|
|
41
|
+
let loadState = 'idle'; // idle | loading | ready | failed
|
|
42
|
+
let loadPromise = null;
|
|
43
|
+
let loadError = null;
|
|
44
|
+
|
|
45
|
+
// ── 模型懒加载(首请求触发;失败后经 warmup 可重试,embed 请求失败快返回) ──
|
|
46
|
+
function ensureLoaded() {
|
|
47
|
+
if (loadState === 'ready') return Promise.resolve();
|
|
48
|
+
if (loadState === 'loading' && loadPromise) return loadPromise;
|
|
49
|
+
loadState = 'loading';
|
|
50
|
+
loadError = null;
|
|
51
|
+
loadPromise = (async () => {
|
|
52
|
+
try {
|
|
53
|
+
if (!RUNTIME_DIR || !MODEL_DIR) throw new Error('workerData 缺少 runtimeDir/modelDir');
|
|
54
|
+
const req = createRequire(path.join(RUNTIME_DIR, 'package.json'));
|
|
55
|
+
const mod = req('@huggingface/transformers');
|
|
56
|
+
if (mod.env) {
|
|
57
|
+
mod.env.allowRemoteModels = false;
|
|
58
|
+
if (mod.env.allowLocalModels !== undefined) mod.env.allowLocalModels = true;
|
|
59
|
+
}
|
|
60
|
+
extractor = await mod.pipeline('feature-extraction', MODEL_DIR, { dtype: DTYPE });
|
|
61
|
+
loadState = 'ready';
|
|
62
|
+
} catch (err) {
|
|
63
|
+
loadState = 'failed';
|
|
64
|
+
loadError = errMsg(err);
|
|
65
|
+
loadPromise = null; // 失败后下一次 warmup 重新走加载
|
|
66
|
+
throw err;
|
|
67
|
+
}
|
|
68
|
+
})();
|
|
69
|
+
return loadPromise;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// ── 推理队列(逐条 job;priority 插队;条间让路) ──
|
|
73
|
+
const queue = []; // [{ request, index, text }]
|
|
74
|
+
let pumping = false;
|
|
75
|
+
|
|
76
|
+
function postReply(msg, transfer) {
|
|
77
|
+
parentPort.postMessage(msg, transfer || []);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** 请求全部条目完成 → 回执(transfer 向量缓冲零拷贝)。 */
|
|
81
|
+
function settle(request) {
|
|
82
|
+
if (request.failed || request.remaining !== 0) return;
|
|
83
|
+
postReply(
|
|
84
|
+
{ id: request.id, ok: true, type: 'embedded', vectors: request.vectors },
|
|
85
|
+
request.vectors.map((v) => v.buffer),
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** 首个失败即回执错误,并移除该请求尚未开跑的条目(正在跑的至多 1 条,算完即弃)。 */
|
|
90
|
+
function failRequest(request, error, stage) {
|
|
91
|
+
if (request.failed) return;
|
|
92
|
+
request.failed = true;
|
|
93
|
+
for (let i = queue.length - 1; i >= 0; i--) {
|
|
94
|
+
if (queue[i].request === request) queue.splice(i, 1);
|
|
95
|
+
}
|
|
96
|
+
postReply({ id: request.id, ok: false, stage: stage, error: errMsg(error) });
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async function pump() {
|
|
100
|
+
if (pumping) return;
|
|
101
|
+
pumping = true;
|
|
102
|
+
try {
|
|
103
|
+
while (queue.length > 0) {
|
|
104
|
+
const job = queue.shift();
|
|
105
|
+
if (job.request.failed) continue;
|
|
106
|
+
try {
|
|
107
|
+
const result = await extractor([job.text.slice(0, MAX_INPUT_CHARS)], { pooling: POOLING, normalize: true });
|
|
108
|
+
const data = result[0] && result[0].data;
|
|
109
|
+
job.request.vectors[job.index] = data instanceof Float32Array ? data : new Float32Array(data);
|
|
110
|
+
job.request.remaining -= 1;
|
|
111
|
+
settle(job.request);
|
|
112
|
+
} catch (err) {
|
|
113
|
+
failRequest(job.request, err, 'infer');
|
|
114
|
+
}
|
|
115
|
+
// 条间让路:交还本线程事件循环,新消息(含 priority 插队)可在批间被处理
|
|
116
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
117
|
+
}
|
|
118
|
+
} finally {
|
|
119
|
+
pumping = false;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function handleEmbed(msg) {
|
|
124
|
+
const texts = Array.isArray(msg.texts) ? msg.texts : [];
|
|
125
|
+
if (texts.length === 0) {
|
|
126
|
+
postReply({ id: msg.id, ok: true, type: 'embedded', vectors: [] });
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
const request = { id: msg.id, vectors: new Array(texts.length), remaining: texts.length, failed: false };
|
|
130
|
+
// 单条请求(召回 query)插队到队首;批次追加到队尾(批内条目间仍会被后来的插队)
|
|
131
|
+
const priority = msg.priority === true && texts.length === 1;
|
|
132
|
+
for (let i = 0; i < texts.length; i++) {
|
|
133
|
+
const job = { request: request, index: i, text: String(texts[i] == null ? '' : texts[i]) };
|
|
134
|
+
if (priority) queue.unshift(job);
|
|
135
|
+
else queue.push(job);
|
|
136
|
+
}
|
|
137
|
+
void pump();
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
parentPort.on('message', (msg) => {
|
|
141
|
+
if (!msg || typeof msg.id !== 'number' || typeof msg.type !== 'string') return;
|
|
142
|
+
if (msg.type === 'ping') {
|
|
143
|
+
postReply({ id: msg.id, ok: true, type: 'pong' });
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
if (msg.type === 'warmup') {
|
|
147
|
+
ensureLoaded().then(
|
|
148
|
+
() => postReply({ id: msg.id, ok: true, type: 'ready' }),
|
|
149
|
+
() => postReply({ id: msg.id, ok: false, stage: 'load', error: loadError ?? '模型加载失败' }),
|
|
150
|
+
);
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
if (msg.type === 'embed') {
|
|
154
|
+
void (async () => {
|
|
155
|
+
try {
|
|
156
|
+
await ensureLoaded();
|
|
157
|
+
} catch {
|
|
158
|
+
postReply({ id: msg.id, ok: false, stage: 'load', error: loadError ?? '模型加载失败' });
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
handleEmbed(msg);
|
|
162
|
+
})();
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
// 未捕获异常:尽力通知主线程后强制退出(主线程的 exit 处理会拒绝全部 pending
|
|
168
|
+
// 并转入 failed 态)。不保持半死状态——半死的 worker 只会让调用方挂到超时。
|
|
169
|
+
process.on('uncaughtException', (err) => {
|
|
170
|
+
try {
|
|
171
|
+
postReply({ type: 'fatal', error: errMsg(err) });
|
|
172
|
+
} catch {
|
|
173
|
+
/* 端口已坏,exit 处理兜底 */
|
|
174
|
+
}
|
|
175
|
+
setTimeout(() => process.exit(1), 50);
|
|
176
|
+
});
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { MemoryRecord } from '../types.js';
|
|
2
|
+
import type { GraphEdge, GraphNode, GraphProjectionResult } from './types.js';
|
|
3
|
+
export interface ApplyGraphProjectionOptions {
|
|
4
|
+
/** 可变异的合并 scope(现有节点/边;调用方决定 scope 大小)。 */
|
|
5
|
+
nodes: GraphNode[];
|
|
6
|
+
edges: GraphEdge[];
|
|
7
|
+
/** 本批认领的记录(族判定与时间锚的证据来源)。 */
|
|
8
|
+
records: readonly MemoryRecord[];
|
|
9
|
+
result: GraphProjectionResult;
|
|
10
|
+
allowedRecordIds: ReadonlySet<string>;
|
|
11
|
+
/** 本次投影时刻(ISO;时间锚兜底与 updatedAt 的时间基准)。 */
|
|
12
|
+
now: string;
|
|
13
|
+
idFactory: (prefix: 'node' | 'edge' | 'gfact') => string;
|
|
14
|
+
}
|
|
15
|
+
export interface ApplyGraphProjectionOutcome {
|
|
16
|
+
nodeIds: string[];
|
|
17
|
+
edgeIds: string[];
|
|
18
|
+
/** 因无来源/形状非法被丢弃的提案条数(节点+fact+边;诊断日志用)。 */
|
|
19
|
+
dropped: number;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* 实体名归一(NFKC + 小写 + 去空白/下划线/连字符):全角/半角、大小写、
|
|
23
|
+
* "张 三"与"张三"、"AI-agent"与"aiagent"视作同一实体的拼写变体。
|
|
24
|
+
* 归一后空串返回原串(防止空名互相合并)。
|
|
25
|
+
*/
|
|
26
|
+
export declare function normalizeEntityName(name: string): string;
|
|
27
|
+
/**
|
|
28
|
+
* 时间锚:有效期为先,提及/入库时间为后。
|
|
29
|
+
*
|
|
30
|
+
* 逐条记录按六级证据取最晚,任何一级无法解析(缺字段/非法日期)即落到下一级:
|
|
31
|
+
* validTo → validFrom(时间增强列,新写入路径)→ metadata.activity_end_time →
|
|
32
|
+
* metadata.activity_start_time(列迁移前的存量数据)→ timestamps 最新 → createdAt。
|
|
33
|
+
* 全部无证据才用 fallback(now),绝不猜测。
|
|
34
|
+
*/
|
|
35
|
+
export declare function anchorTimeFromRecords(records: readonly MemoryRecord[], fallbackIso: string): string;
|
|
36
|
+
/** 应用一次投影提案(硬校验 + 消歧 + supersede + 状态重建)。 */
|
|
37
|
+
export declare function applyGraphProjection(options: ApplyGraphProjectionOptions): ApplyGraphProjectionOutcome;
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 投影提案硬校验与落库语义(纯函数,无 I/O)。
|
|
3
|
+
*
|
|
4
|
+
* 这是图谱最核心的守门代码,唯一不变量:**零无来源事实**——每个节点/fact/边的
|
|
5
|
+
* sourceRecordIds 必须非空且全部属于本 job 认领的记录批次,否则整条提案静默丢弃。
|
|
6
|
+
* 图谱里不存在无法回溯到 L1 记录(进而到 JSONL 事实源)的结论。
|
|
7
|
+
*
|
|
8
|
+
* 其余语义:
|
|
9
|
+
* - 实体消歧:normalizeEntityName(NFKC + 小写 + 去空白/下划线/连字符)归一后同名
|
|
10
|
+
* 且类型一致才合并(别名累积去重);子串包含不算同一实体;
|
|
11
|
+
* - 状态历史:同 key 的新 fact 把旧 active fact 标 superseded 并闭合 validTo;
|
|
12
|
+
* 同 (from,relation) 指向新目标的旧边同样 supersede——保留时间区间,不覆盖历史;
|
|
13
|
+
* - currentState 只由 active facts 重建;
|
|
14
|
+
* - 时间锚四级链:activity_start_time → activity_end_time → timestamps 最新 →
|
|
15
|
+
* createdAt,跨来源取最晚(ISO);无任何证据时才落到 now,不猜日期。
|
|
16
|
+
*
|
|
17
|
+
* 函数直接变异传入的 nodes/edges(merge scope),返回 touched id 集合——调用方
|
|
18
|
+
* (GraphStore.complete)在单事务内 load scope → apply → 写回,LLM 调用永远
|
|
19
|
+
* 不发生在该事务内。
|
|
20
|
+
*/
|
|
21
|
+
import { familyForType } from '../types.js';
|
|
22
|
+
import { GRAPH_CAP, GRAPH_NODE_TYPE_SET, GRAPH_SOURCE_RECORDS_CAP, GRAPH_STATUS_SET, } from './constraints.js';
|
|
23
|
+
/** 字符串字段归一:trim、折叠空白、截断;非字符串一律空串。 */
|
|
24
|
+
function textOf(value, limit) {
|
|
25
|
+
return typeof value === 'string' ? value.trim().replace(/\s+/g, ' ').slice(0, limit) : '';
|
|
26
|
+
}
|
|
27
|
+
/** 字符串数组字段归一:逐项截断去空、跨项去重、截到条数上限。 */
|
|
28
|
+
function stringsOf(value, itemCap, countCap) {
|
|
29
|
+
if (!Array.isArray(value))
|
|
30
|
+
return [];
|
|
31
|
+
const out = [];
|
|
32
|
+
for (const item of value) {
|
|
33
|
+
const t = textOf(item, itemCap);
|
|
34
|
+
if (t && !out.includes(t))
|
|
35
|
+
out.push(t);
|
|
36
|
+
if (out.length >= countCap)
|
|
37
|
+
break;
|
|
38
|
+
}
|
|
39
|
+
return out;
|
|
40
|
+
}
|
|
41
|
+
/** 置信度钳制:非法值回到 0.8 中性档(提案没填 = 模型默认自信度)。 */
|
|
42
|
+
function clampConfidence(value) {
|
|
43
|
+
return typeof value === 'number' && Number.isFinite(value) ? Math.max(0, Math.min(1, value)) : 0.8;
|
|
44
|
+
}
|
|
45
|
+
/** 状态归一:非法值回 active(四态之外的表达一律不采信)。 */
|
|
46
|
+
function clampStatus(value) {
|
|
47
|
+
return typeof value === 'string' && GRAPH_STATUS_SET.has(value) ? value : 'active';
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* 实体名归一(NFKC + 小写 + 去空白/下划线/连字符):全角/半角、大小写、
|
|
51
|
+
* "张 三"与"张三"、"AI-agent"与"aiagent"视作同一实体的拼写变体。
|
|
52
|
+
* 归一后空串返回原串(防止空名互相合并)。
|
|
53
|
+
*/
|
|
54
|
+
export function normalizeEntityName(name) {
|
|
55
|
+
const normalized = name.normalize('NFKC').toLowerCase().replace(/[\s_-]+/g, '');
|
|
56
|
+
return normalized || name;
|
|
57
|
+
}
|
|
58
|
+
/** 记录族兜底(record.family 缺失时按 type 前缀推导,与写入侧同一兜底链)。 */
|
|
59
|
+
function recordFamily(r) {
|
|
60
|
+
return r.family ?? familyForType(r.type);
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* 时间锚:有效期为先,提及/入库时间为后。
|
|
64
|
+
*
|
|
65
|
+
* 逐条记录按六级证据取最晚,任何一级无法解析(缺字段/非法日期)即落到下一级:
|
|
66
|
+
* validTo → validFrom(时间增强列,新写入路径)→ metadata.activity_end_time →
|
|
67
|
+
* metadata.activity_start_time(列迁移前的存量数据)→ timestamps 最新 → createdAt。
|
|
68
|
+
* 全部无证据才用 fallback(now),绝不猜测。
|
|
69
|
+
*/
|
|
70
|
+
export function anchorTimeFromRecords(records, fallbackIso) {
|
|
71
|
+
let latest = Number.NaN;
|
|
72
|
+
const consider = (raw) => {
|
|
73
|
+
const t = typeof raw === 'number' ? raw : typeof raw === 'string' ? Date.parse(raw) : Number.NaN;
|
|
74
|
+
if (Number.isFinite(t) && t > 0 && (Number.isNaN(latest) || t > latest))
|
|
75
|
+
latest = t;
|
|
76
|
+
};
|
|
77
|
+
for (const r of records) {
|
|
78
|
+
// 时间增强列优先(新写入路径),再回落 metadata.activity_*(列迁移前的存量数据)
|
|
79
|
+
consider(r.validTo);
|
|
80
|
+
consider(r.validFrom);
|
|
81
|
+
const meta = r.metadata;
|
|
82
|
+
consider(meta?.activity_start_time);
|
|
83
|
+
consider(meta?.activity_end_time);
|
|
84
|
+
for (const t of r.timestamps ?? [])
|
|
85
|
+
consider(t);
|
|
86
|
+
consider(r.createdAt);
|
|
87
|
+
}
|
|
88
|
+
return Number.isNaN(latest) ? fallbackIso : new Date(latest).toISOString();
|
|
89
|
+
}
|
|
90
|
+
/** 来源集并集封顶(保最新:新引用追加在尾部,溢出丢头部最老引用)。 */
|
|
91
|
+
function unionSourcesCapped(old, add) {
|
|
92
|
+
const merged = [...new Set([...old, ...add])];
|
|
93
|
+
return merged.length > GRAPH_SOURCE_RECORDS_CAP ? merged.slice(-GRAPH_SOURCE_RECORDS_CAP) : merged;
|
|
94
|
+
}
|
|
95
|
+
/** 同一实体判定:类型一致 + 名称/别名归一后的集合相交(子串包含不算)。 */
|
|
96
|
+
function isSameEntity(node, name, aliases, type) {
|
|
97
|
+
if (node.type !== type)
|
|
98
|
+
return false;
|
|
99
|
+
const keys = new Set([normalizeEntityName(name), ...aliases.map(normalizeEntityName)]);
|
|
100
|
+
return [node.name, ...node.aliases].some((candidate) => keys.has(normalizeEntityName(candidate)));
|
|
101
|
+
}
|
|
102
|
+
/** 应用一次投影提案(硬校验 + 消歧 + supersede + 状态重建)。 */
|
|
103
|
+
export function applyGraphProjection(options) {
|
|
104
|
+
const { nodes, edges, records, allowedRecordIds, now, idFactory } = options;
|
|
105
|
+
const refs = new Map();
|
|
106
|
+
const touchedNodes = new Set();
|
|
107
|
+
const touchedEdges = new Set();
|
|
108
|
+
let dropped = 0;
|
|
109
|
+
/** 来源引用校验:非空且全部属于本批(任一越批 = 整条提案不可信,返回空)。 */
|
|
110
|
+
const validSources = (value) => {
|
|
111
|
+
const requested = stringsOf(value, GRAPH_CAP.ref, GRAPH_CAP.proposalSources);
|
|
112
|
+
return requested.length > 0 && requested.every((id) => allowedRecordIds.has(id)) ? requested : [];
|
|
113
|
+
};
|
|
114
|
+
const recordById = new Map(records.map((r) => [r.id, r]));
|
|
115
|
+
const familiesOf = (sourceIds) => {
|
|
116
|
+
const families = new Set();
|
|
117
|
+
for (const id of sourceIds) {
|
|
118
|
+
const r = recordById.get(id);
|
|
119
|
+
if (r)
|
|
120
|
+
families.add(recordFamily(r));
|
|
121
|
+
}
|
|
122
|
+
return [...families];
|
|
123
|
+
};
|
|
124
|
+
// ── 节点提案 ──
|
|
125
|
+
for (const raw of Array.isArray(options.result.nodes) ? options.result.nodes : []) {
|
|
126
|
+
const ref = textOf(raw?.ref, GRAPH_CAP.ref);
|
|
127
|
+
const name = textOf(raw?.name, GRAPH_CAP.name);
|
|
128
|
+
const sources = validSources(raw?.sourceRecordIds);
|
|
129
|
+
if (!ref || !name || !GRAPH_NODE_TYPE_SET.has(raw?.type) || sources.length === 0) {
|
|
130
|
+
dropped++;
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
const aliases = stringsOf(raw?.aliases, GRAPH_CAP.alias, GRAPH_CAP.aliases).filter((alias) => normalizeEntityName(alias) !== normalizeEntityName(name));
|
|
134
|
+
const tags = stringsOf(raw?.tags, GRAPH_CAP.tag, GRAPH_CAP.tags);
|
|
135
|
+
const validFrom = textOf(raw?.validFrom, GRAPH_CAP.date) || anchorTimeFromRecords(sources.map((id) => recordById.get(id)).filter(Boolean), now);
|
|
136
|
+
const validTo = textOf(raw?.validTo, GRAPH_CAP.date) || undefined;
|
|
137
|
+
const status = clampStatus(raw?.status);
|
|
138
|
+
const confidence = clampConfidence(raw?.confidence);
|
|
139
|
+
// 消歧合并:同名同型已有节点就地更新;否则新建
|
|
140
|
+
let node = nodes.find((candidate) => isSameEntity(candidate, name, aliases, raw.type));
|
|
141
|
+
if (!node) {
|
|
142
|
+
node = {
|
|
143
|
+
id: idFactory('node'),
|
|
144
|
+
name,
|
|
145
|
+
type: raw.type,
|
|
146
|
+
aliases: [],
|
|
147
|
+
currentState: '',
|
|
148
|
+
facts: [],
|
|
149
|
+
status,
|
|
150
|
+
confidence,
|
|
151
|
+
sourceRecordIds: [],
|
|
152
|
+
families: [],
|
|
153
|
+
createdAt: now,
|
|
154
|
+
updatedAt: now,
|
|
155
|
+
};
|
|
156
|
+
nodes.push(node);
|
|
157
|
+
}
|
|
158
|
+
node.aliases = [...new Set([...node.aliases, ...aliases])];
|
|
159
|
+
if (tags.length > 0)
|
|
160
|
+
node.tags = [...new Set([...(node.tags ?? []), ...tags])].slice(0, GRAPH_CAP.tags);
|
|
161
|
+
node.status = status;
|
|
162
|
+
node.confidence = confidence;
|
|
163
|
+
node.sourceRecordIds = unionSourcesCapped(node.sourceRecordIds, sources);
|
|
164
|
+
node.families = [...new Set([...node.families, ...familiesOf(sources)])];
|
|
165
|
+
// 事实集:单行状态(state)归一为「状态」fact + 显式 facts 数组
|
|
166
|
+
const stateLine = textOf(raw?.state, GRAPH_CAP.state);
|
|
167
|
+
const explicitFacts = Array.isArray(raw?.facts) ? raw.facts : [];
|
|
168
|
+
const factProposals = [
|
|
169
|
+
...(stateLine ? [{ key: '状态', value: stateLine }] : []),
|
|
170
|
+
...explicitFacts,
|
|
171
|
+
];
|
|
172
|
+
for (const fp of factProposals) {
|
|
173
|
+
const key = textOf(fp?.key, GRAPH_CAP.factKey);
|
|
174
|
+
const value = Array.isArray(fp?.value)
|
|
175
|
+
? stringsOf(fp.value, GRAPH_CAP.factValue, GRAPH_CAP.factValueItems)
|
|
176
|
+
: textOf(fp?.value, GRAPH_CAP.factValue);
|
|
177
|
+
if (!key || (Array.isArray(value) ? value.length === 0 : !value))
|
|
178
|
+
continue;
|
|
179
|
+
// fact 级来源可选:未带则继承节点提案的来源(同样过批校验)
|
|
180
|
+
const factSources = fp && 'sourceRecordIds' in fp && fp.sourceRecordIds !== undefined
|
|
181
|
+
? validSources(fp.sourceRecordIds)
|
|
182
|
+
: sources;
|
|
183
|
+
if (factSources.length === 0) {
|
|
184
|
+
dropped++;
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
// supersede:同 key 的旧 active fact 闭合(保留完整时间区间)
|
|
188
|
+
for (const old of node.facts) {
|
|
189
|
+
if (old.status === 'active' && old.key === key) {
|
|
190
|
+
old.status = 'superseded';
|
|
191
|
+
if (!old.validTo)
|
|
192
|
+
old.validTo = validFrom;
|
|
193
|
+
old.updatedAt = now;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
const fact = {
|
|
197
|
+
id: idFactory('gfact'),
|
|
198
|
+
key,
|
|
199
|
+
value,
|
|
200
|
+
status,
|
|
201
|
+
validFrom,
|
|
202
|
+
...(validTo ? { validTo } : {}),
|
|
203
|
+
confidence,
|
|
204
|
+
sourceRecordIds: unionSourcesCapped([], factSources),
|
|
205
|
+
createdAt: now,
|
|
206
|
+
updatedAt: now,
|
|
207
|
+
};
|
|
208
|
+
node.facts.push(fact);
|
|
209
|
+
}
|
|
210
|
+
// currentState 只由 active facts 重建(superseded/disputed/archived 不进状态串)
|
|
211
|
+
node.currentState = node.facts
|
|
212
|
+
.filter((f) => f.status === 'active')
|
|
213
|
+
.map((f) => `${f.key}: ${Array.isArray(f.value) ? f.value.join('、') : f.value}`)
|
|
214
|
+
.join('\n');
|
|
215
|
+
node.updatedAt = now;
|
|
216
|
+
refs.set(ref, node);
|
|
217
|
+
touchedNodes.add(node.id);
|
|
218
|
+
}
|
|
219
|
+
// ── 边提案 ──
|
|
220
|
+
for (const raw of Array.isArray(options.result.edges) ? options.result.edges : []) {
|
|
221
|
+
const from = refs.get(textOf(raw?.fromRef, GRAPH_CAP.ref));
|
|
222
|
+
const to = refs.get(textOf(raw?.toRef, GRAPH_CAP.ref));
|
|
223
|
+
const relation = textOf(raw?.relation, GRAPH_CAP.relation);
|
|
224
|
+
const sources = validSources(raw?.sourceRecordIds);
|
|
225
|
+
if (!from || !to || from.id === to.id || !relation || sources.length === 0) {
|
|
226
|
+
dropped++;
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
const status = clampStatus(raw?.status);
|
|
230
|
+
const confidence = clampConfidence(raw?.confidence);
|
|
231
|
+
const validFrom = textOf(raw?.validFrom, GRAPH_CAP.date) || anchorTimeFromRecords(sources.map((id) => recordById.get(id)).filter(Boolean), now);
|
|
232
|
+
const validTo = textOf(raw?.validTo, GRAPH_CAP.date) || undefined;
|
|
233
|
+
// supersede:同 (from,relation) 指向新目标 → 旧 active 边闭合
|
|
234
|
+
for (const old of edges) {
|
|
235
|
+
if (old.status === 'active' && old.fromNodeId === from.id && old.relation === relation && old.toNodeId !== to.id) {
|
|
236
|
+
old.status = 'superseded';
|
|
237
|
+
if (!old.validTo)
|
|
238
|
+
old.validTo = validFrom;
|
|
239
|
+
old.updatedAt = now;
|
|
240
|
+
touchedEdges.add(old.id);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
// 同 (from,to,relation,status) 已有边 → 合并来源;否则新建
|
|
244
|
+
const existing = edges.find((e) => e.fromNodeId === from.id && e.toNodeId === to.id && e.relation === relation && e.status === status);
|
|
245
|
+
if (existing) {
|
|
246
|
+
existing.sourceRecordIds = unionSourcesCapped(existing.sourceRecordIds, sources);
|
|
247
|
+
existing.confidence = confidence;
|
|
248
|
+
existing.updatedAt = now;
|
|
249
|
+
touchedEdges.add(existing.id);
|
|
250
|
+
}
|
|
251
|
+
else {
|
|
252
|
+
const edge = {
|
|
253
|
+
id: idFactory('edge'),
|
|
254
|
+
fromNodeId: from.id,
|
|
255
|
+
toNodeId: to.id,
|
|
256
|
+
relation,
|
|
257
|
+
status,
|
|
258
|
+
validFrom,
|
|
259
|
+
...(validTo ? { validTo } : {}),
|
|
260
|
+
confidence,
|
|
261
|
+
sourceRecordIds: unionSourcesCapped([], sources),
|
|
262
|
+
createdAt: now,
|
|
263
|
+
updatedAt: now,
|
|
264
|
+
};
|
|
265
|
+
edges.push(edge);
|
|
266
|
+
touchedEdges.add(edge.id);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
return { nodeIds: [...touchedNodes], edgeIds: [...touchedEdges], dropped };
|
|
270
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 图谱域约束常量的单一事实源。
|
|
3
|
+
*
|
|
4
|
+
* 提示词(src/prompts/graph-projection.ts)与校验(src/graph/apply.ts)两侧都从
|
|
5
|
+
* 这里内插/引用——防"提示词说一套、校验执行另一套"的漂移。上限值从本项目自身
|
|
6
|
+
* 的序列化约束推导(L1 content 上限、fact 行渲染宽度、上下文装配预算),不对照
|
|
7
|
+
* 外部实现抄数。
|
|
8
|
+
*/
|
|
9
|
+
import type { GraphNodeType, GraphRecordStatus } from './types.js';
|
|
10
|
+
/** 节点类型词汇表(提示词枚举与 apply 校验共用;超集即丢弃提案)。 */
|
|
11
|
+
export declare const GRAPH_NODE_TYPES: readonly GraphNodeType[];
|
|
12
|
+
export declare const GRAPH_NODE_TYPE_SET: ReadonlySet<string>;
|
|
13
|
+
/** 记录生命周期四态(节点/fact/边通用)。 */
|
|
14
|
+
export declare const GRAPH_STATUSES: readonly GraphRecordStatus[];
|
|
15
|
+
export declare const GRAPH_STATUS_SET: ReadonlySet<string>;
|
|
16
|
+
/** 规范中文关系词(提示词给枚举;校验不做白名单——关系是开放词表,只做长度钳制)。 */
|
|
17
|
+
export declare const GRAPH_RELATION_WORDS: readonly ["使用", "属于", "创建", "参与", "贡献", "依赖", "位于", "相关"];
|
|
18
|
+
/** 提案字段上限(校验侧截断/丢弃与提示词内插共用同一组数字)。 */
|
|
19
|
+
export declare const GRAPH_CAP: {
|
|
20
|
+
/** 提案内临时引用(ref)长度。 */
|
|
21
|
+
readonly ref: 120;
|
|
22
|
+
/** 节点显示名。 */
|
|
23
|
+
readonly name: 160;
|
|
24
|
+
/** 单个别名/标签/关系词。 */
|
|
25
|
+
readonly alias: 80;
|
|
26
|
+
readonly tag: 60;
|
|
27
|
+
readonly relation: 60;
|
|
28
|
+
/** 节点状态行与单条 fact 值(与召回注入的单条 maxCharsPerMemory 默认 500 同量级,给足余量)。 */
|
|
29
|
+
readonly state: 1200;
|
|
30
|
+
readonly factKey: 120;
|
|
31
|
+
readonly factValue: 1200;
|
|
32
|
+
/** fact 值为数组时的元素数与每元素长度。 */
|
|
33
|
+
readonly factValueItems: 40;
|
|
34
|
+
/** 单节点提案的别名/标签条数上限。 */
|
|
35
|
+
readonly aliases: 20;
|
|
36
|
+
readonly tags: 12;
|
|
37
|
+
/** 单提案允许引用的来源记录数(正常远小于批大小;钳到批上限的整数倍防畸形输出)。 */
|
|
38
|
+
readonly proposalSources: 64;
|
|
39
|
+
/** 单个日期字符串(validFrom/validTo)长度。 */
|
|
40
|
+
readonly date: 80;
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* 节点累计 sourceRecordIds 的并集封顶(保最新):图谱可重建,来源集只增会让
|
|
44
|
+
* 长寿命节点(如"用户本人")的来源数组无界增长;封顶后丢的是最老的来源引用,
|
|
45
|
+
* 状态仍由 active facts 决定,不影响可追溯性主体。
|
|
46
|
+
*/
|
|
47
|
+
export declare const GRAPH_SOURCE_RECORDS_CAP = 128;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/** 节点类型词汇表(提示词枚举与 apply 校验共用;超集即丢弃提案)。 */
|
|
2
|
+
export const GRAPH_NODE_TYPES = ['person', 'project', 'organization', 'tool', 'place'];
|
|
3
|
+
export const GRAPH_NODE_TYPE_SET = new Set(GRAPH_NODE_TYPES);
|
|
4
|
+
/** 记录生命周期四态(节点/fact/边通用)。 */
|
|
5
|
+
export const GRAPH_STATUSES = ['active', 'superseded', 'disputed', 'archived'];
|
|
6
|
+
export const GRAPH_STATUS_SET = new Set(GRAPH_STATUSES);
|
|
7
|
+
/** 规范中文关系词(提示词给枚举;校验不做白名单——关系是开放词表,只做长度钳制)。 */
|
|
8
|
+
export const GRAPH_RELATION_WORDS = ['使用', '属于', '创建', '参与', '贡献', '依赖', '位于', '相关'];
|
|
9
|
+
/** 提案字段上限(校验侧截断/丢弃与提示词内插共用同一组数字)。 */
|
|
10
|
+
export const GRAPH_CAP = {
|
|
11
|
+
/** 提案内临时引用(ref)长度。 */
|
|
12
|
+
ref: 120,
|
|
13
|
+
/** 节点显示名。 */
|
|
14
|
+
name: 160,
|
|
15
|
+
/** 单个别名/标签/关系词。 */
|
|
16
|
+
alias: 80,
|
|
17
|
+
tag: 60,
|
|
18
|
+
relation: 60,
|
|
19
|
+
/** 节点状态行与单条 fact 值(与召回注入的单条 maxCharsPerMemory 默认 500 同量级,给足余量)。 */
|
|
20
|
+
state: 1_200,
|
|
21
|
+
factKey: 120,
|
|
22
|
+
factValue: 1_200,
|
|
23
|
+
/** fact 值为数组时的元素数与每元素长度。 */
|
|
24
|
+
factValueItems: 40,
|
|
25
|
+
/** 单节点提案的别名/标签条数上限。 */
|
|
26
|
+
aliases: 20,
|
|
27
|
+
tags: 12,
|
|
28
|
+
/** 单提案允许引用的来源记录数(正常远小于批大小;钳到批上限的整数倍防畸形输出)。 */
|
|
29
|
+
proposalSources: 64,
|
|
30
|
+
/** 单个日期字符串(validFrom/validTo)长度。 */
|
|
31
|
+
date: 80,
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* 节点累计 sourceRecordIds 的并集封顶(保最新):图谱可重建,来源集只增会让
|
|
35
|
+
* 长寿命节点(如"用户本人")的来源数组无界增长;封顶后丢的是最老的来源引用,
|
|
36
|
+
* 状态仍由 active facts 决定,不影响可追溯性主体。
|
|
37
|
+
*/
|
|
38
|
+
export const GRAPH_SOURCE_RECORDS_CAP = 128;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { GraphEdge, GraphNode, GraphNodeSearchResult } from './types.js';
|
|
2
|
+
/** 字段权重(检索命中计分;名称最高、关系词最低)。 */
|
|
3
|
+
export declare const GRAPH_FIELD_WEIGHTS: {
|
|
4
|
+
readonly name: 6;
|
|
5
|
+
readonly aliases: 5;
|
|
6
|
+
readonly tags: 5;
|
|
7
|
+
readonly currentState: 4;
|
|
8
|
+
readonly facts: 4;
|
|
9
|
+
readonly relations: 3;
|
|
10
|
+
readonly type: 2;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* 图谱节点检索:返回按 score 降序的命中(最多 limit 条;limit 钳制 1~20)。
|
|
14
|
+
* 候选节点 = active | disputed(superseded/archived 不进检索)。
|
|
15
|
+
*/
|
|
16
|
+
export declare function searchGraphNodes(nodes: readonly GraphNode[], edges: readonly GraphEdge[], query: string, limit: number): GraphNodeSearchResult[];
|