@a9i5k4/dsh-auto-memory 0.1.28 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +352 -266
- package/README.zh-CN.md +370 -264
- package/cordis.patch.yml +9 -9
- package/lib/activation-host.js +455 -0
- package/lib/activation-inbox-state.js +261 -0
- package/lib/activation-inbox.js +426 -0
- package/lib/client.js +1395 -50
- package/lib/context-bridge.js +619 -0
- package/lib/context-host.js +712 -0
- package/lib/context-sink-python.js +90 -0
- package/lib/episodic-store.js +316 -0
- package/lib/evidence-store.js +272 -0
- package/lib/fact-store.js +418 -0
- package/lib/index-sync.js +160 -0
- package/lib/index.js +2386 -105
- package/lib/intent-clean.js +74 -0
- package/lib/m4-corpus.js +169 -0
- package/lib/m7-index-sync-host.js +194 -0
- package/lib/m7-wire.js +268 -0
- package/lib/memory-anchor.js +451 -0
- package/lib/memory-hub.js +259 -0
- package/lib/memory-index.js +145 -0
- package/lib/memory-writer.js +391 -0
- package/lib/policies/activation_policy_v2.json +88 -0
- package/lib/policies/recall_intent_lr_v1.json +1 -0
- package/lib/procedure-store.js +406 -0
- package/lib/python-sidecar-client.js +326 -0
- package/lib/semantic-decide.js +265 -0
- package/lib/semantic-js.js +381 -0
- package/lib/shadow-host.js +361 -0
- package/lib/shadow-retrieval.js +673 -0
- package/lib/storage-manage.js +203 -0
- package/package.json +2 -2
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* M7-0/M7-1 PythonContextSinkPre(docs/PYTHON-SIDECAR-CONTRACT.md §5.2,§8.2,§13.1)。
|
|
3
|
+
* 实现 M5 ContextSinkPre 接口:消费现有 ContextPushEnvelopePre 原样字段(零 schema 改动),
|
|
4
|
+
* 在 deadlineAt 预算内返回兼容 ContextAckPre;worker 主动推送的 activation_request 帧经
|
|
5
|
+
* onActivation 上抛(交给现有 M6 validator/inbox 路径,本模块不构建 Packet)。
|
|
6
|
+
* 失败映射为结构化 ContextAckPre(accepted:false + reason 枚举);异常绝不冒泡到基础对话。
|
|
7
|
+
* 本模块不 spawn:进程生命周期完全属于共享的 SidecarClient。UTF-8 无 BOM。
|
|
8
|
+
*/
|
|
9
|
+
import { validateContextAckPre } from './context-bridge.js'
|
|
10
|
+
import { ackMatchesObservationPre } from './m7-wire.js'
|
|
11
|
+
|
|
12
|
+
export const PYTHON_CONTEXT_SINK_KIND_PRE = 'python'
|
|
13
|
+
|
|
14
|
+
const ACK_REASON_FOR_CODE_PRE = Object.freeze({
|
|
15
|
+
timeout: 'busy',
|
|
16
|
+
aborted: 'busy',
|
|
17
|
+
circuitOpen: 'unsupported',
|
|
18
|
+
'circuit-open': 'unsupported',
|
|
19
|
+
unavailable: 'unsupported',
|
|
20
|
+
crashed: 'unsupported',
|
|
21
|
+
disposed: 'unsupported',
|
|
22
|
+
backpressure: 'busy',
|
|
23
|
+
protocol: 'unsupported',
|
|
24
|
+
'line-oversize': 'unsupported',
|
|
25
|
+
'worker-error': 'unsupported',
|
|
26
|
+
'unsupported-frame': 'unsupported',
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
export function createPythonContextSinkPre(opts = {}) {
|
|
30
|
+
const client = opts.client
|
|
31
|
+
if (!client || typeof client.request !== 'function') throw new Error('context-sink-python: client required')
|
|
32
|
+
const stats = { pushed: 0, accepted: 0, rejected: 0, staleDeadline: 0, invalidAck: 0, errors: 0 }
|
|
33
|
+
const requestTimeoutMs = Number(opts.requestTimeoutMs) || 5000
|
|
34
|
+
// 2026-08-30 canary 实证修复:传入的 onActivation 从未注册(只暴露注册方法,无人调用),
|
|
35
|
+
// worker activation_request 帧到达客户端后静默蒸发(activationsReceived++ 但 offers=0)。
|
|
36
|
+
// 自动注册,修通 Python emit → M6 的最后一厘米。
|
|
37
|
+
if (typeof opts.onActivation === 'function') client.onActivation(opts.onActivation)
|
|
38
|
+
|
|
39
|
+
/** 注册 worker 主动激活的下游处理器(M6 路径);返回解绑函数。 */
|
|
40
|
+
function onActivation(handler) { return client.onActivation(handler) }
|
|
41
|
+
|
|
42
|
+
async function push(frame, signal) {
|
|
43
|
+
stats.pushed++
|
|
44
|
+
const observationId = frame && typeof frame.observationId === 'string' ? frame.observationId : ''
|
|
45
|
+
try {
|
|
46
|
+
if (!frame || !observationId) { stats.rejected++; stats.errors++; return { observationId, accepted: false, reason: 'unsupported' } }
|
|
47
|
+
const now = Date.now()
|
|
48
|
+
const deadline = Number(frame.deadlineAt)
|
|
49
|
+
let timeoutMs = requestTimeoutMs
|
|
50
|
+
if (Number.isFinite(deadline)) {
|
|
51
|
+
const remaining = deadline - now
|
|
52
|
+
if (remaining <= 0) { stats.rejected++; stats.staleDeadline++; return { observationId, accepted: false, reason: 'stale' } }
|
|
53
|
+
timeoutMs = Math.min(timeoutMs, remaining)
|
|
54
|
+
}
|
|
55
|
+
// §8.2:payload 即完整 ContextPushEnvelopePre,原样透传(零字段增删)
|
|
56
|
+
const res = await client.request('context_push', frame, { timeoutMs, signal })
|
|
57
|
+
if (res.ok) {
|
|
58
|
+
const ack = res.frame && res.frame.payload
|
|
59
|
+
const v = validateContextAckPre(ack)
|
|
60
|
+
if (!v.ok || !ackMatchesObservationPre(ack, observationId)) {
|
|
61
|
+
stats.rejected++; stats.invalidAck++
|
|
62
|
+
return { observationId, accepted: false, reason: 'unsupported' }
|
|
63
|
+
}
|
|
64
|
+
if (ack.accepted) stats.accepted++; else stats.rejected++
|
|
65
|
+
return ack
|
|
66
|
+
}
|
|
67
|
+
stats.rejected++
|
|
68
|
+
const reason = ACK_REASON_FOR_CODE_PRE[res.code] || 'unsupported'
|
|
69
|
+
return { observationId, accepted: false, reason }
|
|
70
|
+
} catch (_) {
|
|
71
|
+
stats.errors++; stats.rejected++
|
|
72
|
+
return { observationId, accepted: false, reason: 'unsupported' }
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async function closeSession(sessionId) {
|
|
77
|
+
try { client.notify('close_session', { sessionId: String(sessionId || '') }) } catch (_) {}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async function dispose(reason) {
|
|
81
|
+
void reason
|
|
82
|
+
// 客户端为 engine 级共享;sink 只解除自身注册(进程销毁由客户端 disposer 负责)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function debugView() {
|
|
86
|
+
return { kind: PYTHON_CONTEXT_SINK_KIND_PRE, stats: { ...stats }, client: client.debugView() }
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return { kind: PYTHON_CONTEXT_SINK_KIND_PRE, push, closeSession, dispose, onActivation, debugView, _statsForTest: stats }
|
|
90
|
+
}
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* M8-1 Episodic Store 纯核心(docs/proactive-associative-memory-system-map.html M-02 Episodic)。
|
|
3
|
+
* 纯内存状态机,零 IO、零依赖(node:crypto 仅作确定性身份);持久化通过可注入 IO 接口,
|
|
4
|
+
* Host 接线时才接真实文件(本模块自身不读写磁盘,测试用内存 IO)。
|
|
5
|
+
*
|
|
6
|
+
* 设计目标(M-02 元代码逐行落地):
|
|
7
|
+
* - Episode 六元组: {intent, actions, entities, unresolved, outcome, provenance}
|
|
8
|
+
* - 轻量事件先落 sidecar;会话结束或空闲期再摘要和巩固。
|
|
9
|
+
* - 失败经验默认是 candidate,不直接是事实(防错误自我解释污染长期层)。
|
|
10
|
+
*
|
|
11
|
+
* 生命周期:
|
|
12
|
+
* append(segment) → episode 累积(intent/actions/entities/unresolved 从对话段提取)
|
|
13
|
+
* consolidate() → 会话结束/空闲期调用:把 episode 摘要并固化,产出 episodic_candidate
|
|
14
|
+
* 供 M-03 Semantic(fact candidate)/M-04 Procedural(procedure candidate)消费。
|
|
15
|
+
*
|
|
16
|
+
* 与 M5 evidence 的衔接:每段都有 eventSeq/contextVersion 溯源;consolidate 后 evidence
|
|
17
|
+
* 挂钩供 Procedure 晋升复用(复用 fact-store 的 evidenceFor 语义,本模块自带轻量聚合)。
|
|
18
|
+
*
|
|
19
|
+
* 与 M7 judgement-shadow 的衔接:episodic_candidate 是 judgement-shadow 8 类之一,
|
|
20
|
+
* 本模块是它的 JS 侧真实来源(Python 侧仅建议,JS 侧才固化)。
|
|
21
|
+
*
|
|
22
|
+
* 全部同输入确定; UTF-8 无 BOM。
|
|
23
|
+
*/
|
|
24
|
+
import { createHash } from 'node:crypto'
|
|
25
|
+
|
|
26
|
+
// ========== 冻结常量 ==========
|
|
27
|
+
|
|
28
|
+
export const EPISODIC_POLICY_VERSION = 'episodic_store_v1'
|
|
29
|
+
export const EPISODE_ID_PREFIX = 'epi_'
|
|
30
|
+
export const EPISODE_ID_RE = /^epi_[0-9a-f]{32}$/
|
|
31
|
+
|
|
32
|
+
/** episode 累积上限(防无限增长;超过后最旧段被丢弃)。 */
|
|
33
|
+
export const EPISODE_SEGMENT_CAP_V1 = 64
|
|
34
|
+
|
|
35
|
+
/** 巩固后保留的最多 episode 数(超出按时间淘汰,保留最近)。 */
|
|
36
|
+
export const EPISODE_RETENTION_V1 = 256
|
|
37
|
+
|
|
38
|
+
/** 巩固阈值:至少多少段才算一个可巩固 episode(少于=丢弃,噪声太多)。 */
|
|
39
|
+
export const EPISODE_MIN_SEGMENTS_V1 = 2
|
|
40
|
+
|
|
41
|
+
/** outcome 枚举。 */
|
|
42
|
+
export const EPISODE_OUTCOMES_V1 = Object.freeze(['unknown', 'success', 'failure', 'partial'])
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* EpisodeSegment 校验(append 输入)。
|
|
46
|
+
* 最小字段: kind/userText 至少一个非空; eventSeq 非负; contextVersion 非负。
|
|
47
|
+
*/
|
|
48
|
+
export function validateEpisodeSegmentPre(seg) {
|
|
49
|
+
const p = []
|
|
50
|
+
if (!seg || typeof seg !== 'object' || Array.isArray(seg)) return { ok: false, reason: 'not-object' }
|
|
51
|
+
const hasText = typeof seg.userText === 'string' && seg.userText.trim() ||
|
|
52
|
+
typeof seg.assistantText === 'string' && seg.assistantText.trim()
|
|
53
|
+
if (!hasText) p.push('no-text')
|
|
54
|
+
if (seg.kind !== undefined && !['user', 'assistant', 'reasoning', 'tool'].includes(seg.kind)) p.push('kind')
|
|
55
|
+
if (seg.eventSeq !== undefined && (!Number.isInteger(seg.eventSeq) || seg.eventSeq < 0)) p.push('eventSeq')
|
|
56
|
+
if (seg.contextVersion !== undefined && (!Number.isInteger(seg.contextVersion) || seg.contextVersion < 0)) p.push('contextVersion')
|
|
57
|
+
if (p.length) return { ok: false, reason: 'invalid:' + p.join(',') }
|
|
58
|
+
return { ok: true, segment: seg }
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Episode 校验(consolidate 产出 / 持久化读回)。 */
|
|
62
|
+
export function validateEpisodePre(ep) {
|
|
63
|
+
const p = []
|
|
64
|
+
if (!ep || typeof ep !== 'object' || Array.isArray(ep)) return { ok: false, reason: 'not-object' }
|
|
65
|
+
if (typeof ep.episodeId !== 'string' || !EPISODE_ID_RE.test(ep.episodeId)) p.push('episodeId')
|
|
66
|
+
if (typeof ep.sessionRef !== 'string' || !ep.sessionRef) p.push('sessionRef')
|
|
67
|
+
if (typeof ep.intent !== 'string') p.push('intent')
|
|
68
|
+
if (!Array.isArray(ep.actions)) p.push('actions')
|
|
69
|
+
if (!Array.isArray(ep.entities)) p.push('entities')
|
|
70
|
+
if (!Array.isArray(ep.unresolved)) p.push('unresolved')
|
|
71
|
+
if (!EPISODE_OUTCOMES_V1.includes(ep.outcome)) p.push('outcome')
|
|
72
|
+
if (!Array.isArray(ep.provenance)) p.push('provenance')
|
|
73
|
+
if (typeof ep.startedAt !== 'number' || !Number.isFinite(ep.startedAt)) p.push('startedAt')
|
|
74
|
+
if (ep.consolidatedAt !== undefined && (typeof ep.consolidatedAt !== 'number' || !Number.isFinite(ep.consolidatedAt))) p.push('consolidatedAt')
|
|
75
|
+
if (ep.success !== undefined && typeof ep.success !== 'boolean') p.push('success')
|
|
76
|
+
if (p.length) return { ok: false, reason: 'invalid:' + p.join(',') }
|
|
77
|
+
return { ok: true, episode: ep }
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Episodic Store 工厂。
|
|
82
|
+
* @param {object} opts
|
|
83
|
+
* @param {object} opts.io 可选持久化 { save(snapshot), load() → snapshot, clear() }
|
|
84
|
+
* @param {function} opts.now 可选时钟
|
|
85
|
+
* @param {object} opts.config { minSegments?, retention?, segmentCap? } — 可调参数(默认走冻结常量)
|
|
86
|
+
*/
|
|
87
|
+
export function createEpisodicStorePre(opts = {}) {
|
|
88
|
+
const io = opts.io || { save() {}, load() { return null }, clear() {} }
|
|
89
|
+
const nowFn = typeof opts.now === 'function' ? opts.now : () => Date.now()
|
|
90
|
+
const cfg = Object.assign({
|
|
91
|
+
segmentCap: EPISODE_SEGMENT_CAP_V1,
|
|
92
|
+
retention: EPISODE_RETENTION_V1,
|
|
93
|
+
minSegments: EPISODE_MIN_SEGMENTS_V1,
|
|
94
|
+
}, opts.config || {})
|
|
95
|
+
|
|
96
|
+
let episodes = [] // 已巩固 episode(含 candidate 状态)
|
|
97
|
+
let current = null // 当前会话累积中(未巩固)
|
|
98
|
+
let disposed = false
|
|
99
|
+
const stats = { segmentsAppended: 0, consolidated: 0, droppedTooShort: 0, retained: 0 }
|
|
100
|
+
|
|
101
|
+
function defaultEpisodeId(sessionRef, startedAt) {
|
|
102
|
+
const h = createHash('sha256').update(['episode-pre-v1', String(sessionRef), String(startedAt)].join('\u0000')).digest('hex')
|
|
103
|
+
return EPISODE_ID_PREFIX + h.slice(0, 32)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ---- 段追加(会话进行中实时累积) ----
|
|
107
|
+
function append(seg) {
|
|
108
|
+
if (disposed) return { ok: false, reason: 'disposed' }
|
|
109
|
+
const v = validateEpisodeSegmentPre(seg)
|
|
110
|
+
if (!v.ok) return { ok: false, reason: v.reason }
|
|
111
|
+
const s = v.segment
|
|
112
|
+
if (!current) {
|
|
113
|
+
current = {
|
|
114
|
+
sessionRef: String(s.sessionRef || 'unknown'),
|
|
115
|
+
startedAt: nowFn(),
|
|
116
|
+
segments: [],
|
|
117
|
+
userTexts: [], assistantTexts: [],
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
// 累积文本(只保留文本特征,不保留原文全文 —— 隐私最小化)
|
|
121
|
+
const ut = typeof s.userText === 'string' ? s.userText : ''
|
|
122
|
+
const at = typeof s.assistantText === 'string' ? s.assistantText : ''
|
|
123
|
+
if (ut.trim()) current.userTexts.push(ut.trim().slice(0, 200))
|
|
124
|
+
if (at.trim()) current.assistantTexts.push(at.trim().slice(0, 200))
|
|
125
|
+
current.segments.push({
|
|
126
|
+
kind: s.kind || 'unknown',
|
|
127
|
+
eventSeq: s.eventSeq || 0,
|
|
128
|
+
contextVersion: s.contextVersion || 0,
|
|
129
|
+
userText: ut.slice(0, 200),
|
|
130
|
+
assistantText: at.slice(0, 200),
|
|
131
|
+
ts: nowFn(),
|
|
132
|
+
})
|
|
133
|
+
// 超 cap:丢最旧段(保留最近上下文)
|
|
134
|
+
while (current.segments.length > cfg.segmentCap) {
|
|
135
|
+
current.segments.shift()
|
|
136
|
+
if (current.userTexts.length) current.userTexts.shift()
|
|
137
|
+
if (current.assistantTexts.length) current.assistantTexts.shift()
|
|
138
|
+
}
|
|
139
|
+
stats.segmentsAppended++
|
|
140
|
+
return { ok: true, segments: current.segments.length }
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** 从累积文本提取 intent(2026-08-28 提纯):跳过本插件注入快照污染的段
|
|
144
|
+
* (「Current runtime context. This snapshot…」/auto-memory 标记),取第一条
|
|
145
|
+
* 真实用户文本;全被污染则取最后一条(退化)。 */
|
|
146
|
+
function extractIntent(ep) {
|
|
147
|
+
const INJECTED_RE = /^current runtime context\.|^this snapshot supersedes|auto-memory|\[自动沉淀\]/i
|
|
148
|
+
const texts = (ep.userTexts || []).map((t) => String(t || '').trim()).filter((t) => t)
|
|
149
|
+
const clean = texts.filter((t) => !INJECTED_RE.test(t))
|
|
150
|
+
const pick = clean.length ? clean[0] : (texts.length ? texts[texts.length - 1] : '')
|
|
151
|
+
return pick ? pick.slice(0, 60) : '(未提取)'
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** 从累积文本提取 entities(简单启发式:用户文本中的 CJK 2-4 字符 token,高频优先)。 */
|
|
155
|
+
function extractEntities(userTexts) {
|
|
156
|
+
const joined = userTexts.join(' ')
|
|
157
|
+
const grams = new Map()
|
|
158
|
+
const tokens = String(joined).match(/[A-Za-z0-9_\u4e00-\u9fff]+/g) || []
|
|
159
|
+
for (const tok of tokens) {
|
|
160
|
+
if (tok.length < 2 || tok.length > 20) continue
|
|
161
|
+
if (/^[\u4e00-\u9fff]+$/.test(tok) && tok.length > 4) {
|
|
162
|
+
// 中文长 token 按 2-3 字滑窗
|
|
163
|
+
for (let i = 0; i + 2 <= tok.length && i < tok.length - 2 + 1 && i < 12; i++) {
|
|
164
|
+
const g = tok.slice(i, i + 2)
|
|
165
|
+
if (/[\u4e00-\u9fff]/.test(g)) grams.set(g, (grams.get(g) || 0) + 1)
|
|
166
|
+
}
|
|
167
|
+
} else {
|
|
168
|
+
grams.set(tok, (grams.get(tok) || 0) + 1)
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
return [...grams.entries()].sort((a, b) => b[1] - a[1]).slice(0, 8).map(([g]) => g)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** 推断 outcome(启发式:助手文本含失败信号→failure, 成功信号→success)。 */
|
|
175
|
+
function inferOutcome(ep) {
|
|
176
|
+
const at = ep.assistantTexts.join(' ').toLowerCase()
|
|
177
|
+
const failSig = ['失败', '错误', '报错', '无法', 'error', 'failed', 'exception']
|
|
178
|
+
const succSig = ['成功', '完成', '已修复', '搞定', 'ok', 'done', 'success']
|
|
179
|
+
if (failSig.some((s) => at.includes(s))) return 'failure'
|
|
180
|
+
if (succSig.some((s) => at.includes(s))) return 'success'
|
|
181
|
+
return 'unknown'
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** 提取未决事项(启发式:用户文本含问句/待办)。 */
|
|
185
|
+
function extractUnresolved(ep) {
|
|
186
|
+
const ut = ep.userTexts.join(' ')
|
|
187
|
+
const out = []
|
|
188
|
+
const q = ut.match(/[^。!?\n]*[??][^。!?\n]*/g) || []
|
|
189
|
+
for (const s of q) if (s.trim() && out.length < 5) out.push(s.trim().slice(0, 80))
|
|
190
|
+
return out
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* 巩固: 会话结束/空闲期调用,把当前累积固化成 episode。
|
|
195
|
+
* 产出 outcome=intent/entities/unresolved 等特征,并挂上 success 标志(供 Procedure 用)。
|
|
196
|
+
* 少于 minSegments 的丢弃(噪声)。
|
|
197
|
+
*/
|
|
198
|
+
function consolidate() {
|
|
199
|
+
if (disposed) return { ok: false, reason: 'disposed' }
|
|
200
|
+
if (!current) return { ok: false, reason: 'nothing-to-consolidate' }
|
|
201
|
+
if (current.segments.length < cfg.minSegments) {
|
|
202
|
+
stats.droppedTooShort++
|
|
203
|
+
const segs = current.segments.length
|
|
204
|
+
current = null
|
|
205
|
+
return { ok: false, reason: 'too-short', segments: segs }
|
|
206
|
+
}
|
|
207
|
+
const outcome = inferOutcome(current)
|
|
208
|
+
const ep = {
|
|
209
|
+
episodeId: defaultEpisodeId(current.sessionRef, current.startedAt),
|
|
210
|
+
sessionRef: current.sessionRef,
|
|
211
|
+
intent: extractIntent(current),
|
|
212
|
+
actions: current.segments.map((s) => s.kind).filter(Boolean),
|
|
213
|
+
entities: extractEntities(current.userTexts),
|
|
214
|
+
unresolved: extractUnresolved(current),
|
|
215
|
+
outcome,
|
|
216
|
+
success: outcome === 'success',
|
|
217
|
+
provenance: current.segments.map((s) => 'seg:' + String(s.eventSeq)),
|
|
218
|
+
startedAt: current.startedAt,
|
|
219
|
+
consolidatedAt: nowFn(),
|
|
220
|
+
}
|
|
221
|
+
const v = validateEpisodePre(ep)
|
|
222
|
+
if (!v.ok) { current = null; return { ok: false, reason: 'invalid:' + v.reason } }
|
|
223
|
+
episodes.push(v.episode)
|
|
224
|
+
// 保留策略: 超 retention 淘汰最旧
|
|
225
|
+
if (episodes.length > cfg.retention) {
|
|
226
|
+
episodes = episodes.slice(-cfg.retention)
|
|
227
|
+
stats.retained++
|
|
228
|
+
}
|
|
229
|
+
current = null
|
|
230
|
+
stats.consolidated++
|
|
231
|
+
void persist()
|
|
232
|
+
return { ok: true, episode: v.episode }
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** 会话中途强制巩固(跨天续接/会话切换时调用)。 */
|
|
236
|
+
function flush() {
|
|
237
|
+
if (current) return consolidate()
|
|
238
|
+
return { ok: false, reason: 'nothing-to-consolidate' }
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// ---- 查询 ----
|
|
242
|
+
function query(q = {}) {
|
|
243
|
+
if (disposed) return []
|
|
244
|
+
return episodes
|
|
245
|
+
.filter((e) =>
|
|
246
|
+
(q.sessionRef === undefined || e.sessionRef === q.sessionRef) &&
|
|
247
|
+
(q.outcome === undefined || e.outcome === q.outcome) &&
|
|
248
|
+
(q.success === undefined || e.success === q.success))
|
|
249
|
+
.map((e) => ({ ...e }))
|
|
250
|
+
}
|
|
251
|
+
function recent(n = 10) {
|
|
252
|
+
return episodes.slice(-n).map((e) => ({ ...e }))
|
|
253
|
+
}
|
|
254
|
+
function get(episodeId) {
|
|
255
|
+
const e = episodes.find((x) => x.episodeId === episodeId)
|
|
256
|
+
return e ? { ...e } : null
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// ---- M-04 挂钩: 供 Procedure 晋升用的事故/成功统计 ----
|
|
260
|
+
function statsFor(sessionRef) {
|
|
261
|
+
const all = episodes.filter((e) => e.sessionRef === sessionRef)
|
|
262
|
+
return {
|
|
263
|
+
total: all.length,
|
|
264
|
+
success: all.filter((e) => e.success).length,
|
|
265
|
+
failure: all.filter((e) => e.outcome === 'failure').length,
|
|
266
|
+
distinctSessions: new Set(episodes.map((e) => e.sessionRef)).size,
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// ---- 持久化 ----
|
|
271
|
+
function snapshot() {
|
|
272
|
+
return {
|
|
273
|
+
schemaVersion: 1, namespace: 'dsh-auto-memory', policyVersion: EPISODIC_POLICY_VERSION,
|
|
274
|
+
savedAt: nowFn(),
|
|
275
|
+
episodes: episodes.map((e) => ({ ...e })),
|
|
276
|
+
current: current ? {
|
|
277
|
+
sessionRef: current.sessionRef, startedAt: current.startedAt,
|
|
278
|
+
segments: current.segments, userTexts: current.userTexts, assistantTexts: current.assistantTexts,
|
|
279
|
+
} : null,
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
function restore(data) {
|
|
283
|
+
if (!data || data.schemaVersion !== 1) return { ok: false, reason: 'bad-schema' }
|
|
284
|
+
if (!Array.isArray(data.episodes)) return { ok: false, reason: 'bad-episodes' }
|
|
285
|
+
episodes = []
|
|
286
|
+
for (const e of data.episodes) {
|
|
287
|
+
const v = validateEpisodePre(e)
|
|
288
|
+
if (!v.ok) continue
|
|
289
|
+
episodes.push(v.episode)
|
|
290
|
+
}
|
|
291
|
+
current = data.current || null
|
|
292
|
+
return { ok: true, restored: episodes.length }
|
|
293
|
+
}
|
|
294
|
+
function clear() {
|
|
295
|
+
episodes = []; current = null
|
|
296
|
+
try { io.clear() } catch (_) {}
|
|
297
|
+
return { ok: true }
|
|
298
|
+
}
|
|
299
|
+
function dispose(reason) {
|
|
300
|
+
if (disposed) return
|
|
301
|
+
disposed = true
|
|
302
|
+
try { io.save(snapshot()) } catch (_) {}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function persist() {
|
|
306
|
+
try { io.save(snapshot()) } catch (_) {}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
return {
|
|
310
|
+
append, consolidate, flush, query, recent, get, statsFor,
|
|
311
|
+
snapshot, restore, clear, dispose,
|
|
312
|
+
getStats: () => ({ ...stats }),
|
|
313
|
+
get size() { return episodes.length },
|
|
314
|
+
get hasCurrent() { return !!current },
|
|
315
|
+
}
|
|
316
|
+
}
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* M5-2 Evidence Store + Aggregate(docs/M5-CONTRACT.md §8-§11)。
|
|
3
|
+
* append-only events(JSONL 按日分片)+隐私投影(sessionRef/workspaceRef 哈希;无原文/无绝对路径/
|
|
4
|
+
* 无裸 sessionId/workspaceKey/excerpt)+retention(30 天/32MiB)+aggregate rebuild(fresh/stale/unknown)。
|
|
5
|
+
* events 为唯一权威;aggregate 是可重建派生物。默认不构造不落盘;测试用临时 DSH_HOME 注入 root。
|
|
6
|
+
* 零第三方依赖(node:fs/path/crypto);UTF-8 无 BOM。
|
|
7
|
+
*/
|
|
8
|
+
import { mkdirSync, appendFileSync, readdirSync, readFileSync, statSync, rmSync, writeFileSync } from 'node:fs'
|
|
9
|
+
import path from 'node:path'
|
|
10
|
+
import { createHash } from 'node:crypto'
|
|
11
|
+
import {
|
|
12
|
+
validateAccessEvidencePre, validateEvidenceAggregatePre, EVIDENCE_POLICY_VERSION, NAMESPACE, BoundedIdSet,
|
|
13
|
+
} from './context-bridge.js'
|
|
14
|
+
|
|
15
|
+
const sha256Str = (s) => createHash('sha256').update(Buffer.from(String(s), 'utf8')).digest('hex')
|
|
16
|
+
const first32 = (h) => h.slice(0, 32)
|
|
17
|
+
|
|
18
|
+
/** §10 持久化策略(冻结)。 */
|
|
19
|
+
export const EVIDENCE_STORE_POLICY_V1 = Object.freeze({
|
|
20
|
+
schemaVersion: 1,
|
|
21
|
+
storePolicyVersion: 'evidence_store_v1',
|
|
22
|
+
keepDays: 30,
|
|
23
|
+
maxTotalBytes: 32 * 1024 * 1024,
|
|
24
|
+
eventMaxBytes: 16 * 1024,
|
|
25
|
+
appendedIdCache: 4096,
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
/** 稳定 workspaceKey 规范化(与 m4-corpus canonical 同规则:resolve+正斜杠+小写)。 */
|
|
29
|
+
export function canonicalWorkspaceKey(key) {
|
|
30
|
+
return path.resolve(String(key == null ? '' : key)).replace(/\\/g, '/').toLowerCase()
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** §10 隐私投影:sessionId → plugin-local sessionRef(不可逆哈希;跨文件稳定)。 */
|
|
34
|
+
export function sessionRefOf(sessionId) {
|
|
35
|
+
return 'sesr_' + first32(sha256Str('evidence-sesref-pre-v1\u0000' + String(sessionId || '')))
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** §10 隐私投影:workspace 绝对键 → 稳定 workspaceRef(不落盘任何绝对路径)。 */
|
|
39
|
+
export function workspaceRefOf(workspaceKey) {
|
|
40
|
+
return 'wsr_' + first32(sha256Str('evidence-wsref-pre-v1\u0000' + canonicalWorkspaceKey(workspaceKey)))
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* §10 durable projection:evidence → 落盘形态。删除裸 sessionId/workspaceKey;
|
|
45
|
+
* sourceRef 已由 validator 限定为相对引用(user:/workspace:/workspace-log:+文件名)。
|
|
46
|
+
* 返回 {ok, projected} 或 {ok:false, reason}(invalid-evidence/oversize)。
|
|
47
|
+
*/
|
|
48
|
+
export function projectEvidenceForDurable(ev, opts = {}) {
|
|
49
|
+
const v = validateAccessEvidencePre(ev)
|
|
50
|
+
if (!v.ok) return { ok: false, reason: 'invalid-evidence:' + v.reason }
|
|
51
|
+
const maxBytes = Number(opts.eventMaxBytes) || EVIDENCE_STORE_POLICY_V1.eventMaxBytes
|
|
52
|
+
const projected = {
|
|
53
|
+
schemaVersion: 1,
|
|
54
|
+
namespace: NAMESPACE,
|
|
55
|
+
storePolicyVersion: EVIDENCE_STORE_POLICY_V1.storePolicyVersion,
|
|
56
|
+
evidenceId: ev.evidenceId,
|
|
57
|
+
kind: ev.kind,
|
|
58
|
+
memoryId: ev.memoryId,
|
|
59
|
+
anchorId: ev.anchorId,
|
|
60
|
+
scope: ev.scope,
|
|
61
|
+
workspaceRef: workspaceRefOf(ev.workspaceKey),
|
|
62
|
+
event: {
|
|
63
|
+
sessionRef: sessionRefOf(ev.event.sessionId),
|
|
64
|
+
eventSeq: ev.event.eventSeq,
|
|
65
|
+
nativeSeq: ev.event.nativeSeq === undefined ? undefined : ev.event.nativeSeq,
|
|
66
|
+
contextVersion: ev.event.contextVersion,
|
|
67
|
+
callId: ev.event.callId === undefined ? undefined : ev.event.callId,
|
|
68
|
+
ts: ev.event.ts,
|
|
69
|
+
},
|
|
70
|
+
source: {
|
|
71
|
+
sourceRef: ev.source.sourceRef,
|
|
72
|
+
sourceEpoch: ev.source.sourceEpoch,
|
|
73
|
+
sourceVersion: ev.source.sourceVersion,
|
|
74
|
+
fileDigest: ev.source.fileDigest,
|
|
75
|
+
recordDigest: ev.source.recordDigest,
|
|
76
|
+
},
|
|
77
|
+
policyVersion: ev.policyVersion,
|
|
78
|
+
recordedAt: Number.isFinite(opts.now) ? opts.now : Date.now(),
|
|
79
|
+
}
|
|
80
|
+
if (ev.coverage !== undefined) projected.coverage = ev.coverage
|
|
81
|
+
if (ev.episodeId !== undefined && ev.episodeId !== null) projected.episodeId = ev.episodeId
|
|
82
|
+
const json = JSON.stringify(projected)
|
|
83
|
+
if (Buffer.byteLength(json, 'utf8') > maxBytes) return { ok: false, reason: 'event-oversize' }
|
|
84
|
+
return { ok: true, projected, line: json }
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** 从投影行还原成 rebuild 输入(无原始 sessionId;distinctSessions 以 sessionRef 计数)。 */
|
|
88
|
+
export function parseDurableEventLine(line) {
|
|
89
|
+
try {
|
|
90
|
+
const j = JSON.parse(line)
|
|
91
|
+
if (!j || typeof j !== 'object' || !j.evidenceId || !j.kind || !j.memoryId) return { ok: false }
|
|
92
|
+
return { ok: true, event: j }
|
|
93
|
+
} catch (_) { return { ok: false } }
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// ========== Store(appen-only JSONL,engine 级串行链) ==========
|
|
97
|
+
|
|
98
|
+
export class EvidenceEventStore {
|
|
99
|
+
constructor(opts = {}) {
|
|
100
|
+
this.root = opts.root || null
|
|
101
|
+
this.eventsDir = opts.eventsDir || (this.root ? path.join(this.root, 'events') : null)
|
|
102
|
+
this._chain = Promise.resolve()
|
|
103
|
+
this._appended = new BoundedIdSet(EVIDENCE_STORE_POLICY_V1.appendedIdCache)
|
|
104
|
+
this._swept = false
|
|
105
|
+
this.stats = { appended: 0, duplicates: 0, oversize: 0, invalid: 0, writeFailed: 0, sweptFiles: 0, lastWriteError: null }
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** §9 幂等:同 evidenceId 进程内只落盘一次(seed replay 与 live 双喂安全网;磁盘侧靠 rebuild 去重兜底)。 */
|
|
109
|
+
async append(evidence, opts = {}) {
|
|
110
|
+
if (!this.eventsDir) return { ok: false, reason: 'store-not-configured' }
|
|
111
|
+
const id = evidence && evidence.evidenceId
|
|
112
|
+
if (id && this._appended.has(id)) { this.stats.duplicates++; return { ok: false, reason: 'duplicate-evidence', evidenceId: id } }
|
|
113
|
+
const proj = projectEvidenceForDurable(evidence, { now: opts.now })
|
|
114
|
+
if (!proj.ok) {
|
|
115
|
+
if (proj.reason === 'event-oversize') this.stats.oversize++
|
|
116
|
+
else this.stats.invalid++
|
|
117
|
+
return proj
|
|
118
|
+
}
|
|
119
|
+
if (id) this._appended.add(id)
|
|
120
|
+
this._chain = this._chain.then(() => this._writeLine(proj.line))
|
|
121
|
+
return this._chain.then((written) => ({ ok: written, reason: written ? 'ok' : 'write-failed', evidenceId: id, projected: proj.projected }))
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
async _writeLine(line) {
|
|
125
|
+
try {
|
|
126
|
+
mkdirSync(this.eventsDir, { recursive: true })
|
|
127
|
+
const d = new Date()
|
|
128
|
+
const fname = d.getFullYear() + '-' + String(d.getMonth() + 1).padStart(2, '0') + '-' + String(d.getDate()).padStart(2, '0') + '.jsonl'
|
|
129
|
+
appendFileSync(path.join(this.eventsDir, fname), line + '\n', 'utf8')
|
|
130
|
+
this.stats.appended++
|
|
131
|
+
this.sweepRetention()
|
|
132
|
+
return true
|
|
133
|
+
} catch (e) {
|
|
134
|
+
this.stats.writeFailed++
|
|
135
|
+
this.stats.lastWriteError = String(e && e.message ? e.message : e)
|
|
136
|
+
try { console.error('[evidence-store] write-failed: ' + this.stats.lastWriteError) } catch (_) {}
|
|
137
|
+
return false
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** §10 retention:保留 keepDays 天且总量 ≤maxTotalBytes;只清 events 分片。每进程至多全扫一次。 */
|
|
142
|
+
sweepRetention(force = false) {
|
|
143
|
+
if (this._swept && !force) return
|
|
144
|
+
this._swept = true
|
|
145
|
+
try {
|
|
146
|
+
const now = Date.now()
|
|
147
|
+
let total = 0
|
|
148
|
+
const files = []
|
|
149
|
+
for (const f of readdirSync(this.eventsDir)) {
|
|
150
|
+
if (!f.endsWith('.jsonl')) continue
|
|
151
|
+
const fp = path.join(this.eventsDir, f)
|
|
152
|
+
const st = statSync(fp)
|
|
153
|
+
total += st.size
|
|
154
|
+
files.push({ fp, mtimeMs: st.mtimeMs, size: st.size })
|
|
155
|
+
}
|
|
156
|
+
files.sort((a, b) => a.mtimeMs - b.mtimeMs)
|
|
157
|
+
for (const f of files) {
|
|
158
|
+
const ageDays = (now - f.mtimeMs) / 86400000
|
|
159
|
+
if (ageDays > EVIDENCE_STORE_POLICY_V1.keepDays || total > EVIDENCE_STORE_POLICY_V1.maxTotalBytes) {
|
|
160
|
+
try { rmSync(f.fp, { force: true }); this.stats.sweptFiles++; total -= f.size } catch (_) {}
|
|
161
|
+
} else break
|
|
162
|
+
}
|
|
163
|
+
} catch (_) {}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** 读回全部事件(旧→新按文件名排序;坏行跳过计数)。 */
|
|
167
|
+
loadEvents() {
|
|
168
|
+
const out = []
|
|
169
|
+
let badLines = 0
|
|
170
|
+
try {
|
|
171
|
+
const files = readdirSync(this.eventsDir).filter((f) => f.endsWith('.jsonl')).sort()
|
|
172
|
+
for (const f of files) {
|
|
173
|
+
const text = readFileSync(path.join(this.eventsDir, f), 'utf8')
|
|
174
|
+
for (const line of text.split('\n')) {
|
|
175
|
+
if (!line.trim()) continue
|
|
176
|
+
const p = parseDurableEventLine(line)
|
|
177
|
+
if (p.ok) out.push(p.event)
|
|
178
|
+
else badLines++
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
} catch (_) {}
|
|
182
|
+
return { events: out, badLines }
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
dispose(reason) {
|
|
186
|
+
void reason
|
|
187
|
+
this._appended.clear()
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// ========== §11 Aggregate rebuild(events 为唯一权威) ==========
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* aggregate rebuild:从 durable events 重建 per-memoryId 聚合。
|
|
195
|
+
* - Session scope 不进 durable aggregate(§11 scope 仅 Workspace|User;privacy)。
|
|
196
|
+
* - evidenceId 去重(首见保留;重复计账)——seed replay/live 双写安全网。
|
|
197
|
+
* - freshness:提供 corpusRecords 时,digest+sourceVersion 与当前记录一致=fresh;
|
|
198
|
+
* memoryId 存在但不匹配=stale;语料缺失=unknown。不提供 corpusRecords → 全部 unknown。
|
|
199
|
+
*/
|
|
200
|
+
export function rebuildAggregates(durableEvents, corpusRecords) {
|
|
201
|
+
const list = Array.isArray(durableEvents)
|
|
202
|
+
? durableEvents
|
|
203
|
+
: (durableEvents && Array.isArray(durableEvents.events) ? durableEvents.events : [])
|
|
204
|
+
const byKey = new Map() // memoryId → state
|
|
205
|
+
const seenIds = new Set()
|
|
206
|
+
let duplicates = 0
|
|
207
|
+
let sessionScoped = 0
|
|
208
|
+
for (const ev of list) {
|
|
209
|
+
if (seenIds.has(ev.evidenceId)) { duplicates++; continue }
|
|
210
|
+
seenIds.add(ev.evidenceId)
|
|
211
|
+
if (ev.scope === 'Session') { sessionScoped++; continue }
|
|
212
|
+
if (ev.scope !== 'Workspace' && ev.scope !== 'User') continue
|
|
213
|
+
const wsRef = ev.workspaceRef || 'unknown'
|
|
214
|
+
const key = ev.memoryId + '|' + ev.scope + '|' + wsRef
|
|
215
|
+
let st = byKey.get(key)
|
|
216
|
+
if (!st) {
|
|
217
|
+
st = { memoryId: ev.memoryId, scope: ev.scope, workspaceRef: wsRef, sessions: new Set(), counts: { seen: 0, read: 0, cite: 0, reuse: 0, success: 0, correction: 0 }, lastEvidenceAt: 0, latest: null }
|
|
218
|
+
byKey.set(key, st)
|
|
219
|
+
}
|
|
220
|
+
if (st.counts[ev.kind] === undefined) continue // 未知 kind 不计入(policy 升级兼容)
|
|
221
|
+
st.counts[ev.kind]++
|
|
222
|
+
st.sessions.add(ev.event && ev.event.sessionRef ? ev.event.sessionRef : 'unknown')
|
|
223
|
+
const ts = Number(ev.event && ev.event.ts) || 0
|
|
224
|
+
if (ts >= st.lastEvidenceAt) { st.lastEvidenceAt = ts; st.latest = ev }
|
|
225
|
+
}
|
|
226
|
+
const recsByMemory = new Map()
|
|
227
|
+
for (const r of Array.isArray(corpusRecords) ? corpusRecords : []) {
|
|
228
|
+
if (!recsByMemory.has(r.memoryId)) recsByMemory.set(r.memoryId, [])
|
|
229
|
+
recsByMemory.get(r.memoryId).push(r)
|
|
230
|
+
}
|
|
231
|
+
const aggregates = []
|
|
232
|
+
const byWorkspaceRef = new Map() // workspaceRef → aggregates(§9 跨工作区零泄漏的推送侧过滤依据)
|
|
233
|
+
for (const st of byKey.values()) {
|
|
234
|
+
let freshness = 'unknown'
|
|
235
|
+
const recs = recsByMemory.get(st.memoryId)
|
|
236
|
+
if (recs && recs.length && st.latest) {
|
|
237
|
+
const match = recs.some((r) => r.recordDigest === st.latest.source.recordDigest && r.sourceVersion === st.latest.source.sourceVersion)
|
|
238
|
+
freshness = match ? 'fresh' : 'stale'
|
|
239
|
+
}
|
|
240
|
+
const agg = {
|
|
241
|
+
memoryId: st.memoryId,
|
|
242
|
+
scope: st.scope,
|
|
243
|
+
freshness,
|
|
244
|
+
distinctSessions: st.sessions.size,
|
|
245
|
+
seen: st.counts.seen,
|
|
246
|
+
read: st.counts.read,
|
|
247
|
+
cite: st.counts.cite,
|
|
248
|
+
reuse: st.counts.reuse,
|
|
249
|
+
success: st.counts.success,
|
|
250
|
+
correction: st.counts.correction,
|
|
251
|
+
lastEvidenceAt: st.lastEvidenceAt,
|
|
252
|
+
policyVersion: EVIDENCE_POLICY_VERSION,
|
|
253
|
+
}
|
|
254
|
+
const v = validateEvidenceAggregatePre(agg)
|
|
255
|
+
if (v.ok) {
|
|
256
|
+
aggregates.push(v.aggregate)
|
|
257
|
+
if (!byWorkspaceRef.has(st.workspaceRef)) byWorkspaceRef.set(st.workspaceRef, [])
|
|
258
|
+
byWorkspaceRef.get(st.workspaceRef).push(v.aggregate)
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
aggregates.sort((a, b) => (a.memoryId < b.memoryId ? -1 : a.memoryId > b.memoryId ? 1 : a.scope < b.scope ? -1 : 1))
|
|
262
|
+
return { aggregates, byWorkspaceRef, duplicates, sessionScoped }
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** aggregate 快照持久化(evidence/aggregates/index.json;可重建派生物,仅调试/导出用)。 */
|
|
266
|
+
export function persistAggregatesSnapshot(root, aggregates) {
|
|
267
|
+
const dir = path.join(root, 'aggregates')
|
|
268
|
+
mkdirSync(dir, { recursive: true })
|
|
269
|
+
const payload = { schemaVersion: 1, namespace: NAMESPACE, storePolicyVersion: EVIDENCE_STORE_POLICY_V1.storePolicyVersion, rebuiltAt: Date.now(), aggregates }
|
|
270
|
+
writeFileSync(path.join(dir, 'index.json'), JSON.stringify(payload, null, 2) + '\n', 'utf8')
|
|
271
|
+
return path.join(dir, 'index.json')
|
|
272
|
+
}
|