@a9i5k4/dsh-auto-memory 0.1.29 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +352 -266
- package/README.zh-CN.md +370 -264
- package/cordis.patch.yml +9 -9
- package/lib/activation-host.js +455 -0
- package/lib/activation-inbox-state.js +261 -0
- package/lib/activation-inbox.js +426 -0
- package/lib/client.js +1379 -50
- package/lib/context-bridge.js +619 -0
- package/lib/context-host.js +712 -0
- package/lib/context-sink-python.js +90 -0
- package/lib/episodic-store.js +316 -0
- package/lib/evidence-store.js +272 -0
- package/lib/fact-store.js +418 -0
- package/lib/index-sync.js +160 -0
- package/lib/index.js +2357 -102
- package/lib/intent-clean.js +74 -0
- package/lib/m4-corpus.js +169 -0
- package/lib/m7-index-sync-host.js +194 -0
- package/lib/m7-wire.js +268 -0
- package/lib/memory-anchor.js +451 -0
- package/lib/memory-hub.js +259 -0
- package/lib/memory-index.js +145 -0
- package/lib/memory-writer.js +391 -0
- package/lib/policies/activation_policy_v2.json +88 -0
- package/lib/policies/recall_intent_lr_v1.json +1 -0
- package/lib/procedure-store.js +406 -0
- package/lib/python-sidecar-client.js +326 -0
- package/lib/semantic-decide.js +265 -0
- package/lib/semantic-js.js +381 -0
- package/lib/shadow-host.js +361 -0
- package/lib/shadow-retrieval.js +673 -0
- package/lib/storage-manage.js +203 -0
- package/package.json +2 -2
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* M8 采集侧 intent 清洗(2026-08-30 P1,docs/HANDOFF-M8-M9-M10.md §2 P1)。
|
|
3
|
+
*
|
|
4
|
+
* 背景:episode 的 intent 来自 consolidateTurn 取到的「本轮最后一条 user 文本」,而该文本
|
|
5
|
+
* 在真实运行时常被三种形态污染(已实录于 ~/.dsh/memory/hub/episodes.json):
|
|
6
|
+
* ① harness 注入的上下文快照 —— 以 "Current runtime context. This snapshot supersedes…" 开头
|
|
7
|
+
* ② 工具回包 —— role 也是 user,但 eventType='tool/result',正文是 JSON 转储
|
|
8
|
+
* ③ 行号引用文本 —— 同属工具回包("436: ## 2026-08-25\n437: …")
|
|
9
|
+
*
|
|
10
|
+
* 三层修复原本内联在 lib/index.js consolidateTurn 的 for 循环里,只能靠「用户自然对话两轮」
|
|
11
|
+
* 实机验证,无法回归锁定。抽出为纯函数后,上述真实形态全部变成可重复执行的断言。
|
|
12
|
+
*
|
|
13
|
+
* 抽取时发现并修掉的实质缺陷(2026-08-30):
|
|
14
|
+
* 原第 2 层「正文含 <memory_system> 就整条跳过」会抢在第 3 层之前生效,于是第 3 层的
|
|
15
|
+
* 快照剥离**永远轮不到**——真人问题一旦与注入快照拼在同一条消息里,整条被丢,intent
|
|
16
|
+
* 只能退化成上一轮的旧问题。现改为「先剥离块,再看剩下什么」:剥离后为空或只剩
|
|
17
|
+
* harness 前缀才判为合成消息,否则剩下的就是真人问题(严格优于原行为:原行为会丢问题,
|
|
18
|
+
* 新行为只是把问题捞回来;纯快照消息两种行为都跳过)。
|
|
19
|
+
*
|
|
20
|
+
* 设计约束:
|
|
21
|
+
* - 纯函数、零 IO、零依赖;同输入同输出。
|
|
22
|
+
* - 只认 eventType='user/message' 的真人消息(messageOfEvent 只产出
|
|
23
|
+
* user/message | assistant/message | tool/result 三类)。
|
|
24
|
+
* 命名空间:_pre 隔离。UTF-8 无 BOM。
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
/** 完整的注入快照块(开闭标签配对)。 */
|
|
28
|
+
const SNAPSHOT_BLOCK_RE = /<memory_system>[\s\S]*?<\/memory_system>/g
|
|
29
|
+
/** harness 合成前缀(剥离块后若只剩这段,说明整条都是注入)。 */
|
|
30
|
+
const RUNTIME_CONTEXT_RE = /^current runtime context\./i
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* 剥离注入内容,返回剩余的真人文本:
|
|
34
|
+
* (a) 完整的 <memory_system>…</memory_system> 块整体移除(可多块);
|
|
35
|
+
* (b) 只剩闭合标签(快照被拆条/半截注入)→ 取其后内容;
|
|
36
|
+
* (c) 其余原样。
|
|
37
|
+
*/
|
|
38
|
+
export function stripInjectedBlockPre(text) {
|
|
39
|
+
let s = String(text == null ? '' : text)
|
|
40
|
+
s = s.replace(SNAPSHOT_BLOCK_RE, '')
|
|
41
|
+
const marker = '</memory_system>'
|
|
42
|
+
const idx = s.lastIndexOf(marker)
|
|
43
|
+
if (idx >= 0) s = s.slice(idx + marker.length)
|
|
44
|
+
// harness 的 "Current runtime context. …" 导语是整行噪声,且可能出现在任一行
|
|
45
|
+
// (快照在前真问题在后 / 真问题在前快照在后,两种拼法都要能剥干净)
|
|
46
|
+
return s.split(/\r?\n/).filter((ln) => !RUNTIME_CONTEXT_RE.test(ln.trim())).join('\n')
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** 合成注入消息识别:剥离注入内容后什么都不剩 → 整条都是注入。 */
|
|
50
|
+
export function isInjectedContextTextPre(text) {
|
|
51
|
+
return !stripInjectedBlockPre(text).trim()
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* 从会话消息序列挑选本轮沉淀用的 user/assistant 文本。
|
|
56
|
+
* 第 1 层:只认 role=user && eventType='user/message'(滤掉工具回包);
|
|
57
|
+
* 第 2 层:剥离注入块后,空或仅剩 runtime-context 前缀 → 判为合成消息并跳过;
|
|
58
|
+
* 第 3 层:取**最后一条**真人消息(与原逻辑一致);assistant 取最后一条非空文本。
|
|
59
|
+
*/
|
|
60
|
+
export function pickConsolidationTextPre(messages) {
|
|
61
|
+
let userText = ''
|
|
62
|
+
let assistantText = ''
|
|
63
|
+
for (const m of Array.isArray(messages) ? messages : []) {
|
|
64
|
+
if (!m || typeof m !== 'object') continue
|
|
65
|
+
if (m.role === 'user' && m.eventType === 'user/message') {
|
|
66
|
+
const rest = stripInjectedBlockPre(m.text).trim()
|
|
67
|
+
if (!rest) continue
|
|
68
|
+
userText = rest
|
|
69
|
+
} else if (m.role === 'assistant' && m.text) {
|
|
70
|
+
assistantText = String(m.text)
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return { userText, assistantText }
|
|
74
|
+
}
|
package/lib/m4-corpus.js
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* M4-2 Corpus Adapter — SourceCatalog/M3b sidecar 校验/CorpusRegistry(docs/M4-CONTRACT.md §7/§8)。
|
|
3
|
+
* 纯适配层:输入为受控 source paths 与磁盘 shadow-copy;不接触 live Host、不做 audit、不注入。
|
|
4
|
+
*
|
|
5
|
+
* 职责:
|
|
6
|
+
* 1) buildSourceCatalog({workspaceKey, userMemoryPath, workspaceMemoryPath, todayLogPath}):
|
|
7
|
+
* 固定顺序三源 catalog(user/workspace/workspace-log),canonical 化,sourceRef 稳定相对引用。
|
|
8
|
+
* 2) canonicalScopeGuard:sidecar.sourceFile 必须与 catalog canonical 完全一致;
|
|
9
|
+
* symlink/reparse 解析真实路径不得逃逸允许根(realpathSync)。
|
|
10
|
+
* 3) loadCorpusSnapshot(catalog, fsApi):逐源 parseSidecar→stat/read→fileDigest 比对→
|
|
11
|
+
* record byte range/digest 校验→CorpusRecord[](legacy/conflict/stale fail closed 记原因);
|
|
12
|
+
* memoryIndexVersion(canonical tuples);预算(sources≤3/records≤512/corpusBytes≤64MiB/单文件 5MiB)。
|
|
13
|
+
* 4) CorpusRegistry:fingerprint(stat size+mtimeMs)缓存;fingerprint 未变化复用 snapshot;
|
|
14
|
+
* 变化只 reload 受影响 source 并整体替换;completedKeys/recentHits 上限由调用方持有。
|
|
15
|
+
*/
|
|
16
|
+
import { readFileSync, existsSync, statSync, realpathSync } from 'node:fs'
|
|
17
|
+
import path from 'node:path'
|
|
18
|
+
import { createHash } from 'node:crypto'
|
|
19
|
+
import { parseSidecar } from './memory-anchor.js'
|
|
20
|
+
import { memoryIndexVersion as computeIndexVersion, SHADOW_LEXICAL_BUDGET_V1 as BUDGET } from './shadow-retrieval.js'
|
|
21
|
+
|
|
22
|
+
const sha256Hex = (buf) => createHash('sha256').update(buf).digest('hex')
|
|
23
|
+
const INDEX_MAX_FILE_BYTES = 5 * 1024 * 1024
|
|
24
|
+
|
|
25
|
+
/** Windows 稳定 canonical:resolve + 正斜杠 + 小写(大小写不敏感 FS)。 */
|
|
26
|
+
export function canonicalize(p) {
|
|
27
|
+
return path.resolve(String(p == null ? '' : p)).replace(/\\/g, '/').toLowerCase()
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** sourceRef 稳定相对引用(契约 §15.3:audit 只允许这三类前缀)。 */
|
|
31
|
+
function refFor(kind, file) {
|
|
32
|
+
const base = path.basename(file)
|
|
33
|
+
if (kind === 'user') return 'user:' + base
|
|
34
|
+
if (kind === 'workspace-log') return 'workspace-log:' + base
|
|
35
|
+
return 'workspace:' + base
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
/** §7.2 SourceCatalog:三源固定顺序;canonical 化;scope 授权由调用方受控路径决定。 */
|
|
40
|
+
export function buildSourceCatalog({ workspaceKey, userMemoryPath, workspaceMemoryPath, todayLogPath }) {
|
|
41
|
+
const entries = []
|
|
42
|
+
if (userMemoryPath) entries.push({ kind: 'user', scope: 'User', sourceClass: 'user-memory', file: userMemoryPath })
|
|
43
|
+
if (workspaceMemoryPath) entries.push({ kind: 'workspace', scope: 'Workspace', sourceClass: 'workspace-notes', file: workspaceMemoryPath })
|
|
44
|
+
if (todayLogPath) entries.push({ kind: 'workspace-log', scope: 'Workspace', sourceClass: 'workspace-log', file: todayLogPath })
|
|
45
|
+
const sources = entries.map((e) => ({
|
|
46
|
+
kind: e.kind, scope: e.scope, sourceClass: e.sourceClass,
|
|
47
|
+
file: path.resolve(e.file),
|
|
48
|
+
canonicalFile: canonicalize(e.file),
|
|
49
|
+
sourceRef: refFor(e.kind, e.file),
|
|
50
|
+
workspaceKey: String(workspaceKey || ''),
|
|
51
|
+
}))
|
|
52
|
+
return { workspaceKey: String(workspaceKey || ''), sources }
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* §7.3 scope guard:sidecar.sourceFile 必须与 catalog canonical 完全一致;
|
|
57
|
+
* 真实路径(realpath,解析 symlink/reparse/8.3)不得逃逸声明目录。
|
|
58
|
+
*/
|
|
59
|
+
export function canonicalScopeGuard(source, sidecarSourceFile, fsApi = { realpathSync }) {
|
|
60
|
+
if (canonicalize(sidecarSourceFile) !== canonicalize(source.file)) {
|
|
61
|
+
return { ok: false, reason: 'source-mismatch' }
|
|
62
|
+
}
|
|
63
|
+
let real
|
|
64
|
+
try { real = canonicalize(fsApi.realpathSync(source.file)) } catch (_) { return { ok: false, reason: 'sidecar-invalid' } }
|
|
65
|
+
const declaredRoot = canonicalize(path.dirname(path.resolve(source.file)))
|
|
66
|
+
if (!real.startsWith(declaredRoot)) return { ok: false, reason: 'cross-workspace' }
|
|
67
|
+
return { ok: true }
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// —— IO 抽象(测试注入;默认同步 node:fs)——
|
|
71
|
+
function existsSyncIo(io, p) { return io && io.existsSync ? io.existsSync(p) : existsSync(p) }
|
|
72
|
+
function readFileSyncIo(io, p) { return io && io.readFileSync ? io.readFileSync(p) : readFileSync(p) }
|
|
73
|
+
|
|
74
|
+
/** §7.3/§8 loader:sidecar→guard→文件 digest→record 校验→CorpusSnapshot(预算 fail closed)。 */
|
|
75
|
+
export function loadCorpusSnapshot(catalog, io = {}, opts = {}) {
|
|
76
|
+
const sidecarDir = io.sidecarDir
|
|
77
|
+
const dropped = []
|
|
78
|
+
const records = []
|
|
79
|
+
const versionSources = []
|
|
80
|
+
let loadedSources = 0
|
|
81
|
+
for (const source of catalog.sources) {
|
|
82
|
+
const canonHash = sha256Hex(Buffer.from(canonicalize(source.file), 'utf8'))
|
|
83
|
+
const sp = path.join(sidecarDir, canonHash + '.json')
|
|
84
|
+
if (!existsSyncIo(io, sp)) { dropped.push({ stage: 'corpus', reason: 'sidecar-missing', sourceRef: source.sourceRef }); continue }
|
|
85
|
+
let sideText
|
|
86
|
+
try { sideText = readFileSyncIo(io, sp) } catch (_) { dropped.push({ stage: 'corpus', reason: 'sidecar-invalid', sourceRef: source.sourceRef }); continue }
|
|
87
|
+
const parsed = parseSidecar(String(sideText))
|
|
88
|
+
if (!parsed.ok) { dropped.push({ stage: 'corpus', reason: 'sidecar-invalid', sourceRef: source.sourceRef }); continue }
|
|
89
|
+
const sc = parsed.sidecar
|
|
90
|
+
const g = canonicalScopeGuard(source, sc.sourceFile)
|
|
91
|
+
if (!g.ok) { dropped.push({ stage: 'corpus', reason: g.reason, sourceRef: source.sourceRef }); continue }
|
|
92
|
+
let buf
|
|
93
|
+
try { buf = readFileSyncIo(io, source.file) } catch (_) { dropped.push({ stage: 'corpus', reason: 'stale-source', sourceRef: source.sourceRef }); continue }
|
|
94
|
+
if (buf.length > INDEX_MAX_FILE_BYTES) { dropped.push({ stage: 'corpus', reason: 'oversized', sourceRef: source.sourceRef }); continue }
|
|
95
|
+
const fileDigest = sha256Hex(buf)
|
|
96
|
+
if (fileDigest !== sc.fileDigest) { dropped.push({ stage: 'corpus', reason: 'stale-source', sourceRef: source.sourceRef }); continue }
|
|
97
|
+
for (const r of (sc.records || [])) {
|
|
98
|
+
if (!Number.isInteger(r.byteStart) || !Number.isInteger(r.byteEnd) || r.byteStart < 0 || r.byteEnd > buf.length) {
|
|
99
|
+
dropped.push({ stage: 'corpus', reason: 'record-stale', memoryId: r.memoryId, sourceRef: source.sourceRef }); continue
|
|
100
|
+
}
|
|
101
|
+
if (sha256Hex(buf.subarray(r.byteStart, r.byteEnd)) !== r.recordDigest) {
|
|
102
|
+
dropped.push({ stage: 'corpus', reason: 'record-stale', memoryId: r.memoryId, sourceRef: source.sourceRef }); continue
|
|
103
|
+
}
|
|
104
|
+
records.push({
|
|
105
|
+
memoryId: r.memoryId, anchorId: r.anchorId,
|
|
106
|
+
scope: source.scope, sourceClass: source.sourceClass, sourceRef: source.sourceRef,
|
|
107
|
+
sourceEpoch: sc.sourceEpoch, sourceVersion: sc.sourceVersion,
|
|
108
|
+
fileDigest: sc.fileDigest, recordDigest: r.recordDigest,
|
|
109
|
+
lineStart: r.lineStart || 0, lineEnd: r.lineEnd || 0,
|
|
110
|
+
byteStart: r.byteStart, byteEnd: r.byteEnd,
|
|
111
|
+
heading: r.heading != null ? r.heading : null,
|
|
112
|
+
text: buf.toString('utf8', r.byteStart, Math.min(r.byteEnd, r.byteStart + BUDGET.recordScanKiB * 1024)),
|
|
113
|
+
bytes: r.bytes != null ? r.bytes : (r.byteEnd - r.byteStart),
|
|
114
|
+
})
|
|
115
|
+
}
|
|
116
|
+
loadedSources++
|
|
117
|
+
versionSources.push({ scope: source.scope, sourceRef: source.sourceRef, sourceEpoch: sc.sourceEpoch, sourceVersion: sc.sourceVersion, fileDigest: sc.fileDigest })
|
|
118
|
+
}
|
|
119
|
+
if (loadedSources > BUDGET.sourceFiles) return { ok: false, reason: 'source-budget', dropped }
|
|
120
|
+
if (records.length > BUDGET.corpusRecords) return { ok: false, reason: 'record-budget', dropped }
|
|
121
|
+
let totalBytes = 0
|
|
122
|
+
for (const r of records) totalBytes += Number(r.bytes || 0)
|
|
123
|
+
if (totalBytes > BUDGET.corpusBytes) return { ok: false, reason: 'corpus-byte-budget', dropped }
|
|
124
|
+
const miv = computeIndexVersion(versionSources)
|
|
125
|
+
return {
|
|
126
|
+
ok: true,
|
|
127
|
+
snapshot: {
|
|
128
|
+
memoryIndexVersion: miv,
|
|
129
|
+
sources: versionSources,
|
|
130
|
+
records,
|
|
131
|
+
counts: { sources: loadedSources, records: records.length, legacyConflicts: dropped.filter((d) => d.reason === 'legacy-conflict').length, rawHits: 0, kept: 0, dropped: dropped.length },
|
|
132
|
+
dropped,
|
|
133
|
+
},
|
|
134
|
+
dropped,
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** §14.3 stat fingerprint(size+mtimeMs)。 */
|
|
139
|
+
export function sourceFingerprint(file, fsApi = {}) {
|
|
140
|
+
try {
|
|
141
|
+
const st = (fsApi.statSync || statSync)(file)
|
|
142
|
+
return st.size + ':' + st.mtimeMs
|
|
143
|
+
} catch (_) { return null }
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** CorpusRegistry:fingerprint 缓存;未变化复用(零重读),变化整体重建。 */
|
|
147
|
+
export class CorpusRegistry {
|
|
148
|
+
constructor(opts = {}) { this.sidecarDir = opts.sidecarDir || null; this._cache = new Map() }
|
|
149
|
+
|
|
150
|
+
get(catalog) {
|
|
151
|
+
const key = canonicalize(catalog.workspaceKey || '__all__')
|
|
152
|
+
const cached = this._cache.get(key)
|
|
153
|
+
const fingerprints = new Map()
|
|
154
|
+
const changed = []
|
|
155
|
+
for (const s of catalog.sources) {
|
|
156
|
+
const fp = sourceFingerprint(s.file)
|
|
157
|
+
fingerprints.set(s.canonicalFile, fp)
|
|
158
|
+
const prevFp = cached && cached.fingerprints.get(s.canonicalFile)
|
|
159
|
+
if (prevFp !== fp) changed.push(s)
|
|
160
|
+
}
|
|
161
|
+
if (cached && changed.length === 0) return { ok: true, snapshot: cached.snapshot, reloaded: [], fromCache: true }
|
|
162
|
+
const res = loadCorpusSnapshot(catalog, { sidecarDir: this.sidecarDir }, {})
|
|
163
|
+
if (!res.ok) return res
|
|
164
|
+
this._cache.set(key, { fingerprints, snapshot: res.snapshot })
|
|
165
|
+
return { ok: true, snapshot: res.snapshot, reloaded: changed.map((c) => c.sourceRef), fromCache: false }
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
invalidate() { this._cache.clear() }
|
|
169
|
+
}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* M7-8 Host Index Sync Orchestrator(docs/PYTHON-SIDECAR-CONTRACT.md §19.10;修复 live blocker)。
|
|
3
|
+
*
|
|
4
|
+
* 根因(M7-8 live Phase E 实证):M7-1 的 index_sync 只实现了 plan/client 层,生产 Host
|
|
5
|
+
* 从未调用 buildIndexSyncPlansPre/sendIndexSyncPlanPre → Python worker 收不到全库语料,
|
|
6
|
+
* 无法建库;context_push 的 memoryRefs(top-8 lexical)不足以做语义检索。
|
|
7
|
+
*
|
|
8
|
+
* 本模块把授权 corpus snapshot → index_sync begin/page/commit 的编排接进 Host,并保证:
|
|
9
|
+
* - 默认关闭零 IO(assoc∧bridge∧pythonBackend∧sink='python' 四门全开才启用);
|
|
10
|
+
* - 输入必须是 loadCorpus(paths) 得到的已授权 M4 CorpusSnapshot(绝不自行读文件);
|
|
11
|
+
* - 每个 (workspaceRef, scope, memoryIndexVersion, workerEpoch) 最多成功同步一次;
|
|
12
|
+
* - worker 重启/epoch 变化 → 重新同步当前 index;
|
|
13
|
+
* - 新 memoryIndexVersion latest-wins;旧 in-flight sync abort/cancel;
|
|
14
|
+
* - 同一 workspace 的 Workspace/User plans 按确定顺序发送;
|
|
15
|
+
* - 任一失败结构化记录,允许下一有效 Segment 重试,绝不向未 ready 的 index 发 context_push;
|
|
16
|
+
* - 禁止在每个 Segment 重复全量 sync(成功后缓存 ready identity);
|
|
17
|
+
* - dispose 清理 in-flight/ready cache/abort controller,不删除 derived cache。
|
|
18
|
+
*
|
|
19
|
+
* 可观察性(最小投影,不泄内容):capturedPathKeys(≤8)、lastSegmentRuntimeKey、
|
|
20
|
+
* lastSegmentSessionRef、lastDrop{reason,contextVersion,runtimeKey}、readyState。
|
|
21
|
+
* UTF-8 无 BOM。
|
|
22
|
+
*/
|
|
23
|
+
import { buildIndexSyncPlansPre, sendIndexSyncPlanPre } from './index-sync.js'
|
|
24
|
+
import { workspaceRefOf } from './evidence-store.js'
|
|
25
|
+
|
|
26
|
+
export const M7_INDEX_SYNC_HOST_POLICY_VERSION = 'm7_index_sync_host_v1'
|
|
27
|
+
const MAX_PATH_KEYS = 8
|
|
28
|
+
const MAX_DROPS = 16
|
|
29
|
+
|
|
30
|
+
export function createIndexSyncHostPre(opts = {}) {
|
|
31
|
+
const engine = opts.engine
|
|
32
|
+
if (!engine) throw new Error('index-sync-host: engine required')
|
|
33
|
+
const readyCache = new Map() // key=(wsRef,scope) -> { miv, epoch, at }
|
|
34
|
+
const inFlight = new Map() // key=(wsRef,scope) -> { controller, promise, miv }
|
|
35
|
+
const enabledKeys = new Set() // 显式 disable 的 key(同一 miv 内不再重试;miv 变化自动解除)
|
|
36
|
+
const volatileDrops = [] // ≤16 条最小投影(无文本)
|
|
37
|
+
const stats = { syncsStarted: 0, syncsOk: 0, syncsFailed: 0, skippedCached: 0,
|
|
38
|
+
epochReset: 0, mivReplaced: 0, aborted: 0, drops: 0, readyHits: 0 }
|
|
39
|
+
|
|
40
|
+
function enabled() {
|
|
41
|
+
return engine.config.associativeMemoryEnabled === true &&
|
|
42
|
+
engine.config.contextBridgeEnabled === true &&
|
|
43
|
+
engine.config.pythonBackendEnabled === true &&
|
|
44
|
+
String(engine.config.contextSinkMode || 'null') === 'python'
|
|
45
|
+
}
|
|
46
|
+
function client() { return engine._pythonSidecar || null }
|
|
47
|
+
function keyOf(wsRef, scope) { return wsRef + '|' + scope }
|
|
48
|
+
function drop(reason, contextVersion, runtimeKey) {
|
|
49
|
+
volatileDrops.push({ at: Date.now(), reason, contextVersion, runtimeKey: String(runtimeKey || '').slice(0, 40) })
|
|
50
|
+
if (volatileDrops.length > MAX_DROPS) volatileDrops.shift()
|
|
51
|
+
stats.drops++
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** 当前 worker epoch(可能未启动);null 表示未启动。 */
|
|
55
|
+
function currentEpoch() {
|
|
56
|
+
const c = client()
|
|
57
|
+
if (!c || typeof c.currentEpoch !== 'function') return null
|
|
58
|
+
try { return c.currentEpoch() } catch (_) { return null }
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* 使指定 workspace/scope 的 index 就绪(幂等)。返回 {ok, ready, reason}。
|
|
63
|
+
* - ready=true 表示该 (wsRef,scope,miv,epoch) 已同步过(缓存命中)。
|
|
64
|
+
* - 新 epoch → 清除该 key 的缓存并重同步(worker 重启后内存态清零)。
|
|
65
|
+
* - 新 miv → latest-wins:abort 旧 in-flight,替换缓存,重同步。
|
|
66
|
+
* - 失败不抛,结构化记录,允许重试。
|
|
67
|
+
*/
|
|
68
|
+
async function ensureIndexReady(snapshot, paths, scope, opts = {}) {
|
|
69
|
+
const signal = (opts && opts.signal) || undefined
|
|
70
|
+
const ctxKey = String((opts && opts.runtimeKey) || '')
|
|
71
|
+
if (!enabled()) return { ok: false, ready: false, reason: 'disabled' }
|
|
72
|
+
const c = client()
|
|
73
|
+
if (!c) return { ok: false, ready: false, reason: 'no-client' }
|
|
74
|
+
if (!snapshot || !snapshot.records || !snapshot.records.length) return { ok: false, ready: false, reason: 'empty-corpus' }
|
|
75
|
+
const wsRef = workspaceRefOf(paths.workspaceKey)
|
|
76
|
+
const miv = String(snapshot.memoryIndexVersion || '')
|
|
77
|
+
if (!miv.startsWith('idx_')) return { ok: false, ready: false, reason: 'bad-miv' }
|
|
78
|
+
const epoch = currentEpoch()
|
|
79
|
+
const k = keyOf(wsRef, scope)
|
|
80
|
+
const cached = readyCache.get(k)
|
|
81
|
+
// epoch 变化(worker 重启) → 缓存失效,必须重同步
|
|
82
|
+
if (cached && epoch && cached.epoch !== epoch) {
|
|
83
|
+
readyCache.delete(k)
|
|
84
|
+
stats.epochReset++
|
|
85
|
+
drop('epoch-reset', 0, ctxKey)
|
|
86
|
+
}
|
|
87
|
+
// miv 变化 → 旧缓存/旧 in-flight 作废,latest-wins
|
|
88
|
+
if (cached && cached.miv !== miv) {
|
|
89
|
+
readyCache.delete(k)
|
|
90
|
+
stats.mivReplaced++
|
|
91
|
+
const infl = inFlight.get(k)
|
|
92
|
+
if (infl) { try { infl.controller.abort() } catch (_) {}; stats.aborted++ }
|
|
93
|
+
drop('miv-replaced', 0, ctxKey)
|
|
94
|
+
}
|
|
95
|
+
if (readyCache.has(k)) { stats.readyHits++; return { ok: true, ready: true, miv, epoch } }
|
|
96
|
+
if (inFlight.has(k)) {
|
|
97
|
+
const infl = inFlight.get(k)
|
|
98
|
+
// 同 key 已在同步中:若 miv 相同则等待;否则 abort 旧的(latest-wins)
|
|
99
|
+
if (infl.miv === miv) {
|
|
100
|
+
try { await infl.promise; return infl.result } catch (_) { return { ok: false, ready: false, reason: 'inflight-failed' } }
|
|
101
|
+
}
|
|
102
|
+
try { infl.controller.abort() } catch (_) {}
|
|
103
|
+
stats.aborted++
|
|
104
|
+
drop('miv-replaced-inflight', 0, ctxKey)
|
|
105
|
+
}
|
|
106
|
+
// 构建计划(Workspace→User 固定序;本函数只处理单个 scope 的计划)
|
|
107
|
+
const built = buildIndexSyncPlansPre({ snapshot, workspaceKey: paths.workspaceKey })
|
|
108
|
+
if (!built.ok) { drop('plan:' + built.reason, 0, ctxKey); return { ok: false, ready: false, reason: built.reason } }
|
|
109
|
+
const plan = built.plans.find((p) => p.scope === scope)
|
|
110
|
+
if (!plan) { drop('no-plan:' + scope, 0, ctxKey); return { ok: false, ready: false, reason: 'no-plan:' + scope } }
|
|
111
|
+
const controller = new AbortController()
|
|
112
|
+
const signal2 = signal || controller.signal
|
|
113
|
+
stats.syncsStarted++
|
|
114
|
+
// M7-8 live 修复:sync 帧用独立长超时(覆盖 BGE 加载+全量建库),不套 client 默认 5s——
|
|
115
|
+
// 否则 worker 首次加载/建库期间 begin/page 帧超时→重生成风暴(实测 syncsOk=0 死循环)
|
|
116
|
+
const syncTimeoutMs = Number((opts && opts.syncTimeoutMs) || 120000)
|
|
117
|
+
const promise = sendIndexSyncPlanPre(c, plan, { signal: signal2, timeoutMs: syncTimeoutMs })
|
|
118
|
+
.then((res) => {
|
|
119
|
+
if (res.ok) {
|
|
120
|
+
readyCache.set(k, { miv, epoch, at: Date.now() })
|
|
121
|
+
stats.syncsOk++
|
|
122
|
+
return { ok: true, ready: true, miv, epoch }
|
|
123
|
+
}
|
|
124
|
+
stats.syncsFailed++
|
|
125
|
+
drop('sync:' + (res.reason || res.phase || 'failed'), 0, ctxKey)
|
|
126
|
+
return { ok: false, ready: false, reason: (res.reason || res.phase || 'sync-failed') }
|
|
127
|
+
})
|
|
128
|
+
.catch((err) => {
|
|
129
|
+
const aborted = err && err.name === 'AbortError'
|
|
130
|
+
if (aborted) stats.aborted++
|
|
131
|
+
else stats.syncsFailed++
|
|
132
|
+
drop('sync:' + (aborted ? 'aborted' : String(err && err.message || 'error')), 0, ctxKey)
|
|
133
|
+
return { ok: false, ready: false, reason: aborted ? 'aborted' : 'sync-error' }
|
|
134
|
+
})
|
|
135
|
+
const entry = { controller, promise, miv, result: null }
|
|
136
|
+
entry.result = promise
|
|
137
|
+
inFlight.set(k, entry)
|
|
138
|
+
try {
|
|
139
|
+
const r = await promise
|
|
140
|
+
if (inFlight.get(k) === entry) inFlight.delete(k)
|
|
141
|
+
return r
|
|
142
|
+
} catch (_) {
|
|
143
|
+
if (inFlight.get(k) === entry) inFlight.delete(k)
|
|
144
|
+
return { ok: false, ready: false, reason: 'sync-error' }
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** 一次调用同步一个 workspace 的全部 scope(Workspace→User 固定序)。返回逐 scope 结果。 */
|
|
149
|
+
async function ensureWorkspaceIndexReady(snapshot, paths, opts = {}) {
|
|
150
|
+
if (!enabled()) return [{ ok: false, ready: false, reason: 'disabled' }]
|
|
151
|
+
const results = []
|
|
152
|
+
const scopes = ['Workspace', 'User']
|
|
153
|
+
for (const scope of scopes) {
|
|
154
|
+
const has = snapshot.records.some((r) => r.scope === scope)
|
|
155
|
+
if (!has) continue
|
|
156
|
+
results.push(await ensureIndexReady(snapshot, paths, scope, opts))
|
|
157
|
+
}
|
|
158
|
+
return results
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function debugView() {
|
|
162
|
+
if (!enabled()) return { enabled: false }
|
|
163
|
+
const c = client()
|
|
164
|
+
return {
|
|
165
|
+
enabled: true,
|
|
166
|
+
policyVersion: M7_INDEX_SYNC_HOST_POLICY_VERSION,
|
|
167
|
+
ready: [...readyCache.entries()].map(([k, v]) => ({ key: k, miv: v.miv, epoch: v.epoch ? v.epoch.slice(0, 12) : null })),
|
|
168
|
+
inFlightCount: inFlight.size,
|
|
169
|
+
capturedPathKeys: [...enabledKeys].slice(0, MAX_PATH_KEYS),
|
|
170
|
+
stats: { ...stats },
|
|
171
|
+
recentDrops: volatileDrops.slice(-4),
|
|
172
|
+
epoch: c && typeof c.currentEpoch === 'function' ? (c.currentEpoch() || '').slice(0, 12) : null,
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function dispose(reason) {
|
|
177
|
+
for (const [, infl] of inFlight) { try { infl.controller.abort() } catch (_) {} }
|
|
178
|
+
inFlight.clear()
|
|
179
|
+
readyCache.clear()
|
|
180
|
+
enabledKeys.clear()
|
|
181
|
+
volatileDrops.length = 0
|
|
182
|
+
stats.drops = 0
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
return {
|
|
186
|
+
ensureIndexReady,
|
|
187
|
+
ensureWorkspaceIndexReady,
|
|
188
|
+
debugView,
|
|
189
|
+
dispose,
|
|
190
|
+
_stats: stats,
|
|
191
|
+
_readyCacheForTest: readyCache,
|
|
192
|
+
_inFlightForTest: inFlight,
|
|
193
|
+
}
|
|
194
|
+
}
|