@kira905/dsh-memory-toolkit 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/CHANGELOG.md +42 -0
  2. package/LICENSE +661 -0
  3. package/README.en.md +110 -0
  4. package/README.md +102 -2
  5. package/dsh-memory-local/FORK-BASE.json +24 -0
  6. package/dsh-memory-local/README.md +113 -0
  7. package/dsh-memory-local/build.mjs +101 -0
  8. package/dsh-memory-local/cordis.patch.yml +20 -0
  9. package/dsh-memory-local/deploy.mjs +306 -0
  10. package/dsh-memory-local/entity-status.mjs +75 -0
  11. package/dsh-memory-local/lib/bridge.mjs +108 -0
  12. package/dsh-memory-local/lib/host-adapter.js +1171 -0
  13. package/dsh-memory-local/lib/index.d.ts +128 -0
  14. package/dsh-memory-local/lib/index.js +944 -0
  15. package/dsh-memory-local/lib/injector.mjs +400 -0
  16. package/dsh-memory-local/lib/memory-read.mjs +116 -0
  17. package/dsh-memory-local/lib/module-gate.mjs +68 -0
  18. package/dsh-memory-local/lib/pointer.mjs +152 -0
  19. package/dsh-memory-local/lib/prod-gate.mjs +121 -0
  20. package/dsh-memory-local/lib/recall-log.mjs +118 -0
  21. package/dsh-memory-local/lib/recall.mjs +192 -0
  22. package/dsh-memory-local/lib/schema-migrate.mjs +365 -0
  23. package/dsh-memory-local/lib/user-message.mjs +32 -0
  24. package/dsh-memory-local/lib/vectors-read.mjs +110 -0
  25. package/dsh-memory-local/package.json +51 -0
  26. package/dsh-memory-local/restore.mjs +259 -0
  27. package/dsh-memory-local/rollback-injector-fix.mjs +166 -0
  28. package/dsh-memory-local/src/bridge.mjs +108 -0
  29. package/dsh-memory-local/src/host-adapter.js +1171 -0
  30. package/dsh-memory-local/src/index.d.ts +128 -0
  31. package/dsh-memory-local/src/index.js +944 -0
  32. package/dsh-memory-local/src/injector.mjs +400 -0
  33. package/dsh-memory-local/src/memory-read.mjs +116 -0
  34. package/dsh-memory-local/src/module-gate.mjs +68 -0
  35. package/dsh-memory-local/src/pointer.mjs +152 -0
  36. package/dsh-memory-local/src/recall-log.mjs +118 -0
  37. package/dsh-memory-local/src/recall.mjs +192 -0
  38. package/dsh-memory-local/src/schema-migrate.mjs +365 -0
  39. package/dsh-memory-local/src/user-message.mjs +32 -0
  40. package/dsh-memory-local/src/vectors-read.mjs +110 -0
  41. package/dsh-memory-local/test/fixtures-s2b.mjs +216 -0
  42. package/dsh-memory-local/test/probe-entity.mjs +170 -0
  43. package/dsh-memory-local/test/regression.mjs +270 -0
  44. package/dsh-memory-local/test/s2b.test.mjs +568 -0
  45. package/dsh-memory-local/test/schema-migrate.test.mjs +334 -0
  46. package/dsh-memory-local/test/test-adapter-converge.mjs +127 -0
  47. package/dsh-memory-local/test/visible-injections.test.mjs +239 -0
  48. package/dsh-memory-local/verify-after-restart.mjs +181 -0
  49. package/lib/deploy-guard.mjs +391 -0
  50. package/lib/guard-http.mjs +93 -0
  51. package/lib/pitfall-form-map.mjs +130 -0
  52. package/lib/pitfall-score.mjs +185 -0
  53. package/lib/session-file-name.mjs +160 -0
  54. package/lib/zstd-frames.mjs +78 -0
  55. package/memory-db-tools/README.md +68 -0
  56. package/memory-db-tools/_test/test-memory-scope.mjs +302 -0
  57. package/memory-db-tools/audit-analyze.mjs +352 -0
  58. package/memory-db-tools/audit-dump-ids.mjs +245 -0
  59. package/memory-db-tools/audit-export.mjs +270 -0
  60. package/memory-db-tools/audit-hit-probe.mjs +227 -0
  61. package/memory-db-tools/audit-ledger-probe.mjs +236 -0
  62. package/memory-db-tools/audit-persona-overlap.mjs +277 -0
  63. package/memory-db-tools/audit-struct-probe.mjs +213 -0
  64. package/memory-db-tools/audit-verify-unchanged.mjs +276 -0
  65. package/memory-db-tools/check-memory-db.mjs +8 -0
  66. package/memory-db-tools/dump-memory-db.mjs +12 -0
  67. package/memory-db-tools/lib/memory-scope.mjs +446 -0
  68. package/memory-db-tools/list-memory-summary.mjs +477 -0
  69. package/memory-db-tools/memory-identity-check.mjs +101 -0
  70. package/memory-db-tools/memory-schema-migrate.mjs +290 -0
  71. package/memory-db-tools/memory-scope-fixture.mjs +197 -0
  72. package/memory-db-tools/read-memories.mjs +26 -0
  73. package/memory-db-tools/read-memory-db.mjs +20 -0
  74. package/memory-db-tools/refine-memories.mjs +247 -0
  75. package/memory-db-tools/show-mem-times.mjs +22 -0
  76. package/memory-engine/README.md +70 -0
  77. package/memory-engine/bench/bench.mjs +221 -0
  78. package/memory-engine/cli/distill-rollback.mjs +86 -0
  79. package/memory-engine/cli/distill.mjs +156 -0
  80. package/memory-engine/cli/doc-index-rebuild.mjs +155 -0
  81. package/memory-engine/cli/doc-index.mjs +48 -0
  82. package/memory-engine/cli/domain-tags.mjs +251 -0
  83. package/memory-engine/cli/pitfall-hook.mjs +91 -0
  84. package/memory-engine/cli/recall.mjs +132 -0
  85. package/memory-engine/cli/selfcheck.mjs +57 -0
  86. package/memory-engine/cli/serve-start.mjs +247 -0
  87. package/memory-engine/cli/sweep.mjs +58 -0
  88. package/memory-engine/cli/sync.mjs +69 -0
  89. package/memory-engine/lib/cli-util.mjs +197 -0
  90. package/memory-engine/lib/config.mjs +135 -0
  91. package/memory-engine/lib/dirty-ledger.mjs +174 -0
  92. package/memory-engine/lib/distill-rollback.mjs +178 -0
  93. package/memory-engine/lib/distill.mjs +898 -0
  94. package/memory-engine/lib/doc-index-store.mjs +325 -0
  95. package/memory-engine/lib/doc-index.mjs +529 -0
  96. package/memory-engine/lib/doc-scan.mjs +271 -0
  97. package/memory-engine/lib/doc-summarize.mjs +305 -0
  98. package/memory-engine/lib/domain-tags.mjs +429 -0
  99. package/memory-engine/lib/embedder.mjs +242 -0
  100. package/memory-engine/lib/hashes.mjs +12 -0
  101. package/memory-engine/lib/memory-source.mjs +127 -0
  102. package/memory-engine/lib/pitfall-source.mjs +170 -0
  103. package/memory-engine/lib/policy-source.mjs +371 -0
  104. package/memory-engine/lib/recall.mjs +331 -0
  105. package/memory-engine/lib/selfcheck.mjs +131 -0
  106. package/memory-engine/lib/sweep.mjs +159 -0
  107. package/memory-engine/lib/sync.mjs +180 -0
  108. package/memory-engine/lib/tick-hook.mjs +218 -0
  109. package/memory-engine/lib/vectors-store.mjs +292 -0
  110. package/memory-engine/package-lock.json +1086 -0
  111. package/memory-engine/package.json +22 -0
  112. package/memory-engine/serve.mjs +171 -0
  113. package/memory-engine/test/01-store.test.mjs +127 -0
  114. package/memory-engine/test/02-sync.test.mjs +196 -0
  115. package/memory-engine/test/03-sweep.test.mjs +145 -0
  116. package/memory-engine/test/04-recall.test.mjs +214 -0
  117. package/memory-engine/test/05-selfcheck.test.mjs +175 -0
  118. package/memory-engine/test/06-doc-index.test.mjs +928 -0
  119. package/memory-engine/test/07-domain-tags.test.mjs +298 -0
  120. package/memory-engine/test/08-distill.test.mjs +686 -0
  121. package/memory-engine/test/09-pitfall-source.test.mjs +365 -0
  122. package/memory-engine/test/10-tick-hook.test.mjs +252 -0
  123. package/memory-engine/test/11-policy-source.test.mjs +224 -0
  124. package/memory-engine/test/_tickhook-worker.mjs +56 -0
  125. package/memory-engine/test/fixtures.mjs +205 -0
  126. package/memory-engine/test/harness.mjs +73 -0
  127. package/memory-engine/test/run.mjs +35 -0
  128. package/memory-engine/tools/bridge-smoke.mjs +80 -0
  129. package/memory-engine/tools/fetch-model.mjs +128 -0
  130. package/memory-engine/tools/smoke-recall.mjs +76 -0
  131. package/package.json +53 -4
  132. package/persona-memory-audit.mjs +217 -0
@@ -0,0 +1,529 @@
1
+ // 记忆引擎 v2 · 离线层 —— 文档索引器主编排(方案 §3.2 / §5.2 / §5.4 / §7)
2
+ //
3
+ // 一轮做的事(顺序即语义):
4
+ // ① 扫描面(`docs/**/*.md`,排除 4 类)+ 取范围(首期 10 篇 / --all / --only)
5
+ // ② 差分:比对 state 里的 `mtime/size` + 库中是否已有条 ⇒ **没变的一篇都不碰**
6
+ // (「变了才 UPSERT 该篇」= §5.4 去重键口径;幂等靠 path 主键)
7
+ // ③ 变了 → 生成摘要(rule 确定性 / llm 受 §3.5 成本闸)→ 逐批 UPSERT(busy_timeout + 重试 3 次 / 退避 ≤60s)
8
+ // ④ 清孤条(文档变短后残留的 `path#§N`)
9
+ // ⑤ 写 state(原子) + 日报一行;**任何失败都只记不抛**(卡要求"不报错给主人")
10
+ import fs from 'node:fs';
11
+ import os from 'node:os';
12
+ import path from 'node:path';
13
+ import { textHash } from './hashes.mjs';
14
+ import {
15
+ DEFAULT_DOCS_ROOT,
16
+ DEFAULT_DSH_ROOT,
17
+ FIRST_BATCH,
18
+ FIRST_BATCH_SOURCE,
19
+ parseStructure,
20
+ planRows,
21
+ readDoc,
22
+ scanDocs,
23
+ } from './doc-scan.mjs';
24
+ import { emptyCost, resolveApiKey, rolloverCost, summarizeAll } from './doc-summarize.mjs';
25
+ import {
26
+ META_KEYS,
27
+ deleteStaleRows,
28
+ hasDocRows,
29
+ listRows,
30
+ openDocIndexStore,
31
+ setMeta,
32
+ upsertDocs,
33
+ withBusyRetry,
34
+ } from './doc-index-store.mjs';
35
+
36
+ export const DOC_INDEX_STATE_SCHEMA = 1;
37
+ export const HISTORY_MAX = 20;
38
+ /** 小批量(方案 §5.2:独立库各自加锁 + 小批量)。 */
39
+ export const DEFAULT_BATCH = 5;
40
+
41
+ // ---------------------------------------------------------------- state
42
+
43
+ export function readState(file) {
44
+ try {
45
+ const obj = JSON.parse(fs.readFileSync(file, 'utf8'));
46
+ return obj && typeof obj === 'object' ? obj : null;
47
+ } catch {
48
+ return null; // 缺失/损坏一律当"首次运行"(state 是可重建载体)
49
+ }
50
+ }
51
+
52
+ export function writeState(file, obj) {
53
+ try {
54
+ fs.mkdirSync(path.dirname(file), { recursive: true });
55
+ const tmp = `${file}.tmp-${process.pid}`;
56
+ fs.writeFileSync(tmp, `${JSON.stringify(obj, null, 2)}\n`, 'utf8');
57
+ fs.renameSync(tmp, file); // 同目录 rename:原子且不跨盘(跨盘 rename 必炸,铁律)
58
+ return true;
59
+ } catch {
60
+ return false;
61
+ }
62
+ }
63
+
64
+ export function pushHistory(prev, entry, max = HISTORY_MAX) {
65
+ const hist = Array.isArray(prev?.history) ? prev.history.slice(-(max - 1)) : [];
66
+ hist.push(entry);
67
+ while (hist.length > max) hist.shift();
68
+ return hist;
69
+ }
70
+
71
+ // ---------------------------------------------------------------- 范围与差分
72
+
73
+ /** 解析本轮范围:`--all` 全扫描面 / `--only a,b` 指定 / 默认首期 10 篇。 */
74
+ export function resolveScope({ scanned, flags = {} }) {
75
+ const byRel = new Map(scanned.map((d) => [d.rel, d]));
76
+ if (flags.all || flags.scope === 'all') {
77
+ return { mode: 'all', source: '扫描面全量(--all)', docs: scanned, missing: [] };
78
+ }
79
+ const only =
80
+ typeof flags.only === 'string' && flags.only.trim()
81
+ ? flags.only.split(',').map((s) => s.trim()).filter(Boolean)
82
+ : null;
83
+ const want = only ?? FIRST_BATCH;
84
+ const docs = [];
85
+ const missing = [];
86
+ for (const rel of want) {
87
+ const hit = byRel.get(rel);
88
+ if (hit) docs.push(hit);
89
+ else missing.push(rel);
90
+ }
91
+ return {
92
+ mode: only ? 'only' : 'first-batch',
93
+ source: only ? '--only 显式指定' : FIRST_BATCH_SOURCE,
94
+ docs,
95
+ missing,
96
+ };
97
+ }
98
+
99
+ /**
100
+ * 是否要重建该篇。判据(方案 §3.2「比对 mtime/size/text_hash,变了才 UPSERT 该篇」):
101
+ * state 里 mtime/size 未变 **且** 上次结果 ok **且** 库中已有该篇主条 ⇒ 跳过,一个字节都不碰。
102
+ * @param {{doc:object, prevEntry:object|null, hasRow:boolean, force:boolean}} p
103
+ */
104
+ export function needsRebuild({ doc, prevEntry, hasRow, force = false }) {
105
+ if (force) return { rebuild: true, reason: 'forced' };
106
+ if (!prevEntry) return { rebuild: true, reason: 'new' };
107
+ if (Number(prevEntry.mtime) !== Number(doc.mtime) || Number(prevEntry.size) !== Number(doc.size)) {
108
+ return { rebuild: true, reason: 'mtime-or-size-changed' };
109
+ }
110
+ // ⚠️ 只有 **failed** 才强制重算;'ok'(本轮重建过)与 'skipped'(本轮已确认最新)都表示"已同步"。
111
+ // 实测踩过:把 'skipped' 也当"未同步" ⇒ 跳过轮与重建轮来回振荡(永远有一半的轮次在重算摘要)。
112
+ if (prevEntry.lastResult === 'failed') return { rebuild: true, reason: 'last-failed' };
113
+ if (!hasRow) return { rebuild: true, reason: 'missing-in-db' };
114
+ return { rebuild: false, reason: 'unchanged' };
115
+ }
116
+
117
+ // ---------------------------------------------------------------- 日报一行
118
+
119
+ /** 日报一行文案(纯函数;state 与日报段共用)。 */
120
+ export function renderDailyLine(state) {
121
+ const r = state?.lastRun ?? {};
122
+ const cost = state?.cost ?? { calls: 0, tokens: 0 };
123
+ const failed = Number(r.failedCount ?? (Array.isArray(r.failed) ? r.failed.length : 0));
124
+ const head = `📄 文档索引(doc-index):${r.totalDocs ?? 0} 篇 / ${r.totalRows ?? 0} 条`;
125
+ const detail = `本轮 重建 ${r.rebuilt ?? 0} · 跳过 ${r.skipped ?? 0} · 写库 ${r.upserted ?? 0} · 清孤条 ${r.staleDeleted ?? 0}`;
126
+ const tail = `失败 ${failed} · 成本 ${cost.calls ?? 0} 次调用 ${cost.tokens ?? 0} token`;
127
+ const warn = failed ? ` ⚠️ ${String(r.failed?.[0]?.error ?? '').slice(0, 80)}` : '';
128
+ return `${head} · ${detail} · ${tail}${warn}`;
129
+ }
130
+
131
+ /** 当日日报路径(与 report.mjs 同口径:`<docsDir>\管家报告\<YYYY-MM-DD>-<host>.md`)。 */
132
+ export function dailyReportPath({ docsDir = DEFAULT_DOCS_ROOT, host, now = Date.now() } = {}) {
133
+ const d = new Date(now);
134
+ const p = (n) => String(n).padStart(2, '0');
135
+ const date = `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`;
136
+ return path.join(docsDir, '管家报告', `${date}-${host || os.hostname()}.md`);
137
+ }
138
+
139
+ export const DAILY_SECTION_TITLE = '## 📄 文档索引(doc-index)';
140
+
141
+ /**
142
+ * 幂等更新当日日报里的「📄 文档索引」段:**文件不存在则不新建**
143
+ * (照既有先例「日报缺失 → 跳过不报错」;日报是 report.mjs 的产物,本模块只往里补一行)。
144
+ * @returns {{ok:boolean, file:string, reason?:string, created?:boolean}}
145
+ */
146
+ export function updateDailySection({ file, line, dryRun = false }) {
147
+ let txt;
148
+ try {
149
+ txt = fs.readFileSync(file, 'utf8');
150
+ } catch {
151
+ return { ok: false, file, reason: 'daily-missing' };
152
+ }
153
+ const lines = txt.split(/\r?\n/);
154
+ const idx = lines.findIndex((l) => l.trim() === DAILY_SECTION_TITLE);
155
+ const block = [DAILY_SECTION_TITLE, '', `> ${line}`, ''];
156
+ let out;
157
+ let created = false;
158
+ if (idx < 0) {
159
+ out = `${txt.replace(/\s*$/, '')}\n\n${block.join('\n')}`;
160
+ created = true;
161
+ } else {
162
+ // 段体 = 标题之后到下一个 `## ` 之前
163
+ let end = lines.length;
164
+ for (let j = idx + 1; j < lines.length; j += 1) {
165
+ if (/^##\s/.test(lines[j])) {
166
+ end = j;
167
+ break;
168
+ }
169
+ }
170
+ out = [...lines.slice(0, idx), ...block, ...lines.slice(end)].join('\n');
171
+ if (out === txt) return { ok: true, file, reason: 'unchanged', created: false };
172
+ }
173
+ if (dryRun) return { ok: true, file, reason: 'dry-run', created };
174
+ try {
175
+ const tmp = `${file}.tmp-${process.pid}`;
176
+ fs.writeFileSync(tmp, out, 'utf8');
177
+ fs.renameSync(tmp, file);
178
+ } catch (e) {
179
+ return { ok: false, file, reason: `write-failed:${String((e && e.message) || e)}` };
180
+ }
181
+ return { ok: true, file, created };
182
+ }
183
+
184
+ // ---------------------------------------------------------------- 主流程
185
+
186
+ /**
187
+ * 跑一轮文档索引(纯编排;外部依赖可注入 ⇒ 单测零副作用、零网络)。
188
+ * @returns {Promise<{ok:boolean, exitCode:number, status:string, stats:object, state:object, lines:string[]}>}
189
+ */
190
+ export async function runDocIndexOnce({ flags = {}, env = process.env, now = () => Date.now(), deps = {} } = {}) {
191
+ const lines = [];
192
+ const log = (m) => lines.push(String(m));
193
+ const dryRun = Boolean(flags['dry-run']);
194
+ const force = Boolean(flags.rebuild) || Boolean(flags.force);
195
+ const batchSize = Number(flags.batch ?? DEFAULT_BATCH) || DEFAULT_BATCH;
196
+ const useLlm = Boolean(flags.llm);
197
+ const startedAt = now();
198
+
199
+ const docsRoot = path.resolve(String(flags['docs-root'] || env.DOC_INDEX_DOCS_ROOT || DEFAULT_DOCS_ROOT));
200
+ const dshRoot = path.resolve(String(flags['dsh-root'] || DEFAULT_DSH_ROOT));
201
+ const home = path.resolve(String(flags.home || env.DSH_HOME || path.join(os.homedir(), '.dsh')));
202
+ const dbPath = path.resolve(
203
+ String(flags['doc-index-db'] || env.DOC_INDEX_DB || path.join(home, 'memory-vectors', 'doc-index.db')),
204
+ );
205
+ const stateFile = path.resolve(String(flags['state-file'] || path.join(home, 'butler-doc-index-state.json')));
206
+ const hostId = flags.host || os.hostname();
207
+
208
+ const readStateFn = deps.readState ?? readState;
209
+ const writeStateFn = deps.writeState ?? writeState;
210
+ const openStore = deps.openStore ?? openDocIndexStore;
211
+ const scanFn = deps.scanDocs ?? scanDocs;
212
+ const readDocFn = deps.readDoc ?? readDoc;
213
+ const summarizeFn = deps.summarizeAll ?? summarizeAll;
214
+ const upsertFn = deps.upsertDocs ?? upsertDocs;
215
+ const deleteStaleFn = deps.deleteStaleRows ?? deleteStaleRows;
216
+
217
+ const prev = readStateFn(stateFile) ?? {};
218
+ const d = new Date(startedAt);
219
+ const dateKey = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`;
220
+ const cost = rolloverCost(prev.cost ?? emptyCost(dateKey), dateKey);
221
+
222
+ const stats = {
223
+ scope: null,
224
+ scopeSource: null,
225
+ docsRoot,
226
+ dbPath,
227
+ stateFile,
228
+ scanned: 0,
229
+ considered: 0,
230
+ totalDocs: 0,
231
+ totalRows: 0,
232
+ rebuilt: 0,
233
+ skipped: 0,
234
+ upserted: 0,
235
+ inserted: 0,
236
+ updated: 0,
237
+ unchanged: 0,
238
+ staleDeleted: 0,
239
+ longDocs: 0,
240
+ failures: [],
241
+ failed: [],
242
+ failedCount: 0,
243
+ missing: [],
244
+ batches: 0,
245
+ busyWaits: 0,
246
+ llmCalls: 0,
247
+ llmTokens: 0,
248
+ rowsByDoc: {},
249
+ decisions: {},
250
+ startedAt,
251
+ finishedAt: 0,
252
+ elapsedMs: 0,
253
+ };
254
+
255
+ const finish = (status, extra = {}) => {
256
+ stats.finishedAt = now();
257
+ stats.elapsedMs = stats.finishedAt - stats.startedAt;
258
+ const state = {
259
+ ...prev,
260
+ schemaVersion: DOC_INDEX_STATE_SCHEMA,
261
+ hostId,
262
+ docsRoot,
263
+ dbPath,
264
+ stateFile,
265
+ runs: (prev.runs ?? 0) + 1,
266
+ lastRunAt: new Date(stats.finishedAt).toISOString(),
267
+ lastStatus: status,
268
+ lastRun: stats,
269
+ paths: extra.paths ?? prev.paths ?? {},
270
+ history: pushHistory(prev, {
271
+ at: new Date(stats.finishedAt).toISOString(),
272
+ status,
273
+ rebuilt: stats.rebuilt,
274
+ skipped: stats.skipped,
275
+ upserted: stats.upserted,
276
+ failed: stats.failedCount,
277
+ }),
278
+ cost: extra.cost ?? cost,
279
+ };
280
+ state.lastDailyLine = renderDailyLine(state);
281
+ if (!dryRun) writeStateFn(stateFile, state);
282
+ return state;
283
+ };
284
+
285
+ // ① 扫描面 + 范围
286
+ const scanned = scanFn(docsRoot, { dshRoot });
287
+ stats.scanned = scanned.length;
288
+ const scope = resolveScope({ scanned, flags });
289
+ stats.scope = scope.mode;
290
+ stats.scopeSource = scope.source;
291
+ stats.considered = scope.docs.length;
292
+ stats.missing = scope.missing;
293
+ log(
294
+ `[doc-index] 扫描面 ${scanned.length} 篇(已排除 变更流水/管家报告/坑库/_archive)`
295
+ + ` · 本轮范围 ${scope.mode} = ${scope.docs.length} 篇(${scope.source})`,
296
+ );
297
+ if (scope.missing.length) {
298
+ log(`[doc-index] ⚠️ 清单里 ${scope.missing.length} 篇不在扫描面内:${scope.missing.join('、')}`);
299
+ }
300
+
301
+ if (!scope.docs.length) {
302
+ const state = finish('empty', { paths: prev.paths ?? {} });
303
+ return { ok: true, exitCode: 0, status: 'empty', stats, state, lines };
304
+ }
305
+
306
+ // 打开独立库(dry-run 且库不存在 ⇒ 不建库、不建目录)
307
+ let store = null;
308
+ if (!dryRun || fs.existsSync(dbPath)) {
309
+ store = openStore(dbPath, { create: !dryRun, hostId });
310
+ }
311
+ const db = store?.db ?? null;
312
+ let summaries = { byKey: new Map(), cost, calls: 0, tokens: 0, stopped: null, failures: [], skippedForFamily: [] };
313
+
314
+ try {
315
+ // ② 差分
316
+ const prevPaths = prev.paths ?? {};
317
+ const rebuildList = [];
318
+ for (const doc of scope.docs) {
319
+ const hasRow = db ? hasDocRows(db, doc.rel) : Boolean(prevPaths[doc.rel]);
320
+ const decision = needsRebuild({ doc, prevEntry: prevPaths[doc.rel] ?? null, hasRow, force });
321
+ stats.decisions[doc.rel] = decision;
322
+ if (!decision.rebuild) {
323
+ stats.skipped += 1;
324
+ log(`[doc-index] 跳过 ${doc.rel}(未变:mtime/size 与 state 一致且库中已有条)`);
325
+ continue;
326
+ }
327
+ rebuildList.push({ doc, reason: decision.reason });
328
+ }
329
+
330
+ // ③ 读文档 + 解析 + 切条 + 生成摘要
331
+ const planMap = new Map();
332
+ const docsForSummary = [];
333
+ for (const item of rebuildList) {
334
+ try {
335
+ const { text, chars } = readDocFn(item.doc.abs);
336
+ const structure = parseStructure(text);
337
+ const doc = { ...item.doc, chars, structure };
338
+ planMap.set(doc.rel, { doc, rows: planRows(doc), reason: item.reason });
339
+ docsForSummary.push(doc);
340
+ } catch (e) {
341
+ stats.failures.push({ path: item.doc.rel, error: `读取/解析失败:${String((e && e.message) || e)}` });
342
+ }
343
+ }
344
+
345
+ if (docsForSummary.length) {
346
+ summaries = await summarizeFn({
347
+ docs: docsForSummary,
348
+ plans: new Map([...planMap].map(([rel, v]) => [rel, v.rows])),
349
+ mode: useLlm ? 'llm' : 'rule',
350
+ cost,
351
+ date: dateKey,
352
+ llm: useLlm ? buildLlmConfig(flags, env, home) : null,
353
+ deps,
354
+ });
355
+ for (const f of summaries.failures ?? []) stats.failures.push(f);
356
+ stats.llmCalls = Number(summaries.calls) || 0;
357
+ stats.llmTokens = Number(summaries.tokens) || 0;
358
+ if (summaries.stopped) log(`[doc-index] ⛔ 成本闸:${summaries.stopped} ⇒ 本轮剩余文档停止(不重试)`);
359
+ }
360
+
361
+ // ④ 分批 UPSERT
362
+ const payload = [];
363
+ for (const [rel, entry] of planMap) {
364
+ const usable = entry.rows.filter((r) => summaries.byKey.has(r.key));
365
+ if (usable.length !== entry.rows.length) {
366
+ stats.failures.push({
367
+ path: rel,
368
+ error: `摘要缺失 ${entry.rows.length - usable.length}/${entry.rows.length} 条(成本闸中止或上游失败)⇒ 该篇跳过,下轮重试`,
369
+ });
370
+ }
371
+ if (!usable.length) continue;
372
+ stats.rebuilt += 1;
373
+ stats.rowsByDoc[rel] = usable.length;
374
+ if (entry.rows.length > 1) stats.longDocs += 1;
375
+ for (const r of usable) {
376
+ const s = summaries.byKey.get(r.key);
377
+ payload.push({
378
+ path: r.key,
379
+ mtime: entry.doc.mtime,
380
+ size: entry.doc.size,
381
+ textHash: s.textHash,
382
+ summary: s.summary,
383
+ ownerUser: flags['owner-user'] || 'LiJianXiao',
384
+ visibility: flags.visibility || 'private_owner',
385
+ updatedAt: entry.doc.mtime, // 索引条描述的是**该文档的 mtime**(方案 §3.2 索引条形态)
386
+ });
387
+ }
388
+ }
389
+ stats.totalRows = Object.values(stats.rowsByDoc).reduce((a, b) => a + b, 0);
390
+ stats.totalDocs = Object.keys(stats.rowsByDoc).length;
391
+
392
+ if (payload.length && !dryRun) {
393
+ for (let i = 0; i < payload.length; i += batchSize) {
394
+ const chunk = payload.slice(i, i + batchSize);
395
+ stats.batches += 1;
396
+ const r = await withBusyRetry(() => upsertFn(db, chunk), {
397
+ sleep: deps.sleep,
398
+ delaysMs: deps.retryDelaysMs,
399
+ onRetry: (n, e) => {
400
+ stats.busyWaits += 1;
401
+ log(`[doc-index] 库被锁,退避重试第 ${n} 次(${String((e && e.message) || e)})`);
402
+ },
403
+ });
404
+ if (!r.ok) {
405
+ // 超限记 failed(不报错给主人);细到每篇,下轮只重跑这些
406
+ for (const c of chunk) {
407
+ const rel = String(c.path).split('#§')[0];
408
+ if (!stats.failures.some((f) => f.path === rel)) stats.failures.push({ path: rel, error: r.error });
409
+ }
410
+ log(`[doc-index] ❌ 批 ${stats.batches} 写库失败(放弃该批,下轮重试):${r.error}`);
411
+ continue;
412
+ }
413
+ stats.upserted += r.value.inserted + r.value.updated;
414
+ stats.inserted += r.value.inserted;
415
+ stats.updated += r.value.updated;
416
+ stats.unchanged += r.value.unchanged;
417
+ }
418
+ } else if (payload.length && dryRun) {
419
+ log(`[doc-index] --dry-run:本应写 ${payload.length} 条(未写库)`);
420
+ }
421
+
422
+ // ⑤ 清孤条(只清已重建的篇;跳过的与失败的都不动)
423
+ if (!dryRun && db) {
424
+ for (const [rel, entry] of planMap) {
425
+ const keep = entry.rows.filter((r) => summaries.byKey.has(r.key)).map((r) => r.key);
426
+ if (!keep.length) continue;
427
+ const r = await withBusyRetry(() => deleteStaleFn(db, rel, keep), { sleep: deps.sleep, delaysMs: deps.retryDelaysMs });
428
+ if (r.ok) stats.staleDeleted += Number(r.value) || 0;
429
+ else stats.failures.push({ path: rel, error: `清孤条失败:${r.error}` });
430
+ }
431
+ }
432
+
433
+ // ⑥ meta
434
+ if (!dryRun && db) {
435
+ await withBusyRetry(
436
+ () => {
437
+ setMeta(db, META_KEYS.hostId, hostId);
438
+ setMeta(db, META_KEYS.lastRunAt, new Date(stats.finishedAt || now()).toISOString());
439
+ setMeta(
440
+ db,
441
+ META_KEYS.lastRunStats,
442
+ JSON.stringify({
443
+ rebuilt: stats.rebuilt,
444
+ skipped: stats.skipped,
445
+ upserted: stats.upserted,
446
+ failed: stats.failures.length,
447
+ }),
448
+ );
449
+ },
450
+ { sleep: deps.sleep },
451
+ );
452
+ // 规模以**库内真实值**为准:跳过轮(本轮没重建任何一篇)也必须显示真实规模,
453
+ // 不能按"本轮写了多少"算 —— 否则日报一行会在跳过轮显示 "0 篇 / 0 条"(实测踩过)。
454
+ try {
455
+ const all = (deps.listRows ?? listRows)(db);
456
+ stats.totalRows = all.length;
457
+ stats.totalDocs = new Set(all.map((r) => String(r.path).split('#§')[0])).size;
458
+ } catch {
459
+ /* 查询失败保持既有值 */
460
+ }
461
+ }
462
+ } catch (e) {
463
+ stats.failures.push({ path: '(run)', error: `运行异常:${String((e && e.message) || e)}` });
464
+ log(`[doc-index] ❌ 运行异常(只记失败,不影响其它机制):${String((e && e.message) || e)}`);
465
+ } finally {
466
+ try {
467
+ if (store) store.db.close();
468
+ } catch {
469
+ /* ignore */
470
+ }
471
+ }
472
+
473
+ // per-path state(mtime / size / lastResult / failCount —— 卡要求四字段)
474
+ // lastResult 语义:'ok' 本轮重建过 / 'skipped' 本轮确认未变 / 'failed' 本轮失败(下轮强制重算)
475
+ const paths = { ...(prev.paths ?? {}) };
476
+ const failedMap = new Map(stats.failures.map((f) => [f.path, f.error]));
477
+ const stampIso = new Date(stats.finishedAt || now()).toISOString();
478
+ for (const doc of scope.docs) {
479
+ const prevEntry = paths[doc.rel] ?? { failCount: 0 };
480
+ const rebuiltDone = Object.prototype.hasOwnProperty.call(stats.rowsByDoc, doc.rel);
481
+ const decision = stats.decisions[doc.rel] ?? {};
482
+ let lastResult = rebuiltDone ? 'ok' : 'skipped';
483
+ if (failedMap.has(doc.rel)) lastResult = 'failed';
484
+ paths[doc.rel] = {
485
+ mtime: doc.mtime,
486
+ size: doc.size,
487
+ lastResult,
488
+ failCount: lastResult === 'failed' ? Number(prevEntry.failCount || 0) + 1 : 0,
489
+ rows: stats.rowsByDoc[doc.rel] ?? prevEntry.rows ?? 0,
490
+ reason: decision.reason ?? null,
491
+ updatedAt: stampIso,
492
+ error: failedMap.get(doc.rel) ?? null,
493
+ };
494
+ }
495
+ stats.failed = stats.failures.filter((f) => f.path !== '(run)');
496
+ stats.failedCount = stats.failed.length;
497
+ const status = stats.failedCount ? 'failed' : stats.rebuilt === 0 ? 'noop' : 'ok';
498
+ const state = finish(status, {
499
+ paths,
500
+ cost: { date: summaries.cost?.date ?? dateKey, calls: stats.llmCalls, tokens: stats.llmTokens },
501
+ });
502
+
503
+ // 日报一行(默认写;`--no-daily` 关闭。日报文件不存在 ⇒ 不新建,只记 reason)
504
+ const dailyFile = dailyReportPath({ docsDir: docsRoot, host: hostId, now: startedAt });
505
+ const daily = updateDailySection({
506
+ file: dailyFile,
507
+ line: state.lastDailyLine,
508
+ dryRun: dryRun || Boolean(flags['no-daily']),
509
+ });
510
+ stats.daily = daily;
511
+ log(`[doc-index] 日报一行:${state.lastDailyLine}`);
512
+ log(`[doc-index] 日报落点:${dailyFile}(${daily.ok ? daily.reason ?? 'written' : daily.reason})`);
513
+ log(
514
+ `[doc-index] ${status === 'failed' ? '⚠️' : '✅'} 重建 ${stats.rebuilt} / 跳过 ${stats.skipped}`
515
+ + ` · 写库 ${stats.upserted}(新 ${stats.inserted} / 更 ${stats.updated} / 同哈希 ${stats.unchanged})`
516
+ + ` · 清孤条 ${stats.staleDeleted} · 失败 ${stats.failedCount} · ${stats.elapsedMs}ms`,
517
+ );
518
+ return { ok: true, exitCode: 0, status, stats, state, lines };
519
+ }
520
+
521
+ function buildLlmConfig(flags, env, home) {
522
+ const apiKey =
523
+ flags['llm-key'] || resolveApiKey({ env, credentialsFile: path.join(home, '.credentials.yaml') });
524
+ return {
525
+ apiKey,
526
+ baseUrl: String(flags['llm-base-url'] || env.DOC_INDEX_LLM_BASE_URL || 'https://api.deepseek.com'),
527
+ model: String(flags['llm-model'] || env.DOC_INDEX_LLM_MODEL || 'deepseek-flash'),
528
+ };
529
+ }