@yolk_vat-y/dsh-project-memory 0.5.11 → 0.5.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +225 -1
- package/LICENSE +1 -1
- package/README.md +38 -35
- package/README.zh-CN.md +38 -35
- package/client/client.js +73 -73
- package/client/client.js.map +1 -1
- package/package.json +12 -11
- package/scripts/bench-load.mjs +47 -0
- package/scripts/bench.mjs +35 -4
- package/src/audit.js +29 -3
- package/src/auto-inject.js +5 -3
- package/src/enhancer.js +170 -86
- package/src/llm.js +4 -1
- package/src/store.js +139 -22
- package/src/tools/index-repo.js +12 -2
- package/src/tools/query-memory.js +26 -3
- package/src/util/fs.js +65 -1
- package/src/watch.js +41 -7
package/src/store.js
CHANGED
|
@@ -18,33 +18,122 @@ const GITIGNORE_FILE = '.gitignore'
|
|
|
18
18
|
const LEGACY_TYPE_CACHE_DIR = 'type-cache'
|
|
19
19
|
|
|
20
20
|
const storeCache = new Map()
|
|
21
|
+
/** 单实例超预算的告警只打一次(key = 解析后的 store 目录)。 */
|
|
22
|
+
const warnedOverBudget = new Set()
|
|
21
23
|
/** 条目数上限(兜底)。 */
|
|
22
24
|
const STORE_CACHE_MAX = 32
|
|
23
25
|
/**
|
|
24
|
-
* 驻留内存的估算上限。只数"几个 store"是不够的:单个大仓库 store
|
|
25
|
-
*
|
|
26
|
+
* 驻留内存的估算上限。只数"几个 store"是不够的:单个大仓库 store 实测驻留
|
|
27
|
+
* 68MB(`load()` 后)~110MB(物化 `searchText` + 建 IDF/符号索引后),
|
|
28
|
+
* 语料是 deepseek-harness / 70119 条目。32 个就是几 GB。
|
|
29
|
+
* 估算口径见 `_estimateResidentBytes`(对内存里的对象图精确计数,不再用读盘文本量换算)。
|
|
26
30
|
*/
|
|
27
|
-
|
|
31
|
+
let STORE_CACHE_MAX_BYTES = 256 * 1024 * 1024
|
|
28
32
|
/** 每轮 save 最多补写多少个待压实的老分片(有界,避免升级后一次重写整库)。 */
|
|
29
33
|
const COMPACT_BATCH = 200
|
|
30
34
|
|
|
31
|
-
/**
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
35
|
+
/**
|
|
36
|
+
* 一个值的驻留字节:形状开销 + 字符串的**编码字节数**。
|
|
37
|
+
*
|
|
38
|
+
* 用 `Buffer.byteLength()` 而不是 `String.length`:CJK 在 UTF-8 里 3 字节/字符,而 V8
|
|
39
|
+
* 双字节串 2 字节/字符——UTF-8 对非 ASCII 是偏高的代理指标,对纯 ASCII 精确。缓存预算
|
|
40
|
+
* 容得下这个误差(实测两端 ±13%,见 `test/store-cache.test.mjs` 与 PLAN 附七)。
|
|
41
|
+
*
|
|
42
|
+
* 递归用深度上限而不是 `Set` 去环:store 的对象图来自 `JSON.parse`,本就无环;深度上限
|
|
43
|
+
* 既挡畸形数据,又比逐对象 `Set` 查重快 ~30%(70119 条目实测 36ms vs 52ms)。
|
|
44
|
+
*/
|
|
45
|
+
const RESIDENT_OBJ_BYTES = 60
|
|
46
|
+
const RESIDENT_ARR_BYTES = 40
|
|
47
|
+
const RESIDENT_KEY_BYTES = 40
|
|
48
|
+
const RESIDENT_STR_BYTES = 16
|
|
49
|
+
const RESIDENT_MAX_DEPTH = 8
|
|
50
|
+
|
|
51
|
+
function residentBytesOf(value, depth = 0) {
|
|
52
|
+
if (depth > RESIDENT_MAX_DEPTH) return 0
|
|
53
|
+
const type = typeof value
|
|
54
|
+
if (type === 'string') return RESIDENT_STR_BYTES + Buffer.byteLength(value)
|
|
55
|
+
if (type === 'number' || type === 'bigint') return 16
|
|
56
|
+
if (type === 'boolean' || value === null) return 8
|
|
57
|
+
if (Array.isArray(value)) {
|
|
58
|
+
let sum = RESIDENT_ARR_BYTES
|
|
59
|
+
for (const item of value) sum += 8 + residentBytesOf(item, depth + 1)
|
|
60
|
+
return sum
|
|
61
|
+
}
|
|
62
|
+
if (type === 'object') {
|
|
63
|
+
let sum = RESIDENT_OBJ_BYTES
|
|
64
|
+
for (const key in value) sum += RESIDENT_KEY_BYTES + residentBytesOf(value[key], depth + 1)
|
|
65
|
+
return sum
|
|
66
|
+
}
|
|
67
|
+
return 0
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* 单个 store 的驻留估算:数**内存里留下的对象图**,按版本记忆化。
|
|
72
|
+
*
|
|
73
|
+
* 为什么不是"读盘文本量换算"(旧实现):那个口径有两个错。
|
|
74
|
+
* 1. 时序错位——它量的是 `load()` 读进来的 JSON 文本,而 `searchText` 要到
|
|
75
|
+
* `allEntries()` 才物化,同一份数据在物化前后估出两个数(实测 76MB → 107MB)。
|
|
76
|
+
* 2. 账不对物——旧分片里还带着 `searchText` / `linkedSymbols`,`load()` 会
|
|
77
|
+
* `stripPersistedDerived` 把它们删掉,可文本量已经把被删的字节算进去了。
|
|
78
|
+
* `/home/sxt/project` 的 store 实测:读盘 212.6MB,真实驻留 101.7MB。
|
|
79
|
+
*
|
|
80
|
+
* 两个版本键都参与失效:`_entriesVersion`(`setEntries` / `removeFile` / `_loadSharded`)
|
|
81
|
+
* 与 `_materializeVersion`(`allEntries()` 物化 `searchText`)。漏掉后者就是旧实现的错。
|
|
82
|
+
*
|
|
83
|
+
* 代价:版本变化后的**一次**全量遍历(70119 条目 ~40ms)。必须记忆化——`evictStoreCache`
|
|
84
|
+
* 是对**整个缓存**逐 store 求和的,不记忆化等于每次冷加载都把缓存里所有 store 重数一遍。
|
|
85
|
+
*/
|
|
86
|
+
export function _estimateResidentBytes(store) {
|
|
87
|
+
const stamp = `${store._entriesVersion}:${store._materializeVersion}`
|
|
88
|
+
if (store._residentStamp !== stamp) {
|
|
89
|
+
store._residentBytes =
|
|
90
|
+
residentBytesOf(store.entries) +
|
|
91
|
+
residentBytesOf(store.files) +
|
|
92
|
+
residentBytesOf(store.experience) +
|
|
93
|
+
residentBytesOf(store.tasks) +
|
|
94
|
+
residentBytesOf(store.insights) +
|
|
95
|
+
residentBytesOf(store.watchlist) +
|
|
96
|
+
residentBytesOf(store.binding)
|
|
97
|
+
store._residentStamp = stamp
|
|
98
|
+
}
|
|
99
|
+
return store._residentBytes
|
|
36
100
|
}
|
|
37
101
|
|
|
38
102
|
/** 按 LRU 逐出,直到同时满足条数与字节预算;刚加入的 keepKey 即使超预算也保留。 */
|
|
39
103
|
function evictStoreCache(keepKey) {
|
|
40
104
|
let total = 0
|
|
41
|
-
for (const store of storeCache.values()) total +=
|
|
105
|
+
for (const store of storeCache.values()) total += _estimateResidentBytes(store)
|
|
42
106
|
while (storeCache.size > STORE_CACHE_MAX || total > STORE_CACHE_MAX_BYTES) {
|
|
43
107
|
const oldest = storeCache.keys().next().value
|
|
44
108
|
if (oldest === undefined || oldest === keepKey) break
|
|
45
|
-
total -=
|
|
109
|
+
total -= _estimateResidentBytes(storeCache.get(oldest))
|
|
46
110
|
storeCache.delete(oldest)
|
|
47
111
|
}
|
|
112
|
+
// keepKey 豁免是**有意**的:刚加载的 store 必须立刻可用。但它的代价是"单实例超预算时
|
|
113
|
+
// 预算不再是上界"——这个代价不能是隐式的,所以每个这样的 store 告警一次(每 root 一次)。
|
|
114
|
+
const kept = _estimateResidentBytes(storeCache.get(keepKey))
|
|
115
|
+
if (kept > STORE_CACHE_MAX_BYTES && !warnedOverBudget.has(keepKey)) {
|
|
116
|
+
warnedOverBudget.add(keepKey)
|
|
117
|
+
console.error(
|
|
118
|
+
`[dsh-project-memory] store ${keepKey} 单实例驻留约 ${Math.round(kept / 1048576)}MB,` +
|
|
119
|
+
`超过 storeCache 预算 ${Math.round(STORE_CACHE_MAX_BYTES / 1048576)}MB;` +
|
|
120
|
+
'刚加载的 store 不逐出(有意豁免),多 root 场景下其他 store 会被更频繁地逐出。',
|
|
121
|
+
)
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** 仅供测试:临时收窄字节预算(返回恢复函数),免得测试真去分配 256MB 字符串。 */
|
|
126
|
+
export function _setStoreCacheBudgetForTest(bytes) {
|
|
127
|
+
const previous = STORE_CACHE_MAX_BYTES
|
|
128
|
+
STORE_CACHE_MAX_BYTES = bytes
|
|
129
|
+
return () => {
|
|
130
|
+
STORE_CACHE_MAX_BYTES = previous
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** 仅供测试:当前缓存 key 的 LRU 顺序(最旧在前)。 */
|
|
135
|
+
export function _storeCacheKeysForTest() {
|
|
136
|
+
return [...storeCache.keys()]
|
|
48
137
|
}
|
|
49
138
|
|
|
50
139
|
/** 已就"无法迁移的旧 store"告警过的目录:避免每次 load() 都刷一行。 */
|
|
@@ -78,14 +167,13 @@ function withoutPersistedDerived(entry) {
|
|
|
78
167
|
return out
|
|
79
168
|
}
|
|
80
169
|
|
|
81
|
-
function loadJson(filePath, fallback
|
|
170
|
+
function loadJson(filePath, fallback) {
|
|
82
171
|
let raw
|
|
83
172
|
try {
|
|
84
173
|
raw = readFileSync(filePath, 'utf8')
|
|
85
174
|
} catch {
|
|
86
175
|
return fallback
|
|
87
176
|
}
|
|
88
|
-
if (sizeSink) sizeSink.bytes += raw.length
|
|
89
177
|
try {
|
|
90
178
|
return JSON.parse(raw)
|
|
91
179
|
} catch {
|
|
@@ -142,8 +230,14 @@ export class ProjectMemoryStore {
|
|
|
142
230
|
this._entriesVersion = 0
|
|
143
231
|
/** 待压实的老分片(≤0.5.10 落盘时带 linkedSymbols/searchText);save() 每轮有界补写。 */
|
|
144
232
|
this._compactQueue = new Set()
|
|
145
|
-
/**
|
|
146
|
-
this.
|
|
233
|
+
/** 驻留字节估算的记忆化结果与版本戳(见 `_estimateResidentBytes`)。 */
|
|
234
|
+
this._residentBytes = 0
|
|
235
|
+
this._residentStamp = ''
|
|
236
|
+
/**
|
|
237
|
+
* `searchText` 的物化计数。它和 `_entriesVersion` 一起构成驻留估算的失效键:
|
|
238
|
+
* 物化只改 entry 对象本身,不改 entries 的构成,所以不会 bump `_entriesVersion`。
|
|
239
|
+
*/
|
|
240
|
+
this._materializeVersion = 0
|
|
147
241
|
}
|
|
148
242
|
|
|
149
243
|
load() {
|
|
@@ -243,9 +337,8 @@ export class ProjectMemoryStore {
|
|
|
243
337
|
} catch {
|
|
244
338
|
shardNames = []
|
|
245
339
|
}
|
|
246
|
-
const sizeSink = { bytes: 0 }
|
|
247
340
|
for (const name of shardNames) {
|
|
248
|
-
const shard = loadJson(path.join(this.dir, SHARDS_DIR, name), null
|
|
341
|
+
const shard = loadJson(path.join(this.dir, SHARDS_DIR, name), null)
|
|
249
342
|
if (!shard || typeof shard.relPath !== 'string' || !isRecord(shard.record)) continue
|
|
250
343
|
this.files[shard.relPath] = shard.record
|
|
251
344
|
// 畸形 shard(entries 被写成对象/null)不能让 allEntries() 在 `for…of` 上抛错,
|
|
@@ -256,11 +349,10 @@ export class ProjectMemoryStore {
|
|
|
256
349
|
this.entries[shard.relPath] = list.map(stripPersistedDerived)
|
|
257
350
|
}
|
|
258
351
|
this._entriesVersion++
|
|
259
|
-
this.experience = loadJson(path.join(this.dir, EXPERIENCE_FILE), []
|
|
260
|
-
this.tasks = loadJson(path.join(this.dir, TASKS_FILE), []
|
|
261
|
-
this.binding = loadJson(path.join(this.dir, BINDING_FILE), {}
|
|
262
|
-
this.watchlist = loadJson(path.join(this.dir, WATCH_FILE), []
|
|
263
|
-
this._residentChars = sizeSink.bytes
|
|
352
|
+
this.experience = loadJson(path.join(this.dir, EXPERIENCE_FILE), [])
|
|
353
|
+
this.tasks = loadJson(path.join(this.dir, TASKS_FILE), [])
|
|
354
|
+
this.binding = loadJson(path.join(this.dir, BINDING_FILE), {})
|
|
355
|
+
this.watchlist = loadJson(path.join(this.dir, WATCH_FILE), [])
|
|
264
356
|
this._formatWritten = existsSafe(path.join(this.dir, FORMAT_FILE))
|
|
265
357
|
}
|
|
266
358
|
|
|
@@ -563,6 +655,24 @@ export class ProjectMemoryStore {
|
|
|
563
655
|
return this._entriesVersion
|
|
564
656
|
}
|
|
565
657
|
|
|
658
|
+
/**
|
|
659
|
+
* 这个 store 里有没有任何"事实"。
|
|
660
|
+
*
|
|
661
|
+
* 用于判断"值不值得为一条记账把它落盘":一个从没被索引过、也没写过任何东西的根,
|
|
662
|
+
* 不该因为一次**全局** insight 命中就被建出一个 store 目录(审计侧同理,见 audit.js)。
|
|
663
|
+
* 刻意不调 `stats()`——那个会 `allEntries()` 物化 `searchText`。
|
|
664
|
+
*/
|
|
665
|
+
get hasContent() {
|
|
666
|
+
return (
|
|
667
|
+
Object.keys(this.files).length > 0 ||
|
|
668
|
+
this.experience.length > 0 ||
|
|
669
|
+
this.tasks.length > 0 ||
|
|
670
|
+
this.insights.items.length > 0 ||
|
|
671
|
+
Object.keys(this.binding).length > 0 ||
|
|
672
|
+
this.watchlist.length > 0
|
|
673
|
+
)
|
|
674
|
+
}
|
|
675
|
+
|
|
566
676
|
/** 还有多少个老分片等着被 save() 压实(0 = 存量已收敛)。 */
|
|
567
677
|
get pendingCompaction() {
|
|
568
678
|
return this._compactQueue.size
|
|
@@ -570,13 +680,20 @@ export class ProjectMemoryStore {
|
|
|
570
680
|
|
|
571
681
|
allEntries() {
|
|
572
682
|
const out = []
|
|
683
|
+
let materialized = 0
|
|
573
684
|
for (const list of Object.values(this.entries)) {
|
|
574
685
|
for (const entry of list) {
|
|
575
686
|
// searchText 不落盘,首次用到时按需物化并挂在对象上(同一 entry 只算一次)。
|
|
576
|
-
if (entry.searchText === undefined)
|
|
687
|
+
if (entry.searchText === undefined) {
|
|
688
|
+
entry.searchText = makeSearchText(entry)
|
|
689
|
+
materialized++
|
|
690
|
+
}
|
|
577
691
|
out.push(entry)
|
|
578
692
|
}
|
|
579
693
|
}
|
|
694
|
+
// 物化只改 entry 对象、不改 entries 的构成,所以 `_entriesVersion` 不会动;但驻留会实打实
|
|
695
|
+
// 增加(70119 条目实测 +34MB)。这里是它的第二个失效键——漏了它,估算就停在物化前的水位。
|
|
696
|
+
if (materialized) this._materializeVersion++
|
|
580
697
|
return out
|
|
581
698
|
}
|
|
582
699
|
|
package/src/tools/index-repo.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import { statSync } from 'node:fs'
|
|
4
|
-
import { assertIndexRoot, assertSafeRoot, memoryRootFor, relativePath, scanLimits, storeKey, walkDir } from '../util/fs.js'
|
|
4
|
+
import { assertIndexRoot, assertSafeRoot, memoryDirName, memoryRootFor, relativePath, scanLimits, storeKey, walkDir } from '../util/fs.js'
|
|
5
5
|
import { DROP, OVERSIZE, UNCHANGED, commitFileUpdates, fileKind, planFileIndex, toFileUpdate } from '../index-pipeline.js'
|
|
6
6
|
import { ProjectMemoryStore } from '../store.js'
|
|
7
7
|
import { onFileIndexed, isTypeScriptFile } from '../enhancer.js'
|
|
@@ -38,7 +38,10 @@ export async function indexRepository(ctx, config, root, { reindex = false, allo
|
|
|
38
38
|
batch = []
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
const { files, truncated } = walkDir(root,
|
|
41
|
+
const { files, truncated, skipped: nestedRoots } = walkDir(root, {
|
|
42
|
+
...scanLimits(config),
|
|
43
|
+
nestedStoreName: memoryDirName(config),
|
|
44
|
+
})
|
|
42
45
|
for (const filePath of files) {
|
|
43
46
|
const rel = storeKey(relativePath(root, filePath))
|
|
44
47
|
seen.add(rel)
|
|
@@ -91,6 +94,13 @@ export async function indexRepository(ctx, config, root, { reindex = false, allo
|
|
|
91
94
|
`\nscan truncated at the safety limit (maxFiles=${maxFiles}, maxDepth=${maxDepth}): only part of the tree was indexed.` +
|
|
92
95
|
' Raise maxScanFiles/maxScanDepth if this project is legitimately that large.'
|
|
93
96
|
}
|
|
97
|
+
if (nestedRoots.length) {
|
|
98
|
+
const rels = nestedRoots.map((p) => storeKey(relativePath(root, p)))
|
|
99
|
+
report +=
|
|
100
|
+
`\nskipped ${nestedRoots.length} nested project root(s) with their own store: ${rels.join(', ')}` +
|
|
101
|
+
"\n(their content lives in their own root's index; this root keeps no duplicate copy)"
|
|
102
|
+
if (removed) report += `, ${removed} stale duplicate entr${removed === 1 ? 'y' : 'ies'} removed`
|
|
103
|
+
}
|
|
94
104
|
if (failures.length) {
|
|
95
105
|
report += `\nfailed to index ${failures.length} file(s): ${failures.join(', ')}`
|
|
96
106
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
2
2
|
import path from 'node:path'
|
|
3
|
-
import { memoryRootFor, resolveSafeIndexRoot } from '../util/fs.js'
|
|
3
|
+
import { memoryDirName, memoryRootFor, nestedStoreRoots, resolveSafeIndexRoot } from '../util/fs.js'
|
|
4
4
|
import { ProjectMemoryStore, storeOverview } from '../store.js'
|
|
5
5
|
import { expandQuery } from '../llm.js'
|
|
6
6
|
import { resolveRoute } from '../llm-route.js'
|
|
@@ -13,6 +13,22 @@ function toAbs(root, rel) {
|
|
|
13
13
|
return path.isAbsolute(rel) ? rel : path.join(root, rel)
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
+
/**
|
|
17
|
+
* 本根未命中记忆层时,把"这里有哪几个独立子索引"告诉模型。
|
|
18
|
+
*
|
|
19
|
+
* 父根不再重复保存子项目的内容(`walkDir()` 跳过自带 store 的嵌套根),所以"在父根下问子项目的
|
|
20
|
+
* 事"必然查不到;换 `root=` 再问一次是唯一正确的下一步。只在未命中时算,不在热路径上。
|
|
21
|
+
*/
|
|
22
|
+
function nestedIndexNote(root, config) {
|
|
23
|
+
const nested = nestedStoreRoots(root, memoryDirName(config))
|
|
24
|
+
if (!nested.length) return ''
|
|
25
|
+
return (
|
|
26
|
+
`\n\nNote: this root keeps no copy of ${nested.length} nested project(s) — each has its own index. ` +
|
|
27
|
+
'Re-run query_memory with `root: <path>` for the one you want:\n' +
|
|
28
|
+
nested.map((p) => `- ${p}`).join('\n')
|
|
29
|
+
)
|
|
30
|
+
}
|
|
31
|
+
|
|
16
32
|
export function queryMemoryTool(ctx, config) {
|
|
17
33
|
return defineTool({
|
|
18
34
|
name: 'query_memory',
|
|
@@ -66,6 +82,8 @@ export function queryMemoryTool(ctx, config) {
|
|
|
66
82
|
const boundTaskId = sessionId && typeof store.getBoundTaskId === 'function' ? store.getBoundTaskId(sessionId) : null
|
|
67
83
|
const globalStore = new GlobalStore(cfgInsight(config).globalFile || defaultGlobalFile()).load()
|
|
68
84
|
const wantMemory = type === 'all' || type === 'doc' || type === 'symbol'
|
|
85
|
+
/** 本根记忆层(doc/symbol)的命中数:0 且存在子索引时,末尾要指路。 */
|
|
86
|
+
let memHitCount = 0
|
|
69
87
|
const wantInsight = type === 'all' || type === 'insight'
|
|
70
88
|
// 一次 recall 覆盖全部层:doc/symbol/experience/insight 共用同一个检索核心(src/recall.js)。
|
|
71
89
|
const recalled = recallItems({
|
|
@@ -100,6 +118,7 @@ export function queryMemoryTool(ctx, config) {
|
|
|
100
118
|
.flatMap((l) => l.hits)
|
|
101
119
|
.sort((a, b) => b.weightedScore - a.weightedScore)
|
|
102
120
|
.slice(0, limit)
|
|
121
|
+
memHitCount = memHits.length
|
|
103
122
|
if (memHits.length) {
|
|
104
123
|
const top = memHits[0].weightedScore || 1
|
|
105
124
|
lines.push(`## Memory (${type === 'all' ? 'docs + symbols' : type})`)
|
|
@@ -196,12 +215,16 @@ export function queryMemoryTool(ctx, config) {
|
|
|
196
215
|
: overview.latest
|
|
197
216
|
? `, last indexed at ${overview.latest}. Use memory_stats to see what the store contains.`
|
|
198
217
|
: '. Use memory_stats to see what the store contains.'
|
|
218
|
+
const nestedNote = wantMemory && memHitCount === 0 ? nestedIndexNote(root, config) : ''
|
|
199
219
|
return (
|
|
200
220
|
`No memory matches for "${args.query}" in ${root}. ${hint}\n` +
|
|
201
|
-
`Store overview: ${overview.files} files indexed, ${overview.entries} entries, ${overview.experience} experience notes${tail}`
|
|
221
|
+
`Store overview: ${overview.files} files indexed, ${overview.entries} entries, ${overview.experience} experience notes${tail}${nestedNote}`
|
|
202
222
|
)
|
|
203
223
|
}
|
|
204
|
-
|
|
224
|
+
// 提示放在**最前**:`truncate()` 截的是尾部,而 insight/procedure 类条目单条就能到 600+
|
|
225
|
+
// 字符,放末尾会被整段截掉(实测 8000 字符预算下必被截)。
|
|
226
|
+
const note = wantMemory && memHitCount === 0 ? nestedIndexNote(root, config).trimStart() + '\n\n' : ''
|
|
227
|
+
return truncate(note + lines.join('\n\n'), config.maxOutputChars)
|
|
205
228
|
},
|
|
206
229
|
})
|
|
207
230
|
}
|
package/src/util/fs.js
CHANGED
|
@@ -130,11 +130,62 @@ export function scanLimits(config) {
|
|
|
130
130
|
* - `truncated` 让调用方区分「这棵树扫完了」和「只扫了一部分」——后者绝不能拿
|
|
131
131
|
* 本轮未见到的文件去删旧条目(见 index-repo/watch 的 unseen 用法)。
|
|
132
132
|
*/
|
|
133
|
+
/**
|
|
134
|
+
* 一个目录是否**自带 store**(即它自己已经是一个被索引过的项目根)。
|
|
135
|
+
*
|
|
136
|
+
* 判据用 `format.json`:任何走过一次 `save()` 的 store 都必然有它,而误建/刚建的空
|
|
137
|
+
* `.dsh-project-memory` 目录没有——那种不能作为跳过的依据,否则父 store 一跳过,内容两头都没了。
|
|
138
|
+
*/
|
|
139
|
+
export function hasOwnStore(dir, memoryDirName) {
|
|
140
|
+
return existsSync(path.join(dir, memoryDirName, 'format.json'))
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* 列出 `root` 下**自带 store** 的子目录 —— 也就是"这部分内容有自己的独立索引"。
|
|
145
|
+
*
|
|
146
|
+
* 存在的理由:`walkDir()` 会跳过这些子树(父根不再重复保存它们的内容),于是"在父根下问子项目
|
|
147
|
+
* 的事"必须换个 `root=` 再问一次。这个函数就是用来在**查询未命中时给模型指路**的,不是文件系统
|
|
148
|
+
* 巡检:BFS、限深 3 层、最多列 8 个、最多看 300 个目录,且不往已知的子根里下探。
|
|
149
|
+
*/
|
|
150
|
+
export function nestedStoreRoots(root, memoryDirName, { maxDepth = 3, maxEntries = 8, maxVisited = 300 } = {}) {
|
|
151
|
+
const found = []
|
|
152
|
+
const queue = [[root, 0]]
|
|
153
|
+
let visited = 0
|
|
154
|
+
while (queue.length && found.length < maxEntries && visited < maxVisited) {
|
|
155
|
+
const [dir, depth] = queue.shift()
|
|
156
|
+
visited++
|
|
157
|
+
let entries
|
|
158
|
+
try {
|
|
159
|
+
entries = readdirSync(dir, { withFileTypes: true })
|
|
160
|
+
} catch {
|
|
161
|
+
continue
|
|
162
|
+
}
|
|
163
|
+
for (const entry of entries) {
|
|
164
|
+
if (found.length >= maxEntries) break
|
|
165
|
+
if (!entry.isDirectory() || entry.name.startsWith('.')) continue
|
|
166
|
+
if (DEFAULT_IGNORE.has(entry.name)) continue
|
|
167
|
+
const full = path.join(dir, entry.name)
|
|
168
|
+
if (hasOwnStore(full, memoryDirName)) {
|
|
169
|
+
found.push(full)
|
|
170
|
+
continue // 已知的子根不再往里下探
|
|
171
|
+
}
|
|
172
|
+
if (depth + 1 <= maxDepth) queue.push([full, depth + 1])
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return found.sort()
|
|
176
|
+
}
|
|
177
|
+
|
|
133
178
|
export function walkDir(root, opts = {}) {
|
|
134
179
|
const ignoreNames = opts.ignoreNames || DEFAULT_IGNORE
|
|
135
180
|
const maxFiles = Number.isFinite(opts.maxFiles) && opts.maxFiles > 0 ? opts.maxFiles : Infinity
|
|
136
181
|
const maxDepth = Number.isFinite(opts.maxDepth) && opts.maxDepth > 0 ? opts.maxDepth : Infinity
|
|
182
|
+
// 传 memoryDir 名即开启「嵌套根不重复索引」:子目录自带 store 时不再往下走。
|
|
183
|
+
// 理由是跟解析规则保持一致——`findProjectRoot()` 对子树里的文件返回的正是那个子根,
|
|
184
|
+
// 扫描也该同意这一点;否则父根会把子项目的全部内容再存一份(实测 `/home/sxt/project`
|
|
185
|
+
// 的 store 里 9075/9081 个文件是子根 store 的重复副本)。
|
|
186
|
+
const nestedStoreName = opts.nestedStoreName || null
|
|
137
187
|
const out = []
|
|
188
|
+
const skipped = []
|
|
138
189
|
const stack = [[root, 0]]
|
|
139
190
|
let truncated = false
|
|
140
191
|
let hitFileCap = false
|
|
@@ -151,6 +202,10 @@ export function walkDir(root, opts = {}) {
|
|
|
151
202
|
const full = path.join(dir, entry.name)
|
|
152
203
|
if (entry.isDirectory()) {
|
|
153
204
|
if (ignoreNames.has(entry.name)) continue
|
|
205
|
+
if (nestedStoreName && hasOwnStore(full, nestedStoreName)) {
|
|
206
|
+
skipped.push(full)
|
|
207
|
+
continue
|
|
208
|
+
}
|
|
154
209
|
if (depth + 1 > maxDepth) {
|
|
155
210
|
truncated = true
|
|
156
211
|
continue
|
|
@@ -168,7 +223,8 @@ export function walkDir(root, opts = {}) {
|
|
|
168
223
|
if (hitFileCap) break
|
|
169
224
|
}
|
|
170
225
|
out.sort()
|
|
171
|
-
|
|
226
|
+
skipped.sort()
|
|
227
|
+
return { files: out, truncated, skipped }
|
|
172
228
|
}
|
|
173
229
|
|
|
174
230
|
export function isSupportedDoc(ext) {
|
|
@@ -206,6 +262,14 @@ export function memoryRootFor(indexRoot, memoryDir) {
|
|
|
206
262
|
return path.join(indexRoot, memoryDir)
|
|
207
263
|
}
|
|
208
264
|
|
|
265
|
+
/** store 目录名(配置项 `memoryDir`;调用方可能来自工具参数,缺省时兜底成默认值)。 */
|
|
266
|
+
export const DEFAULT_MEMORY_DIR = '.dsh-project-memory'
|
|
267
|
+
|
|
268
|
+
export function memoryDirName(config) {
|
|
269
|
+
const raw = config?.memoryDir
|
|
270
|
+
return typeof raw === 'string' && raw.trim() ? raw : DEFAULT_MEMORY_DIR
|
|
271
|
+
}
|
|
272
|
+
|
|
209
273
|
/**
|
|
210
274
|
* 危险根目录:整体扫描/监听它们会直接吃满内存(issue #5 实测把 DSH 撑死的就是
|
|
211
275
|
* 家目录与 `/opt/homebrew`)。判定是**精确匹配**:只拒绝这些目录**本身**,不拒绝
|
package/src/watch.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { existsSync, statSync } from 'node:fs'
|
|
2
2
|
import path from 'node:path'
|
|
3
|
-
import { isUnsafeRoot, memoryRootFor, relativePath, scanLimits, storeKey, walkDir } from './util/fs.js'
|
|
3
|
+
import { isUnsafeRoot, memoryDirName, memoryRootFor, relativePath, scanLimits, storeKey, walkDir } from './util/fs.js'
|
|
4
4
|
import { OVERSIZE, UNCHANGED, commitFileUpdates, fileKind, planFileIndex, toFileUpdate } from './index-pipeline.js'
|
|
5
5
|
import { ProjectMemoryStore } from './store.js'
|
|
6
6
|
import { onFileChanged, isTypeScriptFile } from './enhancer.js'
|
|
@@ -13,6 +13,8 @@ export class WatchManager {
|
|
|
13
13
|
this.timer = null
|
|
14
14
|
this._polling = false
|
|
15
15
|
this._truncationWarned = new Set()
|
|
16
|
+
/** 已就「跳过了嵌套项目根」告警过的根:同上,只提示一次。 */
|
|
17
|
+
this._nestedStoreWarned = new Set()
|
|
16
18
|
}
|
|
17
19
|
|
|
18
20
|
/** 包含该路径的已注册根(最长前缀):懒索引据此把文件归到显式 watch 过的无标记项目里。 */
|
|
@@ -64,8 +66,11 @@ export class WatchManager {
|
|
|
64
66
|
// 文件系统根 / 家目录 / 系统前缀不整体监听(它们的子目录允许),除非显式放行。
|
|
65
67
|
if (!this.unsafeAllowed() && isUnsafeRoot(root)) return false
|
|
66
68
|
if (!this.roots.has(root)) {
|
|
69
|
+
// **不在这里 load()**:`restorePersisted()` 会对 cwd store 的整条 watchlist 逐个 addRoot,
|
|
70
|
+
// 急切读盘等于「watch 过多少个根,启动就同步读多少个 store 的全部 shard」——一个万文件
|
|
71
|
+
// 的 store 就是秒级,几十个根就是启动卡死 + 内存无上界。这里只登记,store 到第一次
|
|
72
|
+
// pollRoot 真正需要时才取(见 `storeFor`)。
|
|
67
73
|
this.roots.set(root, {
|
|
68
|
-
store: new ProjectMemoryStore(memoryRootFor(root, this.config.memoryDir)).load(),
|
|
69
74
|
snapshot: {},
|
|
70
75
|
// rel → 上次已上报的错误信息。坏文件每轮都会重试,逐轮打印会刷屏。
|
|
71
76
|
failures: new Map(),
|
|
@@ -75,6 +80,21 @@ export class WatchManager {
|
|
|
75
80
|
return false
|
|
76
81
|
}
|
|
77
82
|
|
|
83
|
+
/**
|
|
84
|
+
* 每个根**按需**取 store:命中 `storeCache` 时是 O(1)(一次 Map 查找 + LRU 挪位)。
|
|
85
|
+
*
|
|
86
|
+
* 刻意不让本类长期持有实例。旧实现把 `store` 塞进 `this.roots` 里,等于给每个 watch 根
|
|
87
|
+
* 加了一个**进程级强引用**,于是:
|
|
88
|
+
* - 逐出对 watch 根完全不省内存(`storeCache` 丢了 key,watcher 还攥着);
|
|
89
|
+
* - 一旦真被逐出,下一次 `load()` 会从盘上**再建一份**,同一个 root 在同一进程里出现两份
|
|
90
|
+
* 互不可见的内存状态,两份各自 `save()` 脏分片 → 最后写者赢,存在丢更新的窗口。
|
|
91
|
+
* 现在唯一的活实例就是缓存里那一个:逐出即真正释放,下一轮 poll 重新读盘(缓存该有的语义),
|
|
92
|
+
* 而内存上界回到 `STORE_CACHE_MAX_BYTES`,不再由「watch 了多少个根」决定。
|
|
93
|
+
*/
|
|
94
|
+
storeFor(root) {
|
|
95
|
+
return new ProjectMemoryStore(memoryRootFor(root, this.config.memoryDir)).load()
|
|
96
|
+
}
|
|
97
|
+
|
|
78
98
|
removeRoot(root) {
|
|
79
99
|
return this.roots.delete(root)
|
|
80
100
|
}
|
|
@@ -142,8 +162,22 @@ export class WatchManager {
|
|
|
142
162
|
const updates = []
|
|
143
163
|
// rel → 本轮采集的 mtime:size。提交成功后才落进快照;CAS 失败的条目留在原地等下一轮。
|
|
144
164
|
const signatures = new Map()
|
|
165
|
+
// 本轮用的 store:命中缓存即 O(1),被逐出则重新读盘。整个 pollRoot 期间共用同一个实例。
|
|
166
|
+
const store = this.storeFor(root)
|
|
145
167
|
|
|
146
|
-
const { files, truncated } = walkDir(root,
|
|
168
|
+
const { files, truncated, skipped: nestedRoots } = walkDir(root, {
|
|
169
|
+
...scanLimits(this.config),
|
|
170
|
+
nestedStoreName: memoryDirName(this.config),
|
|
171
|
+
})
|
|
172
|
+
if (nestedRoots.length && !this._nestedStoreWarned.has(root)) {
|
|
173
|
+
// 同上,只提示一次。这里同时说明"旧副本会被本轮清掉"——升级后的存量重复就是这么收敛的。
|
|
174
|
+
this._nestedStoreWarned.add(root)
|
|
175
|
+
const rels = nestedRoots.map((p) => storeKey(relativePath(root, p)))
|
|
176
|
+
console.error(
|
|
177
|
+
`[dsh-project-memory] watch of ${root} skips nested project root(s) with their own store: ${rels.join(', ')}; ` +
|
|
178
|
+
'their content is refreshed by their own root, and any duplicate copy previously indexed here is being removed.',
|
|
179
|
+
)
|
|
180
|
+
}
|
|
147
181
|
if (truncated && !this._truncationWarned.has(root)) {
|
|
148
182
|
// 只提示一次:这是一条**永久**的降级说明,不是每轮都要刷屏的故障。
|
|
149
183
|
this._truncationWarned.add(root)
|
|
@@ -169,7 +203,7 @@ export class WatchManager {
|
|
|
169
203
|
const sig = `${stats.mtimeMs}:${stats.size}`
|
|
170
204
|
if (state.snapshot[rel] === sig) continue
|
|
171
205
|
|
|
172
|
-
const record =
|
|
206
|
+
const record = store.fileRecord(rel)
|
|
173
207
|
let plan
|
|
174
208
|
try {
|
|
175
209
|
plan = await planFileIndex({
|
|
@@ -178,7 +212,7 @@ export class WatchManager {
|
|
|
178
212
|
kind,
|
|
179
213
|
config: this.config,
|
|
180
214
|
record,
|
|
181
|
-
existingEntries:
|
|
215
|
+
existingEntries: store.entries[rel],
|
|
182
216
|
size: stats.size,
|
|
183
217
|
})
|
|
184
218
|
} catch (err) {
|
|
@@ -204,7 +238,7 @@ export class WatchManager {
|
|
|
204
238
|
// 单事务写盘;CAS 失败的条目本轮不落快照,下一轮自然重试。
|
|
205
239
|
// 截断时**不传 unseen**:没扫到的文件不等于被删了,否则一份被上限截掉的树每轮都会
|
|
206
240
|
// 把自己的记忆删掉一半(先删再下轮重新索引,纯粹的抖动)。
|
|
207
|
-
const { stale, removed } = commitFileUpdates(
|
|
241
|
+
const { stale, removed } = commitFileUpdates(store, {
|
|
208
242
|
updates,
|
|
209
243
|
unseen: truncated ? null : seen,
|
|
210
244
|
})
|
|
@@ -217,7 +251,7 @@ export class WatchManager {
|
|
|
217
251
|
if (update.type !== 'code') continue
|
|
218
252
|
const filePath = path.join(root, update.rel)
|
|
219
253
|
if (isTypeScriptFile(filePath)) {
|
|
220
|
-
onFileChanged(
|
|
254
|
+
onFileChanged(store, update.rel, filePath, this.config, root)
|
|
221
255
|
}
|
|
222
256
|
}
|
|
223
257
|
// 有变化 → 下一轮回到 base 间隔;纯空转 → 退避。
|