dsh-knit 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/README.en.md +61 -17
- package/README.md +50 -14
- package/package.json +3 -2
- package/src/client/client.js +91 -4
- package/src/host/feedback.js +641 -0
- package/src/host/index.js +148 -1
- package/src/host/tool.js +64 -3
- package/test/client.test.mjs +99 -0
- package/test/context-delta.test.mjs +309 -0
- package/test/context-feedback-eval.test.mjs +189 -0
- package/test/feedback.test.mjs +247 -0
- package/test/host-http.test.mjs +193 -2
- package/test/i18n.test.mjs +3 -2
- package/test/tool.test.mjs +129 -4
- package/tools/context-feedback-eval.mjs +330 -0
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Knit v0.15 · **Context Snapshot / Delta / Usage 的行为测试**
|
|
3
|
+
*
|
|
4
|
+
* 这里测的是「Knit 把上下文交出去之后,到底发生了什么」——
|
|
5
|
+
* Context Snapshot(交出去那一刻的三层排序)· Context Delta(两次之间谁进谁出)
|
|
6
|
+
* · Usage(某次真实的读落在哪一层)。
|
|
7
|
+
*
|
|
8
|
+
* 六条纪律,每一条都对应一组用例:
|
|
9
|
+
* 1. **无变化不记新快照** —— 否则 5 秒轮询会把同一份上下文记成几十份,Delta churn 全是噪音;
|
|
10
|
+
* 2. **游标幂等** —— 同一批事件喂两遍,读只算一遍;
|
|
11
|
+
* 3. **会话隔离** —— A 会话的读不许进 B 会话的报告;
|
|
12
|
+
* 4. **内存有上限** —— 会话数 / 读过的篇数 / Delta 条数都有硬上限(LRU);
|
|
13
|
+
* 5. **失败降级** —— store 给错了、事件是 null、pack 是 undefined,都不许抛,只返回空结果;
|
|
14
|
+
* 6. **没有分数** —— Usage 里只有计数与事实(`primaryFollowThrough` 是布尔,不是成功率)。
|
|
15
|
+
*/
|
|
16
|
+
import test from 'node:test'
|
|
17
|
+
import assert from 'node:assert/strict'
|
|
18
|
+
|
|
19
|
+
import {
|
|
20
|
+
createStore, activateAudit, isAudited, ingestEvents, noteSnapshot, usageFor, usageFrom,
|
|
21
|
+
auditPayload, getAuditSummary, resetStore,
|
|
22
|
+
MAX_SESSIONS, MAX_READS_PER_SESSION, MAX_DELTAS_PER_SESSION,
|
|
23
|
+
} from '../src/host/feedback.js'
|
|
24
|
+
|
|
25
|
+
const ROOT = '/w'
|
|
26
|
+
|
|
27
|
+
/** 造一份 Context Pack(只给测试要用的字段)。 */
|
|
28
|
+
function pack(options = {}) {
|
|
29
|
+
const rows = (list) => (list || []).map((rel) => ({ rel }))
|
|
30
|
+
return {
|
|
31
|
+
mode: 'relevance',
|
|
32
|
+
topic: options.topic || 'knit',
|
|
33
|
+
task: options.task || '',
|
|
34
|
+
primary: rows(options.primary),
|
|
35
|
+
supporting: rows(options.supporting),
|
|
36
|
+
related: rows(options.related),
|
|
37
|
+
totals: { matched: 0, total: Number.isInteger(options.total) ? options.total : 10 },
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** 造一批 read 事件:`pairs` 是 `[rel, seq]`。 */
|
|
42
|
+
function readEvents(pairs, startSeq = 100) {
|
|
43
|
+
const events = []
|
|
44
|
+
let seq = startSeq
|
|
45
|
+
for (const [rel, at] of pairs) {
|
|
46
|
+
const callId = `c${seq}`
|
|
47
|
+
events.push({
|
|
48
|
+
type: 'tool/call',
|
|
49
|
+
seq,
|
|
50
|
+
time: (at || seq) * 1,
|
|
51
|
+
data: { turn: 1, step: 1, callId, name: 'read', arguments: JSON.stringify({ file_path: rel }) },
|
|
52
|
+
})
|
|
53
|
+
seq += 1
|
|
54
|
+
events.push({
|
|
55
|
+
type: 'tool/result',
|
|
56
|
+
seq,
|
|
57
|
+
time: (at || seq) * 1,
|
|
58
|
+
data: { message: { role: 'tool', toolCallId: callId, source: { kind: 'tool', callId }, isError: false } },
|
|
59
|
+
})
|
|
60
|
+
seq += 1
|
|
61
|
+
}
|
|
62
|
+
return events
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
test('快照:第一次交出去只记快照,没有 Delta', () => {
|
|
66
|
+
const store = createStore()
|
|
67
|
+
const first = noteSnapshot(store, 's1', pack({ primary: ['a.md'], supporting: ['b.md'] }), { seq: 5, at: 100 })
|
|
68
|
+
assert.ok(first.snapshot)
|
|
69
|
+
assert.equal(first.delta, null)
|
|
70
|
+
const usage = usageFor(store, 's1')
|
|
71
|
+
assert.equal(usage.items.length, 2)
|
|
72
|
+
assert.equal(usage.stats.churn.deltas, 0)
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
test('快照:内容完全相同就不记新快照(否则轮询会灌进几十份)', () => {
|
|
76
|
+
const store = createStore()
|
|
77
|
+
const p = pack({ primary: ['a.md'], supporting: ['b.md'], related: ['c.md'] })
|
|
78
|
+
noteSnapshot(store, 's1', p, { seq: 5, at: 100 })
|
|
79
|
+
assert.equal(noteSnapshot(store, 's1', p, { seq: 6, at: 200 }), null)
|
|
80
|
+
assert.equal(noteSnapshot(store, 's1', p, { seq: 7, at: 300 }), null)
|
|
81
|
+
const usage = usageFor(store, 's1')
|
|
82
|
+
assert.equal(usage.stats.churn.deltas, 0)
|
|
83
|
+
// 时间戳要跟着往前推,不能停在第一次
|
|
84
|
+
assert.equal(usage.at, 300)
|
|
85
|
+
assert.equal(usage.seq, 7)
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
test('变化:进 / 出 / 换层都记下来,churn 是累计值', () => {
|
|
89
|
+
const store = createStore()
|
|
90
|
+
noteSnapshot(store, 's1', pack({ primary: ['a.md'], supporting: ['b.md'], related: ['c.md'] }), { seq: 5, at: 100 })
|
|
91
|
+
const second = noteSnapshot(store, 's1', pack({ primary: ['b.md'], supporting: ['d.md'], related: [] }), { seq: 20, at: 200 })
|
|
92
|
+
assert.deepEqual(second.delta.appeared.map((x) => x.rel), ['d.md'])
|
|
93
|
+
assert.deepEqual(second.delta.disappeared.map((x) => x.rel), ['a.md', 'c.md'])
|
|
94
|
+
assert.deepEqual(second.delta.moved.map((x) => x.rel), ['b.md'])
|
|
95
|
+
|
|
96
|
+
const usage = usageFor(store, 's1')
|
|
97
|
+
assert.equal(usage.stats.churn.deltas, 1)
|
|
98
|
+
assert.equal(usage.stats.churn.appeared, 1)
|
|
99
|
+
assert.equal(usage.stats.churn.disappeared, 2)
|
|
100
|
+
assert.equal(usage.stats.churn.moved, 1)
|
|
101
|
+
assert.deepEqual(usage.delta.appeared.map((x) => x.rel), ['d.md'])
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
test('使用情况:读了 Primary 就是 primaryFollowThrough;读的是 Supporting 记进覆盖率', () => {
|
|
105
|
+
const store = createStore()
|
|
106
|
+
activateAudit(store, 's1')
|
|
107
|
+
noteSnapshot(store, 's1', pack({
|
|
108
|
+
primary: ['a.md'], supporting: ['b.md', 'c.md', 'd.md'], related: ['e.md'],
|
|
109
|
+
}), { seq: 5, at: 100 })
|
|
110
|
+
|
|
111
|
+
ingestEvents(store, 's1', readEvents([['a.md', 10], ['b.md', 11]], 100), { root: ROOT })
|
|
112
|
+
const usage = usageFor(store, 's1')
|
|
113
|
+
|
|
114
|
+
assert.equal(usage.stats.primaryRel, 'a.md')
|
|
115
|
+
assert.equal(usage.stats.primaryFollowThrough, true)
|
|
116
|
+
assert.equal(usage.stats.firstReadTier, 'primary')
|
|
117
|
+
assert.equal(usage.stats.firstReadRel, 'a.md')
|
|
118
|
+
assert.deepEqual(usage.stats.byTier.supporting, { total: 3, read: 1 })
|
|
119
|
+
assert.equal(Number(usage.stats.supportingCoverage.toFixed(3)), 0.333)
|
|
120
|
+
assert.equal(usage.stats.outside, 0)
|
|
121
|
+
assert.equal(usage.stats.reads, 2)
|
|
122
|
+
assert.equal(usage.stats.distinct, 2)
|
|
123
|
+
})
|
|
124
|
+
|
|
125
|
+
test('使用情况:快照之外的读如实记成 outside,不解释成「失败」', () => {
|
|
126
|
+
const store = createStore()
|
|
127
|
+
noteSnapshot(store, 's1', pack({ primary: ['a.md'] }), { seq: 5, at: 100 })
|
|
128
|
+
ingestEvents(store, 's1', readEvents([['z.md', 50], ['y.md', 51]], 200), { root: ROOT })
|
|
129
|
+
|
|
130
|
+
const usage = usageFor(store, 's1')
|
|
131
|
+
assert.equal(usage.stats.outside, 2)
|
|
132
|
+
assert.deepEqual(usage.stats.outsideReads.sort(), ['y.md', 'z.md'])
|
|
133
|
+
assert.equal(usage.stats.primaryFollowThrough, false)
|
|
134
|
+
assert.equal(usage.stats.firstReadTier, null)
|
|
135
|
+
const outside = usage.reads.find((r) => r.rel === 'z.md')
|
|
136
|
+
assert.equal(outside.tier, null)
|
|
137
|
+
assert.equal(outside.outside, true)
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
test('使用情况:Knit 还没给过上下文就自己读了 ⇒ 那一篇算 outside(tier 为 null)', () => {
|
|
141
|
+
const store = createStore()
|
|
142
|
+
activateAudit(store, 's1')
|
|
143
|
+
ingestEvents(store, 's1', readEvents([['a.md', 10]], 100), { root: ROOT })
|
|
144
|
+
const usage = usageFor(store, 's1')
|
|
145
|
+
assert.equal(usage.stats.outside, 1)
|
|
146
|
+
assert.equal(usage.stats.firstReadTier, null)
|
|
147
|
+
assert.equal(usage.stats.primaryFollowThrough, false)
|
|
148
|
+
})
|
|
149
|
+
|
|
150
|
+
test('使用情况:读的时候用的是**当时**那份快照,不是最新那份', () => {
|
|
151
|
+
const store = createStore()
|
|
152
|
+
// 第一份:a.md 在 Primary
|
|
153
|
+
noteSnapshot(store, 's1', pack({ primary: ['a.md'], supporting: ['b.md'] }), { seq: 5, at: 100 })
|
|
154
|
+
// 读发生在 seq 150 —— 此时生效的是第一份
|
|
155
|
+
ingestEvents(store, 's1', readEvents([['a.md', 300]], 300), { root: ROOT })
|
|
156
|
+
// 之后上下文换了:a.md 掉到 Related
|
|
157
|
+
noteSnapshot(store, 's1', pack({ supporting: ['b.md'], related: ['a.md'] }), { seq: 400, at: 400 })
|
|
158
|
+
|
|
159
|
+
const usage = usageFor(store, 's1')
|
|
160
|
+
const row = usage.reads.find((r) => r.rel === 'a.md')
|
|
161
|
+
// 行上是「当时」的层(primary),不是现在这份(related)
|
|
162
|
+
assert.equal(row.firstTier, 'primary')
|
|
163
|
+
assert.equal(row.tier, 'related')
|
|
164
|
+
})
|
|
165
|
+
|
|
166
|
+
test('幂等:同一批事件喂两遍,读只算一遍', () => {
|
|
167
|
+
const store = createStore()
|
|
168
|
+
noteSnapshot(store, 's1', pack({ primary: ['a.md'] }), { seq: 1, at: 1 })
|
|
169
|
+
const events = readEvents([['a.md', 10]], 100)
|
|
170
|
+
const first = ingestEvents(store, 's1', events, { root: ROOT })
|
|
171
|
+
const second = ingestEvents(store, 's1', events, { root: ROOT })
|
|
172
|
+
assert.equal(first.reads, 1)
|
|
173
|
+
assert.equal(second.reads, 0)
|
|
174
|
+
const usage = usageFor(store, 's1')
|
|
175
|
+
assert.equal(usage.stats.reads, 1)
|
|
176
|
+
assert.equal(usage.reads[0].count, 1)
|
|
177
|
+
})
|
|
178
|
+
|
|
179
|
+
test('幂等:同一篇读两次是 count 2,但仍然只算一个 distinct 篇目', () => {
|
|
180
|
+
const store = createStore()
|
|
181
|
+
noteSnapshot(store, 's1', pack({ supporting: ['b.md'] }), { seq: 1, at: 1 })
|
|
182
|
+
ingestEvents(store, 's1', readEvents([['b.md', 10], ['b.md', 20]], 100), { root: ROOT })
|
|
183
|
+
const usage = usageFor(store, 's1')
|
|
184
|
+
assert.equal(usage.stats.reads, 2)
|
|
185
|
+
assert.equal(usage.stats.distinct, 1)
|
|
186
|
+
assert.equal(usage.reads[0].count, 2)
|
|
187
|
+
assert.deepEqual(usage.stats.byTier.supporting, { total: 1, read: 1 })
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
test('隔离:两个会话各记各的,互不串味', () => {
|
|
191
|
+
const store = createStore()
|
|
192
|
+
noteSnapshot(store, 's1', pack({ primary: ['a.md'] }), { seq: 1, at: 1 })
|
|
193
|
+
noteSnapshot(store, 's2', pack({ primary: ['x.md'] }), { seq: 1, at: 1 })
|
|
194
|
+
ingestEvents(store, 's1', readEvents([['a.md', 10]], 100), { root: ROOT })
|
|
195
|
+
|
|
196
|
+
assert.equal(usageFor(store, 's1').stats.primaryFollowThrough, true)
|
|
197
|
+
assert.equal(usageFor(store, 's2').stats.reads, 0)
|
|
198
|
+
assert.equal(usageFor(store, 's2').stats.primaryFollowThrough, false)
|
|
199
|
+
assert.equal(usageFor(store, 's3'), null)
|
|
200
|
+
|
|
201
|
+
resetStore(store, 's1')
|
|
202
|
+
assert.equal(usageFor(store, 's1'), null)
|
|
203
|
+
assert.ok(usageFor(store, 's2'))
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
test('审计开关:只对主动开启的会话记账(默认不攒数据)', () => {
|
|
207
|
+
const store = createStore()
|
|
208
|
+
assert.equal(isAudited(store, 's1'), false)
|
|
209
|
+
assert.equal(activateAudit(store, 's1'), true)
|
|
210
|
+
assert.equal(isAudited(store, 's1'), true)
|
|
211
|
+
assert.equal(isAudited(store, 's2'), false)
|
|
212
|
+
resetStore(store)
|
|
213
|
+
assert.equal(isAudited(store, 's1'), false)
|
|
214
|
+
})
|
|
215
|
+
|
|
216
|
+
test('内存上限:读过的篇目按 LRU 砍到上限', () => {
|
|
217
|
+
const store = createStore()
|
|
218
|
+
noteSnapshot(store, 's1', pack({ related: [] }), { seq: 1, at: 1 })
|
|
219
|
+
const pairs = []
|
|
220
|
+
for (let i = 0; i < MAX_READS_PER_SESSION + 25; i += 1) pairs.push([`doc${i}.md`, i])
|
|
221
|
+
ingestEvents(store, 's1', readEvents(pairs, 10), { root: ROOT })
|
|
222
|
+
|
|
223
|
+
const usage = usageFor(store, 's1')
|
|
224
|
+
assert.equal(usage.reads.length, MAX_READS_PER_SESSION)
|
|
225
|
+
// 保留的是**最近**读的那批
|
|
226
|
+
assert.equal(usage.reads[usage.reads.length - 1].rel, `doc${MAX_READS_PER_SESSION + 24}.md`)
|
|
227
|
+
assert.equal(usage.reads.find((r) => r.rel === 'doc0.md'), undefined)
|
|
228
|
+
})
|
|
229
|
+
|
|
230
|
+
test('内存上限:会话数超上限时淘汰最久没碰的', () => {
|
|
231
|
+
const store = createStore()
|
|
232
|
+
for (let i = 0; i < MAX_SESSIONS + 3; i += 1) {
|
|
233
|
+
noteSnapshot(store, `s${i}`, pack({ primary: ['a.md'] }), { seq: i, at: i })
|
|
234
|
+
}
|
|
235
|
+
assert.equal(store.sessions.size, MAX_SESSIONS)
|
|
236
|
+
assert.equal(store.sessions.has('s0'), false)
|
|
237
|
+
assert.equal(store.sessions.has(`s${MAX_SESSIONS + 2}`), true)
|
|
238
|
+
})
|
|
239
|
+
|
|
240
|
+
test('内存上限:Delta 只留最近若干条,churn 仍是累计值', () => {
|
|
241
|
+
const store = createStore()
|
|
242
|
+
noteSnapshot(store, 's1', pack({ primary: ['base.md'] }), { seq: 0, at: 0 })
|
|
243
|
+
for (let i = 0; i < MAX_DELTAS_PER_SESSION + 5; i += 1) {
|
|
244
|
+
noteSnapshot(store, 's1', pack({ primary: [`d${i}.md`] }), { seq: i + 1, at: i + 1 })
|
|
245
|
+
}
|
|
246
|
+
const usage = usageFor(store, 's1')
|
|
247
|
+
const changes = MAX_DELTAS_PER_SESSION + 5
|
|
248
|
+
// churn 是**累计**的:明细数组被砍到上限,但「变过几次」不砍
|
|
249
|
+
assert.equal(usage.stats.churn.deltas, changes)
|
|
250
|
+
assert.equal(usage.stats.churn.retained, MAX_DELTAS_PER_SESSION)
|
|
251
|
+
assert.equal(store.sessions.get('s1').deltas.length, MAX_DELTAS_PER_SESSION)
|
|
252
|
+
// 每一次替换都是「进一篇、出一篇」
|
|
253
|
+
assert.equal(usage.stats.churn.appeared, changes)
|
|
254
|
+
assert.equal(usage.stats.churn.disappeared, changes)
|
|
255
|
+
})
|
|
256
|
+
|
|
257
|
+
test('降级:store / 会话 / 事件给错,一律返回空结果,不抛', () => {
|
|
258
|
+
const store = createStore()
|
|
259
|
+
assert.deepEqual(ingestEvents(null, 's1', []), { ok: false, reads: 0, cursor: -1 })
|
|
260
|
+
assert.equal(ingestEvents(store, '', []).ok, false)
|
|
261
|
+
assert.equal(ingestEvents(store, 's1', null).ok, true)
|
|
262
|
+
assert.equal(noteSnapshot(null, 's1', pack()), null)
|
|
263
|
+
assert.equal(noteSnapshot(store, 's1', null), null)
|
|
264
|
+
assert.equal(usageFor(null, 's1'), null)
|
|
265
|
+
assert.equal(usageFor(store, 'nope'), null)
|
|
266
|
+
assert.equal(activateAudit(null, 's1'), false)
|
|
267
|
+
assert.equal(activateAudit(store, ''), false)
|
|
268
|
+
// 坏事件不会把整批作废
|
|
269
|
+
const bad = [null, { type: 'tool/call' }, { type: 'tool/call', seq: 1, data: { name: 'read', callId: 'c', arguments: 'not json' } }]
|
|
270
|
+
assert.equal(ingestEvents(store, 's1', bad).ok, true)
|
|
271
|
+
})
|
|
272
|
+
|
|
273
|
+
test('给 Agent 的摘要:只有事实、没有分数;没可说的就返回空串', () => {
|
|
274
|
+
const store = createStore()
|
|
275
|
+
noteSnapshot(store, 's1', pack({ primary: ['a.md'], supporting: ['b.md', 'c.md'] }), { seq: 1, at: 1 })
|
|
276
|
+
ingestEvents(store, 's1', readEvents([['a.md', 10], ['z.md', 11]], 100), { root: ROOT })
|
|
277
|
+
|
|
278
|
+
const usage = usageFor(store, 's1')
|
|
279
|
+
const summary = getAuditSummary(usage)
|
|
280
|
+
assert.match(summary, /primary read \(a\.md\)/)
|
|
281
|
+
assert.match(summary, /supporting 0\/2/)
|
|
282
|
+
assert.match(summary, /1 read outside the pack/)
|
|
283
|
+
assert.doesNotMatch(summary, /%|score|confidence/i)
|
|
284
|
+
|
|
285
|
+
const payload = auditPayload(usage)
|
|
286
|
+
assert.deepEqual(payload, {
|
|
287
|
+
primaryRead: true,
|
|
288
|
+
primaryRel: 'a.md',
|
|
289
|
+
supportingRead: 0,
|
|
290
|
+
supportingTotal: 2,
|
|
291
|
+
readsOutside: 1,
|
|
292
|
+
contextChanges: 0,
|
|
293
|
+
})
|
|
294
|
+
assert.equal(getAuditSummary(null), '')
|
|
295
|
+
assert.equal(auditPayload(null), null)
|
|
296
|
+
})
|
|
297
|
+
|
|
298
|
+
test('Usage 的形状是稳定的(给面板与工具用的字段一个不少)', () => {
|
|
299
|
+
const store = createStore()
|
|
300
|
+
noteSnapshot(store, 's1', pack({ primary: ['a.md'], supporting: ['b.md'], related: ['c.md'] }), { seq: 1, at: 1 })
|
|
301
|
+
const usage = usageFrom(store.sessions.get('s1'))
|
|
302
|
+
assert.deepEqual(Object.keys(usage.stats).sort(), [
|
|
303
|
+
'byTier', 'churn', 'distinct', 'firstReadAt', 'firstReadRel', 'firstReadTier',
|
|
304
|
+
'outside', 'outsideReads', 'primaryFollowThrough', 'primaryRel', 'reads',
|
|
305
|
+
'supportingCoverage',
|
|
306
|
+
])
|
|
307
|
+
assert.equal(usage.stats.supportingCoverage, 0)
|
|
308
|
+
assert.equal(usage.stats.churn.snapshots, 0)
|
|
309
|
+
})
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context Feedback Eval 的**确定性**对照(v0.15)。
|
|
3
|
+
*
|
|
4
|
+
* 这里不是「再测一遍 feedback.js」—— 那些在 `test/feedback.test.mjs` 与
|
|
5
|
+
* `test/context-delta.test.mjs` 里。这里测的是**量尺本身在真实装配管线上成立**:
|
|
6
|
+
* 同一个工作区、同一批事件,只改一个变量 —— **包交没交给 agent**。
|
|
7
|
+
*
|
|
8
|
+
* Control(`deliver:false`) :没人给过上下文 ⇒ 那次读必须落在包外
|
|
9
|
+
* Treatment(`deliver:true`) :读了「先看」的那一篇 ⇒ 必须记成 follow-through
|
|
10
|
+
*
|
|
11
|
+
* 这条对照是这个功能**唯一**能自证的地方:如果 Control 臂也能测出「读命中 primary」,
|
|
12
|
+
* 那这个量尺就是在自说自话(`tools/context-feedback-eval.mjs --control` 里同一条判据)。
|
|
13
|
+
*
|
|
14
|
+
* ⚠️ 工作区一律来自 `makeWorkspace()`(AGENTS.md §6.8:样本不许借仓库里的真实文件),
|
|
15
|
+
* 且用 `quantum-anchor` 这种**只出现在一篇里**的词把 primary 钉死 —— 靠「BM25 大概会选它」
|
|
16
|
+
* 的断言会随排序引擎的调整漂移。
|
|
17
|
+
*/
|
|
18
|
+
import test from 'node:test'
|
|
19
|
+
import assert from 'node:assert/strict'
|
|
20
|
+
|
|
21
|
+
import { makeWorkspace } from './fixture.mjs'
|
|
22
|
+
import { metricsFor, parseEvents, replaySession, scopeFor } from '../tools/context-feedback-eval.mjs'
|
|
23
|
+
|
|
24
|
+
/** 目标文档:`quantum-anchor` 只在这里出现,所以它必须是 primary。 */
|
|
25
|
+
const TARGET = 'docs/target.md'
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* 造一个工作区。`extra` 走 `makeWorkspace()` —— 里面已经有 README / 笔记 / CHANGELOG 三篇。
|
|
29
|
+
* @returns {string} 工作区绝对路径
|
|
30
|
+
*/
|
|
31
|
+
function workspace() {
|
|
32
|
+
return makeWorkspace([
|
|
33
|
+
[TARGET, '# 排序目标\n\n唯一命中词 quantum-anchor 出现在这篇里,相关性排序也提到了。\n'],
|
|
34
|
+
])
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** 用户消息(`isSubstantiveTask` 认的那种)。 */
|
|
38
|
+
function userMessage(seq, text) {
|
|
39
|
+
return {
|
|
40
|
+
type: 'user/message',
|
|
41
|
+
seq,
|
|
42
|
+
time: 1000 + seq,
|
|
43
|
+
data: { role: 'user', content: [{ type: 'text', text }], source: { kind: 'user' } },
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** 一次真实的 `read` 调用(`arguments` 是 JSON **字符串** —— 宿主就是这样存的)。 */
|
|
48
|
+
function readCall(seq, filePath) {
|
|
49
|
+
const callId = `c${seq}`
|
|
50
|
+
return {
|
|
51
|
+
type: 'tool/call',
|
|
52
|
+
seq,
|
|
53
|
+
time: 1000 + seq,
|
|
54
|
+
data: { turn: 1, step: 1, callId, name: 'read', arguments: JSON.stringify({ file_path: filePath }) },
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** 对应的成功返回。 */
|
|
59
|
+
function toolResult(seq, callSeq) {
|
|
60
|
+
const callId = `c${callSeq}`
|
|
61
|
+
return {
|
|
62
|
+
type: 'tool/result',
|
|
63
|
+
seq,
|
|
64
|
+
time: 1000 + seq,
|
|
65
|
+
sourceEventSeqs: [callSeq],
|
|
66
|
+
data: {
|
|
67
|
+
turn: 1, step: 1,
|
|
68
|
+
message: {
|
|
69
|
+
role: 'tool',
|
|
70
|
+
source: { kind: 'tool', callId },
|
|
71
|
+
toolCallId: callId,
|
|
72
|
+
content: [{ type: 'text', text: '<path>…</path>' }],
|
|
73
|
+
isError: false,
|
|
74
|
+
id: `m${seq}`,
|
|
75
|
+
},
|
|
76
|
+
},
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* 一份完整的事件流:一句话 → 读「先看」那篇 → 再读一个 Knit 根本索引不到的文件。
|
|
82
|
+
* @returns {object[]} 事件
|
|
83
|
+
*/
|
|
84
|
+
function treatmentEvents() {
|
|
85
|
+
return [
|
|
86
|
+
userMessage(1, '项目的相关性排序是怎么算的?quantum-anchor 在哪一篇里'),
|
|
87
|
+
readCall(2, TARGET),
|
|
88
|
+
toolResult(3, 2),
|
|
89
|
+
readCall(4, 'package.json'),
|
|
90
|
+
toolResult(5, 4),
|
|
91
|
+
]
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
test('评测:交过包才能记成 follow-through —— Control 与 Treatment 只差这一个变量', async () => {
|
|
95
|
+
const root = workspace()
|
|
96
|
+
const events = treatmentEvents()
|
|
97
|
+
|
|
98
|
+
const control = await replaySession(events, { root, sessionId: 'ctl', deliver: false })
|
|
99
|
+
const treated = await replaySession(events, { root, sessionId: 'trt', deliver: true })
|
|
100
|
+
|
|
101
|
+
// 前提:这篇确实是「先看」的那一篇(否则下面两条断言测的是别的东西)
|
|
102
|
+
assert.equal(treated.metrics.primaryRel, TARGET, 'quantum-anchor 应当把这篇钉成 primary')
|
|
103
|
+
|
|
104
|
+
// Treatment:读了 primary ⇒ follow-through,第一次读落在 primary 层
|
|
105
|
+
assert.equal(treated.packs > 0, true, 'Treatment 必须真的交过包')
|
|
106
|
+
assert.equal(treated.metrics.primaryRead, true)
|
|
107
|
+
assert.equal(treated.metrics.firstReadTier, 'primary')
|
|
108
|
+
assert.equal(treated.metrics.firstReadRel, TARGET)
|
|
109
|
+
|
|
110
|
+
// Control:一份包都没交 ⇒ 那次读只能落在包外,且**不许**记成 follow-through
|
|
111
|
+
assert.equal(control.packs, 0)
|
|
112
|
+
assert.equal(control.metrics.primaryRead, false)
|
|
113
|
+
assert.equal(control.metrics.firstReadTier, null)
|
|
114
|
+
assert.equal(control.metrics.outside, 2, '两次读都该在包外')
|
|
115
|
+
|
|
116
|
+
// 两边的读**次数**一样 —— 变的只是「有没有人给过上下文」
|
|
117
|
+
assert.equal(control.metrics.reads, treated.metrics.reads)
|
|
118
|
+
assert.equal(control.metrics.distinct, treated.metrics.distinct)
|
|
119
|
+
})
|
|
120
|
+
|
|
121
|
+
test('评测:包外的读要拆成「Knit 看得到却没进包」与「Knit 根本不索引」', async () => {
|
|
122
|
+
const root = workspace()
|
|
123
|
+
const result = await replaySession(treatmentEvents(), { root, sessionId: 'scope' })
|
|
124
|
+
|
|
125
|
+
// `package.json` 不是 `.md` ⇒ 永远不可能进包。它**不算漏**。
|
|
126
|
+
assert.deepEqual(result.scope.outOfScopeReads, ['package.json'])
|
|
127
|
+
assert.equal(result.scope.outOfScope, 1)
|
|
128
|
+
// 被读的那篇 `.md` 在扫描结果里、也在包里 ⇒ 既不是漏也不是域外
|
|
129
|
+
assert.equal(result.scope.inScope, 1)
|
|
130
|
+
assert.deepEqual(result.scope.missed, [])
|
|
131
|
+
|
|
132
|
+
// Control 臂里,同一篇 `.md` 就变成了**真的漏**(Knit 看得到它,却没人把包交出去)
|
|
133
|
+
const control = await replaySession(treatmentEvents(), { root, sessionId: 'scope-ctl', deliver: false })
|
|
134
|
+
assert.deepEqual(control.scope.missed, [TARGET])
|
|
135
|
+
assert.equal(control.scope.outOfScope, 1)
|
|
136
|
+
})
|
|
137
|
+
|
|
138
|
+
test('评测:话题换了之后快照跟着换(churn 记的是事实,不是估算)', async () => {
|
|
139
|
+
const root = workspace()
|
|
140
|
+
const events = [
|
|
141
|
+
...treatmentEvents(),
|
|
142
|
+
userMessage(6, '换个话题:把发布记录整理一下'),
|
|
143
|
+
readCall(7, TARGET),
|
|
144
|
+
toolResult(8, 7),
|
|
145
|
+
]
|
|
146
|
+
const result = await replaySession(events, { root, sessionId: 'churn' })
|
|
147
|
+
|
|
148
|
+
assert.equal(result.metrics.contextChanges >= 1, true, '任务变了必须换一份包')
|
|
149
|
+
assert.equal(result.metrics.taskChanged >= 1, true)
|
|
150
|
+
assert.equal(result.metrics.reads, 3)
|
|
151
|
+
})
|
|
152
|
+
|
|
153
|
+
test('评测:同样的输入必须给出逐字一样的读数(两次回放 deepEqual)', async () => {
|
|
154
|
+
const root = workspace()
|
|
155
|
+
const first = await replaySession(treatmentEvents(), { root, sessionId: 'det' })
|
|
156
|
+
const second = await replaySession(treatmentEvents(), { root, sessionId: 'det' })
|
|
157
|
+
assert.deepEqual(first.metrics, second.metrics)
|
|
158
|
+
assert.deepEqual(first.scope, second.scope)
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
test('评测:坏行不废整批(半截的尾行很常见)', () => {
|
|
162
|
+
const text = [
|
|
163
|
+
'{"type":"session","id":"s","cwd":"/tmp"}',
|
|
164
|
+
'{"type":"user/message","seq":1,"data":{}}',
|
|
165
|
+
'{"broken"',
|
|
166
|
+
'',
|
|
167
|
+
'不是 JSON',
|
|
168
|
+
].join('\n')
|
|
169
|
+
const events = parseEvents(text)
|
|
170
|
+
assert.equal(events.length, 2)
|
|
171
|
+
assert.equal(events[0].type, 'session')
|
|
172
|
+
})
|
|
173
|
+
|
|
174
|
+
test('评测:metricsFor 的键是稳定的(下游脚本靠它)', () => {
|
|
175
|
+
const metrics = metricsFor(null)
|
|
176
|
+
assert.deepEqual(Object.keys(metrics).sort(), [
|
|
177
|
+
'appeared', 'contextChanges', 'disappeared', 'distinct', 'firstReadRel', 'firstReadTier',
|
|
178
|
+
'moved', 'outside', 'outsideReads', 'primaryRead', 'primaryRel', 'reads',
|
|
179
|
+
'supportingCoverage', 'taskChanged',
|
|
180
|
+
])
|
|
181
|
+
assert.equal(metrics.reads, 0)
|
|
182
|
+
assert.equal(metrics.primaryRead, false)
|
|
183
|
+
assert.equal(metrics.firstReadTier, null)
|
|
184
|
+
})
|
|
185
|
+
|
|
186
|
+
test('评测:scopeFor 不认识 usage 时给空拆分,不抛', () => {
|
|
187
|
+
const scope = scopeFor(null, ['a.md'])
|
|
188
|
+
assert.deepEqual(scope, { inScope: 0, missed: [], missedReads: 0, outOfScope: 0, outOfScopeReads: [] })
|
|
189
|
+
})
|