@devflow-tools/benchmark 0.17.7 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +2 -2
  3. package/dist/accuracy-command-options.d.ts +3 -0
  4. package/dist/accuracy-command-options.d.ts.map +1 -0
  5. package/dist/accuracy-command-options.js +17 -0
  6. package/dist/accuracy-command-options.js.map +1 -0
  7. package/dist/context-symbol-quality-command.d.ts +59 -0
  8. package/dist/context-symbol-quality-command.d.ts.map +1 -0
  9. package/dist/context-symbol-quality-command.js +482 -0
  10. package/dist/context-symbol-quality-command.js.map +1 -0
  11. package/dist/evaluation-bundle-builder.d.ts +48 -0
  12. package/dist/evaluation-bundle-builder.d.ts.map +1 -0
  13. package/dist/evaluation-bundle-builder.js +496 -0
  14. package/dist/evaluation-bundle-builder.js.map +1 -0
  15. package/dist/evaluation-bundle-command.d.ts +2 -0
  16. package/dist/evaluation-bundle-command.d.ts.map +1 -0
  17. package/dist/evaluation-bundle-command.js +15 -0
  18. package/dist/evaluation-bundle-command.js.map +1 -0
  19. package/dist/evaluation-bundle.d.ts +55 -0
  20. package/dist/evaluation-bundle.d.ts.map +1 -0
  21. package/dist/evaluation-bundle.js +272 -0
  22. package/dist/evaluation-bundle.js.map +1 -0
  23. package/dist/evaluation-run-state.d.ts +23 -0
  24. package/dist/evaluation-run-state.d.ts.map +1 -0
  25. package/dist/evaluation-run-state.js +115 -0
  26. package/dist/evaluation-run-state.js.map +1 -0
  27. package/dist/evidence-quality-metrics.d.ts +15 -0
  28. package/dist/evidence-quality-metrics.d.ts.map +1 -1
  29. package/dist/evidence-quality-metrics.js +51 -1
  30. package/dist/evidence-quality-metrics.js.map +1 -1
  31. package/dist/graders/deterministic.d.ts.map +1 -1
  32. package/dist/graders/deterministic.js +5 -3
  33. package/dist/graders/deterministic.js.map +1 -1
  34. package/dist/index.d.ts +15 -2
  35. package/dist/index.d.ts.map +1 -1
  36. package/dist/index.js +12 -1
  37. package/dist/index.js.map +1 -1
  38. package/dist/release-quality-bundle-command.d.ts +3 -0
  39. package/dist/release-quality-bundle-command.d.ts.map +1 -0
  40. package/dist/release-quality-bundle-command.js +72 -0
  41. package/dist/release-quality-bundle-command.js.map +1 -0
  42. package/dist/release-quality-bundle-evaluator.d.ts +61 -0
  43. package/dist/release-quality-bundle-evaluator.d.ts.map +1 -0
  44. package/dist/release-quality-bundle-evaluator.js +321 -0
  45. package/dist/release-quality-bundle-evaluator.js.map +1 -0
  46. package/dist/release-quality-bundle-materializer.d.ts +79 -0
  47. package/dist/release-quality-bundle-materializer.d.ts.map +1 -0
  48. package/dist/release-quality-bundle-materializer.js +395 -0
  49. package/dist/release-quality-bundle-materializer.js.map +1 -0
  50. package/dist/release-quality-comparison.d.ts +34 -0
  51. package/dist/release-quality-comparison.d.ts.map +1 -0
  52. package/dist/release-quality-comparison.js +60 -0
  53. package/dist/release-quality-comparison.js.map +1 -0
  54. package/dist/release-quality-evaluation-command.d.ts +42 -0
  55. package/dist/release-quality-evaluation-command.d.ts.map +1 -0
  56. package/dist/release-quality-evaluation-command.js +91 -0
  57. package/dist/release-quality-evaluation-command.js.map +1 -0
  58. package/dist/release-quality-host-proof.d.ts +30 -0
  59. package/dist/release-quality-host-proof.d.ts.map +1 -0
  60. package/dist/release-quality-host-proof.js +109 -0
  61. package/dist/release-quality-host-proof.js.map +1 -0
  62. package/dist/release-quality-output-cleanup.d.ts +2 -0
  63. package/dist/release-quality-output-cleanup.d.ts.map +1 -0
  64. package/dist/release-quality-output-cleanup.js +18 -0
  65. package/dist/release-quality-output-cleanup.js.map +1 -0
  66. package/dist/release-quality-policy.d.ts +87 -0
  67. package/dist/release-quality-policy.d.ts.map +1 -0
  68. package/dist/release-quality-policy.js +179 -0
  69. package/dist/release-quality-policy.js.map +1 -0
  70. package/dist/release-quality-query-sampling.d.ts +36 -0
  71. package/dist/release-quality-query-sampling.d.ts.map +1 -0
  72. package/dist/release-quality-query-sampling.js +87 -0
  73. package/dist/release-quality-query-sampling.js.map +1 -0
  74. package/dist/release-quality-source-snapshot.d.ts +18 -0
  75. package/dist/release-quality-source-snapshot.d.ts.map +1 -0
  76. package/dist/release-quality-source-snapshot.js +46 -0
  77. package/dist/release-quality-source-snapshot.js.map +1 -0
  78. package/dist/retrieval-quality-corpus.d.ts +0 -1
  79. package/dist/retrieval-quality-corpus.d.ts.map +1 -1
  80. package/dist/retrieval-quality-corpus.js +1 -1
  81. package/dist/retrieval-quality-corpus.js.map +1 -1
  82. package/dist/retrieval-quality.d.ts.map +1 -1
  83. package/dist/retrieval-quality.js +29 -26
  84. package/dist/retrieval-quality.js.map +1 -1
  85. package/dist/run-accuracy-tests.js +82 -2
  86. package/dist/run-accuracy-tests.js.map +1 -1
  87. package/dist/suite-runner.js +1 -1
  88. package/dist/suite-runner.js.map +1 -1
  89. package/dist/unified-runtime-corpus.d.ts +64 -0
  90. package/dist/unified-runtime-corpus.d.ts.map +1 -0
  91. package/dist/unified-runtime-corpus.js +161 -0
  92. package/dist/unified-runtime-corpus.js.map +1 -0
  93. package/dist/unified-runtime-evaluation.d.ts +133 -0
  94. package/dist/unified-runtime-evaluation.d.ts.map +1 -0
  95. package/dist/unified-runtime-evaluation.js +388 -0
  96. package/dist/unified-runtime-evaluation.js.map +1 -0
  97. package/dist/unified-runtime-observation.d.ts +101 -0
  98. package/dist/unified-runtime-observation.d.ts.map +1 -0
  99. package/dist/unified-runtime-observation.js +2 -0
  100. package/dist/unified-runtime-observation.js.map +1 -0
  101. package/dist/unified-runtime-runner.d.ts +73 -0
  102. package/dist/unified-runtime-runner.d.ts.map +1 -0
  103. package/dist/unified-runtime-runner.js +620 -0
  104. package/dist/unified-runtime-runner.js.map +1 -0
  105. package/fixtures/evidence-runtime-corpus.json +0 -100
  106. package/fixtures/release-quality/bundles.json +1 -0
  107. package/fixtures/release-quality/calibration.json +602 -0
  108. package/fixtures/release-quality/ecc/corpus.json +257 -0
  109. package/fixtures/release-quality/ecc/input.json +62 -0
  110. package/fixtures/release-quality/llm-wiki/corpus.json +263 -0
  111. package/fixtures/release-quality/llm-wiki/input.json +72 -0
  112. package/fixtures/release-quality/mem0/corpus.json +288 -0
  113. package/fixtures/release-quality/mem0/input.json +68 -0
  114. package/fixtures/release-quality/projects.json +29 -0
  115. package/fixtures/release-quality/release-quality-policy.json +232 -0
  116. package/fixtures/release-quality/rsbuild-react/corpus.json +263 -0
  117. package/fixtures/release-quality/rsbuild-react/input.json +62 -0
  118. package/fixtures/retrieval-quality-corpus.json +7 -7
  119. package/fixtures/semantic-regex-inventory.json +3 -10
  120. package/fixtures/unified-runtime/baselines/2026-08-08.json +37 -0
  121. package/fixtures/unified-runtime/context.json +206 -0
  122. package/fixtures/unified-runtime/knowledge.json +224 -0
  123. package/fixtures/unified-runtime/memory.json +224 -0
  124. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/context.json +32 -0
  125. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/knowledge.json +32 -0
  126. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/memory.json +32 -0
  127. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/semantic.json +409 -0
  128. package/fixtures/unified-runtime/runs/rsbuild-react-2026-08-05.json +54 -0
  129. package/fixtures/unified-runtime/semantic.json +409 -0
  130. package/package.json +24 -6
@@ -0,0 +1,257 @@
1
+ {
2
+ "schemaVersion": "devflow.evaluation-corpus.v1",
3
+ "cases": [
4
+ {
5
+ "caseId": "ecc/context/path-safety",
6
+ "channel": "context",
7
+ "caseClass": "positive",
8
+ "prompt": "ECC 如何 canonicalize 尚不存在的目标路径并阻止 symlink 越过 trusted install root?",
9
+ "querySampling": {
10
+ "status": "sampled",
11
+ "provider": "deepseek",
12
+ "model": "deepseek-v4-pro",
13
+ "drafts": [
14
+ {
15
+ "channel": "code",
16
+ "kind": "rewrite",
17
+ "query": "canonicalize path symlink resolution trusted install root",
18
+ "sourceClaimIds": [
19
+ "semantic-claim:970fff7c8ae54297ba5d76510dcfc066"
20
+ ],
21
+ "confidence": 0.6
22
+ },
23
+ {
24
+ "channel": "code",
25
+ "kind": "expansion",
26
+ "query": "canonicalize nonexistent target path prevent symlink escape install root",
27
+ "sourceClaimIds": [
28
+ "semantic-claim:970fff7c8ae54297ba5d76510dcfc066"
29
+ ],
30
+ "confidence": 0.5
31
+ },
32
+ {
33
+ "channel": "code",
34
+ "kind": "expansion",
35
+ "query": "resolve symlink containment check trusted root path traversal",
36
+ "sourceClaimIds": [
37
+ "semantic-claim:970fff7c8ae54297ba5d76510dcfc066"
38
+ ],
39
+ "confidence": 0.4
40
+ },
41
+ {
42
+ "channel": "knowledge",
43
+ "kind": "rewrite",
44
+ "query": "ECC canonicalize path symlink trusted install root",
45
+ "sourceClaimIds": [
46
+ "semantic-claim:970fff7c8ae54297ba5d76510dcfc066"
47
+ ],
48
+ "confidence": 0.5
49
+ },
50
+ {
51
+ "channel": "knowledge",
52
+ "kind": "expansion",
53
+ "query": "canonicalize nonexistent path prevent symlink escape install root",
54
+ "sourceClaimIds": [
55
+ "semantic-claim:970fff7c8ae54297ba5d76510dcfc066"
56
+ ],
57
+ "confidence": 0.4
58
+ },
59
+ {
60
+ "channel": "memory",
61
+ "kind": "rewrite",
62
+ "query": "ECC canonicalize path symlink trusted install root",
63
+ "sourceClaimIds": [
64
+ "semantic-claim:970fff7c8ae54297ba5d76510dcfc066"
65
+ ],
66
+ "confidence": 0.4
67
+ }
68
+ ]
69
+ }
70
+ },
71
+ {
72
+ "caseId": "ecc/memory/trusted-root",
73
+ "channel": "memory",
74
+ "caseClass": "positive",
75
+ "prompt": "ECC 安装状态驱动的写入和删除必须复用哪些路径安全函数?",
76
+ "querySampling": {
77
+ "status": "sampled",
78
+ "provider": "deepseek",
79
+ "model": "deepseek-v4-pro",
80
+ "drafts": [
81
+ {
82
+ "channel": "code",
83
+ "kind": "rewrite",
84
+ "query": "ECC install state-driven write and delete path safety functions",
85
+ "sourceClaimIds": [
86
+ "semantic-claim:2b65f2fc07cb89c25ca6ef31931a1b7c",
87
+ "semantic-claim:c8bd22fc098d708b1918a7160e225c32"
88
+ ],
89
+ "confidence": 0.85
90
+ },
91
+ {
92
+ "channel": "code",
93
+ "kind": "expansion",
94
+ "query": "ECC install write path safe write function caller entry point",
95
+ "sourceClaimIds": [
96
+ "semantic-claim:2b65f2fc07cb89c25ca6ef31931a1b7c",
97
+ "semantic-claim:c8bd22fc098d708b1918a7160e225c32"
98
+ ],
99
+ "confidence": 0.7
100
+ },
101
+ {
102
+ "channel": "code",
103
+ "kind": "expansion",
104
+ "query": "ECC install delete path safe delete function pipeline orchestration",
105
+ "sourceClaimIds": [
106
+ "semantic-claim:2b65f2fc07cb89c25ca6ef31931a1b7c",
107
+ "semantic-claim:c8bd22fc098d708b1918a7160e225c32"
108
+ ],
109
+ "confidence": 0.7
110
+ },
111
+ {
112
+ "channel": "memory",
113
+ "kind": "rewrite",
114
+ "query": "ECC 安装状态 写入 删除 路径安全函数",
115
+ "sourceClaimIds": [
116
+ "semantic-claim:2b65f2fc07cb89c25ca6ef31931a1b7c",
117
+ "semantic-claim:c8bd22fc098d708b1918a7160e225c32"
118
+ ],
119
+ "confidence": 0.6
120
+ },
121
+ {
122
+ "channel": "knowledge",
123
+ "kind": "rewrite",
124
+ "query": "ECC installation state write delete path safety functions",
125
+ "sourceClaimIds": [
126
+ "semantic-claim:2b65f2fc07cb89c25ca6ef31931a1b7c",
127
+ "semantic-claim:c8bd22fc098d708b1918a7160e225c32"
128
+ ],
129
+ "confidence": 0.55
130
+ }
131
+ ]
132
+ }
133
+ },
134
+ {
135
+ "caseId": "ecc/memory/explicit-save",
136
+ "channel": "memory",
137
+ "caseClass": "lifecycle",
138
+ "prompt": "当前 ECC 安装状态驱动的写入和删除必须复用哪些路径安全函数并遵守什么 containment?",
139
+ "expectedLifecycle": "latest_episode_selected",
140
+ "querySampling": {
141
+ "status": "sampled",
142
+ "provider": "deepseek",
143
+ "model": "deepseek-v4-pro",
144
+ "drafts": [
145
+ {
146
+ "channel": "code",
147
+ "kind": "rewrite",
148
+ "query": "ECC install status driven write and delete path safety functions containment",
149
+ "sourceClaimIds": [
150
+ "semantic-claim:67e48fd84f7954d08fcc93d2877e91df",
151
+ "semantic-claim:6b6adabad513697b8e5977ad559d5ab0"
152
+ ],
153
+ "confidence": 0.72
154
+ },
155
+ {
156
+ "channel": "code",
157
+ "kind": "expansion",
158
+ "query": "ECC installation state write path safe write function caller entry point",
159
+ "sourceClaimIds": [
160
+ "semantic-claim:67e48fd84f7954d08fcc93d2877e91df",
161
+ "semantic-claim:6b6adabad513697b8e5977ad559d5ab0"
162
+ ],
163
+ "confidence": 0.55
164
+ },
165
+ {
166
+ "channel": "code",
167
+ "kind": "expansion",
168
+ "query": "ECC install status delete path containment orchestration pipeline data flow",
169
+ "sourceClaimIds": [
170
+ "semantic-claim:67e48fd84f7954d08fcc93d2877e91df",
171
+ "semantic-claim:6b6adabad513697b8e5977ad559d5ab0"
172
+ ],
173
+ "confidence": 0.5
174
+ },
175
+ {
176
+ "channel": "memory",
177
+ "kind": "rewrite",
178
+ "query": "ECC 安装状态 写入 删除 路径安全函数 containment",
179
+ "sourceClaimIds": [
180
+ "semantic-claim:67e48fd84f7954d08fcc93d2877e91df",
181
+ "semantic-claim:6b6adabad513697b8e5977ad559d5ab0"
182
+ ],
183
+ "confidence": 0.5
184
+ },
185
+ {
186
+ "channel": "knowledge",
187
+ "kind": "rewrite",
188
+ "query": "ECC installation status write delete path safety functions containment requirements",
189
+ "sourceClaimIds": [
190
+ "semantic-claim:67e48fd84f7954d08fcc93d2877e91df",
191
+ "semantic-claim:6b6adabad513697b8e5977ad559d5ab0"
192
+ ],
193
+ "confidence": 0.45
194
+ }
195
+ ]
196
+ }
197
+ },
198
+ {
199
+ "caseId": "ecc/knowledge/v2-control-plane",
200
+ "channel": "knowledge",
201
+ "caseClass": "positive",
202
+ "prompt": "ECC 2.0 稳定版包含哪些 control-plane、session 和 worktree 能力?",
203
+ "querySampling": {
204
+ "status": "sampled",
205
+ "provider": "deepseek",
206
+ "model": "deepseek-v4-pro",
207
+ "drafts": [
208
+ {
209
+ "channel": "code",
210
+ "kind": "rewrite",
211
+ "query": "ECC 2.0 stable control-plane session worktree capabilities",
212
+ "sourceClaimIds": [
213
+ "semantic-claim:55c173a4b8c5119643192489f1be9bda"
214
+ ],
215
+ "confidence": 0.6
216
+ },
217
+ {
218
+ "channel": "code",
219
+ "kind": "expansion",
220
+ "query": "ECC control plane session management worktree operations",
221
+ "sourceClaimIds": [
222
+ "semantic-claim:55c173a4b8c5119643192489f1be9bda"
223
+ ],
224
+ "confidence": 0.5
225
+ },
226
+ {
227
+ "channel": "code",
228
+ "kind": "expansion",
229
+ "query": "ECC session worktree lifecycle entry point orchestration",
230
+ "sourceClaimIds": [
231
+ "semantic-claim:55c173a4b8c5119643192489f1be9bda"
232
+ ],
233
+ "confidence": 0.4
234
+ },
235
+ {
236
+ "channel": "memory",
237
+ "kind": "rewrite",
238
+ "query": "ECC 2.0 stable control-plane session worktree capabilities",
239
+ "sourceClaimIds": [
240
+ "semantic-claim:55c173a4b8c5119643192489f1be9bda"
241
+ ],
242
+ "confidence": 0.5
243
+ },
244
+ {
245
+ "channel": "knowledge",
246
+ "kind": "rewrite",
247
+ "query": "ECC 2.0 stable control-plane session worktree capabilities",
248
+ "sourceClaimIds": [
249
+ "semantic-claim:55c173a4b8c5119643192489f1be9bda"
250
+ ],
251
+ "confidence": 0.5
252
+ }
253
+ ]
254
+ }
255
+ }
256
+ ]
257
+ }
@@ -0,0 +1,62 @@
1
+ {
2
+ "schemaVersion": "devflow.release-quality-project-input.v1",
3
+ "project": {
4
+ "projectId": "github:affaan-m/ecc",
5
+ "repository": "https://github.com/affaan-m/ECC.git",
6
+ "commit": "a3130f9ebfaeed075df5d5b52538acb0ee4bcdf8",
7
+ "tree": "685a0abce1f4e1544ea8a71026a0233675a78528",
8
+ "checkoutDirectory": "ecc",
9
+ "sourceSnapshot": { "manifestSha256": "7b30c405fdbf4ca1a5d4e639538a6a7c31419b16d7b8a4696b8682509c8cd921" }
10
+ },
11
+ "context": {
12
+ "evidence": [
13
+ {
14
+ "path": "scripts/lib/path-safety.js",
15
+ "sha256": "09b5208a8ff3f40c5f69921bdd0da5441a29582f75d159c47792c65be6c27ac0",
16
+ "symbol": "assertWithinTrustedRoot",
17
+ "startLine": 65
18
+ }
19
+ ]
20
+ },
21
+ "memory": {
22
+ "actorId": "actor:release-quality-reviewer",
23
+ "sessionId": "session:ecc-release-quality",
24
+ "prompt": "请记住:ECC 安装状态驱动的写入与删除统一复用 realpathNearestExisting、isWithinRoot 和 assertWithinTrustedRoot,禁止绕过 trusted root containment。",
25
+ "explicitContent": "ECC 安装状态驱动的写入与删除统一复用 realpathNearestExisting、isWithinRoot 和 assertWithinTrustedRoot,禁止绕过 trusted root containment。",
26
+ "temporal": {
27
+ "olderContent": "ECC 安装状态驱动的写入与删除必须复用 assertWithinTrustedRoot 并遵守 trusted root containment。",
28
+ "currentContent": "ECC 安装状态驱动的写入与删除统一复用 realpathNearestExisting、isWithinRoot 和 assertWithinTrustedRoot,禁止绕过 trusted root containment。",
29
+ "query": "当前 ECC 安装状态驱动的写入和删除必须复用哪些路径安全函数并遵守什么 containment?"
30
+ },
31
+ "sourceEvidence": [
32
+ {
33
+ "path": "scripts/lib/path-safety.js",
34
+ "sha256": "09b5208a8ff3f40c5f69921bdd0da5441a29582f75d159c47792c65be6c27ac0"
35
+ }
36
+ ]
37
+ },
38
+ "knowledge": {
39
+ "sourceId": "ecc-maintainer-guide",
40
+ "version": "2.0.0",
41
+ "pluginName": "release-quality",
42
+ "goldText": "the control-pane substrate (session adapters + MCP inventory), the worktree-lifecycle service, the `orch-*` orchestrator family",
43
+ "provenance": {
44
+ "repository": "https://github.com/affaan-m/ECC.git",
45
+ "commit": "a3130f9ebfaeed075df5d5b52538acb0ee4bcdf8"
46
+ },
47
+ "documents": [
48
+ {
49
+ "path": "README.md",
50
+ "title": "ECC maintainer guide",
51
+ "sourceUri": "https://github.com/affaan-m/ECC/blob/a3130f9ebfaeed075df5d5b52538acb0ee4bcdf8/README.md",
52
+ "sha256": "3a4c5c55a3ce1b8b854b62f939999c503c3f6d137d483b39c64600e75f138d04"
53
+ }
54
+ ]
55
+ },
56
+ "requiredSlices": [
57
+ { "channel": "context", "caseClass": "positive", "caseIds": ["ecc/context/path-safety"] },
58
+ { "channel": "memory", "caseClass": "positive", "caseIds": ["ecc/memory/trusted-root"] },
59
+ { "channel": "memory", "caseClass": "lifecycle", "caseIds": ["ecc/memory/explicit-save"] },
60
+ { "channel": "knowledge", "caseClass": "positive", "caseIds": ["ecc/knowledge/v2-control-plane"] }
61
+ ]
62
+ }
@@ -0,0 +1,263 @@
1
+ {
2
+ "schemaVersion": "devflow.evaluation-corpus.v1",
3
+ "cases": [
4
+ {
5
+ "caseId": "llm-wiki/context/explicit-files",
6
+ "channel": "context",
7
+ "caseClass": "positive",
8
+ "prompt": "LLM Wiki 如何安全加载用户显式选择的上下文文件,并在发送给模型前限制文件和最终上下文大小?",
9
+ "querySampling": {
10
+ "status": "sampled",
11
+ "provider": "deepseek",
12
+ "model": "deepseek-v4-pro",
13
+ "drafts": [
14
+ {
15
+ "channel": "code",
16
+ "kind": "rewrite",
17
+ "query": "load user-selected context file safely limit file size and final context size before sending to model",
18
+ "sourceClaimIds": [
19
+ "semantic-claim:d6e11cd2b3876ba1b9892c2334ac8366",
20
+ "semantic-claim:f784540c93e0d438bcbfdc259c091e1c"
21
+ ],
22
+ "confidence": 0.55
23
+ },
24
+ {
25
+ "channel": "code",
26
+ "kind": "expansion",
27
+ "query": "read context file validate size truncate content build prompt context",
28
+ "sourceClaimIds": [
29
+ "semantic-claim:d6e11cd2b3876ba1b9892c2334ac8366"
30
+ ],
31
+ "confidence": 0.45
32
+ },
33
+ {
34
+ "channel": "code",
35
+ "kind": "expansion",
36
+ "query": "context file loading pipeline orchestration size limit enforcement before model request",
37
+ "sourceClaimIds": [
38
+ "semantic-claim:f784540c93e0d438bcbfdc259c091e1c"
39
+ ],
40
+ "confidence": 0.4
41
+ },
42
+ {
43
+ "channel": "memory",
44
+ "kind": "rewrite",
45
+ "query": "LLM Wiki safe loading of user-selected context file with size limits",
46
+ "sourceClaimIds": [
47
+ "semantic-claim:d6e11cd2b3876ba1b9892c2334ac8366",
48
+ "semantic-claim:f784540c93e0d438bcbfdc259c091e1c"
49
+ ],
50
+ "confidence": 0.5
51
+ },
52
+ {
53
+ "channel": "memory",
54
+ "kind": "expansion",
55
+ "query": "context file size limit final context size constraint model input",
56
+ "sourceClaimIds": [
57
+ "semantic-claim:d6e11cd2b3876ba1b9892c2334ac8366"
58
+ ],
59
+ "confidence": 0.4
60
+ },
61
+ {
62
+ "channel": "knowledge",
63
+ "kind": "rewrite",
64
+ "query": "how to safely load user-selected context file and limit file and final context size before sending to LLM",
65
+ "sourceClaimIds": [
66
+ "semantic-claim:d6e11cd2b3876ba1b9892c2334ac8366",
67
+ "semantic-claim:f784540c93e0d438bcbfdc259c091e1c"
68
+ ],
69
+ "confidence": 0.5
70
+ },
71
+ {
72
+ "channel": "knowledge",
73
+ "kind": "expansion",
74
+ "query": "LLM context window size limits file size limits safe file loading best practices",
75
+ "sourceClaimIds": [
76
+ "semantic-claim:d6e11cd2b3876ba1b9892c2334ac8366"
77
+ ],
78
+ "confidence": 0.45
79
+ },
80
+ {
81
+ "channel": "knowledge",
82
+ "kind": "expansion",
83
+ "query": "wiki knowledge base context file selection size enforcement model prompt",
84
+ "sourceClaimIds": [
85
+ "semantic-claim:f784540c93e0d438bcbfdc259c091e1c"
86
+ ],
87
+ "confidence": 0.4
88
+ }
89
+ ]
90
+ }
91
+ },
92
+ {
93
+ "caseId": "llm-wiki/memory/context-safety",
94
+ "channel": "memory",
95
+ "caseClass": "positive",
96
+ "prompt": "维护显式上下文加载时必须保留哪些安全与预算约束?",
97
+ "querySampling": {
98
+ "status": "sampled",
99
+ "provider": "deepseek",
100
+ "model": "deepseek-v4-pro",
101
+ "drafts": [
102
+ {
103
+ "channel": "code",
104
+ "kind": "rewrite",
105
+ "query": "explicit context loading security constraints budget constraints",
106
+ "sourceClaimIds": [
107
+ "semantic-claim:340472b14ef8c1717bf964a0a39ad3f5"
108
+ ],
109
+ "confidence": 0.55
110
+ },
111
+ {
112
+ "channel": "code",
113
+ "kind": "expansion",
114
+ "query": "context loader security validation budget limit enforcement",
115
+ "sourceClaimIds": [
116
+ "semantic-claim:340472b14ef8c1717bf964a0a39ad3f5"
117
+ ],
118
+ "confidence": 0.45
119
+ },
120
+ {
121
+ "channel": "code",
122
+ "kind": "expansion",
123
+ "query": "explicit context load pipeline caller entry point budget check",
124
+ "sourceClaimIds": [
125
+ "semantic-claim:340472b14ef8c1717bf964a0a39ad3f5"
126
+ ],
127
+ "confidence": 0.4
128
+ },
129
+ {
130
+ "channel": "memory",
131
+ "kind": "rewrite",
132
+ "query": "显式上下文加载 安全约束 预算约束",
133
+ "sourceClaimIds": [
134
+ "semantic-claim:340472b14ef8c1717bf964a0a39ad3f5"
135
+ ],
136
+ "confidence": 0.5
137
+ },
138
+ {
139
+ "channel": "memory",
140
+ "kind": "expansion",
141
+ "query": "上下文加载 安全策略 预算限制 保留约束",
142
+ "sourceClaimIds": [
143
+ "semantic-claim:340472b14ef8c1717bf964a0a39ad3f5"
144
+ ],
145
+ "confidence": 0.4
146
+ },
147
+ {
148
+ "channel": "knowledge",
149
+ "kind": "rewrite",
150
+ "query": "explicit context loading security and budget constraints to preserve",
151
+ "sourceClaimIds": [
152
+ "semantic-claim:340472b14ef8c1717bf964a0a39ad3f5"
153
+ ],
154
+ "confidence": 0.5
155
+ },
156
+ {
157
+ "channel": "knowledge",
158
+ "kind": "expansion",
159
+ "query": "context loading safety constraints budget limits best practices",
160
+ "sourceClaimIds": [
161
+ "semantic-claim:340472b14ef8c1717bf964a0a39ad3f5"
162
+ ],
163
+ "confidence": 0.4
164
+ }
165
+ ]
166
+ }
167
+ },
168
+ {
169
+ "caseId": "llm-wiki/memory/explicit-save",
170
+ "channel": "memory",
171
+ "caseClass": "lifecycle",
172
+ "prompt": "当前显式上下文加载必须保留哪些文件数量、路径安全和模型窗口预算约束?",
173
+ "expectedLifecycle": "latest_episode_selected",
174
+ "querySampling": {
175
+ "status": "not_requested",
176
+ "drafts": []
177
+ }
178
+ },
179
+ {
180
+ "caseId": "llm-wiki/knowledge/mcp-boundary",
181
+ "channel": "knowledge",
182
+ "caseClass": "positive",
183
+ "prompt": "LLM Wiki MCP Server 是否复制搜索逻辑或直接扫描项目,它如何继承桌面 API 的安全边界?",
184
+ "querySampling": {
185
+ "status": "sampled",
186
+ "provider": "deepseek",
187
+ "model": "deepseek-v4-pro",
188
+ "drafts": [
189
+ {
190
+ "channel": "code",
191
+ "kind": "rewrite",
192
+ "query": "LLM Wiki MCP Server search logic vs direct project scan",
193
+ "sourceClaimIds": [
194
+ "semantic-claim:390911f30fd36681214cf39c135b7df9",
195
+ "semantic-claim:b6f56cbb644e040db91410c4cfdb3b3a",
196
+ "semantic-claim:ddd919c616b12b638d51f5bce3c5a6a2",
197
+ "semantic-claim:5c010b7342d9374ed94385a5edb9e4ac"
198
+ ],
199
+ "confidence": 0.7
200
+ },
201
+ {
202
+ "channel": "code",
203
+ "kind": "expansion",
204
+ "query": "MCP server tool handler search implementation wiki query",
205
+ "sourceClaimIds": [
206
+ "semantic-claim:ddd919c616b12b638d51f5bce3c5a6a2",
207
+ "semantic-claim:5c010b7342d9374ed94385a5edb9e4ac",
208
+ "semantic-claim:b6f56cbb644e040db91410c4cfdb3b3a"
209
+ ],
210
+ "confidence": 0.6
211
+ },
212
+ {
213
+ "channel": "code",
214
+ "kind": "expansion",
215
+ "query": "MCP server entry point request routing security boundary desktop API",
216
+ "sourceClaimIds": [
217
+ "semantic-claim:ddd919c616b12b638d51f5bce3c5a6a2",
218
+ "semantic-claim:5c010b7342d9374ed94385a5edb9e4ac",
219
+ "semantic-claim:3ae2f14aaffd10543afee5ef7f3e3589"
220
+ ],
221
+ "confidence": 0.55
222
+ },
223
+ {
224
+ "channel": "knowledge",
225
+ "kind": "rewrite",
226
+ "query": "LLM Wiki MCP Server search logic versus direct project scanning security boundary inheritance",
227
+ "sourceClaimIds": [
228
+ "semantic-claim:390911f30fd36681214cf39c135b7df9",
229
+ "semantic-claim:b6f56cbb644e040db91410c4cfdb3b3a",
230
+ "semantic-claim:ddd919c616b12b638d51f5bce3c5a6a2",
231
+ "semantic-claim:5c010b7342d9374ed94385a5edb9e4ac",
232
+ "semantic-claim:3ae2f14aaffd10543afee5ef7f3e3589"
233
+ ],
234
+ "confidence": 0.5
235
+ },
236
+ {
237
+ "channel": "knowledge",
238
+ "kind": "expansion",
239
+ "query": "MCP server desktop API security boundary permission model",
240
+ "sourceClaimIds": [
241
+ "semantic-claim:ddd919c616b12b638d51f5bce3c5a6a2",
242
+ "semantic-claim:3ae2f14aaffd10543afee5ef7f3e3589",
243
+ "semantic-claim:5c010b7342d9374ed94385a5edb9e4ac"
244
+ ],
245
+ "confidence": 0.45
246
+ },
247
+ {
248
+ "channel": "memory",
249
+ "kind": "rewrite",
250
+ "query": "LLM Wiki MCP Server search logic direct scan security boundary",
251
+ "sourceClaimIds": [
252
+ "semantic-claim:390911f30fd36681214cf39c135b7df9",
253
+ "semantic-claim:b6f56cbb644e040db91410c4cfdb3b3a",
254
+ "semantic-claim:ddd919c616b12b638d51f5bce3c5a6a2",
255
+ "semantic-claim:5c010b7342d9374ed94385a5edb9e4ac"
256
+ ],
257
+ "confidence": 0.4
258
+ }
259
+ ]
260
+ }
261
+ }
262
+ ]
263
+ }
@@ -0,0 +1,72 @@
1
+ {
2
+ "schemaVersion": "devflow.release-quality-project-input.v1",
3
+ "project": {
4
+ "projectId": "github:nashsu/llm_wiki",
5
+ "repository": "https://github.com/nashsu/llm_wiki.git",
6
+ "commit": "58e60b699a96a892baa72896d33577e70d373448",
7
+ "tree": "092d0bed8903885a7ef8d07555ac56adf4c65c5d",
8
+ "checkoutDirectory": "llm-wiki",
9
+ "sourceSnapshot": { "manifestSha256": "65b74c95eceda2d9714e3cbc825d88c1d7f9f1dc916f3b210e8ae5880ed7337c" }
10
+ },
11
+ "context": {
12
+ "evidence": [
13
+ {
14
+ "path": "src-tauri/src/agent/context.rs",
15
+ "sha256": "c6f828939b572883b2c9294f52eb1527e9dca3aa417ed765a7c9fd0cb328a514",
16
+ "symbol": "load_explicit_context_files",
17
+ "startLine": 273
18
+ },
19
+ {
20
+ "path": "src-tauri/src/agent/runtime.rs",
21
+ "sha256": "30f2fff2a8cf92ff4f6b4c5f2f463d9445384d3a0ff7432ce765cbd28675522b",
22
+ "symbol": "run_agent_loop",
23
+ "startLine": 1294
24
+ }
25
+ ]
26
+ },
27
+ "memory": {
28
+ "actorId": "actor:release-quality-reviewer",
29
+ "sessionId": "session:llm-wiki-release-quality",
30
+ "prompt": "请记住:维护显式上下文加载时必须同时保留最多 8 个文件、路径 containment、XML 转义和最终模型窗口预算。",
31
+ "explicitContent": "维护显式上下文加载时必须同时保留最多 8 个文件、路径 containment、XML 转义和最终模型窗口预算。",
32
+ "temporal": {
33
+ "olderContent": "维护显式上下文加载时最多保留 8 个文件并执行路径 containment。",
34
+ "currentContent": "维护显式上下文加载时必须同时保留最多 8 个文件、路径 containment、XML 转义和最终模型窗口预算。",
35
+ "query": "当前显式上下文加载必须保留哪些文件数量、路径安全和模型窗口预算约束?"
36
+ },
37
+ "sourceEvidence": [
38
+ {
39
+ "path": "src-tauri/src/agent/context.rs",
40
+ "sha256": "c6f828939b572883b2c9294f52eb1527e9dca3aa417ed765a7c9fd0cb328a514"
41
+ },
42
+ {
43
+ "path": "src-tauri/src/agent/runtime.rs",
44
+ "sha256": "30f2fff2a8cf92ff4f6b4c5f2f463d9445384d3a0ff7432ce765cbd28675522b"
45
+ }
46
+ ]
47
+ },
48
+ "knowledge": {
49
+ "sourceId": "llm-wiki-mcp-docs",
50
+ "version": "0.4.25",
51
+ "pluginName": "release-quality",
52
+ "goldText": "does **not** scan project folders directly and does **not** copy the app's search or graph logic",
53
+ "provenance": {
54
+ "repository": "https://github.com/nashsu/llm_wiki.git",
55
+ "commit": "58e60b699a96a892baa72896d33577e70d373448"
56
+ },
57
+ "documents": [
58
+ {
59
+ "path": "mcp-server/README.md",
60
+ "title": "LLM Wiki MCP Server",
61
+ "sourceUri": "https://github.com/nashsu/llm_wiki/blob/58e60b699a96a892baa72896d33577e70d373448/mcp-server/README.md",
62
+ "sha256": "e22e529aacaa745c2f282d2c710c3dcc9a3f4bad8babb91b105df346b2bea2a7"
63
+ }
64
+ ]
65
+ },
66
+ "requiredSlices": [
67
+ { "channel": "context", "caseClass": "positive", "caseIds": ["llm-wiki/context/explicit-files"] },
68
+ { "channel": "memory", "caseClass": "positive", "caseIds": ["llm-wiki/memory/context-safety"] },
69
+ { "channel": "memory", "caseClass": "lifecycle", "caseIds": ["llm-wiki/memory/explicit-save"] },
70
+ { "channel": "knowledge", "caseClass": "positive", "caseIds": ["llm-wiki/knowledge/mcp-boundary"] }
71
+ ]
72
+ }