@devflow-tools/benchmark 0.17.8 → 0.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/LICENSE +21 -0
  2. package/dist/accuracy-command-options.d.ts +3 -0
  3. package/dist/accuracy-command-options.d.ts.map +1 -0
  4. package/dist/accuracy-command-options.js +17 -0
  5. package/dist/accuracy-command-options.js.map +1 -0
  6. package/dist/context-symbol-quality-command.d.ts +59 -0
  7. package/dist/context-symbol-quality-command.d.ts.map +1 -0
  8. package/dist/context-symbol-quality-command.js +482 -0
  9. package/dist/context-symbol-quality-command.js.map +1 -0
  10. package/dist/evaluation-bundle-builder.d.ts +48 -0
  11. package/dist/evaluation-bundle-builder.d.ts.map +1 -0
  12. package/dist/evaluation-bundle-builder.js +496 -0
  13. package/dist/evaluation-bundle-builder.js.map +1 -0
  14. package/dist/evaluation-bundle-command.d.ts +2 -0
  15. package/dist/evaluation-bundle-command.d.ts.map +1 -0
  16. package/dist/evaluation-bundle-command.js +15 -0
  17. package/dist/evaluation-bundle-command.js.map +1 -0
  18. package/dist/evaluation-bundle.d.ts +55 -0
  19. package/dist/evaluation-bundle.d.ts.map +1 -0
  20. package/dist/evaluation-bundle.js +272 -0
  21. package/dist/evaluation-bundle.js.map +1 -0
  22. package/dist/evaluation-run-state.d.ts +23 -0
  23. package/dist/evaluation-run-state.d.ts.map +1 -0
  24. package/dist/evaluation-run-state.js +115 -0
  25. package/dist/evaluation-run-state.js.map +1 -0
  26. package/dist/evidence-quality-metrics.d.ts +15 -0
  27. package/dist/evidence-quality-metrics.d.ts.map +1 -1
  28. package/dist/evidence-quality-metrics.js +51 -1
  29. package/dist/evidence-quality-metrics.js.map +1 -1
  30. package/dist/graders/deterministic.d.ts.map +1 -1
  31. package/dist/graders/deterministic.js +5 -3
  32. package/dist/graders/deterministic.js.map +1 -1
  33. package/dist/index.d.ts +15 -2
  34. package/dist/index.d.ts.map +1 -1
  35. package/dist/index.js +12 -1
  36. package/dist/index.js.map +1 -1
  37. package/dist/release-quality-bundle-command.d.ts +3 -0
  38. package/dist/release-quality-bundle-command.d.ts.map +1 -0
  39. package/dist/release-quality-bundle-command.js +72 -0
  40. package/dist/release-quality-bundle-command.js.map +1 -0
  41. package/dist/release-quality-bundle-evaluator.d.ts +61 -0
  42. package/dist/release-quality-bundle-evaluator.d.ts.map +1 -0
  43. package/dist/release-quality-bundle-evaluator.js +321 -0
  44. package/dist/release-quality-bundle-evaluator.js.map +1 -0
  45. package/dist/release-quality-bundle-materializer.d.ts +79 -0
  46. package/dist/release-quality-bundle-materializer.d.ts.map +1 -0
  47. package/dist/release-quality-bundle-materializer.js +395 -0
  48. package/dist/release-quality-bundle-materializer.js.map +1 -0
  49. package/dist/release-quality-comparison.d.ts +34 -0
  50. package/dist/release-quality-comparison.d.ts.map +1 -0
  51. package/dist/release-quality-comparison.js +60 -0
  52. package/dist/release-quality-comparison.js.map +1 -0
  53. package/dist/release-quality-evaluation-command.d.ts +42 -0
  54. package/dist/release-quality-evaluation-command.d.ts.map +1 -0
  55. package/dist/release-quality-evaluation-command.js +91 -0
  56. package/dist/release-quality-evaluation-command.js.map +1 -0
  57. package/dist/release-quality-host-proof.d.ts +30 -0
  58. package/dist/release-quality-host-proof.d.ts.map +1 -0
  59. package/dist/release-quality-host-proof.js +109 -0
  60. package/dist/release-quality-host-proof.js.map +1 -0
  61. package/dist/release-quality-output-cleanup.d.ts +2 -0
  62. package/dist/release-quality-output-cleanup.d.ts.map +1 -0
  63. package/dist/release-quality-output-cleanup.js +18 -0
  64. package/dist/release-quality-output-cleanup.js.map +1 -0
  65. package/dist/release-quality-policy.d.ts +87 -0
  66. package/dist/release-quality-policy.d.ts.map +1 -0
  67. package/dist/release-quality-policy.js +179 -0
  68. package/dist/release-quality-policy.js.map +1 -0
  69. package/dist/release-quality-query-sampling.d.ts +36 -0
  70. package/dist/release-quality-query-sampling.d.ts.map +1 -0
  71. package/dist/release-quality-query-sampling.js +87 -0
  72. package/dist/release-quality-query-sampling.js.map +1 -0
  73. package/dist/release-quality-source-snapshot.d.ts +18 -0
  74. package/dist/release-quality-source-snapshot.d.ts.map +1 -0
  75. package/dist/release-quality-source-snapshot.js +46 -0
  76. package/dist/release-quality-source-snapshot.js.map +1 -0
  77. package/dist/retrieval-quality-corpus.d.ts +0 -1
  78. package/dist/retrieval-quality-corpus.d.ts.map +1 -1
  79. package/dist/retrieval-quality-corpus.js +1 -1
  80. package/dist/retrieval-quality-corpus.js.map +1 -1
  81. package/dist/retrieval-quality.d.ts.map +1 -1
  82. package/dist/retrieval-quality.js +29 -26
  83. package/dist/retrieval-quality.js.map +1 -1
  84. package/dist/run-accuracy-tests.js +82 -2
  85. package/dist/run-accuracy-tests.js.map +1 -1
  86. package/dist/suite-runner.js +1 -1
  87. package/dist/suite-runner.js.map +1 -1
  88. package/dist/unified-runtime-corpus.d.ts +64 -0
  89. package/dist/unified-runtime-corpus.d.ts.map +1 -0
  90. package/dist/unified-runtime-corpus.js +161 -0
  91. package/dist/unified-runtime-corpus.js.map +1 -0
  92. package/dist/unified-runtime-evaluation.d.ts +133 -0
  93. package/dist/unified-runtime-evaluation.d.ts.map +1 -0
  94. package/dist/unified-runtime-evaluation.js +388 -0
  95. package/dist/unified-runtime-evaluation.js.map +1 -0
  96. package/dist/unified-runtime-observation.d.ts +101 -0
  97. package/dist/unified-runtime-observation.d.ts.map +1 -0
  98. package/dist/unified-runtime-observation.js +2 -0
  99. package/dist/unified-runtime-observation.js.map +1 -0
  100. package/dist/unified-runtime-runner.d.ts +73 -0
  101. package/dist/unified-runtime-runner.d.ts.map +1 -0
  102. package/dist/unified-runtime-runner.js +620 -0
  103. package/dist/unified-runtime-runner.js.map +1 -0
  104. package/fixtures/evidence-runtime-corpus.json +0 -100
  105. package/fixtures/release-quality/bundles.json +1 -0
  106. package/fixtures/release-quality/calibration.json +602 -0
  107. package/fixtures/release-quality/ecc/corpus.json +257 -0
  108. package/fixtures/release-quality/ecc/input.json +62 -0
  109. package/fixtures/release-quality/llm-wiki/corpus.json +263 -0
  110. package/fixtures/release-quality/llm-wiki/input.json +72 -0
  111. package/fixtures/release-quality/mem0/corpus.json +288 -0
  112. package/fixtures/release-quality/mem0/input.json +68 -0
  113. package/fixtures/release-quality/projects.json +29 -0
  114. package/fixtures/release-quality/release-quality-policy.json +232 -0
  115. package/fixtures/release-quality/rsbuild-react/corpus.json +263 -0
  116. package/fixtures/release-quality/rsbuild-react/input.json +62 -0
  117. package/fixtures/retrieval-quality-corpus.json +7 -7
  118. package/fixtures/semantic-regex-inventory.json +3 -10
  119. package/fixtures/unified-runtime/baselines/2026-08-08.json +37 -0
  120. package/fixtures/unified-runtime/context.json +206 -0
  121. package/fixtures/unified-runtime/knowledge.json +224 -0
  122. package/fixtures/unified-runtime/memory.json +224 -0
  123. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/context.json +32 -0
  124. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/knowledge.json +32 -0
  125. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/memory.json +32 -0
  126. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/semantic.json +409 -0
  127. package/fixtures/unified-runtime/runs/rsbuild-react-2026-08-05.json +54 -0
  128. package/fixtures/unified-runtime/semantic.json +409 -0
  129. package/package.json +24 -6
package/fixtures/release-quality/bundles.json ADDED
@@ -0,0 +1 @@
1
+ {"bundles":[{"builderIdentity":"devflow:588bc6b61fc4c4dbf8cb97a69cbb005f5e0c2e0e","bundleId":"bundle:a0d7e887f8e1662a59e8c0e68df466e73ffbf164ca76abbbb714b1d09b96e88f","channelSnapshotHashes":{"context":"e548087a5e2ca8c820339a22234280cd6595dc1e282b6284fe26ccbfa6f3e50d","knowledge":"f9d53e78a4d459d74a4cda022b83057e41672787b56fe9b602b244ff0b6a2dc6","memory":"148c341b581ee072b063d08f86cf814b260e6295f7d1372f41c9e0cb1358eeb8"},"datasetDirectory":"rsbuild-react","projectId":"github:shilongfeicool/rsbuild-react","projectSnapshotId":"project:9460d5413204b3fc13beb05047010c5b59cb721a3c18b235de301f522810b839"},{"builderIdentity":"devflow:588bc6b61fc4c4dbf8cb97a69cbb005f5e0c2e0e","bundleId":"bundle:7fbeb709b2febaee7dc63e1270dd2244956ad0c439d22bbafd8f12f247ebcd47","channelSnapshotHashes":{"context":"ae5e153719ac2632a1f520a3af75e3a346b32924febf033533af52d5d604f527","knowledge":"f7f769244a616949ba965a488961d01f28fd6019463fe45addc303d7b7c25991","memory":"50b206c7129d3ba1b324e57bf980844b151a783966222875b29592ff709ee218"},"datasetDirectory":"llm-wiki","projectId":"github:nashsu/llm_wiki","projectSnapshotId":"project:92aeea3502386fb1cc5ac12824c82dac346a39de8c3fbfaa79c25def981cac3a"},{"builderIdentity":"devflow:588bc6b61fc4c4dbf8cb97a69cbb005f5e0c2e0e","bundleId":"bundle:e9ec35871a8be61a08ac838a88fc49d7d2c8da12aaea2c1404364714168d83c2","channelSnapshotHashes":{"context":"9e130c2a41de7573eac7b0ff51168f551e2c1121e361216a7c74f2a50a388793","knowledge":"93870ccd0e1f916647f5b9d53155f65fde12f311a3677df2d22374b3c8974258","memory":"20a9f6c78dee20b4026ef4a8c5394eb369712a8f7c71a2528aa7fc12e429e25f"},"datasetDirectory":"ecc","projectId":"github:affaan-m/ecc","projectSnapshotId":"project:4057e05c924f3b1470ead6d24a3381feeef4debd9a9cafc7cc4bfc7dc5e80c6a"},{"builderIdentity":"devflow:588bc6b61fc4c4dbf8cb97a69cbb005f5e0c2e0e","bundleId":"bundle:827ac58ac3ee1f6cd153171ee1f1a888b066235a02dc071da0b1fc66a547c681","channelSnapshotHashes":{"context":"06a0e1016e16706f01cfe6671753cfdea4105dfc6ca10006abc4c1b3535bf16c","knowledge":"398855a0c318b254fa02d5d0ac6d0db6337e1498ca558d86bdb4a504e69e60b8","memory":"5e3ff77ec4eaa8e035de2521132aea92b0e65fad64370eadc66c7ac349502004"},"datasetDirectory":"mem0","projectId":"github:mem0ai/mem0","projectSnapshotId":"project:d6ea4efb87381667965efe270d4c76913f23d2bda42708d74bcdccf279396482"}],"schemaVersion":"devflow.release-quality-bundles.v1"}
package/fixtures/release-quality/calibration.json ADDED
@@ -0,0 +1,602 @@
1
+ {
2
+ "schemaVersion": "devflow.release-quality-calibration.v1",
3
+ "sourceReports": [
4
+ "/private/tmp/devflow-release-quality-k.json",
5
+ "/private/tmp/devflow-release-quality-l.json"
6
+ ],
7
+ "bundleIds": [
8
+ "bundle:7fbeb709b2febaee7dc63e1270dd2244956ad0c439d22bbafd8f12f247ebcd47",
9
+ "bundle:827ac58ac3ee1f6cd153171ee1f1a888b066235a02dc071da0b1fc66a547c681",
10
+ "bundle:a0d7e887f8e1662a59e8c0e68df466e73ffbf164ca76abbbb714b1d09b96e88f",
11
+ "bundle:e9ec35871a8be61a08ac838a88fc49d7d2c8da12aaea2c1404364714168d83c2"
12
+ ],
13
+ "builderIdentity": "devflow:588bc6b61fc4c4dbf8cb97a69cbb005f5e0c2e0e",
14
+ "comparisonProfile": {
15
+ "providerModel": {
16
+ "provider": "deepseek",
17
+ "model": "deepseek-v4-pro"
18
+ },
19
+ "readerProfileHash": "66c128c341ad8d9159932389bae16eab6efeaf497065028c456bf3ff1a14a267",
20
+ "embeddingGeneration": 3,
21
+ "budgetHash": "e44e88e280fd2da77b687a89d791dda18ac3842ae0076cabe37e6ecf41e7b75c",
22
+ "candidateTopK": 20,
23
+ "selectedTopK": 5
24
+ },
25
+ "budget": {
26
+ "hookDeadlineMs": 3000,
27
+ "contextTokens": 8000,
28
+ "responseReserveTokens": 2000,
29
+ "perChannelMs": {
30
+ "code": 1500,
31
+ "memory": 800,
32
+ "knowledge": 1500
33
+ }
34
+ },
35
+ "projects": [
36
+ {
37
+ "projectId": "github:shilongfeicool/rsbuild-react",
38
+ "bundleId": "bundle:a0d7e887f8e1662a59e8c0e68df466e73ffbf164ca76abbbb714b1d09b96e88f",
39
+ "caseCounts": {
40
+ "context/positive": 1,
41
+ "context/lifecycle": 0,
42
+ "memory/positive": 1,
43
+ "memory/lifecycle": 1,
44
+ "knowledge/positive": 1,
45
+ "knowledge/lifecycle": 0
46
+ },
47
+ "channels": {
48
+ "context": {
49
+ "candidateRecall": {
50
+ "numerator": 1,
51
+ "denominator": 1,
52
+ "value": 1
53
+ },
54
+ "recallAt5": {
55
+ "numerator": 1,
56
+ "denominator": 1,
57
+ "value": 1
58
+ },
59
+ "precisionAt5": {
60
+ "numerator": 1,
61
+ "denominator": 5,
62
+ "value": 0.2
63
+ },
64
+ "forbiddenContamination": {
65
+ "numerator": 0,
66
+ "denominator": 5,
67
+ "value": 0
68
+ },
69
+ "provenanceCompleteness": {
70
+ "numerator": 195,
71
+ "denominator": 195,
72
+ "value": 1
73
+ }
74
+ },
75
+ "memory": {
76
+ "candidateRecall": {
77
+ "numerator": 1,
78
+ "denominator": 1,
79
+ "value": 1
80
+ },
81
+ "recallAt5": {
82
+ "numerator": 1,
83
+ "denominator": 1,
84
+ "value": 1
85
+ },
86
+ "precisionAt5": {
87
+ "numerator": 1,
88
+ "denominator": 5,
89
+ "value": 0.2
90
+ },
91
+ "forbiddenContamination": {
92
+ "numerator": 0,
93
+ "denominator": 5,
94
+ "value": 0
95
+ },
96
+ "provenanceCompleteness": {
97
+ "numerator": 2,
98
+ "denominator": 2,
99
+ "value": 1
100
+ }
101
+ },
102
+ "knowledge": {
103
+ "candidateRecall": {
104
+ "numerator": 1,
105
+ "denominator": 1,
106
+ "value": 1
107
+ },
108
+ "recallAt5": {
109
+ "numerator": 1,
110
+ "denominator": 1,
111
+ "value": 1
112
+ },
113
+ "precisionAt5": {
114
+ "numerator": 1,
115
+ "denominator": 5,
116
+ "value": 0.2
117
+ },
118
+ "forbiddenContamination": {
119
+ "numerator": 0,
120
+ "denominator": 5,
121
+ "value": 0
122
+ },
123
+ "provenanceCompleteness": {
124
+ "numerator": 3,
125
+ "denominator": 3,
126
+ "value": 1
127
+ }
128
+ }
129
+ },
130
+ "memoryDurability": {
131
+ "numerator": 1,
132
+ "denominator": 1,
133
+ "value": 1
134
+ },
135
+ "memoryTemporalCorrectness": {
136
+ "numerator": 1,
137
+ "denominator": 1,
138
+ "value": 1
139
+ },
140
+ "knowledgeVersionCorrectness": {
141
+ "numerator": 3,
142
+ "denominator": 3,
143
+ "value": 1
144
+ }
145
+ },
146
+ {
147
+ "projectId": "github:nashsu/llm_wiki",
148
+ "bundleId": "bundle:7fbeb709b2febaee7dc63e1270dd2244956ad0c439d22bbafd8f12f247ebcd47",
149
+ "caseCounts": {
150
+ "context/positive": 1,
151
+ "context/lifecycle": 0,
152
+ "memory/positive": 1,
153
+ "memory/lifecycle": 1,
154
+ "knowledge/positive": 1,
155
+ "knowledge/lifecycle": 0
156
+ },
157
+ "channels": {
158
+ "context": {
159
+ "candidateRecall": {
160
+ "numerator": 2,
161
+ "denominator": 2,
162
+ "value": 1
163
+ },
164
+ "recallAt5": {
165
+ "numerator": 2,
166
+ "denominator": 2,
167
+ "value": 1
168
+ },
169
+ "precisionAt5": {
170
+ "numerator": 2,
171
+ "denominator": 5,
172
+ "value": 0.4
173
+ },
174
+ "forbiddenContamination": {
175
+ "numerator": 0,
176
+ "denominator": 5,
177
+ "value": 0
178
+ },
179
+ "provenanceCompleteness": {
180
+ "numerator": 317,
181
+ "denominator": 317,
182
+ "value": 1
183
+ }
184
+ },
185
+ "memory": {
186
+ "candidateRecall": {
187
+ "numerator": 1,
188
+ "denominator": 1,
189
+ "value": 1
190
+ },
191
+ "recallAt5": {
192
+ "numerator": 1,
193
+ "denominator": 1,
194
+ "value": 1
195
+ },
196
+ "precisionAt5": {
197
+ "numerator": 1,
198
+ "denominator": 5,
199
+ "value": 0.2
200
+ },
201
+ "forbiddenContamination": {
202
+ "numerator": 0,
203
+ "denominator": 5,
204
+ "value": 0
205
+ },
206
+ "provenanceCompleteness": {
207
+ "numerator": 3,
208
+ "denominator": 3,
209
+ "value": 1
210
+ }
211
+ },
212
+ "knowledge": {
213
+ "candidateRecall": {
214
+ "numerator": 1,
215
+ "denominator": 1,
216
+ "value": 1
217
+ },
218
+ "recallAt5": {
219
+ "numerator": 1,
220
+ "denominator": 1,
221
+ "value": 1
222
+ },
223
+ "precisionAt5": {
224
+ "numerator": 1,
225
+ "denominator": 5,
226
+ "value": 0.2
227
+ },
228
+ "forbiddenContamination": {
229
+ "numerator": 0,
230
+ "denominator": 5,
231
+ "value": 0
232
+ },
233
+ "provenanceCompleteness": {
234
+ "numerator": 21,
235
+ "denominator": 21,
236
+ "value": 1
237
+ }
238
+ }
239
+ },
240
+ "memoryDurability": {
241
+ "numerator": 1,
242
+ "denominator": 1,
243
+ "value": 1
244
+ },
245
+ "memoryTemporalCorrectness": {
246
+ "numerator": 1,
247
+ "denominator": 1,
248
+ "value": 1
249
+ },
250
+ "knowledgeVersionCorrectness": {
251
+ "numerator": 21,
252
+ "denominator": 21,
253
+ "value": 1
254
+ }
255
+ },
256
+ {
257
+ "projectId": "github:affaan-m/ecc",
258
+ "bundleId": "bundle:e9ec35871a8be61a08ac838a88fc49d7d2c8da12aaea2c1404364714168d83c2",
259
+ "caseCounts": {
260
+ "context/positive": 1,
261
+ "context/lifecycle": 0,
262
+ "memory/positive": 1,
263
+ "memory/lifecycle": 1,
264
+ "knowledge/positive": 1,
265
+ "knowledge/lifecycle": 0
266
+ },
267
+ "channels": {
268
+ "context": {
269
+ "candidateRecall": {
270
+ "numerator": 1,
271
+ "denominator": 1,
272
+ "value": 1
273
+ },
274
+ "recallAt5": {
275
+ "numerator": 1,
276
+ "denominator": 1,
277
+ "value": 1
278
+ },
279
+ "precisionAt5": {
280
+ "numerator": 1,
281
+ "denominator": 5,
282
+ "value": 0.2
283
+ },
284
+ "forbiddenContamination": {
285
+ "numerator": 0,
286
+ "denominator": 5,
287
+ "value": 0
288
+ },
289
+ "provenanceCompleteness": {
290
+ "numerator": 438,
291
+ "denominator": 438,
292
+ "value": 1
293
+ }
294
+ },
295
+ "memory": {
296
+ "candidateRecall": {
297
+ "numerator": 1,
298
+ "denominator": 1,
299
+ "value": 1
300
+ },
301
+ "recallAt5": {
302
+ "numerator": 1,
303
+ "denominator": 1,
304
+ "value": 1
305
+ },
306
+ "precisionAt5": {
307
+ "numerator": 1,
308
+ "denominator": 5,
309
+ "value": 0.2
310
+ },
311
+ "forbiddenContamination": {
312
+ "numerator": 0,
313
+ "denominator": 5,
314
+ "value": 0
315
+ },
316
+ "provenanceCompleteness": {
317
+ "numerator": 2,
318
+ "denominator": 2,
319
+ "value": 1
320
+ }
321
+ },
322
+ "knowledge": {
323
+ "candidateRecall": {
324
+ "numerator": 1,
325
+ "denominator": 1,
326
+ "value": 1
327
+ },
328
+ "recallAt5": {
329
+ "numerator": 1,
330
+ "denominator": 1,
331
+ "value": 1
332
+ },
333
+ "precisionAt5": {
334
+ "numerator": 1,
335
+ "denominator": 5,
336
+ "value": 0.2
337
+ },
338
+ "forbiddenContamination": {
339
+ "numerator": 0,
340
+ "denominator": 5,
341
+ "value": 0
342
+ },
343
+ "provenanceCompleteness": {
344
+ "numerator": 14,
345
+ "denominator": 14,
346
+ "value": 1
347
+ }
348
+ }
349
+ },
350
+ "memoryDurability": {
351
+ "numerator": 1,
352
+ "denominator": 1,
353
+ "value": 1
354
+ },
355
+ "memoryTemporalCorrectness": {
356
+ "numerator": 1,
357
+ "denominator": 1,
358
+ "value": 1
359
+ },
360
+ "knowledgeVersionCorrectness": {
361
+ "numerator": 14,
362
+ "denominator": 14,
363
+ "value": 1
364
+ }
365
+ },
366
+ {
367
+ "projectId": "github:mem0ai/mem0",
368
+ "bundleId": "bundle:827ac58ac3ee1f6cd153171ee1f1a888b066235a02dc071da0b1fc66a547c681",
369
+ "caseCounts": {
370
+ "context/positive": 1,
371
+ "context/lifecycle": 0,
372
+ "memory/positive": 1,
373
+ "memory/lifecycle": 1,
374
+ "knowledge/positive": 1,
375
+ "knowledge/lifecycle": 0
376
+ },
377
+ "channels": {
378
+ "context": {
379
+ "candidateRecall": {
380
+ "numerator": 1,
381
+ "denominator": 1,
382
+ "value": 1
383
+ },
384
+ "recallAt5": {
385
+ "numerator": 1,
386
+ "denominator": 1,
387
+ "value": 1
388
+ },
389
+ "precisionAt5": {
390
+ "numerator": 1,
391
+ "denominator": 5,
392
+ "value": 0.2
393
+ },
394
+ "forbiddenContamination": {
395
+ "numerator": 0,
396
+ "denominator": 5,
397
+ "value": 0
398
+ },
399
+ "provenanceCompleteness": {
400
+ "numerator": 335,
401
+ "denominator": 335,
402
+ "value": 1
403
+ }
404
+ },
405
+ "memory": {
406
+ "candidateRecall": {
407
+ "numerator": 1,
408
+ "denominator": 1,
409
+ "value": 1
410
+ },
411
+ "recallAt5": {
412
+ "numerator": 1,
413
+ "denominator": 1,
414
+ "value": 1
415
+ },
416
+ "precisionAt5": {
417
+ "numerator": 1,
418
+ "denominator": 5,
419
+ "value": 0.2
420
+ },
421
+ "forbiddenContamination": {
422
+ "numerator": 0,
423
+ "denominator": 5,
424
+ "value": 0
425
+ },
426
+ "provenanceCompleteness": {
427
+ "numerator": 2,
428
+ "denominator": 2,
429
+ "value": 1
430
+ }
431
+ },
432
+ "knowledge": {
433
+ "candidateRecall": {
434
+ "numerator": 1,
435
+ "denominator": 1,
436
+ "value": 1
437
+ },
438
+ "recallAt5": {
439
+ "numerator": 1,
440
+ "denominator": 1,
441
+ "value": 1
442
+ },
443
+ "precisionAt5": {
444
+ "numerator": 1,
445
+ "denominator": 5,
446
+ "value": 0.2
447
+ },
448
+ "forbiddenContamination": {
449
+ "numerator": 0,
450
+ "denominator": 5,
451
+ "value": 0
452
+ },
453
+ "provenanceCompleteness": {
454
+ "numerator": 21,
455
+ "denominator": 21,
456
+ "value": 1
457
+ }
458
+ }
459
+ },
460
+ "memoryDurability": {
461
+ "numerator": 1,
462
+ "denominator": 1,
463
+ "value": 1
464
+ },
465
+ "memoryTemporalCorrectness": {
466
+ "numerator": 1,
467
+ "denominator": 1,
468
+ "value": 1
469
+ },
470
+ "knowledgeVersionCorrectness": {
471
+ "numerator": 21,
472
+ "denominator": 21,
473
+ "value": 1
474
+ }
475
+ }
476
+ ],
477
+ "macro": {
478
+ "context": {
479
+ "candidateRecall": 1,
480
+ "forbiddenContamination": 0,
481
+ "precisionAt5": 0.25,
482
+ "provenanceCompleteness": 1,
483
+ "recallAt5": 1
484
+ },
485
+ "knowledge": {
486
+ "candidateRecall": 1,
487
+ "forbiddenContamination": 0,
488
+ "precisionAt5": 0.2,
489
+ "provenanceCompleteness": 1,
490
+ "recallAt5": 1
491
+ },
492
+ "memory": {
493
+ "candidateRecall": 1,
494
+ "forbiddenContamination": 0,
495
+ "precisionAt5": 0.2,
496
+ "provenanceCompleteness": 1,
497
+ "recallAt5": 1
498
+ }
499
+ },
500
+ "micro": {
501
+ "context": {
502
+ "candidateRecall": {
503
+ "numerator": 5,
504
+ "denominator": 5,
505
+ "value": 1
506
+ },
507
+ "forbiddenContamination": {
508
+ "numerator": 0,
509
+ "denominator": 20,
510
+ "value": 0
511
+ },
512
+ "precisionAt5": {
513
+ "numerator": 5,
514
+ "denominator": 20,
515
+ "value": 0.25
516
+ },
517
+ "provenanceCompleteness": {
518
+ "numerator": 1285,
519
+ "denominator": 1285,
520
+ "value": 1
521
+ },
522
+ "recallAt5": {
523
+ "numerator": 5,
524
+ "denominator": 5,
525
+ "value": 1
526
+ }
527
+ },
528
+ "knowledge": {
529
+ "candidateRecall": {
530
+ "numerator": 4,
531
+ "denominator": 4,
532
+ "value": 1
533
+ },
534
+ "forbiddenContamination": {
535
+ "numerator": 0,
536
+ "denominator": 20,
537
+ "value": 0
538
+ },
539
+ "precisionAt5": {
540
+ "numerator": 4,
541
+ "denominator": 20,
542
+ "value": 0.2
543
+ },
544
+ "provenanceCompleteness": {
545
+ "numerator": 59,
546
+ "denominator": 59,
547
+ "value": 1
548
+ },
549
+ "recallAt5": {
550
+ "numerator": 4,
551
+ "denominator": 4,
552
+ "value": 1
553
+ }
554
+ },
555
+ "memory": {
556
+ "candidateRecall": {
557
+ "numerator": 4,
558
+ "denominator": 4,
559
+ "value": 1
560
+ },
561
+ "forbiddenContamination": {
562
+ "numerator": 0,
563
+ "denominator": 20,
564
+ "value": 0
565
+ },
566
+ "precisionAt5": {
567
+ "numerator": 4,
568
+ "denominator": 20,
569
+ "value": 0.2
570
+ },
571
+ "provenanceCompleteness": {
572
+ "numerator": 9,
573
+ "denominator": 9,
574
+ "value": 1
575
+ },
576
+ "recallAt5": {
577
+ "numerator": 4,
578
+ "denominator": 4,
579
+ "value": 1
580
+ }
581
+ }
582
+ },
583
+ "memoryDurability": {
584
+ "numerator": 4,
585
+ "denominator": 4,
586
+ "value": 1
587
+ },
588
+ "memoryTemporalCorrectness": {
589
+ "numerator": 4,
590
+ "denominator": 4,
591
+ "value": 1
592
+ },
593
+ "knowledgeVersionCorrectness": {
594
+ "numerator": 59,
595
+ "denominator": 59,
596
+ "value": 1
597
+ },
598
+ "hostProof": {
599
+ "consumptionRate": null,
600
+ "verificationRate": null
601
+ }
602
+ }