@devflow-tools/benchmark 0.17.8 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/LICENSE +21 -0
  2. package/dist/accuracy-command-options.d.ts +3 -0
  3. package/dist/accuracy-command-options.d.ts.map +1 -0
  4. package/dist/accuracy-command-options.js +17 -0
  5. package/dist/accuracy-command-options.js.map +1 -0
  6. package/dist/context-symbol-quality-command.d.ts +59 -0
  7. package/dist/context-symbol-quality-command.d.ts.map +1 -0
  8. package/dist/context-symbol-quality-command.js +482 -0
  9. package/dist/context-symbol-quality-command.js.map +1 -0
  10. package/dist/evaluation-bundle-builder.d.ts +48 -0
  11. package/dist/evaluation-bundle-builder.d.ts.map +1 -0
  12. package/dist/evaluation-bundle-builder.js +496 -0
  13. package/dist/evaluation-bundle-builder.js.map +1 -0
  14. package/dist/evaluation-bundle-command.d.ts +2 -0
  15. package/dist/evaluation-bundle-command.d.ts.map +1 -0
  16. package/dist/evaluation-bundle-command.js +15 -0
  17. package/dist/evaluation-bundle-command.js.map +1 -0
  18. package/dist/evaluation-bundle.d.ts +55 -0
  19. package/dist/evaluation-bundle.d.ts.map +1 -0
  20. package/dist/evaluation-bundle.js +272 -0
  21. package/dist/evaluation-bundle.js.map +1 -0
  22. package/dist/evaluation-run-state.d.ts +23 -0
  23. package/dist/evaluation-run-state.d.ts.map +1 -0
  24. package/dist/evaluation-run-state.js +115 -0
  25. package/dist/evaluation-run-state.js.map +1 -0
  26. package/dist/evidence-quality-metrics.d.ts +15 -0
  27. package/dist/evidence-quality-metrics.d.ts.map +1 -1
  28. package/dist/evidence-quality-metrics.js +51 -1
  29. package/dist/evidence-quality-metrics.js.map +1 -1
  30. package/dist/graders/deterministic.d.ts.map +1 -1
  31. package/dist/graders/deterministic.js +5 -3
  32. package/dist/graders/deterministic.js.map +1 -1
  33. package/dist/index.d.ts +15 -2
  34. package/dist/index.d.ts.map +1 -1
  35. package/dist/index.js +12 -1
  36. package/dist/index.js.map +1 -1
  37. package/dist/release-quality-bundle-command.d.ts +3 -0
  38. package/dist/release-quality-bundle-command.d.ts.map +1 -0
  39. package/dist/release-quality-bundle-command.js +72 -0
  40. package/dist/release-quality-bundle-command.js.map +1 -0
  41. package/dist/release-quality-bundle-evaluator.d.ts +61 -0
  42. package/dist/release-quality-bundle-evaluator.d.ts.map +1 -0
  43. package/dist/release-quality-bundle-evaluator.js +321 -0
  44. package/dist/release-quality-bundle-evaluator.js.map +1 -0
  45. package/dist/release-quality-bundle-materializer.d.ts +79 -0
  46. package/dist/release-quality-bundle-materializer.d.ts.map +1 -0
  47. package/dist/release-quality-bundle-materializer.js +395 -0
  48. package/dist/release-quality-bundle-materializer.js.map +1 -0
  49. package/dist/release-quality-comparison.d.ts +34 -0
  50. package/dist/release-quality-comparison.d.ts.map +1 -0
  51. package/dist/release-quality-comparison.js +60 -0
  52. package/dist/release-quality-comparison.js.map +1 -0
  53. package/dist/release-quality-evaluation-command.d.ts +42 -0
  54. package/dist/release-quality-evaluation-command.d.ts.map +1 -0
  55. package/dist/release-quality-evaluation-command.js +91 -0
  56. package/dist/release-quality-evaluation-command.js.map +1 -0
  57. package/dist/release-quality-host-proof.d.ts +30 -0
  58. package/dist/release-quality-host-proof.d.ts.map +1 -0
  59. package/dist/release-quality-host-proof.js +109 -0
  60. package/dist/release-quality-host-proof.js.map +1 -0
  61. package/dist/release-quality-output-cleanup.d.ts +2 -0
  62. package/dist/release-quality-output-cleanup.d.ts.map +1 -0
  63. package/dist/release-quality-output-cleanup.js +18 -0
  64. package/dist/release-quality-output-cleanup.js.map +1 -0
  65. package/dist/release-quality-policy.d.ts +87 -0
  66. package/dist/release-quality-policy.d.ts.map +1 -0
  67. package/dist/release-quality-policy.js +179 -0
  68. package/dist/release-quality-policy.js.map +1 -0
  69. package/dist/release-quality-query-sampling.d.ts +36 -0
  70. package/dist/release-quality-query-sampling.d.ts.map +1 -0
  71. package/dist/release-quality-query-sampling.js +87 -0
  72. package/dist/release-quality-query-sampling.js.map +1 -0
  73. package/dist/release-quality-source-snapshot.d.ts +18 -0
  74. package/dist/release-quality-source-snapshot.d.ts.map +1 -0
  75. package/dist/release-quality-source-snapshot.js +46 -0
  76. package/dist/release-quality-source-snapshot.js.map +1 -0
  77. package/dist/retrieval-quality-corpus.d.ts +0 -1
  78. package/dist/retrieval-quality-corpus.d.ts.map +1 -1
  79. package/dist/retrieval-quality-corpus.js +1 -1
  80. package/dist/retrieval-quality-corpus.js.map +1 -1
  81. package/dist/retrieval-quality.d.ts.map +1 -1
  82. package/dist/retrieval-quality.js +29 -26
  83. package/dist/retrieval-quality.js.map +1 -1
  84. package/dist/run-accuracy-tests.js +82 -2
  85. package/dist/run-accuracy-tests.js.map +1 -1
  86. package/dist/suite-runner.js +1 -1
  87. package/dist/suite-runner.js.map +1 -1
  88. package/dist/unified-runtime-corpus.d.ts +64 -0
  89. package/dist/unified-runtime-corpus.d.ts.map +1 -0
  90. package/dist/unified-runtime-corpus.js +161 -0
  91. package/dist/unified-runtime-corpus.js.map +1 -0
  92. package/dist/unified-runtime-evaluation.d.ts +133 -0
  93. package/dist/unified-runtime-evaluation.d.ts.map +1 -0
  94. package/dist/unified-runtime-evaluation.js +388 -0
  95. package/dist/unified-runtime-evaluation.js.map +1 -0
  96. package/dist/unified-runtime-observation.d.ts +101 -0
  97. package/dist/unified-runtime-observation.d.ts.map +1 -0
  98. package/dist/unified-runtime-observation.js +2 -0
  99. package/dist/unified-runtime-observation.js.map +1 -0
  100. package/dist/unified-runtime-runner.d.ts +73 -0
  101. package/dist/unified-runtime-runner.d.ts.map +1 -0
  102. package/dist/unified-runtime-runner.js +620 -0
  103. package/dist/unified-runtime-runner.js.map +1 -0
  104. package/fixtures/evidence-runtime-corpus.json +0 -100
  105. package/fixtures/release-quality/bundles.json +1 -0
  106. package/fixtures/release-quality/calibration.json +602 -0
  107. package/fixtures/release-quality/ecc/corpus.json +257 -0
  108. package/fixtures/release-quality/ecc/input.json +62 -0
  109. package/fixtures/release-quality/llm-wiki/corpus.json +263 -0
  110. package/fixtures/release-quality/llm-wiki/input.json +72 -0
  111. package/fixtures/release-quality/mem0/corpus.json +288 -0
  112. package/fixtures/release-quality/mem0/input.json +68 -0
  113. package/fixtures/release-quality/projects.json +29 -0
  114. package/fixtures/release-quality/release-quality-policy.json +232 -0
  115. package/fixtures/release-quality/rsbuild-react/corpus.json +263 -0
  116. package/fixtures/release-quality/rsbuild-react/input.json +62 -0
  117. package/fixtures/retrieval-quality-corpus.json +7 -7
  118. package/fixtures/semantic-regex-inventory.json +3 -10
  119. package/fixtures/unified-runtime/baselines/2026-08-08.json +37 -0
  120. package/fixtures/unified-runtime/context.json +206 -0
  121. package/fixtures/unified-runtime/knowledge.json +224 -0
  122. package/fixtures/unified-runtime/memory.json +224 -0
  123. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/context.json +32 -0
  124. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/knowledge.json +32 -0
  125. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/memory.json +32 -0
  126. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/semantic.json +409 -0
  127. package/fixtures/unified-runtime/runs/rsbuild-react-2026-08-05.json +54 -0
  128. package/fixtures/unified-runtime/semantic.json +409 -0
  129. package/package.json +24 -6
@@ -0,0 +1,288 @@
1
+ {
2
+ "schemaVersion": "devflow.evaluation-corpus.v1",
3
+ "cases": [
4
+ {
5
+ "caseId": "mem0/context/add-pipeline",
6
+ "channel": "context",
7
+ "caseClass": "positive",
8
+ "prompt": "在 OSS Memory.add infer=True 中,输入规范化、既有记忆检索和 LLM 抽取依次在哪里执行?",
9
+ "querySampling": {
10
+ "status": "sampled",
11
+ "provider": "deepseek",
12
+ "model": "deepseek-v4-pro",
13
+ "drafts": [
14
+ {
15
+ "channel": "code",
16
+ "kind": "rewrite",
17
+ "query": "Memory.add infer=True input normalization memory retrieval LLM extraction execution order",
18
+ "sourceClaimIds": [
19
+ "semantic-claim:6bb8b4442ad8ba114f58e438f7683e6b",
20
+ "semantic-claim:9ba3172dcd1f9d9fe6bae66190521609",
21
+ "semantic-claim:dae6f22804a1209d6e156dc2dad4b10b",
22
+ "semantic-claim:f818d19326ce1448ddfab72bd13e19e5"
23
+ ],
24
+ "confidence": 0.85
25
+ },
26
+ {
27
+ "channel": "code",
28
+ "kind": "expansion",
29
+ "query": "Memory.add method implementation normalize input retrieve existing memories extract with LLM pipeline",
30
+ "sourceClaimIds": [
31
+ "semantic-claim:6bb8b4442ad8ba114f58e438f7683e6b",
32
+ "semantic-claim:9ba3172dcd1f9d9fe6bae66190521609",
33
+ "semantic-claim:dae6f22804a1209d6e156dc2dad4b10b"
34
+ ],
35
+ "confidence": 0.7
36
+ },
37
+ {
38
+ "channel": "code",
39
+ "kind": "expansion",
40
+ "query": "Memory.add caller entry point orchestration data flow normalization recall extraction",
41
+ "sourceClaimIds": [
42
+ "semantic-claim:6bb8b4442ad8ba114f58e438f7683e6b",
43
+ "semantic-claim:9ba3172dcd1f9d9fe6bae66190521609"
44
+ ],
45
+ "confidence": 0.6
46
+ },
47
+ {
48
+ "channel": "memory",
49
+ "kind": "rewrite",
50
+ "query": "Memory.add infer=True 输入规范化 记忆检索 LLM 抽取 执行顺序",
51
+ "sourceClaimIds": [
52
+ "semantic-claim:6bb8b4442ad8ba114f58e438f7683e6b",
53
+ "semantic-claim:9ba3172dcd1f9d9fe6bae66190521609",
54
+ "semantic-claim:dae6f22804a1209d6e156dc2dad4b10b"
55
+ ],
56
+ "confidence": 0.8
57
+ },
58
+ {
59
+ "channel": "knowledge",
60
+ "kind": "rewrite",
61
+ "query": "OSS Memory.add infer=True input normalization existing memory retrieval LLM extraction execution order",
62
+ "sourceClaimIds": [
63
+ "semantic-claim:c55f54e5d90828085192151436199c5a",
64
+ "semantic-claim:6bb8b4442ad8ba114f58e438f7683e6b",
65
+ "semantic-claim:9ba3172dcd1f9d9fe6bae66190521609",
66
+ "semantic-claim:dae6f22804a1209d6e156dc2dad4b10b"
67
+ ],
68
+ "confidence": 0.75
69
+ }
70
+ ]
71
+ }
72
+ },
73
+ {
74
+ "caseId": "mem0/memory/add-order",
75
+ "channel": "memory",
76
+ "caseClass": "positive",
77
+ "prompt": "维护 Memory.add infer=True 时必须保留怎样的处理顺序?",
78
+ "querySampling": {
79
+ "status": "sampled",
80
+ "provider": "deepseek",
81
+ "model": "deepseek-v4-pro",
82
+ "drafts": [
83
+ {
84
+ "channel": "code",
85
+ "kind": "rewrite",
86
+ "query": "Memory.add infer=True processing order",
87
+ "sourceClaimIds": [
88
+ "semantic-claim:aa940991c0abbc78ab7e7946072c6b2e",
89
+ "semantic-claim:258fba0b9afa186f55129c777cfd1093"
90
+ ],
91
+ "confidence": 0.7
92
+ },
93
+ {
94
+ "channel": "code",
95
+ "kind": "expansion",
96
+ "query": "Memory.add infer parameter handling sequence",
97
+ "sourceClaimIds": [
98
+ "semantic-claim:aa940991c0abbc78ab7e7946072c6b2e"
99
+ ],
100
+ "confidence": 0.55
101
+ },
102
+ {
103
+ "channel": "code",
104
+ "kind": "expansion",
105
+ "query": "Memory.add caller entry point infer flow",
106
+ "sourceClaimIds": [
107
+ "semantic-claim:a2f9f4a46105e98d702a1f75bcd6fd08",
108
+ "semantic-claim:aa940991c0abbc78ab7e7946072c6b2e"
109
+ ],
110
+ "confidence": 0.5
111
+ },
112
+ {
113
+ "channel": "memory",
114
+ "kind": "rewrite",
115
+ "query": "Memory.add infer=True 处理顺序",
116
+ "sourceClaimIds": [
117
+ "semantic-claim:aa940991c0abbc78ab7e7946072c6b2e",
118
+ "semantic-claim:258fba0b9afa186f55129c777cfd1093"
119
+ ],
120
+ "confidence": 0.6
121
+ },
122
+ {
123
+ "channel": "knowledge",
124
+ "kind": "rewrite",
125
+ "query": "Memory.add infer=True processing order requirement",
126
+ "sourceClaimIds": [
127
+ "semantic-claim:aa940991c0abbc78ab7e7946072c6b2e",
128
+ "semantic-claim:258fba0b9afa186f55129c777cfd1093"
129
+ ],
130
+ "confidence": 0.5
131
+ }
132
+ ]
133
+ }
134
+ },
135
+ {
136
+ "caseId": "mem0/memory/explicit-save",
137
+ "channel": "memory",
138
+ "caseClass": "lifecycle",
139
+ "prompt": "当前 OSS Memory.add infer=True 的输入规范化、既有记忆检索、LLM 抽取和向量写入顺序是什么?",
140
+ "expectedLifecycle": "latest_episode_selected",
141
+ "querySampling": {
142
+ "status": "sampled",
143
+ "provider": "deepseek",
144
+ "model": "deepseek-v4-pro",
145
+ "drafts": [
146
+ {
147
+ "channel": "code",
148
+ "kind": "rewrite",
149
+ "query": "Memory.add infer=True input normalization memory retrieval LLM extraction vector write order",
150
+ "sourceClaimIds": [
151
+ "semantic-claim:3c79162c87eb1329e6bcd52774c077e5",
152
+ "semantic-claim:5862455596f167faa50b7fd6027cca74",
153
+ "semantic-claim:906e6e4f38cf1b9a35ce8bec3f467a09",
154
+ "semantic-claim:a4a412fc3d5ddaa7e124b657ed239a4f"
155
+ ],
156
+ "confidence": 0.85
157
+ },
158
+ {
159
+ "channel": "code",
160
+ "kind": "expansion",
161
+ "query": "Memory.add method implementation infer parameter pipeline",
162
+ "sourceClaimIds": [
163
+ "semantic-claim:3c79162c87eb1329e6bcd52774c077e5",
164
+ "semantic-claim:a4a412fc3d5ddaa7e124b657ed239a4f"
165
+ ],
166
+ "confidence": 0.7
167
+ },
168
+ {
169
+ "channel": "code",
170
+ "kind": "expansion",
171
+ "query": "Memory.add caller entry point orchestration data flow",
172
+ "sourceClaimIds": [
173
+ "semantic-claim:3c79162c87eb1329e6bcd52774c077e5",
174
+ "semantic-claim:906e6e4f38cf1b9a35ce8bec3f467a09"
175
+ ],
176
+ "confidence": 0.6
177
+ },
178
+ {
179
+ "channel": "memory",
180
+ "kind": "rewrite",
181
+ "query": "Memory.add infer=True 输入规范化 记忆检索 LLM 抽取 向量写入 顺序",
182
+ "sourceClaimIds": [
183
+ "semantic-claim:3c79162c87eb1329e6bcd52774c077e5",
184
+ "semantic-claim:5862455596f167faa50b7fd6027cca74",
185
+ "semantic-claim:906e6e4f38cf1b9a35ce8bec3f467a09",
186
+ "semantic-claim:a4a412fc3d5ddaa7e124b657ed239a4f"
187
+ ],
188
+ "confidence": 0.8
189
+ },
190
+ {
191
+ "channel": "knowledge",
192
+ "kind": "rewrite",
193
+ "query": "OSS Memory.add infer=True input normalization existing memory retrieval LLM extraction vector write sequence",
194
+ "sourceClaimIds": [
195
+ "semantic-claim:b801fac16c7f7a28605b6fc068fdc699",
196
+ "semantic-claim:3c79162c87eb1329e6bcd52774c077e5",
197
+ "semantic-claim:5862455596f167faa50b7fd6027cca74",
198
+ "semantic-claim:a4a412fc3d5ddaa7e124b657ed239a4f"
199
+ ],
200
+ "confidence": 0.75
201
+ },
202
+ {
203
+ "channel": "knowledge",
204
+ "kind": "expansion",
205
+ "query": "Memory.add infer=True pipeline stages normalization retrieval LLM extraction vector write",
206
+ "sourceClaimIds": [
207
+ "semantic-claim:3c79162c87eb1329e6bcd52774c077e5",
208
+ "semantic-claim:5862455596f167faa50b7fd6027cca74",
209
+ "semantic-claim:a4a412fc3d5ddaa7e124b657ed239a4f"
210
+ ],
211
+ "confidence": 0.65
212
+ }
213
+ ]
214
+ }
215
+ },
216
+ {
217
+ "caseId": "mem0/knowledge/identity-metadata",
218
+ "channel": "knowledge",
219
+ "caseClass": "positive",
220
+ "prompt": "Mem0 Python SDK v2.0.13 对 update() 中的 identity metadata 做了什么修复?",
221
+ "querySampling": {
222
+ "status": "sampled",
223
+ "provider": "deepseek",
224
+ "model": "deepseek-v4-pro",
225
+ "drafts": [
226
+ {
227
+ "channel": "code",
228
+ "kind": "rewrite",
229
+ "query": "Mem0 Python SDK update identity metadata fix",
230
+ "sourceClaimIds": [
231
+ "semantic-claim:f35e337eca3cd93788f73baf6bb6437e",
232
+ "semantic-claim:32130f9a79b1b844431e7897f2c44567",
233
+ "semantic-claim:df0a5f37b0858bbd9c559e924a7463d7"
234
+ ],
235
+ "confidence": 0.7
236
+ },
237
+ {
238
+ "channel": "code",
239
+ "kind": "expansion",
240
+ "query": "Mem0 update method identity metadata handling",
241
+ "sourceClaimIds": [
242
+ "semantic-claim:f35e337eca3cd93788f73baf6bb6437e"
243
+ ],
244
+ "confidence": 0.6
245
+ },
246
+ {
247
+ "channel": "code",
248
+ "kind": "expansion",
249
+ "query": "Mem0 client update identity metadata merge logic",
250
+ "sourceClaimIds": [
251
+ "semantic-claim:f35e337eca3cd93788f73baf6bb6437e",
252
+ "semantic-claim:f045a4fda88ef90a98afd33ddf2f2249"
253
+ ],
254
+ "confidence": 0.5
255
+ },
256
+ {
257
+ "channel": "knowledge",
258
+ "kind": "rewrite",
259
+ "query": "Mem0 Python SDK v2.0.13 update identity metadata fix changelog",
260
+ "sourceClaimIds": [
261
+ "semantic-claim:f35e337eca3cd93788f73baf6bb6437e",
262
+ "semantic-claim:32130f9a79b1b844431e7897f2c44567"
263
+ ],
264
+ "confidence": 0.7
265
+ },
266
+ {
267
+ "channel": "knowledge",
268
+ "kind": "expansion",
269
+ "query": "Mem0 update identity metadata bug fix release notes",
270
+ "sourceClaimIds": [
271
+ "semantic-claim:f35e337eca3cd93788f73baf6bb6437e"
272
+ ],
273
+ "confidence": 0.5
274
+ },
275
+ {
276
+ "channel": "memory",
277
+ "kind": "rewrite",
278
+ "query": "Mem0 update identity metadata fix",
279
+ "sourceClaimIds": [
280
+ "semantic-claim:f35e337eca3cd93788f73baf6bb6437e"
281
+ ],
282
+ "confidence": 0.4
283
+ }
284
+ ]
285
+ }
286
+ }
287
+ ]
288
+ }
@@ -0,0 +1,68 @@
1
+ {
2
+ "schemaVersion": "devflow.release-quality-project-input.v1",
3
+ "project": {
4
+ "projectId": "github:mem0ai/mem0",
5
+ "repository": "https://github.com/mem0ai/mem0.git",
6
+ "commit": "ca2abca2b884e038d3e525070e79d3057ef2012c",
7
+ "tree": "7ef14215330c040767e941686af06426941bf963",
8
+ "checkoutDirectory": "mem0",
9
+ "sourceSnapshot": { "manifestSha256": "2dc574a39b7fc1ed6dcf53946dcee5885ab0cede812681962588b295df668f37" }
10
+ },
11
+ "context": {
12
+ "evidence": [
13
+ {
14
+ "path": "mem0/memory/main.py",
15
+ "sha256": "7aa1a0026056abaea7b3a7653ea0d175a3a461d90b83a66c33c680fdac7cb46e",
16
+ "symbol": "add",
17
+ "startLine": 735
18
+ }
19
+ ]
20
+ },
21
+ "memory": {
22
+ "actorId": "actor:release-quality-reviewer",
23
+ "sessionId": "session:mem0-release-quality",
24
+ "prompt": "请记住:维护 OSS Memory.add infer=True 时必须保留输入规范化、检索既有记忆、LLM 抽取再写入向量库的处理顺序。",
25
+ "explicitContent": "维护 OSS Memory.add infer=True 时必须保留输入规范化、检索既有记忆、LLM 抽取再写入向量库的处理顺序。",
26
+ "temporal": {
27
+ "olderContent": "维护 OSS Memory.add infer=True 时必须先规范化输入,再写入记忆。",
28
+ "currentContent": "维护 OSS Memory.add infer=True 时必须保留输入规范化、检索既有记忆、LLM 抽取再写入向量库的处理顺序。",
29
+ "query": "当前 OSS Memory.add infer=True 的输入规范化、既有记忆检索、LLM 抽取和向量写入顺序是什么?"
30
+ },
31
+ "sourceEvidence": [
32
+ {
33
+ "path": "mem0/memory/main.py",
34
+ "sha256": "7aa1a0026056abaea7b3a7653ea0d175a3a461d90b83a66c33c680fdac7cb46e"
35
+ }
36
+ ]
37
+ },
38
+ "knowledge": {
39
+ "sourceId": "mem0-sdk-changelog",
40
+ "version": "2.0.13",
41
+ "pluginName": "release-quality",
42
+ "goldText": "<Update label=\"2026-07-22\" description=\"v2.0.13\">",
43
+ "provenance": {
44
+ "repository": "https://github.com/mem0ai/mem0.git",
45
+ "commit": "ca2abca2b884e038d3e525070e79d3057ef2012c"
46
+ },
47
+ "documents": [
48
+ {
49
+ "path": "docs/changelog/sdk.mdx",
50
+ "title": "Mem0 SDK changelog",
51
+ "sourceUri": "https://github.com/mem0ai/mem0/blob/ca2abca2b884e038d3e525070e79d3057ef2012c/docs/changelog/sdk.mdx",
52
+ "sha256": "72295da0b08b8b6cf5835404ca207378b995c5cc03640a43c005e88207509457"
53
+ },
54
+ {
55
+ "path": "pyproject.toml",
56
+ "title": "Mem0 Python package metadata",
57
+ "sourceUri": "https://github.com/mem0ai/mem0/blob/ca2abca2b884e038d3e525070e79d3057ef2012c/pyproject.toml",
58
+ "sha256": "adb1d694b022c1ad9e8bc8b532a44fe99aa5fdb678bcc07a786fb8c9ae2756c3"
59
+ }
60
+ ]
61
+ },
62
+ "requiredSlices": [
63
+ { "channel": "context", "caseClass": "positive", "caseIds": ["mem0/context/add-pipeline"] },
64
+ { "channel": "memory", "caseClass": "positive", "caseIds": ["mem0/memory/add-order"] },
65
+ { "channel": "memory", "caseClass": "lifecycle", "caseIds": ["mem0/memory/explicit-save"] },
66
+ { "channel": "knowledge", "caseClass": "positive", "caseIds": ["mem0/knowledge/identity-metadata"] }
67
+ ]
68
+ }
@@ -0,0 +1,29 @@
1
+ {
2
+ "schemaVersion": "devflow.release-quality-projects.v1",
3
+ "projects": [
4
+ {
5
+ "projectId": "github:shilongfeicool/rsbuild-react",
6
+ "datasetDirectory": "rsbuild-react",
7
+ "input": "input.json",
8
+ "corpus": "corpus.json"
9
+ },
10
+ {
11
+ "projectId": "github:nashsu/llm_wiki",
12
+ "datasetDirectory": "llm-wiki",
13
+ "input": "input.json",
14
+ "corpus": "corpus.json"
15
+ },
16
+ {
17
+ "projectId": "github:affaan-m/ecc",
18
+ "datasetDirectory": "ecc",
19
+ "input": "input.json",
20
+ "corpus": "corpus.json"
21
+ },
22
+ {
23
+ "projectId": "github:mem0ai/mem0",
24
+ "datasetDirectory": "mem0",
25
+ "input": "input.json",
26
+ "corpus": "corpus.json"
27
+ }
28
+ ]
29
+ }
@@ -0,0 +1,232 @@
1
+ {
2
+ "schemaVersion": "devflow.release-quality-policy.v1",
3
+ "bundleIds": [
4
+ "bundle:ca9cffebe3ec9cb7c7d468476e1c7b9c28011dfe80e4285909309dec23df8948",
5
+ "bundle:cb7ced4af9a57e90868365a90b3f573750f5c9c1fee1a234a8753de44ce53022",
6
+ "bundle:ca07e8f07062515bbd6498e44d87378b03ff16f394f488e2086dcf330fa71ef0",
7
+ "bundle:b5813b7c77e0087fad548d0c2d5804967a8c30c24d2cd473c7c3366d7f37c553"
8
+ ],
9
+ "requiredSlices": [
10
+ {
11
+ "key": "github:shilongfeicool/rsbuild-react/context/positive",
12
+ "minimumCases": 1,
13
+ "thresholds": {
14
+ "candidateRecallAt20": 1,
15
+ "recallAt5": 1,
16
+ "forbiddenContamination": 0,
17
+ "provenanceCompleteness": 1
18
+ }
19
+ },
20
+ {
21
+ "key": "github:shilongfeicool/rsbuild-react/memory/positive",
22
+ "minimumCases": 1,
23
+ "thresholds": {
24
+ "recallAt5": 1,
25
+ "provenanceCompleteness": 1
26
+ }
27
+ },
28
+ {
29
+ "key": "github:shilongfeicool/rsbuild-react/memory/lifecycle",
30
+ "minimumCases": 1,
31
+ "thresholds": {
32
+ "temporalCorrectness": 1,
33
+ "explicitSaveDurability": 1
34
+ }
35
+ },
36
+ {
37
+ "key": "github:shilongfeicool/rsbuild-react/knowledge/positive",
38
+ "minimumCases": 1,
39
+ "thresholds": {
40
+ "recallAt5": 1,
41
+ "knowledgeVersionCorrectness": 1,
42
+ "forbiddenContamination": 0,
43
+ "provenanceCompleteness": 1
44
+ }
45
+ },
46
+ {
47
+ "key": "github:nashsu/llm_wiki/context/positive",
48
+ "minimumCases": 1,
49
+ "thresholds": {
50
+ "candidateRecallAt20": 1,
51
+ "recallAt5": 1,
52
+ "forbiddenContamination": 0,
53
+ "provenanceCompleteness": 1
54
+ }
55
+ },
56
+ {
57
+ "key": "github:nashsu/llm_wiki/memory/positive",
58
+ "minimumCases": 1,
59
+ "thresholds": {
60
+ "recallAt5": 1,
61
+ "provenanceCompleteness": 1
62
+ }
63
+ },
64
+ {
65
+ "key": "github:nashsu/llm_wiki/memory/lifecycle",
66
+ "minimumCases": 1,
67
+ "thresholds": {
68
+ "temporalCorrectness": 1,
69
+ "explicitSaveDurability": 1
70
+ }
71
+ },
72
+ {
73
+ "key": "github:nashsu/llm_wiki/knowledge/positive",
74
+ "minimumCases": 1,
75
+ "thresholds": {
76
+ "recallAt5": 1,
77
+ "knowledgeVersionCorrectness": 1,
78
+ "forbiddenContamination": 0,
79
+ "provenanceCompleteness": 1
80
+ }
81
+ },
82
+ {
83
+ "key": "github:affaan-m/ecc/context/positive",
84
+ "minimumCases": 1,
85
+ "thresholds": {
86
+ "candidateRecallAt20": 1,
87
+ "recallAt5": 1,
88
+ "forbiddenContamination": 0,
89
+ "provenanceCompleteness": 1
90
+ }
91
+ },
92
+ {
93
+ "key": "github:affaan-m/ecc/memory/positive",
94
+ "minimumCases": 1,
95
+ "thresholds": {
96
+ "recallAt5": 1,
97
+ "provenanceCompleteness": 1
98
+ }
99
+ },
100
+ {
101
+ "key": "github:affaan-m/ecc/memory/lifecycle",
102
+ "minimumCases": 1,
103
+ "thresholds": {
104
+ "temporalCorrectness": 1,
105
+ "explicitSaveDurability": 1
106
+ }
107
+ },
108
+ {
109
+ "key": "github:affaan-m/ecc/knowledge/positive",
110
+ "minimumCases": 1,
111
+ "thresholds": {
112
+ "recallAt5": 1,
113
+ "knowledgeVersionCorrectness": 1,
114
+ "forbiddenContamination": 0,
115
+ "provenanceCompleteness": 1
116
+ }
117
+ },
118
+ {
119
+ "key": "github:mem0ai/mem0/context/positive",
120
+ "minimumCases": 1,
121
+ "thresholds": {
122
+ "candidateRecallAt20": 1,
123
+ "recallAt5": 1,
124
+ "forbiddenContamination": 0,
125
+ "provenanceCompleteness": 1
126
+ }
127
+ },
128
+ {
129
+ "key": "github:mem0ai/mem0/memory/positive",
130
+ "minimumCases": 1,
131
+ "thresholds": {
132
+ "recallAt5": 1,
133
+ "provenanceCompleteness": 1
134
+ }
135
+ },
136
+ {
137
+ "key": "github:mem0ai/mem0/memory/lifecycle",
138
+ "minimumCases": 1,
139
+ "thresholds": {
140
+ "temporalCorrectness": 1,
141
+ "explicitSaveDurability": 1
142
+ }
143
+ },
144
+ {
145
+ "key": "github:mem0ai/mem0/knowledge/positive",
146
+ "minimumCases": 1,
147
+ "thresholds": {
148
+ "recallAt5": 1,
149
+ "knowledgeVersionCorrectness": 1,
150
+ "forbiddenContamination": 0,
151
+ "provenanceCompleteness": 1
152
+ }
153
+ },
154
+ {
155
+ "key": "host",
156
+ "minimumCases": 1,
157
+ "thresholds": {
158
+ "consumptionRate": 1,
159
+ "verificationRate": 1
160
+ }
161
+ }
162
+ ],
163
+ "baselineRuntimeIdentity": {
164
+ "commit": "5fa4179f8ed5850ad7cfc9e8c374fb1c54da3f53",
165
+ "lockfileHash": "0266f38eba6e0bc294b9166c00a97a9854080bc944bddf4565345fb6b42820cf",
166
+ "buildConfigHash": "38ce99ccc860698c47fff0f0a757d788dd003bda",
167
+ "tarballHashes": {
168
+ "@devflow-tools/adapters": "d97d7b819e8975d4ad5616db65d2a9c80fb5722f87e2e61ce9636482398cb914",
169
+ "@devflow-tools/benchmark": "ab1f45cd5d08099260a6dff13e995a9bfaae6b1f507efe9bb6acea40c3921a97",
170
+ "@devflow-tools/claude-code-plugin": "19e4d0b25be4af8476692bb4f3e58613e84754384dcdfe358eb093151d0b211a",
171
+ "@devflow-tools/cli": "9310eaad69d8604cbedf32b643f1967dabcec1c18bcfa2cc974d2ada1642952a",
172
+ "@devflow-tools/context-engine": "3ecc6b72cb84f3786a61040b4c694e2d88fe595b3abad3c9dea7133ff4df8e6b",
173
+ "@devflow-tools/database": "b81d2da5cecb11d9016f119dcea7b3c3eb33c6ec3b9e299fa5b80a330d681280",
174
+ "@devflow-tools/devflow": "45430a458e98e92ff4283cf745cd3691d5fa65e8e708e3927dfec57f4e8827ce",
175
+ "@devflow-tools/knowledge-engine": "4f5a2e453d7c91c4d12e84837ca5ec2e607d017c6bc2274c43477ab24af5e1a8",
176
+ "@devflow-tools/mcp-server": "7d9e3320659dfcc0f6921a6c982193a44666bcdc2a638695d0fc08f90944107e",
177
+ "@devflow-tools/memory-engine": "5ca8fde9aee69c97e39b881203408f8ee6650fa3aa124b010f4be109870905d2",
178
+ "@devflow-tools/plugin-animation": "b6e6c14454d0d36868a8aafdabb40c8221e52f051fd35b6d8d478f2e23c5da7c",
179
+ "@devflow-tools/plugin-css": "0f460fa0dcf1922b039c003ec01240a4739f58caed001488bf2b47bc9f3e9241",
180
+ "@devflow-tools/plugin-docker": "eb73c3a870a975b7ee4420e3e8fab753fe13ba2d1b7d54c8021c91d5e451365d",
181
+ "@devflow-tools/plugin-electron": "760bf40b6320c8a18192682355912a3b0d776a43a7cee52c611cac53203d59a2",
182
+ "@devflow-tools/plugin-git": "26476ba9033e8c136b1f8de9b5973e6fa6d819a641655301a5f0e08af14d89a4",
183
+ "@devflow-tools/plugin-graphql": "e9c1ba776e19fc96b6541ba6e4f7322c7ada56d4a064138192354fac2c7f96f3",
184
+ "@devflow-tools/plugin-nest": "396e88577cd09778ee55e4d8b1a41901a6718feb925f1933b043b4e074a24f8b",
185
+ "@devflow-tools/plugin-nextjs": "9324b8bddae061e0372fd2fe0b59815bf1171ddeaa615d26cab17a5f3ce69c62",
186
+ "@devflow-tools/plugin-performance": "03613e2e32a158b048e3a12e27886ef9baa4cbd941c504d34379899e78727828",
187
+ "@devflow-tools/plugin-react": "7094acb4898aab9ac050dc48496c47f3a1b1db4e3555bf252749be204c4b5ecc",
188
+ "@devflow-tools/plugin-tailwind": "4cda3ebe12184fa49ac96f036f828111e1ff4181bfa470c59493703e0831a474",
189
+ "@devflow-tools/plugin-taro": "9b775abeffa4fbbac508ddf2601b803227b774c2e99e706a4f702e25afe2e3e9",
190
+ "@devflow-tools/plugin-ui-layout": "277ec9401293b810d7f94282595956a1ae4b66b842e15c591288e7a6b789000e",
191
+ "@devflow-tools/plugin-vue": "0fa96b1b78997b33ef8d10ca99ffa82280bc3a3f598e9a014601c04d2095bfef",
192
+ "@devflow-tools/provider-runtime": "30eda206d3ca5cf66ad65ce6d043dfc96a66c6e658501fcc56a3c747bc2c0fbf",
193
+ "@devflow-tools/retrieval-engine": "ada0a3fc8e96a5420a750de68258e6d8f956f8cbe1f279a03036d401e81bb5b8",
194
+ "@devflow-tools/sdk": "5273238abaecc3f7d923821812f2458163cab2d0bc2afc863b7c097c241326d9",
195
+ "@devflow-tools/semantic-engine": "e3011c88cabdf223a5498993dea31ef8bf93559df2f7c163c1bf321ed6e0fce3",
196
+ "@devflow-tools/server": "35bf16b0a7c97cd3b97181582091ef189d396fabe7269e020d798ea98f6505fb",
197
+ "@devflow-tools/task-runtime": "9eb87fe91dc92d4c844799b44bbe2b97d15551b556c9f03d7423671817be88a5",
198
+ "@devflow-tools/telemetry": "8abd3f5baf2b8dc4e6934a40b2baf109481ad368eb4daccc188ec7725ab0867f",
199
+ "@devflow-tools/workflow-engine": "a7343f8ea6ed58f31609c9a3403c98a48ab61763b021e902d5dc6407982d19f2"
200
+ }
201
+ },
202
+ "comparisonProfile": {
203
+ "providerModel": {
204
+ "provider": "deepseek",
205
+ "model": "deepseek-v4-pro"
206
+ },
207
+ "readerProfileHash": "66c128c341ad8d9159932389bae16eab6efeaf497065028c456bf3ff1a14a267",
208
+ "budgetHash": "e44e88e280fd2da77b687a89d791dda18ac3842ae0076cabe37e6ecf41e7b75c",
209
+ "candidateTopK": 20,
210
+ "selectedTopK": 5
211
+ },
212
+ "externalHostProfile": {
213
+ "host": "claude-code",
214
+ "version": "2.1.228",
215
+ "permissions": [
216
+ "mcp__devflow__get_project_context",
217
+ "mcp__devflow__get_memory",
218
+ "mcp__devflow__get_knowledge",
219
+ "mcp__devflow__run_workflow",
220
+ "mcp__devflow__devflow_report_step"
221
+ ],
222
+ "maxTurns": 8,
223
+ "timeoutMs": 600000,
224
+ "promptHashes": {
225
+ "bundle:ca9cffebe3ec9cb7c7d468476e1c7b9c28011dfe80e4285909309dec23df8948": "683a7f69ac778d23c47a037543702f0e72e455c006c72a76f0fe20ad74421352",
226
+ "bundle:cb7ced4af9a57e90868365a90b3f573750f5c9c1fee1a234a8753de44ce53022": "94dc18f0881d4b2dfb22a8dcec741f62664c25bdfb2ef749fb6a9cbf33190205",
227
+ "bundle:ca07e8f07062515bbd6498e44d87378b03ff16f394f488e2086dcf330fa71ef0": "d54fcd60f0933018a44b2103754c3129faca9d0f4b40e66890c488f1c9341152",
228
+ "bundle:b5813b7c77e0087fad548d0c2d5804967a8c30c24d2cd473c7c3366d7f37c553": "a1acd8f71e415fce0acb6b0eec4a02829e4ad7261939889694944793e96dc115"
229
+ }
230
+ },
231
+ "policyHash": "0bf2bcba8bad959cd38301372545f7ac5d5e1cbb759d8ca6420e6d6a9dfb5581"
232
+ }