@devflow-tools/benchmark 0.16.27 → 0.16.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/repeated-trials.d.ts +1 -0
- package/dist/repeated-trials.d.ts.map +1 -1
- package/dist/repeated-trials.js +1 -0
- package/dist/repeated-trials.js.map +1 -1
- package/dist/retrieval-quality-corpus.d.ts +23 -0
- package/dist/retrieval-quality-corpus.d.ts.map +1 -1
- package/dist/retrieval-quality-corpus.js +72 -9
- package/dist/retrieval-quality-corpus.js.map +1 -1
- package/dist/retrieval-quality.d.ts +47 -0
- package/dist/retrieval-quality.d.ts.map +1 -1
- package/dist/retrieval-quality.js +146 -2
- package/dist/retrieval-quality.js.map +1 -1
- package/fixtures/projects/react-video/.devflow/.force-init +1 -1
- package/fixtures/projects/react-video/.devflow/knowledge/knowledge.db +0 -0
- package/fixtures/projects/react-video/.devflow/memory/memories.db +0 -0
- package/fixtures/projects/rsbuild-react-quality/.codegraph/.gitignore +5 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/.force-init +1 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/knowledge/angular/manifest.json +13 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/knowledge/knowledge.db +0 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/knowledge/nest/manifest.json +13 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/knowledge/nextjs/manifest.json +13 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/knowledge/node/manifest.json +13 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/knowledge/prisma/manifest.json +13 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/knowledge/react/chunk-001.json +8 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/knowledge/react/manifest.json +13 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/knowledge/tailwind/manifest.json +13 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/knowledge/typescript/manifest.json +13 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/knowledge/vitest/manifest.json +13 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/knowledge/vue/manifest.json +13 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/memory/memories.db +0 -0
- package/fixtures/projects/rsbuild-react-quality/.devflow/symbols-vec.db +0 -0
- package/fixtures/projects/rsbuild-react-quality/components.json +7 -0
- package/fixtures/projects/rsbuild-react-quality/package.json +17 -0
- package/fixtures/projects/rsbuild-react-quality/src/components/Login.tsx +14 -0
- package/fixtures/retrieval-quality-corpus.json +57 -2
- package/fixtures/retrieval-quality-devflow.json +5 -3
- package/fixtures/retrieval-quality-react-video.json +63 -1
- package/fixtures/retrieval-quality-rsbuild-react.json +93 -0
- package/fixtures/retrieval-quality.json +5 -2
- package/package.json +5 -5
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repeated-trials.d.ts","sourceRoot":"","sources":["../src/repeated-trials.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC5C,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;IACf,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC;IAClB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,WAAW,CAAC;IACpB,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IACT,KAAK,EAAE,uBAAuB,EAAE,CAAC;IACjC,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,OAAO,EAAE;QACP,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;QAClD,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC3C,CAAC;CACH;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IACT,MAAM,EAAE,aAAa,EAAE,CAAC;CACzB,GAAG,mBAAmB,
|
|
1
|
+
{"version":3,"file":"repeated-trials.d.ts","sourceRoot":"","sources":["../src/repeated-trials.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC5C,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;IACf,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,WAAW,CAAC;IACpB,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IACT,KAAK,EAAE,uBAAuB,EAAE,CAAC;IACjC,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,OAAO,EAAE;QACP,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;QAClD,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC3C,CAAC;CACH;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IACT,MAAM,EAAE,aAAa,EAAE,CAAC;CACzB,GAAG,mBAAmB,CA0DtB;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC,GAAG,MAAM,CAExF;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,mBAAmB,EAAE,MAAM,GAAE,MAAM,GAAG,MAAe,GAAG,MAAM,CAc/G"}
|
package/dist/repeated-trials.js
CHANGED
|
@@ -18,6 +18,7 @@ export function aggregateRepeatedTrials(input) {
|
|
|
18
18
|
unavailableTrials: trials.filter(trial => trial.status === 'unavailable').length,
|
|
19
19
|
passAtK: successfulTrials > 0 ? 1 : 0,
|
|
20
20
|
passPowerK: successfulTrials === input.k ? 1 : 0,
|
|
21
|
+
passAt1: successfulTrials / trials.length,
|
|
21
22
|
trialIds: trials.map(trial => trial.trialId),
|
|
22
23
|
};
|
|
23
24
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repeated-trials.js","sourceRoot":"","sources":["../src/repeated-trials.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"repeated-trials.js","sourceRoot":"","sources":["../src/repeated-trials.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AA2DrD,MAAM,UAAU,uBAAuB,CAAC,KAOvC;IACC,cAAc,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;IACnD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE;QACjH,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,qBAAqB,KAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC;QAChH,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;QAClF,OAAO;YACL,MAAM;YACN,eAAe,EAAE,MAAM,CAAC,MAAM;YAC9B,gBAAgB;YAChB,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,aAAa,CAAC,CAAC,MAAM;YAChF,OAAO,EAAE,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,CAAU,CAAC,CAAC,CAAC,CAAU;YACvD,UAAU,EAAE,gBAAgB,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAU,CAAC,CAAC,CAAC,CAAU;YAClE,OAAO,EAAE,gBAAgB,GAAG,MAAM,CAAC,MAAM;YACzC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC;SAC7C,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5C,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACxF,MAAM,iBAAiB,GAA2B,EAAE,CAAC;IACrD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjC,IAAI,CAAC,KAAK,CAAC,eAAe;YAAE,SAAS;QACrC,iBAAiB,CAAC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,iBAAiB,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,aAAa,CAAC,CAAC;IAC/E,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IACrE,OAAO;QACL,KAAK,EAAE,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QACvD,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,MAAM,EAAE,WAAW;QACnB,CAAC,EAAE,KAAK,CAAC,CAAC;QACV,KAAK;QACL,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE;YACP,cAAc,EAAE,KAAK,CAAC,MAAM;YAC5B,eAAe;YACf,gBAAgB;YAChB,OAAO,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;YACrE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACvD,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACnE,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACnE,YAAY,EAAE,eAAe,GAAG,CAAC;gBAC/B,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,eAAe;gBAChF,CAAC,CAAC,CAAC;YACL,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ;YACxE,iBAAiB;SAClB;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAA6C;IAC9E,OAAO,SAAS,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AAC9F,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,MAA2B,EAAE,SAA0B,MAAM;IACrG,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,OAAO;QACL,uBAAuB,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,YAAY,GAAG;QAChE,QAAQ,MAAM,CAAC,KAAK,EAAE;QACtB,YAAY,MAAM,CAAC,WAAW,EAAE;QAChC,aAAa,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE;QACzC,WAAW,MAAM,CAAC,OAAO,CAAC,eAAe,KAAK,MAAM,CAAC,OAAO,CAAC,cAAc,cAAc,MAAM,CAAC,CAAC,GAAG;QACpG,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE;QACtF,QAAQ,MAAM,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;QACtH,QAAQ,MAAM,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;QACzH,kBAAkB,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE;QAC/C,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;KACpE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,MAAuB,EAAE,CAAS;IACxD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IACrF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;IACjD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;QACzF,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,OAAO,2CAA2C,CAAC,CAAC;QACrF,CAAC;QACD,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACxF,IAAI,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QACxG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,UAAU,CAAC;YAC1B,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,cAAc,EAAE,KAAK,CAAC,cAAc;SACrC,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,QAAQ,IAAI,QAAQ,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACxG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7C,CAAC;IACD,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,IAAI,KAAK,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,qBAAqB,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAC;IACrG,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,MAAuB;IAC1C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1F,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACxE,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC;IAChF,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AACrH,CAAC;AAED,SAAS,OAAO,CAAC,MAAgB;IAC/B,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/F,CAAC;AAED,SAAS,QAAQ,CAAC,SAAiB,EAAE,WAAmB;IACtD,OAAO,GAAG,SAAS,IAAI,WAAW,KAAK,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC;AAClH,CAAC"}
|
|
@@ -6,11 +6,31 @@ export interface RetrievalQualityCorpusProject {
|
|
|
6
6
|
optional?: boolean;
|
|
7
7
|
prepareCodeGraph?: boolean;
|
|
8
8
|
deferred?: boolean;
|
|
9
|
+
acceptanceClasses?: string[];
|
|
10
|
+
}
|
|
11
|
+
export interface AuthoritativeScenario {
|
|
12
|
+
id: string;
|
|
13
|
+
acceptanceClasses: string[];
|
|
14
|
+
expected: {
|
|
15
|
+
memoryIds?: string[];
|
|
16
|
+
memoryFacets?: string[];
|
|
17
|
+
knowledgeSources?: string[];
|
|
18
|
+
knowledgeChunks?: string[];
|
|
19
|
+
coverageGaps?: string[];
|
|
20
|
+
codePaths?: string[];
|
|
21
|
+
codeRoles?: string[];
|
|
22
|
+
domainAction?: string;
|
|
23
|
+
verification?: string[];
|
|
24
|
+
ledgerTransitions?: string[];
|
|
25
|
+
};
|
|
9
26
|
}
|
|
10
27
|
export interface RetrievalQualityCorpusFixture {
|
|
11
28
|
schemaVersion: 'devflow.retrieval-quality-corpus.v1';
|
|
12
29
|
projects: RetrievalQualityCorpusProject[];
|
|
13
30
|
maxTokenGrowthRatio: number;
|
|
31
|
+
trialCount?: 1 | 3;
|
|
32
|
+
requiredAcceptanceClasses?: string[];
|
|
33
|
+
authoritativeScenarios?: AuthoritativeScenario[];
|
|
14
34
|
}
|
|
15
35
|
export interface RetrievalQualityCorpusReport {
|
|
16
36
|
schemaVersion: 'devflow.retrieval-quality-corpus-report.v1';
|
|
@@ -21,6 +41,9 @@ export interface RetrievalQualityCorpusReport {
|
|
|
21
41
|
status: 'passed' | 'failed' | 'unavailable';
|
|
22
42
|
baseline?: RetrievalQualityReport;
|
|
23
43
|
candidate?: RetrievalQualityReport;
|
|
44
|
+
candidateTrials?: RetrievalQualityReport[];
|
|
45
|
+
passAt1?: number;
|
|
46
|
+
passPowerK?: 0 | 1;
|
|
24
47
|
tokenGrowthRatio?: number;
|
|
25
48
|
nonInferior?: boolean;
|
|
26
49
|
reason?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retrieval-quality-corpus.d.ts","sourceRoot":"","sources":["../src/retrieval-quality-corpus.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAGrE,MAAM,WAAW,6BAA6B;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"retrieval-quality-corpus.d.ts","sourceRoot":"","sources":["../src/retrieval-quality-corpus.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAGrE,MAAM,WAAW,6BAA6B;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,QAAQ,EAAE;QACR,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QACxB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC5B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QACxB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH;AAED,MAAM,WAAW,6BAA6B;IAC5C,aAAa,EAAE,qCAAqC,CAAC;IACrD,QAAQ,EAAE,6BAA6B,EAAE,CAAC;IAC1C,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IACnB,yBAAyB,CAAC,EAAE,MAAM,EAAE,CAAC;IACrC,sBAAsB,CAAC,EAAE,qBAAqB,EAAE,CAAC;CAClD;AAED,MAAM,WAAW,4BAA4B;IAC3C,aAAa,EAAE,4CAA4C,CAAC;IAC5D,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,QAAQ,EAAE,KAAK,CAAC;QACd,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;QAC5C,QAAQ,CAAC,EAAE,sBAAsB,CAAC;QAClC,SAAS,CAAC,EAAE,sBAAsB,CAAC;QACnC,eAAe,CAAC,EAAE,sBAAsB,EAAE,CAAC;QAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,UAAU,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,wBAAsB,yBAAyB,CAAC,OAAO,EAAE;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,GAAG,OAAO,CAAC,4BAA4B,CAAC,CA6GxC"}
|
|
@@ -6,6 +6,7 @@ export async function runRetrievalQualityCorpus(options) {
|
|
|
6
6
|
const corpus = loadCorpus(corpusPath);
|
|
7
7
|
const baseDir = dirname(corpusPath);
|
|
8
8
|
const projects = [];
|
|
9
|
+
const trialCount = corpus.trialCount ?? 1;
|
|
9
10
|
const previous = process.env.DEVFLOW_ROLLOUT_RETRIEVAL_SESSIONS;
|
|
10
11
|
try {
|
|
11
12
|
for (const project of corpus.projects) {
|
|
@@ -26,15 +27,30 @@ export async function runRetrievalQualityCorpus(options) {
|
|
|
26
27
|
process.env.DEVFLOW_ROLLOUT_RETRIEVAL_SESSIONS = 'off';
|
|
27
28
|
const baseline = await runRetrievalQualityBenchmark({ projectRoot, fixturePath, persist: false });
|
|
28
29
|
process.env.DEVFLOW_ROLLOUT_RETRIEVAL_SESSIONS = 'on';
|
|
29
|
-
const
|
|
30
|
-
|
|
31
|
-
|
|
30
|
+
const candidateTrials = [];
|
|
31
|
+
for (let trial = 0; trial < trialCount; trial++) {
|
|
32
|
+
candidateTrials.push(await runRetrievalQualityBenchmark({
|
|
33
|
+
projectRoot,
|
|
34
|
+
fixturePath,
|
|
35
|
+
persist: options.persist && trial === trialCount - 1,
|
|
36
|
+
}));
|
|
37
|
+
}
|
|
38
|
+
const candidate = candidateTrials[0];
|
|
39
|
+
const growth = candidateTrials.map(trial => ratioGrowth(baseline.metrics.averageContextTokens, trial.metrics.averageContextTokens));
|
|
40
|
+
const nonInferiorTrials = candidateTrials.map(trial => qualityNonInferior(baseline, trial));
|
|
41
|
+
const successfulTrials = candidateTrials.filter((trial, index) => trial.status === 'passed'
|
|
42
|
+
&& nonInferiorTrials[index] && growth[index] <= corpus.maxTokenGrowthRatio).length;
|
|
43
|
+
const tokenGrowthRatio = Math.max(...growth);
|
|
44
|
+
const nonInferior = nonInferiorTrials.every(Boolean);
|
|
32
45
|
projects.push({
|
|
33
46
|
id: project.id,
|
|
34
|
-
status:
|
|
47
|
+
status: successfulTrials === trialCount
|
|
35
48
|
? 'passed' : 'failed',
|
|
36
49
|
baseline,
|
|
37
50
|
candidate,
|
|
51
|
+
candidateTrials,
|
|
52
|
+
passAt1: successfulTrials / trialCount,
|
|
53
|
+
passPowerK: successfulTrials === trialCount ? 1 : 0,
|
|
38
54
|
tokenGrowthRatio,
|
|
39
55
|
nonInferior,
|
|
40
56
|
});
|
|
@@ -48,9 +64,12 @@ export async function runRetrievalQualityCorpus(options) {
|
|
|
48
64
|
}
|
|
49
65
|
const available = projects.filter(project => project.status !== 'unavailable' && project.candidate);
|
|
50
66
|
const metricNames = [
|
|
51
|
-
'codeFileRecallAt5', 'codeFilePrecisionAt5', 'symbolNdcgAt5',
|
|
52
|
-
'
|
|
53
|
-
'
|
|
67
|
+
'codeFileRecallAt5', 'codeFilePrecisionAt5', 'symbolNdcgAt5',
|
|
68
|
+
'memoryRecallAt5', 'policyMemoryRecallAt5', 'memoryPrecisionAt5',
|
|
69
|
+
'expectedEmptyMemoryAccuracy', 'knowledgeNdcgAt5', 'knowledgePrecisionAt3', 'knowledgeVectorCoverage',
|
|
70
|
+
'knowledgeVersionMatch', 'knowledgeInjectionViolations', 'staleSelectedFiles',
|
|
71
|
+
'averageContextTokens', 'maxRetrievalCycle',
|
|
72
|
+
'policyActionAccuracy', 'ledgerCompleteness', 'sameSessionMemoryReadyMs',
|
|
54
73
|
];
|
|
55
74
|
const macroMetrics = Object.fromEntries(metricNames.map(name => [name, average(available.map(project => Number(project.candidate.metrics[name])))]));
|
|
56
75
|
const gates = {
|
|
@@ -61,6 +80,25 @@ export async function runRetrievalQualityCorpus(options) {
|
|
|
61
80
|
nonInferior: available.every(project => project.nonInferior === true),
|
|
62
81
|
boundedTokenGrowth: available.every(project => (project.tokenGrowthRatio ?? Infinity) <= corpus.maxTokenGrowthRatio),
|
|
63
82
|
boundedCycles: available.every(project => project.candidate.metrics.maxRetrievalCycle <= 3),
|
|
83
|
+
memoryRecall: macroMetrics.memoryRecallAt5 >= 0.90,
|
|
84
|
+
policyMemoryRecall: macroMetrics.policyMemoryRecallAt5 >= 1,
|
|
85
|
+
memoryPrecision: macroMetrics.memoryPrecisionAt5 >= 0.95,
|
|
86
|
+
knowledgePrecision: available.every(project => project.candidate.metrics.knowledgePrecisionAt3 >= 0.80),
|
|
87
|
+
knowledgeNdcg: available.every(project => project.candidate.metrics.knowledgeNdcgAt5 >= 0.85),
|
|
88
|
+
noKnowledgeInjection: available.every(project => project.candidate.metrics.knowledgeInjectionViolations === 0),
|
|
89
|
+
codeSelectionPrecision: available.every(project => project.candidate.metrics.codeFilePrecisionAt5 >= 0.85),
|
|
90
|
+
noStaleSelectedFiles: available.every(project => project.candidate.metrics.staleSelectedFiles === 0),
|
|
91
|
+
policyActionAccuracy: available.every(project => project.candidate.metrics.policyActionAccuracy >= 1),
|
|
92
|
+
ledgerCompleteness: available.every(project => project.candidate.metrics.ledgerCompleteness >= 1),
|
|
93
|
+
positiveSamplesPresent: available.length === 0 || (available.reduce((total, project) => total + project.candidate.samples.memoryPositive, 0) > 0
|
|
94
|
+
&& available.reduce((total, project) => total + project.candidate.samples.policyMemoryPositive, 0) > 0
|
|
95
|
+
&& available.reduce((total, project) => total + project.candidate.samples.knowledgePositive, 0) > 0
|
|
96
|
+
&& available.reduce((total, project) => total + project.candidate.samples.policyAction, 0) > 0),
|
|
97
|
+
sameSessionMemoryReady: available.every(project => project.candidate.metrics.sameSessionMemoryReadyMs <= 5_000),
|
|
98
|
+
acceptanceClassesCovered: requiredAcceptanceClassesCovered(corpus),
|
|
99
|
+
authoritativeScenariosComplete: authoritativeScenariosComplete(corpus.authoritativeScenarios ?? []),
|
|
100
|
+
passAt1: available.every(project => project.passAt1 === 1),
|
|
101
|
+
passPowerK: available.every(project => project.passPowerK === 1),
|
|
64
102
|
};
|
|
65
103
|
return {
|
|
66
104
|
schemaVersion: 'devflow.retrieval-quality-corpus-report.v1',
|
|
@@ -71,6 +109,27 @@ export async function runRetrievalQualityCorpus(options) {
|
|
|
71
109
|
gates,
|
|
72
110
|
};
|
|
73
111
|
}
|
|
112
|
+
function requiredAcceptanceClassesCovered(corpus) {
|
|
113
|
+
const required = new Set(corpus.requiredAcceptanceClasses ?? []);
|
|
114
|
+
if (required.size === 0)
|
|
115
|
+
return true;
|
|
116
|
+
const covered = new Set(corpus.projects.flatMap(project => project.acceptanceClasses ?? []));
|
|
117
|
+
return [...required].every(value => covered.has(value));
|
|
118
|
+
}
|
|
119
|
+
function authoritativeScenariosComplete(scenarios) {
|
|
120
|
+
if (scenarios.length === 0)
|
|
121
|
+
return true;
|
|
122
|
+
if (scenarios.length !== 7 || new Set(scenarios.map(item => item.id)).size !== 7)
|
|
123
|
+
return false;
|
|
124
|
+
return scenarios.every(scenario => scenario.acceptanceClasses.length > 0
|
|
125
|
+
&& (scenario.expected.memoryIds !== undefined || scenario.expected.memoryFacets !== undefined)
|
|
126
|
+
&& (scenario.expected.knowledgeSources !== undefined || scenario.expected.coverageGaps !== undefined)
|
|
127
|
+
&& (scenario.expected.codePaths?.length ?? 0) > 0
|
|
128
|
+
&& (scenario.expected.codeRoles?.length ?? 0) > 0
|
|
129
|
+
&& Boolean(scenario.expected.domainAction)
|
|
130
|
+
&& (scenario.expected.verification?.length ?? 0) > 0
|
|
131
|
+
&& (scenario.expected.ledgerTransitions?.length ?? 0) > 0);
|
|
132
|
+
}
|
|
74
133
|
async function ensureCodeGraph(projectRoot) {
|
|
75
134
|
const module = await import('@colbymchenry/codegraph');
|
|
76
135
|
const CodeGraph = (module.default?.default ?? module.default);
|
|
@@ -97,9 +156,13 @@ function loadCorpus(path) {
|
|
|
97
156
|
function qualityNonInferior(baseline, candidate) {
|
|
98
157
|
const higher = [
|
|
99
158
|
'codeFileRecallAt5', 'codeFilePrecisionAt5', 'symbolNdcgAt5', 'memoryPrecisionAt5',
|
|
100
|
-
'expectedEmptyMemoryAccuracy', 'knowledgeNdcgAt5', '
|
|
159
|
+
'expectedEmptyMemoryAccuracy', 'knowledgeNdcgAt5', 'knowledgePrecisionAt3',
|
|
160
|
+
'knowledgeVectorCoverage', 'knowledgeVersionMatch',
|
|
161
|
+
];
|
|
162
|
+
const lower = [
|
|
163
|
+
'falseCrossFileEdgeRate', 'knowledgeChunkP95', 'knowledgeInjectionViolations',
|
|
164
|
+
'staleSelectedFiles', 'lifecycleFalseClosed', 'workflowFalseSuccess',
|
|
101
165
|
];
|
|
102
|
-
const lower = ['falseCrossFileEdgeRate', 'knowledgeChunkP95', 'lifecycleFalseClosed', 'workflowFalseSuccess'];
|
|
103
166
|
return higher.every(name => candidate.metrics[name] + 1e-9 >= baseline.metrics[name])
|
|
104
167
|
&& lower.every(name => candidate.metrics[name] <= baseline.metrics[name] + 1e-9);
|
|
105
168
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retrieval-quality-corpus.js","sourceRoot":"","sources":["../src/retrieval-quality-corpus.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"retrieval-quality-corpus.js","sourceRoot":"","sources":["../src/retrieval-quality-corpus.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AA0DtE,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,OAI/C;IACC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,MAAM,QAAQ,GAA6C,EAAE,CAAC;IAC9D,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC;IAChE,IAAI,CAAC;QACH,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBACjD,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,oCAAoC,EAAE,CAAC,CAAC;gBACvG,SAAS;YACX,CAAC;YACD,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;YAC1D,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;YAC1D,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBACzD,IAAI,CAAC,OAAO,CAAC,QAAQ;oBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC7F,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC,CAAC;gBAC/F,SAAS;YACX,CAAC;YACD,IAAI,OAAO,CAAC,gBAAgB;gBAAE,MAAM,eAAe,CAAC,WAAW,CAAC,CAAC;YACjE,OAAO,CAAC,GAAG,CAAC,kCAAkC,GAAG,KAAK,CAAC;YACvD,MAAM,QAAQ,GAAG,MAAM,4BAA4B,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAClG,OAAO,CAAC,GAAG,CAAC,kCAAkC,GAAG,IAAI,CAAC;YACtD,MAAM,eAAe,GAA6B,EAAE,CAAC;YACrD,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,EAAE,KAAK,EAAE,EAAE,CAAC;gBAChD,eAAe,CAAC,IAAI,CAAC,MAAM,4BAA4B,CAAC;oBACtD,WAAW;oBACX,WAAW;oBACX,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,KAAK,KAAK,UAAU,GAAG,CAAC;iBACrD,CAAC,CAAC,CAAC;YACN,CAAC;YACD,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAE,CAAC;YACtC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,WAAW,CACrD,QAAQ,CAAC,OAAO,CAAC,oBAAoB,EAAE,KAAK,CAAC,OAAO,CAAC,oBAAoB,CAC1E,CAAC,CAAC;YACH,MAAM,iBAAiB,GAAG,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,kBAAkB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;YAC5F,MAAM,gBAAgB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ;mBACtF,iBAAiB,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAE,IAAI,MAAM,CAAC,mBAAmB,CAAC,CAAC,MAAM,CAAC;YACtF,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YAC7C,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACrD,QAAQ,CAAC,IAAI,CAAC;gBACZ,EAAE,EAAE,OAAO,CAAC,EAAE;gBACd,MAAM,EAAE,gBAAgB,KAAK,UAAU;oBACrC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;gBACvB,QAAQ;gBACR,SAAS;gBACT,eAAe;gBACf,OAAO,EAAE,gBAAgB,GAAG,UAAU;gBACtC,UAAU,EAAE,gBAAgB,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnD,gBAAgB;gBAChB,WAAW;aACZ,CAAC,CAAC;QACL,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC;;YAC7E,OAAO,CAAC,GAAG,CAAC,kCAAkC,GAAG,QAAQ,CAAC;IACjE,CAAC;IAED,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,aAAa,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC;IACpG,MAAM,WAAW,GAAG;QAClB,mBAAmB,EAAE,sBAAsB,EAAE,eAAe;QAC5D,iBAAiB,EAAE,uBAAuB,EAAE,oBAAoB;QAChE,6BAA6B,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,yBAAyB;QACrG,uBAAuB,EAAE,8BAA8B,EAAE,oBAAoB;QAC7E,sBAAsB,EAAE,mBAAmB;QAC3C,sBAAsB,EAAE,oBAAoB,EAAE,0BAA0B;KAChE,CAAC;IACX,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAC5E,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CACnE,CAAC,CAAC,CAAC,CAAC;IACL,MAAM,KAAK,GAAG;QACZ,yBAAyB,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,aAAa;eAChF,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,OAAO,CAAC,EAAE,CAAC,EAAE,QAAQ,KAAK,IAAI;eACzE,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,OAAO,CAAC,EAAE,CAAC,EAAE,QAAQ,KAAK,IAAI,CAAC;QAC/E,oBAAoB,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,MAAM,KAAK,QAAQ,CAAC;QACxF,WAAW,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,WAAW,KAAK,IAAI,CAAC;QACrE,kBAAkB,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,IAAI,QAAQ,CAAC,IAAI,MAAM,CAAC,mBAAmB,CAAC;QACpH,aAAa,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,iBAAiB,IAAI,CAAC,CAAC;QAC5F,YAAY,EAAE,YAAY,CAAC,eAAe,IAAI,IAAI;QAClD,kBAAkB,EAAE,YAAY,CAAC,qBAAqB,IAAI,CAAC;QAC3D,eAAe,EAAE,YAAY,CAAC,kBAAkB,IAAI,IAAI;QACxD,kBAAkB,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC;QACxG,aAAa,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC;QAC9F,oBAAoB,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,4BAA4B,KAAK,CAAC,CAAC;QAC/G,sBAAsB,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,oBAAoB,IAAI,IAAI,CAAC;QAC3G,oBAAoB,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,kBAAkB,KAAK,CAAC,CAAC;QACrG,oBAAoB,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,oBAAoB,IAAI,CAAC,CAAC;QACtG,kBAAkB,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,kBAAkB,IAAI,CAAC,CAAC;QAClG,sBAAsB,EAAE,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,CAChD,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,GAAG,CAAC;eAC3F,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC,GAAG,CAAC;eACpG,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,GAAG,CAAC;eACjG,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,GAAG,CAAC,CAChG;QACD,sBAAsB,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,wBAAwB,IAAI,KAAK,CAAC;QAChH,wBAAwB,EAAE,gCAAgC,CAAC,MAAM,CAAC;QAClE,8BAA8B,EAAE,8BAA8B,CAAC,MAAM,CAAC,sBAAsB,IAAI,EAAE,CAAC;QACnG,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,KAAK,CAAC,CAAC;QAC1D,UAAU,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,UAAU,KAAK,CAAC,CAAC;KACjE,CAAC;IACF,OAAO;QACL,aAAa,EAAE,4CAA4C;QAC3D,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACjE,QAAQ;QACR,YAAY;QACZ,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,gCAAgC,CAAC,MAAqC;IAC7E,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;IACjE,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7F,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,8BAA8B,CAAC,SAAkC;IACxE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/F,OAAO,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC;WACnE,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC;WAC3F,CAAC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,KAAK,SAAS,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC;WAClG,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;WAC9C,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;WAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC;WACvC,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;WACjD,CAAC,QAAQ,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,WAAmB;IAChD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,CAAE,MAAM,CAAC,OAAe,EAAE,OAAO,IAAI,MAAM,CAAC,OAAO,CAIpE,CAAC;IACF,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjE,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,OAAO;IACT,CAAC;IACD,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC9C,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,CAAC;IAChB,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAkC,CAAC;IACtF,IAAI,KAAK,CAAC,aAAa,KAAK,qCAAqC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpG,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAgC,EAAE,SAAiC;IAC7F,MAAM,MAAM,GAAG;QACb,mBAAmB,EAAE,sBAAsB,EAAE,eAAe,EAAE,oBAAoB;QAClF,6BAA6B,EAAE,kBAAkB,EAAE,uBAAuB;QAC1E,yBAAyB,EAAE,uBAAuB;KAC1C,CAAC;IACX,MAAM,KAAK,GAAG;QACZ,wBAAwB,EAAE,mBAAmB,EAAE,8BAA8B;QAC7E,oBAAoB,EAAE,sBAAsB,EAAE,sBAAsB;KAC5D,CAAC;IACX,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;WAChF,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,WAAW,CAAC,QAAgB,EAAE,SAAiB;IACtD,IAAI,QAAQ,IAAI,CAAC;QAAE,OAAO,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IACxD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,OAAO,CAAC,MAAgB;IAC/B,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/F,CAAC"}
|
|
@@ -8,19 +8,41 @@ export interface RetrievalQualityFixture {
|
|
|
8
8
|
targetFile: string;
|
|
9
9
|
kind?: string;
|
|
10
10
|
}>;
|
|
11
|
+
setup?: {
|
|
12
|
+
resetProjectState?: boolean;
|
|
13
|
+
explicitMemories?: string[];
|
|
14
|
+
knowledgeSources?: Array<{
|
|
15
|
+
name: string;
|
|
16
|
+
pluginName?: string;
|
|
17
|
+
version: string;
|
|
18
|
+
chunks: Array<{
|
|
19
|
+
title: string;
|
|
20
|
+
content: string;
|
|
21
|
+
sourceUrl?: string;
|
|
22
|
+
}>;
|
|
23
|
+
}>;
|
|
24
|
+
};
|
|
11
25
|
}
|
|
12
26
|
export interface RetrievalQualityThresholds {
|
|
13
27
|
codeFileRecallAt5: number;
|
|
14
28
|
codeFilePrecisionAt5: number;
|
|
15
29
|
symbolNdcgAt5: number;
|
|
16
30
|
falseCrossFileEdgeRate: number;
|
|
31
|
+
memoryRecallAt5?: number;
|
|
32
|
+
policyMemoryRecallAt5?: number;
|
|
17
33
|
memoryPrecisionAt5: number;
|
|
18
34
|
expectedEmptyMemoryAccuracy: number;
|
|
19
35
|
knowledgeNdcgAt5: number;
|
|
36
|
+
knowledgePrecisionAt3: number;
|
|
20
37
|
knowledgeVectorCoverage: number;
|
|
21
38
|
knowledgeChunkP95: number;
|
|
22
39
|
knowledgeVersionMatch: number;
|
|
40
|
+
maxKnowledgeInjectionViolations?: number;
|
|
41
|
+
maxStaleSelectedFiles?: number;
|
|
23
42
|
maxRetrievalCycle?: number;
|
|
43
|
+
minPolicyActionAccuracy?: number;
|
|
44
|
+
minLedgerCompleteness?: number;
|
|
45
|
+
maxSameSessionMemoryReadyMs?: number;
|
|
24
46
|
}
|
|
25
47
|
export interface RetrievalQualityCase {
|
|
26
48
|
id: string;
|
|
@@ -37,6 +59,14 @@ export interface RetrievalQualityCase {
|
|
|
37
59
|
relevantKnowledgeIds?: string[];
|
|
38
60
|
relevantKnowledgeSources?: string[];
|
|
39
61
|
expectedVersions?: Record<string, string>;
|
|
62
|
+
expectedEmptyKnowledge?: boolean;
|
|
63
|
+
forbiddenKnowledgeSources?: string[];
|
|
64
|
+
forbiddenKnowledgeKinds?: Array<'api' | 'behavior' | 'migration' | 'error' | 'overview'>;
|
|
65
|
+
degradation?: 'embedding_unavailable' | 'reranker_unavailable' | 'none';
|
|
66
|
+
activeSkill?: string;
|
|
67
|
+
expectedPolicyFacets?: string[];
|
|
68
|
+
expectedDomainAction?: string;
|
|
69
|
+
expectedVerificationKinds?: string[];
|
|
40
70
|
}
|
|
41
71
|
export interface RetrievalQualityReport {
|
|
42
72
|
schemaVersion: 'devflow.retrieval-quality-report.v1';
|
|
@@ -49,16 +79,30 @@ export interface RetrievalQualityReport {
|
|
|
49
79
|
codeFilePrecisionAt5: number;
|
|
50
80
|
symbolNdcgAt5: number;
|
|
51
81
|
falseCrossFileEdgeRate: number;
|
|
82
|
+
memoryRecallAt5: number;
|
|
83
|
+
policyMemoryRecallAt5: number;
|
|
52
84
|
memoryPrecisionAt5: number;
|
|
53
85
|
expectedEmptyMemoryAccuracy: number;
|
|
54
86
|
knowledgeNdcgAt5: number;
|
|
87
|
+
knowledgePrecisionAt3: number;
|
|
55
88
|
knowledgeVectorCoverage: number;
|
|
56
89
|
knowledgeChunkP95: number;
|
|
57
90
|
knowledgeVersionMatch: number;
|
|
91
|
+
knowledgeInjectionViolations: number;
|
|
92
|
+
staleSelectedFiles: number;
|
|
58
93
|
lifecycleFalseClosed: number;
|
|
59
94
|
workflowFalseSuccess: number;
|
|
60
95
|
averageContextTokens: number;
|
|
61
96
|
maxRetrievalCycle: number;
|
|
97
|
+
policyActionAccuracy: number;
|
|
98
|
+
ledgerCompleteness: number;
|
|
99
|
+
sameSessionMemoryReadyMs: number;
|
|
100
|
+
};
|
|
101
|
+
samples: {
|
|
102
|
+
memoryPositive: number;
|
|
103
|
+
policyMemoryPositive: number;
|
|
104
|
+
knowledgePositive: number;
|
|
105
|
+
policyAction: number;
|
|
62
106
|
};
|
|
63
107
|
thresholds: RetrievalQualityThresholds;
|
|
64
108
|
gates: Record<string, boolean>;
|
|
@@ -72,6 +116,9 @@ export interface RetrievalQualityReport {
|
|
|
72
116
|
memoryPrecisionAt5?: number;
|
|
73
117
|
expectedEmptyMemoryCorrect?: boolean;
|
|
74
118
|
knowledgeNdcgAt5?: number;
|
|
119
|
+
knowledgePrecisionAt3?: number;
|
|
120
|
+
knowledgeInjectionViolations?: number;
|
|
121
|
+
staleSelectedFiles: number;
|
|
75
122
|
tokenCount: number;
|
|
76
123
|
retrievalCycle: number;
|
|
77
124
|
}>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retrieval-quality.d.ts","sourceRoot":"","sources":["../src/retrieval-quality.ts"],"names":[],"mappings":"AAKA,OAAO,EAAsB,KAAK,eAAe,EAA4B,MAAM,oBAAoB,CAAC;AAExG,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,8BAA8B,CAAC;IAC9C,UAAU,EAAE,0BAA0B,CAAC;IACvC,OAAO,EAAE,oBAAoB,EAAE,CAAC;IAChC,kBAAkB,CAAC,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"retrieval-quality.d.ts","sourceRoot":"","sources":["../src/retrieval-quality.ts"],"names":[],"mappings":"AAKA,OAAO,EAAsB,KAAK,eAAe,EAA4B,MAAM,oBAAoB,CAAC;AAExG,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,8BAA8B,CAAC;IAC9C,UAAU,EAAE,0BAA0B,CAAC;IACvC,OAAO,EAAE,oBAAoB,EAAE,CAAC;IAChC,kBAAkB,CAAC,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACtF,KAAK,CAAC,EAAE;QACN,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC5B,gBAAgB,CAAC,EAAE,KAAK,CAAC;YACvB,IAAI,EAAE,MAAM,CAAC;YACb,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,OAAO,EAAE,MAAM,CAAC;YAChB,MAAM,EAAE,KAAK,CAAC;gBAAE,KAAK,EAAE,MAAM,CAAC;gBAAC,OAAO,EAAE,MAAM,CAAC;gBAAC,SAAS,CAAC,EAAE,MAAM,CAAA;aAAE,CAAC,CAAC;SACvE,CAAC,CAAC;KACJ,CAAC;CACH;AAED,MAAM,WAAW,0BAA0B;IACzC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,2BAA2B,EAAE,MAAM,CAAC;IACpC,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,+BAA+B,CAAC,EAAE,MAAM,CAAC;IACzC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,eAAe,CAAA;KAAE,CAAC,CAAC;IAC9D,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC,wBAAwB,CAAC,EAAE,MAAM,EAAE,CAAC;IACpC,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,yBAAyB,CAAC,EAAE,MAAM,EAAE,CAAC;IACrC,uBAAuB,CAAC,EAAE,KAAK,CAAC,KAAK,GAAG,UAAU,GAAG,WAAW,GAAG,OAAO,GAAG,UAAU,CAAC,CAAC;IACzF,WAAW,CAAC,EAAE,uBAAuB,GAAG,sBAAsB,GAAG,MAAM,CAAC;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,yBAAyB,CAAC,EAAE,MAAM,EAAE,CAAC;CACtC;AAED,MAAM,WAAW,sBAAsB;IACrC,aAAa,EAAE,qCAAqC,CAAC;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,OAAO,EAAE;QACP,iBAAiB,EAAE,MAAM,CAAC;QAC1B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,aAAa,EAAE,MAAM,CAAC;QACtB,sBAAsB,EAAE,MAAM,CAAC;QAC/B,eAAe,EAAE,MAAM,CAAC;QACxB,qBAAqB,EAAE,MAAM,CAAC;QAC9B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,2BAA2B,EAAE,MAAM,CAAC;QACpC,gBAAgB,EAAE,MAAM,CAAC;QACzB,qBAAqB,EAAE,MAAM,CAAC;QAC9B,uBAAuB,EAAE,MAAM,CAAC;QAChC,iBAAiB,EAAE,MAAM,CAAC;QAC1B,qBAAqB,EAAE,MAAM,CAAC;QAC9B,4BAA4B,EAAE,MAAM,CAAC;QACrC,kBAAkB,EAAE,MAAM,CAAC;QAC3B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,wBAAwB,EAAE,MAAM,CAAC;KAClC,CAAC;IACF,OAAO,EAAE;QACP,cAAc,EAAE,MAAM,CAAC;QACvB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,UAAU,EAAE,0BAA0B,CAAC;IACvC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,KAAK,EAAE,KAAK,CAAC;QACX,EAAE,EAAE,MAAM,CAAC;QACX,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,iBAAiB,EAAE,MAAM,EAAE,CAAC;QAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,0BAA0B,CAAC,EAAE,OAAO,CAAC;QACrC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B,4BAA4B,CAAC,EAAE,MAAM,CAAC;QACtC,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;QACnB,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC,CAAC;IACH,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,EAAE;QACV,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;QAC/B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;QACnC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;QACnC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;QAC/B,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;QACrC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;QACrC,0BAA0B,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1C,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;QAC/B,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;KAC5C,CAAC;CACH;AAED,wBAAsB,4BAA4B,CAAC,OAAO,EAAE;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CA4KlC"}
|