oh-my-knowledge 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +205 -8
- package/README.zh.md +200 -7
- package/dist/src/analysis/failure-clusterer.d.ts +96 -0
- package/dist/src/analysis/failure-clusterer.d.ts.map +1 -0
- package/dist/src/analysis/failure-clusterer.js +298 -0
- package/dist/src/analysis/failure-clusterer.js.map +1 -0
- package/dist/src/analysis/sample-diagnostics.d.ts +78 -0
- package/dist/src/analysis/sample-diagnostics.d.ts.map +1 -0
- package/dist/src/analysis/sample-diagnostics.js +259 -0
- package/dist/src/analysis/sample-diagnostics.js.map +1 -0
- package/dist/src/analysis/saturation.d.ts +85 -0
- package/dist/src/analysis/saturation.d.ts.map +1 -0
- package/dist/src/analysis/saturation.js +174 -0
- package/dist/src/analysis/saturation.js.map +1 -0
- package/dist/src/cli.js +751 -5
- package/dist/src/cli.js.map +1 -1
- package/dist/src/eval-core/bootstrap.d.ts +72 -0
- package/dist/src/eval-core/bootstrap.d.ts.map +1 -0
- package/dist/src/eval-core/bootstrap.js +174 -0
- package/dist/src/eval-core/bootstrap.js.map +1 -0
- package/dist/src/eval-core/evaluation-execution.d.ts +15 -1
- package/dist/src/eval-core/evaluation-execution.d.ts.map +1 -1
- package/dist/src/eval-core/evaluation-execution.js +37 -3
- package/dist/src/eval-core/evaluation-execution.js.map +1 -1
- package/dist/src/eval-core/evaluation-job.d.ts +8 -2
- package/dist/src/eval-core/evaluation-job.d.ts.map +1 -1
- package/dist/src/eval-core/evaluation-job.js +7 -1
- package/dist/src/eval-core/evaluation-job.js.map +1 -1
- package/dist/src/eval-core/evaluation-reporting.d.ts.map +1 -1
- package/dist/src/eval-core/evaluation-reporting.js +90 -0
- package/dist/src/eval-core/evaluation-reporting.js.map +1 -1
- package/dist/src/eval-core/schema.d.ts.map +1 -1
- package/dist/src/eval-core/schema.js +69 -0
- package/dist/src/eval-core/schema.js.map +1 -1
- package/dist/src/eval-core/verdict.d.ts +74 -0
- package/dist/src/eval-core/verdict.d.ts.map +1 -0
- package/dist/src/eval-core/verdict.js +283 -0
- package/dist/src/eval-core/verdict.js.map +1 -0
- package/dist/src/eval-workflows/each-evaluation-workflow.d.ts +10 -1
- package/dist/src/eval-workflows/each-evaluation-workflow.d.ts.map +1 -1
- package/dist/src/eval-workflows/each-evaluation-workflow.js +4 -1
- package/dist/src/eval-workflows/each-evaluation-workflow.js.map +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts +13 -1
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.js +41 -3
- package/dist/src/eval-workflows/evaluation-pipeline.js.map +1 -1
- package/dist/src/eval-workflows/run-evaluation.d.ts +17 -2
- package/dist/src/eval-workflows/run-evaluation.d.ts.map +1 -1
- package/dist/src/eval-workflows/run-evaluation.js +97 -5
- package/dist/src/eval-workflows/run-evaluation.js.map +1 -1
- package/dist/src/grading/assertions.d.ts +16 -0
- package/dist/src/grading/assertions.d.ts.map +1 -1
- package/dist/src/grading/assertions.js +385 -111
- package/dist/src/grading/assertions.js.map +1 -1
- package/dist/src/grading/debias-validate.d.ts +84 -0
- package/dist/src/grading/debias-validate.d.ts.map +1 -0
- package/dist/src/grading/debias-validate.js +173 -0
- package/dist/src/grading/debias-validate.js.map +1 -0
- package/dist/src/grading/gold-cli.d.ts +88 -0
- package/dist/src/grading/gold-cli.d.ts.map +1 -0
- package/dist/src/grading/gold-cli.js +251 -0
- package/dist/src/grading/gold-cli.js.map +1 -0
- package/dist/src/grading/gold-dataset.d.ts +73 -0
- package/dist/src/grading/gold-dataset.d.ts.map +1 -0
- package/dist/src/grading/gold-dataset.js +161 -0
- package/dist/src/grading/gold-dataset.js.map +1 -0
- package/dist/src/grading/human-gold.d.ts +102 -0
- package/dist/src/grading/human-gold.d.ts.map +1 -0
- package/dist/src/grading/human-gold.js +188 -0
- package/dist/src/grading/human-gold.js.map +1 -0
- package/dist/src/grading/index.d.ts +27 -2
- package/dist/src/grading/index.d.ts.map +1 -1
- package/dist/src/grading/index.js +36 -18
- package/dist/src/grading/index.js.map +1 -1
- package/dist/src/grading/judge.d.ts +65 -2
- package/dist/src/grading/judge.d.ts.map +1 -1
- package/dist/src/grading/judge.js +280 -23
- package/dist/src/grading/judge.js.map +1 -1
- package/dist/src/inputs/eval-config.js +19 -0
- package/dist/src/inputs/eval-config.js.map +1 -1
- package/dist/src/renderer/html-renderer.d.ts.map +1 -1
- package/dist/src/renderer/html-renderer.js +16 -4
- package/dist/src/renderer/html-renderer.js.map +1 -1
- package/dist/src/renderer/layout.d.ts.map +1 -1
- package/dist/src/renderer/layout.js +32 -4
- package/dist/src/renderer/layout.js.map +1 -1
- package/dist/src/renderer/summary.d.ts +28 -1
- package/dist/src/renderer/summary.d.ts.map +1 -1
- package/dist/src/renderer/summary.js +321 -7
- package/dist/src/renderer/summary.js.map +1 -1
- package/dist/src/renderer/table.d.ts.map +1 -1
- package/dist/src/renderer/table.js +63 -2
- package/dist/src/renderer/table.js.map +1 -1
- package/dist/src/types.d.ts +241 -0
- package/dist/src/types.d.ts.map +1 -1
- package/package.json +15 -4
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sample-diagnostics.js","sourceRoot":"","sources":["../../../src/analysis/sample-diagnostics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAGH,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAoDlD,MAAM,QAAQ,GAA+C;IAC3D,cAAc,EAAE,GAAG;IACnB,eAAe,EAAE,GAAG;IACpB,YAAY,EAAE,CAAC;IACf,eAAe,EAAE,CAAC;IAClB,aAAa,EAAE,GAAG;CACnB,CAAC;AAEF,MAAM,aAAa,GAA4C,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AAEjG,MAAM,UAAU,eAAe,CAAC,MAAc,EAAE,UAA2B,EAAE;IAC3E,MAAM,GAAG,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,OAAO,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC7C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IAErC,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC;IAED,wEAAwE;IACxE,0CAA0C;IAC1C,MAAM,WAAW,GAOZ,EAAE,CAAC;IAER,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9B,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,IAAI,OAAO,CAAC,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,CAAC,cAAc,GAAG,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;YAChG,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;YACvB,SAAS,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;YACtD,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK;gBAAE,MAAM,EAAE,CAAC;YAC7B,IAAI,OAAO,CAAC,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,CAAC,cAAc,GAAG,CAAC;gBAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;QACxG,CAAC;QACD,WAAW,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QAE3E,2DAA2D;QAC3D,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YAChC,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBAC3B,MAAM,CAAC,IAAI,CAAC;oBACV,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU;oBAC9D,OAAO,EAAE,kCAAkC;oBAC3C,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACjD,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC;oBACV,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU;oBAC/D,OAAO,EAAE,8CAA8C;oBACvD,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACjD,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;gBACzC,MAAM,CAAC,IAAI,CAAC;oBACV,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa;oBACpE,OAAO,EAAE,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,aAAa,mBAAmB;oBAC/E,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE;iBACzF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,qFAAqF;QACrF,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,aAAa;gBAC3G,OAAO,EAAE,GAAG,MAAM,IAAI,QAAQ,CAAC,MAAM,+BAA+B;gBACpE,QAAQ,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE;aAChE,CAAC,CAAC;QACL,CAAC;QAED,8EAA8E;QAC9E,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,IAAI,SAAS,IAAI,GAAG,CAAC,eAAe,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC;gBACV,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,kBAAkB;gBACzE,OAAO,EAAE,qBAAqB,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,eAAe,kBAAkB;gBAC7F,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;aAC9G,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yDAAyD;IACzD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxF,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjG,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7F,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,IAAI,UAAU,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,YAAY,GAAG,UAAU,EAAE,CAAC;YAC9D,MAAM,CAAC,IAAI,CAAC;gBACV,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc;gBACpE,OAAO,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,YAAY,UAAU,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;gBAC3F,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE;aACzF,CAAC,CAAC;QACL,CAAC;QACD,IAAI,aAAa,GAAG,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC,eAAe,GAAG,aAAa,EAAE,CAAC;YAC5E,MAAM,CAAC,IAAI,CAAC;gBACV,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB;gBACvE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,eAAe,SAAS,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;gBACxH,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE;aAC9D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,wEAAwE;IACxE,wEAAwE;IACxE,mDAAmD;IACnD,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACnF,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC9C,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO;YAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACjE,MAAM,OAAO,GAAwC,EAAE,CAAC;QACxD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAChD,IAAI,MAAM,EAAE,MAAM;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5C,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC1D,IAAI,KAAK,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;oBAChC,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC5D,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,SAAS;oBAChC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAClB,MAAM,CAAC,IAAI,CAAC;wBACV,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB;wBACrE,OAAO,EAAE,kBAAkB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,cAAc,SAAS,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ;wBACjG,QAAQ,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE;qBAC3E,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAEvH,MAAM,MAAM,GAAqC,EAAE,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;YAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,MAAM,GAAG;QACb,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,OAAO,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACrD,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM;QAC3D,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,MAAM;QAC/D,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM;KAC1D,CAAC;IAEF,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC;IAChF,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E,6DAA6D;IAC7D,yDAAyD;IACzD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAElF,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AACjD,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO;QACL,MAAM,EAAE,EAAE;QACV,WAAW,EAAE,GAAG;QAChB,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;KAClE,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,KAAkB,EAAE,QAAkB;IACvD,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;YAC9C,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAA4B,EAAE,UAA6B,EAAE;IACnG,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,2BAA2B,IAAI,CAAC,WAAW,MAAM,CAAC,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,OAAO,cAAc,IAAI,CAAC,MAAM,CAAC,OAAO,YAAY,IAAI,CAAC,MAAM,CAAC,MAAM,cAAc,IAAI,CAAC,MAAM,CAAC,QAAQ,WAAW,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC;IAC3K,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,iCAAiC;IACjC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAsB,CAAC;IAC5D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC/B,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,MAAM,YAAY,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC1D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YACxF,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,IAAI,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,cAAc,QAAQ,CAAC,MAAM,GAAG,IAAI,kBAAkB,CAAC,CAAC;QACrE,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,iDAAiD;IACjD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC;QACpF,KAAK,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACtE,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC;QAC1F,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;IACxD,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Saturation curve analysis — answers "have I run enough samples?"
|
|
3
|
+
*
|
|
4
|
+
* Why this exists
|
|
5
|
+
* ---------------
|
|
6
|
+
* The most common omk-user question after "is the difference significant?"
|
|
7
|
+
* is "is N=30 enough, or should I keep running?". Without a principled
|
|
8
|
+
* answer, users either over-pay (running 200 when 50 was enough) or
|
|
9
|
+
* under-pay (calling skill effects null at N=30 when N=80 would have
|
|
10
|
+
* shown clearly significant — Bootstrap CI from Phase 1 just hasn't
|
|
11
|
+
* converged yet).
|
|
12
|
+
*
|
|
13
|
+
* Saturation analysis fits the right tool to the question: track a
|
|
14
|
+
* convergence-of-evidence metric across cumulative sample counts; when
|
|
15
|
+
* the metric stops moving meaningfully, you've saturated. Three metric
|
|
16
|
+
* choices are exposed because each has a failure mode:
|
|
17
|
+
*
|
|
18
|
+
* - **slope** (simplest): mean's rate of change between consecutive
|
|
19
|
+
* checkpoints. Easy to explain. Fragile to outlier samples.
|
|
20
|
+
* - **bootstrap-ci-width** (default): CI shrinks as O(1/√N); when its
|
|
21
|
+
* decay rate flattens, more samples buy little. Statistically
|
|
22
|
+
* grounded; pairs naturally with Phase 1.
|
|
23
|
+
* - **plateau-height**: range of mean across the last K checkpoints.
|
|
24
|
+
* Conservative — slow to declare saturation, hard to fool.
|
|
25
|
+
*
|
|
26
|
+
* The function never says "saturated" off a single observation: a 3-
|
|
27
|
+
* window run-of-success is required. This guards against random dips
|
|
28
|
+
* that look like convergence but aren't.
|
|
29
|
+
*/
|
|
30
|
+
export type SaturationMethod = 'slope' | 'bootstrap-ci-width' | 'plateau-height';
|
|
31
|
+
/**
|
|
32
|
+
* One observation in a saturation curve. `n` is the cumulative sample
|
|
33
|
+
* count at this checkpoint; `mean` and (optional) `ciWidth` come from
|
|
34
|
+
* applying the chosen metric to all samples up to and including `n`.
|
|
35
|
+
*/
|
|
36
|
+
export interface SaturationCheckpoint {
|
|
37
|
+
n: number;
|
|
38
|
+
mean: number;
|
|
39
|
+
ciWidth?: number;
|
|
40
|
+
}
|
|
41
|
+
export interface SaturationResult {
|
|
42
|
+
/** True only after threshold satisfied for `windowSize` consecutive transitions. */
|
|
43
|
+
saturated: boolean;
|
|
44
|
+
/** Sample count at which saturation was first declared. null if not saturated. */
|
|
45
|
+
atN: number | null;
|
|
46
|
+
/** 'high' for ≥ 50 cumulative samples; 'medium' for [20, 50); 'low' below 20. */
|
|
47
|
+
confidence: 'high' | 'medium' | 'low';
|
|
48
|
+
/** Method that produced the verdict. */
|
|
49
|
+
method: SaturationMethod;
|
|
50
|
+
/** Threshold used. */
|
|
51
|
+
threshold: number;
|
|
52
|
+
/** Per-checkpoint metric trace, useful for plotting. */
|
|
53
|
+
trace: Array<{
|
|
54
|
+
n: number;
|
|
55
|
+
metric: number;
|
|
56
|
+
}>;
|
|
57
|
+
/** Human-readable explanation of why we said yes / no. */
|
|
58
|
+
reason: string;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Compute saturation from a sequence of cumulative score arrays.
|
|
62
|
+
*
|
|
63
|
+
* `cumulativeScores[i]` is the array of all scores observed by checkpoint i
|
|
64
|
+
* (the i-th element is the LATEST cumulative slice; arrays grow). Each slice
|
|
65
|
+
* is fed to bootstrapMeanCI to get a (mean, CI) pair. The metric trajectory
|
|
66
|
+
* is then evaluated by the chosen method.
|
|
67
|
+
*
|
|
68
|
+
* @param cumulativeScores Cumulative score arrays in chronological order.
|
|
69
|
+
* @param method Detection algorithm (default 'bootstrap-ci-width').
|
|
70
|
+
* @param threshold Method-specific cutoff (defaults match the method).
|
|
71
|
+
* @param windowSize Number of consecutive transitions that must all
|
|
72
|
+
* satisfy the threshold (default 3).
|
|
73
|
+
* @param bootstrapSamples Used only by 'bootstrap-ci-width' (default 1000).
|
|
74
|
+
* @param seed Optional seed for reproducible CIs.
|
|
75
|
+
*/
|
|
76
|
+
export declare function findSaturationPoint(cumulativeScores: number[][], method?: SaturationMethod, threshold?: number, windowSize?: number, bootstrapSamples?: number, seed?: number): SaturationResult;
|
|
77
|
+
/**
|
|
78
|
+
* Build cumulative score arrays from a per-run flat list. `runs[i]` is the
|
|
79
|
+
* array of per-sample scores from the i-th repeat. Returns an array where
|
|
80
|
+
* the j-th entry contains all scores from runs 0..j (concatenated).
|
|
81
|
+
*
|
|
82
|
+
* Use this when feeding `runMultiple` results to `findSaturationPoint`.
|
|
83
|
+
*/
|
|
84
|
+
export declare function buildCumulativeScores(runs: number[][]): number[][];
|
|
85
|
+
//# sourceMappingURL=saturation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"saturation.d.ts","sourceRoot":"","sources":["../../../src/analysis/saturation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAIH,MAAM,MAAM,gBAAgB,GAAG,OAAO,GAAG,oBAAoB,GAAG,gBAAgB,CAAC;AAEjF;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACnC,CAAC,EAAE,MAAM,CAAC;IACV,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,oFAAoF;IACpF,SAAS,EAAE,OAAO,CAAC;IACnB,kFAAkF;IAClF,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,iFAAiF;IACjF,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,wCAAwC;IACxC,MAAM,EAAE,gBAAgB,CAAC;IACzB,sBAAsB;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,KAAK,EAAE,KAAK,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5C,0DAA0D;IAC1D,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,mBAAmB,CACjC,gBAAgB,EAAE,MAAM,EAAE,EAAE,EAC5B,MAAM,GAAE,gBAAuC,EAC/C,SAAS,CAAC,EAAE,MAAM,EAClB,UAAU,SAAI,EACd,gBAAgB,SAAO,EACvB,IAAI,CAAC,EAAE,MAAM,GACZ,gBAAgB,CA4GlB;AAUD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,EAAE,CAQlE"}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Saturation curve analysis — answers "have I run enough samples?"
|
|
3
|
+
*
|
|
4
|
+
* Why this exists
|
|
5
|
+
* ---------------
|
|
6
|
+
* The most common omk-user question after "is the difference significant?"
|
|
7
|
+
* is "is N=30 enough, or should I keep running?". Without a principled
|
|
8
|
+
* answer, users either over-pay (running 200 when 50 was enough) or
|
|
9
|
+
* under-pay (calling skill effects null at N=30 when N=80 would have
|
|
10
|
+
* shown clearly significant — Bootstrap CI from Phase 1 just hasn't
|
|
11
|
+
* converged yet).
|
|
12
|
+
*
|
|
13
|
+
* Saturation analysis fits the right tool to the question: track a
|
|
14
|
+
* convergence-of-evidence metric across cumulative sample counts; when
|
|
15
|
+
* the metric stops moving meaningfully, you've saturated. Three metric
|
|
16
|
+
* choices are exposed because each has a failure mode:
|
|
17
|
+
*
|
|
18
|
+
* - **slope** (simplest): mean's rate of change between consecutive
|
|
19
|
+
* checkpoints. Easy to explain. Fragile to outlier samples.
|
|
20
|
+
* - **bootstrap-ci-width** (default): CI shrinks as O(1/√N); when its
|
|
21
|
+
* decay rate flattens, more samples buy little. Statistically
|
|
22
|
+
* grounded; pairs naturally with Phase 1.
|
|
23
|
+
* - **plateau-height**: range of mean across the last K checkpoints.
|
|
24
|
+
* Conservative — slow to declare saturation, hard to fool.
|
|
25
|
+
*
|
|
26
|
+
* The function never says "saturated" off a single observation: a 3-
|
|
27
|
+
* window run-of-success is required. This guards against random dips
|
|
28
|
+
* that look like convergence but aren't.
|
|
29
|
+
*/
|
|
30
|
+
import { bootstrapMeanCI } from '../eval-core/bootstrap.js';
|
|
31
|
+
/**
|
|
32
|
+
* Compute saturation from a sequence of cumulative score arrays.
|
|
33
|
+
*
|
|
34
|
+
* `cumulativeScores[i]` is the array of all scores observed by checkpoint i
|
|
35
|
+
* (the i-th element is the LATEST cumulative slice; arrays grow). Each slice
|
|
36
|
+
* is fed to bootstrapMeanCI to get a (mean, CI) pair. The metric trajectory
|
|
37
|
+
* is then evaluated by the chosen method.
|
|
38
|
+
*
|
|
39
|
+
* @param cumulativeScores Cumulative score arrays in chronological order.
|
|
40
|
+
* @param method Detection algorithm (default 'bootstrap-ci-width').
|
|
41
|
+
* @param threshold Method-specific cutoff (defaults match the method).
|
|
42
|
+
* @param windowSize Number of consecutive transitions that must all
|
|
43
|
+
* satisfy the threshold (default 3).
|
|
44
|
+
* @param bootstrapSamples Used only by 'bootstrap-ci-width' (default 1000).
|
|
45
|
+
* @param seed Optional seed for reproducible CIs.
|
|
46
|
+
*/
|
|
47
|
+
export function findSaturationPoint(cumulativeScores, method = 'bootstrap-ci-width', threshold, windowSize = 3, bootstrapSamples = 1000, seed) {
|
|
48
|
+
const trace = [];
|
|
49
|
+
const checkpoints = [];
|
|
50
|
+
for (const scores of cumulativeScores) {
|
|
51
|
+
if (scores.length === 0)
|
|
52
|
+
continue;
|
|
53
|
+
const ci = bootstrapMeanCI(scores, 0.05, bootstrapSamples, seed);
|
|
54
|
+
checkpoints.push({
|
|
55
|
+
n: scores.length,
|
|
56
|
+
mean: ci.estimate,
|
|
57
|
+
ciWidth: ci.high - ci.low,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
// Default thresholds chosen for typical Likert-1-5 evaluations:
|
|
61
|
+
// - slope: |Δmean| / |ΔN| < 0.005 (5% of one rubric step over 100 samples)
|
|
62
|
+
// - bootstrap-ci-width: relative CI width shrink < 5% per checkpoint
|
|
63
|
+
// - plateau-height: max-min(mean) < 0.1 over the last 3 windows
|
|
64
|
+
const t = threshold ?? defaultThreshold(method);
|
|
65
|
+
const lastN = checkpoints.length > 0 ? checkpoints[checkpoints.length - 1].n : 0;
|
|
66
|
+
const confidence = lastN >= 50 ? 'high'
|
|
67
|
+
: lastN >= 20 ? 'medium'
|
|
68
|
+
: 'low';
|
|
69
|
+
const baseShell = (overrides) => ({
|
|
70
|
+
saturated: false,
|
|
71
|
+
atN: null,
|
|
72
|
+
confidence,
|
|
73
|
+
method,
|
|
74
|
+
threshold: t,
|
|
75
|
+
trace,
|
|
76
|
+
reason: '',
|
|
77
|
+
...overrides,
|
|
78
|
+
});
|
|
79
|
+
if (checkpoints.length < windowSize + 1) {
|
|
80
|
+
return baseShell({
|
|
81
|
+
reason: `数据点不足:有 ${checkpoints.length} 个,至少需要 ${windowSize + 1} 个`
|
|
82
|
+
+ ` (windowSize=${windowSize})`,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
// Compute per-method metric series. metric[i] is the "amount of change at
|
|
86
|
+
// transition i" — how much the metric moved from checkpoint i-1 to i.
|
|
87
|
+
// Rounded to 4 decimals to match bootstrap's precision and dodge IEEE-754
|
|
88
|
+
// surprises (e.g. 4.0 - 3.9 = 0.10000000000000009 > 0.1).
|
|
89
|
+
const round4 = (x) => Number.isFinite(x) ? Math.round(x * 10000) / 10000 : x;
|
|
90
|
+
let metricSeries;
|
|
91
|
+
let metricLabel;
|
|
92
|
+
switch (method) {
|
|
93
|
+
case 'slope': {
|
|
94
|
+
metricSeries = checkpoints.slice(1).map((cp, i) => ({
|
|
95
|
+
n: cp.n,
|
|
96
|
+
metric: round4(Math.abs(cp.mean - checkpoints[i].mean) / Math.max(1, cp.n - checkpoints[i].n)),
|
|
97
|
+
}));
|
|
98
|
+
metricLabel = 'mean 变化斜率 |Δmean/ΔN|';
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
case 'bootstrap-ci-width': {
|
|
102
|
+
metricSeries = checkpoints.slice(1).map((cp, i) => {
|
|
103
|
+
const prev = checkpoints[i].ciWidth ?? 0;
|
|
104
|
+
const curr = cp.ciWidth ?? 0;
|
|
105
|
+
// Relative shrink: (prev - curr) / prev. Negative = CI grew (still finding signal).
|
|
106
|
+
const rel = prev > 0 ? Math.abs(prev - curr) / prev : 0;
|
|
107
|
+
return { n: cp.n, metric: round4(rel) };
|
|
108
|
+
});
|
|
109
|
+
metricLabel = 'CI 宽度相对衰减率';
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
case 'plateau-height': {
|
|
113
|
+
// For each checkpoint i with i >= windowSize, compute max-min over the
|
|
114
|
+
// last windowSize+1 means. Earlier checkpoints get a Number.POSITIVE
|
|
115
|
+
// sentinel so they can never satisfy the threshold.
|
|
116
|
+
metricSeries = checkpoints.map((cp, i) => {
|
|
117
|
+
if (i < windowSize)
|
|
118
|
+
return { n: cp.n, metric: Number.POSITIVE_INFINITY };
|
|
119
|
+
const window = checkpoints.slice(i - windowSize, i + 1).map((c) => c.mean);
|
|
120
|
+
const range = Math.max(...window) - Math.min(...window);
|
|
121
|
+
return { n: cp.n, metric: round4(range) };
|
|
122
|
+
});
|
|
123
|
+
metricLabel = 'mean 极差 (最近 K 窗口)';
|
|
124
|
+
break;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
trace.push(...metricSeries);
|
|
128
|
+
// Saturated when the last `windowSize` consecutive metric values are all
|
|
129
|
+
// <= threshold. Walk forward; record the earliest qualifying checkpoint.
|
|
130
|
+
for (let i = windowSize - 1; i < metricSeries.length; i++) {
|
|
131
|
+
let allBelow = true;
|
|
132
|
+
for (let k = 0; k < windowSize; k++) {
|
|
133
|
+
if (metricSeries[i - k].metric > t) {
|
|
134
|
+
allBelow = false;
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
if (allBelow) {
|
|
139
|
+
const atN = metricSeries[i].n;
|
|
140
|
+
return baseShell({
|
|
141
|
+
saturated: true,
|
|
142
|
+
atN,
|
|
143
|
+
reason: `连续 ${windowSize} 个窗口 ${metricLabel} ≤ ${t},于 N=${atN} 判定饱和`,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return baseShell({
|
|
148
|
+
reason: `${metricLabel} 尚未满足"连续 ${windowSize} 个窗口 ≤ ${t}"`,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
function defaultThreshold(method) {
|
|
152
|
+
switch (method) {
|
|
153
|
+
case 'slope': return 0.005;
|
|
154
|
+
case 'bootstrap-ci-width': return 0.05;
|
|
155
|
+
case 'plateau-height': return 0.1;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Build cumulative score arrays from a per-run flat list. `runs[i]` is the
|
|
160
|
+
* array of per-sample scores from the i-th repeat. Returns an array where
|
|
161
|
+
* the j-th entry contains all scores from runs 0..j (concatenated).
|
|
162
|
+
*
|
|
163
|
+
* Use this when feeding `runMultiple` results to `findSaturationPoint`.
|
|
164
|
+
*/
|
|
165
|
+
export function buildCumulativeScores(runs) {
|
|
166
|
+
const acc = [];
|
|
167
|
+
const out = [];
|
|
168
|
+
for (const r of runs) {
|
|
169
|
+
acc.push(...r);
|
|
170
|
+
out.push([...acc]);
|
|
171
|
+
}
|
|
172
|
+
return out;
|
|
173
|
+
}
|
|
174
|
+
//# sourceMappingURL=saturation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"saturation.js","sourceRoot":"","sources":["../../../src/analysis/saturation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAgC5D;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,mBAAmB,CACjC,gBAA4B,EAC5B,SAA2B,oBAAoB,EAC/C,SAAkB,EAClB,UAAU,GAAG,CAAC,EACd,gBAAgB,GAAG,IAAI,EACvB,IAAa;IAEb,MAAM,KAAK,GAA8B,EAAE,CAAC;IAC5C,MAAM,WAAW,GAA2B,EAAE,CAAC;IAE/C,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACtC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAClC,MAAM,EAAE,GAAG,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;QACjE,WAAW,CAAC,IAAI,CAAC;YACf,CAAC,EAAE,MAAM,CAAC,MAAM;YAChB,IAAI,EAAE,EAAE,CAAC,QAAQ;YACjB,OAAO,EAAE,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,GAAG;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,gEAAgE;IAChE,4EAA4E;IAC5E,sEAAsE;IACtE,iEAAiE;IACjE,MAAM,CAAC,GAAG,SAAS,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjF,MAAM,UAAU,GACd,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM;QACpB,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ;YACxB,CAAC,CAAC,KAAK,CAAC;IAEV,MAAM,SAAS,GAAG,CAAC,SAAoC,EAAoB,EAAE,CAAC,CAAC;QAC7E,SAAS,EAAE,KAAK;QAChB,GAAG,EAAE,IAAI;QACT,UAAU;QACV,MAAM;QACN,SAAS,EAAE,CAAC;QACZ,KAAK;QACL,MAAM,EAAE,EAAE;QACV,GAAG,SAAS;KACb,CAAC,CAAC;IAEH,IAAI,WAAW,CAAC,MAAM,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;QACxC,OAAO,SAAS,CAAC;YACf,MAAM,EAAE,WAAW,WAAW,CAAC,MAAM,WAAW,UAAU,GAAG,CAAC,IAAI;kBAC9D,gBAAgB,UAAU,GAAG;SAClC,CAAC,CAAC;IACL,CAAC;IAED,0EAA0E;IAC1E,sEAAsE;IACtE,0EAA0E;IAC1E,0DAA0D;IAC1D,MAAM,MAAM,GAAG,CAAC,CAAS,EAAU,EAAE,CACnC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,IAAI,YAAkD,CAAC;IACvD,IAAI,WAAmB,CAAC;IACxB,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClD,CAAC,EAAE,EAAE,CAAC,CAAC;gBACP,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAC/F,CAAC,CAAC,CAAC;YACJ,WAAW,GAAG,sBAAsB,CAAC;YACrC,MAAM;QACR,CAAC;QACD,KAAK,oBAAoB,CAAC,CAAC,CAAC;YAC1B,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE;gBAChD,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;gBACzC,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,IAAI,CAAC,CAAC;gBAC7B,oFAAoF;gBACpF,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxD,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1C,CAAC,CAAC,CAAC;YACH,WAAW,GAAG,YAAY,CAAC;YAC3B,MAAM;QACR,CAAC;QACD,KAAK,gBAAgB,CAAC,CAAC,CAAC;YACtB,uEAAuE;YACvE,qEAAqE;YACrE,oDAAoD;YACpD,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvC,IAAI,CAAC,GAAG,UAAU;oBAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBACzE,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC3E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;gBACxD,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5C,CAAC,CAAC,CAAC;YACH,WAAW,GAAG,mBAAmB,CAAC;YAClC,MAAM;QACR,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAE5B,yEAAyE;IACzE,yEAAyE;IACzE,KAAK,IAAI,CAAC,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1D,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAAC,QAAQ,GAAG,KAAK,CAAC;gBAAC,MAAM;YAAC,CAAC;QAClE,CAAC;QACD,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9B,OAAO,SAAS,CAAC;gBACf,SAAS,EAAE,IAAI;gBACf,GAAG;gBACH,MAAM,EAAE,MAAM,UAAU,QAAQ,WAAW,MAAM,CAAC,QAAQ,GAAG,OAAO;aACrE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;QACf,MAAM,EAAE,GAAG,WAAW,YAAY,UAAU,UAAU,CAAC,GAAG;KAC3D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAwB;IAChD,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,OAAO,CAAC,CAAC,OAAO,KAAK,CAAC;QAC3B,KAAK,oBAAoB,CAAC,CAAC,OAAO,IAAI,CAAC;QACvC,KAAK,gBAAgB,CAAC,CAAC,OAAO,GAAG,CAAC;IACpC,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAgB;IACpD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACf,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|