@devflow-tools/benchmark 0.17.8 → 0.18.0

package/dist/graders/deterministic.js

@@ -153,10 +153,12 @@ function memoryOutcomeGrader() {
             if (!isRecord(outcome))
                 return result('memory_outcome', 'unavailable', 'Memory outcome is unavailable.', startedAt, []);
             const selected = Number(outcome.selected ?? 0);
-            const adopted = Number(outcome.adopted ?? 0);
+            const consumed = Number(outcome.consumed ?? 0);
+            const verified = Number(outcome.verified ?? 0);
             const status = outcome.status;
-            const valid = (status === 'evaluated' || status === 'positive') && selected >= 0 && adopted >= 0;
-            return result('memory_outcome', valid ? 'passed' : 'failed', valid ? `Memory outcome evaluated (${adopted}/${selected} adopted).` : 'Memory outcome was not verified.', startedAt, [{ type: 'memory_outcome', value: JSON.stringify({ status, selected, adopted }) }], valid ? 1 : 0, 1);
+            const valid = ['positive_outcome', 'negative_outcome', 'unknown_outcome'].includes(String(status))
+                && selected >= consumed && consumed >= verified && verified >= 0;
+            return result('memory_outcome', valid ? 'passed' : 'failed', valid ? `Memory outcome evaluated (${consumed}/${selected} consumed, ${verified} verified).` : 'Memory outcome was not verified.', startedAt, [{ type: 'memory_outcome', value: JSON.stringify({ status, selected, consumed, verified }) }], valid ? 1 : 0, 1);
         },
     };
 }

package/dist/graders/deterministic.js.map

@@ -1 +1 @@
-{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../src/graders/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,MAAM,wBAAwB,GAAG,EAAE,GAAG,IAAI,CAAC;AAC3C,MAAM,0BAA0B,GAAG,MAAM,CAAC;AAE1C,MAAM,CAAC,MAAM,qBAAqB,GAAuB;IACvD,qBAAqB,CAAC,mBAAmB,EAAE,iBAAiB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAC3F,qBAAqB,CAAC,qBAAqB,EAAE,mBAAmB,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACxF,qBAAqB,CAAC,gBAAgB,EAAE,oBAAoB,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACpF,aAAa,EAAE;IACf,qBAAqB,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,CAAC,WAAW,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IACvG,qBAAqB,CAAC,mBAAmB,EAAE,aAAa,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC9E,qBAAqB,CAAC,uBAAuB,EAAE,mBAAmB,EAAE,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAC9F,mBAAmB,EAAE;IACrB,qBAAqB,CAAC,mBAAmB,EAAE,kBAAkB,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACvF,mBAAmB,EAAE;CACtB,CAAC;AAEF,MAAM,UAAU,uBAAuB;IACrC,OAAO;QACL,EAAE,EAAE,qBAAqB;QACzB,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC;QACnD,gBAAgB,EAAE,CAAC,SAAS,CAAC;QAC7B,SAAS,EAAE,0BAA0B;QACrC,IAAI,EAAE,KAAK;QACX,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,KAAK,EAAC,OAAO,EAAC,EAAE;YACrB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;YACzC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnF,OAAO,MAAM,CAAC,qBAAqB,EAAE,aAAa,EAAE,6CAA6C,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YACpH,CAAC;YACD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;gBACtC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;gBACzE,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,EAAE,0BAA0B,EAAE,0BAA0B,CAAC,CAAC;YAChH,MAAM,cAAc,GAAG,aAAa,CAAC,OAAO,CAAC,cAAc,EAAE,KAAK,EAAE,wBAAwB,EAAE,wBAAwB,CAAC,CAAC;YACxH,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE;gBAC5D,GAAG,EAAE,OAAO,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW;gBACxE,SAAS;gBACT,cAAc;aACf,CAAC,CAAC;YACH,OAAO,MAAM,CACX,qBAAqB,EACrB,SAAS,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EACrE,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,4BAA4B,SAAS,CAAC,QAAQ,GAAG,EAC7F,SAAS,EACT;gBACE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBAC7D,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,UAAU,EAAE;aACrD,EACD,SAAS,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,IAAc,EACd,OAAqE;IAErE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC;IAClE,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;IAC1E,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;QAC3B,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE;YACjC,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,cAAc,GAAG,CAAC;YAC7B,QAAQ,EAAE,QAAQ;SACnB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;YACrB,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;YAC7D,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;YAC7D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;YACpE,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,OAAQ,KAAsC,EAAE,IAAI,KAAK,QAAQ;gBAC5E,CAAC,CAAC,MAAM,CAAE,KAA+B,CAAC,IAAI,CAAC;gBAC/C,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClB,OAAO,CAAC;gBACN,QAAQ,EAAE,IAAI;gBACd,QAAQ;gBACR,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC9D,SAAS,EAAE,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;aAC5C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS;gBAAE,QAAQ,GAAG,IAAI,CAAC;QACpE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB,CAAC,EAAU,EAAE,GAAW,EAAE,SAAmB;IACzE,OAAO;QACL,EAAE;QACF,IAAI,EAAE,eAAe;QACrB,SAAS;QACT,gBAAgB,EAAE,CAAC,GAAG,CAAC;QACvB,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,EAAE;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC/B,OAAO,MAAM,CAAC,EAAE,EAAE,aAAa,EAAE,qBAAqB,GAAG,kBAAkB,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YAC9F,CAAC;YACD,OAAO,MAAM,CACX,EAAE,EACF,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAC3B,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,EAC3C,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EACrC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACb,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,aAAa;IACpB,OAAO;QACL,EAAE,EAAE,kBAAkB;QACtB,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC;QACpD,gBAAgB,EAAE,CAAC,SAAS,CAAC;QAC7B,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,EAAE;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;YACzC,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,OAAO,MAAM,CAAC,kBAAkB,EAAE,aAAa,EAAE,yBAAyB,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YAC5H,MAAM,KAAK,GAAG,yCAAyC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtE,OAAO,MAAM,CACX,kBAAkB,EAClB,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAC3B,KAAK,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,2BAA2B,EAC/D,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EACrF,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACb,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB;IAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,0BAA0B,CAAC,CAAC,CAAC;IAC7H,OAAO;QACL,EAAE,EAAE,0BAA0B;QAC9B,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC;QAC7C,gBAAgB,EAAE,CAAC,gBAAgB,CAAC;QACpC,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,EAAE;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAO,MAAM,CAAC,0BAA0B,EAAE,aAAa,EAAE,iCAAiC,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YAC1I,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAClC,OAAO,MAAM,CACX,0BAA0B,EAC1B,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAC3B,KAAK,CAAC,CAAC,CAAC,qBAAqB,KAAK,GAAG,CAAC,CAAC,CAAC,qCAAqC,KAAK,GAAG,EACrF,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAC3C,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACb,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB;IAC1B,OAAO;QACL,EAAE,EAAE,gBAAgB;QACpB,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC;QAChC,gBAAgB,EAAE,CAAC,eAAe,CAAC;QACnC,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,EAAE;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC/C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,OAAO,MAAM,CAAC,gBAAgB,EAAE,aAAa,EAAE,gCAAgC,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YACxH,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;YAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,MAAM,KAAK,GAAG,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,KAAK,UAAU,CAAC,IAAI,QAAQ,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,CAAC;YACjG,OAAO,MAAM,CACX,gBAAgB,EAChB,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAC3B,KAAK,CAAC,CAAC,CAAC,6BAA6B,OAAO,IAAI,QAAQ,YAAY,CAAC,CAAC,CAAC,kCAAkC,EACzG,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,EAClF,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACb,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,MAAM,CACpB,QAAgB,EAChB,MAA8B,EAC9B,MAAc,EACd,SAAiB,EACjB,QAAkC,EAClC,KAAc,EACd,SAAkB;IAElB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;IACvD,MAAM,OAAO,GAAG,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;QACnE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS;KACrD,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IACjC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;AAC9G,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,OAAe,EAAE,OAAe,EAAE,QAAgB;IACvF,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QACxD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC,QAAQ,CAAC;AACf,CAAC"}
+{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../src/graders/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,MAAM,wBAAwB,GAAG,EAAE,GAAG,IAAI,CAAC;AAC3C,MAAM,0BAA0B,GAAG,MAAM,CAAC;AAE1C,MAAM,CAAC,MAAM,qBAAqB,GAAuB;IACvD,qBAAqB,CAAC,mBAAmB,EAAE,iBAAiB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAC3F,qBAAqB,CAAC,qBAAqB,EAAE,mBAAmB,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACxF,qBAAqB,CAAC,gBAAgB,EAAE,oBAAoB,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACpF,aAAa,EAAE;IACf,qBAAqB,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,CAAC,WAAW,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IACvG,qBAAqB,CAAC,mBAAmB,EAAE,aAAa,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC9E,qBAAqB,CAAC,uBAAuB,EAAE,mBAAmB,EAAE,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAC9F,mBAAmB,EAAE;IACrB,qBAAqB,CAAC,mBAAmB,EAAE,kBAAkB,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACvF,mBAAmB,EAAE;CACtB,CAAC;AAEF,MAAM,UAAU,uBAAuB;IACrC,OAAO;QACL,EAAE,EAAE,qBAAqB;QACzB,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC;QACnD,gBAAgB,EAAE,CAAC,SAAS,CAAC;QAC7B,SAAS,EAAE,0BAA0B;QACrC,IAAI,EAAE,KAAK;QACX,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,KAAK,EAAC,OAAO,EAAC,EAAE;YACrB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;YACzC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnF,OAAO,MAAM,CAAC,qBAAqB,EAAE,aAAa,EAAE,6CAA6C,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YACpH,CAAC;YACD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;gBACtC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;gBACzE,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,EAAE,0BAA0B,EAAE,0BAA0B,CAAC,CAAC;YAChH,MAAM,cAAc,GAAG,aAAa,CAAC,OAAO,CAAC,cAAc,EAAE,KAAK,EAAE,wBAAwB,EAAE,wBAAwB,CAAC,CAAC;YACxH,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE;gBAC5D,GAAG,EAAE,OAAO,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW;gBACxE,SAAS;gBACT,cAAc;aACf,CAAC,CAAC;YACH,OAAO,MAAM,CACX,qBAAqB,EACrB,SAAS,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EACrE,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,4BAA4B,SAAS,CAAC,QAAQ,GAAG,EAC7F,SAAS,EACT;gBACE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBAC7D,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,UAAU,EAAE;aACrD,EACD,SAAS,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,IAAc,EACd,OAAqE;IAErE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC;IAClE,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;IAC1E,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;QAC3B,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE;YACjC,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,cAAc,GAAG,CAAC;YAC7B,QAAQ,EAAE,QAAQ;SACnB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;YACrB,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;YAC7D,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;YAC7D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;YACpE,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,OAAQ,KAAsC,EAAE,IAAI,KAAK,QAAQ;gBAC5E,CAAC,CAAC,MAAM,CAAE,KAA+B,CAAC,IAAI,CAAC;gBAC/C,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClB,OAAO,CAAC;gBACN,QAAQ,EAAE,IAAI;gBACd,QAAQ;gBACR,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC9D,SAAS,EAAE,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;aAC5C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS;gBAAE,QAAQ,GAAG,IAAI,CAAC;QACpE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB,CAAC,EAAU,EAAE,GAAW,EAAE,SAAmB;IACzE,OAAO;QACL,EAAE;QACF,IAAI,EAAE,eAAe;QACrB,SAAS;QACT,gBAAgB,EAAE,CAAC,GAAG,CAAC;QACvB,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,EAAE;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC/B,OAAO,MAAM,CAAC,EAAE,EAAE,aAAa,EAAE,qBAAqB,GAAG,kBAAkB,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YAC9F,CAAC;YACD,OAAO,MAAM,CACX,EAAE,EACF,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAC3B,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,EAC3C,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EACrC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACb,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,aAAa;IACpB,OAAO;QACL,EAAE,EAAE,kBAAkB;QACtB,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC;QACpD,gBAAgB,EAAE,CAAC,SAAS,CAAC;QAC7B,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,EAAE;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;YACzC,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,OAAO,MAAM,CAAC,kBAAkB,EAAE,aAAa,EAAE,yBAAyB,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YAC5H,MAAM,KAAK,GAAG,yCAAyC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtE,OAAO,MAAM,CACX,kBAAkB,EAClB,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAC3B,KAAK,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,2BAA2B,EAC/D,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EACrF,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACb,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB;IAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,0BAA0B,CAAC,CAAC,CAAC;IAC7H,OAAO;QACL,EAAE,EAAE,0BAA0B;QAC9B,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC;QAC7C,gBAAgB,EAAE,CAAC,gBAAgB,CAAC;QACpC,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,EAAE;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAO,MAAM,CAAC,0BAA0B,EAAE,aAAa,EAAE,iCAAiC,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YAC1I,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAClC,OAAO,MAAM,CACX,0BAA0B,EAC1B,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAC3B,KAAK,CAAC,CAAC,CAAC,qBAAqB,KAAK,GAAG,CAAC,CAAC,CAAC,qCAAqC,KAAK,GAAG,EACrF,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAC3C,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACb,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB;IAC1B,OAAO;QACL,EAAE,EAAE,gBAAgB;QACpB,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC;QAChC,gBAAgB,EAAE,CAAC,eAAe,CAAC;QACnC,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,EAAE;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC/C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,OAAO,MAAM,CAAC,gBAAgB,EAAE,aAAa,EAAE,gCAAgC,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YACxH,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;YAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,MAAM,KAAK,GAAG,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;mBAC7F,QAAQ,IAAI,QAAQ,IAAI,QAAQ,IAAI,QAAQ,IAAI,QAAQ,IAAI,CAAC,CAAC;YACnE,OAAO,MAAM,CACX,gBAAgB,EAChB,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAC3B,KAAK,CAAC,CAAC,CAAC,6BAA6B,QAAQ,IAAI,QAAQ,cAAc,QAAQ,aAAa,CAAC,CAAC,CAAC,kCAAkC,EACjI,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,EAC7F,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACb,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,MAAM,CACpB,QAAgB,EAChB,MAA8B,EAC9B,MAAc,EACd,SAAiB,EACjB,QAAkC,EAClC,KAAc,EACd,SAAkB;IAElB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;IACvD,MAAM,OAAO,GAAG,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;QACnE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS;KACrD,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IACjC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;AAC9G,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,OAAe,EAAE,OAAe,EAAE,QAAgB;IACvF,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QACxD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC,QAAQ,CAAC;AACf,CAAC"}

package/dist/index.d.ts

@@ -16,12 +16,25 @@ export type { RetrievalQualityCase, RetrievalQualityFixture, RetrievalQualityRep
 export { runRetrievalQualityCorpus } from './retrieval-quality-corpus.js';
 export type { EvidenceRuntimeCorpusFixture, EvidenceRuntimeQualityReport, QualityCorpusReport, RetrievalQualityCorpusFixture, RetrievalQualityCorpusProject, RetrievalQualityCorpusReport, } from './retrieval-quality-corpus.js';
 export { runEvidenceRuntimeQualityCorpus } from './retrieval-quality-corpus.js';
-export { evidenceQualityMetric } from './evidence-quality-metrics.js';
-export type { EvidenceQualityMetadata, EvidenceQualityMetric, WilsonInterval95, } from './evidence-quality-metrics.js';
+export * from './evidence-quality-metrics.js';
+export * from './release-quality-comparison.js';
+export * from './release-quality-policy.js';
 export * from './graders/index.js';
 export { aggregateRepeatedTrials, createTrialReceipt, renderRepeatedTrialReport, } from './repeated-trials.js';
 export type { RepeatedTrial, RepeatedTrialReport, RepeatedTrialTaskReport, } from './repeated-trials.js';
 export { evaluateSemanticFixtures } from './semantic-intent-quality.js';
 export { evaluateSemanticCalibration, evaluateSemanticReleaseGates, wilsonInterval95 } from './semantic-intent-quality.js';
 export type { SemanticIntentCorpus, SemanticIntentExpectedFrame, SemanticIntentFixture, SemanticMutationPermission, SemanticQualityReport, SemanticCalibrationReport, SemanticCalibrationSample, SemanticReleaseGates, } from './semantic-intent-quality.js';
+export { loadUnifiedRuntimeCorpus } from './unified-runtime-corpus.js';
+export type { UnifiedAvailability, UnifiedCaseClass, UnifiedCaseSlice, UnifiedChannel, UnifiedLanguage, UnifiedRetrievalCase, UnifiedRuntimeCorpus, UnifiedSemanticCase, UnifiedVariant, } from './unified-runtime-corpus.js';
+export { evaluateUnifiedRuntimeCorpus } from './unified-runtime-evaluation.js';
+export { runUnifiedRuntimeGold } from './unified-runtime-runner.js';
+export type * from './unified-runtime-observation.js';
+export type { EvaluationBundleEvidenceV1, UnifiedEvaluationSlice, UnifiedLatencyMetric, UnifiedRerankerMetric, UnifiedRuntimeEvaluationReport, } from './unified-runtime-evaluation.js';
+export * from './evaluation-bundle.js';
+export * from './evaluation-bundle-builder.js';
+export * from './evaluation-run-state.js';
+export * from './release-quality-bundle-materializer.js';
+export * from './release-quality-bundle-evaluator.js';
+export * from './release-quality-evaluation-command.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,YAAY,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAC3E,YAAY,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AAC1E,YAAY,EACV,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,YAAY,EACV,4BAA4B,EAC5B,4BAA4B,EAC5B,mBAAmB,EACnB,6BAA6B,EAC7B,6BAA6B,EAC7B,4BAA4B,GAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,+BAA+B,EAAE,MAAM,+BAA+B,CAAC;AAChF,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,YAAY,EACV,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,GACjB,MAAM,+BAA+B,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,aAAa,EACb,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAC3H,YAAY,EACV,oBAAoB,EACpB,2BAA2B,EAC3B,qBAAqB,EACrB,0BAA0B,EAC1B,qBAAqB,EACrB,yBAAyB,EACzB,yBAAyB,EACzB,oBAAoB,GACrB,MAAM,8BAA8B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,YAAY,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAC3E,YAAY,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AAC1E,YAAY,EACV,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,YAAY,EACV,4BAA4B,EAC5B,4BAA4B,EAC5B,mBAAmB,EACnB,6BAA6B,EAC7B,6BAA6B,EAC7B,4BAA4B,GAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,+BAA+B,EAAE,MAAM,+BAA+B,CAAC;AAChF,cAAc,+BAA+B,CAAC;AAC9C,cAAc,iCAAiC,CAAC;AAChD,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,aAAa,EACb,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAC3H,YAAY,EACV,oBAAoB,EACpB,2BAA2B,EAC3B,qBAAqB,EACrB,0BAA0B,EAC1B,qBAAqB,EACrB,yBAAyB,EACzB,yBAAyB,EACzB,oBAAoB,GACrB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AACvE,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,cAAc,GACf,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,4BAA4B,EAAE,MAAM,iCAAiC,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,mBAAmB,kCAAkC,CAAC;AACtD,YAAY,EACV,0BAA0B,EAC1B,sBAAsB,EACtB,oBAAoB,EACpB,qBAAqB,EACrB,8BAA8B,GAC/B,MAAM,iCAAiC,CAAC;AACzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,0CAA0C,CAAC;AACzD,cAAc,uCAAuC,CAAC;AACtD,cAAc,yCAAyC,CAAC"}

package/dist/index.js

@@ -8,9 +8,20 @@ export { runRetrievalQualityBenchmark } from './retrieval-quality.js';
 export { createIsolatedQualityProject } from './isolated-quality-store.js';
 export { runRetrievalQualityCorpus } from './retrieval-quality-corpus.js';
 export { runEvidenceRuntimeQualityCorpus } from './retrieval-quality-corpus.js';
-export { evidenceQualityMetric } from './evidence-quality-metrics.js';
+export * from './evidence-quality-metrics.js';
+export * from './release-quality-comparison.js';
+export * from './release-quality-policy.js';
 export * from './graders/index.js';
 export { aggregateRepeatedTrials, createTrialReceipt, renderRepeatedTrialReport, } from './repeated-trials.js';
 export { evaluateSemanticFixtures } from './semantic-intent-quality.js';
 export { evaluateSemanticCalibration, evaluateSemanticReleaseGates, wilsonInterval95 } from './semantic-intent-quality.js';
+export { loadUnifiedRuntimeCorpus } from './unified-runtime-corpus.js';
+export { evaluateUnifiedRuntimeCorpus } from './unified-runtime-evaluation.js';
+export { runUnifiedRuntimeGold } from './unified-runtime-runner.js';
+export * from './evaluation-bundle.js';
+export * from './evaluation-bundle-builder.js';
+export * from './evaluation-run-state.js';
+export * from './release-quality-bundle-materializer.js';
+export * from './release-quality-bundle-evaluator.js';
+export * from './release-quality-evaluation-command.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAQ3E,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAS1E,OAAO,EAAE,+BAA+B,EAAE,MAAM,+BAA+B,CAAC;AAChF,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAMtE,cAAc,oBAAoB,CAAC;AACnC,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAM9B,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAQ3E,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAS1E,OAAO,EAAE,+BAA+B,EAAE,MAAM,+BAA+B,CAAC;AAChF,cAAc,+BAA+B,CAAC;AAC9C,cAAc,iCAAiC,CAAC;AAChD,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAM9B,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAW3H,OAAO,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAYvE,OAAO,EAAE,4BAA4B,EAAE,MAAM,iCAAiC,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AASpE,cAAc,wBAAwB,CAAC;AACvC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,0CAA0C,CAAC;AACzD,cAAc,uCAAuC,CAAC;AACtD,cAAc,yCAAyC,CAAC"}

package/dist/release-quality-bundle-command.d.ts

@@ -0,0 +1,3 @@
+export declare function runReleaseQualityBundleCommand(argv: string[]): Promise<void>;
+export declare function removeReleaseQualityBundleOutput(path: string): void;
+//# sourceMappingURL=release-quality-bundle-command.d.ts.map

package/dist/release-quality-bundle-command.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"release-quality-bundle-command.d.ts","sourceRoot":"","sources":["../src/release-quality-bundle-command.ts"],"names":[],"mappings":"AAgCA,wBAAsB,8BAA8B,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAsClF;AAED,wBAAgB,gCAAgC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAEnE"}

package/dist/release-quality-bundle-command.js

@@ -0,0 +1,72 @@
+import { execFileSync } from 'node:child_process';
+import { existsSync, mkdirSync, realpathSync, writeFileSync, } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { stableRuntimeJson } from '@devflow-tools/sdk';
+import { loadReleaseQualityMaterializationTargets, materializeReleaseQualityBundle, } from './release-quality-bundle-materializer.js';
+import { removeReleaseQualityOutput } from './release-quality-output-cleanup.js';
+export async function runReleaseQualityBundleCommand(argv) {
+    const manifestPath = requiredOption(argv, '--manifest');
+    const sourcesRoot = requiredOption(argv, '--sources');
+    const outputRoot = resolve(requiredOption(argv, '--output'));
+    const builderCommit = requiredCommit(argv, '--builder-commit');
+    const builderIdentity = `devflow:${builderCommit}`;
+    if (existsSync(outputRoot))
+        throw new Error('RELEASE_QUALITY_OUTPUT_EXISTS');
+    mkdirSync(outputRoot, { recursive: true });
+    try {
+        const targets = loadReleaseQualityMaterializationTargets({
+            manifestPath,
+            sourcesRoot,
+            outputRoot,
+        });
+        const bundles = [];
+        for (const target of targets) {
+            const result = await materializeReleaseQualityBundle(target, builderIdentity);
+            bundles.push({
+                projectId: target.input.project.projectId,
+                datasetDirectory: target.outputPath.slice(outputRoot.length + 1),
+                bundleId: result.manifest.bundleId,
+                projectSnapshotId: result.manifest.projectSnapshotId,
+                channelSnapshotHashes: result.manifest.channelSnapshotHashes,
+                builderIdentity: result.manifest.builderIdentity,
+            });
+        }
+        const mapping = {
+            schemaVersion: 'devflow.release-quality-bundles.v1',
+            bundles,
+        };
+        const mappingPath = join(outputRoot, 'bundles.json');
+        writeFileSync(mappingPath, `${stableRuntimeJson(mapping)}\n`);
+        process.stdout.write(`${stableRuntimeJson(mapping)}\n`);
+    }
+    catch (error) {
+        removeReleaseQualityOutput(outputRoot);
+        throw error;
+    }
+}
+export function removeReleaseQualityBundleOutput(path) {
+    removeReleaseQualityOutput(path);
+}
+function requiredOption(argv, name) {
+    const index = argv.indexOf(name);
+    const value = index >= 0 ? argv[index + 1] : undefined;
+    if (!value)
+        throw new Error(`${name} requires a path`);
+    return resolve(value);
+}
+function requiredCommit(argv, name) {
+    const index = argv.indexOf(name);
+    const value = index >= 0 ? argv[index + 1] : undefined;
+    if (!value || !/^[0-9a-f]{40}$/u.test(value))
+        throw new Error(`${name} requires a Git commit`);
+    const head = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
+    if (value !== head)
+        throw new Error('RELEASE_QUALITY_BUILDER_COMMIT_MISMATCH');
+    return value;
+}
+const commandPath = process.argv[1] ? realpathSync(process.argv[1]) : '';
+if (commandPath === realpathSync(fileURLToPath(import.meta.url)) || commandPath.endsWith('/release-quality-bundle-command.ts')) {
+    await runReleaseQualityBundleCommand(process.argv.slice(2));
+}
+//# sourceMappingURL=release-quality-bundle-command.js.map

package/dist/release-quality-bundle-command.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"release-quality-bundle-command.js","sourceRoot":"","sources":["../src/release-quality-bundle-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EACL,UAAU,EACV,SAAS,EACT,YAAY,EACZ,aAAa,GACd,MAAM,SAAS,CAAC;AACjB,OAAO,EAAW,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,wCAAwC,EACxC,+BAA+B,GAChC,MAAM,0CAA0C,CAAC;AAClD,OAAO,EAAE,0BAA0B,EAAE,MAAM,qCAAqC,CAAC;AAkBjF,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAAC,IAAc;IACjE,MAAM,YAAY,GAAG,cAAc,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IACxD,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;IAC7D,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;IAC/D,MAAM,eAAe,GAAG,WAAW,aAAa,EAAE,CAAC;IACnD,IAAI,UAAU,CAAC,UAAU,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IAE7E,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,wCAAwC,CAAC;YACvD,YAAY;YACZ,WAAW;YACX,UAAU;SACX,CAAC,CAAC;QACH,MAAM,OAAO,GAA6C,EAAE,CAAC;QAC7D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,+BAA+B,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;YAC9E,OAAO,CAAC,IAAI,CAAC;gBACX,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS;gBACzC,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;gBAChE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ;gBAClC,iBAAiB,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB;gBACpD,qBAAqB,EAAE,MAAM,CAAC,QAAQ,CAAC,qBAAqB;gBAC5D,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,eAAe;aACjD,CAAC,CAAC;QACL,CAAC;QACD,MAAM,OAAO,GAAkC;YAC7C,aAAa,EAAE,oCAAoC;YACnD,OAAO;SACR,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QACrD,aAAa,CAAC,WAAW,EAAE,GAAG,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,0BAA0B,CAAC,UAAU,CAAC,CAAC;QACvC,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gCAAgC,CAAC,IAAY;IAC3D,0BAA0B,CAAC,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,cAAc,CAAC,IAAc,EAAE,IAAY;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACvD,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,kBAAkB,CAAC,CAAC;IACvD,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,cAAc,CAAC,IAAc,EAAE,IAAY;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACvD,IAAI,CAAC,KAAK,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,wBAAwB,CAAC,CAAC;IAC/F,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACrF,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC/E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACzE,IAAI,WAAW,KAAK,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE,CAAC;IAC/H,MAAM,8BAA8B,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC"}

package/dist/release-quality-bundle-evaluator.d.ts

@@ -0,0 +1,61 @@
+import { type EvidenceRerankerPort, type EvaluationChannel, type ProviderExecutionStatus } from '@devflow-tools/sdk';
+import { type AggregatedEvidenceQualityMetric, type EvidenceQualityMetric } from './evidence-quality-metrics.js';
+import { type ReleaseQualityQuerySamplingEvidence } from './release-quality-query-sampling.js';
+export interface ReleaseQualityBundleCaseResult {
+    caseId: string;
+    channel: EvaluationChannel;
+    caseClass: 'positive' | 'lifecycle';
+    expectedLifecycle?: 'explicit_save_durable' | 'latest_episode_selected';
+    candidateRefs: string[];
+    selectedRefs: string[];
+    candidateRecall: EvidenceQualityMetric;
+    recallAt5: EvidenceQualityMetric;
+    precisionAt5: EvidenceQualityMetric;
+    forbiddenContamination: EvidenceQualityMetric;
+    provenanceCompleteness: EvidenceQualityMetric;
+    sourceVersionCorrectness: EvidenceQualityMetric;
+    querySampling: ReleaseQualityQuerySamplingEvidence;
+    reranker: {
+        profileId: string;
+        profileVersion: string;
+        status: ProviderExecutionStatus;
+        reasonCodes: string[];
+    };
+    durationMs: number;
+}
+interface ReleaseQualityChannelMetrics {
+    candidateRecall: AggregatedEvidenceQualityMetric;
+    recallAt5: AggregatedEvidenceQualityMetric;
+    precisionAt5: AggregatedEvidenceQualityMetric;
+    forbiddenContamination: AggregatedEvidenceQualityMetric;
+    provenanceCompleteness: AggregatedEvidenceQualityMetric;
+}
+export interface ReleaseQualityBundleReport {
+    schemaVersion: 'devflow.release-quality-bundle-report.v1';
+    bundleId: string;
+    projectSnapshotId: string;
+    builderIdentity: string;
+    corpusHash: string;
+    labelHashes: Record<EvaluationChannel, string>;
+    channelSnapshotHashes: Record<EvaluationChannel, string>;
+    cases: ReleaseQualityBundleCaseResult[];
+    channels: Record<EvaluationChannel, ReleaseQualityChannelMetrics>;
+    memoryDurability: EvidenceQualityMetric;
+    memoryTemporalCorrectness: EvidenceQualityMetric;
+    knowledgeVersionCorrectness: EvidenceQualityMetric;
+    consumption: {
+        status: 'unavailable';
+        reasonCode: 'host_ledger_required';
+    };
+    verification: {
+        status: 'unavailable';
+        reasonCode: 'verification_receipt_required';
+    };
+    bundleUnchanged: boolean;
+}
+export declare function evaluateReleaseQualityBundle(input: {
+    bundlePath: string;
+    reranker: EvidenceRerankerPort;
+}): Promise<ReleaseQualityBundleReport>;
+export {};
+//# sourceMappingURL=release-quality-bundle-evaluator.d.ts.map

package/dist/release-quality-bundle-evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"release-quality-bundle-evaluator.d.ts","sourceRoot":"","sources":["../src/release-quality-bundle-evaluator.ts"],"names":[],"mappings":"AAIA,OAAO,EAML,KAAK,oBAAoB,EACzB,KAAK,iBAAiB,EACtB,KAAK,uBAAuB,EAC7B,MAAM,oBAAoB,CAAC;AAM5B,OAAO,EAQL,KAAK,+BAA+B,EACpC,KAAK,qBAAqB,EAC3B,MAAM,+BAA+B,CAAC;AAiBvC,OAAO,EAIL,KAAK,mCAAmC,EACzC,MAAM,qCAAqC,CAAC;AAW7C,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,iBAAiB,CAAC;IAC3B,SAAS,EAAE,UAAU,GAAG,WAAW,CAAC;IACpC,iBAAiB,CAAC,EAAE,uBAAuB,GAAG,yBAAyB,CAAC;IACxE,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,eAAe,EAAE,qBAAqB,CAAC;IACvC,SAAS,EAAE,qBAAqB,CAAC;IACjC,YAAY,EAAE,qBAAqB,CAAC;IACpC,sBAAsB,EAAE,qBAAqB,CAAC;IAC9C,sBAAsB,EAAE,qBAAqB,CAAC;IAC9C,wBAAwB,EAAE,qBAAqB,CAAC;IAChD,aAAa,EAAE,mCAAmC,CAAC;IACnD,QAAQ,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,MAAM,EAAE,uBAAuB,CAAC;QAChC,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,4BAA4B;IACpC,eAAe,EAAE,+BAA+B,CAAC;IACjD,SAAS,EAAE,+BAA+B,CAAC;IAC3C,YAAY,EAAE,+BAA+B,CAAC;IAC9C,sBAAsB,EAAE,+BAA+B,CAAC;IACxD,sBAAsB,EAAE,+BAA+B,CAAC;CACzD;AAED,MAAM,WAAW,0BAA0B;IACzC,aAAa,EAAE,0CAA0C,CAAC;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IAC/C,qBAAqB,EAAE,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IACzD,KAAK,EAAE,8BAA8B,EAAE,CAAC;IACxC,QAAQ,EAAE,MAAM,CAAC,iBAAiB,EAAE,4BAA4B,CAAC,CAAC;IAClE,gBAAgB,EAAE,qBAAqB,CAAC;IACxC,yBAAyB,EAAE,qBAAqB,CAAC;IACjD,2BAA2B,EAAE,qBAAqB,CAAC;IACnD,WAAW,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,UAAU,EAAE,sBAAsB,CAAA;KAAE,CAAC;IAC3E,YAAY,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,UAAU,EAAE,+BAA+B,CAAA;KAAE,CAAC;IACrF,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED,wBAAsB,4BAA4B,CAAC,KAAK,EAAE;IACxD,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAmFtC"}

package/dist/release-quality-bundle-evaluator.js

@@ -0,0 +1,321 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import Database from 'better-sqlite3';
+import { DevFlowDatabase } from '@devflow-tools/database';
+import { evaluationContextStableRef, evaluationKnowledgeStableRef, evaluationMemoryStableRef, createRuntimeActivationSnapshotV1, } from '@devflow-tools/sdk';
+import { createEngines } from '@devflow-tools/mcp-server/composition';
+import { RetrievalCoordinator, RetrievalSessionRepository } from '@devflow-tools/retrieval-engine';
+import { resolveTaskSemanticContext, } from '@devflow-tools/semantic-engine';
+import { aggregateEvidenceQualityMetrics, candidateRecallAt20, evidenceQualityMetric, forbiddenContaminationAt5, precisionAt5, recallAt5, selectBestGoldSet, } from './evidence-quality-metrics.js';
+import { createEvaluationRunState } from './evaluation-run-state.js';
+import { captureProducer, identityFor, noOpTaskRuntime, rankedEvidenceRefs, restrictBenchmarkRetrievalChannel, stageCapture, } from './unified-runtime-runner.js';
+import { validateEvaluationBundle, } from './evaluation-bundle.js';
+import { createReleaseQualityQuerySamplingReplay, parseReleaseQualityQuerySampling, } from './release-quality-query-sampling.js';
+export async function evaluateReleaseQualityBundle(input) {
+    const bundlePath = input.bundlePath;
+    const validation = validateEvaluationBundle(bundlePath);
+    if (validation.status !== 'valid')
+        throw new Error(`EVALUATION_BUNDLE_INVALID:${validation.reasonCode}`);
+    const { manifest } = validation;
+    const corpus = readJson(join(bundlePath, 'corpus.json'));
+    const profiles = readJson(join(bundlePath, 'profiles.json'));
+    const labels = readLabels(bundlePath);
+    const runState = createEvaluationRunState({
+        bundlePath,
+        kind: 'retrieval',
+        caseId: manifest.bundleId.slice('bundle:'.length, 'bundle:'.length + 16),
+    });
+    const engines = await createEngines(runState.dataSources.sourceRoot, {
+        warmSymbolVectors: false,
+        dataSources: runState.dataSources,
+    });
+    const memoryDatabase = new Database(runState.dataSources.memoryDatabasePath, { readonly: true });
+    const knowledgeDatabase = new Database(runState.dataSources.knowledgeDatabasePath, { readonly: true });
+    const operationalDatabase = new DevFlowDatabase(runState.dataSources.sourceRoot, {
+        dbPath: runState.dataSources.operationalDatabasePath,
+    });
+    try {
+        engines.knowledge.activatePersistedLocalSources();
+        const cases = [];
+        for (const rawCase of corpus.cases) {
+            const item = releaseQualityCase(rawCase);
+            const label = labels[item.channel].labels.find(candidate => candidate.caseId === item.caseId);
+            if (!label)
+                throw new Error(`EVALUATION_LABEL_CASE_MISSING:${item.caseId}`);
+            cases.push(await evaluateCase({
+                item,
+                label,
+                manifest,
+                engines,
+                memoryDatabase,
+                knowledgeDatabase,
+                operationalDatabase,
+                profiles,
+                reranker: input.reranker,
+                sourceRoot: runState.dataSources.sourceRoot,
+            }));
+        }
+        await runState.verifyBundleUnchanged();
+        const positive = cases.filter(item => item.caseClass === 'positive');
+        const channels = Object.fromEntries(['context', 'memory', 'knowledge'].map(channel => {
+            const channelCases = positive.filter(item => item.channel === channel);
+            return [channel, aggregateChannel(channelCases)];
+        }));
+        const lifecycle = cases.filter(item => item.channel === 'memory' && item.caseClass === 'lifecycle');
+        return {
+            schemaVersion: 'devflow.release-quality-bundle-report.v1',
+            bundleId: manifest.bundleId,
+            projectSnapshotId: manifest.projectSnapshotId,
+            builderIdentity: manifest.builderIdentity,
+            corpusHash: manifest.corpusHash,
+            labelHashes: manifest.labelHashes,
+            channelSnapshotHashes: manifest.channelSnapshotHashes,
+            cases,
+            channels,
+            memoryDurability: evidenceQualityMetric(lifecycle.filter(item => item.recallAt5.numerator === item.recallAt5.denominator).length, lifecycle.length),
+            memoryTemporalCorrectness: evidenceQualityMetric(lifecycle.filter(item => item.expectedLifecycle === 'latest_episode_selected'
+                && item.recallAt5.numerator === item.recallAt5.denominator
+                && item.forbiddenContamination.numerator === 0).length, lifecycle.filter(item => item.expectedLifecycle === 'latest_episode_selected').length),
+            knowledgeVersionCorrectness: combine(cases
+                .filter(item => item.channel === 'knowledge' && item.caseClass === 'positive')
+                .map(item => item.sourceVersionCorrectness)),
+            consumption: { status: 'unavailable', reasonCode: 'host_ledger_required' },
+            verification: { status: 'unavailable', reasonCode: 'verification_receipt_required' },
+            bundleUnchanged: true,
+        };
+    }
+    finally {
+        memoryDatabase.close();
+        knowledgeDatabase.close();
+        operationalDatabase.close();
+        closeEngines(engines);
+        runState.dispose();
+    }
+}
+async function evaluateCase(input) {
+    const startedAt = performance.now();
+    const resolved = await canonicalRefs(input);
+    const candidateRefs = resolved.candidateRefs.slice(0, 20);
+    const selectedRefs = resolved.selectedRefs.slice(0, 5);
+    const gold = new Set(selectBestGoldSet({
+        goldSets: input.label.goldSets,
+        candidateRefs,
+        selectedRefs,
+    }));
+    return {
+        caseId: input.item.caseId,
+        channel: input.item.channel,
+        caseClass: input.item.caseClass,
+        ...(input.item.expectedLifecycle ? { expectedLifecycle: input.item.expectedLifecycle } : {}),
+        candidateRefs,
+        selectedRefs,
+        candidateRecall: candidateRecallAt20(candidateRefs, gold),
+        recallAt5: recallAt5(selectedRefs, gold),
+        precisionAt5: precisionAt5(selectedRefs, gold),
+        forbiddenContamination: forbiddenContaminationAt5(selectedRefs, new Set(input.label.forbiddenRefs)),
+        provenanceCompleteness: evidenceQualityMetric(resolved.resolved, resolved.returned),
+        sourceVersionCorrectness: input.item.channel === 'knowledge'
+            ? evidenceQualityMetric(resolved.versionCorrect, resolved.returned)
+            : evidenceQualityMetric(0, 0),
+        querySampling: resolved.querySampling,
+        reranker: resolved.reranker,
+        durationMs: Math.max(0, Math.round((performance.now() - startedAt) * 1000) / 1000),
+    };
+}
+async function canonicalRefs(input) {
+    const identity = identityFor(input.item.caseId, 'benchmark:release-quality', input.sourceRoot, input.manifest.projectId);
+    const querySampling = createReleaseQualityQuerySamplingReplay(input.item.querySampling);
+    const semantic = await resolveTaskSemanticContext({
+        normalizedPrompt: input.item.prompt,
+        identity,
+        revision: 1,
+        activation: createRuntimeActivationSnapshotV1({
+            semantic: 'authoritative',
+            retrieval: 'authoritative',
+            memoryExtraction: 'deferred',
+            knowledgeCompilation: 'deferred',
+            embeddingProfiles: {
+                code_retrieval: input.profiles.embeddingProfiles.code_retrieval.profileHash,
+            },
+            rolloutPolicyVersion: 'release-quality-bundle.v1',
+            createdAt: 1,
+        }),
+        router: input.engines.semanticRouter,
+        retrievalQuerySampling: {
+            sample: querySampling.sample,
+        },
+        policy: {
+            policyHash: 'release-quality-bundle.v1',
+            budgets: {
+                hookDeadlineMs: 3_000,
+                contextTokens: 8_000,
+                responseReserveTokens: 2_000,
+                perChannelMs: { code: 1_500, memory: 800, knowledge: 1_500 },
+            },
+        },
+        now: 1,
+    });
+    const querySamplingEvidence = querySampling.evidence();
+    const context = restrictBenchmarkRetrievalChannel(semantic.context, input.item.channel, input.item.prompt.normalize('NFKC'));
+    const stage = stageCapture();
+    const result = await new RetrievalCoordinator({
+        code: captureProducer(input.engines.codeEvidence, stage),
+        memory: captureProducer(input.engines.memoryQuery, stage),
+        knowledge: captureProducer(input.engines.knowledgeQuery, stage),
+        sessions: new RetrievalSessionRepository(input.operationalDatabase),
+        reranker: input.reranker,
+        taskRuntime: noOpTaskRuntime(),
+    }).retrieve({ context, taskRevision: 1, tokenBudget: 8_000 });
+    const channel = input.item.channel === 'context' ? 'code' : input.item.channel;
+    const reranker = result.receipt.reranker?.providerReceipt;
+    if (!reranker || reranker.status !== 'ready') {
+        const legs = (result.receipt.legs ?? [])
+            .filter(leg => leg.channel === channel)
+            .map(leg => `${leg.status}:${leg.candidateCount}:${leg.reasonCode ?? 'none'}`)
+            .join(',');
+        throw new Error(`RELEASE_QUALITY_RERANKER_UNAVAILABLE:${input.item.caseId}:${channel}:`
+            + `${reranker?.status ?? 'missing'}:${reranker?.reasonCodes.join(',') ?? 'none'}`
+            + `:candidates=${stage.candidates.length}:legs=${legs || 'none'}`);
+    }
+    const candidates = stage.candidates.filter(candidate => candidate.channel === channel);
+    const mapped = new Map();
+    let resolved = 0;
+    let versionCorrect = 0;
+    for (const candidate of candidates) {
+        const reference = evaluationRef(input, candidate);
+        if (!reference)
+            continue;
+        mapped.set(candidate.stableRef, reference.ref);
+        resolved += 1;
+        versionCorrect += reference.versionCorrect;
+    }
+    return {
+        candidateRefs: unique(rankedEvidenceRefs(result.receipt, input.item.channel)
+            .flatMap(stableRef => mapped.get(stableRef) ?? [])),
+        selectedRefs: unique((result.receipt.selectedEvidence ?? [])
+            .filter(candidate => candidate.channel === channel)
+            .flatMap(candidate => mapped.get(candidate.stableRef) ?? [])),
+        returned: candidates.length,
+        resolved,
+        versionCorrect,
+        querySampling: querySamplingEvidence,
+        reranker: {
+            profileId: result.receipt.reranker.profileId,
+            profileVersion: result.receipt.reranker.profileVersion,
+            status: reranker.status,
+            reasonCodes: reranker.reasonCodes,
+        },
+    };
+}
+function evaluationRef(input, candidate) {
+    if (candidate.channel === 'code') {
+        const path = stringPayload(candidate, 'path');
+        const symbolId = stringPayload(candidate, 'symbolId');
+        if (!path || !symbolId)
+            return undefined;
+        return { ref: evaluationContextStableRef({
+                projectSnapshotId: input.manifest.projectSnapshotId,
+                normalizedPath: path,
+                symbolId,
+                range: [integerPayload(candidate, 'startLine'), integerPayload(candidate, 'endLine')],
+            }), versionCorrect: 0 };
+    }
+    if (candidate.channel === 'memory') {
+        const id = stringPayload(candidate, 'id');
+        if (!id)
+            return undefined;
+        if (candidate.payload.type === 'cold_episode') {
+            const row = input.memoryDatabase.prepare('SELECT episode_hash FROM memory_episodes_v1 WHERE episode_id = ?')
+                .get(id);
+            return row ? { ref: evaluationMemoryStableRef({
+                    channelSnapshotHash: input.manifest.channelSnapshotHashes.memory,
+                    kind: 'episode',
+                    immutableId: id,
+                    version: row.episode_hash,
+                }), versionCorrect: 0 } : undefined;
+        }
+        const row = input.memoryDatabase.prepare('SELECT version FROM memories WHERE id = ?')
+            .get(id);
+        return row ? { ref: evaluationMemoryStableRef({
+                channelSnapshotHash: input.manifest.channelSnapshotHashes.memory,
+                kind: 'fact',
+                immutableId: id,
+                version: row.version,
+            }), versionCorrect: 0 } : undefined;
+    }
+    const id = stringPayload(candidate, 'id');
+    if (!id)
+        return undefined;
+    const row = input.knowledgeDatabase.prepare(`SELECT c.content_hash, d.source, d.version
+    FROM knowledge_chunks c JOIN knowledge_docs d ON d.id = c.doc_id WHERE c.id = ?`);
+    const knowledge = row.get(id);
+    if (!knowledge)
+        return undefined;
+    return { ref: evaluationKnowledgeStableRef({
+            sourceId: knowledge.source,
+            sourceVersion: knowledge.version,
+            contentHash: knowledge.content_hash,
+            chunkId: id,
+        }), versionCorrect: candidate.payload.version === knowledge.version ? 1 : 0 };
+}
+function stringPayload(candidate, key) {
+    const value = candidate.payload[key];
+    return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+function integerPayload(candidate, key) {
+    const value = candidate.payload[key];
+    return Number.isSafeInteger(value) && value >= 0 ? value : 0;
+}
+function unique(refs) {
+    return [...new Set(refs)];
+}
+function aggregateChannel(cases) {
+    return {
+        candidateRecall: aggregateEvidenceQualityMetrics(cases.map(item => item.candidateRecall)),
+        recallAt5: aggregateEvidenceQualityMetrics(cases.map(item => item.recallAt5)),
+        precisionAt5: aggregateEvidenceQualityMetrics(cases.map(item => item.precisionAt5)),
+        forbiddenContamination: aggregateEvidenceQualityMetrics(cases.map(item => item.forbiddenContamination)),
+        provenanceCompleteness: aggregateEvidenceQualityMetrics(cases.map(item => item.provenanceCompleteness)),
+    };
+}
+function combine(metrics) {
+    return evidenceQualityMetric(metrics.reduce((total, item) => total + item.numerator, 0), metrics.reduce((total, item) => total + item.denominator, 0));
+}
+function readLabels(bundlePath) {
+    const entries = ['context', 'memory', 'knowledge'].map(channel => [
+        channel,
+        readJson(join(bundlePath, `labels/${channel}.json`)),
+    ]);
+    return Object.fromEntries(entries);
+}
+function readJson(path) {
+    return JSON.parse(readFileSync(path, 'utf8'));
+}
+function releaseQualityCase(value) {
+    if (typeof value.caseId !== 'string'
+        || !['context', 'memory', 'knowledge'].includes(value.channel)
+        || (value.caseClass !== 'positive' && value.caseClass !== 'lifecycle')
+        || typeof value.prompt !== 'string' || !value.prompt.trim()
+        || (value.expectedLifecycle !== undefined
+            && value.expectedLifecycle !== 'explicit_save_durable'
+            && value.expectedLifecycle !== 'latest_episode_selected')) {
+        throw new Error(`EVALUATION_CORPUS_CASE_INVALID:${String(value.caseId)}`);
+    }
+    return {
+        caseId: value.caseId,
+        channel: value.channel,
+        caseClass: value.caseClass,
+        prompt: value.prompt,
+        querySampling: parseReleaseQualityQuerySampling(value.querySampling),
+        ...(value.expectedLifecycle ? { expectedLifecycle: value.expectedLifecycle } : {}),
+    };
+}
+function closeEngines(engines) {
+    engines.context.close();
+    engines.memory.close();
+    engines.knowledge.close();
+    engines.telemetry.close();
+    engines.codegraph.close();
+    engines.data.close?.();
+}
+//# sourceMappingURL=release-quality-bundle-evaluator.js.map