@jean.gnc/harness-kit 0.14.3 → 0.14.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/check/index.d.ts +11 -4
- package/dist/check/index.d.ts.map +1 -1
- package/dist/check/index.js +47 -89
- package/dist/check/index.js.map +1 -1
- package/dist/check/kinds.d.ts +4 -3
- package/dist/check/kinds.d.ts.map +1 -1
- package/dist/check/kinds.js +8 -20
- package/dist/check/kinds.js.map +1 -1
- package/dist/cli.js +4 -18
- package/dist/cli.js.map +1 -1
- package/dist/eval/claude-env.d.ts +3 -0
- package/dist/eval/claude-env.d.ts.map +1 -0
- package/dist/eval/claude-env.js +12 -0
- package/dist/eval/claude-env.js.map +1 -0
- package/dist/eval/{grade-deterministic.d.ts → grade/deterministic.d.ts} +3 -3
- package/dist/eval/grade/deterministic.d.ts.map +1 -0
- package/dist/eval/{grade-deterministic.js → grade/deterministic.js} +3 -3
- package/dist/eval/grade/deterministic.js.map +1 -0
- package/dist/eval/grade/index.d.ts +4 -0
- package/dist/eval/grade/index.d.ts.map +1 -0
- package/dist/eval/grade/index.js +4 -0
- package/dist/eval/grade/index.js.map +1 -0
- package/dist/eval/grade/results.d.ts +5 -0
- package/dist/eval/grade/results.d.ts.map +1 -0
- package/dist/eval/{grade.js → grade/results.js} +4 -4
- package/dist/eval/grade/results.js.map +1 -0
- package/dist/eval/{grade-judge.d.ts → grade/rubric.d.ts} +3 -3
- package/dist/eval/grade/rubric.d.ts.map +1 -0
- package/dist/eval/{grade-judge.js → grade/rubric.js} +1 -1
- package/dist/eval/grade/rubric.js.map +1 -0
- package/dist/eval/index.d.ts +1 -1
- package/dist/eval/index.d.ts.map +1 -1
- package/dist/eval/index.js +8 -19
- package/dist/eval/index.js.map +1 -1
- package/dist/eval/judge/claude-cli.d.ts +9 -0
- package/dist/eval/judge/claude-cli.d.ts.map +1 -0
- package/dist/eval/judge/claude-cli.js +38 -0
- package/dist/eval/judge/claude-cli.js.map +1 -0
- package/dist/eval/judge/index.d.ts +5 -0
- package/dist/eval/judge/index.d.ts.map +1 -0
- package/dist/eval/judge/index.js +5 -0
- package/dist/eval/judge/index.js.map +1 -0
- package/dist/eval/judge/select.d.ts +9 -0
- package/dist/eval/judge/select.d.ts.map +1 -0
- package/dist/eval/judge/select.js +15 -0
- package/dist/eval/judge/select.js.map +1 -0
- package/dist/eval/judge/subagent.d.ts +13 -0
- package/dist/eval/judge/subagent.d.ts.map +1 -0
- package/dist/eval/judge/subagent.js +65 -0
- package/dist/eval/judge/subagent.js.map +1 -0
- package/dist/eval/{judge.d.ts → judge/verdict.d.ts} +2 -7
- package/dist/eval/judge/verdict.d.ts.map +1 -0
- package/dist/eval/judge/verdict.js +15 -0
- package/dist/eval/judge/verdict.js.map +1 -0
- package/dist/eval/runner.d.ts.map +1 -1
- package/dist/eval/runner.js +2 -6
- package/dist/eval/runner.js.map +1 -1
- package/dist/eval/score.d.ts +1 -2
- package/dist/eval/score.d.ts.map +1 -1
- package/dist/eval/score.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/layout/index.d.ts +1 -0
- package/dist/layout/index.d.ts.map +1 -1
- package/dist/layout/index.js +18 -0
- package/dist/layout/index.js.map +1 -1
- package/package.json +1 -2
- package/dist/eval/grade-deterministic.d.ts.map +0 -1
- package/dist/eval/grade-deterministic.js.map +0 -1
- package/dist/eval/grade-judge.d.ts.map +0 -1
- package/dist/eval/grade-judge.js.map +0 -1
- package/dist/eval/grade.d.ts +0 -5
- package/dist/eval/grade.d.ts.map +0 -1
- package/dist/eval/grade.js.map +0 -1
- package/dist/eval/judge.d.ts.map +0 -1
- package/dist/eval/judge.js +0 -55
- package/dist/eval/judge.js.map +0 -1
package/dist/layout/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/layout/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EACL,2BAA2B,EAC3B,2BAA2B,EAC3B,yBAAyB,GAC1B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,iBAAiB,EAAsC,MAAM,yBAAyB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAe,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,cAAc,CAAC;AAgDpD,SAAS,gBAAgB,CAAC,OAAe,EAAE,UAA8B,EAAE,MAAc;IACvF,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QAC1E,OAAO,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;AAClC,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,SAAiB,EACjB,SAAiB;IAEjB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,yBAAyB,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,2BAA2B,CAAC,CAAC;IAC9D,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAEvF,IAAI,KAAK,IAAI,OAAO,EAAE,CAAC;QACrB,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAyB,CAAC;QAC/E,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,GAAG,CAAC;gBACT,IAAI,EAAE,kBAAkB;gBACxB,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC;aACtC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IACvD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,GAAG,CAAC;YACT,IAAI,EAAE,kBAAkB;YACxB,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC;SACtC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAAe;IAC9C,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC;IACnE,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;QACzC,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;IACvE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,GAAG,CAAC;YACT,IAAI,EAAE,qBAAqB;YAC3B,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC;SACtC,CAAC,CAAC;IACL,CAAC;IACD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC;IAChC,MAAM,UAAU,GAAG,WAAW,CAAC,QAAQ,EAAE,UAAU,CAAC;IAEpD,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,aAAa,GAAmB,EAAE,CAAC;IAEzC,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACrC,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC3E,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;YACnC,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACvE,IAAI,CAAC,cAAc,CAAC,EAAE;YAAE,OAAO,cAAc,CAAC;QAC9C,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC;QAElE,IAAI,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;YACjC,OAAO,GAAG,CAAC;gBACT,IAAI,EAAE,sBAAsB;gBAC5B,SAAS,EAAE,KAAK,CAAC,IAAI;gBACrB,YAAY,EAAE,QAAQ,CAAC,IAAI;gBAC3B,IAAI,EAAE,IAAI,CACR,SAAS,EACT,cAAc,KAAK,WAAW,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,2BAA2B,CACzF;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,SAAS;YACT,QAAQ;YACR,cAAc;YACd,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC;YACpC,WAAW,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,QAAQ,IAAI,UAAU,CAAC;YAC7D,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC;YACvD,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC;SACrD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC;AAC9D,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAsB;IAC1D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,MAAM,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1D,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,MAAM,iBAAiB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/D,QAAQ,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,MAAM,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7D,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AACtC,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,GAAW;IACvC,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IACxC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;YAAE,SAAS;QACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACvC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YACtC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;SACvC,CAAC,CAAC;QACH,IAAI,KAAK,IAAI,KAAK;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW;IACjD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IACxC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAAE,SAAS;QAC9B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,SAAS;QAC1C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/layout/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EACL,2BAA2B,EAC3B,2BAA2B,EAC3B,yBAAyB,GAC1B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,iBAAiB,EAAsC,MAAM,yBAAyB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAe,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,cAAc,CAAC;AAgDpD,MAAM,UAAU,iBAAiB,CAAC,KAAkB;IAClD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,qBAAqB;YACxB,OAAO,qCAAqC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC3D,KAAK,qBAAqB;YACxB,OAAO,2BAA2B,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACxF,KAAK,gBAAgB;YACnB,OAAO,WAAW,KAAK,CAAC,IAAI,kBAAkB,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7D,KAAK,kBAAkB;YACrB,OAAO,WAAW,KAAK,CAAC,IAAI,wBAAwB,KAAK,CAAC,SAAS,EAAE,CAAC;QACxE,KAAK,oBAAoB;YACvB,OAAO,WAAW,KAAK,CAAC,IAAI,2CAA2C,KAAK,CAAC,SAAS,aAAa,CAAC;QACtG,KAAK,kBAAkB;YACrB,OAAO,sBAAsB,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACnF,KAAK,sBAAsB;YACzB,OAAO,iBAAiB,KAAK,CAAC,SAAS,mCAAmC,KAAK,CAAC,YAAY,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;IACrH,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe,EAAE,UAA8B,EAAE,MAAc;IACvF,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QAC1E,OAAO,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;AAClC,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,SAAiB,EACjB,SAAiB;IAEjB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,yBAAyB,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,2BAA2B,CAAC,CAAC;IAC9D,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAEvF,IAAI,KAAK,IAAI,OAAO,EAAE,CAAC;QACrB,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAyB,CAAC;QAC/E,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,GAAG,CAAC;gBACT,IAAI,EAAE,kBAAkB;gBACxB,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC;aACtC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IACvD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,GAAG,CAAC;YACT,IAAI,EAAE,kBAAkB;YACxB,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC;SACtC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAAe;IAC9C,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC;IACnE,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;QACzC,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;IACvE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,GAAG,CAAC;YACT,IAAI,EAAE,qBAAqB;YAC3B,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC;SACtC,CAAC,CAAC;IACL,CAAC;IACD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC;IAChC,MAAM,UAAU,GAAG,WAAW,CAAC,QAAQ,EAAE,UAAU,CAAC;IAEpD,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,aAAa,GAAmB,EAAE,CAAC;IAEzC,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACrC,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC3E,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;YACnC,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACvE,IAAI,CAAC,cAAc,CAAC,EAAE;YAAE,OAAO,cAAc,CAAC;QAC9C,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC;QAElE,IAAI,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;YACjC,OAAO,GAAG,CAAC;gBACT,IAAI,EAAE,sBAAsB;gBAC5B,SAAS,EAAE,KAAK,CAAC,IAAI;gBACrB,YAAY,EAAE,QAAQ,CAAC,IAAI;gBAC3B,IAAI,EAAE,IAAI,CACR,SAAS,EACT,cAAc,KAAK,WAAW,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,2BAA2B,CACzF;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,SAAS;YACT,QAAQ;YACR,cAAc;YACd,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC;YACpC,WAAW,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,QAAQ,IAAI,UAAU,CAAC;YAC7D,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC;YACvD,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC;SACrD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC;AAC9D,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAsB;IAC1D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,MAAM,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1D,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,MAAM,iBAAiB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/D,QAAQ,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,MAAM,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7D,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AACtC,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,GAAW;IACvC,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IACxC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;YAAE,SAAS;QACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACvC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YACtC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;SACvC,CAAC,CAAC;QACH,IAAI,KAAK,IAAI,KAAK;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW;IACjD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IACxC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAAE,SAAS;QAC9B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,SAAS;QAC1C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jean.gnc/harness-kit",
|
|
3
|
-
"version": "0.14.
|
|
3
|
+
"version": "0.14.5",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Build your own multi-agent harness: typed toolkit for authoring plugins (skills, agents, commands, hooks) and shipping them to Claude Code and Codex from a single source tree.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -68,7 +68,6 @@
|
|
|
68
68
|
]
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@anthropic-ai/sdk": "^0.100.1",
|
|
72
71
|
"citty": "^0.2.2",
|
|
73
72
|
"js-yaml": "^4.1.1",
|
|
74
73
|
"markdownlint-cli2": "^0.18.1",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"grade-deterministic.d.ts","sourceRoot":"","sources":["../../src/eval/grade-deterministic.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,SAAS,SAAS,EAAE,EAChC,OAAO,EAAE,cAAc,GACtB,eAAe,EAAE,CAEnB"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"grade-deterministic.js","sourceRoot":"","sources":["../../src/eval/grade-deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,cAAc,CAAC;AAEpD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AASvC,MAAM,UAAU,eAAe,CAC7B,UAAgC,EAChC,OAAuB;IAEvB,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,KAAK,CAAC,SAAoB,EAAE,OAAuB;IAC1D,QAAQ,SAAS,CAAC,IAAI,EAAE,CAAC;QACvB,KAAK,eAAe;YAClB,OAAO,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,UAAU,EAAE;gBAChD,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,SAAS,CAAC,OAAO,GAAG,CAAC;gBACpE,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,SAAS,CAAC,OAAO,GAAG,CAAC;aACzE,CAAC,CAAC;QAEL,KAAK,gBAAgB;YACnB,OAAO,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,UAAU,EAAE;gBAChD,WAAW,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,oBAAoB,GAAG,GAAG,CAAC;gBACjE,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,SAAS,CAAC,OAAO,GAAG,CAAC;aACpE,CAAC,CAAC;QAEL,KAAK,UAAU;YACb,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC5C,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,SAAS,CAAC,IAAI,EAAE,CAAC;gBAC3C,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEpE,KAAK,eAAe;YAClB,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC5C,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,SAAS,CAAC,IAAI,EAAE,CAAC;gBACxD,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;QAEvD,KAAK,WAAW;YACd,OAAO,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC;AAOD,SAAS,WAAW,CAClB,SAA2E,EAC3E,IAAY,EACZ,OAAqB;IAErB,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;IACnE,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,OAAO,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,cAAc,CACrB,SAAoD,EACpD,OAAuB;IAEvB,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,OAAO,IAAI,CACT,SAAS,EACT,GAAG,SAAS,CAAC,IAAI,yBAAyB,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAC/E,CAAC;IACJ,CAAC;IACD,IAAI,SAAS,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC,SAAS,EAAE,SAAS,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;IACrF,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;IAC1E,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACvD,OAAO,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI;QACpC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,IAAI,kBAAkB,CAAC;QACtD,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,IAAI,2BAA2B,SAAS,CAAC,cAAc,GAAG,CAAC,CAAC;AAC/F,CAAC;AAID,SAAS,cAAc,CAAC,OAAe,EAAE,KAAc;IACrD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACH,QAAQ,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,GAAG,CAAC,kBAAkB,OAAO,MAAO,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,OAAO,CAAC,OAAuB,EAAE,IAAY;IACpD,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,YAAY,CAAC,OAAuB;IAC3C,OAAO,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACzC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CACtD,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM,CAAC;AAC7C,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC;IAC1C,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAChD,CAAC;AAED,SAAS,KAAK,CAAC,OAAuB;IACpC,OAAO,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,IAAI,CAAC,SAAoB,EAAE,QAAgB;IAClD,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;AAC7C,CAAC;AAED,SAAS,IAAI,CAAC,SAAoB,EAAE,QAAgB;IAClD,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC9C,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"grade-judge.d.ts","sourceRoot":"","sources":["../../src/eval/grade-judge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,KAAK,EAAe,MAAM,EAAE,MAAM,aAAa,CAAC;AAEvD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,UAAU,EAAE,SAAS,eAAe,EAAE,CAAC;IAChD,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;CACxB;AAED,wBAAsB,WAAW,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,KAAK,GACX,OAAO,CAAC,YAAY,CAAC,CAQvB"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"grade-judge.js","sourceRoot":"","sources":["../../src/eval/grade-judge.ts"],"names":[],"mappings":"AAaA,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAc,EACd,MAAc,EACd,KAAY;IAEZ,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;QACzD,SAAS;QACT,OAAO,EAAE,MAAM,KAAK,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;KACvD,CAAC,CAAC,CACJ,CAAC;IACF,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,OAAO,CAAC,IAAiB,EAAE,UAAsC;IACxE,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACxE,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK;QAAE,OAAO,MAAM,KAAK,UAAU,CAAC,MAAM,CAAC;IAChE,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;AAC/E,CAAC"}
|
package/dist/eval/grade.d.ts
DELETED
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import type { Judge } from "./judge.js";
|
|
2
|
-
import type { CaseReport } from "./report.js";
|
|
3
|
-
import type { CaseResult } from "./runner.js";
|
|
4
|
-
export declare function gradeResults(results: readonly CaseResult[], judge: Judge | undefined): Promise<CaseReport[]>;
|
|
5
|
-
//# sourceMappingURL=grade.d.ts.map
|
package/dist/eval/grade.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"grade.d.ts","sourceRoot":"","sources":["../../src/eval/grade.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,wBAAsB,YAAY,CAChC,OAAO,EAAE,SAAS,UAAU,EAAE,EAC9B,KAAK,EAAE,KAAK,GAAG,SAAS,GACvB,OAAO,CAAC,UAAU,EAAE,CAAC,CAEvB"}
|
package/dist/eval/grade.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"grade.js","sourceRoot":"","sources":["../../src/eval/grade.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAqB,MAAM,kBAAkB,CAAC;AAIlE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAyB,MAAM,YAAY,CAAC;AAG5E,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAA8B,EAC9B,KAAwB;IAExB,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,MAAkB,EAAE,KAAwB;IAClE,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QAClC,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;IACnF,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAC5F,CAAC;IACF,OAAO;QACL,QAAQ;QACR,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC;QAC/C,OAAO,EAAE,EAAE,MAAM,EAAE;KACpB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,UAAgC,EAChC,MAA0B,EAC1B,OAAuB,EACvB,KAAwB;IAExB,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACpD,MAAM,YAAY,GAChB,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChF,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;AACtD,CAAC"}
|
package/dist/eval/judge.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,OAAO;;;;;;;;;EAGlB,CAAC;AAEH,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,OAAO,CAAC,CAAC;AAE9C,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAMhE,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAIlD;AAwBD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,qBAAqB,GAAG,KAAK,CAmB1E"}
|
package/dist/eval/judge.js
DELETED
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
import Anthropic from "@anthropic-ai/sdk";
|
|
2
|
-
import { z } from "zod";
|
|
3
|
-
export const Verdict = z.object({
|
|
4
|
-
pass: z.union([z.boolean(), z.literal("unknown")]),
|
|
5
|
-
evidence: z.string(),
|
|
6
|
-
});
|
|
7
|
-
function abstain(evidence) {
|
|
8
|
-
return { pass: "unknown", evidence };
|
|
9
|
-
}
|
|
10
|
-
export function parseVerdict(raw) {
|
|
11
|
-
if (raw === undefined || raw === null)
|
|
12
|
-
return abstain("judge returned no verdict");
|
|
13
|
-
const result = Verdict.safeParse(raw);
|
|
14
|
-
return result.success ? result.data : abstain("judge returned an unparseable verdict");
|
|
15
|
-
}
|
|
16
|
-
const SYSTEM_PREAMBLE = "You are a strict grader for AI agent outputs. Judge the output against the single criterion " +
|
|
17
|
-
"you are given, in isolation. Record a verdict via the tool: pass=true only if the criterion is " +
|
|
18
|
-
'clearly met, pass=false if clearly unmet, and pass="unknown" if you genuinely cannot tell. ' +
|
|
19
|
-
"Always cite concrete evidence from the output.";
|
|
20
|
-
const VERDICT_TOOL = {
|
|
21
|
-
name: "record_verdict",
|
|
22
|
-
description: "Record the grading verdict for one rubric dimension.",
|
|
23
|
-
input_schema: {
|
|
24
|
-
type: "object",
|
|
25
|
-
properties: {
|
|
26
|
-
pass: {
|
|
27
|
-
description: 'true if met, false if unmet, "unknown" if undeterminable',
|
|
28
|
-
anyOf: [{ type: "boolean" }, { type: "string", enum: ["unknown"] }],
|
|
29
|
-
},
|
|
30
|
-
evidence: { type: "string", description: "concrete evidence from the output" },
|
|
31
|
-
},
|
|
32
|
-
required: ["pass", "evidence"],
|
|
33
|
-
},
|
|
34
|
-
};
|
|
35
|
-
export function createAnthropicJudge(options) {
|
|
36
|
-
const client = new Anthropic({ apiKey: options.apiKey });
|
|
37
|
-
return async ({ dimension, criterion, output }) => {
|
|
38
|
-
const response = await client.messages.create({
|
|
39
|
-
model: options.model,
|
|
40
|
-
max_tokens: options.maxTokens ?? 1024,
|
|
41
|
-
tools: [VERDICT_TOOL],
|
|
42
|
-
tool_choice: { type: "tool", name: VERDICT_TOOL.name },
|
|
43
|
-
system: [{ type: "text", text: SYSTEM_PREAMBLE, cache_control: { type: "ephemeral" } }],
|
|
44
|
-
messages: [
|
|
45
|
-
{
|
|
46
|
-
role: "user",
|
|
47
|
-
content: `Dimension: ${dimension}\nCriterion: ${criterion}\n\nOutput to grade:\n${output}`,
|
|
48
|
-
},
|
|
49
|
-
],
|
|
50
|
-
});
|
|
51
|
-
const block = response.content.find((b) => b.type === "tool_use");
|
|
52
|
-
return parseVerdict(block?.input);
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
//# sourceMappingURL=judge.js.map
|
package/dist/eval/judge.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;IAClD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;CACrB,CAAC,CAAC;AAYH,SAAS,OAAO,CAAC,QAAgB;IAC/B,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAY;IACvC,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,OAAO,CAAC,2BAA2B,CAAC,CAAC;IACnF,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;AACzF,CAAC;AAED,MAAM,eAAe,GACnB,8FAA8F;IAC9F,iGAAiG;IACjG,6FAA6F;IAC7F,gDAAgD,CAAC;AAEnD,MAAM,YAAY,GAAmB;IACnC,IAAI,EAAE,gBAAgB;IACtB,WAAW,EAAE,sDAAsD;IACnE,YAAY,EAAE;QACZ,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,IAAI,EAAE;gBACJ,WAAW,EAAE,0DAA0D;gBACvE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;aACpE;YACD,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,mCAAmC,EAAE;SAC/E;QACD,QAAQ,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC;KAC/B;CACF,CAAC;AAQF,MAAM,UAAU,oBAAoB,CAAC,OAA8B;IACjE,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACzD,OAAO,KAAK,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE;QAChD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC5C,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,UAAU,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI;YACrC,KAAK,EAAE,CAAC,YAAY,CAAC;YACrB,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE;YACtD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,CAAC;YACvF,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,cAAc,SAAS,gBAAgB,SAAS,yBAAyB,MAAM,EAAE;iBAC3F;aACF;SACF,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;QAClE,OAAO,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC"}
|