@michaelfromyeg/loom-eval 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/index.js +1 -1
  2. package/dist/index.js.map +1 -1
  3. package/package.json +4 -4
package/dist/index.js CHANGED
@@ -255,7 +255,7 @@ async function runCli(cmd, args, opts) {
255
255
  }
256
256
 
257
257
  // src/drivers/claude.ts
258
- var ALLOWED_TOOLS = "Read,Edit,Write,Bash,Grep,Glob,WebFetch";
258
+ var ALLOWED_TOOLS = "Read,Edit,Write,Bash,Grep,Glob,WebFetch,Skill";
259
259
  var claudeDriver = {
260
260
  target: "claude",
261
261
  available: () => cliAvailable("claude", ["--version"]),
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/assert.ts","../src/baselines.ts","../src/drivers/parse.ts","../src/drivers/util.ts","../src/drivers/claude.ts","../src/drivers/codex.ts","../src/drivers/copilot.ts","../src/drivers/cursor.ts","../src/drivers/opencode.ts","../src/drivers/index.ts","../src/runner.ts"],"names":["execa","drivers","join"],"mappings":";;;;;;;;AAwCA,SAAS,aAAA,CAAc,KAAe,GAAA,EAAwB;AAC5D,EAAA,IAAI,CAAA,GAAI,CAAA;AACR,EAAA,KAAA,MAAW,QAAQ,GAAA,EAAK;AACtB,IAAA,IAAI,IAAA,KAAS,GAAA,CAAI,CAAC,CAAA,EAAG,CAAA,EAAA;AACrB,IAAA,IAAI,CAAA,KAAM,GAAA,CAAI,MAAA,EAAQ,OAAO,IAAA;AAAA,EAC/B;AACA,EAAA,OAAO,IAAI,MAAA,KAAW,CAAA;AACxB;AAEA,SAAS,iBAAA,CAAkB,GAAgB,CAAA,EAAwB;AACjE,EAAA,MAAM,QAAQ,CAAA,CAAE,SAAA,CAAU,IAAI,CAAC,CAAA,KAAM,EAAE,IAAI,CAAA;AAC3C,EAAA,IAAI,CAAA,CAAE,cAAc,CAAC,KAAA,CAAM,SAAS,CAAA,CAAE,UAAU,GAAG,OAAO,KAAA;AAC1D,EAAA,IAAI,EAAE,aAAA,IAAiB,KAAA,CAAM,SAAS,CAAA,CAAE,aAAa,GAAG,OAAO,KAAA;AAC/D,EAAA,IAAI,CAAA,CAAE,aAAa,MAAA,IAAa,CAAA,CAAE,UAAU,MAAA,GAAS,CAAA,CAAE,UAAU,OAAO,KAAA;AACxE,EAAA,IAAI,CAAA,CAAE,YAAY,CAAC,aAAA,CAAc,EAAE,QAAA,EAAU,KAAK,GAAG,OAAO,KAAA;AAC5D,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,aAAA,CAAc,GAAgB,WAAA,EAAyC;AAC9E,EAAA,IAAI,WAAA,CAAY,SAAS,CAAA,IAAK,WAAA,CAAY,MAAM,CAAC,CAAA,KAAM,CAAA,CAAE,gBAAgB,CAAA,EAAG;AAC1E,IAAA,OAAO;AAAA,MACL,IAAA,EAAM,OAAA;AAAA,MACN,MAAA,EAAQ,UAAA;AAAA,MACR,MAAA,EAAQ;AAAA,KACV;AAAA,EACF;AACA,EAAA,MAAM,MAAA,GAAS,YAAY,MAAA,CAAO,CAAC,MAAM,iBAAA,CAAkB,CAAA,EAAG,CAAC,CAAC,CAAA,CAAE,MAAA;AAClE,EAAA,MAAM,OAAO,WAAA,CAAY,MAAA,GAAS,CAAA,GAAI,MAAA,GAAS,YAAY,MAAA,GAAS,CAAA;AACpE,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,OAAA;AAAA,IACN,MAAA,EAAQ,IAAA,IAAQ,CAAA,CAAE,WAAA,GAAc,MAAA,GAAS,MAAA;AAAA,IACzC,MAAA,EAAQ,GAAG,MAAM,CAAA,CAAA,EAAI,YAAY,MAAM,CAAA,6BAAA,EAAgC,EAAE,WAAW,CAAA,CAAA;AAAA,GACtF;AACF;AAEA,SAAS,aAAa,OAAA,EAAgC;AACpD,EAAA,MAAM,CAAA,GAAI,iBAAA,CAAkB,IAAA,CAAK,OAAO,CAAA;AACxC,EAAA,IAAI;AACF,IAAA,OAAO,IAAI,IAAI,MAAA,CAAO,OAAA,CAAQ,KAAA,CAAM,EAAE,CAAC,CAAA,CAAE,MAAM,CAAA,EAAG,EAAE,CAAC,CAAC,CAAA,GAAI,IAAI,OAAO,OAAO,CAAA;AAAA,EAC9E,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,IAAA;AAAA,EACT;AACF;AAEA,SAAS,kBAAA,CAAmB,GAAiB,CAAA,EAAwB;AACnE,EAAA,MAAM,OAAO,CAAA,CAAE,SAAA;AACf,EAAA,IAAI,CAAA,CAAE,MAAA,KAAW,MAAA,IAAa,IAAA,CAAK,IAAA,OAAW,CAAA,CAAE,MAAA,CAAO,IAAA,EAAK,EAAG,OAAO,KAAA;AACtE,EAAA,IAAI,CAAA,CAAE,OAAA,KAAY,MAAA,IAAa,CAAC,YAAA,CAAa,CAAA,CAAE,OAAO,CAAA,EAAG,IAAA,CAAK,IAAI,CAAA,EAAG,OAAO,KAAA;AAC5E,EAAA,IAAI,CAAA,CAAE,eAAe,MAAA,EAAW;AAC9B,IAAA,IAAI;AACF,MAAA,IAAA,CAAK,MAAM,IAAI,CAAA;AAAA,IACjB,CAAA,CAAA,MAAQ;AACN,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AACA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,cAAA,CAAe,GAAiB,WAAA,EAAyC;AAChF,EAAA,IAAI,WAAA,CAAY,MAAA,KAAW,CAAA,EAAG,OAAO,EAAE,MAAM,QAAA,EAAU,MAAA,EAAQ,MAAA,EAAQ,MAAA,EAAQ,YAAA,EAAa;AAC5F,EAAA,MAAM,MAAA,GAAS,YAAY,MAAA,CAAO,CAAC,MAAM,kBAAA,CAAmB,CAAA,EAAG,CAAC,CAAC,CAAA,CAAE,MAAA;AACnE,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,QAAA;AAAA,IACN,MAAA,EAAQ,MAAA,KAAW,WAAA,CAAY,MAAA,GAAS,MAAA,GAAS,MAAA;AAAA,IACjD,MAAA,EAAQ,CAAA,EAAG,MAAM,CAAA,CAAA,EAAI,YAAY,MAAM,CAAA,gBAAA;AAAA,GACzC;AACF;AAEA,eAAe,aAAA,CACb,CAAA,EACA,WAAA,EACA,GAAA,EACuB;AACvB,EAAA,IAAI,CAAC,IAAI,KAAA,EAAO;AACd,IAAA,OAAO,EAAE,IAAA,EAAM,OAAA,EAAS,MAAA,EAAQ,SAAA,EAAW,QAAQ,sCAAA,EAAuC;AAAA,EAC5F;AACA,EAAA,IAAI,MAAA,GAAS,CAAA;AACb,EAAA,KAAA,MAAW,KAAK,WAAA,EAAa;AAC3B,IAAA,MAAM,CAAA,GAAI,MAAM,GAAA,CAAI,KAAA,CAAM;AAAA,MACxB,WAAW,CAAA,CAAE,SAAA;AAAA,MACb,WAAW,CAAA,CAAE,SAAA;AAAA,MACb,QAAQ,CAAA,CAAE,MAAA;AAAA,MACV,MAAM,CAAA,CAAE,IAAA;AAAA,MACR,SAAS,CAAA,CAAE;AAAA,KACZ,CAAA;AACD,IAAA,IAAI,EAAE,IAAA,EAAM,MAAA,EAAA;AAAA,EACd;AACA,EAAA,MAAM,EAAA,GAAK,MAAA,GAAS,CAAA,GAAI,WAAA,CAAY,MAAA;AAEpC,EAAA,MAAM,SAAuB,CAAA,CAAE,IAAA,GAAQ,KAAK,MAAA,GAAS,MAAA,GAAU,KAAK,MAAA,GAAS,SAAA;AAC7E,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,OAAA;AAAA,IACN,MAAA;AAAA,IACA,MAAA,EAAQ,CAAA,EAAG,MAAM,CAAA,CAAA,EAAI,WAAA,CAAY,MAAM,CAAA,oBAAA,EAAuB,CAAA,CAAE,IAAA,GAAO,EAAA,GAAK,aAAa,CAAA;AAAA,GAC3F;AACF;AAEA,SAAS,oBAAA,CAAqB,GAAuB,GAAA,EAAkC;AACrF,EAAA,IAAI,GAAA,CAAI,kBAAkB,MAAA,EAAW;AACnC,IAAA,OAAO;AAAA,MACL,IAAA,EAAM,cAAA;AAAA,MACN,MAAA,EAAQ,SAAA;AAAA,MACR,MAAA,EAAQ;AAAA,KACV;AAAA,EACF;AACA,EAAA,MAAM,KAAA,GAAQ,IAAI,SAAA,IAAa,CAAA;AAC/B,EAAA,MAAM,EAAA,GAAK,KAAA,GAAQ,GAAA,CAAI,aAAA,IAAiB,CAAC,CAAA,CAAE,WAAA;AAC3C,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,cAAA;AAAA,IACN,MAAA,EAAQ,KAAK,MAAA,GAAS,MAAA;AAAA,IACtB,MAAA,EAAQ,CAAA,MAAA,EAAS,KAAA,CAAM,OAAA,CAAQ,CAAC,CAAC,CAAA,aAAA,EAAgB,GAAA,CAAI,aAAA,CAAc,OAAA,CAAQ,CAAC,CAAC,CAAA,cAAA,EAAiB,EAAE,WAAW,CAAA,CAAA;AAAA,GAC7G;AACF;AASA,eAAsB,iBAAA,CACpB,CAAA,EACA,WAAA,EACA,GAAA,GAAqB,EAAC,EACC;AACvB,EAAA,QAAQ,EAAE,IAAA;AAAM,IACd,KAAK,OAAA;AACH,MAAA,OAAO,aAAA,CAAc,GAAG,WAAW,CAAA;AAAA,IACrC,KAAK,QAAA;AACH,MAAA,OAAO,cAAA,CAAe,GAAG,WAAW,CAAA;AAAA,IACtC,KAAK,OAAA;AACH,MAAA,OAAO,aAAA,CAAc,CAAA,EAAG,WAAA,EAAa,GAAG,CAAA;AAAA,IAC1C,KAAK,cAAA;AACH,MAAA,OAAO,oBAAA,CAAqB,GAAG,GAAG,CAAA;AAAA;AAExC;ACvKA,SAAS,YAAA,CAAa,SAAA,EAAmB,SAAA,EAAmB,OAAA,EAAyB;AACnF,EAAA,MAAM,IAAA,GAAO,SAAA,CAAU,OAAA,CAAQ,UAAA,EAAY,GAAG,CAAA;AAC9C,EAAA,OAAO,KAAK,SAAA,EAAW,OAAA,EAAS,cAAc,IAAA,EAAM,CAAA,EAAG,OAAO,CAAA,KAAA,CAAO,CAAA;AACvE;AAGO,SAAS,YAAA,CACd,SAAA,EACA,SAAA,EACA,OAAA,EACiB;AACjB,EAAA,MAAM,CAAA,GAAI,YAAA,CAAa,SAAA,EAAW,SAAA,EAAW,OAAO,CAAA;AACpD,EAAA,IAAI,CAAC,UAAA,CAAW,CAAC,CAAA,EAAG,OAAO,IAAA;AAC3B,EAAA,IAAI;AACF,IAAA,OAAO,IAAA,CAAK,KAAA,CAAM,YAAA,CAAa,CAAA,EAAG,MAAM,CAAC,CAAA;AAAA,EAC3C,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,IAAA;AAAA,EACT;AACF;AAMO,SAAS,aAAA,CACd,SAAA,EACA,SAAA,EACA,OAAA,EACA,QAAA,EACQ;AACR,EAAA,MAAM,CAAA,GAAI,YAAA,CAAa,SAAA,EAAW,SAAA,EAAW,OAAO,CAAA;AACpD,EAAA,SAAA,CAAU,QAAQ,CAAC,CAAA,EAAG,EAAE,SAAA,EAAW,MAAM,CAAA;AACzC,EAAA,aAAA,CAAc,GAAG,CAAA,EAAG,IAAA,CAAK,UAAU,QAAA,EAAU,IAAA,EAAM,CAAC,CAAC;AAAA,CAAI,CAAA;AACzD,EAAA,OAAO,CAAA;AACT;;;ACxCO,SAAS,WAAW,GAAA,EAAwC;AACjE,EAAA,MAAM,MAAiC,EAAC;AACxC,EAAA,KAAA,MAAW,IAAA,IAAQ,GAAA,CAAI,KAAA,CAAM,IAAI,CAAA,EAAG;AAClC,IAAA,MAAM,CAAA,GAAI,KAAK,IAAA,EAAK;AACpB,IAAA,IAAI,CAAC,CAAA,EAAG;AACR,IAAA,IAAI;AACF,MAAA,GAAA,CAAI,IAAA,CAAK,IAAA,CAAK,KAAA,CAAM,CAAC,CAAC,CAAA;AAAA,IACxB,CAAA,CAAA,MAAQ;AAAA,IAER;AAAA,EACF;AACA,EAAA,OAAO,GAAA;AACT;AAOO,SAAS,kBAAkB,GAAA,EAA2D;AAC3F,EAAA,MAAM,YAAwB,EAAC;AAC/B,EAAA,IAAI,SAAA,GAAY,EAAA;AAChB,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,MAAW,GAAA,IAAO,UAAA,CAAW,GAAG,CAAA,EAAG;AACjC,IAAA,IAAI,GAAA,CAAI,SAAS,WAAA,EAAa;AAC5B,MAAA,MAAM,OAAA,GAAW,GAAA,CAAI,OAAA,EAAiD,OAAA,IAAW,EAAC;AAClF,MAAA,KAAA,MAAW,SAAS,OAAA,EAAsC;AACxD,QAAA,IAAI,KAAA,EAAO,SAAS,UAAA,EAAY;AAC9B,UAAA,SAAA,CAAU,IAAA,CAAK,EAAE,IAAA,EAAM,MAAA,CAAO,KAAA,CAAM,IAAI,CAAA,EAAG,IAAA,EAAM,KAAA,CAAM,KAAA,EAAO,EAAA,EAAI,EAAA,EAAA,EAAM,CAAA;AAAA,QAC1E;AAAA,MACF;AAAA,IACF,WAAW,GAAA,CAAI,IAAA,KAAS,YAAY,OAAO,GAAA,CAAI,WAAW,QAAA,EAAU;AAClE,MAAA,SAAA,GAAY,GAAA,CAAI,MAAA;AAAA,IAClB;AAAA,EACF;AACA,EAAA,OAAO,EAAE,WAAW,SAAA,EAAU;AAChC;AAOO,SAAS,iBAAiB,GAAA,EAA2D;AAC1F,EAAA,MAAM,YAAwB,EAAC;AAC/B,EAAA,IAAI,SAAA,GAAY,EAAA;AAChB,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,MAAW,GAAA,IAAO,UAAA,CAAW,GAAG,CAAA,EAAG;AACjC,IAAA,IAAI,GAAA,CAAI,SAAS,gBAAA,EAAkB;AACnC,IAAA,MAAM,OAAO,GAAA,CAAI,IAAA;AACjB,IAAA,IAAI,CAAC,IAAA,EAAM;AACX,IAAA,MAAM,IAAA,GAAO,MAAA,CAAO,IAAA,CAAK,IAAI,CAAA;AAC7B,IAAA,IAAI,SAAS,eAAA,EAAiB;AAC5B,MAAA,SAAA,GAAY,MAAA,CAAO,IAAA,CAAK,IAAA,IAAQ,IAAA,CAAK,WAAW,SAAS,CAAA;AAAA,IAC3D,CAAA,MAAA,IAAW,SAAS,WAAA,EAAa;AAC/B,MAAA,MAAM,IAAA,GAAO,SAAS,eAAA,GAAkB,MAAA,CAAO,KAAK,IAAA,IAAQ,IAAA,CAAK,IAAA,IAAQ,IAAI,CAAA,GAAI,IAAA;AACjF,MAAA,SAAA,CAAU,KAAK,EAAE,IAAA,EAAM,MAAM,IAAA,EAAM,EAAA,EAAI,MAAM,CAAA;AAAA,IAC/C;AAAA,EACF;AACA,EAAA,OAAO,EAAE,WAAW,SAAA,EAAU;AAChC;AAGA,SAAS,eAAe,GAAA,EAAsC;AAC5D,EAAA,MAAM,EAAA,GAAM,GAAA,CAAI,SAAA,IAAa,GAAA,CAAI,QAAA;AACjC,EAAA,IAAI,EAAA,IAAM,OAAO,EAAA,KAAO,QAAA,EAAU;AAEhC,IAAA,MAAM,GAAA,GAAM,MAAA,CAAO,IAAA,CAAK,EAAE,CAAA,CAAE,IAAA,CAAK,CAAC,CAAA,KAAM,WAAA,CAAY,IAAA,CAAK,CAAC,CAAC,CAAA;AAC3D,IAAA,IAAI,GAAA,EAAK,OAAO,GAAA,CAAI,OAAA,CAAQ,aAAa,EAAE,CAAA;AAC3C,IAAA,IAAI,OAAO,EAAA,CAAG,IAAA,KAAS,QAAA,SAAiB,EAAA,CAAG,IAAA;AAAA,EAC7C;AACA,EAAA,OAAO,OAAO,GAAA,CAAI,IAAA,KAAS,QAAA,GAAW,IAAI,IAAA,GAAO,MAAA;AACnD;AAMO,SAAS,kBAAkB,GAAA,EAA2D;AAC3F,EAAA,MAAM,YAAwB,EAAC;AAC/B,EAAA,IAAI,SAAA,GAAY,EAAA;AAChB,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,MAAW,GAAA,IAAO,UAAA,CAAW,GAAG,CAAA,EAAG;AACjC,IAAA,IAAI,GAAA,CAAI,SAAS,WAAA,KAAgB,GAAA,CAAI,YAAY,WAAA,IAAe,GAAA,CAAI,YAAY,MAAA,CAAA,EAAY;AAC1F,MAAA,SAAA,CAAU,IAAA,CAAK,EAAE,IAAA,EAAM,cAAA,CAAe,GAAG,CAAA,EAAG,IAAA,EAAM,GAAA,CAAI,SAAA,IAAa,GAAA,EAAK,EAAA,EAAI,EAAA,EAAA,EAAM,CAAA;AAAA,IACpF,WAAW,GAAA,CAAI,IAAA,KAAS,YAAY,OAAO,GAAA,CAAI,WAAW,QAAA,EAAU;AAClE,MAAA,SAAA,GAAY,GAAA,CAAI,MAAA;AAAA,IAClB;AAAA,EACF;AACA,EAAA,OAAO,EAAE,WAAW,SAAA,EAAU;AAChC;AAOO,SAAS,oBAAoB,GAAA,EAA2D;AAC7F,EAAA,MAAM,YAAwB,EAAC;AAC/B,EAAA,IAAI,SAAA,GAAY,EAAA;AAChB,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,MAAW,GAAA,IAAO,UAAA,CAAW,GAAG,CAAA,EAAG;AACjC,IAAA,IAAI,GAAA,CAAI,SAAS,UAAA,EAAY;AAC3B,MAAA,MAAM,IAAA,GAAQ,IAAI,IAAA,IAAQ,GAAA;AAC1B,MAAA,MAAM,QAAQ,IAAA,CAAK,KAAA;AACnB,MAAA,SAAA,CAAU,IAAA,CAAK;AAAA,QACb,IAAA,EAAM,MAAA,CAAO,IAAA,CAAK,IAAA,IAAQ,MAAM,CAAA;AAAA,QAChC,MAAM,KAAA,EAAO,KAAA;AAAA,QACb,QAAQ,KAAA,EAAO,MAAA;AAAA,QACf,EAAA,EAAI,EAAA;AAAA,OACL,CAAA;AAAA,IACH,CAAA,MAAA,IAAW,GAAA,CAAI,IAAA,KAAS,MAAA,EAAQ;AAC9B,MAAA,MAAM,OAAO,GAAA,CAAI,IAAA;AACjB,MAAA,SAAA,IAAa,MAAA,CAAO,GAAA,CAAI,IAAA,IAAQ,IAAA,EAAM,QAAQ,EAAE,CAAA;AAAA,IAClD;AAAA,EACF;AACA,EAAA,OAAO,EAAE,WAAW,SAAA,EAAU;AAChC;ACrHA,eAAsB,YAAA,CAAa,KAAa,WAAA,EAAyC;AACvF,EAAA,IAAI;AACF,IAAA,MAAM,CAAA,GAAI,MAAM,KAAA,CAAM,GAAA,EAAK,WAAA,EAAa,EAAE,MAAA,EAAQ,KAAA,EAAO,OAAA,EAAS,GAAA,EAAQ,CAAA;AAC1E,IAAA,OAAO,EAAE,QAAA,KAAa,CAAA;AAAA,EACxB,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,KAAA;AAAA,EACT;AACF;AASA,eAAsB,MAAA,CACpB,GAAA,EACA,IAAA,EACA,IAAA,EACoB;AACpB,EAAA,IAAI;AACF,IAAA,MAAM,CAAA,GAAI,MAAM,KAAA,CAAM,GAAA,EAAK,IAAA,EAAM;AAAA,MAC/B,KAAK,IAAA,CAAK,GAAA;AAAA,MACV,MAAA,EAAQ,KAAA;AAAA,MACR,OAAA,EAAS,KAAK,SAAA,IAAa,IAAA;AAAA,MAC3B,GAAA,EAAK,IAAA,CAAK,MAAA,IAAU;AAAC,KACtB,CAAA;AACD,IAAA,OAAO,EAAE,MAAA,EAAQ,CAAA,CAAE,MAAA,IAAU,EAAA,EAAI,MAAA,EAAQ,CAAA,CAAE,MAAA,IAAU,EAAA,EAAI,QAAA,EAAU,CAAA,CAAE,QAAA,IAAY,CAAA,EAAE;AAAA,EACrF,SAAS,GAAA,EAAK;AACZ,IAAA,OAAO,EAAE,MAAA,EAAQ,EAAA,EAAI,QAAS,GAAA,CAAc,OAAA,EAAS,UAAU,CAAA,EAAE;AAAA,EACnE;AACF;;;AC/BA,IAAM,aAAA,GAAgB,yCAAA;AAEf,IAAM,YAAA,GAA8B;AAAA,EACzC,MAAA,EAAQ,QAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,QAAA,EAAU,CAAC,WAAW,CAAC,CAAA;AAAA,EACrD,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,MAAM,MAAM,MAAA;AAAA,MAChB,QAAA;AAAA,MACA;AAAA,QACE,IAAA;AAAA,QACA,MAAA;AAAA,QACA,iBAAA;AAAA,QACA,aAAA;AAAA,QACA,WAAA;AAAA,QACA,mBAAA;AAAA,QACA,aAAA;AAAA,QACA,gBAAA;AAAA,QACA;AAAA,OACF;AAAA,MACA,EAAE,GAAA,EAAK,MAAA,EAAQ,SAAA;AAAU,KAC3B;AACA,IAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAU,GAAI,iBAAA,CAAkB,IAAI,MAAM,CAAA;AAC7D,IAAA,OAAO,EAAE,WAAW,SAAA,EAAW,QAAA,EAAU,IAAI,QAAA,EAAU,GAAA,EAAK,IAAI,MAAA,EAAO;AAAA,EACzE;AACF;;;ACxBO,IAAM,WAAA,GAA6B;AAAA,EACxC,MAAA,EAAQ,OAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,OAAA,EAAS,CAAC,WAAW,CAAC,CAAA;AAAA,EACpD,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,MAAM,MAAM,MAAA;AAAA,MAChB,OAAA;AAAA,MACA,CAAC,MAAA,EAAQ,QAAA,EAAU,4CAAA,EAA8C,MAAM,CAAA;AAAA,MACvE,EAAE,GAAA,EAAK,MAAA,EAAQ,SAAA;AAAU,KAC3B;AACA,IAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAU,GAAI,gBAAA,CAAiB,IAAI,MAAM,CAAA;AAC5D,IAAA,OAAO,EAAE,WAAW,SAAA,EAAW,QAAA,EAAU,IAAI,QAAA,EAAU,GAAA,EAAK,IAAI,MAAA,EAAO;AAAA,EACzE;AACF;;;ACPO,IAAM,aAAA,GAA+B;AAAA,EAC1C,MAAA,EAAQ,SAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,SAAA,EAAW,CAAC,WAAW,CAAC,CAAA;AAAA,EACtD,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,GAAA,GAAM,MAAM,MAAA,CAAO,SAAA,EAAW,CAAC,IAAA,EAAM,MAAA,EAAQ,IAAA,EAAM,aAAa,CAAA,EAAG;AAAA,MACvE,GAAA;AAAA,MACA,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AACD,IAAA,OAAO;AAAA,MACL,SAAA,EAAW,GAAA,CAAI,MAAA,CAAO,IAAA,EAAK;AAAA,MAC3B,WAAW,EAAC;AAAA,MACZ,UAAU,GAAA,CAAI,QAAA;AAAA,MACd,KAAK,GAAA,CAAI,MAAA;AAAA,MACT,gBAAA,EAAkB;AAAA,KACpB;AAAA,EACF;AACF;;;ACtBO,IAAM,YAAA,GAA8B;AAAA,EACzC,MAAA,EAAQ,QAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,cAAA,EAAgB,CAAC,WAAW,CAAC,CAAA;AAAA,EAC3D,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,MAAM,MAAM,MAAA;AAAA,MAChB,cAAA;AAAA,MACA,CAAC,IAAA,EAAM,MAAA,EAAQ,SAAA,EAAW,mBAAmB,aAAa,CAAA;AAAA,MAC1D,EAAE,GAAA,EAAK,MAAA,EAAQ,SAAA;AAAU,KAC3B;AACA,IAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAU,GAAI,iBAAA,CAAkB,IAAI,MAAM,CAAA;AAC7D,IAAA,OAAO,EAAE,WAAW,SAAA,EAAW,QAAA,EAAU,IAAI,QAAA,EAAU,GAAA,EAAK,IAAI,MAAA,EAAO;AAAA,EACzE;AACF;;;ACZO,IAAM,cAAA,GAAgC;AAAA,EAC3C,MAAA,EAAQ,UAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,UAAA,EAAY,CAAC,WAAW,CAAC,CAAA;AAAA,EACvD,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,GAAA,GAAM,MAAM,MAAA,CAAO,UAAA,EAAY,CAAC,KAAA,EAAO,MAAA,EAAQ,UAAA,EAAY,MAAM,CAAA,EAAG;AAAA,MACxE,GAAA;AAAA,MACA,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AACD,IAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAU,GAAI,mBAAA,CAAoB,IAAI,MAAM,CAAA;AAC/D,IAAA,OAAO,EAAE,WAAW,SAAA,EAAW,QAAA,EAAU,IAAI,QAAA,EAAU,GAAA,EAAK,IAAI,MAAA,EAAO;AAAA,EACzE;AACF;;;ACOO,IAAM,OAAA,GAAyC;AAAA,EACpD,MAAA,EAAQ,YAAA;AAAA,EACR,KAAA,EAAO,WAAA;AAAA,EACP,MAAA,EAAQ,YAAA;AAAA,EACR,OAAA,EAAS,aAAA;AAAA,EACT,QAAA,EAAU;AACZ;ACmBO,SAAS,cAAc,SAAA,EAAqC;AACjE,EAAA,MAAM,MAAA,GAAS,cAAc,SAAS,CAAA;AACtC,EAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AACd,IAAA,MAAM,IAAI,KAAA,CAAM,MAAA,CAAO,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,EAAG,CAAA,CAAE,IAAI,KAAK,CAAA,CAAE,OAAO,EAAE,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,EAChF;AACA,EAAA,MAAM,MAAwB,EAAC;AAC/B,EAAA,KAAA,MAAW,CAAA,IAAK,MAAA,CAAO,KAAA,CAAM,MAAA,CAAO,UAAA,EAAY;AAC9C,IAAA,MAAM,QAAA,GAAW,OAAA,IAAW,CAAA,GAAI,CAAA,CAAE,KAAA,GAAQ,MAAA;AAC1C,IAAA,IAAI,CAAC,QAAA,EAAU;AACf,IAAA,MAAM,OAAO,MAAA,CAAO,KAAA,CAAM,KAAK,QAAQ,CAAA,CAAE,SAAS,MAAM,CAAA;AACxD,IAAA,MAAM,SAAS,YAAA,CAAa,QAAA,EAAU,MAAM,EAAE,QAAA,EAAU,UAAU,CAAA;AAClE,IAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AACd,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,eAAA,EAAkB,QAAQ,CAAA,GAAA,EAAM,MAAA,CAAO,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,OAChF;AAAA,IACF;AACA,IAAA,GAAA,CAAI,IAAA,CAAK,EAAE,aAAA,EAAe,UAAA,CAAW,CAAC,CAAA,EAAG,SAAA,EAAW,QAAA,EAAU,QAAA,EAAU,MAAA,CAAO,KAAA,EAAO,CAAA;AAAA,EACxF;AACA,EAAA,OAAO,GAAA;AACT;AAEA,eAAe,EAAA,CAAG,KAAa,GAAA,EAA8B;AAC3D,EAAA,MAAM,CAAA,GAAI,MAAMA,KAAAA,CAAM,MAAA,EAAQ,CAAC,KAAA,EAAO,GAAG,CAAA,EAAG,EAAE,GAAA,EAAK,MAAA,EAAQ,KAAA,EAAO,OAAA,EAAS,KAAQ,CAAA;AACnF,EAAA,OAAO,EAAE,QAAA,IAAY,CAAA;AACvB;AAQA,eAAe,OAAA,CACb,CAAA,EACA,MAAA,EACA,GAAA,EACA,GAAA,EACqB;AACrB,EAAA,IAAI,EAAE,KAAA,EAAO,MAAM,EAAA,CAAG,CAAA,CAAE,OAAO,GAAG,CAAA;AAElC,EAAA,MAAM,cAA4B,EAAC;AACnC,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,SAAS,CAAA,EAAA,EAAK;AAClC,IAAA,WAAA,CAAY,IAAA,CAAK,MAAM,MAAA,CAAO,GAAA,CAAI,EAAE,MAAA,EAAQ,CAAA,CAAE,MAAA,EAAQ,GAAA,EAAK,SAAA,EAAW,GAAA,CAAI,SAAA,EAAW,CAAC,CAAA;AAAA,EACxF;AAGA,EAAA,MAAM,OAAA,GAAwC,IAAI,KAAA,CAAM,CAAA,CAAE,OAAO,MAAM,CAAA;AACvE,EAAA,IAAI,QAAA,GAAW,CAAA;AACf,EAAA,IAAI,OAAA,GAAU,CAAA;AACd,EAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,CAAO,QAAQ,CAAA,EAAA,EAAK;AACxC,IAAA,MAAM,CAAA,GAAI,CAAA,CAAE,MAAA,CAAO,CAAC,CAAA;AACpB,IAAA,IAAI,CAAA,CAAE,IAAA,KAAS,OAAA,IAAW,CAAA,CAAE,SAAS,QAAA,EAAU;AAC7C,MAAA,MAAM,CAAA,GAAI,MAAM,iBAAA,CAAkB,CAAA,EAAG,WAAW,CAAA;AAChD,MAAA,OAAA,CAAQ,CAAC,CAAA,GAAI,CAAA;AACb,MAAA,QAAA,EAAA;AACA,MAAA,IAAI,CAAA,CAAE,WAAW,MAAA,EAAQ,OAAA,EAAA;AAAA,IAC3B;AAAA,EACF;AACA,EAAA,MAAM,KAAA,GAAQ,QAAA,GAAW,CAAA,GAAI,OAAA,GAAU,QAAA,GAAW,CAAA;AAGlD,EAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,CAAO,QAAQ,CAAA,EAAA,EAAK;AACxC,IAAA,MAAM,CAAA,GAAI,CAAA,CAAE,MAAA,CAAO,CAAC,CAAA;AACpB,IAAA,IAAI,CAAA,CAAE,IAAA,KAAS,OAAA,IAAW,CAAA,CAAE,SAAS,cAAA,EAAgB;AACnD,MAAA,OAAA,CAAQ,CAAC,CAAA,GAAI,MAAM,iBAAA,CAAkB,GAAG,WAAA,EAAa;AAAA,QACnD,OAAO,GAAA,CAAI,KAAA;AAAA,QACX,SAAA,EAAW,KAAA;AAAA,QACX,eAAe,GAAA,CAAI;AAAA,OACpB,CAAA;AAAA,IACH;AAAA,EACF;AACA,EAAA,MAAM,aAAa,OAAA,CAAQ,MAAA,CAAO,CAAC,CAAA,KAAyB,MAAM,MAAS,CAAA;AAE3E,EAAA,IAAI,YAAA;AACJ,EAAA,IAAI,CAAA,CAAE,QAAQ,YAAA,GAAgB,MAAM,GAAG,CAAA,CAAE,MAAA,EAAQ,GAAG,CAAA,KAAO,CAAA;AAC3D,EAAA,IAAI,EAAE,OAAA,EAAS,MAAM,EAAA,CAAG,CAAA,CAAE,SAAS,GAAG,CAAA;AAItC,EAAA,MAAM,IAAA,GAAO,CAAC,UAAA,CAAW,IAAA,CAAK,CAAC,MAAM,CAAA,CAAE,MAAA,KAAW,MAAM,CAAA,KAAM,YAAA,IAAgB,IAAA,CAAA;AAC9E,EAAA,OAAO,EAAE,IAAA,EAAM,CAAA,CAAE,MAAM,UAAA,EAAY,YAAA,EAAc,OAAO,IAAA,EAAK;AAC/D;AAsBA,eAAsB,QAAQ,IAAA,EAA2C;AACvE,EAAA,MAAM,EAAE,QAAA,EAAU,OAAA,EAAAC,QAAAA,EAAQ,GAAI,IAAA;AAC9B,EAAA,MAAM,YAA6B,EAAC;AAEpC,EAAA,KAAA,MAAW,OAAA,IAAW,SAAS,SAAA,EAAW;AACxC,IAAA,MAAM,MAAA,GAASA,SAAQ,OAAO,CAAA;AAC9B,IAAA,IAAI,EAAE,MAAA,IAAW,MAAM,MAAA,CAAO,WAAU,CAAA,EAAK;AAC3C,MAAA,SAAA,CAAU,IAAA,CAAK;AAAA,QACb,OAAA;AAAA,QACA,MAAA,EAAQ,UAAA;AAAA,QACR,MAAA,EAAQ,SAAS,2CAAA,GAA8C,4BAAA;AAAA,QAC/D,OAAO,EAAC;AAAA,QACR,KAAA,EAAO,CAAA;AAAA,QACP,IAAA,EAAM;AAAA,OACP,CAAA;AACD,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,gBAAgB,YAAA,CAAa,IAAA,CAAK,WAAW,QAAA,CAAS,SAAA,EAAW,OAAO,CAAA,EAAG,KAAA;AACjF,IAAA,MAAM,OAAA,GAAU,WAAA,CAAYC,IAAAA,CAAK,IAAA,CAAK,WAAA,IAAe,QAAO,EAAG,CAAA,UAAA,EAAa,OAAO,CAAA,CAAA,CAAG,CAAC,CAAA;AACvF,IAAA,IAAI;AACF,MAAA,MAAM,OAAA,CAAQ;AAAA,QACZ,WAAW,IAAA,CAAK,SAAA;AAAA,QAChB,KAAA,EAAO,SAAA;AAAA,QACP,GAAA,EAAK,OAAA;AAAA,QACL,UAAU,IAAA,CAAK,QAAA;AAAA,QACf,OAAA,EAAS,CAAC,OAAO,CAAA;AAAA,QACjB,IAAA,EAAM,CAAC,IAAA,CAAK,aAAa,CAAA;AAAA;AAAA,QAEzB,OAAA,EAAS;AAAA,OACV,CAAA;AACD,MAAA,MAAM,QAAsB,EAAC;AAC7B,MAAA,KAAA,MAAW,CAAA,IAAK,SAAS,KAAA,EAAO;AAC9B,QAAA,KAAA,CAAM,IAAA;AAAA,UACJ,MAAM,OAAA,CAAQ,CAAA,EAAG,MAAA,EAAQ,OAAA,EAAS;AAAA,YAChC,OAAO,IAAA,CAAK,KAAA;AAAA,YACZ,aAAA;AAAA,YACA,WAAW,IAAA,CAAK;AAAA,WACjB;AAAA,SACH;AAAA,MACF;AACA,MAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,MAAA,GAAS,CAAA,GAAI,MAAM,MAAA,CAAO,CAAC,CAAA,EAAG,CAAA,KAAM,IAAI,CAAA,CAAE,KAAA,EAAO,CAAC,CAAA,GAAI,MAAM,MAAA,GAAS,CAAA;AACzF,MAAA,IAAI,KAAK,iBAAA,EAAmB;AAC1B,QAAA,MAAM,QAAA,GAAqB,EAAE,OAAA,EAAS,SAAA,EAAW,KAAA,EAAM;AACvD,QAAA,aAAA,CAAc,IAAA,CAAK,SAAA,EAAW,QAAA,CAAS,SAAA,EAAW,SAAS,QAAQ,CAAA;AAAA,MACrE;AACA,MAAA,SAAA,CAAU,IAAA,CAAK,EAAE,OAAA,EAAS,MAAA,EAAQ,UAAU,KAAA,EAAO,KAAA,EAAO,IAAA,EAAM,KAAA,CAAM,MAAM,CAAC,CAAA,KAAM,CAAA,CAAE,IAAI,GAAG,CAAA;AAAA,IAC9F,SAAS,GAAA,EAAK;AACZ,MAAA,SAAA,CAAU,IAAA,CAAK;AAAA,QACb,OAAA;AAAA,QACA,MAAA,EAAQ,UAAA;AAAA,QACR,MAAA,EAAQ,CAAA,gBAAA,EAAoB,GAAA,CAAc,OAAO,CAAA,CAAA;AAAA,QACjD,OAAO,EAAC;AAAA,QACR,KAAA,EAAO,CAAA;AAAA,QACP,IAAA,EAAM;AAAA,OACP,CAAA;AAAA,IACH,CAAA,SAAE;AACA,MAAA,MAAA,CAAO,SAAS,EAAE,SAAA,EAAW,IAAA,EAAM,KAAA,EAAO,MAAM,CAAA;AAAA,IAClD;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,SAAA,EAAW,QAAA,CAAS,SAAA,EAAW,SAAA,EAAU;AACpD","file":"index.js","sourcesContent":["import type { Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport type {\n Assertion,\n DifferentialAssert,\n JudgeAssert,\n OutputAssert,\n TraceAssert,\n} from \"@michaelfromyeg/loom-schema\";\n\nexport type AssertStatus = \"pass\" | \"fail\" | \"degraded\" | \"skipped\";\n\nexport interface AssertResult {\n kind: Assertion[\"kind\"];\n status: AssertStatus;\n detail: string;\n}\n\nexport interface JudgeInput {\n candidate: string;\n reference?: string;\n rubric: string;\n mode: \"absolute\" | \"pairwise\";\n samples: number;\n}\nexport interface JudgeVerdict {\n pass: boolean;\n detail?: string;\n}\n/** A judge model. Injected so evals run offline/deterministically in tests. */\nexport type JudgeFn = (input: JudgeInput) => Promise<JudgeVerdict>;\n\nexport interface AssertContext {\n judge?: JudgeFn;\n /** Deterministic score of the case (fraction of trace/output passing) -- differential. */\n caseScore?: number;\n /** Baseline score for (component, harness) from evals/.baselines/ -- differential. */\n baselineScore?: number;\n}\n\n/** True iff `seq` appears as an ordered subsequence of `arr`. */\nfunction isSubsequence(seq: string[], arr: string[]): boolean {\n let i = 0;\n for (const item of arr) {\n if (item === seq[i]) i++;\n if (i === seq.length) return true;\n }\n return seq.length === 0;\n}\n\nfunction traceSamplePasses(a: TraceAssert, t: Transcript): boolean {\n const names = t.toolCalls.map((c) => c.name);\n if (a.toolCalled && !names.includes(a.toolCalled)) return false;\n if (a.toolNotCalled && names.includes(a.toolNotCalled)) return false;\n if (a.maxCalls !== undefined && t.toolCalls.length > a.maxCalls) return false;\n if (a.sequence && !isSubsequence(a.sequence, names)) return false;\n return true;\n}\n\nfunction evaluateTrace(a: TraceAssert, transcripts: Transcript[]): AssertResult {\n if (transcripts.length > 0 && transcripts.every((t) => t.traceUnavailable)) {\n return {\n kind: \"trace\",\n status: \"degraded\",\n detail: \"harness exposes no tool-call trace; trace assertion not evaluated\",\n };\n }\n const passes = transcripts.filter((t) => traceSamplePasses(a, t)).length;\n const rate = transcripts.length > 0 ? passes / transcripts.length : 0;\n return {\n kind: \"trace\",\n status: rate >= a.minPassRate ? \"pass\" : \"fail\",\n detail: `${passes}/${transcripts.length} samples passed (minPassRate ${a.minPassRate})`,\n };\n}\n\nfunction compileRegex(pattern: string): RegExp | null {\n const m = /^\\(\\?([a-z]+)\\)/.exec(pattern);\n try {\n return m ? new RegExp(pattern.slice(m[0].length), m[1]) : new RegExp(pattern);\n } catch {\n return null;\n }\n}\n\nfunction outputSamplePasses(a: OutputAssert, t: Transcript): boolean {\n const text = t.finalText;\n if (a.equals !== undefined && text.trim() !== a.equals.trim()) return false;\n if (a.matches !== undefined && !compileRegex(a.matches)?.test(text)) return false;\n if (a.jsonSchema !== undefined) {\n try {\n JSON.parse(text);\n } catch {\n return false;\n }\n }\n return true;\n}\n\nfunction evaluateOutput(a: OutputAssert, transcripts: Transcript[]): AssertResult {\n if (transcripts.length === 0) return { kind: \"output\", status: \"fail\", detail: \"no samples\" };\n const passes = transcripts.filter((t) => outputSamplePasses(a, t)).length;\n return {\n kind: \"output\",\n status: passes === transcripts.length ? \"pass\" : \"fail\",\n detail: `${passes}/${transcripts.length} samples matched`,\n };\n}\n\nasync function evaluateJudge(\n a: JudgeAssert,\n transcripts: Transcript[],\n ctx: AssertContext,\n): Promise<AssertResult> {\n if (!ctx.judge) {\n return { kind: \"judge\", status: \"skipped\", detail: \"no judge model configured (advisory)\" };\n }\n let passes = 0;\n for (const t of transcripts) {\n const v = await ctx.judge({\n candidate: t.finalText,\n reference: a.reference,\n rubric: a.rubric,\n mode: a.mode,\n samples: a.samples,\n });\n if (v.pass) passes++;\n }\n const ok = passes * 2 > transcripts.length; // majority\n // Advisory unless `gate:true` -- a non-gating judge never fails the case.\n const status: AssertStatus = a.gate ? (ok ? \"pass\" : \"fail\") : ok ? \"pass\" : \"skipped\";\n return {\n kind: \"judge\",\n status,\n detail: `${passes}/${transcripts.length} judge verdicts pass${a.gate ? \"\" : \" (advisory)\"}`,\n };\n}\n\nfunction evaluateDifferential(a: DifferentialAssert, ctx: AssertContext): AssertResult {\n if (ctx.baselineScore === undefined) {\n return {\n kind: \"differential\",\n status: \"skipped\",\n detail: \"no baseline (run loom publish to snapshot one)\",\n };\n }\n const score = ctx.caseScore ?? 0;\n const ok = score - ctx.baselineScore >= -a.noWorseThan;\n return {\n kind: \"differential\",\n status: ok ? \"pass\" : \"fail\",\n detail: `score ${score.toFixed(2)} vs baseline ${ctx.baselineScore.toFixed(2)} (noWorseThan ${a.noWorseThan})`,\n };\n}\n\n/**\n * Evaluate one assertion against a case's sample transcripts (spec §9.5). Trace\n * and output are deterministic; judge is advisory unless `gate:true` and needs an\n * injected judge model; differential compares the case score to a committed\n * baseline (the \"vibes\" no-regression gate).\n */\n// biome-ignore lint/suspicious/useAwait: branches mix sync (trace/output) and async (judge/differential) results; async normalizes the return to Promise<AssertResult>.\nexport async function evaluateAssertion(\n a: Assertion,\n transcripts: Transcript[],\n ctx: AssertContext = {},\n): Promise<AssertResult> {\n switch (a.kind) {\n case \"trace\":\n return evaluateTrace(a, transcripts);\n case \"output\":\n return evaluateOutput(a, transcripts);\n case \"judge\":\n return evaluateJudge(a, transcripts, ctx);\n case \"differential\":\n return evaluateDifferential(a, ctx);\n }\n}\n","import { existsSync, mkdirSync, readFileSync, writeFileSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport type { Target } from \"@michaelfromyeg/loom-schema\";\n\nexport interface Baseline {\n version: string;\n score: number;\n}\n\nfunction baselineFile(pluginDir: string, component: string, harness: Target): string {\n const safe = component.replace(/[^\\w.-]/g, \"_\");\n return join(pluginDir, \"evals\", \".baselines\", safe, `${harness}.json`);\n}\n\n/** Load the committed baseline score for (component, harness), or null. */\nexport function loadBaseline(\n pluginDir: string,\n component: string,\n harness: Target,\n): Baseline | null {\n const f = baselineFile(pluginDir, component, harness);\n if (!existsSync(f)) return null;\n try {\n return JSON.parse(readFileSync(f, \"utf8\")) as Baseline;\n } catch {\n return null;\n }\n}\n\n/**\n * Snapshot a baseline (spec §9.5). Called on `loom publish` so the next release's\n * differential evals compare against this score.\n */\nexport function writeBaseline(\n pluginDir: string,\n component: string,\n harness: Target,\n baseline: Baseline,\n): string {\n const f = baselineFile(pluginDir, component, harness);\n mkdirSync(dirname(f), { recursive: true });\n writeFileSync(f, `${JSON.stringify(baseline, null, 2)}\\n`);\n return f;\n}\n","import type { ToolCall } from \"@michaelfromyeg/loom-adapter-kit\";\n\n/** Split NDJSON/JSONL into parsed objects, skipping blank/garbage lines. */\nexport function parseLines(raw: string): Record<string, unknown>[] {\n const out: Record<string, unknown>[] = [];\n for (const line of raw.split(\"\\n\")) {\n const t = line.trim();\n if (!t) continue;\n try {\n out.push(JSON.parse(t));\n } catch {\n // tolerate non-JSON noise lines\n }\n }\n return out;\n}\n\n/**\n * Parse Claude's `--output-format stream-json --verbose` NDJSON. Tool calls are\n * `assistant` message content blocks `{type:\"tool_use\", name, input}`; the final\n * text is the `result` event's `result` field (see harness-research.md).\n */\nexport function parseClaudeStream(raw: string): { finalText: string; toolCalls: ToolCall[] } {\n const toolCalls: ToolCall[] = [];\n let finalText = \"\";\n let ts = 0;\n for (const evt of parseLines(raw)) {\n if (evt.type === \"assistant\") {\n const content = (evt.message as { content?: unknown[] } | undefined)?.content ?? [];\n for (const block of content as Record<string, unknown>[]) {\n if (block?.type === \"tool_use\") {\n toolCalls.push({ name: String(block.name), args: block.input, ts: ts++ });\n }\n }\n } else if (evt.type === \"result\" && typeof evt.result === \"string\") {\n finalText = evt.result;\n }\n }\n return { finalText, toolCalls };\n}\n\n/**\n * Parse Codex `exec --json` JSONL. Tool calls live in `item.completed` events by\n * `item.type`; the final text is the `agent_message` item. Only completed items\n * are counted to avoid double-counting the matching `item.started`.\n */\nexport function parseCodexStream(raw: string): { finalText: string; toolCalls: ToolCall[] } {\n const toolCalls: ToolCall[] = [];\n let finalText = \"\";\n let ts = 0;\n for (const evt of parseLines(raw)) {\n if (evt.type !== \"item.completed\") continue;\n const item = evt.item as Record<string, unknown> | undefined;\n if (!item) continue;\n const kind = String(item.type);\n if (kind === \"agent_message\") {\n finalText = String(item.text ?? item.message ?? finalText);\n } else if (kind !== \"reasoning\") {\n const name = kind === \"mcp_tool_call\" ? String(item.tool ?? item.name ?? kind) : kind;\n toolCalls.push({ name, args: item, ts: ts++ });\n }\n }\n return { finalText, toolCalls };\n}\n\n/** Best-effort tool name from a Cursor `tool_call` event (e.g. {readToolCall:{}} -> \"read\"). */\nfunction cursorToolName(evt: Record<string, unknown>): string {\n const tc = (evt.tool_call ?? evt.toolCall) as Record<string, unknown> | undefined;\n if (tc && typeof tc === \"object\") {\n // TODO(verify): the exact tool-name field of cursor-agent stream-json events.\n const key = Object.keys(tc).find((k) => /ToolCall$/.test(k));\n if (key) return key.replace(/ToolCall$/, \"\");\n if (typeof tc.name === \"string\") return tc.name;\n }\n return typeof evt.name === \"string\" ? evt.name : \"tool\";\n}\n\n/**\n * Parse Cursor `--output-format stream-json` NDJSON. Tool calls are `tool_call`\n * events (count the `completed` subtype); the final text is the `result` event.\n */\nexport function parseCursorStream(raw: string): { finalText: string; toolCalls: ToolCall[] } {\n const toolCalls: ToolCall[] = [];\n let finalText = \"\";\n let ts = 0;\n for (const evt of parseLines(raw)) {\n if (evt.type === \"tool_call\" && (evt.subtype === \"completed\" || evt.subtype === undefined)) {\n toolCalls.push({ name: cursorToolName(evt), args: evt.tool_call ?? evt, ts: ts++ });\n } else if (evt.type === \"result\" && typeof evt.result === \"string\") {\n finalText = evt.result;\n }\n }\n return { finalText, toolCalls };\n}\n\n/**\n * Parse OpenCode `run --format json` JSONL. Tool calls are `tool_use` events\n * carrying a ToolPart (`part.tool`, `part.state.{input,output}`); the final text\n * is the concatenation of `text` events.\n */\nexport function parseOpencodeStream(raw: string): { finalText: string; toolCalls: ToolCall[] } {\n const toolCalls: ToolCall[] = [];\n let finalText = \"\";\n let ts = 0;\n for (const evt of parseLines(raw)) {\n if (evt.type === \"tool_use\") {\n const part = (evt.part ?? evt) as Record<string, unknown>;\n const state = part.state as Record<string, unknown> | undefined;\n toolCalls.push({\n name: String(part.tool ?? \"tool\"),\n args: state?.input,\n result: state?.output,\n ts: ts++,\n });\n } else if (evt.type === \"text\") {\n const part = evt.part as { text?: string } | undefined;\n finalText += String(evt.text ?? part?.text ?? \"\");\n }\n }\n return { finalText, toolCalls };\n}\n","import { execa } from \"execa\";\n\n/** True iff `cmd <versionArgs>` exits 0 (CLI installed). Never throws. */\nexport async function cliAvailable(cmd: string, versionArgs: string[]): Promise<boolean> {\n try {\n const r = await execa(cmd, versionArgs, { reject: false, timeout: 10_000 });\n return r.exitCode === 0;\n } catch {\n return false;\n }\n}\n\nexport interface CliResult {\n stdout: string;\n stderr: string;\n exitCode: number;\n}\n\n/** Run a harness CLI headlessly. Resolved config is passed as env; never throws. */\nexport async function runCli(\n cmd: string,\n args: string[],\n opts: { cwd: string; config?: Record<string, string>; timeoutMs?: number },\n): Promise<CliResult> {\n try {\n const r = await execa(cmd, args, {\n cwd: opts.cwd,\n reject: false,\n timeout: opts.timeoutMs ?? 120_000,\n env: opts.config ?? {},\n });\n return { stdout: r.stdout ?? \"\", stderr: r.stderr ?? \"\", exitCode: r.exitCode ?? 1 };\n } catch (err) {\n return { stdout: \"\", stderr: (err as Error).message, exitCode: 1 };\n }\n}\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { parseClaudeStream } from \"./parse\";\nimport { cliAvailable, runCli } from \"./util\";\n\nconst ALLOWED_TOOLS = \"Read,Edit,Write,Bash,Grep,Glob,WebFetch\";\n\nexport const claudeDriver: HarnessDriver = {\n target: \"claude\",\n available: () => cliAvailable(\"claude\", [\"--version\"]),\n async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n const res = await runCli(\n \"claude\",\n [\n \"-p\",\n prompt,\n \"--output-format\",\n \"stream-json\",\n \"--verbose\",\n \"--permission-mode\",\n \"acceptEdits\",\n \"--allowedTools\",\n ALLOWED_TOOLS,\n ],\n { cwd, config, timeoutMs },\n );\n const { finalText, toolCalls } = parseClaudeStream(res.stdout);\n return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };\n },\n};\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { parseCodexStream } from \"./parse\";\nimport { cliAvailable, runCli } from \"./util\";\n\nexport const codexDriver: HarnessDriver = {\n target: \"codex\",\n available: () => cliAvailable(\"codex\", [\"--version\"]),\n async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n const res = await runCli(\n \"codex\",\n [\"exec\", \"--json\", \"--dangerously-bypass-approvals-and-sandbox\", prompt],\n { cwd, config, timeoutMs },\n );\n const { finalText, toolCalls } = parseCodexStream(res.stdout);\n return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };\n },\n};\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { cliAvailable, runCli } from \"./util\";\n\n/**\n * GitHub Copilot's headless `-p` mode exposes NO structured tool-call trace today\n * (only a Markdown transcript via --share). So `run` returns the final text with\n * `traceUnavailable: true`, and the runner degrades `trace` assertions to `output`\n * for this harness rather than faking a pass (spec §14, harness-research.md).\n */\nexport const copilotDriver: HarnessDriver = {\n target: \"copilot\",\n available: () => cliAvailable(\"copilot\", [\"--version\"]),\n async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n const res = await runCli(\"copilot\", [\"-p\", prompt, \"-s\", \"--allow-all\"], {\n cwd,\n config,\n timeoutMs,\n });\n return {\n finalText: res.stdout.trim(),\n toolCalls: [],\n exitCode: res.exitCode,\n raw: res.stdout,\n traceUnavailable: true,\n };\n },\n};\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { parseCursorStream } from \"./parse\";\nimport { cliAvailable, runCli } from \"./util\";\n\nexport const cursorDriver: HarnessDriver = {\n target: \"cursor\",\n available: () => cliAvailable(\"cursor-agent\", [\"--version\"]),\n async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n const res = await runCli(\n \"cursor-agent\",\n [\"-p\", prompt, \"--force\", \"--output-format\", \"stream-json\"],\n { cwd, config, timeoutMs },\n );\n const { finalText, toolCalls } = parseCursorStream(res.stdout);\n return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };\n },\n};\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { parseOpencodeStream } from \"./parse\";\nimport { cliAvailable, runCli } from \"./util\";\n\nexport const opencodeDriver: HarnessDriver = {\n target: \"opencode\",\n available: () => cliAvailable(\"opencode\", [\"--version\"]),\n async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n const res = await runCli(\"opencode\", [\"run\", prompt, \"--format\", \"json\"], {\n cwd,\n config,\n timeoutMs,\n });\n const { finalText, toolCalls } = parseOpencodeStream(res.stdout);\n return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };\n },\n};\n","import type { HarnessDriver } from \"@michaelfromyeg/loom-adapter-kit\";\nimport type { Target } from \"@michaelfromyeg/loom-schema\";\nimport { claudeDriver } from \"./claude\";\nimport { codexDriver } from \"./codex\";\nimport { copilotDriver } from \"./copilot\";\nimport { cursorDriver } from \"./cursor\";\nimport { opencodeDriver } from \"./opencode\";\n\nexport { claudeDriver } from \"./claude\";\nexport { codexDriver } from \"./codex\";\nexport { copilotDriver } from \"./copilot\";\nexport { cursorDriver } from \"./cursor\";\nexport { opencodeDriver } from \"./opencode\";\nexport {\n parseClaudeStream,\n parseCodexStream,\n parseCursorStream,\n parseLines,\n parseOpencodeStream,\n} from \"./parse\";\nexport type { CliResult } from \"./util\";\n\n/** Every built-in headless driver, keyed by Target. */\nexport const drivers: Record<Target, HarnessDriver> = {\n claude: claudeDriver,\n codex: codexDriver,\n cursor: cursorDriver,\n copilot: copilotDriver,\n opencode: opencodeDriver,\n};\n","import { mkdtempSync, rmSync } from \"node:fs\";\nimport { tmpdir } from \"node:os\";\nimport { join } from \"node:path\";\nimport type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { type AdapterRegistry, install, loadPluginDir } from \"@michaelfromyeg/loom-core\";\nimport {\n type Case,\n EvalFile,\n leafNameOf,\n loadManifest,\n type Target,\n} from \"@michaelfromyeg/loom-schema\";\nimport { execa } from \"execa\";\nimport { type AssertResult, evaluateAssertion, type JudgeFn } from \"./assert\";\nimport { type Baseline, loadBaseline, writeBaseline } from \"./baselines\";\n\nexport interface CaseResult {\n name: string;\n assertions: AssertResult[];\n verifyPassed?: boolean;\n /** Deterministic score: fraction of trace/output assertions that passed. */\n score: number;\n pass: boolean;\n}\n\nexport interface HarnessReport {\n harness: Target;\n status: \"tested\" | \"untested\";\n /** Why a harness was not tested (no driver / CLI absent / install failed). */\n reason?: string;\n cases: CaseResult[];\n /** Mean deterministic case score (snapshotted as the next release's baseline). */\n score: number;\n pass: boolean;\n}\n\nexport interface EvalReport {\n component: string;\n harnesses: HarnessReport[];\n}\n\nexport interface DiscoveredEval {\n componentLeaf: string;\n evalsPath: string;\n evalFile: EvalFile;\n}\n\n/** Find the components in a plugin that declare an `evals` file and load each. */\nexport function discoverEvals(pluginDir: string): DiscoveredEval[] {\n const loaded = loadPluginDir(pluginDir);\n if (!loaded.ok) {\n throw new Error(loaded.issues.map((i) => `${i.path}: ${i.message}`).join(\"\\n\"));\n }\n const out: DiscoveredEval[] = [];\n for (const c of loaded.value.plugin.components) {\n const evalsRel = \"evals\" in c ? c.evals : undefined;\n if (!evalsRel) continue;\n const text = loaded.value.read(evalsRel).toString(\"utf8\");\n const parsed = loadManifest(EvalFile, text, { filename: evalsRel });\n if (!parsed.ok) {\n throw new Error(\n `invalid evals \"${evalsRel}\": ${parsed.issues.map((i) => i.message).join(\"; \")}`,\n );\n }\n out.push({ componentLeaf: leafNameOf(c), evalsPath: evalsRel, evalFile: parsed.value });\n }\n return out;\n}\n\nasync function sh(cmd: string, cwd: string): Promise<number> {\n const r = await execa(\"bash\", [\"-lc\", cmd], { cwd, reject: false, timeout: 60_000 });\n return r.exitCode ?? 1;\n}\n\ninterface CaseContext {\n judge?: JudgeFn;\n baselineScore?: number;\n timeoutMs?: number;\n}\n\nasync function runCase(\n c: Case,\n driver: HarnessDriver,\n cwd: string,\n ctx: CaseContext,\n): Promise<CaseResult> {\n if (c.setup) await sh(c.setup, cwd);\n\n const transcripts: Transcript[] = [];\n for (let i = 0; i < c.samples; i++) {\n transcripts.push(await driver.run({ prompt: c.prompt, cwd, timeoutMs: ctx.timeoutMs }));\n }\n\n // Pass 1: evaluate the deterministic tier (trace/output) to get the case score.\n const results: (AssertResult | undefined)[] = new Array(c.assert.length);\n let detTotal = 0;\n let detPass = 0;\n for (let i = 0; i < c.assert.length; i++) {\n const a = c.assert[i];\n if (a.kind === \"trace\" || a.kind === \"output\") {\n const r = await evaluateAssertion(a, transcripts);\n results[i] = r;\n detTotal++;\n if (r.status === \"pass\") detPass++;\n }\n }\n const score = detTotal > 0 ? detPass / detTotal : 1;\n\n // Pass 2: judge (advisory unless gated) + differential (score vs baseline).\n for (let i = 0; i < c.assert.length; i++) {\n const a = c.assert[i];\n if (a.kind === \"judge\" || a.kind === \"differential\") {\n results[i] = await evaluateAssertion(a, transcripts, {\n judge: ctx.judge,\n caseScore: score,\n baselineScore: ctx.baselineScore,\n });\n }\n }\n const assertions = results.filter((r): r is AssertResult => r !== undefined);\n\n let verifyPassed: boolean | undefined;\n if (c.verify) verifyPassed = (await sh(c.verify, cwd)) === 0;\n if (c.cleanup) await sh(c.cleanup, cwd);\n\n // A case passes iff no assertion FAILED and any post-state verify passed.\n // degraded/skipped assertions do not fail the case (they are reported).\n const pass = !assertions.some((a) => a.status === \"fail\") && (verifyPassed ?? true);\n return { name: c.name, assertions, verifyPassed, score, pass };\n}\n\nexport interface RunEvalOptions {\n evalFile: EvalFile;\n pluginDir: string;\n componentLeaf: string;\n registry: AdapterRegistry;\n drivers: Record<Target, HarnessDriver>;\n scratchRoot?: string;\n timeoutMs?: number;\n /** Judge model for `judge` assertions (advisory unless gated). Omit to skip. */\n judge?: JudgeFn;\n /** Snapshot each harness's mean score into evals/.baselines/ (spec §9.5). */\n snapshotBaselines?: boolean;\n}\n\n/**\n * Drive the real harnesses headlessly and assert over what each did (spec §9.5).\n * A harness with no available driver is reported UNTESTED -- never faked. For each\n * tested harness the component is installed into a throwaway scratch project so the\n * harness loads it, then every case runs and is evaluated.\n */\nexport async function runEval(opts: RunEvalOptions): Promise<EvalReport> {\n const { evalFile, drivers } = opts;\n const harnesses: HarnessReport[] = [];\n\n for (const harness of evalFile.harnesses) {\n const driver = drivers[harness];\n if (!(driver && (await driver.available()))) {\n harnesses.push({\n harness,\n status: \"untested\",\n reason: driver ? \"CLI not installed or not headless-capable\" : \"no driver for this harness\",\n cases: [],\n score: 0,\n pass: false,\n });\n continue;\n }\n\n const baselineScore = loadBaseline(opts.pluginDir, evalFile.component, harness)?.score;\n const scratch = mkdtempSync(join(opts.scratchRoot ?? tmpdir(), `loom-eval-${harness}-`));\n try {\n await install({\n pluginDir: opts.pluginDir,\n scope: \"project\",\n cwd: scratch,\n registry: opts.registry,\n targets: [harness],\n only: [opts.componentLeaf],\n // Keep the source plugin pristine -- write the eval lock into the scratch dir.\n lockDir: scratch,\n });\n const cases: CaseResult[] = [];\n for (const c of evalFile.cases) {\n cases.push(\n await runCase(c, driver, scratch, {\n judge: opts.judge,\n baselineScore,\n timeoutMs: opts.timeoutMs,\n }),\n );\n }\n const score = cases.length > 0 ? cases.reduce((s, c) => s + c.score, 0) / cases.length : 1;\n if (opts.snapshotBaselines) {\n const snapshot: Baseline = { version: \"current\", score };\n writeBaseline(opts.pluginDir, evalFile.component, harness, snapshot);\n }\n harnesses.push({ harness, status: \"tested\", cases, score, pass: cases.every((c) => c.pass) });\n } catch (err) {\n harnesses.push({\n harness,\n status: \"untested\",\n reason: `install failed: ${(err as Error).message}`,\n cases: [],\n score: 0,\n pass: false,\n });\n } finally {\n rmSync(scratch, { recursive: true, force: true });\n }\n }\n\n return { component: evalFile.component, harnesses };\n}\n"]}
1
+ {"version":3,"sources":["../src/assert.ts","../src/baselines.ts","../src/drivers/parse.ts","../src/drivers/util.ts","../src/drivers/claude.ts","../src/drivers/codex.ts","../src/drivers/copilot.ts","../src/drivers/cursor.ts","../src/drivers/opencode.ts","../src/drivers/index.ts","../src/runner.ts"],"names":["execa","drivers","join"],"mappings":";;;;;;;;AAwCA,SAAS,aAAA,CAAc,KAAe,GAAA,EAAwB;AAC5D,EAAA,IAAI,CAAA,GAAI,CAAA;AACR,EAAA,KAAA,MAAW,QAAQ,GAAA,EAAK;AACtB,IAAA,IAAI,IAAA,KAAS,GAAA,CAAI,CAAC,CAAA,EAAG,CAAA,EAAA;AACrB,IAAA,IAAI,CAAA,KAAM,GAAA,CAAI,MAAA,EAAQ,OAAO,IAAA;AAAA,EAC/B;AACA,EAAA,OAAO,IAAI,MAAA,KAAW,CAAA;AACxB;AAEA,SAAS,iBAAA,CAAkB,GAAgB,CAAA,EAAwB;AACjE,EAAA,MAAM,QAAQ,CAAA,CAAE,SAAA,CAAU,IAAI,CAAC,CAAA,KAAM,EAAE,IAAI,CAAA;AAC3C,EAAA,IAAI,CAAA,CAAE,cAAc,CAAC,KAAA,CAAM,SAAS,CAAA,CAAE,UAAU,GAAG,OAAO,KAAA;AAC1D,EAAA,IAAI,EAAE,aAAA,IAAiB,KAAA,CAAM,SAAS,CAAA,CAAE,aAAa,GAAG,OAAO,KAAA;AAC/D,EAAA,IAAI,CAAA,CAAE,aAAa,MAAA,IAAa,CAAA,CAAE,UAAU,MAAA,GAAS,CAAA,CAAE,UAAU,OAAO,KAAA;AACxE,EAAA,IAAI,CAAA,CAAE,YAAY,CAAC,aAAA,CAAc,EAAE,QAAA,EAAU,KAAK,GAAG,OAAO,KAAA;AAC5D,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,aAAA,CAAc,GAAgB,WAAA,EAAyC;AAC9E,EAAA,IAAI,WAAA,CAAY,SAAS,CAAA,IAAK,WAAA,CAAY,MAAM,CAAC,CAAA,KAAM,CAAA,CAAE,gBAAgB,CAAA,EAAG;AAC1E,IAAA,OAAO;AAAA,MACL,IAAA,EAAM,OAAA;AAAA,MACN,MAAA,EAAQ,UAAA;AAAA,MACR,MAAA,EAAQ;AAAA,KACV;AAAA,EACF;AACA,EAAA,MAAM,MAAA,GAAS,YAAY,MAAA,CAAO,CAAC,MAAM,iBAAA,CAAkB,CAAA,EAAG,CAAC,CAAC,CAAA,CAAE,MAAA;AAClE,EAAA,MAAM,OAAO,WAAA,CAAY,MAAA,GAAS,CAAA,GAAI,MAAA,GAAS,YAAY,MAAA,GAAS,CAAA;AACpE,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,OAAA;AAAA,IACN,MAAA,EAAQ,IAAA,IAAQ,CAAA,CAAE,WAAA,GAAc,MAAA,GAAS,MAAA;AAAA,IACzC,MAAA,EAAQ,GAAG,MAAM,CAAA,CAAA,EAAI,YAAY,MAAM,CAAA,6BAAA,EAAgC,EAAE,WAAW,CAAA,CAAA;AAAA,GACtF;AACF;AAEA,SAAS,aAAa,OAAA,EAAgC;AACpD,EAAA,MAAM,CAAA,GAAI,iBAAA,CAAkB,IAAA,CAAK,OAAO,CAAA;AACxC,EAAA,IAAI;AACF,IAAA,OAAO,IAAI,IAAI,MAAA,CAAO,OAAA,CAAQ,KAAA,CAAM,EAAE,CAAC,CAAA,CAAE,MAAM,CAAA,EAAG,EAAE,CAAC,CAAC,CAAA,GAAI,IAAI,OAAO,OAAO,CAAA;AAAA,EAC9E,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,IAAA;AAAA,EACT;AACF;AAEA,SAAS,kBAAA,CAAmB,GAAiB,CAAA,EAAwB;AACnE,EAAA,MAAM,OAAO,CAAA,CAAE,SAAA;AACf,EAAA,IAAI,CAAA,CAAE,MAAA,KAAW,MAAA,IAAa,IAAA,CAAK,IAAA,OAAW,CAAA,CAAE,MAAA,CAAO,IAAA,EAAK,EAAG,OAAO,KAAA;AACtE,EAAA,IAAI,CAAA,CAAE,OAAA,KAAY,MAAA,IAAa,CAAC,YAAA,CAAa,CAAA,CAAE,OAAO,CAAA,EAAG,IAAA,CAAK,IAAI,CAAA,EAAG,OAAO,KAAA;AAC5E,EAAA,IAAI,CAAA,CAAE,eAAe,MAAA,EAAW;AAC9B,IAAA,IAAI;AACF,MAAA,IAAA,CAAK,MAAM,IAAI,CAAA;AAAA,IACjB,CAAA,CAAA,MAAQ;AACN,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AACA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,cAAA,CAAe,GAAiB,WAAA,EAAyC;AAChF,EAAA,IAAI,WAAA,CAAY,MAAA,KAAW,CAAA,EAAG,OAAO,EAAE,MAAM,QAAA,EAAU,MAAA,EAAQ,MAAA,EAAQ,MAAA,EAAQ,YAAA,EAAa;AAC5F,EAAA,MAAM,MAAA,GAAS,YAAY,MAAA,CAAO,CAAC,MAAM,kBAAA,CAAmB,CAAA,EAAG,CAAC,CAAC,CAAA,CAAE,MAAA;AACnE,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,QAAA;AAAA,IACN,MAAA,EAAQ,MAAA,KAAW,WAAA,CAAY,MAAA,GAAS,MAAA,GAAS,MAAA;AAAA,IACjD,MAAA,EAAQ,CAAA,EAAG,MAAM,CAAA,CAAA,EAAI,YAAY,MAAM,CAAA,gBAAA;AAAA,GACzC;AACF;AAEA,eAAe,aAAA,CACb,CAAA,EACA,WAAA,EACA,GAAA,EACuB;AACvB,EAAA,IAAI,CAAC,IAAI,KAAA,EAAO;AACd,IAAA,OAAO,EAAE,IAAA,EAAM,OAAA,EAAS,MAAA,EAAQ,SAAA,EAAW,QAAQ,sCAAA,EAAuC;AAAA,EAC5F;AACA,EAAA,IAAI,MAAA,GAAS,CAAA;AACb,EAAA,KAAA,MAAW,KAAK,WAAA,EAAa;AAC3B,IAAA,MAAM,CAAA,GAAI,MAAM,GAAA,CAAI,KAAA,CAAM;AAAA,MACxB,WAAW,CAAA,CAAE,SAAA;AAAA,MACb,WAAW,CAAA,CAAE,SAAA;AAAA,MACb,QAAQ,CAAA,CAAE,MAAA;AAAA,MACV,MAAM,CAAA,CAAE,IAAA;AAAA,MACR,SAAS,CAAA,CAAE;AAAA,KACZ,CAAA;AACD,IAAA,IAAI,EAAE,IAAA,EAAM,MAAA,EAAA;AAAA,EACd;AACA,EAAA,MAAM,EAAA,GAAK,MAAA,GAAS,CAAA,GAAI,WAAA,CAAY,MAAA;AAEpC,EAAA,MAAM,SAAuB,CAAA,CAAE,IAAA,GAAQ,KAAK,MAAA,GAAS,MAAA,GAAU,KAAK,MAAA,GAAS,SAAA;AAC7E,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,OAAA;AAAA,IACN,MAAA;AAAA,IACA,MAAA,EAAQ,CAAA,EAAG,MAAM,CAAA,CAAA,EAAI,WAAA,CAAY,MAAM,CAAA,oBAAA,EAAuB,CAAA,CAAE,IAAA,GAAO,EAAA,GAAK,aAAa,CAAA;AAAA,GAC3F;AACF;AAEA,SAAS,oBAAA,CAAqB,GAAuB,GAAA,EAAkC;AACrF,EAAA,IAAI,GAAA,CAAI,kBAAkB,MAAA,EAAW;AACnC,IAAA,OAAO;AAAA,MACL,IAAA,EAAM,cAAA;AAAA,MACN,MAAA,EAAQ,SAAA;AAAA,MACR,MAAA,EAAQ;AAAA,KACV;AAAA,EACF;AACA,EAAA,MAAM,KAAA,GAAQ,IAAI,SAAA,IAAa,CAAA;AAC/B,EAAA,MAAM,EAAA,GAAK,KAAA,GAAQ,GAAA,CAAI,aAAA,IAAiB,CAAC,CAAA,CAAE,WAAA;AAC3C,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,cAAA;AAAA,IACN,MAAA,EAAQ,KAAK,MAAA,GAAS,MAAA;AAAA,IACtB,MAAA,EAAQ,CAAA,MAAA,EAAS,KAAA,CAAM,OAAA,CAAQ,CAAC,CAAC,CAAA,aAAA,EAAgB,GAAA,CAAI,aAAA,CAAc,OAAA,CAAQ,CAAC,CAAC,CAAA,cAAA,EAAiB,EAAE,WAAW,CAAA,CAAA;AAAA,GAC7G;AACF;AASA,eAAsB,iBAAA,CACpB,CAAA,EACA,WAAA,EACA,GAAA,GAAqB,EAAC,EACC;AACvB,EAAA,QAAQ,EAAE,IAAA;AAAM,IACd,KAAK,OAAA;AACH,MAAA,OAAO,aAAA,CAAc,GAAG,WAAW,CAAA;AAAA,IACrC,KAAK,QAAA;AACH,MAAA,OAAO,cAAA,CAAe,GAAG,WAAW,CAAA;AAAA,IACtC,KAAK,OAAA;AACH,MAAA,OAAO,aAAA,CAAc,CAAA,EAAG,WAAA,EAAa,GAAG,CAAA;AAAA,IAC1C,KAAK,cAAA;AACH,MAAA,OAAO,oBAAA,CAAqB,GAAG,GAAG,CAAA;AAAA;AAExC;ACvKA,SAAS,YAAA,CAAa,SAAA,EAAmB,SAAA,EAAmB,OAAA,EAAyB;AACnF,EAAA,MAAM,IAAA,GAAO,SAAA,CAAU,OAAA,CAAQ,UAAA,EAAY,GAAG,CAAA;AAC9C,EAAA,OAAO,KAAK,SAAA,EAAW,OAAA,EAAS,cAAc,IAAA,EAAM,CAAA,EAAG,OAAO,CAAA,KAAA,CAAO,CAAA;AACvE;AAGO,SAAS,YAAA,CACd,SAAA,EACA,SAAA,EACA,OAAA,EACiB;AACjB,EAAA,MAAM,CAAA,GAAI,YAAA,CAAa,SAAA,EAAW,SAAA,EAAW,OAAO,CAAA;AACpD,EAAA,IAAI,CAAC,UAAA,CAAW,CAAC,CAAA,EAAG,OAAO,IAAA;AAC3B,EAAA,IAAI;AACF,IAAA,OAAO,IAAA,CAAK,KAAA,CAAM,YAAA,CAAa,CAAA,EAAG,MAAM,CAAC,CAAA;AAAA,EAC3C,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,IAAA;AAAA,EACT;AACF;AAMO,SAAS,aAAA,CACd,SAAA,EACA,SAAA,EACA,OAAA,EACA,QAAA,EACQ;AACR,EAAA,MAAM,CAAA,GAAI,YAAA,CAAa,SAAA,EAAW,SAAA,EAAW,OAAO,CAAA;AACpD,EAAA,SAAA,CAAU,QAAQ,CAAC,CAAA,EAAG,EAAE,SAAA,EAAW,MAAM,CAAA;AACzC,EAAA,aAAA,CAAc,GAAG,CAAA,EAAG,IAAA,CAAK,UAAU,QAAA,EAAU,IAAA,EAAM,CAAC,CAAC;AAAA,CAAI,CAAA;AACzD,EAAA,OAAO,CAAA;AACT;;;ACxCO,SAAS,WAAW,GAAA,EAAwC;AACjE,EAAA,MAAM,MAAiC,EAAC;AACxC,EAAA,KAAA,MAAW,IAAA,IAAQ,GAAA,CAAI,KAAA,CAAM,IAAI,CAAA,EAAG;AAClC,IAAA,MAAM,CAAA,GAAI,KAAK,IAAA,EAAK;AACpB,IAAA,IAAI,CAAC,CAAA,EAAG;AACR,IAAA,IAAI;AACF,MAAA,GAAA,CAAI,IAAA,CAAK,IAAA,CAAK,KAAA,CAAM,CAAC,CAAC,CAAA;AAAA,IACxB,CAAA,CAAA,MAAQ;AAAA,IAER;AAAA,EACF;AACA,EAAA,OAAO,GAAA;AACT;AAOO,SAAS,kBAAkB,GAAA,EAA2D;AAC3F,EAAA,MAAM,YAAwB,EAAC;AAC/B,EAAA,IAAI,SAAA,GAAY,EAAA;AAChB,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,MAAW,GAAA,IAAO,UAAA,CAAW,GAAG,CAAA,EAAG;AACjC,IAAA,IAAI,GAAA,CAAI,SAAS,WAAA,EAAa;AAC5B,MAAA,MAAM,OAAA,GAAW,GAAA,CAAI,OAAA,EAAiD,OAAA,IAAW,EAAC;AAClF,MAAA,KAAA,MAAW,SAAS,OAAA,EAAsC;AACxD,QAAA,IAAI,KAAA,EAAO,SAAS,UAAA,EAAY;AAC9B,UAAA,SAAA,CAAU,IAAA,CAAK,EAAE,IAAA,EAAM,MAAA,CAAO,KAAA,CAAM,IAAI,CAAA,EAAG,IAAA,EAAM,KAAA,CAAM,KAAA,EAAO,EAAA,EAAI,EAAA,EAAA,EAAM,CAAA;AAAA,QAC1E;AAAA,MACF;AAAA,IACF,WAAW,GAAA,CAAI,IAAA,KAAS,YAAY,OAAO,GAAA,CAAI,WAAW,QAAA,EAAU;AAClE,MAAA,SAAA,GAAY,GAAA,CAAI,MAAA;AAAA,IAClB;AAAA,EACF;AACA,EAAA,OAAO,EAAE,WAAW,SAAA,EAAU;AAChC;AAOO,SAAS,iBAAiB,GAAA,EAA2D;AAC1F,EAAA,MAAM,YAAwB,EAAC;AAC/B,EAAA,IAAI,SAAA,GAAY,EAAA;AAChB,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,MAAW,GAAA,IAAO,UAAA,CAAW,GAAG,CAAA,EAAG;AACjC,IAAA,IAAI,GAAA,CAAI,SAAS,gBAAA,EAAkB;AACnC,IAAA,MAAM,OAAO,GAAA,CAAI,IAAA;AACjB,IAAA,IAAI,CAAC,IAAA,EAAM;AACX,IAAA,MAAM,IAAA,GAAO,MAAA,CAAO,IAAA,CAAK,IAAI,CAAA;AAC7B,IAAA,IAAI,SAAS,eAAA,EAAiB;AAC5B,MAAA,SAAA,GAAY,MAAA,CAAO,IAAA,CAAK,IAAA,IAAQ,IAAA,CAAK,WAAW,SAAS,CAAA;AAAA,IAC3D,CAAA,MAAA,IAAW,SAAS,WAAA,EAAa;AAC/B,MAAA,MAAM,IAAA,GAAO,SAAS,eAAA,GAAkB,MAAA,CAAO,KAAK,IAAA,IAAQ,IAAA,CAAK,IAAA,IAAQ,IAAI,CAAA,GAAI,IAAA;AACjF,MAAA,SAAA,CAAU,KAAK,EAAE,IAAA,EAAM,MAAM,IAAA,EAAM,EAAA,EAAI,MAAM,CAAA;AAAA,IAC/C;AAAA,EACF;AACA,EAAA,OAAO,EAAE,WAAW,SAAA,EAAU;AAChC;AAGA,SAAS,eAAe,GAAA,EAAsC;AAC5D,EAAA,MAAM,EAAA,GAAM,GAAA,CAAI,SAAA,IAAa,GAAA,CAAI,QAAA;AACjC,EAAA,IAAI,EAAA,IAAM,OAAO,EAAA,KAAO,QAAA,EAAU;AAEhC,IAAA,MAAM,GAAA,GAAM,MAAA,CAAO,IAAA,CAAK,EAAE,CAAA,CAAE,IAAA,CAAK,CAAC,CAAA,KAAM,WAAA,CAAY,IAAA,CAAK,CAAC,CAAC,CAAA;AAC3D,IAAA,IAAI,GAAA,EAAK,OAAO,GAAA,CAAI,OAAA,CAAQ,aAAa,EAAE,CAAA;AAC3C,IAAA,IAAI,OAAO,EAAA,CAAG,IAAA,KAAS,QAAA,SAAiB,EAAA,CAAG,IAAA;AAAA,EAC7C;AACA,EAAA,OAAO,OAAO,GAAA,CAAI,IAAA,KAAS,QAAA,GAAW,IAAI,IAAA,GAAO,MAAA;AACnD;AAMO,SAAS,kBAAkB,GAAA,EAA2D;AAC3F,EAAA,MAAM,YAAwB,EAAC;AAC/B,EAAA,IAAI,SAAA,GAAY,EAAA;AAChB,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,MAAW,GAAA,IAAO,UAAA,CAAW,GAAG,CAAA,EAAG;AACjC,IAAA,IAAI,GAAA,CAAI,SAAS,WAAA,KAAgB,GAAA,CAAI,YAAY,WAAA,IAAe,GAAA,CAAI,YAAY,MAAA,CAAA,EAAY;AAC1F,MAAA,SAAA,CAAU,IAAA,CAAK,EAAE,IAAA,EAAM,cAAA,CAAe,GAAG,CAAA,EAAG,IAAA,EAAM,GAAA,CAAI,SAAA,IAAa,GAAA,EAAK,EAAA,EAAI,EAAA,EAAA,EAAM,CAAA;AAAA,IACpF,WAAW,GAAA,CAAI,IAAA,KAAS,YAAY,OAAO,GAAA,CAAI,WAAW,QAAA,EAAU;AAClE,MAAA,SAAA,GAAY,GAAA,CAAI,MAAA;AAAA,IAClB;AAAA,EACF;AACA,EAAA,OAAO,EAAE,WAAW,SAAA,EAAU;AAChC;AAOO,SAAS,oBAAoB,GAAA,EAA2D;AAC7F,EAAA,MAAM,YAAwB,EAAC;AAC/B,EAAA,IAAI,SAAA,GAAY,EAAA;AAChB,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,MAAW,GAAA,IAAO,UAAA,CAAW,GAAG,CAAA,EAAG;AACjC,IAAA,IAAI,GAAA,CAAI,SAAS,UAAA,EAAY;AAC3B,MAAA,MAAM,IAAA,GAAQ,IAAI,IAAA,IAAQ,GAAA;AAC1B,MAAA,MAAM,QAAQ,IAAA,CAAK,KAAA;AACnB,MAAA,SAAA,CAAU,IAAA,CAAK;AAAA,QACb,IAAA,EAAM,MAAA,CAAO,IAAA,CAAK,IAAA,IAAQ,MAAM,CAAA;AAAA,QAChC,MAAM,KAAA,EAAO,KAAA;AAAA,QACb,QAAQ,KAAA,EAAO,MAAA;AAAA,QACf,EAAA,EAAI,EAAA;AAAA,OACL,CAAA;AAAA,IACH,CAAA,MAAA,IAAW,GAAA,CAAI,IAAA,KAAS,MAAA,EAAQ;AAC9B,MAAA,MAAM,OAAO,GAAA,CAAI,IAAA;AACjB,MAAA,SAAA,IAAa,MAAA,CAAO,GAAA,CAAI,IAAA,IAAQ,IAAA,EAAM,QAAQ,EAAE,CAAA;AAAA,IAClD;AAAA,EACF;AACA,EAAA,OAAO,EAAE,WAAW,SAAA,EAAU;AAChC;ACrHA,eAAsB,YAAA,CAAa,KAAa,WAAA,EAAyC;AACvF,EAAA,IAAI;AACF,IAAA,MAAM,CAAA,GAAI,MAAM,KAAA,CAAM,GAAA,EAAK,WAAA,EAAa,EAAE,MAAA,EAAQ,KAAA,EAAO,OAAA,EAAS,GAAA,EAAQ,CAAA;AAC1E,IAAA,OAAO,EAAE,QAAA,KAAa,CAAA;AAAA,EACxB,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,KAAA;AAAA,EACT;AACF;AASA,eAAsB,MAAA,CACpB,GAAA,EACA,IAAA,EACA,IAAA,EACoB;AACpB,EAAA,IAAI;AACF,IAAA,MAAM,CAAA,GAAI,MAAM,KAAA,CAAM,GAAA,EAAK,IAAA,EAAM;AAAA,MAC/B,KAAK,IAAA,CAAK,GAAA;AAAA,MACV,MAAA,EAAQ,KAAA;AAAA,MACR,OAAA,EAAS,KAAK,SAAA,IAAa,IAAA;AAAA,MAC3B,GAAA,EAAK,IAAA,CAAK,MAAA,IAAU;AAAC,KACtB,CAAA;AACD,IAAA,OAAO,EAAE,MAAA,EAAQ,CAAA,CAAE,MAAA,IAAU,EAAA,EAAI,MAAA,EAAQ,CAAA,CAAE,MAAA,IAAU,EAAA,EAAI,QAAA,EAAU,CAAA,CAAE,QAAA,IAAY,CAAA,EAAE;AAAA,EACrF,SAAS,GAAA,EAAK;AACZ,IAAA,OAAO,EAAE,MAAA,EAAQ,EAAA,EAAI,QAAS,GAAA,CAAc,OAAA,EAAS,UAAU,CAAA,EAAE;AAAA,EACnE;AACF;;;AC5BA,IAAM,aAAA,GAAgB,+CAAA;AAEf,IAAM,YAAA,GAA8B;AAAA,EACzC,MAAA,EAAQ,QAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,QAAA,EAAU,CAAC,WAAW,CAAC,CAAA;AAAA,EACrD,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,MAAM,MAAM,MAAA;AAAA,MAChB,QAAA;AAAA,MACA;AAAA,QACE,IAAA;AAAA,QACA,MAAA;AAAA,QACA,iBAAA;AAAA,QACA,aAAA;AAAA,QACA,WAAA;AAAA,QACA,mBAAA;AAAA,QACA,aAAA;AAAA,QACA,gBAAA;AAAA,QACA;AAAA,OACF;AAAA,MACA,EAAE,GAAA,EAAK,MAAA,EAAQ,SAAA;AAAU,KAC3B;AACA,IAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAU,GAAI,iBAAA,CAAkB,IAAI,MAAM,CAAA;AAC7D,IAAA,OAAO,EAAE,WAAW,SAAA,EAAW,QAAA,EAAU,IAAI,QAAA,EAAU,GAAA,EAAK,IAAI,MAAA,EAAO;AAAA,EACzE;AACF;;;AC3BO,IAAM,WAAA,GAA6B;AAAA,EACxC,MAAA,EAAQ,OAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,OAAA,EAAS,CAAC,WAAW,CAAC,CAAA;AAAA,EACpD,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,MAAM,MAAM,MAAA;AAAA,MAChB,OAAA;AAAA,MACA,CAAC,MAAA,EAAQ,QAAA,EAAU,4CAAA,EAA8C,MAAM,CAAA;AAAA,MACvE,EAAE,GAAA,EAAK,MAAA,EAAQ,SAAA;AAAU,KAC3B;AACA,IAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAU,GAAI,gBAAA,CAAiB,IAAI,MAAM,CAAA;AAC5D,IAAA,OAAO,EAAE,WAAW,SAAA,EAAW,QAAA,EAAU,IAAI,QAAA,EAAU,GAAA,EAAK,IAAI,MAAA,EAAO;AAAA,EACzE;AACF;;;ACPO,IAAM,aAAA,GAA+B;AAAA,EAC1C,MAAA,EAAQ,SAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,SAAA,EAAW,CAAC,WAAW,CAAC,CAAA;AAAA,EACtD,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,GAAA,GAAM,MAAM,MAAA,CAAO,SAAA,EAAW,CAAC,IAAA,EAAM,MAAA,EAAQ,IAAA,EAAM,aAAa,CAAA,EAAG;AAAA,MACvE,GAAA;AAAA,MACA,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AACD,IAAA,OAAO;AAAA,MACL,SAAA,EAAW,GAAA,CAAI,MAAA,CAAO,IAAA,EAAK;AAAA,MAC3B,WAAW,EAAC;AAAA,MACZ,UAAU,GAAA,CAAI,QAAA;AAAA,MACd,KAAK,GAAA,CAAI,MAAA;AAAA,MACT,gBAAA,EAAkB;AAAA,KACpB;AAAA,EACF;AACF;;;ACtBO,IAAM,YAAA,GAA8B;AAAA,EACzC,MAAA,EAAQ,QAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,cAAA,EAAgB,CAAC,WAAW,CAAC,CAAA;AAAA,EAC3D,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,MAAM,MAAM,MAAA;AAAA,MAChB,cAAA;AAAA,MACA,CAAC,IAAA,EAAM,MAAA,EAAQ,SAAA,EAAW,mBAAmB,aAAa,CAAA;AAAA,MAC1D,EAAE,GAAA,EAAK,MAAA,EAAQ,SAAA;AAAU,KAC3B;AACA,IAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAU,GAAI,iBAAA,CAAkB,IAAI,MAAM,CAAA;AAC7D,IAAA,OAAO,EAAE,WAAW,SAAA,EAAW,QAAA,EAAU,IAAI,QAAA,EAAU,GAAA,EAAK,IAAI,MAAA,EAAO;AAAA,EACzE;AACF;;;ACZO,IAAM,cAAA,GAAgC;AAAA,EAC3C,MAAA,EAAQ,UAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,UAAA,EAAY,CAAC,WAAW,CAAC,CAAA;AAAA,EACvD,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,GAAA,GAAM,MAAM,MAAA,CAAO,UAAA,EAAY,CAAC,KAAA,EAAO,MAAA,EAAQ,UAAA,EAAY,MAAM,CAAA,EAAG;AAAA,MACxE,GAAA;AAAA,MACA,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AACD,IAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAU,GAAI,mBAAA,CAAoB,IAAI,MAAM,CAAA;AAC/D,IAAA,OAAO,EAAE,WAAW,SAAA,EAAW,QAAA,EAAU,IAAI,QAAA,EAAU,GAAA,EAAK,IAAI,MAAA,EAAO;AAAA,EACzE;AACF;;;ACOO,IAAM,OAAA,GAAyC;AAAA,EACpD,MAAA,EAAQ,YAAA;AAAA,EACR,KAAA,EAAO,WAAA;AAAA,EACP,MAAA,EAAQ,YAAA;AAAA,EACR,OAAA,EAAS,aAAA;AAAA,EACT,QAAA,EAAU;AACZ;ACmBO,SAAS,cAAc,SAAA,EAAqC;AACjE,EAAA,MAAM,MAAA,GAAS,cAAc,SAAS,CAAA;AACtC,EAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AACd,IAAA,MAAM,IAAI,KAAA,CAAM,MAAA,CAAO,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,EAAG,CAAA,CAAE,IAAI,KAAK,CAAA,CAAE,OAAO,EAAE,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,EAChF;AACA,EAAA,MAAM,MAAwB,EAAC;AAC/B,EAAA,KAAA,MAAW,CAAA,IAAK,MAAA,CAAO,KAAA,CAAM,MAAA,CAAO,UAAA,EAAY;AAC9C,IAAA,MAAM,QAAA,GAAW,OAAA,IAAW,CAAA,GAAI,CAAA,CAAE,KAAA,GAAQ,MAAA;AAC1C,IAAA,IAAI,CAAC,QAAA,EAAU;AACf,IAAA,MAAM,OAAO,MAAA,CAAO,KAAA,CAAM,KAAK,QAAQ,CAAA,CAAE,SAAS,MAAM,CAAA;AACxD,IAAA,MAAM,SAAS,YAAA,CAAa,QAAA,EAAU,MAAM,EAAE,QAAA,EAAU,UAAU,CAAA;AAClE,IAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AACd,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,eAAA,EAAkB,QAAQ,CAAA,GAAA,EAAM,MAAA,CAAO,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,OAChF;AAAA,IACF;AACA,IAAA,GAAA,CAAI,IAAA,CAAK,EAAE,aAAA,EAAe,UAAA,CAAW,CAAC,CAAA,EAAG,SAAA,EAAW,QAAA,EAAU,QAAA,EAAU,MAAA,CAAO,KAAA,EAAO,CAAA;AAAA,EACxF;AACA,EAAA,OAAO,GAAA;AACT;AAEA,eAAe,EAAA,CAAG,KAAa,GAAA,EAA8B;AAC3D,EAAA,MAAM,CAAA,GAAI,MAAMA,KAAAA,CAAM,MAAA,EAAQ,CAAC,KAAA,EAAO,GAAG,CAAA,EAAG,EAAE,GAAA,EAAK,MAAA,EAAQ,KAAA,EAAO,OAAA,EAAS,KAAQ,CAAA;AACnF,EAAA,OAAO,EAAE,QAAA,IAAY,CAAA;AACvB;AAQA,eAAe,OAAA,CACb,CAAA,EACA,MAAA,EACA,GAAA,EACA,GAAA,EACqB;AACrB,EAAA,IAAI,EAAE,KAAA,EAAO,MAAM,EAAA,CAAG,CAAA,CAAE,OAAO,GAAG,CAAA;AAElC,EAAA,MAAM,cAA4B,EAAC;AACnC,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,SAAS,CAAA,EAAA,EAAK;AAClC,IAAA,WAAA,CAAY,IAAA,CAAK,MAAM,MAAA,CAAO,GAAA,CAAI,EAAE,MAAA,EAAQ,CAAA,CAAE,MAAA,EAAQ,GAAA,EAAK,SAAA,EAAW,GAAA,CAAI,SAAA,EAAW,CAAC,CAAA;AAAA,EACxF;AAGA,EAAA,MAAM,OAAA,GAAwC,IAAI,KAAA,CAAM,CAAA,CAAE,OAAO,MAAM,CAAA;AACvE,EAAA,IAAI,QAAA,GAAW,CAAA;AACf,EAAA,IAAI,OAAA,GAAU,CAAA;AACd,EAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,CAAO,QAAQ,CAAA,EAAA,EAAK;AACxC,IAAA,MAAM,CAAA,GAAI,CAAA,CAAE,MAAA,CAAO,CAAC,CAAA;AACpB,IAAA,IAAI,CAAA,CAAE,IAAA,KAAS,OAAA,IAAW,CAAA,CAAE,SAAS,QAAA,EAAU;AAC7C,MAAA,MAAM,CAAA,GAAI,MAAM,iBAAA,CAAkB,CAAA,EAAG,WAAW,CAAA;AAChD,MAAA,OAAA,CAAQ,CAAC,CAAA,GAAI,CAAA;AACb,MAAA,QAAA,EAAA;AACA,MAAA,IAAI,CAAA,CAAE,WAAW,MAAA,EAAQ,OAAA,EAAA;AAAA,IAC3B;AAAA,EACF;AACA,EAAA,MAAM,KAAA,GAAQ,QAAA,GAAW,CAAA,GAAI,OAAA,GAAU,QAAA,GAAW,CAAA;AAGlD,EAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,CAAO,QAAQ,CAAA,EAAA,EAAK;AACxC,IAAA,MAAM,CAAA,GAAI,CAAA,CAAE,MAAA,CAAO,CAAC,CAAA;AACpB,IAAA,IAAI,CAAA,CAAE,IAAA,KAAS,OAAA,IAAW,CAAA,CAAE,SAAS,cAAA,EAAgB;AACnD,MAAA,OAAA,CAAQ,CAAC,CAAA,GAAI,MAAM,iBAAA,CAAkB,GAAG,WAAA,EAAa;AAAA,QACnD,OAAO,GAAA,CAAI,KAAA;AAAA,QACX,SAAA,EAAW,KAAA;AAAA,QACX,eAAe,GAAA,CAAI;AAAA,OACpB,CAAA;AAAA,IACH;AAAA,EACF;AACA,EAAA,MAAM,aAAa,OAAA,CAAQ,MAAA,CAAO,CAAC,CAAA,KAAyB,MAAM,MAAS,CAAA;AAE3E,EAAA,IAAI,YAAA;AACJ,EAAA,IAAI,CAAA,CAAE,QAAQ,YAAA,GAAgB,MAAM,GAAG,CAAA,CAAE,MAAA,EAAQ,GAAG,CAAA,KAAO,CAAA;AAC3D,EAAA,IAAI,EAAE,OAAA,EAAS,MAAM,EAAA,CAAG,CAAA,CAAE,SAAS,GAAG,CAAA;AAItC,EAAA,MAAM,IAAA,GAAO,CAAC,UAAA,CAAW,IAAA,CAAK,CAAC,MAAM,CAAA,CAAE,MAAA,KAAW,MAAM,CAAA,KAAM,YAAA,IAAgB,IAAA,CAAA;AAC9E,EAAA,OAAO,EAAE,IAAA,EAAM,CAAA,CAAE,MAAM,UAAA,EAAY,YAAA,EAAc,OAAO,IAAA,EAAK;AAC/D;AAsBA,eAAsB,QAAQ,IAAA,EAA2C;AACvE,EAAA,MAAM,EAAE,QAAA,EAAU,OAAA,EAAAC,QAAAA,EAAQ,GAAI,IAAA;AAC9B,EAAA,MAAM,YAA6B,EAAC;AAEpC,EAAA,KAAA,MAAW,OAAA,IAAW,SAAS,SAAA,EAAW;AACxC,IAAA,MAAM,MAAA,GAASA,SAAQ,OAAO,CAAA;AAC9B,IAAA,IAAI,EAAE,MAAA,IAAW,MAAM,MAAA,CAAO,WAAU,CAAA,EAAK;AAC3C,MAAA,SAAA,CAAU,IAAA,CAAK;AAAA,QACb,OAAA;AAAA,QACA,MAAA,EAAQ,UAAA;AAAA,QACR,MAAA,EAAQ,SAAS,2CAAA,GAA8C,4BAAA;AAAA,QAC/D,OAAO,EAAC;AAAA,QACR,KAAA,EAAO,CAAA;AAAA,QACP,IAAA,EAAM;AAAA,OACP,CAAA;AACD,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,gBAAgB,YAAA,CAAa,IAAA,CAAK,WAAW,QAAA,CAAS,SAAA,EAAW,OAAO,CAAA,EAAG,KAAA;AACjF,IAAA,MAAM,OAAA,GAAU,WAAA,CAAYC,IAAAA,CAAK,IAAA,CAAK,WAAA,IAAe,QAAO,EAAG,CAAA,UAAA,EAAa,OAAO,CAAA,CAAA,CAAG,CAAC,CAAA;AACvF,IAAA,IAAI;AACF,MAAA,MAAM,OAAA,CAAQ;AAAA,QACZ,WAAW,IAAA,CAAK,SAAA;AAAA,QAChB,KAAA,EAAO,SAAA;AAAA,QACP,GAAA,EAAK,OAAA;AAAA,QACL,UAAU,IAAA,CAAK,QAAA;AAAA,QACf,OAAA,EAAS,CAAC,OAAO,CAAA;AAAA,QACjB,IAAA,EAAM,CAAC,IAAA,CAAK,aAAa,CAAA;AAAA;AAAA,QAEzB,OAAA,EAAS;AAAA,OACV,CAAA;AACD,MAAA,MAAM,QAAsB,EAAC;AAC7B,MAAA,KAAA,MAAW,CAAA,IAAK,SAAS,KAAA,EAAO;AAC9B,QAAA,KAAA,CAAM,IAAA;AAAA,UACJ,MAAM,OAAA,CAAQ,CAAA,EAAG,MAAA,EAAQ,OAAA,EAAS;AAAA,YAChC,OAAO,IAAA,CAAK,KAAA;AAAA,YACZ,aAAA;AAAA,YACA,WAAW,IAAA,CAAK;AAAA,WACjB;AAAA,SACH;AAAA,MACF;AACA,MAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,MAAA,GAAS,CAAA,GAAI,MAAM,MAAA,CAAO,CAAC,CAAA,EAAG,CAAA,KAAM,IAAI,CAAA,CAAE,KAAA,EAAO,CAAC,CAAA,GAAI,MAAM,MAAA,GAAS,CAAA;AACzF,MAAA,IAAI,KAAK,iBAAA,EAAmB;AAC1B,QAAA,MAAM,QAAA,GAAqB,EAAE,OAAA,EAAS,SAAA,EAAW,KAAA,EAAM;AACvD,QAAA,aAAA,CAAc,IAAA,CAAK,SAAA,EAAW,QAAA,CAAS,SAAA,EAAW,SAAS,QAAQ,CAAA;AAAA,MACrE;AACA,MAAA,SAAA,CAAU,IAAA,CAAK,EAAE,OAAA,EAAS,MAAA,EAAQ,UAAU,KAAA,EAAO,KAAA,EAAO,IAAA,EAAM,KAAA,CAAM,MAAM,CAAC,CAAA,KAAM,CAAA,CAAE,IAAI,GAAG,CAAA;AAAA,IAC9F,SAAS,GAAA,EAAK;AACZ,MAAA,SAAA,CAAU,IAAA,CAAK;AAAA,QACb,OAAA;AAAA,QACA,MAAA,EAAQ,UAAA;AAAA,QACR,MAAA,EAAQ,CAAA,gBAAA,EAAoB,GAAA,CAAc,OAAO,CAAA,CAAA;AAAA,QACjD,OAAO,EAAC;AAAA,QACR,KAAA,EAAO,CAAA;AAAA,QACP,IAAA,EAAM;AAAA,OACP,CAAA;AAAA,IACH,CAAA,SAAE;AACA,MAAA,MAAA,CAAO,SAAS,EAAE,SAAA,EAAW,IAAA,EAAM,KAAA,EAAO,MAAM,CAAA;AAAA,IAClD;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,SAAA,EAAW,QAAA,CAAS,SAAA,EAAW,SAAA,EAAU;AACpD","file":"index.js","sourcesContent":["import type { Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport type {\n Assertion,\n DifferentialAssert,\n JudgeAssert,\n OutputAssert,\n TraceAssert,\n} from \"@michaelfromyeg/loom-schema\";\n\nexport type AssertStatus = \"pass\" | \"fail\" | \"degraded\" | \"skipped\";\n\nexport interface AssertResult {\n kind: Assertion[\"kind\"];\n status: AssertStatus;\n detail: string;\n}\n\nexport interface JudgeInput {\n candidate: string;\n reference?: string;\n rubric: string;\n mode: \"absolute\" | \"pairwise\";\n samples: number;\n}\nexport interface JudgeVerdict {\n pass: boolean;\n detail?: string;\n}\n/** A judge model. Injected so evals run offline/deterministically in tests. */\nexport type JudgeFn = (input: JudgeInput) => Promise<JudgeVerdict>;\n\nexport interface AssertContext {\n judge?: JudgeFn;\n /** Deterministic score of the case (fraction of trace/output passing) -- differential. */\n caseScore?: number;\n /** Baseline score for (component, harness) from evals/.baselines/ -- differential. */\n baselineScore?: number;\n}\n\n/** True iff `seq` appears as an ordered subsequence of `arr`. */\nfunction isSubsequence(seq: string[], arr: string[]): boolean {\n let i = 0;\n for (const item of arr) {\n if (item === seq[i]) i++;\n if (i === seq.length) return true;\n }\n return seq.length === 0;\n}\n\nfunction traceSamplePasses(a: TraceAssert, t: Transcript): boolean {\n const names = t.toolCalls.map((c) => c.name);\n if (a.toolCalled && !names.includes(a.toolCalled)) return false;\n if (a.toolNotCalled && names.includes(a.toolNotCalled)) return false;\n if (a.maxCalls !== undefined && t.toolCalls.length > a.maxCalls) return false;\n if (a.sequence && !isSubsequence(a.sequence, names)) return false;\n return true;\n}\n\nfunction evaluateTrace(a: TraceAssert, transcripts: Transcript[]): AssertResult {\n if (transcripts.length > 0 && transcripts.every((t) => t.traceUnavailable)) {\n return {\n kind: \"trace\",\n status: \"degraded\",\n detail: \"harness exposes no tool-call trace; trace assertion not evaluated\",\n };\n }\n const passes = transcripts.filter((t) => traceSamplePasses(a, t)).length;\n const rate = transcripts.length > 0 ? passes / transcripts.length : 0;\n return {\n kind: \"trace\",\n status: rate >= a.minPassRate ? \"pass\" : \"fail\",\n detail: `${passes}/${transcripts.length} samples passed (minPassRate ${a.minPassRate})`,\n };\n}\n\nfunction compileRegex(pattern: string): RegExp | null {\n const m = /^\\(\\?([a-z]+)\\)/.exec(pattern);\n try {\n return m ? new RegExp(pattern.slice(m[0].length), m[1]) : new RegExp(pattern);\n } catch {\n return null;\n }\n}\n\nfunction outputSamplePasses(a: OutputAssert, t: Transcript): boolean {\n const text = t.finalText;\n if (a.equals !== undefined && text.trim() !== a.equals.trim()) return false;\n if (a.matches !== undefined && !compileRegex(a.matches)?.test(text)) return false;\n if (a.jsonSchema !== undefined) {\n try {\n JSON.parse(text);\n } catch {\n return false;\n }\n }\n return true;\n}\n\nfunction evaluateOutput(a: OutputAssert, transcripts: Transcript[]): AssertResult {\n if (transcripts.length === 0) return { kind: \"output\", status: \"fail\", detail: \"no samples\" };\n const passes = transcripts.filter((t) => outputSamplePasses(a, t)).length;\n return {\n kind: \"output\",\n status: passes === transcripts.length ? \"pass\" : \"fail\",\n detail: `${passes}/${transcripts.length} samples matched`,\n };\n}\n\nasync function evaluateJudge(\n a: JudgeAssert,\n transcripts: Transcript[],\n ctx: AssertContext,\n): Promise<AssertResult> {\n if (!ctx.judge) {\n return { kind: \"judge\", status: \"skipped\", detail: \"no judge model configured (advisory)\" };\n }\n let passes = 0;\n for (const t of transcripts) {\n const v = await ctx.judge({\n candidate: t.finalText,\n reference: a.reference,\n rubric: a.rubric,\n mode: a.mode,\n samples: a.samples,\n });\n if (v.pass) passes++;\n }\n const ok = passes * 2 > transcripts.length; // majority\n // Advisory unless `gate:true` -- a non-gating judge never fails the case.\n const status: AssertStatus = a.gate ? (ok ? \"pass\" : \"fail\") : ok ? \"pass\" : \"skipped\";\n return {\n kind: \"judge\",\n status,\n detail: `${passes}/${transcripts.length} judge verdicts pass${a.gate ? \"\" : \" (advisory)\"}`,\n };\n}\n\nfunction evaluateDifferential(a: DifferentialAssert, ctx: AssertContext): AssertResult {\n if (ctx.baselineScore === undefined) {\n return {\n kind: \"differential\",\n status: \"skipped\",\n detail: \"no baseline (run loom publish to snapshot one)\",\n };\n }\n const score = ctx.caseScore ?? 0;\n const ok = score - ctx.baselineScore >= -a.noWorseThan;\n return {\n kind: \"differential\",\n status: ok ? \"pass\" : \"fail\",\n detail: `score ${score.toFixed(2)} vs baseline ${ctx.baselineScore.toFixed(2)} (noWorseThan ${a.noWorseThan})`,\n };\n}\n\n/**\n * Evaluate one assertion against a case's sample transcripts (spec §9.5). Trace\n * and output are deterministic; judge is advisory unless `gate:true` and needs an\n * injected judge model; differential compares the case score to a committed\n * baseline (the \"vibes\" no-regression gate).\n */\n// biome-ignore lint/suspicious/useAwait: branches mix sync (trace/output) and async (judge/differential) results; async normalizes the return to Promise<AssertResult>.\nexport async function evaluateAssertion(\n a: Assertion,\n transcripts: Transcript[],\n ctx: AssertContext = {},\n): Promise<AssertResult> {\n switch (a.kind) {\n case \"trace\":\n return evaluateTrace(a, transcripts);\n case \"output\":\n return evaluateOutput(a, transcripts);\n case \"judge\":\n return evaluateJudge(a, transcripts, ctx);\n case \"differential\":\n return evaluateDifferential(a, ctx);\n }\n}\n","import { existsSync, mkdirSync, readFileSync, writeFileSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport type { Target } from \"@michaelfromyeg/loom-schema\";\n\nexport interface Baseline {\n version: string;\n score: number;\n}\n\nfunction baselineFile(pluginDir: string, component: string, harness: Target): string {\n const safe = component.replace(/[^\\w.-]/g, \"_\");\n return join(pluginDir, \"evals\", \".baselines\", safe, `${harness}.json`);\n}\n\n/** Load the committed baseline score for (component, harness), or null. */\nexport function loadBaseline(\n pluginDir: string,\n component: string,\n harness: Target,\n): Baseline | null {\n const f = baselineFile(pluginDir, component, harness);\n if (!existsSync(f)) return null;\n try {\n return JSON.parse(readFileSync(f, \"utf8\")) as Baseline;\n } catch {\n return null;\n }\n}\n\n/**\n * Snapshot a baseline (spec §9.5). Called on `loom publish` so the next release's\n * differential evals compare against this score.\n */\nexport function writeBaseline(\n pluginDir: string,\n component: string,\n harness: Target,\n baseline: Baseline,\n): string {\n const f = baselineFile(pluginDir, component, harness);\n mkdirSync(dirname(f), { recursive: true });\n writeFileSync(f, `${JSON.stringify(baseline, null, 2)}\\n`);\n return f;\n}\n","import type { ToolCall } from \"@michaelfromyeg/loom-adapter-kit\";\n\n/** Split NDJSON/JSONL into parsed objects, skipping blank/garbage lines. */\nexport function parseLines(raw: string): Record<string, unknown>[] {\n const out: Record<string, unknown>[] = [];\n for (const line of raw.split(\"\\n\")) {\n const t = line.trim();\n if (!t) continue;\n try {\n out.push(JSON.parse(t));\n } catch {\n // tolerate non-JSON noise lines\n }\n }\n return out;\n}\n\n/**\n * Parse Claude's `--output-format stream-json --verbose` NDJSON. Tool calls are\n * `assistant` message content blocks `{type:\"tool_use\", name, input}`; the final\n * text is the `result` event's `result` field (see harness-research.md).\n */\nexport function parseClaudeStream(raw: string): { finalText: string; toolCalls: ToolCall[] } {\n const toolCalls: ToolCall[] = [];\n let finalText = \"\";\n let ts = 0;\n for (const evt of parseLines(raw)) {\n if (evt.type === \"assistant\") {\n const content = (evt.message as { content?: unknown[] } | undefined)?.content ?? [];\n for (const block of content as Record<string, unknown>[]) {\n if (block?.type === \"tool_use\") {\n toolCalls.push({ name: String(block.name), args: block.input, ts: ts++ });\n }\n }\n } else if (evt.type === \"result\" && typeof evt.result === \"string\") {\n finalText = evt.result;\n }\n }\n return { finalText, toolCalls };\n}\n\n/**\n * Parse Codex `exec --json` JSONL. Tool calls live in `item.completed` events by\n * `item.type`; the final text is the `agent_message` item. Only completed items\n * are counted to avoid double-counting the matching `item.started`.\n */\nexport function parseCodexStream(raw: string): { finalText: string; toolCalls: ToolCall[] } {\n const toolCalls: ToolCall[] = [];\n let finalText = \"\";\n let ts = 0;\n for (const evt of parseLines(raw)) {\n if (evt.type !== \"item.completed\") continue;\n const item = evt.item as Record<string, unknown> | undefined;\n if (!item) continue;\n const kind = String(item.type);\n if (kind === \"agent_message\") {\n finalText = String(item.text ?? item.message ?? finalText);\n } else if (kind !== \"reasoning\") {\n const name = kind === \"mcp_tool_call\" ? String(item.tool ?? item.name ?? kind) : kind;\n toolCalls.push({ name, args: item, ts: ts++ });\n }\n }\n return { finalText, toolCalls };\n}\n\n/** Best-effort tool name from a Cursor `tool_call` event (e.g. {readToolCall:{}} -> \"read\"). */\nfunction cursorToolName(evt: Record<string, unknown>): string {\n const tc = (evt.tool_call ?? evt.toolCall) as Record<string, unknown> | undefined;\n if (tc && typeof tc === \"object\") {\n // TODO(verify): the exact tool-name field of cursor-agent stream-json events.\n const key = Object.keys(tc).find((k) => /ToolCall$/.test(k));\n if (key) return key.replace(/ToolCall$/, \"\");\n if (typeof tc.name === \"string\") return tc.name;\n }\n return typeof evt.name === \"string\" ? evt.name : \"tool\";\n}\n\n/**\n * Parse Cursor `--output-format stream-json` NDJSON. Tool calls are `tool_call`\n * events (count the `completed` subtype); the final text is the `result` event.\n */\nexport function parseCursorStream(raw: string): { finalText: string; toolCalls: ToolCall[] } {\n const toolCalls: ToolCall[] = [];\n let finalText = \"\";\n let ts = 0;\n for (const evt of parseLines(raw)) {\n if (evt.type === \"tool_call\" && (evt.subtype === \"completed\" || evt.subtype === undefined)) {\n toolCalls.push({ name: cursorToolName(evt), args: evt.tool_call ?? evt, ts: ts++ });\n } else if (evt.type === \"result\" && typeof evt.result === \"string\") {\n finalText = evt.result;\n }\n }\n return { finalText, toolCalls };\n}\n\n/**\n * Parse OpenCode `run --format json` JSONL. Tool calls are `tool_use` events\n * carrying a ToolPart (`part.tool`, `part.state.{input,output}`); the final text\n * is the concatenation of `text` events.\n */\nexport function parseOpencodeStream(raw: string): { finalText: string; toolCalls: ToolCall[] } {\n const toolCalls: ToolCall[] = [];\n let finalText = \"\";\n let ts = 0;\n for (const evt of parseLines(raw)) {\n if (evt.type === \"tool_use\") {\n const part = (evt.part ?? evt) as Record<string, unknown>;\n const state = part.state as Record<string, unknown> | undefined;\n toolCalls.push({\n name: String(part.tool ?? \"tool\"),\n args: state?.input,\n result: state?.output,\n ts: ts++,\n });\n } else if (evt.type === \"text\") {\n const part = evt.part as { text?: string } | undefined;\n finalText += String(evt.text ?? part?.text ?? \"\");\n }\n }\n return { finalText, toolCalls };\n}\n","import { execa } from \"execa\";\n\n/** True iff `cmd <versionArgs>` exits 0 (CLI installed). Never throws. */\nexport async function cliAvailable(cmd: string, versionArgs: string[]): Promise<boolean> {\n try {\n const r = await execa(cmd, versionArgs, { reject: false, timeout: 10_000 });\n return r.exitCode === 0;\n } catch {\n return false;\n }\n}\n\nexport interface CliResult {\n stdout: string;\n stderr: string;\n exitCode: number;\n}\n\n/** Run a harness CLI headlessly. Resolved config is passed as env; never throws. */\nexport async function runCli(\n cmd: string,\n args: string[],\n opts: { cwd: string; config?: Record<string, string>; timeoutMs?: number },\n): Promise<CliResult> {\n try {\n const r = await execa(cmd, args, {\n cwd: opts.cwd,\n reject: false,\n timeout: opts.timeoutMs ?? 120_000,\n env: opts.config ?? {},\n });\n return { stdout: r.stdout ?? \"\", stderr: r.stderr ?? \"\", exitCode: r.exitCode ?? 1 };\n } catch (err) {\n return { stdout: \"\", stderr: (err as Error).message, exitCode: 1 };\n }\n}\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { parseClaudeStream } from \"./parse\";\nimport { cliAvailable, runCli } from \"./util\";\n\n// Include \"Skill\" so an installed plugin's skill can actually activate during an\n// eval; without it, headless claude answers from general knowledge and a skill is\n// never exercised (the whole point of the eval).\nconst ALLOWED_TOOLS = \"Read,Edit,Write,Bash,Grep,Glob,WebFetch,Skill\";\n\nexport const claudeDriver: HarnessDriver = {\n target: \"claude\",\n available: () => cliAvailable(\"claude\", [\"--version\"]),\n async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n const res = await runCli(\n \"claude\",\n [\n \"-p\",\n prompt,\n \"--output-format\",\n \"stream-json\",\n \"--verbose\",\n \"--permission-mode\",\n \"acceptEdits\",\n \"--allowedTools\",\n ALLOWED_TOOLS,\n ],\n { cwd, config, timeoutMs },\n );\n const { finalText, toolCalls } = parseClaudeStream(res.stdout);\n return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };\n },\n};\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { parseCodexStream } from \"./parse\";\nimport { cliAvailable, runCli } from \"./util\";\n\nexport const codexDriver: HarnessDriver = {\n target: \"codex\",\n available: () => cliAvailable(\"codex\", [\"--version\"]),\n async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n const res = await runCli(\n \"codex\",\n [\"exec\", \"--json\", \"--dangerously-bypass-approvals-and-sandbox\", prompt],\n { cwd, config, timeoutMs },\n );\n const { finalText, toolCalls } = parseCodexStream(res.stdout);\n return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };\n },\n};\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { cliAvailable, runCli } from \"./util\";\n\n/**\n * GitHub Copilot's headless `-p` mode exposes NO structured tool-call trace today\n * (only a Markdown transcript via --share). So `run` returns the final text with\n * `traceUnavailable: true`, and the runner degrades `trace` assertions to `output`\n * for this harness rather than faking a pass (spec §14, harness-research.md).\n */\nexport const copilotDriver: HarnessDriver = {\n target: \"copilot\",\n available: () => cliAvailable(\"copilot\", [\"--version\"]),\n async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n const res = await runCli(\"copilot\", [\"-p\", prompt, \"-s\", \"--allow-all\"], {\n cwd,\n config,\n timeoutMs,\n });\n return {\n finalText: res.stdout.trim(),\n toolCalls: [],\n exitCode: res.exitCode,\n raw: res.stdout,\n traceUnavailable: true,\n };\n },\n};\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { parseCursorStream } from \"./parse\";\nimport { cliAvailable, runCli } from \"./util\";\n\nexport const cursorDriver: HarnessDriver = {\n target: \"cursor\",\n available: () => cliAvailable(\"cursor-agent\", [\"--version\"]),\n async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n const res = await runCli(\n \"cursor-agent\",\n [\"-p\", prompt, \"--force\", \"--output-format\", \"stream-json\"],\n { cwd, config, timeoutMs },\n );\n const { finalText, toolCalls } = parseCursorStream(res.stdout);\n return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };\n },\n};\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { parseOpencodeStream } from \"./parse\";\nimport { cliAvailable, runCli } from \"./util\";\n\nexport const opencodeDriver: HarnessDriver = {\n target: \"opencode\",\n available: () => cliAvailable(\"opencode\", [\"--version\"]),\n async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n const res = await runCli(\"opencode\", [\"run\", prompt, \"--format\", \"json\"], {\n cwd,\n config,\n timeoutMs,\n });\n const { finalText, toolCalls } = parseOpencodeStream(res.stdout);\n return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };\n },\n};\n","import type { HarnessDriver } from \"@michaelfromyeg/loom-adapter-kit\";\nimport type { Target } from \"@michaelfromyeg/loom-schema\";\nimport { claudeDriver } from \"./claude\";\nimport { codexDriver } from \"./codex\";\nimport { copilotDriver } from \"./copilot\";\nimport { cursorDriver } from \"./cursor\";\nimport { opencodeDriver } from \"./opencode\";\n\nexport { claudeDriver } from \"./claude\";\nexport { codexDriver } from \"./codex\";\nexport { copilotDriver } from \"./copilot\";\nexport { cursorDriver } from \"./cursor\";\nexport { opencodeDriver } from \"./opencode\";\nexport {\n parseClaudeStream,\n parseCodexStream,\n parseCursorStream,\n parseLines,\n parseOpencodeStream,\n} from \"./parse\";\nexport type { CliResult } from \"./util\";\n\n/** Every built-in headless driver, keyed by Target. */\nexport const drivers: Record<Target, HarnessDriver> = {\n claude: claudeDriver,\n codex: codexDriver,\n cursor: cursorDriver,\n copilot: copilotDriver,\n opencode: opencodeDriver,\n};\n","import { mkdtempSync, rmSync } from \"node:fs\";\nimport { tmpdir } from \"node:os\";\nimport { join } from \"node:path\";\nimport type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { type AdapterRegistry, install, loadPluginDir } from \"@michaelfromyeg/loom-core\";\nimport {\n type Case,\n EvalFile,\n leafNameOf,\n loadManifest,\n type Target,\n} from \"@michaelfromyeg/loom-schema\";\nimport { execa } from \"execa\";\nimport { type AssertResult, evaluateAssertion, type JudgeFn } from \"./assert\";\nimport { type Baseline, loadBaseline, writeBaseline } from \"./baselines\";\n\nexport interface CaseResult {\n name: string;\n assertions: AssertResult[];\n verifyPassed?: boolean;\n /** Deterministic score: fraction of trace/output assertions that passed. */\n score: number;\n pass: boolean;\n}\n\nexport interface HarnessReport {\n harness: Target;\n status: \"tested\" | \"untested\";\n /** Why a harness was not tested (no driver / CLI absent / install failed). */\n reason?: string;\n cases: CaseResult[];\n /** Mean deterministic case score (snapshotted as the next release's baseline). */\n score: number;\n pass: boolean;\n}\n\nexport interface EvalReport {\n component: string;\n harnesses: HarnessReport[];\n}\n\nexport interface DiscoveredEval {\n componentLeaf: string;\n evalsPath: string;\n evalFile: EvalFile;\n}\n\n/** Find the components in a plugin that declare an `evals` file and load each. */\nexport function discoverEvals(pluginDir: string): DiscoveredEval[] {\n const loaded = loadPluginDir(pluginDir);\n if (!loaded.ok) {\n throw new Error(loaded.issues.map((i) => `${i.path}: ${i.message}`).join(\"\\n\"));\n }\n const out: DiscoveredEval[] = [];\n for (const c of loaded.value.plugin.components) {\n const evalsRel = \"evals\" in c ? c.evals : undefined;\n if (!evalsRel) continue;\n const text = loaded.value.read(evalsRel).toString(\"utf8\");\n const parsed = loadManifest(EvalFile, text, { filename: evalsRel });\n if (!parsed.ok) {\n throw new Error(\n `invalid evals \"${evalsRel}\": ${parsed.issues.map((i) => i.message).join(\"; \")}`,\n );\n }\n out.push({ componentLeaf: leafNameOf(c), evalsPath: evalsRel, evalFile: parsed.value });\n }\n return out;\n}\n\nasync function sh(cmd: string, cwd: string): Promise<number> {\n const r = await execa(\"bash\", [\"-lc\", cmd], { cwd, reject: false, timeout: 60_000 });\n return r.exitCode ?? 1;\n}\n\ninterface CaseContext {\n judge?: JudgeFn;\n baselineScore?: number;\n timeoutMs?: number;\n}\n\nasync function runCase(\n c: Case,\n driver: HarnessDriver,\n cwd: string,\n ctx: CaseContext,\n): Promise<CaseResult> {\n if (c.setup) await sh(c.setup, cwd);\n\n const transcripts: Transcript[] = [];\n for (let i = 0; i < c.samples; i++) {\n transcripts.push(await driver.run({ prompt: c.prompt, cwd, timeoutMs: ctx.timeoutMs }));\n }\n\n // Pass 1: evaluate the deterministic tier (trace/output) to get the case score.\n const results: (AssertResult | undefined)[] = new Array(c.assert.length);\n let detTotal = 0;\n let detPass = 0;\n for (let i = 0; i < c.assert.length; i++) {\n const a = c.assert[i];\n if (a.kind === \"trace\" || a.kind === \"output\") {\n const r = await evaluateAssertion(a, transcripts);\n results[i] = r;\n detTotal++;\n if (r.status === \"pass\") detPass++;\n }\n }\n const score = detTotal > 0 ? detPass / detTotal : 1;\n\n // Pass 2: judge (advisory unless gated) + differential (score vs baseline).\n for (let i = 0; i < c.assert.length; i++) {\n const a = c.assert[i];\n if (a.kind === \"judge\" || a.kind === \"differential\") {\n results[i] = await evaluateAssertion(a, transcripts, {\n judge: ctx.judge,\n caseScore: score,\n baselineScore: ctx.baselineScore,\n });\n }\n }\n const assertions = results.filter((r): r is AssertResult => r !== undefined);\n\n let verifyPassed: boolean | undefined;\n if (c.verify) verifyPassed = (await sh(c.verify, cwd)) === 0;\n if (c.cleanup) await sh(c.cleanup, cwd);\n\n // A case passes iff no assertion FAILED and any post-state verify passed.\n // degraded/skipped assertions do not fail the case (they are reported).\n const pass = !assertions.some((a) => a.status === \"fail\") && (verifyPassed ?? true);\n return { name: c.name, assertions, verifyPassed, score, pass };\n}\n\nexport interface RunEvalOptions {\n evalFile: EvalFile;\n pluginDir: string;\n componentLeaf: string;\n registry: AdapterRegistry;\n drivers: Record<Target, HarnessDriver>;\n scratchRoot?: string;\n timeoutMs?: number;\n /** Judge model for `judge` assertions (advisory unless gated). Omit to skip. */\n judge?: JudgeFn;\n /** Snapshot each harness's mean score into evals/.baselines/ (spec §9.5). */\n snapshotBaselines?: boolean;\n}\n\n/**\n * Drive the real harnesses headlessly and assert over what each did (spec §9.5).\n * A harness with no available driver is reported UNTESTED -- never faked. For each\n * tested harness the component is installed into a throwaway scratch project so the\n * harness loads it, then every case runs and is evaluated.\n */\nexport async function runEval(opts: RunEvalOptions): Promise<EvalReport> {\n const { evalFile, drivers } = opts;\n const harnesses: HarnessReport[] = [];\n\n for (const harness of evalFile.harnesses) {\n const driver = drivers[harness];\n if (!(driver && (await driver.available()))) {\n harnesses.push({\n harness,\n status: \"untested\",\n reason: driver ? \"CLI not installed or not headless-capable\" : \"no driver for this harness\",\n cases: [],\n score: 0,\n pass: false,\n });\n continue;\n }\n\n const baselineScore = loadBaseline(opts.pluginDir, evalFile.component, harness)?.score;\n const scratch = mkdtempSync(join(opts.scratchRoot ?? tmpdir(), `loom-eval-${harness}-`));\n try {\n await install({\n pluginDir: opts.pluginDir,\n scope: \"project\",\n cwd: scratch,\n registry: opts.registry,\n targets: [harness],\n only: [opts.componentLeaf],\n // Keep the source plugin pristine -- write the eval lock into the scratch dir.\n lockDir: scratch,\n });\n const cases: CaseResult[] = [];\n for (const c of evalFile.cases) {\n cases.push(\n await runCase(c, driver, scratch, {\n judge: opts.judge,\n baselineScore,\n timeoutMs: opts.timeoutMs,\n }),\n );\n }\n const score = cases.length > 0 ? cases.reduce((s, c) => s + c.score, 0) / cases.length : 1;\n if (opts.snapshotBaselines) {\n const snapshot: Baseline = { version: \"current\", score };\n writeBaseline(opts.pluginDir, evalFile.component, harness, snapshot);\n }\n harnesses.push({ harness, status: \"tested\", cases, score, pass: cases.every((c) => c.pass) });\n } catch (err) {\n harnesses.push({\n harness,\n status: \"untested\",\n reason: `install failed: ${(err as Error).message}`,\n cases: [],\n score: 0,\n pass: false,\n });\n } finally {\n rmSync(scratch, { recursive: true, force: true });\n }\n }\n\n return { component: evalFile.component, harnesses };\n}\n"]}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@michaelfromyeg/loom-eval",
3
- "version": "0.2.0",
3
+ "version": "0.3.0",
4
4
  "description": "Loom eval runner + headless HarnessDriver implementations (real harnesses, local-first).",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -17,9 +17,9 @@
17
17
  ],
18
18
  "dependencies": {
19
19
  "execa": "^9.5.2",
20
- "@michaelfromyeg/loom-adapter-kit": "0.2.0",
21
- "@michaelfromyeg/loom-schema": "0.2.0",
22
- "@michaelfromyeg/loom-core": "0.2.0"
20
+ "@michaelfromyeg/loom-adapter-kit": "0.3.0",
21
+ "@michaelfromyeg/loom-core": "0.3.0",
22
+ "@michaelfromyeg/loom-schema": "0.3.0"
23
23
  },
24
24
  "license": "MIT",
25
25
  "author": "Michael DeMarco",