@sx4im/skillcheck 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -116,11 +116,10 @@ export class NvidiaNimClient {
116
116
  let lastError;
117
117
  for (let attempt = 0; attempt < this.maxAttempts; attempt += 1) {
118
118
  try {
119
- const extraBody = request.chatTemplateKwargs
120
- ? {
121
- chat_template_kwargs: request.chatTemplateKwargs
122
- }
123
- : undefined;
119
+ // NOTE: do not send `extra_body`. It is a Python-SDK convenience field, not a
120
+ // real OpenAI/NVIDIA parameter — NVIDIA NIM rejects it with
121
+ // "400 Validation: Unsupported parameter(s): `extra_body`". chat_template_kwargs
122
+ // is passed as a real top-level field, which NIM accepts.
124
123
  const requestBody = {
125
124
  model: request.model,
126
125
  messages: request.messages,
@@ -128,7 +127,6 @@ export class NvidiaNimClient {
128
127
  max_tokens: request.maxTokens,
129
128
  response_format: request.responseFormat ? { type: request.responseFormat } : undefined,
130
129
  chat_template_kwargs: request.chatTemplateKwargs,
131
- extra_body: extraBody,
132
130
  stream: false
133
131
  };
134
132
  const response = (await this.runSerializedRequest(() => this.client.chat.completions.create(requestBody)));
@@ -1 +1 @@
1
- {"version":3,"file":"nvidia-nim.js","sourceRoot":"","sources":["../../../packages/cli/src/adapters/nvidia-nim.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAuB5B,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExE,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO;aAClB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAI,IAA2B,CAAC,IAAI,CAAC;gBAC/C,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9C,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;aACD,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,OAAO,KAAK,IAAI,SAAS,CAAC;IAC5B,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACpD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,OAAgF,CAAC;IAChG,OAAO,CACL,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC;QAClC,kBAAkB,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC5C,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CACnC,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;QACrE,MAAM,MAAM,GAAI,KAA8B,CAAC,MAAM,CAAC;QACtD,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACzD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAI,KAA+B,CAAC,OAAO,CAAC;IACzD,MAAM,UAAU,GACd,OAAO,YAAY,OAAO;QACxB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAC5B,CAAC,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,aAAa,IAAI,OAAO;YAC3E,CAAC,CAAE,OAAuC,CAAC,aAAa,CAAC;YACzD,CAAC,CAAC,SAAS,CAAC;IAElB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,OAAO,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,OAAe,EAAE,MAAc,EAAE,WAAmB;IACzF,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,GAAG,EAAE,CAAC;QACjD,OAAO;IACT,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,MAAM,EAAE,CAAC;IACvE,OAAO,CAAC,KAAK,CAAC,sBAAsB,OAAO,GAAG,CAAC,IAAI,WAAW,UAAU,KAAK,aAAa,MAAM,KAAK,CAAC,CAAC;AACzG,CAAC;AAED,MAAM,OAAO,eAAe;IAClB,MAAM,CAAC,YAAY,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IACvD,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;IAEhB,MAAM,CAAS;IACf,cAAc,CAAS;IACvB,WAAW,CAAS;IACpB,eAAe,CAAS;IAEzC,YAAY,MAAoB,EAAE,UAAuD,EAAE;QACzF,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,CAAC;QAC9C,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,MAAM,CAAC,SAAS;YACzB,cAAc,EAAE,OAAO,CAAC,cAAc;SACvC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAI,SAA2B;QAC/D,MAAM,eAAe,GAAG,eAAe,CAAC,YAAY,CAAC;QACrD,IAAI,qBAAqB,GAAe,GAAG,EAAE,CAAC,SAAS,CAAC;QACxD,eAAe,CAAC,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC3D,qBAAqB,GAAG,OAAO,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,MAAM,eAAe,CAAC;QACtB,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,aAAa,CAAC;gBAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;gBAC5D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;oBACf,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,eAAe,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3C,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,qBAAqB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAA0B;QACvC,IAAI,SAAkB,CAAC;QAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YAC/D,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,OAAO,CAAC,kBAAkB;oBAC1C,CAAC,CAAC;wBACE,oBAAoB,EAAE,OAAO,CAAC,kBAAkB;qBACjD;oBACH,CAAC,CAAC,SAAS,CAAC;gBACd,MAAM,WAAW,GAAG;oBAClB,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,UAAU,EAAE,OAAO,CAAC,SAAS;oBAC7B,eAAe,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS;oBACtF,oBAAoB,EAAE,OAAO,CAAC,kBAAkB;oBAChD,UAAU,EAAE,SAAS;oBACrB,MAAM,EAAE,KAAK;iBACd,CAAC;gBACF,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,CACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,CACjD,CAAmB,CAAC;gBAErB,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC;gBAC7C,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC1B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;oBACtE,MAAM,IAAI,KAAK,CAAC,yEAAyE,IAAI,EAAE,CAAC,CAAC;gBACnG,CAAC;gBAED,OAAO;oBACL,OAAO;oBACP,KAAK,EAAE,QAAQ,CAAC,KAAK;oBACrB,KAAK,EAAE;wBACL,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa;wBAC3C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,iBAAiB;wBACnD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY;qBAC1C;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAChC,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,IAAI,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACzE,IAAI,CAAC,SAAS,IAAI,OAAO,KAAK,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;oBACnD,MAAM;gBACR,CAAC;gBAED,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBAC5C,MAAM,WAAW,GAAG,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;gBAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;gBAC/C,MAAM,MAAM,GAAG,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,WAAW,GAAG,CAAC,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC;gBACnG,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACxD,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAC9E,CAAC"}
1
+ {"version":3,"file":"nvidia-nim.js","sourceRoot":"","sources":["../../../packages/cli/src/adapters/nvidia-nim.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAuB5B,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExE,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO;aAClB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAI,IAA2B,CAAC,IAAI,CAAC;gBAC/C,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9C,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;aACD,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,OAAO,KAAK,IAAI,SAAS,CAAC;IAC5B,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACpD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,OAAgF,CAAC;IAChG,OAAO,CACL,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC;QAClC,kBAAkB,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC5C,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CACnC,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;QACrE,MAAM,MAAM,GAAI,KAA8B,CAAC,MAAM,CAAC;QACtD,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACzD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAI,KAA+B,CAAC,OAAO,CAAC;IACzD,MAAM,UAAU,GACd,OAAO,YAAY,OAAO;QACxB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAC5B,CAAC,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,aAAa,IAAI,OAAO;YAC3E,CAAC,CAAE,OAAuC,CAAC,aAAa,CAAC;YACzD,CAAC,CAAC,SAAS,CAAC;IAElB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,OAAO,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,OAAe,EAAE,MAAc,EAAE,WAAmB;IACzF,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,GAAG,EAAE,CAAC;QACjD,OAAO;IACT,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,MAAM,EAAE,CAAC;IACvE,OAAO,CAAC,KAAK,CAAC,sBAAsB,OAAO,GAAG,CAAC,IAAI,WAAW,UAAU,KAAK,aAAa,MAAM,KAAK,CAAC,CAAC;AACzG,CAAC;AAED,MAAM,OAAO,eAAe;IAClB,MAAM,CAAC,YAAY,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IACvD,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;IAEhB,MAAM,CAAS;IACf,cAAc,CAAS;IACvB,WAAW,CAAS;IACpB,eAAe,CAAS;IAEzC,YAAY,MAAoB,EAAE,UAAuD,EAAE;QACzF,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,CAAC;QAC9C,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,MAAM,CAAC,SAAS;YACzB,cAAc,EAAE,OAAO,CAAC,cAAc;SACvC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAI,SAA2B;QAC/D,MAAM,eAAe,GAAG,eAAe,CAAC,YAAY,CAAC;QACrD,IAAI,qBAAqB,GAAe,GAAG,EAAE,CAAC,SAAS,CAAC;QACxD,eAAe,CAAC,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC3D,qBAAqB,GAAG,OAAO,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,MAAM,eAAe,CAAC;QACtB,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,aAAa,CAAC;gBAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;gBAC5D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;oBACf,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,eAAe,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3C,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,qBAAqB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAA0B;QACvC,IAAI,SAAkB,CAAC;QAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YAC/D,IAAI,CAAC;gBACH,8EAA8E;gBAC9E,4DAA4D;gBAC5D,iFAAiF;gBACjF,0DAA0D;gBAC1D,MAAM,WAAW,GAAG;oBAClB,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,UAAU,EAAE,OAAO,CAAC,SAAS;oBAC7B,eAAe,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS;oBACtF,oBAAoB,EAAE,OAAO,CAAC,kBAAkB;oBAChD,MAAM,EAAE,KAAK;iBACd,CAAC;gBACF,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,CACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,CACjD,CAAmB,CAAC;gBAErB,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC;gBAC7C,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC1B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;oBACtE,MAAM,IAAI,KAAK,CAAC,yEAAyE,IAAI,EAAE,CAAC,CAAC;gBACnG,CAAC;gBAED,OAAO;oBACL,OAAO;oBACP,KAAK,EAAE,QAAQ,CAAC,KAAK;oBACrB,KAAK,EAAE;wBACL,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa;wBAC3C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,iBAAiB;wBACnD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY;qBAC1C;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAChC,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,IAAI,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACzE,IAAI,CAAC,SAAS,IAAI,OAAO,KAAK,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;oBACnD,MAAM;gBACR,CAAC;gBAED,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBAC5C,MAAM,WAAW,GAAG,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;gBAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;gBAC/C,MAAM,MAAM,GAAG,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,WAAW,GAAG,CAAC,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC;gBACnG,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACxD,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAC9E,CAAC"}
package/dist/src/grade.js CHANGED
@@ -65,13 +65,16 @@ export async function gradeOutputs(tasks, outputs, config, client, cache) {
65
65
  model: config.graderModel,
66
66
  criterion: task.criterion,
67
67
  output: output.output,
68
- promptVersion: 6,
68
+ promptVersion: 7,
69
69
  responseFormat: 'json_object',
70
70
  attempt
71
71
  }, () => client.complete({
72
72
  model: config.graderModel,
73
73
  temperature: 0,
74
- maxTokens: 240,
74
+ // Reasoning models (e.g. minimax-m2.7) always emit ~500-600 reasoning
75
+ // tokens before the answer; a tight budget leaves `content` empty. Give
76
+ // the JSON verdict room to land after the reasoning.
77
+ maxTokens: 1536,
75
78
  responseFormat: 'json_object',
76
79
  chatTemplateKwargs: { thinking: false },
77
80
  messages: [
@@ -1 +1 @@
1
- {"version":3,"file":"grade.js","sourceRoot":"","sources":["../../packages/cli/src/grade.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAA4B,CAAC;QAChF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;SACpC,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,EAAE,6BAA6B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC7D,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAsB,EACtB,OAAsB,EACtB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC;IAEnD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;gBAChC,GAAG,MAAM;gBACT,GAAG,KAAK;aACT,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,KAAK,CAAC,sBAAsB,MAAM,CAAC,MAAM,UAAU,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,KAA+B,CAAC;QACpC,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CACnC,QAAQ,EACR;gBACE,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,CAAC;gBAChB,cAAc,EAAE,aAAa;gBAC7B,OAAO;aACR,EACD,GAAG,EAAE,CACH,MAAM,CAAC,QAAQ,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,GAAG;gBACd,cAAc,EAAE,aAAa;gBAC7B,kBAAkB,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;gBACvC,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EACL,qMAAqM;qBACxM;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,uBAAuB,IAAI,CAAC,SAAS,yBAAyB,MAAM,CAAC,MAAM,0MAA0M;qBAC/R;iBACF;aACF,CAAC,CACL,CAAC;YAEF,IAAI,CAAC;gBACH,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACrC,MAAM;YACR,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,CAAC,KAAK,CAAC,wDAAwD,OAAO,IAAI,CAAC,CAAC;gBACrF,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAChH,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;YAChC,GAAG,MAAM;YACT,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,IAAI,EAAE,KAAK,CAAC,KAAK,IAAI,GAAG;SACzB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC"}
1
+ {"version":3,"file":"grade.js","sourceRoot":"","sources":["../../packages/cli/src/grade.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAA4B,CAAC;QAChF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;SACpC,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,EAAE,6BAA6B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC7D,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAsB,EACtB,OAAsB,EACtB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC;IAEnD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;gBAChC,GAAG,MAAM;gBACT,GAAG,KAAK;aACT,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,KAAK,CAAC,sBAAsB,MAAM,CAAC,MAAM,UAAU,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,KAA+B,CAAC;QACpC,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CACnC,QAAQ,EACR;gBACE,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,CAAC;gBAChB,cAAc,EAAE,aAAa;gBAC7B,OAAO;aACR,EACD,GAAG,EAAE,CACH,MAAM,CAAC,QAAQ,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,WAAW,EAAE,CAAC;gBACd,sEAAsE;gBACtE,wEAAwE;gBACxE,qDAAqD;gBACrD,SAAS,EAAE,IAAI;gBACf,cAAc,EAAE,aAAa;gBAC7B,kBAAkB,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;gBACvC,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EACL,qMAAqM;qBACxM;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,uBAAuB,IAAI,CAAC,SAAS,yBAAyB,MAAM,CAAC,MAAM,0MAA0M;qBAC/R;iBACF;aACF,CAAC,CACL,CAAC;YAEF,IAAI,CAAC;gBACH,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACrC,MAAM;YACR,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,CAAC,KAAK,CAAC,wDAAwD,OAAO,IAAI,CAAC,CAAC;gBACrF,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAChH,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;YAChC,GAAG,MAAM;YACT,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,IAAI,EAAE,KAAK,CAAC,KAAK,IAAI,GAAG;SACzB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC"}
package/dist/src/run.js CHANGED
@@ -18,11 +18,15 @@ async function runOne(skill, task, trial, arm, config, client, cache) {
18
18
  // so trials 2..K silently read trial 1's cached output. That collapses K
19
19
  // independent stochastic samples (temperature 0.7) into one replicated K times
20
20
  // and makes the paired-bootstrap CI ~sqrt(K) too narrow (pseudo-replication).
21
- const response = await cache.getOrSet('runner', { model: config.runnerModel, temperature: 0.7, maxTokens: 1200, promptVersion: 1, trial, messages }, () => client.complete({
21
+ // Reasoning models (e.g. minimax-m2.7) spend ~500-600 tokens thinking before the
22
+ // answer, so the budget must cover reasoning + a full response or the answer is
23
+ // truncated. maxTokens is part of the cache key, so this also invalidates any
24
+ // answers captured under the old tighter budget.
25
+ const response = await cache.getOrSet('runner', { model: config.runnerModel, temperature: 0.7, maxTokens: 2048, promptVersion: 1, trial, messages }, () => client.complete({
22
26
  model: config.runnerModel,
23
27
  messages,
24
28
  temperature: 0.7,
25
- maxTokens: 1200
29
+ maxTokens: 2048
26
30
  }));
27
31
  const transcriptHash = `sha256:${hashJson({ taskId: task.id, trial, arm, messages, output: response.content })}`;
28
32
  return {
@@ -1 +1 @@
1
- {"version":3,"file":"run.js","sourceRoot":"","sources":["../../packages/cli/src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,SAAS,cAAc,CAAC,KAAsB,EAAE,IAAmB,EAAE,SAAkB;IACrF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,OAAO;QACL;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,mGAAmG,KAAK,CAAC,YAAY,EAAE;SACjI;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE;KACvC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,KAAsB,EACtB,IAAmB,EACnB,KAAa,EACb,GAAuB,EACvB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,YAAY,CAAC,CAAC;IACnE,8EAA8E;IAC9E,+EAA+E;IAC/E,yEAAyE;IACzE,+EAA+E;IAC/E,8EAA8E;IAC9E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CACxJ,MAAM,CAAC,QAAQ,CAAC;QACd,KAAK,EAAE,MAAM,CAAC,WAAW;QACzB,QAAQ;QACR,WAAW,EAAE,GAAG;QAChB,SAAS,EAAE,IAAI;KAChB,CAAC,CACH,CAAC;IACF,MAAM,cAAc,GAAG,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;IAEjH,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK;QACL,GAAG;QACH,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;QAC/C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,gBAAgB,IAAI,CAAC;QACvD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;QAC7C,cAAc;KACf,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAsB,EACtB,KAAsB,EACtB,MAAc,EACd,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,MAAM,aAAa,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;YACpF,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,MAAM,WAAW,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
1
+ {"version":3,"file":"run.js","sourceRoot":"","sources":["../../packages/cli/src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,SAAS,cAAc,CAAC,KAAsB,EAAE,IAAmB,EAAE,SAAkB;IACrF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,OAAO;QACL;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,mGAAmG,KAAK,CAAC,YAAY,EAAE;SACjI;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE;KACvC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,KAAsB,EACtB,IAAmB,EACnB,KAAa,EACb,GAAuB,EACvB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,YAAY,CAAC,CAAC;IACnE,8EAA8E;IAC9E,+EAA+E;IAC/E,yEAAyE;IACzE,+EAA+E;IAC/E,8EAA8E;IAC9E,iFAAiF;IACjF,gFAAgF;IAChF,8EAA8E;IAC9E,iDAAiD;IACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CACxJ,MAAM,CAAC,QAAQ,CAAC;QACd,KAAK,EAAE,MAAM,CAAC,WAAW;QACzB,QAAQ;QACR,WAAW,EAAE,GAAG;QAChB,SAAS,EAAE,IAAI;KAChB,CAAC,CACH,CAAC;IACF,MAAM,cAAc,GAAG,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;IAEjH,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK;QACL,GAAG;QACH,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;QAC/C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,gBAAgB,IAAI,CAAC;QACvD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;QAC7C,cAAc;KACf,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAsB,EACtB,KAAsB,EACtB,MAAc,EACd,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,MAAM,aAAa,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;YACpF,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,MAAM,WAAW,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sx4im/skillcheck",
3
- "version": "0.2.2",
3
+ "version": "0.2.3",
4
4
  "description": "Measure whether agent skills improve task performance.",
5
5
  "type": "module",
6
6
  "license": "MIT",