@lesliechan721/agent-skill-eval 0.1.0-beta.50 → 0.1.0-beta.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/codex-eval-cases.d.ts +0 -2
- package/dist/codex-eval-cases.js +0 -5
- package/dist/codex-eval-cases.js.map +1 -1
- package/dist/codex-eval-rollout.d.ts +8 -0
- package/dist/codex-eval-rollout.js +85 -0
- package/dist/codex-eval-rollout.js.map +1 -0
- package/dist/codex-trigger-eval.d.ts +1 -0
- package/dist/codex-trigger-eval.js +177 -44
- package/dist/codex-trigger-eval.js.map +1 -1
- package/dist/codex-workflow.js +5 -63
- package/dist/codex-workflow.js.map +1 -1
- package/dist/selected-cases.js +0 -2
- package/dist/selected-cases.js.map +1 -1
- package/package.json +3 -3
|
@@ -12,8 +12,6 @@ export interface AwSkillEvalTriggerSuite {
|
|
|
12
12
|
forbidMultipleSkillLoads: boolean;
|
|
13
13
|
profiles: string[];
|
|
14
14
|
defaultRuns: number;
|
|
15
|
-
releaseSplits: Array<'train' | 'validation' | 'sanity' | 'all'>;
|
|
16
|
-
requiredForRelease: boolean;
|
|
17
15
|
perRunTimeoutMs: number;
|
|
18
16
|
}
|
|
19
17
|
export declare const AW_SKILL_EVAL_CASE_SCHEMA_VERSION: 1;
|
package/dist/codex-eval-cases.js
CHANGED
|
@@ -319,11 +319,6 @@ export function validateAwSkillEvalTriggerCaseDocument(value, suite, label = `tr
|
|
|
319
319
|
if (!suite.fixtureModule && cases.some((entry) => entry.workspace_fixture)) {
|
|
320
320
|
throw new Error(`Trigger suite "${suite.id}" requires fixtureModule because cases declare workspace_fixture.`);
|
|
321
321
|
}
|
|
322
|
-
for (const split of suite.releaseSplits) {
|
|
323
|
-
if (split !== 'all' && !cases.some((entry) => entry.split === split)) {
|
|
324
|
-
throw new Error(`Trigger suite "${suite.id}" has no cases for release split "${split}".`);
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
322
|
return { schemaVersion: AW_SKILL_EVAL_CASE_SCHEMA_VERSION, skill, cases };
|
|
328
323
|
}
|
|
329
324
|
//# sourceMappingURL=codex-eval-cases.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"codex-eval-cases.js","sourceRoot":"","sources":["../src/codex-eval-cases.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,EAAE,MAAM,UAAU,CAAA;AAsBzB,MAAM,CAAC,MAAM,iCAAiC,GAAG,CAAU,CAAA;AA0B3D,SAAS,MAAM,CAAC,KAAc,EAAE,KAAa;IAC3C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,qBAAqB,CAAC,CAAA;IAChD,CAAC;IACD,OAAO,KAAgC,CAAA;AACzC,CAAC;AAED,SAAS,QAAQ,CAAC,KAA8B,EAAE,WAA8B,EAAE,KAAa;IAC7F,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IACtE,IAAI,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,uBAAuB,UAAU,IAAI,CAAC,CAAA;AAChF,CAAC;AAED,SAAS,MAAM,CAAC,KAAc,EAAE,KAAa;IAC3C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,8BAA8B,CAAC,CAAA;IACzD,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,OAAO,CAAC,KAAc,EAAE,KAAa;IAC5C,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAA;IAC5E,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,OAAO,CAAC,KAAc,EAAE,KAAa;IAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6BAA6B,CAAC,CAAA;IACxD,CAAC;IACD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC,CAAA;AACzE,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,SAAiB;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;IAC/C,OAAO,QAAQ,KAAK,EAAE,IAAI,CACxB,QAAQ,KAAK,IAAI;WACd,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;WACrC,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAAgB,EAChB,SAAiB,EACjB,KAAa,EACb,aAAqB;IAErB,MAAM,iBAAiB,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;IACnD,MAAM,kBAAkB,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAA;IACrD,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,EAAE,kBAAkB,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,yBAAyB,aAAa,GAAG,CAAC,CAAA;IACpE,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAc,EACd,KAAa,EACb,OAAsC;IAEtC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IAChC,MAAM,WAAW,GAAG,OAAO,CAAC,gBAAgB,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACnE,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;IACtD,IACE,SAAS,KAAK,EAAE;WACb,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC;WACxB,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC;WACxB,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;WACvB,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;WACvB,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;WACvB,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;WACvB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC;WAChC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC;WAChC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,SAAS;WAC7C,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6DAA6D,CAAC,CAAA;IACxF,MAAM,YAAY,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IAC5C,IAAI,YAAY,KAAK,MAAM,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY,KAAK,QAAQ,EAAE,CAAC;QACvF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6CAA6C,CAAC,CAAA;IACxE,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,SAAS,2BAA2B,CAAC,YAAoB,EAAE,OAAe;IACxE,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACnC,OAAO,YAAY,KAAK,MAAM,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,CAAC,CAAA;IACzE,CAAC;IACD,OAAO,YAAY,KAAK,OAAO,CAAA;AACjC,CAAC;AAED,SAAS,qBAAqB,CAAC,WAAmB,EAAE,MAAc,EAAE,KAAa;IAC/E,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;IACvD,IAAI,WAAW,GAAG,WAAW,CAAA;IAC7B,KAAK,MAAM,OAAO,IAAI,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACnE,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;QAC7C,IAAI,EAAE,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,cAAc,EAAE,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,mCAAmC,CAAC,CAAA;QAC9D,CAAC;IACH,CAAC;IACD,oBAAoB,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,CAAC,CAAA;IACrE,MAAM,IAAI,GAAG,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IACjC,IAAI,IAAI,CAAC,MAAM,EAAE;QAAE,OAAM;IACzB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,sDAAsD,CAAC,CAAA;IACjF,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3C,qBAAqB,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;IACrE,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAAC,KAAc,EAAE,KAAa;IAC9D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAM;IACrC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IAClC,QAAQ,CACN,KAAK,EACL,CAAC,kBAAkB,EAAE,yBAAyB,EAAE,4BAA4B,CAAC,EAC7E,KAAK,CACN,CAAA;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,IAAI,CACvE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CACrC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,gDAAgD,CAAC,CAAA;IAC5E,IACE,KAAK,CAAC,uBAAuB,KAAK,SAAS;WACxC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,uBAAuB,CAAC;eAC9C,KAAK,CAAC,uBAAkC,GAAG,CAAC,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,0DAA0D,CAAC,CAAA;IACrF,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,GAAG,KAAK,6BAA6B,CAAC,CAAA;AACjF,CAAC;AAED,SAAS,kCAAkC,CAAC,KAAc,EAAE,KAAa;IACvE,IAAI,KAAK,KAAK,SAAS;QAAE,OAAM;IAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IAClC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACrC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,kCAAkC,CAAC,CAAA;IACrF,KAAK,MAAM,CAAC,YAAY,EAAE,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;QAC9C,kBAAkB,CAAC,YAAY,EAAE,GAAG,KAAK,IAAI,YAAY,EAAE,EAAE,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAA;QACzF,IAAI,OAAO,OAAO,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,IAAI,YAAY,oBAAoB,CAAC,CAAA;IAChG,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,KAAa;IACjD,OAAO,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,CAAA;AACvF,CAAC;AAED,SAAS,+BAA+B,CACtC,KAAc,EACd,KAAa,EACb,WAAmB,EACnB,kBAAkB,IAAI,GAAG,EAAU;IAEnC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAM;IAC/B,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IACxC,QAAQ,CAAC,WAAW,EAAE,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,CAAA;IAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,WAAW,EAAE,GAAG,KAAK,cAAc,CAAC;SACzE,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,kBAAkB,CACvC,KAAK,EACL,GAAG,KAAK,gBAAgB,KAAK,GAAG,EAChC,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAC5B,CAAC,CAAA;IACJ,IAAI,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,MAAM,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,2CAA2C,CAAC,CAAA;IACtE,CAAC;IACD,WAAW,CAAC,OAAO,CAAC,CAAC,YAAY,EAAE,KAAK,EAAE,EAAE;QAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;QAC5D,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,YAAY,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,gBAAgB,KAAK,uCAAuC,CAAC,CAAA;QACvF,CAAC;QACD,qBAAqB,CAAC,WAAW,EAAE,YAAY,EAAE,GAAG,KAAK,gBAAgB,KAAK,GAAG,CAAC,CAAA;QAClF,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;YAChF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,gBAAgB,KAAK,+CAA+C,CAAC,CAAA;QAC/F,CAAC;QACD,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;IACnC,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,2CAA2C,CACzD,KAAc,EACd,aAAqB,EACrB,KAAK,GAAG,oBAAoB,EAC5B,WAAW,GAAG,OAAO,CAAC,GAAG,EAAE,EAC3B,mBAA6B,EAAE;IAE/B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IACrC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,GAAG,KAAK,aAAa,CAAC,CAAA;IAChE,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,gBAAgB,KAAK,iCAAiC,aAAa,IAAI,CAAC,CAAA;IAClG,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,mCAAmC,CAAC,CAAA;IAC9D,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QACzC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,KAAK,UAAU,KAAK,GAAG,CAAC,CAAA;QACrD,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,UAAU,KAAK,MAAM,CAAC,CAAC,IAAI,EAAE,CAAA;QAC/D,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,2BAA2B,EAAE,IAAI,CAAC,CAAA;QAC3E,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACX,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAA;QACrD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,KAAK,SAAS;YACvC,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,UAAU,KAAK,WAAW,CAAC,CAAA;QAC3D,IAAI,OAAO,KAAK,WAAW,IAAI,OAAO,KAAK,iBAAiB,EAAE,CAAC;YAC7D,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,qDAAqD,CAAC,CAAA;QAC/F,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,sCAAsC,CAAC,CAAA;QAChF,CAAC;QACD,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,yBAAyB,CAC3E,KAAK,EACL,GAAG,KAAK,UAAU,KAAK,sBAAsB,UAAU,GAAG,CAC3D,CAAC,CAAA;QACF,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,GAAG,KAAK,UAAU,KAAK,mBAAmB,CAAC,CAAA;QACvE,kCAAkC,CAChC,GAAG,CAAC,eAAe,EACnB,GAAG,KAAK,UAAU,KAAK,mBAAmB,CAC3C,CAAA;QACD,+BAA+B,CAC7B,GAAG,CAAC,WAAW,EACf,GAAG,KAAK,UAAU,KAAK,eAAe,EACtC,WAAW,EACX,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAC1B,CAAA;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAA;QACpE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,KAAK,UAAU,KAAK,iBAAiB,CAAC,CAAA;QAChE,IAAI,MAAM,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;YAC7C,OAAO,CAAC,MAAM,CAAC,mBAAmB,EAAE,GAAG,KAAK,UAAU,KAAK,8BAA8B,CAAC,CAAA;QAC5F,CAAC;QACD,MAAM,uBAAuB,GAAG,MAAM,CAAC,yBAAyB,KAAK,SAAS;eACzE,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,yBAAyB,CAAC,IAAI,MAAM,CAAC,yBAAyB,CAAC,MAAM,KAAK,CAAC,CAAC;YACrG,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,OAAO,CACL,MAAM,CAAC,yBAAyB,EAChC,GAAG,KAAK,UAAU,KAAK,oCAAoC,CAC5D,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CAAC,kBAAkB,CAC5C,KAAK,EACL,GAAG,KAAK,UAAU,KAAK,sCAAsC,SAAS,GAAG,EACzE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAC3B,CAAC,CAAA;QACN,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAA;QACxE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,sCAAsC,CAAC,CAAA;QAChF,CAAC;QACD,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE;YAC9D,MAAM,aAAa,GAAG,GAAG,KAAK,UAAU,KAAK,sBAAsB,aAAa,GAAG,CAAA;YACnF,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAA;YACnD,QAAQ,CAAC,WAAW,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,aAAa,CAAC,CAAA;YAC5F,MAAM,YAAY,GAAG,kBAAkB,CACrC,WAAW,CAAC,IAAI,EAChB,GAAG,aAAa,OAAO,EACvB,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAC5B,CAAA;YACD,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS;gBAAE,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,aAAa,SAAS,CAAC,CAAA;YAC5F,IAAI,WAAW,CAAC,QAAQ,KAAK,SAAS;gBAAE,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,GAAG,aAAa,WAAW,CAAC,CAAA;YAClG,IAAI,WAAW,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;gBAC3C,OAAO,CAAC,WAAW,CAAC,YAAY,EAAE,GAAG,aAAa,eAAe,CAAC,CAAA;YACpE,CAAC;YACD,IAAI,WAAW,CAAC,IAAI,KAAK,SAAS;gBAAE,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,aAAa,OAAO,CAAC,CAAA;YACrF,OAAO,YAAY,CAAA;QACrB,CAAC,CAAC,CAAA;QACF,IAAI,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,8CAA8C,CAAC,CAAA;QACxF,CAAC;QACD,IAAI,OAAO,KAAK,iBAAiB,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,mBAAmB,KAAK,KAAK,EAAE,CAAC;gBACzC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,8DAA8D,CAAC,CAAA;YACxG,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,yBAAyB,CAAC,EAAE,CAAC;gBACrD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,uEAAuE,CAAC,CAAA;YACjH,CAAC;YACD,IAAI,uBAAuB,CAAC,MAAM,GAAG,CAAC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,wDAAwD,CAAC,CAAA;YAClG,CAAC;YACD,iFAAiF;YACjF,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,MAAM,GAAG,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,uBAAuB,CAAC,IAAI,CAChI,CAAC,OAAO,EAAE,EAAE,CAAC,2BAA2B,CAAC,YAAY,EAAE,OAAO,CAAC,CAChE,CAAC,CAAA;YACF,IAAI,iBAAiB,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,UAAU,KAAK,mBAAmB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,GAAG;sBAC5E,8CAA8C,CACjD,CAAA;YACH,CAAC;QACH,CAAC;QACD,OAAO,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,CAAA;IAChC,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,KAAc,EACd,KAAa,EACb,KAA0C;IAE1C,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,iBAAiB,KAAK,GAAG,CAAC,CAAA;IACpD,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,iBAAiB,KAAK,MAAM,CAAC,CAAA;IACvD,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,qCAAqC,CAAC,CAAA;IAC9E,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,SAAS,CAAC,CAAA;IAChE,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,YAAY,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,+CAA+C,CAAC,CAAA;IACxF,CAAC;IACD,MAAM,gBAAgB,GAAG,GAAG,CAAC,kBAAkB,KAAK,SAAS;QAC3D,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,iBAAiB,KAAK,sBAAsB,CAAC,CAAA;IACjF,IAAI,KAAK,CAAC,iBAAiB,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,+DAA+D,CACtF,CAAA;IACH,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC/D,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,uDAAuD,CAAC,CAAA;IAChG,CAAC;IACD,MAAM,gBAAgB,GAAG,GAAG,CAAC,iBAAiB,KAAK,SAAS;QAC1D,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,EAAE,iBAAiB,KAAK,qBAAqB,CAAC,CAAA;IAC9E,MAAM,eAAe,GAAG,GAAG,CAAC,iBAAiB,KAAK,SAAS;QACzD,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,iBAAiB,KAAK,qBAAqB,CAAC,CAAA;IAC/E,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,iBAAiB,KAAK,QAAQ,CAAC,CAAA;IACnG,MAAM,mBAAmB,GAAG,GAAG,CAAC,iBAAiB,KAAK,SAAS,CAAA;IAC/D,MAAM,oBAAoB,GAAG,GAAG,CAAC,sBAAsB,KAAK,SAAS,CAAA;IACrE,MAAM,yBAAyB,GAAG,GAAG,CAAC,2BAA2B,KAAK,SAAS,CAAA;IAC/E,IAAI,CAAC,mBAAmB,EAAE,oBAAoB,EAAE,yBAAyB,CAAC;SACvE,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,mDAAmD;cACvE,yDAAyD,CAC5D,CAAA;IACH,CAAC;IACD,IAAI,oBAA0C,CAAA;IAC9C,IAAI,oBAAoB,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,4CAA4C,CAAC,CAAA;QACrF,CAAC;QACD,oBAAoB,GAAG,GAAG,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,MAAM,CACjF,KAAK,EACL,iBAAiB,KAAK,4BAA4B,UAAU,GAAG,CAChE,CAAC,CAAA;QACF,IAAI,IAAI,GAAG,CAAC,oBAAoB,CAAC,CAAC,IAAI,KAAK,oBAAoB,CAAC,MAAM,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,uDAAuD,CAAC,CAAA;QAChG,CAAC;QACD,MAAM,YAAY,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;QACxF,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,4DAA4D,YAAY,IAAI,CACnG,CAAA;QACH,CAAC;QACD,IAAI,KAAK,CAAC,wBAAwB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,wCAAwC,KAAK,CAAC,EAAE,iCAAiC,CACxG,CAAA;QACH,CAAC;IACH,CAAC;IACD,MAAM,wBAAwB,GAAG,yBAAyB;QACxD,CAAC,CAAC,MAAM,CACN,GAAG,CAAC,2BAA2B,EAC/B,iBAAiB,KAAK,+BAA+B,CACtD;QACD,CAAC,CAAC,SAAS,CAAA;IACb,IAAI,wBAAwB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,+DAA+D;cACnF,IAAI,wBAAwB,IAAI,CACnC,CAAA;IACH,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,SAAS,CAAC,CAAA;IAChE,OAAO;QACL,GAAG,GAAG;QACN,EAAE;QACF,KAAK;QACL,KAAK;QACL,GAAG,CAAC,mBAAmB;YACrB,CAAC,CAAC,EAAE,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,iBAAiB,KAAK,qBAAqB,CAAC,EAAE;YACpG,CAAC,CAAC,oBAAoB;gBACpB,CAAC,CAAC,EAAE,sBAAsB,EAAE,oBAAqB,EAAE;gBACnD,CAAC,CAAC,EAAE,2BAA2B,EAAE,wBAAyB,EAAE,CAAC;QACjE,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,CAAC;QACnF,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,eAAe,EAAE,CAAC;QAChF,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1B,CAAA;AACH,CAAC;AAED,MAAM,UAAU,sCAAsC,CACpD,KAAc,EACd,KAA0C,EAC1C,KAAK,GAAG,4BAA4B,KAAK,CAAC,EAAE,GAAG;IAE/C,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IACrC,IAAI,QAAQ,CAAC,aAAa,KAAK,iCAAiC,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,2BAA2B,iCAAiC,GAAG,CAAC,CAAA;IAC1F,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,KAAK,QAAQ,CAAC,CAAA;IACtD,IAAI,KAAK,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,wBAAwB,KAAK,iCAAiC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;IAChG,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,wCAAwC,CAAC,CAAA;IACnE,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,+BAA+B,CAChF,KAAK,EACL,KAAK,EACL,KAAK,CACN,CAAC,CAAA;IACF,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;IAC/C,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CACb,kBAAkB,KAAK,CAAC,EAAE,mEAAmE,CAC9F,CAAA;IACH,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACxC,IAAI,KAAK,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,EAAE,CAAC;YACrE,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,CAAC,EAAE,qCAAqC,KAAK,IAAI,CAAC,CAAA;QAC3F,CAAC;IACH,CAAC;IACD,OAAO,EAAE,aAAa,EAAE,iCAAiC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;AAC3E,CAAC"}
|
|
1
|
+
{"version":3,"file":"codex-eval-cases.js","sourceRoot":"","sources":["../src/codex-eval-cases.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,EAAE,MAAM,UAAU,CAAA;AAoBzB,MAAM,CAAC,MAAM,iCAAiC,GAAG,CAAU,CAAA;AA0B3D,SAAS,MAAM,CAAC,KAAc,EAAE,KAAa;IAC3C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,qBAAqB,CAAC,CAAA;IAChD,CAAC;IACD,OAAO,KAAgC,CAAA;AACzC,CAAC;AAED,SAAS,QAAQ,CAAC,KAA8B,EAAE,WAA8B,EAAE,KAAa;IAC7F,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IACtE,IAAI,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,uBAAuB,UAAU,IAAI,CAAC,CAAA;AAChF,CAAC;AAED,SAAS,MAAM,CAAC,KAAc,EAAE,KAAa;IAC3C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,8BAA8B,CAAC,CAAA;IACzD,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,OAAO,CAAC,KAAc,EAAE,KAAa;IAC5C,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAA;IAC5E,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,OAAO,CAAC,KAAc,EAAE,KAAa;IAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6BAA6B,CAAC,CAAA;IACxD,CAAC;IACD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC,CAAA;AACzE,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,SAAiB;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;IAC/C,OAAO,QAAQ,KAAK,EAAE,IAAI,CACxB,QAAQ,KAAK,IAAI;WACd,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;WACrC,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAAgB,EAChB,SAAiB,EACjB,KAAa,EACb,aAAqB;IAErB,MAAM,iBAAiB,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;IACnD,MAAM,kBAAkB,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAA;IACrD,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,EAAE,kBAAkB,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,yBAAyB,aAAa,GAAG,CAAC,CAAA;IACpE,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAc,EACd,KAAa,EACb,OAAsC;IAEtC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IAChC,MAAM,WAAW,GAAG,OAAO,CAAC,gBAAgB,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACnE,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;IACtD,IACE,SAAS,KAAK,EAAE;WACb,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC;WACxB,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC;WACxB,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;WACvB,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;WACvB,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;WACvB,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;WACvB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC;WAChC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC;WAChC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,SAAS;WAC7C,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6DAA6D,CAAC,CAAA;IACxF,MAAM,YAAY,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IAC5C,IAAI,YAAY,KAAK,MAAM,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY,KAAK,QAAQ,EAAE,CAAC;QACvF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6CAA6C,CAAC,CAAA;IACxE,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,SAAS,2BAA2B,CAAC,YAAoB,EAAE,OAAe;IACxE,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACnC,OAAO,YAAY,KAAK,MAAM,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,CAAC,CAAA;IACzE,CAAC;IACD,OAAO,YAAY,KAAK,OAAO,CAAA;AACjC,CAAC;AAED,SAAS,qBAAqB,CAAC,WAAmB,EAAE,MAAc,EAAE,KAAa;IAC/E,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;IACvD,IAAI,WAAW,GAAG,WAAW,CAAA;IAC7B,KAAK,MAAM,OAAO,IAAI,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACnE,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;QAC7C,IAAI,EAAE,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,cAAc,EAAE,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,mCAAmC,CAAC,CAAA;QAC9D,CAAC;IACH,CAAC;IACD,oBAAoB,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,CAAC,CAAA;IACrE,MAAM,IAAI,GAAG,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IACjC,IAAI,IAAI,CAAC,MAAM,EAAE;QAAE,OAAM;IACzB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,sDAAsD,CAAC,CAAA;IACjF,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3C,qBAAqB,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;IACrE,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAAC,KAAc,EAAE,KAAa;IAC9D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAM;IACrC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IAClC,QAAQ,CACN,KAAK,EACL,CAAC,kBAAkB,EAAE,yBAAyB,EAAE,4BAA4B,CAAC,EAC7E,KAAK,CACN,CAAA;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,IAAI,CACvE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CACrC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,gDAAgD,CAAC,CAAA;IAC5E,IACE,KAAK,CAAC,uBAAuB,KAAK,SAAS;WACxC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,uBAAuB,CAAC;eAC9C,KAAK,CAAC,uBAAkC,GAAG,CAAC,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,0DAA0D,CAAC,CAAA;IACrF,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,GAAG,KAAK,6BAA6B,CAAC,CAAA;AACjF,CAAC;AAED,SAAS,kCAAkC,CAAC,KAAc,EAAE,KAAa;IACvE,IAAI,KAAK,KAAK,SAAS;QAAE,OAAM;IAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IAClC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACrC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,kCAAkC,CAAC,CAAA;IACrF,KAAK,MAAM,CAAC,YAAY,EAAE,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;QAC9C,kBAAkB,CAAC,YAAY,EAAE,GAAG,KAAK,IAAI,YAAY,EAAE,EAAE,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAA;QACzF,IAAI,OAAO,OAAO,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,IAAI,YAAY,oBAAoB,CAAC,CAAA;IAChG,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,KAAa;IACjD,OAAO,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,CAAA;AACvF,CAAC;AAED,SAAS,+BAA+B,CACtC,KAAc,EACd,KAAa,EACb,WAAmB,EACnB,kBAAkB,IAAI,GAAG,EAAU;IAEnC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAM;IAC/B,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IACxC,QAAQ,CAAC,WAAW,EAAE,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,CAAA;IAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,WAAW,EAAE,GAAG,KAAK,cAAc,CAAC;SACzE,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,kBAAkB,CACvC,KAAK,EACL,GAAG,KAAK,gBAAgB,KAAK,GAAG,EAChC,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAC5B,CAAC,CAAA;IACJ,IAAI,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,MAAM,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,2CAA2C,CAAC,CAAA;IACtE,CAAC;IACD,WAAW,CAAC,OAAO,CAAC,CAAC,YAAY,EAAE,KAAK,EAAE,EAAE;QAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;QAC5D,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,YAAY,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,gBAAgB,KAAK,uCAAuC,CAAC,CAAA;QACvF,CAAC;QACD,qBAAqB,CAAC,WAAW,EAAE,YAAY,EAAE,GAAG,KAAK,gBAAgB,KAAK,GAAG,CAAC,CAAA;QAClF,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;YAChF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,gBAAgB,KAAK,+CAA+C,CAAC,CAAA;QAC/F,CAAC;QACD,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;IACnC,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,2CAA2C,CACzD,KAAc,EACd,aAAqB,EACrB,KAAK,GAAG,oBAAoB,EAC5B,WAAW,GAAG,OAAO,CAAC,GAAG,EAAE,EAC3B,mBAA6B,EAAE;IAE/B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IACrC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,GAAG,KAAK,aAAa,CAAC,CAAA;IAChE,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,gBAAgB,KAAK,iCAAiC,aAAa,IAAI,CAAC,CAAA;IAClG,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,mCAAmC,CAAC,CAAA;IAC9D,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QACzC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,KAAK,UAAU,KAAK,GAAG,CAAC,CAAA;QACrD,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,UAAU,KAAK,MAAM,CAAC,CAAC,IAAI,EAAE,CAAA;QAC/D,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,2BAA2B,EAAE,IAAI,CAAC,CAAA;QAC3E,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACX,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAA;QACrD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,KAAK,SAAS;YACvC,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,UAAU,KAAK,WAAW,CAAC,CAAA;QAC3D,IAAI,OAAO,KAAK,WAAW,IAAI,OAAO,KAAK,iBAAiB,EAAE,CAAC;YAC7D,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,qDAAqD,CAAC,CAAA;QAC/F,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,sCAAsC,CAAC,CAAA;QAChF,CAAC;QACD,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,yBAAyB,CAC3E,KAAK,EACL,GAAG,KAAK,UAAU,KAAK,sBAAsB,UAAU,GAAG,CAC3D,CAAC,CAAA;QACF,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,GAAG,KAAK,UAAU,KAAK,mBAAmB,CAAC,CAAA;QACvE,kCAAkC,CAChC,GAAG,CAAC,eAAe,EACnB,GAAG,KAAK,UAAU,KAAK,mBAAmB,CAC3C,CAAA;QACD,+BAA+B,CAC7B,GAAG,CAAC,WAAW,EACf,GAAG,KAAK,UAAU,KAAK,eAAe,EACtC,WAAW,EACX,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAC1B,CAAA;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAA;QACpE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,KAAK,UAAU,KAAK,iBAAiB,CAAC,CAAA;QAChE,IAAI,MAAM,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;YAC7C,OAAO,CAAC,MAAM,CAAC,mBAAmB,EAAE,GAAG,KAAK,UAAU,KAAK,8BAA8B,CAAC,CAAA;QAC5F,CAAC;QACD,MAAM,uBAAuB,GAAG,MAAM,CAAC,yBAAyB,KAAK,SAAS;eACzE,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,yBAAyB,CAAC,IAAI,MAAM,CAAC,yBAAyB,CAAC,MAAM,KAAK,CAAC,CAAC;YACrG,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,OAAO,CACL,MAAM,CAAC,yBAAyB,EAChC,GAAG,KAAK,UAAU,KAAK,oCAAoC,CAC5D,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CAAC,kBAAkB,CAC5C,KAAK,EACL,GAAG,KAAK,UAAU,KAAK,sCAAsC,SAAS,GAAG,EACzE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAC3B,CAAC,CAAA;QACN,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAA;QACxE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,sCAAsC,CAAC,CAAA;QAChF,CAAC;QACD,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE;YAC9D,MAAM,aAAa,GAAG,GAAG,KAAK,UAAU,KAAK,sBAAsB,aAAa,GAAG,CAAA;YACnF,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAA;YACnD,QAAQ,CAAC,WAAW,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,aAAa,CAAC,CAAA;YAC5F,MAAM,YAAY,GAAG,kBAAkB,CACrC,WAAW,CAAC,IAAI,EAChB,GAAG,aAAa,OAAO,EACvB,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAC5B,CAAA;YACD,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS;gBAAE,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,aAAa,SAAS,CAAC,CAAA;YAC5F,IAAI,WAAW,CAAC,QAAQ,KAAK,SAAS;gBAAE,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,GAAG,aAAa,WAAW,CAAC,CAAA;YAClG,IAAI,WAAW,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;gBAC3C,OAAO,CAAC,WAAW,CAAC,YAAY,EAAE,GAAG,aAAa,eAAe,CAAC,CAAA;YACpE,CAAC;YACD,IAAI,WAAW,CAAC,IAAI,KAAK,SAAS;gBAAE,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,aAAa,OAAO,CAAC,CAAA;YACrF,OAAO,YAAY,CAAA;QACrB,CAAC,CAAC,CAAA;QACF,IAAI,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,8CAA8C,CAAC,CAAA;QACxF,CAAC;QACD,IAAI,OAAO,KAAK,iBAAiB,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,mBAAmB,KAAK,KAAK,EAAE,CAAC;gBACzC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,8DAA8D,CAAC,CAAA;YACxG,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,yBAAyB,CAAC,EAAE,CAAC;gBACrD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,uEAAuE,CAAC,CAAA;YACjH,CAAC;YACD,IAAI,uBAAuB,CAAC,MAAM,GAAG,CAAC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,KAAK,wDAAwD,CAAC,CAAA;YAClG,CAAC;YACD,iFAAiF;YACjF,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,MAAM,GAAG,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,uBAAuB,CAAC,IAAI,CAChI,CAAC,OAAO,EAAE,EAAE,CAAC,2BAA2B,CAAC,YAAY,EAAE,OAAO,CAAC,CAChE,CAAC,CAAA;YACF,IAAI,iBAAiB,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,UAAU,KAAK,mBAAmB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,GAAG;sBAC5E,8CAA8C,CACjD,CAAA;YACH,CAAC;QACH,CAAC;QACD,OAAO,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,CAAA;IAChC,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,KAAc,EACd,KAAa,EACb,KAA0C;IAE1C,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,iBAAiB,KAAK,GAAG,CAAC,CAAA;IACpD,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,iBAAiB,KAAK,MAAM,CAAC,CAAA;IACvD,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,qCAAqC,CAAC,CAAA;IAC9E,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,SAAS,CAAC,CAAA;IAChE,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,YAAY,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,+CAA+C,CAAC,CAAA;IACxF,CAAC;IACD,MAAM,gBAAgB,GAAG,GAAG,CAAC,kBAAkB,KAAK,SAAS;QAC3D,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,iBAAiB,KAAK,sBAAsB,CAAC,CAAA;IACjF,IAAI,KAAK,CAAC,iBAAiB,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,+DAA+D,CACtF,CAAA;IACH,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC/D,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,uDAAuD,CAAC,CAAA;IAChG,CAAC;IACD,MAAM,gBAAgB,GAAG,GAAG,CAAC,iBAAiB,KAAK,SAAS;QAC1D,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,EAAE,iBAAiB,KAAK,qBAAqB,CAAC,CAAA;IAC9E,MAAM,eAAe,GAAG,GAAG,CAAC,iBAAiB,KAAK,SAAS;QACzD,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,iBAAiB,KAAK,qBAAqB,CAAC,CAAA;IAC/E,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,iBAAiB,KAAK,QAAQ,CAAC,CAAA;IACnG,MAAM,mBAAmB,GAAG,GAAG,CAAC,iBAAiB,KAAK,SAAS,CAAA;IAC/D,MAAM,oBAAoB,GAAG,GAAG,CAAC,sBAAsB,KAAK,SAAS,CAAA;IACrE,MAAM,yBAAyB,GAAG,GAAG,CAAC,2BAA2B,KAAK,SAAS,CAAA;IAC/E,IAAI,CAAC,mBAAmB,EAAE,oBAAoB,EAAE,yBAAyB,CAAC;SACvE,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,mDAAmD;cACvE,yDAAyD,CAC5D,CAAA;IACH,CAAC;IACD,IAAI,oBAA0C,CAAA;IAC9C,IAAI,oBAAoB,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,4CAA4C,CAAC,CAAA;QACrF,CAAC;QACD,oBAAoB,GAAG,GAAG,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,MAAM,CACjF,KAAK,EACL,iBAAiB,KAAK,4BAA4B,UAAU,GAAG,CAChE,CAAC,CAAA;QACF,IAAI,IAAI,GAAG,CAAC,oBAAoB,CAAC,CAAC,IAAI,KAAK,oBAAoB,CAAC,MAAM,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,uDAAuD,CAAC,CAAA;QAChG,CAAC;QACD,MAAM,YAAY,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;QACxF,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,4DAA4D,YAAY,IAAI,CACnG,CAAA;QACH,CAAC;QACD,IAAI,KAAK,CAAC,wBAAwB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,wCAAwC,KAAK,CAAC,EAAE,iCAAiC,CACxG,CAAA;QACH,CAAC;IACH,CAAC;IACD,MAAM,wBAAwB,GAAG,yBAAyB;QACxD,CAAC,CAAC,MAAM,CACN,GAAG,CAAC,2BAA2B,EAC/B,iBAAiB,KAAK,+BAA+B,CACtD;QACD,CAAC,CAAC,SAAS,CAAA;IACb,IAAI,wBAAwB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,+DAA+D;cACnF,IAAI,wBAAwB,IAAI,CACnC,CAAA;IACH,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,SAAS,CAAC,CAAA;IAChE,OAAO;QACL,GAAG,GAAG;QACN,EAAE;QACF,KAAK;QACL,KAAK;QACL,GAAG,CAAC,mBAAmB;YACrB,CAAC,CAAC,EAAE,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,iBAAiB,KAAK,qBAAqB,CAAC,EAAE;YACpG,CAAC,CAAC,oBAAoB;gBACpB,CAAC,CAAC,EAAE,sBAAsB,EAAE,oBAAqB,EAAE;gBACnD,CAAC,CAAC,EAAE,2BAA2B,EAAE,wBAAyB,EAAE,CAAC;QACjE,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,CAAC;QACnF,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,eAAe,EAAE,CAAC;QAChF,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1B,CAAA;AACH,CAAC;AAED,MAAM,UAAU,sCAAsC,CACpD,KAAc,EACd,KAA0C,EAC1C,KAAK,GAAG,4BAA4B,KAAK,CAAC,EAAE,GAAG;IAE/C,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IACrC,IAAI,QAAQ,CAAC,aAAa,KAAK,iCAAiC,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,2BAA2B,iCAAiC,GAAG,CAAC,CAAA;IAC1F,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,KAAK,QAAQ,CAAC,CAAA;IACtD,IAAI,KAAK,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,wBAAwB,KAAK,iCAAiC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;IAChG,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,wCAAwC,CAAC,CAAA;IACnE,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,+BAA+B,CAChF,KAAK,EACL,KAAK,EACL,KAAK,CACN,CAAC,CAAA;IACF,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;IAC/C,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CACb,kBAAkB,KAAK,CAAC,EAAE,mEAAmE,CAC9F,CAAA;IACH,CAAC;IACD,OAAO,EAAE,aAAa,EAAE,iCAAiC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;AAC3E,CAAC"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import fs from 'fs-extra';
|
|
3
|
+
const MAX_ROLLOUT_BYTES = 64 * 1024 * 1024;
|
|
4
|
+
const MAX_ROLLOUT_LINES = 100_000;
|
|
5
|
+
function isRecord(value) {
|
|
6
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
7
|
+
}
|
|
8
|
+
export function selectedCodexSkillInstruction(event) {
|
|
9
|
+
if (!isRecord(event) || event.type !== 'response_item' || !isRecord(event.payload))
|
|
10
|
+
return null;
|
|
11
|
+
const payload = event.payload;
|
|
12
|
+
if (payload.type !== 'message' || payload.role !== 'user')
|
|
13
|
+
return null;
|
|
14
|
+
const metadata = isRecord(payload.internal_chat_message_metadata_passthrough)
|
|
15
|
+
? payload.internal_chat_message_metadata_passthrough
|
|
16
|
+
: null;
|
|
17
|
+
if (!metadata || !Array.isArray(metadata.content_item_kinds)
|
|
18
|
+
|| !metadata.content_item_kinds.includes('skills.selected_skill_instructions'))
|
|
19
|
+
return null;
|
|
20
|
+
if (!Array.isArray(payload.content))
|
|
21
|
+
return null;
|
|
22
|
+
const text = payload.content.filter(isRecord)
|
|
23
|
+
.filter((entry) => entry.type === 'input_text' && typeof entry.text === 'string')
|
|
24
|
+
.map((entry) => entry.text).join('\n').replaceAll('\\', '/');
|
|
25
|
+
const name = text.match(/<name>([^/<>]+)<\/name>/u)?.[1];
|
|
26
|
+
const skillPath = text.match(/<path>([^<>]+)<\/path>/u)?.[1];
|
|
27
|
+
return name && skillPath ? { name, path: skillPath } : null;
|
|
28
|
+
}
|
|
29
|
+
export async function readCodexEvalRollouts(codexHome) {
|
|
30
|
+
const files = [];
|
|
31
|
+
async function visit(current) {
|
|
32
|
+
const stat = await fs.lstat(current).catch((error) => {
|
|
33
|
+
if (error.code === 'ENOENT')
|
|
34
|
+
return null;
|
|
35
|
+
throw error;
|
|
36
|
+
});
|
|
37
|
+
if (!stat)
|
|
38
|
+
return;
|
|
39
|
+
if (stat.isSymbolicLink())
|
|
40
|
+
throw new Error('Codex rollout tree contains a symbolic link.');
|
|
41
|
+
for (const entry of await fs.readdir(current, { withFileTypes: true })) {
|
|
42
|
+
const target = path.join(current, entry.name);
|
|
43
|
+
if (entry.isDirectory())
|
|
44
|
+
await visit(target);
|
|
45
|
+
else if (entry.isFile() && target.endsWith('.jsonl'))
|
|
46
|
+
files.push(target);
|
|
47
|
+
else if (entry.isSymbolicLink())
|
|
48
|
+
throw new Error('Codex rollout tree contains a symbolic link.');
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
await visit(path.join(codexHome, 'sessions'));
|
|
52
|
+
const records = [];
|
|
53
|
+
let totalBytes = 0;
|
|
54
|
+
let totalLines = 0;
|
|
55
|
+
for (const file of files.sort()) {
|
|
56
|
+
const stat = await fs.lstat(file);
|
|
57
|
+
if (!stat.isFile())
|
|
58
|
+
throw new Error('Codex rollout must be a regular file.');
|
|
59
|
+
totalBytes += stat.size;
|
|
60
|
+
if (totalBytes > MAX_ROLLOUT_BYTES)
|
|
61
|
+
throw new Error('Codex rollout evidence is too large.');
|
|
62
|
+
const bytes = await fs.readFile(file);
|
|
63
|
+
if (bytes.length !== stat.size)
|
|
64
|
+
throw new Error('Codex rollout changed during evidence collection.');
|
|
65
|
+
const lines = bytes.toString('utf8').split('\n').filter((line) => line.trim() !== '');
|
|
66
|
+
totalLines += lines.length;
|
|
67
|
+
if (totalLines > MAX_ROLLOUT_LINES)
|
|
68
|
+
throw new Error('Codex rollout evidence has too many events.');
|
|
69
|
+
const events = lines.map((line, index) => {
|
|
70
|
+
let value;
|
|
71
|
+
try {
|
|
72
|
+
value = JSON.parse(line);
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
throw new Error(`Codex rollout contains invalid JSON at line ${index + 1}.`);
|
|
76
|
+
}
|
|
77
|
+
if (!isRecord(value))
|
|
78
|
+
throw new Error(`Codex rollout event ${index + 1} must be an object.`);
|
|
79
|
+
return value;
|
|
80
|
+
});
|
|
81
|
+
records.push({ file, events });
|
|
82
|
+
}
|
|
83
|
+
return records;
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=codex-eval-rollout.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"codex-eval-rollout.js","sourceRoot":"","sources":["../src/codex-eval-rollout.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,EAAE,MAAM,UAAU,CAAA;AAEzB,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AAC1C,MAAM,iBAAiB,GAAG,OAAO,CAAA;AAEjC,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC7E,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,KAAc;IAI1D,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAA;IAC/F,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAA;IAC7B,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,IAAI,CAAA;IACtE,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,0CAA0C,CAAC;QAC3E,CAAC,CAAC,OAAO,CAAC,0CAA0C;QACpD,CAAC,CAAC,IAAI,CAAA;IACR,IAAI,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;WACvD,CAAC,QAAQ,CAAC,kBAAkB,CAAC,QAAQ,CAAC,oCAAoC,CAAC;QAAE,OAAO,IAAI,CAAA;IAC7F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAA;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;SAC1C,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;SAChF,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IACxE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,yBAAyB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAC5D,OAAO,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,SAAiB;IAI3D,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,KAAK,UAAU,KAAK,CAAC,OAAe;QAClC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAA4B,EAAE,EAAE;YAC1E,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;gBAAE,OAAO,IAAI,CAAA;YACxC,MAAM,KAAK,CAAA;QACb,CAAC,CAAC,CAAA;QACF,IAAI,CAAC,IAAI;YAAE,OAAM;QACjB,IAAI,IAAI,CAAC,cAAc,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;QAC1F,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACvE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;YAC7C,IAAI,KAAK,CAAC,WAAW,EAAE;gBAAE,MAAM,KAAK,CAAC,MAAM,CAAC,CAAA;iBACvC,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;iBACnE,IAAI,KAAK,CAAC,cAAc,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;QAClG,CAAC;IACH,CAAC;IACD,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAA;IAC7C,MAAM,OAAO,GAAG,EAAE,CAAA;IAClB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACjC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAA;QAC5E,UAAU,IAAI,IAAI,CAAC,IAAI,CAAA;QACvB,IAAI,UAAU,GAAG,iBAAiB;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QAC3F,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACrC,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAA;QACpG,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QACrF,UAAU,IAAI,KAAK,CAAC,MAAM,CAAA;QAC1B,IAAI,UAAU,GAAG,iBAAiB;YAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAA;QAClG,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YACvC,IAAI,KAAc,CAAA;YAClB,IAAI,CAAC;gBACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,+CAA+C,KAAK,GAAG,CAAC,GAAG,CAAC,CAAA;YAC9E,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,KAAK,GAAG,CAAC,qBAAqB,CAAC,CAAA;YAC5F,OAAO,KAAK,CAAA;QACd,CAAC,CAAC,CAAA;QACF,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAA;IAChC,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC"}
|
|
@@ -63,6 +63,7 @@ export declare function inspectTriggerAppServerFrame(frame: Record<string, unkno
|
|
|
63
63
|
turnStatus?: string;
|
|
64
64
|
spawnCall?: Record<string, unknown>;
|
|
65
65
|
requestUserInput?: boolean;
|
|
66
|
+
loadEvidenceUnavailable?: string;
|
|
66
67
|
};
|
|
67
68
|
export declare function inspectTriggerAppServerConfigWarning(frame: Record<string, unknown>): string | null;
|
|
68
69
|
export declare function runCodexTriggerProviderCase(options: {
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
+
import { realpathSync } from 'node:fs';
|
|
2
3
|
import path from 'node:path';
|
|
3
4
|
import { pathToFileURL } from 'node:url';
|
|
4
5
|
import { computeSkillDigest } from './runtime-skill.js';
|
|
5
6
|
import fs from 'fs-extra';
|
|
6
7
|
import { AppServerRequestTimeoutError, CodexEvalAttemptControl, EvalAppServerSession, extractAppServerThreadId, inspectAppServerConfigWarning, } from './codex-eval-app-server.js';
|
|
7
|
-
import { assertCodexEvalCanStartWork, applyCodexEvalProviderEnvironment, createCodexEvalTemporaryRoot, materializeCodexEvalRuntimeHome, } from './codex-eval-runtime-config.js';
|
|
8
|
+
import { assertCodexEvalCanStartWork, applyCodexEvalProviderEnvironment, createCodexEvalTemporaryRoot, materializeCodexEvalRuntimeHome, redactCodexEvalValue, } from './codex-eval-runtime-config.js';
|
|
8
9
|
import { applyCodexEvalGitIsolation } from './codex-eval-isolation.js';
|
|
10
|
+
import { readCodexEvalRollouts, selectedCodexSkillInstruction } from './codex-eval-rollout.js';
|
|
9
11
|
async function createTriggerTemporaryRoot(workRoot, strict = false) {
|
|
10
12
|
if (strict || !workRoot)
|
|
11
13
|
return createCodexEvalTemporaryRoot('aw-trigger-eval-');
|
|
@@ -101,10 +103,6 @@ async function snapshotWorkspace(root) {
|
|
|
101
103
|
await visit(root);
|
|
102
104
|
return snapshot;
|
|
103
105
|
}
|
|
104
|
-
function explicitSkillRequested(query, skillName) {
|
|
105
|
-
const escaped = skillName.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
|
|
106
|
-
return new RegExp(`(?:^|\\s)\\$${escaped}(?:\\s|$)`, 'u').test(query);
|
|
107
|
-
}
|
|
108
106
|
function expectedLoadedSkills(suite, evalCase) {
|
|
109
107
|
if (evalCase.expected_loaded_skills) {
|
|
110
108
|
return [...evalCase.expected_loaded_skills];
|
|
@@ -119,27 +117,59 @@ function routeLabel(skills) {
|
|
|
119
117
|
return 'neither';
|
|
120
118
|
return [...skills].sort((left, right) => left.localeCompare(right)).join('+');
|
|
121
119
|
}
|
|
122
|
-
function normalizedCommandReadsSkill(command, skillPath) {
|
|
123
|
-
const
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
120
|
+
function normalizedCommandReadsSkill(command, skillPath, commandCwd, outputProvesRead) {
|
|
121
|
+
const tokenPattern = /"[^"\r\n]*"|'[^'\r\n]*'|[^\s"';&|]+/gu;
|
|
122
|
+
const tokens = command.match(tokenPattern) ?? [];
|
|
123
|
+
const unquote = (value) => /^["']/u.test(value)
|
|
124
|
+
? value.slice(1, -1) : value;
|
|
125
|
+
const words = tokens.map(unquote);
|
|
126
|
+
if (/^(?:ba|z|da)?sh(?:\.exe)?$/iu.test(path.basename(words[0] ?? ''))
|
|
127
|
+
&& /^-[a-z]*c$/u.test(words[1] ?? '') && words.length === 3) {
|
|
128
|
+
return normalizedCommandReadsSkill(words[2], skillPath, commandCwd, outputProvesRead);
|
|
129
|
+
}
|
|
130
|
+
const mayReadSkill = command.includes('SKILL.md')
|
|
131
|
+
&& /\b(?:cat|sed|head|tail|less|more|bat|awk|grep|rg|type|get-content)\b/iu.test(command);
|
|
132
|
+
// Shell classifications and aggregate output do not prove that a conditional read ran.
|
|
133
|
+
if (command.replace(tokenPattern, '').trim() !== ''
|
|
134
|
+
|| /[$`;&|<>()\r\n*?\[\]{}]/u.test(command)
|
|
135
|
+
|| (process.platform !== 'win32' && command.includes('\\'))) {
|
|
136
|
+
return mayReadSkill ? null : false;
|
|
137
|
+
}
|
|
138
|
+
const executable = path.basename(words[0] ?? '').toLowerCase();
|
|
139
|
+
if (!/^(?:cat|sed|head|tail|less|more|bat|type|get-content)$/u.test(executable)) {
|
|
140
|
+
return /^(?:echo|printf|ls|dir)$/u.test(executable) ? false : mayReadSkill ? null : false;
|
|
141
|
+
}
|
|
142
|
+
const argumentsList = words.slice(1);
|
|
143
|
+
const options = argumentsList.slice(0, -1);
|
|
144
|
+
const simpleRead = (executable !== 'sed' && (options.length === 0 || (options.length === 1 && options[0] === '--')))
|
|
145
|
+
|| (/^(?:head|tail)$/u.test(executable) && options.length === 2
|
|
146
|
+
&& /^(?:-n|-c|--lines|--bytes)$/u.test(options[0]) && /^[1-9]\d*$/u.test(options[1]))
|
|
147
|
+
|| (executable === 'sed' && options.length === 2 && options[0] === '-n'
|
|
148
|
+
&& /^\d+(?:,\d+)?p$/u.test(options[1]));
|
|
149
|
+
if (!simpleRead || argumentsList.length === 0)
|
|
150
|
+
return mayReadSkill ? null : false;
|
|
151
|
+
const argument = argumentsList.at(-1);
|
|
152
|
+
if (!path.isAbsolute(argument)
|
|
153
|
+
&& (typeof commandCwd !== 'string' || !path.isAbsolute(commandCwd))) {
|
|
154
|
+
return path.basename(argument) === 'SKILL.md' ? null : false;
|
|
155
|
+
}
|
|
156
|
+
let target = argument;
|
|
157
|
+
if (!path.isAbsolute(argument)) {
|
|
158
|
+
try {
|
|
159
|
+
if (!fs.statSync(commandCwd).isDirectory())
|
|
160
|
+
return null;
|
|
161
|
+
target = path.resolve(realpathSync(commandCwd), argument);
|
|
127
162
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
163
|
+
catch {
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
try {
|
|
168
|
+
return realpathSync(target) === realpathSync(skillPath) && (executable === 'cat' || outputProvesRead);
|
|
133
169
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
const managedSkillDir = `.agents/skills/${path.basename(path.dirname(skillPath))}`;
|
|
137
|
-
if (normalized.includes(normalizedSkillPath) ||
|
|
138
|
-
normalized.includes(`${managedSkillDir}/SKILL.md`)) {
|
|
139
|
-
return true;
|
|
170
|
+
catch {
|
|
171
|
+
return false;
|
|
140
172
|
}
|
|
141
|
-
return ((normalized.includes(normalizedSkillDir) || normalized.includes(managedSkillDir)) &&
|
|
142
|
-
/(?:^|[\s;&|/])SKILL\.md(?:$|[\s;&|])/u.test(normalized));
|
|
143
173
|
}
|
|
144
174
|
function commandOutputContainsSkill(item, skillPath) {
|
|
145
175
|
if (typeof item.aggregatedOutput !== 'string')
|
|
@@ -155,11 +185,16 @@ export function inspectTriggerAppServerFrame(frame, skillPath) {
|
|
|
155
185
|
if (item?.type === 'commandExecution' && typeof item.command === 'string') {
|
|
156
186
|
const completed = method === 'item/completed';
|
|
157
187
|
const statusPassed = item.status === undefined || item.status === 'completed';
|
|
158
|
-
const exitPassed = item.exitCode ===
|
|
188
|
+
const exitPassed = item.exitCode === 0;
|
|
159
189
|
const outputProvesRead = commandOutputContainsSkill(item, skillPath);
|
|
160
|
-
if (completed
|
|
161
|
-
|
|
162
|
-
|
|
190
|
+
if (!completed)
|
|
191
|
+
return {};
|
|
192
|
+
const readsSkill = normalizedCommandReadsSkill(item.command, skillPath, item.cwd, outputProvesRead);
|
|
193
|
+
if (readsSkill === null)
|
|
194
|
+
return {
|
|
195
|
+
loadEvidenceUnavailable: 'Trigger eval cannot prove a participant read from this command and its cwd.',
|
|
196
|
+
};
|
|
197
|
+
if (readsSkill && ((statusPassed && exitPassed) || outputProvesRead)) {
|
|
163
198
|
return { loaded: true, loadEvidence: item.command };
|
|
164
199
|
}
|
|
165
200
|
}
|
|
@@ -411,15 +446,12 @@ async function runTriggerOnce(options) {
|
|
|
411
446
|
}
|
|
412
447
|
}
|
|
413
448
|
const beforeState = await captureState(options.fixture, context);
|
|
414
|
-
const allLoadedSkills =
|
|
415
|
-
const loadEvidenceBySkill =
|
|
416
|
-
skillName,
|
|
417
|
-
`explicit $${skillName} invocation`,
|
|
418
|
-
]));
|
|
449
|
+
const allLoadedSkills = [];
|
|
450
|
+
const loadEvidenceBySkill = {};
|
|
419
451
|
const observed = {
|
|
420
|
-
loaded:
|
|
452
|
+
loaded: false,
|
|
421
453
|
spawned: false,
|
|
422
|
-
loadEvidence:
|
|
454
|
+
loadEvidence: null,
|
|
423
455
|
spawnEvidence: null,
|
|
424
456
|
};
|
|
425
457
|
let requestUserInputCalled = false;
|
|
@@ -439,6 +471,10 @@ async function runTriggerOnce(options) {
|
|
|
439
471
|
const spawnCalls = [];
|
|
440
472
|
const spawnCallIds = new Set();
|
|
441
473
|
let rootThreadId;
|
|
474
|
+
let rootTurnId;
|
|
475
|
+
const skillFrames = [];
|
|
476
|
+
let skillReadUnavailable = false;
|
|
477
|
+
const readLoads = [];
|
|
442
478
|
let turnStatus;
|
|
443
479
|
let finishRun = () => undefined;
|
|
444
480
|
let rejectRun = () => undefined;
|
|
@@ -458,6 +494,29 @@ async function runTriggerOnce(options) {
|
|
|
458
494
|
};
|
|
459
495
|
});
|
|
460
496
|
void runDone.catch(() => undefined);
|
|
497
|
+
const inspectSkillFrame = (frame) => {
|
|
498
|
+
const params = isRecord(frame.params) ? frame.params : {};
|
|
499
|
+
if (!rootTurnId || params.threadId !== rootThreadId
|
|
500
|
+
|| (params.turnId !== undefined && params.turnId !== rootTurnId))
|
|
501
|
+
return;
|
|
502
|
+
for (const [skillName, skillPath] of skillPaths) {
|
|
503
|
+
const skillEvidence = inspectTriggerAppServerFrame(frame, skillPath);
|
|
504
|
+
if (skillEvidence.loadEvidenceUnavailable) {
|
|
505
|
+
skillReadUnavailable = true;
|
|
506
|
+
if (!diagnostics.includes(skillEvidence.loadEvidenceUnavailable))
|
|
507
|
+
diagnostics.push(skillEvidence.loadEvidenceUnavailable);
|
|
508
|
+
}
|
|
509
|
+
if (!skillEvidence.loaded || !skillEvidence.loadEvidence)
|
|
510
|
+
continue;
|
|
511
|
+
const item = isRecord(params.item) ? params.item : {};
|
|
512
|
+
readLoads.push({
|
|
513
|
+
skill: skillName,
|
|
514
|
+
evidence: skillEvidence.loadEvidence,
|
|
515
|
+
callId: typeof item.id === 'string' ? item.id : null,
|
|
516
|
+
});
|
|
517
|
+
recordSkillLoad(skillName, skillEvidence.loadEvidence);
|
|
518
|
+
}
|
|
519
|
+
};
|
|
461
520
|
const abort = () => {
|
|
462
521
|
finishRun(control.timeoutExpired ? 'timeout' : 'aborted');
|
|
463
522
|
void session?.cleanup().catch((error) => {
|
|
@@ -510,12 +569,8 @@ async function runTriggerOnce(options) {
|
|
|
510
569
|
return;
|
|
511
570
|
}
|
|
512
571
|
const evidence = inspectTriggerAppServerFrame(frame, ownerSkillPath);
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
if (skillEvidence.loaded) {
|
|
516
|
-
recordSkillLoad(skillName, skillEvidence.loadEvidence);
|
|
517
|
-
}
|
|
518
|
-
}
|
|
572
|
+
skillFrames.push(frame);
|
|
573
|
+
inspectSkillFrame(frame);
|
|
519
574
|
if (evidence.spawned) {
|
|
520
575
|
observed.spawned = true;
|
|
521
576
|
observed.spawnEvidence ??= evidence.spawnEvidence ?? null;
|
|
@@ -565,7 +620,7 @@ async function runTriggerOnce(options) {
|
|
|
565
620
|
}, control.rpcTimeout('thread/start'));
|
|
566
621
|
rootThreadId = extractAppServerThreadId(threadResult);
|
|
567
622
|
control.throwIfStopped('turn/start');
|
|
568
|
-
await session.sendRequest('turn/start', {
|
|
623
|
+
const turnResult = await session.sendRequest('turn/start', {
|
|
569
624
|
threadId: rootThreadId,
|
|
570
625
|
input: [{ type: 'text', text: options.evalCase.query, text_elements: [] }],
|
|
571
626
|
cwd: workspaceRoot,
|
|
@@ -573,6 +628,13 @@ async function runTriggerOnce(options) {
|
|
|
573
628
|
sandboxPolicy: { type: 'readOnly', networkAccess: true },
|
|
574
629
|
...(options.model ? { model: options.model } : {}),
|
|
575
630
|
}, control.rpcTimeout('turn/start'));
|
|
631
|
+
const turn = isRecord(turnResult) && isRecord(turnResult.turn) ? turnResult.turn : {};
|
|
632
|
+
rootTurnId = typeof turn.id === 'string' && turn.id !== '' ? turn.id : undefined;
|
|
633
|
+
for (const frame of skillFrames)
|
|
634
|
+
inspectSkillFrame(frame);
|
|
635
|
+
const loadedStop = expectedEarlyStop(options.suite, options.evalCase, observed, allLoadedSkills);
|
|
636
|
+
if (loadedStop)
|
|
637
|
+
finishRun(loadedStop);
|
|
576
638
|
const termination = await runDone;
|
|
577
639
|
control.throwIfStopped('turn wait');
|
|
578
640
|
removeHandler();
|
|
@@ -588,6 +650,75 @@ async function runTriggerOnce(options) {
|
|
|
588
650
|
}
|
|
589
651
|
if (!cleanupFailed)
|
|
590
652
|
control.throwIfStopped('cleanup');
|
|
653
|
+
let rolloutUnavailable = false;
|
|
654
|
+
let firstLoadedSkill = allLoadedSkills[0] ?? null;
|
|
655
|
+
try {
|
|
656
|
+
if (!rootTurnId)
|
|
657
|
+
throw new Error('app-server turn/start response is missing turn.id.');
|
|
658
|
+
const rollouts = await readCodexEvalRollouts(codexHome);
|
|
659
|
+
const roots = rollouts.filter(({ events }) => events.some((event) => event.type === 'session_meta' && isRecord(event.payload) && event.payload.id === rootThreadId));
|
|
660
|
+
if (roots.length !== 1)
|
|
661
|
+
throw new Error('Codex rollout must contain one app-server root thread.');
|
|
662
|
+
const root = roots[0];
|
|
663
|
+
eventTrace.push(...root.events.map((event) => ({
|
|
664
|
+
source: 'codex_rollout', file: path.relative(codexHome, root.file), event,
|
|
665
|
+
})));
|
|
666
|
+
if (root.events.filter((event) => event.type === 'session_meta').length !== 1) {
|
|
667
|
+
throw new Error('Codex rollout contains conflicting thread identities.');
|
|
668
|
+
}
|
|
669
|
+
const turnIds = new Set(root.events.flatMap((event) => {
|
|
670
|
+
if (!isRecord(event.payload) || (event.type !== 'turn_context' && event.type !== 'event_msg'))
|
|
671
|
+
return [];
|
|
672
|
+
return typeof event.payload.turn_id === 'string' ? [event.payload.turn_id] : [];
|
|
673
|
+
}));
|
|
674
|
+
if (turnIds.size !== 1 || !turnIds.has(rootTurnId)) {
|
|
675
|
+
throw new Error('Codex rollout cannot be associated with the app-server root turn.');
|
|
676
|
+
}
|
|
677
|
+
const orderedLoads = [];
|
|
678
|
+
const canonicalSkillPaths = new Map(await Promise.all([...skillPaths].map(async ([name, skillPath]) => [name, await fs.realpath(skillPath)])));
|
|
679
|
+
for (const [index, event] of root.events.entries()) {
|
|
680
|
+
const selected = selectedCodexSkillInstruction(event);
|
|
681
|
+
if (!selected && event.type === 'response_item' && isRecord(event.payload)) {
|
|
682
|
+
const metadata = event.payload.internal_chat_message_metadata_passthrough;
|
|
683
|
+
if (isRecord(metadata) && Array.isArray(metadata.content_item_kinds)
|
|
684
|
+
&& metadata.content_item_kinds.includes('skills.selected_skill_instructions')) {
|
|
685
|
+
throw new Error('Codex selected Skill instruction metadata has an unsupported message format.');
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
if (!selected)
|
|
689
|
+
continue;
|
|
690
|
+
const participantPath = canonicalSkillPaths.get(selected.name);
|
|
691
|
+
const selectedPath = path.isAbsolute(selected.path)
|
|
692
|
+
? await fs.realpath(selected.path).catch(() => null) : null;
|
|
693
|
+
if (!participantPath && ![...canonicalSkillPaths.values()].includes(selectedPath ?? ''))
|
|
694
|
+
continue;
|
|
695
|
+
if (!participantPath || selectedPath !== participantPath) {
|
|
696
|
+
throw new Error('Codex selected Skill instructions do not match the participant name and path.');
|
|
697
|
+
}
|
|
698
|
+
orderedLoads.push({ skill: selected.name, evidence: `rollout:selected_skill_instructions:${selected.path}`, order: index });
|
|
699
|
+
}
|
|
700
|
+
for (const [index, load] of readLoads.entries()) {
|
|
701
|
+
const order = load.callId === null ? -1 : root.events.findIndex((event) => event.type === 'response_item' && isRecord(event.payload)
|
|
702
|
+
&& ['function_call', 'custom_tool_call'].includes(String(event.payload.type))
|
|
703
|
+
&& event.payload.call_id === load.callId);
|
|
704
|
+
orderedLoads.push({ ...load, order: order < 0 ? root.events.length + index : order });
|
|
705
|
+
}
|
|
706
|
+
const unlinkedRead = readLoads.some((load) => load.callId === null || !root.events.some((event) => event.type === 'response_item' && isRecord(event.payload) && event.payload.call_id === load.callId));
|
|
707
|
+
const hasInjection = orderedLoads.length > readLoads.length;
|
|
708
|
+
orderedLoads.sort((left, right) => left.order - right.order);
|
|
709
|
+
allLoadedSkills.length = 0;
|
|
710
|
+
for (const load of orderedLoads)
|
|
711
|
+
recordSkillLoad(load.skill, load.evidence);
|
|
712
|
+
firstLoadedSkill = hasInjection && unlinkedRead && allLoadedSkills.length > 1
|
|
713
|
+
? null : allLoadedSkills[0] ?? null;
|
|
714
|
+
if (firstLoadedSkill === null && allLoadedSkills.length > 0) {
|
|
715
|
+
diagnostics.push('Codex load evidence does not establish an order between injected and read Skills.');
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
catch (error) {
|
|
719
|
+
rolloutUnavailable = true;
|
|
720
|
+
diagnostics.push(`Trigger eval rollout evidence is unavailable: ${error instanceof Error ? error.message : String(error)}`);
|
|
721
|
+
}
|
|
591
722
|
const afterState = await captureState(options.fixture, context);
|
|
592
723
|
let skillsUnchanged = true;
|
|
593
724
|
for (const [skillName, skillPath] of skillPaths) {
|
|
@@ -617,10 +748,12 @@ async function runTriggerOnce(options) {
|
|
|
617
748
|
termination === 'app_server_error' ||
|
|
618
749
|
(!expectedStop && termination === 'completed' && turnStatus !== 'completed') ||
|
|
619
750
|
!fixtureUnchanged ||
|
|
751
|
+
skillReadUnavailable ||
|
|
752
|
+
rolloutUnavailable ||
|
|
620
753
|
cleanupFailed;
|
|
621
754
|
evidence = {
|
|
622
755
|
loaded: observed.loaded,
|
|
623
|
-
first_loaded_skill:
|
|
756
|
+
first_loaded_skill: firstLoadedSkill,
|
|
624
757
|
all_loaded_skills: allLoadedSkills,
|
|
625
758
|
load_evidence_by_skill: loadEvidenceBySkill,
|
|
626
759
|
spawned: observed.spawned,
|
|
@@ -702,9 +835,9 @@ async function runTriggerOnce(options) {
|
|
|
702
835
|
evidence.process_tree_snapshots = session.processTreeSnapshots;
|
|
703
836
|
if (session.stderr.trim())
|
|
704
837
|
evidence.stderr = session.stderr.trim();
|
|
705
|
-
return session.scrub(evidence);
|
|
838
|
+
return session.scrub(redactCodexEvalValue(evidence, options.codexRuntime));
|
|
706
839
|
}
|
|
707
|
-
return evidence;
|
|
840
|
+
return redactCodexEvalValue(evidence, options.codexRuntime);
|
|
708
841
|
}
|
|
709
842
|
export async function runCodexTriggerProviderCase(options) {
|
|
710
843
|
const fixture = await loadFixtureModule(options.fixtureRoot, options.suite);
|