@namzu/sdk 36.0.0 → 37.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +180 -0
- package/README.md +68 -41
- package/dist/bridge/a2a/mapper.d.ts.map +1 -1
- package/dist/bridge/a2a/mapper.js +2 -0
- package/dist/bridge/a2a/mapper.js.map +1 -1
- package/dist/bridge/sse/mapper.d.ts.map +1 -1
- package/dist/bridge/sse/mapper.js +2 -0
- package/dist/bridge/sse/mapper.js.map +1 -1
- package/dist/compaction/extractor.d.ts +1 -1
- package/dist/compaction/extractor.d.ts.map +1 -1
- package/dist/compaction/extractor.js +15 -7
- package/dist/compaction/extractor.js.map +1 -1
- package/dist/compaction/manager.d.ts +2 -0
- package/dist/compaction/manager.d.ts.map +1 -1
- package/dist/compaction/manager.js +12 -1
- package/dist/compaction/manager.js.map +1 -1
- package/dist/compaction/verifier.d.ts.map +1 -1
- package/dist/compaction/verifier.js +82 -15
- package/dist/compaction/verifier.js.map +1 -1
- package/dist/contracts/schemas.d.ts +10 -10
- package/dist/eval/harness-verification.d.ts +63 -0
- package/dist/eval/harness-verification.d.ts.map +1 -0
- package/dist/eval/harness-verification.js +165 -0
- package/dist/eval/harness-verification.js.map +1 -0
- package/dist/eval/index.d.ts +2 -0
- package/dist/eval/index.d.ts.map +1 -1
- package/dist/eval/index.js +1 -0
- package/dist/eval/index.js.map +1 -1
- package/dist/execution/code-runtime/json-value.d.ts +10 -0
- package/dist/execution/code-runtime/json-value.d.ts.map +1 -0
- package/dist/execution/code-runtime/json-value.js +89 -0
- package/dist/execution/code-runtime/json-value.js.map +1 -0
- package/dist/execution/code-runtime/worker-program.d.ts +8 -0
- package/dist/execution/code-runtime/worker-program.d.ts.map +1 -0
- package/dist/execution/code-runtime/worker-program.js +240 -0
- package/dist/execution/code-runtime/worker-program.js.map +1 -0
- package/dist/execution/code-runtime/worker.d.ts +22 -0
- package/dist/execution/code-runtime/worker.d.ts.map +1 -1
- package/dist/execution/code-runtime/worker.js +148 -111
- package/dist/execution/code-runtime/worker.js.map +1 -1
- package/dist/manager/agent/lifecycle.d.ts +12 -0
- package/dist/manager/agent/lifecycle.d.ts.map +1 -1
- package/dist/manager/agent/lifecycle.js +306 -37
- package/dist/manager/agent/lifecycle.js.map +1 -1
- package/dist/manager/run/persistence.d.ts +2 -0
- package/dist/manager/run/persistence.d.ts.map +1 -1
- package/dist/manager/run/persistence.js +6 -0
- package/dist/manager/run/persistence.js.map +1 -1
- package/dist/provider/capabilities.d.ts +3 -1
- package/dist/provider/capabilities.d.ts.map +1 -1
- package/dist/provider/capabilities.js +13 -0
- package/dist/provider/capabilities.js.map +1 -1
- package/dist/provider/fallback.d.ts.map +1 -1
- package/dist/provider/fallback.js +2 -0
- package/dist/provider/fallback.js.map +1 -1
- package/dist/public-runtime.d.ts +3 -1
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +2 -1
- package/dist/public-runtime.js.map +1 -1
- package/dist/public-tools.d.ts +1 -1
- package/dist/public-tools.d.ts.map +1 -1
- package/dist/public-tools.js +1 -1
- package/dist/public-tools.js.map +1 -1
- package/dist/public-types.d.ts +1 -0
- package/dist/public-types.d.ts.map +1 -1
- package/dist/registry/tool/execute.d.ts +3 -0
- package/dist/registry/tool/execute.d.ts.map +1 -1
- package/dist/registry/tool/execute.js +11 -80
- package/dist/registry/tool/execute.js.map +1 -1
- package/dist/run/command-gate.d.ts.map +1 -1
- package/dist/run/command-gate.js +65 -12
- package/dist/run/command-gate.js.map +1 -1
- package/dist/run/index.d.ts +2 -0
- package/dist/run/index.d.ts.map +1 -1
- package/dist/run/index.js +1 -0
- package/dist/run/index.js.map +1 -1
- package/dist/run/memory-promoter.d.ts +3 -35
- package/dist/run/memory-promoter.d.ts.map +1 -1
- package/dist/run/memory-promoter.js +32 -40
- package/dist/run/memory-promoter.js.map +1 -1
- package/dist/run/memory-recall.d.ts +25 -0
- package/dist/run/memory-recall.d.ts.map +1 -0
- package/dist/run/memory-recall.js +228 -0
- package/dist/run/memory-recall.js.map +1 -0
- package/dist/run/reporter.d.ts.map +1 -1
- package/dist/run/reporter.js +1 -0
- package/dist/run/reporter.js.map +1 -1
- package/dist/run/token-budget.d.ts +2 -0
- package/dist/run/token-budget.d.ts.map +1 -1
- package/dist/run/token-budget.js +4 -0
- package/dist/run/token-budget.js.map +1 -1
- package/dist/run/workspace-fingerprint.d.ts +10 -3
- package/dist/run/workspace-fingerprint.d.ts.map +1 -1
- package/dist/run/workspace-fingerprint.js +17 -8
- package/dist/run/workspace-fingerprint.js.map +1 -1
- package/dist/runtime/query/checkpoint.d.ts +14 -0
- package/dist/runtime/query/checkpoint.d.ts.map +1 -1
- package/dist/runtime/query/checkpoint.js +82 -1
- package/dist/runtime/query/checkpoint.js.map +1 -1
- package/dist/runtime/query/executor.d.ts +5 -0
- package/dist/runtime/query/executor.d.ts.map +1 -1
- package/dist/runtime/query/executor.js +70 -14
- package/dist/runtime/query/executor.js.map +1 -1
- package/dist/runtime/query/index.d.ts +5 -8
- package/dist/runtime/query/index.d.ts.map +1 -1
- package/dist/runtime/query/index.js +54 -7
- package/dist/runtime/query/index.js.map +1 -1
- package/dist/runtime/query/iteration/index.d.ts +10 -0
- package/dist/runtime/query/iteration/index.d.ts.map +1 -1
- package/dist/runtime/query/iteration/index.js +325 -36
- package/dist/runtime/query/iteration/index.js.map +1 -1
- package/dist/runtime/query/iteration/native-output.d.ts +10 -0
- package/dist/runtime/query/iteration/native-output.d.ts.map +1 -0
- package/dist/runtime/query/iteration/native-output.js +32 -0
- package/dist/runtime/query/iteration/native-output.js.map +1 -0
- package/dist/runtime/query/iteration/phases/advisory.d.ts.map +1 -1
- package/dist/runtime/query/iteration/phases/advisory.js +18 -13
- package/dist/runtime/query/iteration/phases/advisory.js.map +1 -1
- package/dist/runtime/query/iteration/phases/compaction.d.ts +2 -0
- package/dist/runtime/query/iteration/phases/compaction.d.ts.map +1 -1
- package/dist/runtime/query/iteration/phases/compaction.js +32 -11
- package/dist/runtime/query/iteration/phases/compaction.js.map +1 -1
- package/dist/runtime/query/iteration/phases/context.d.ts +9 -0
- package/dist/runtime/query/iteration/phases/context.d.ts.map +1 -1
- package/dist/runtime/query/iteration/phases/context.js.map +1 -1
- package/dist/runtime/query/iteration/phases/tool-review.d.ts.map +1 -1
- package/dist/runtime/query/iteration/phases/tool-review.js +1 -1
- package/dist/runtime/query/iteration/phases/tool-review.js.map +1 -1
- package/dist/runtime/query/iteration/phases/working-memory.d.ts +1 -1
- package/dist/runtime/query/iteration/phases/working-memory.d.ts.map +1 -1
- package/dist/runtime/query/iteration/phases/working-memory.js +23 -6
- package/dist/runtime/query/iteration/phases/working-memory.js.map +1 -1
- package/dist/runtime/query/iteration/stream-turn.d.ts.map +1 -1
- package/dist/runtime/query/iteration/stream-turn.js +2 -0
- package/dist/runtime/query/iteration/stream-turn.js.map +1 -1
- package/dist/runtime/query/steering.d.ts +6 -2
- package/dist/runtime/query/steering.d.ts.map +1 -1
- package/dist/runtime/query/steering.js +17 -2
- package/dist/runtime/query/steering.js.map +1 -1
- package/dist/runtime/query/tool-call-budget.d.ts +19 -0
- package/dist/runtime/query/tool-call-budget.d.ts.map +1 -0
- package/dist/runtime/query/tool-call-budget.js +110 -0
- package/dist/runtime/query/tool-call-budget.js.map +1 -0
- package/dist/runtime/query/tooling.d.ts +2 -0
- package/dist/runtime/query/tooling.d.ts.map +1 -1
- package/dist/runtime/query/tooling.js +4 -0
- package/dist/runtime/query/tooling.js.map +1 -1
- package/dist/scheduler/local.d.ts.map +1 -1
- package/dist/scheduler/local.js +1 -0
- package/dist/scheduler/local.js.map +1 -1
- package/dist/session/handoff/capacity.d.ts +2 -2
- package/dist/session/handoff/capacity.d.ts.map +1 -1
- package/dist/session/handoff/capacity.js +3 -2
- package/dist/session/handoff/capacity.js.map +1 -1
- package/dist/store/memory/disk.d.ts +6 -2
- package/dist/store/memory/disk.d.ts.map +1 -1
- package/dist/store/memory/disk.js +42 -25
- package/dist/store/memory/disk.js.map +1 -1
- package/dist/store/memory/index.d.ts +4 -0
- package/dist/store/memory/index.d.ts.map +1 -1
- package/dist/store/memory/index.js +55 -19
- package/dist/store/memory/index.js.map +1 -1
- package/dist/store/memory/memory.d.ts +3 -2
- package/dist/store/memory/memory.d.ts.map +1 -1
- package/dist/store/memory/memory.js +11 -2
- package/dist/store/memory/memory.js.map +1 -1
- package/dist/store/memory/operation-lock.d.ts +11 -0
- package/dist/store/memory/operation-lock.d.ts.map +1 -0
- package/dist/store/memory/operation-lock.js +94 -0
- package/dist/store/memory/operation-lock.js.map +1 -0
- package/dist/tools/builtins/computer-use.d.ts.map +1 -1
- package/dist/tools/builtins/computer-use.js +60 -10
- package/dist/tools/builtins/computer-use.js.map +1 -1
- package/dist/tools/builtins/run-code.d.ts +6 -0
- package/dist/tools/builtins/run-code.d.ts.map +1 -1
- package/dist/tools/builtins/run-code.js +15 -4
- package/dist/tools/builtins/run-code.js.map +1 -1
- package/dist/tools/builtins/search-tools.d.ts.map +1 -1
- package/dist/tools/builtins/search-tools.js +14 -2
- package/dist/tools/builtins/search-tools.js.map +1 -1
- package/dist/tools/defineTool.d.ts +2 -0
- package/dist/tools/defineTool.d.ts.map +1 -1
- package/dist/tools/defineTool.js +3 -0
- package/dist/tools/defineTool.js.map +1 -1
- package/dist/tools/memory/delete.d.ts +4 -0
- package/dist/tools/memory/delete.d.ts.map +1 -0
- package/dist/tools/memory/delete.js +26 -0
- package/dist/tools/memory/delete.js.map +1 -0
- package/dist/tools/memory/index.d.ts +2 -0
- package/dist/tools/memory/index.d.ts.map +1 -1
- package/dist/tools/memory/index.js +6 -0
- package/dist/tools/memory/index.js.map +1 -1
- package/dist/tools/memory/save.d.ts.map +1 -1
- package/dist/tools/memory/save.js +3 -2
- package/dist/tools/memory/save.js.map +1 -1
- package/dist/tools/memory/search.d.ts.map +1 -1
- package/dist/tools/memory/search.js +12 -5
- package/dist/tools/memory/search.js.map +1 -1
- package/dist/tools/memory/update.d.ts +4 -0
- package/dist/tools/memory/update.d.ts.map +1 -0
- package/dist/tools/memory/update.js +47 -0
- package/dist/tools/memory/update.js.map +1 -0
- package/dist/types/agent/scheduler.d.ts +5 -0
- package/dist/types/agent/scheduler.d.ts.map +1 -1
- package/dist/types/agent/task.d.ts +10 -0
- package/dist/types/agent/task.d.ts.map +1 -1
- package/dist/types/computer-use/index.d.ts +6 -0
- package/dist/types/computer-use/index.d.ts.map +1 -1
- package/dist/types/computer-use/index.js.map +1 -1
- package/dist/types/errors/catalog.js +1 -1
- package/dist/types/errors/catalog.js.map +1 -1
- package/dist/types/hitl/index.d.ts +11 -1
- package/dist/types/hitl/index.d.ts.map +1 -1
- package/dist/types/hitl/index.js.map +1 -1
- package/dist/types/memory/index.d.ts +14 -1
- package/dist/types/memory/index.d.ts.map +1 -1
- package/dist/types/project/entity.d.ts +1 -0
- package/dist/types/project/entity.d.ts.map +1 -1
- package/dist/types/provider/config.d.ts +2 -0
- package/dist/types/provider/config.d.ts.map +1 -1
- package/dist/types/provider/model.d.ts +4 -0
- package/dist/types/provider/model.d.ts.map +1 -1
- package/dist/types/run/answer-review.d.ts +2 -0
- package/dist/types/run/answer-review.d.ts.map +1 -1
- package/dist/types/run/events.d.ts +9 -0
- package/dist/types/run/events.d.ts.map +1 -1
- package/dist/types/run/events.js.map +1 -1
- package/dist/types/run/prepare-step.d.ts +19 -1
- package/dist/types/run/prepare-step.d.ts.map +1 -1
- package/dist/types/structured-output/index.d.ts +8 -2
- package/dist/types/structured-output/index.d.ts.map +1 -1
- package/dist/types/tool/index.d.ts +15 -0
- package/dist/types/tool/index.d.ts.map +1 -1
- package/dist/types/tool/index.js.map +1 -1
- package/dist/utils/json-snapshot.d.ts +3 -0
- package/dist/utils/json-snapshot.d.ts.map +1 -0
- package/dist/utils/json-snapshot.js +79 -0
- package/dist/utils/json-snapshot.js.map +1 -0
- package/package.json +4 -2
- package/src/bridge/a2a/mapper.ts +2 -0
- package/src/bridge/sse/mapper.ts +2 -0
- package/src/compaction/extractor.ts +21 -8
- package/src/compaction/manager.ts +11 -1
- package/src/compaction/verifier.ts +80 -18
- package/src/eval/harness-verification.ts +270 -0
- package/src/eval/index.ts +10 -0
- package/src/execution/code-runtime/json-value.ts +82 -0
- package/src/execution/code-runtime/worker-program.ts +239 -0
- package/src/execution/code-runtime/worker.ts +164 -111
- package/src/manager/agent/lifecycle.ts +340 -45
- package/src/manager/run/persistence.ts +7 -0
- package/src/provider/capabilities.ts +25 -1
- package/src/provider/fallback.ts +2 -0
- package/src/public-runtime.ts +4 -0
- package/src/public-tools.ts +5 -1
- package/src/public-types.ts +1 -0
- package/src/registry/tool/execute.ts +17 -79
- package/src/run/command-gate.ts +61 -13
- package/src/run/index.ts +2 -0
- package/src/run/memory-promoter.ts +32 -40
- package/src/run/memory-recall.ts +254 -0
- package/src/run/reporter.ts +1 -0
- package/src/run/token-budget.ts +4 -0
- package/src/run/workspace-fingerprint.ts +17 -8
- package/src/runtime/query/checkpoint.ts +96 -2
- package/src/runtime/query/executor.ts +77 -14
- package/src/runtime/query/index.ts +73 -16
- package/src/runtime/query/iteration/index.ts +377 -44
- package/src/runtime/query/iteration/native-output.ts +35 -0
- package/src/runtime/query/iteration/phases/advisory.ts +21 -11
- package/src/runtime/query/iteration/phases/compaction.ts +36 -14
- package/src/runtime/query/iteration/phases/context.ts +9 -0
- package/src/runtime/query/iteration/phases/tool-review.ts +5 -1
- package/src/runtime/query/iteration/phases/working-memory.ts +23 -5
- package/src/runtime/query/iteration/stream-turn.ts +2 -0
- package/src/runtime/query/steering.ts +19 -2
- package/src/runtime/query/tool-call-budget.ts +115 -0
- package/src/runtime/query/tooling.ts +6 -0
- package/src/scheduler/local.ts +1 -0
- package/src/session/handoff/capacity.ts +7 -3
- package/src/store/memory/disk.ts +52 -30
- package/src/store/memory/index.ts +75 -27
- package/src/store/memory/memory.ts +18 -6
- package/src/store/memory/operation-lock.ts +99 -0
- package/src/tools/builtins/computer-use.ts +61 -7
- package/src/tools/builtins/run-code.ts +24 -5
- package/src/tools/builtins/search-tools.ts +18 -3
- package/src/tools/defineTool.ts +5 -0
- package/src/tools/memory/delete.ts +29 -0
- package/src/tools/memory/index.ts +6 -0
- package/src/tools/memory/save.ts +4 -2
- package/src/tools/memory/search.ts +12 -5
- package/src/tools/memory/update.ts +50 -0
- package/src/types/agent/scheduler.ts +6 -0
- package/src/types/agent/task.ts +13 -0
- package/src/types/computer-use/index.ts +6 -0
- package/src/types/errors/catalog.ts +1 -1
- package/src/types/hitl/index.ts +11 -1
- package/src/types/memory/index.ts +16 -1
- package/src/types/project/entity.ts +1 -0
- package/src/types/provider/config.ts +2 -0
- package/src/types/provider/model.ts +4 -0
- package/src/types/run/answer-review.ts +2 -0
- package/src/types/run/events.ts +10 -0
- package/src/types/run/prepare-step.ts +19 -1
- package/src/types/structured-output/index.ts +15 -2
- package/src/types/tool/index.ts +16 -0
- package/src/utils/json-snapshot.ts +76 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manager.js","sourceRoot":"","sources":["../../src/compaction/manager.ts"],"names":[],"mappings":"AAUA,mFAAmF;AACnF,MAAM,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAA;AAC1B,qEAAqE;AACrE,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,CAAA;AAEhC,SAAS,gBAAgB;IACxB,OAAO;QACN,IAAI,EAAE,EAAE;QACR,IAAI,EAAE,EAAE;QACR,IAAI,EAAE,IAAI,GAAG,EAAmB;QAChC,KAAK,EAAE,IAAI,GAAG,EAAoB;QAClC,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,EAAE;QACZ,WAAW,EAAE,EAAE;QACf,WAAW,EAAE,EAAE;QACf,WAAW,EAAE,EAAE;QACf,gBAAgB,EAAE,EAAE;QACpB,cAAc,EAAE,EAAE;QAClB,OAAO,EAAE,EAAE;KACX,CAAA;AACF,CAAC;AAED,MAAM,OAAO,mBAAmB;IACvB,KAAK,CAAc;IACV,MAAM,CAAkB;IAEzC,YAAY,MAAwB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,KAAK,GAAG,gBAAgB,EAAE,CAAA;IAChC,CAAC;IAED,OAAO,CAAC,IAAY;QACnB,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAA;IAC7D,CAAC;IAED,OAAO,CAAC,IAAgB;QACvB,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAA;IACvB,CAAC;IAED,+EAA+E;IAC/E,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,MAAc;QAC5C,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;QACpB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAA;QAChD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;YAC1B,OAAM;QACP,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI;YAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAA;QACjD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;QACzB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;QAC7E,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;YACzF,IAAI,CAAC,MAAM;gBAAE,MAAK;YAClB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAClC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QAC7D,CAAC;IACF,CAAC;IAED,KAAK,CAAC,GAAW;QAChB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;IACpC,CAAC;IAED,SAAS,CAAC,IAAY,EAAE,MAAkB;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC3C,IAAI,QAAQ,EAAE,CAAC;YACd,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAC9B,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACxD,CAAC;IACF,CAAC;IAED,WAAW,CAAC,QAAgB;QAC3B,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IAC5F,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,UAAU,CAAC,OAAe;QACzB,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;IAC5F,CAAC;IAED,YAAY,CAAC,SAAiB;QAC7B,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IACjG,CAAC;IAED,cAAc,CAAC,GAAW;QACzB,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IAC3F,CAAC;IAED,aAAa,CAAC,MAAsB;QACnC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACnC,qEAAqE;QACrE,qEAAqE;QACrE,8DAA8D;QAC9D,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACnE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;YAC9B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,kBAAkB,CAAC,WAAmB;QACrC,MAAM,SAAS,GAAG,
|
|
1
|
+
{"version":3,"file":"manager.js","sourceRoot":"","sources":["../../src/compaction/manager.ts"],"names":[],"mappings":"AAUA,mFAAmF;AACnF,MAAM,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAA;AAC1B,qEAAqE;AACrE,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,CAAA;AAEhC,gFAAgF;AAChF,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,GAAW;IAC1D,IAAI,GAAG,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAA;IACxD,MAAM,MAAM,GAAG,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAA;IACpE,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAA;IAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAC7C,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM;QAAE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAC9D,OAAO,GAAG,IAAI,GAAG,MAAM,EAAE,CAAA;AAC1B,CAAC;AAED,SAAS,gBAAgB;IACxB,OAAO;QACN,IAAI,EAAE,EAAE;QACR,IAAI,EAAE,EAAE;QACR,IAAI,EAAE,IAAI,GAAG,EAAmB;QAChC,KAAK,EAAE,IAAI,GAAG,EAAoB;QAClC,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,EAAE;QACZ,WAAW,EAAE,EAAE;QACf,WAAW,EAAE,EAAE;QACf,WAAW,EAAE,EAAE;QACf,gBAAgB,EAAE,EAAE;QACpB,cAAc,EAAE,EAAE;QAClB,OAAO,EAAE,EAAE;KACX,CAAA;AACF,CAAC;AAED,MAAM,OAAO,mBAAmB;IACvB,KAAK,CAAc;IACV,MAAM,CAAkB;IAEzC,YAAY,MAAwB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,KAAK,GAAG,gBAAgB,EAAE,CAAA;IAChC,CAAC;IAED,OAAO,CAAC,IAAY;QACnB,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAA;IAC7D,CAAC;IAED,OAAO,CAAC,IAAgB;QACvB,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAA;IACvB,CAAC;IAED,+EAA+E;IAC/E,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,MAAc;QAC5C,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;QACpB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAA;QAChD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;YAC1B,OAAM;QACP,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI;YAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAA;QACjD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;QACzB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;QAC7E,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;YACzF,IAAI,CAAC,MAAM;gBAAE,MAAK;YAClB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAClC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QAC7D,CAAC;IACF,CAAC;IAED,KAAK,CAAC,GAAW;QAChB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;IACpC,CAAC;IAED,SAAS,CAAC,IAAY,EAAE,MAAkB;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC3C,IAAI,QAAQ,EAAE,CAAC;YACd,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAC9B,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACxD,CAAC;IACF,CAAC;IAED,WAAW,CAAC,QAAgB;QAC3B,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IAC5F,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,UAAU,CAAC,OAAe;QACzB,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;IAC5F,CAAC;IAED,YAAY,CAAC,SAAiB;QAC7B,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IACjG,CAAC;IAED,cAAc,CAAC,GAAW;QACzB,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IAC3F,CAAC;IAED,aAAa,CAAC,MAAsB;QACnC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACnC,qEAAqE;QACrE,qEAAqE;QACrE,8DAA8D;QAC9D,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACnE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;YAC9B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,kBAAkB,CAAC,WAAmB;QACrC,MAAM,SAAS,GAAG,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAA;QACpF,IAAI,CAAC,gBAAgB,CACpB,kBAAkB,EAClB,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAC3B,SAAS,EACT,IAAI,CAAC,MAAM,CAAC,WAAW,CACvB,CAAA;IACF,CAAC;IAED,gBAAgB,CAAC,IAAY;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAA;QAC5D,IAAI,CAAC,gBAAgB,CACpB,gBAAgB,EAChB,IAAI,CAAC,KAAK,CAAC,cAAc,EACzB,SAAS,EACT,IAAI,CAAC,MAAM,CAAC,WAAW,CACvB,CAAA;IACF,CAAC;IAED,SAAS;QACR,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;YAAE,KAAK,EAAE,CAAA;QAC5B,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAA;QAC/B,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,CAAA;QACnC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAA;QAC9B,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAA;QACpC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAA;QACnC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAA;QACtC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAA;QACtC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAA;QACtC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAA;QAC3C,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAA;QACzC,OAAO,KAAK,CAAA;IACb,CAAC;IAED,QAAQ;QACP,OAAO,IAAI,CAAC,KAAK,CAAA;IAClB,CAAC;IAED;;;;;;;OAOG;IACH,YAAY,CAAC,KAAmB;QAC/B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACnB,CAAC;IAED,KAAK;QACJ,IAAI,CAAC,KAAK,GAAG,gBAAgB,EAAE,CAAA;IAChC,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACK,gBAAgB,CACvB,IAAY,EACZ,IAAc,EACd,IAAY,EACZ,GAAW;IACX;;;;OAIG;IACH,iBAA0B;QAE1B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACzB,iBAAiB,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,EACjD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CACpB,CAAA;QACD,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;YACzB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verifier.d.ts","sourceRoot":"","sources":["../../src/compaction/verifier.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAG5D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,KAAK,EAAE,OAAO,
|
|
1
|
+
{"version":3,"file":"verifier.d.ts","sourceRoot":"","sources":["../../src/compaction/verifier.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAG5D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,KAAK,EAAE,OAAO,EAAsC,MAAM,2BAA2B,CAAA;AAC5F,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAA;AACjE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AAsGvD;;;;;;;;GAQG;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAA;AAEnD,8EAA8E;AAC9E,MAAM,WAAW,6BAA6B;IAC7C,kFAAkF;IAClF,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAA;IAC7B;;;;OAIG;IACH,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAA;CACrC;AAED,wBAAsB,oBAAoB,CACzC,OAAO,EAAE,mBAAmB,EAC5B,aAAa,EAAE,OAAO,EAAE,EACxB,QAAQ,EAAE,WAAW,EACrB,MAAM,EAAE,gBAAgB,EACxB,OAAO,CAAC,EAAE,SAAS;AACnB;;;;;;GAMG;AACH,KAAK,CAAC,EAAE,MAAM,EACd,OAAO,GAAE,6BAAkC,GACzC,OAAO,CAAC,MAAM,CAAC,CAcjB;AAED;;;;;GAKG;AACH,wBAAsB,uCAAuC,CAC5D,OAAO,EAAE,mBAAmB,EAC5B,aAAa,EAAE,OAAO,EAAE,EACxB,QAAQ,EAAE,WAAW,EACrB,MAAM,EAAE,gBAAgB,EACxB,OAAO,CAAC,EAAE,SAAS,EACnB,KAAK,CAAC,EAAE,MAAM,EACd,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CAWjB"}
|
|
@@ -13,25 +13,92 @@ Your job: determine if the structured state captures all important information f
|
|
|
13
13
|
If the structured state is complete, respond with exactly: COMPLETE
|
|
14
14
|
|
|
15
15
|
If something important is missing, respond with a brief bullet list of the missing items (no preamble, just the bullets). Each bullet should be a single concise sentence.`;
|
|
16
|
-
function
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
16
|
+
function* attachmentExcerpt(attachment) {
|
|
17
|
+
const kind = attachment.type === 'stored'
|
|
18
|
+
? attachment.kind
|
|
19
|
+
: attachment.type === 'document'
|
|
20
|
+
? 'document'
|
|
21
|
+
: 'image';
|
|
22
|
+
yield `[${kind}`;
|
|
23
|
+
if ('name' in attachment && attachment.name) {
|
|
24
|
+
yield ' "';
|
|
25
|
+
yield attachment.name;
|
|
26
|
+
yield '"';
|
|
27
|
+
}
|
|
28
|
+
yield ': ';
|
|
29
|
+
yield attachment.mediaType;
|
|
30
|
+
yield ' — not included in this text excerpt]';
|
|
31
|
+
}
|
|
32
|
+
/** Visible evidence only: never stringify a whole message or its attachment bytes. */
|
|
33
|
+
function* conversationExcerpt(messages) {
|
|
34
|
+
let first = true;
|
|
35
|
+
for (const message of messages) {
|
|
36
|
+
const hasDetails = message.role === 'tool' ||
|
|
37
|
+
(message.role === 'assistant' && message.toolCalls?.length) ||
|
|
38
|
+
(message.role === 'user' && message.attachments?.length);
|
|
39
|
+
if (!message.content && !hasDetails)
|
|
21
40
|
continue;
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
41
|
+
if (!first)
|
|
42
|
+
yield '\n\n';
|
|
43
|
+
first = false;
|
|
44
|
+
yield `[${message.role}]: `;
|
|
45
|
+
if (message.role === 'tool') {
|
|
46
|
+
yield '[tool result ';
|
|
47
|
+
yield message.toolCallId;
|
|
48
|
+
yield `; isError=${message.isError === true}]\n`;
|
|
49
|
+
}
|
|
50
|
+
if (typeof message.content === 'string')
|
|
51
|
+
yield message.content;
|
|
52
|
+
else if (message.content) {
|
|
53
|
+
for (const [index, block] of message.content.entries()) {
|
|
54
|
+
if (index > 0)
|
|
55
|
+
yield '\n';
|
|
56
|
+
if (block.type === 'text')
|
|
57
|
+
yield block.text;
|
|
58
|
+
else
|
|
59
|
+
yield* attachmentExcerpt(block);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (message.role === 'assistant') {
|
|
63
|
+
for (const call of message.toolCalls ?? []) {
|
|
64
|
+
yield '\n[tool call ';
|
|
65
|
+
yield call.id;
|
|
66
|
+
yield '] ';
|
|
67
|
+
yield call.function.name;
|
|
68
|
+
yield '(';
|
|
69
|
+
yield call.function.arguments;
|
|
70
|
+
yield ')';
|
|
28
71
|
}
|
|
29
|
-
break;
|
|
30
72
|
}
|
|
31
|
-
|
|
32
|
-
|
|
73
|
+
if (message.role === 'user') {
|
|
74
|
+
for (const attachment of message.attachments ?? []) {
|
|
75
|
+
yield '\n';
|
|
76
|
+
yield* attachmentExcerpt(attachment);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
function truncateMessages(messages, budget) {
|
|
82
|
+
if (budget <= 0)
|
|
83
|
+
return '';
|
|
84
|
+
const parts = [];
|
|
85
|
+
let remaining = budget;
|
|
86
|
+
for (const part of conversationExcerpt(messages)) {
|
|
87
|
+
if (part.length > remaining) {
|
|
88
|
+
parts.push(part.slice(0, remaining));
|
|
89
|
+
// Role labels, separators and the omission marker all consume the
|
|
90
|
+
// same budget. Consume fragments lazily so huge arguments and rich
|
|
91
|
+
// results cannot bypass the cap or require a full serialized copy.
|
|
92
|
+
let prefix = parts.join('').slice(0, budget - 1);
|
|
93
|
+
const last = prefix.charCodeAt(prefix.length - 1);
|
|
94
|
+
if (last >= 0xd800 && last <= 0xdbff)
|
|
95
|
+
prefix = prefix.slice(0, -1);
|
|
96
|
+
return `${prefix}…`;
|
|
97
|
+
}
|
|
98
|
+
parts.push(part);
|
|
99
|
+
remaining -= part.length;
|
|
33
100
|
}
|
|
34
|
-
return
|
|
101
|
+
return parts.join('');
|
|
35
102
|
}
|
|
36
103
|
export async function buildVerifiedSummary(manager, olderMessages, provider, config, onUsage,
|
|
37
104
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verifier.js","sourceRoot":"","sources":["../../src/compaction/verifier.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,wCAAwC,CAAA;AAC9E,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAA;AAK/F,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAEhD,MAAM,mBAAmB,GAAG;;;;;;;;;;;2KAW+I,CAAA;AAE3K,
|
|
1
|
+
{"version":3,"file":"verifier.js","sourceRoot":"","sources":["../../src/compaction/verifier.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,wCAAwC,CAAA;AAC9E,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAA;AAK/F,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAEhD,MAAM,mBAAmB,GAAG;;;;;;;;;;;2KAW+I,CAAA;AAE3K,QAAQ,CAAC,CAAC,iBAAiB,CAC1B,UAA0E;IAE1E,MAAM,IAAI,GACT,UAAU,CAAC,IAAI,KAAK,QAAQ;QAC3B,CAAC,CAAC,UAAU,CAAC,IAAI;QACjB,CAAC,CAAC,UAAU,CAAC,IAAI,KAAK,UAAU;YAC/B,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,OAAO,CAAA;IACZ,MAAM,IAAI,IAAI,EAAE,CAAA;IAChB,IAAI,MAAM,IAAI,UAAU,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,IAAI,CAAA;QACV,MAAM,UAAU,CAAC,IAAI,CAAA;QACrB,MAAM,GAAG,CAAA;IACV,CAAC;IACD,MAAM,IAAI,CAAA;IACV,MAAM,UAAU,CAAC,SAAS,CAAA;IAC1B,MAAM,uCAAuC,CAAA;AAC9C,CAAC;AAED,sFAAsF;AACtF,QAAQ,CAAC,CAAC,mBAAmB,CAAC,QAA4B;IACzD,IAAI,KAAK,GAAG,IAAI,CAAA;IAChB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,UAAU,GACf,OAAO,CAAC,IAAI,KAAK,MAAM;YACvB,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;YAC3D,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;QACzD,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,UAAU;YAAE,SAAQ;QAC7C,IAAI,CAAC,KAAK;YAAE,MAAM,MAAM,CAAA;QACxB,KAAK,GAAG,KAAK,CAAA;QACb,MAAM,IAAI,OAAO,CAAC,IAAI,KAAK,CAAA;QAC3B,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,MAAM,eAAe,CAAA;YACrB,MAAM,OAAO,CAAC,UAAU,CAAA;YACxB,MAAM,aAAa,OAAO,CAAC,OAAO,KAAK,IAAI,KAAK,CAAA;QACjD,CAAC;QACD,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;YAAE,MAAM,OAAO,CAAC,OAAO,CAAA;aACzD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YAC1B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBACxD,IAAI,KAAK,GAAG,CAAC;oBAAE,MAAM,IAAI,CAAA;gBACzB,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;oBAAE,MAAM,KAAK,CAAC,IAAI,CAAA;;oBACtC,KAAK,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAA;YACrC,CAAC;QACF,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAClC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;gBAC5C,MAAM,eAAe,CAAA;gBACrB,MAAM,IAAI,CAAC,EAAE,CAAA;gBACb,MAAM,IAAI,CAAA;gBACV,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAA;gBACxB,MAAM,GAAG,CAAA;gBACT,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAA;gBAC7B,MAAM,GAAG,CAAA;YACV,CAAC;QACF,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,KAAK,MAAM,UAAU,IAAI,OAAO,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC;gBACpD,MAAM,IAAI,CAAA;gBACV,KAAK,CAAC,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAA;YACrC,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED,SAAS,gBAAgB,CAAC,QAA4B,EAAE,MAAc;IACrE,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,EAAE,CAAA;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,IAAI,SAAS,GAAG,MAAM,CAAA;IACtB,KAAK,MAAM,IAAI,IAAI,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClD,IAAI,IAAI,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAA;YACpC,kEAAkE;YAClE,mEAAmE;YACnE,mEAAmE;YACnE,IAAI,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAA;YAChD,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;YACjD,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM;gBAAE,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAClE,OAAO,GAAG,MAAM,GAAG,CAAA;QACpB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChB,SAAS,IAAI,IAAI,CAAC,MAAM,CAAA;IACzB,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AACtB,CAAC;AAyBD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,OAA4B,EAC5B,aAAwB,EACxB,QAAqB,EACrB,MAAwB,EACxB,OAAmB;AACnB;;;;;;GAMG;AACH,KAAc,EACd,UAAyC,EAAE;IAE3C,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;IAChC,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE;QAC7D,aAAa,EAAE,0BAA0B,CAAC,OAAO,CAAC,mBAAmB,CAAC;KACtE,CAAC,CAAA;IACF,OAAO,MAAM,gCAAgC,CAC5C,OAAO,EACP,aAAa,EACb,qBAAqB,EACrB,MAAM,EACN,OAAO,EACP,KAAK,EACL,OAAO,CAAC,MAAM,CACd,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,uCAAuC,CAC5D,OAA4B,EAC5B,aAAwB,EACxB,QAAqB,EACrB,MAAwB,EACxB,OAAmB,EACnB,KAAc,EACd,MAAoB;IAEpB,MAAM,EAAE,cAAc,EAAE,CAAA;IACxB,OAAO,MAAM,gCAAgC,CAC5C,OAAO,EACP,aAAa,EACb,QAAQ,EACR,MAAM,EACN,OAAO,EACP,KAAK,EACL,MAAM,CACN,CAAA;AACF,CAAC;AAED,KAAK,UAAU,gCAAgC,CAC9C,OAA4B,EAC5B,aAAwB,EACxB,QAAqB,EACrB,MAAwB,EACxB,OAAmB,EACnB,KAAc,EACd,MAAoB;IAEpB,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAA;IAErD,IAAI,OAAO,CAAC,SAAS,EAAE,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;QACtD,OAAO,UAAU,CAAA;IAClB,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAA;IAClB,CAAC;IAED,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,aAAa,EAAE,MAAM,CAAC,eAAe,CAAC,CAAA;IAEnF,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,EAAE,CAAC;QACjC,OAAO,UAAU,CAAA;IAClB,CAAC;IAED,MAAM,oBAAoB,GAAc;QACvC;YACC,IAAI,EAAE,QAAiB;YACvB,OAAO,EAAE,mBAAmB;SAC5B;QACD;YACC,IAAI,EAAE,MAAe;YACrB,OAAO,EAAE,0BAA0B,UAAU,kCAAkC,mBAAmB,EAAE;SACpG;KACD,CAAA;IAED,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAC3C,QAAQ,CAAC,UAAU,CAAC;QACnB,KAAK,EAAE,KAAK,IAAI,EAAE;QAClB,QAAQ,EAAE,oBAAoB;QAC9B,SAAS,EAAE,MAAM,CAAC,wBAAwB;QAC1C,WAAW,EAAE,CAAC;QACd,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC7B,CAAC,CACF,CAAA;IAED,OAAO,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IAEzB,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IAE3D,4EAA4E;IAC5E,wDAAwD;IACxD,wEAAwE;IACxE,4EAA4E;IAC5E,2EAA2E;IAC3E,6EAA6E;IAC7E,4EAA4E;IAC5E,6EAA6E;IAC7E,sBAAsB;IACtB,IAAI,YAAY,KAAK,UAAU,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9D,OAAO,UAAU,CAAA;IAClB,CAAC;IAED,OAAO,GAAG,UAAU,wCAAwC,YAAY,EAAE,CAAA;AAC3E,CAAC"}
|
|
@@ -19,20 +19,20 @@ export declare const RunConfigSchema: z.ZodObject<{
|
|
|
19
19
|
temperature?: number | undefined;
|
|
20
20
|
tokenBudget?: number | undefined;
|
|
21
21
|
maxResponseTokens?: number | undefined;
|
|
22
|
-
systemPrompt?: string | undefined;
|
|
23
22
|
streamIdleTimeoutMs?: number | undefined;
|
|
24
23
|
maxRequestRichContentBytes?: number | undefined;
|
|
25
24
|
permissionMode?: "plan" | "auto" | undefined;
|
|
25
|
+
systemPrompt?: string | undefined;
|
|
26
26
|
}, {
|
|
27
27
|
model?: string | undefined;
|
|
28
28
|
timeoutMs?: number | undefined;
|
|
29
29
|
temperature?: number | undefined;
|
|
30
30
|
tokenBudget?: number | undefined;
|
|
31
31
|
maxResponseTokens?: number | undefined;
|
|
32
|
-
systemPrompt?: string | undefined;
|
|
33
32
|
streamIdleTimeoutMs?: number | undefined;
|
|
34
33
|
maxRequestRichContentBytes?: number | undefined;
|
|
35
34
|
permissionMode?: "plan" | "auto" | undefined;
|
|
35
|
+
systemPrompt?: string | undefined;
|
|
36
36
|
}>;
|
|
37
37
|
export declare const CreateMessageSchema: z.ZodObject<{
|
|
38
38
|
role: z.ZodLiteral<"user">;
|
|
@@ -65,20 +65,20 @@ export declare const CreateRunSchema: z.ZodObject<{
|
|
|
65
65
|
temperature?: number | undefined;
|
|
66
66
|
tokenBudget?: number | undefined;
|
|
67
67
|
maxResponseTokens?: number | undefined;
|
|
68
|
-
systemPrompt?: string | undefined;
|
|
69
68
|
streamIdleTimeoutMs?: number | undefined;
|
|
70
69
|
maxRequestRichContentBytes?: number | undefined;
|
|
71
70
|
permissionMode?: "plan" | "auto" | undefined;
|
|
71
|
+
systemPrompt?: string | undefined;
|
|
72
72
|
}, {
|
|
73
73
|
model?: string | undefined;
|
|
74
74
|
timeoutMs?: number | undefined;
|
|
75
75
|
temperature?: number | undefined;
|
|
76
76
|
tokenBudget?: number | undefined;
|
|
77
77
|
maxResponseTokens?: number | undefined;
|
|
78
|
-
systemPrompt?: string | undefined;
|
|
79
78
|
streamIdleTimeoutMs?: number | undefined;
|
|
80
79
|
maxRequestRichContentBytes?: number | undefined;
|
|
81
80
|
permissionMode?: "plan" | "auto" | undefined;
|
|
81
|
+
systemPrompt?: string | undefined;
|
|
82
82
|
}>;
|
|
83
83
|
env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
84
84
|
stream: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -89,10 +89,10 @@ export declare const CreateRunSchema: z.ZodObject<{
|
|
|
89
89
|
temperature?: number | undefined;
|
|
90
90
|
tokenBudget?: number | undefined;
|
|
91
91
|
maxResponseTokens?: number | undefined;
|
|
92
|
-
systemPrompt?: string | undefined;
|
|
93
92
|
streamIdleTimeoutMs?: number | undefined;
|
|
94
93
|
maxRequestRichContentBytes?: number | undefined;
|
|
95
94
|
permissionMode?: "plan" | "auto" | undefined;
|
|
95
|
+
systemPrompt?: string | undefined;
|
|
96
96
|
};
|
|
97
97
|
agent_id: string;
|
|
98
98
|
env?: Record<string, string> | undefined;
|
|
@@ -104,10 +104,10 @@ export declare const CreateRunSchema: z.ZodObject<{
|
|
|
104
104
|
temperature?: number | undefined;
|
|
105
105
|
tokenBudget?: number | undefined;
|
|
106
106
|
maxResponseTokens?: number | undefined;
|
|
107
|
-
systemPrompt?: string | undefined;
|
|
108
107
|
streamIdleTimeoutMs?: number | undefined;
|
|
109
108
|
maxRequestRichContentBytes?: number | undefined;
|
|
110
109
|
permissionMode?: "plan" | "auto" | undefined;
|
|
110
|
+
systemPrompt?: string | undefined;
|
|
111
111
|
};
|
|
112
112
|
agent_id: string;
|
|
113
113
|
env?: Record<string, string> | undefined;
|
|
@@ -132,20 +132,20 @@ export declare const CreateStatelessRunSchema: z.ZodObject<{
|
|
|
132
132
|
temperature?: number | undefined;
|
|
133
133
|
tokenBudget?: number | undefined;
|
|
134
134
|
maxResponseTokens?: number | undefined;
|
|
135
|
-
systemPrompt?: string | undefined;
|
|
136
135
|
streamIdleTimeoutMs?: number | undefined;
|
|
137
136
|
maxRequestRichContentBytes?: number | undefined;
|
|
138
137
|
permissionMode?: "plan" | "auto" | undefined;
|
|
138
|
+
systemPrompt?: string | undefined;
|
|
139
139
|
}, {
|
|
140
140
|
model?: string | undefined;
|
|
141
141
|
timeoutMs?: number | undefined;
|
|
142
142
|
temperature?: number | undefined;
|
|
143
143
|
tokenBudget?: number | undefined;
|
|
144
144
|
maxResponseTokens?: number | undefined;
|
|
145
|
-
systemPrompt?: string | undefined;
|
|
146
145
|
streamIdleTimeoutMs?: number | undefined;
|
|
147
146
|
maxRequestRichContentBytes?: number | undefined;
|
|
148
147
|
permissionMode?: "plan" | "auto" | undefined;
|
|
148
|
+
systemPrompt?: string | undefined;
|
|
149
149
|
}>;
|
|
150
150
|
env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
151
151
|
}, "strict", z.ZodTypeAny, {
|
|
@@ -156,10 +156,10 @@ export declare const CreateStatelessRunSchema: z.ZodObject<{
|
|
|
156
156
|
temperature?: number | undefined;
|
|
157
157
|
tokenBudget?: number | undefined;
|
|
158
158
|
maxResponseTokens?: number | undefined;
|
|
159
|
-
systemPrompt?: string | undefined;
|
|
160
159
|
streamIdleTimeoutMs?: number | undefined;
|
|
161
160
|
maxRequestRichContentBytes?: number | undefined;
|
|
162
161
|
permissionMode?: "plan" | "auto" | undefined;
|
|
162
|
+
systemPrompt?: string | undefined;
|
|
163
163
|
};
|
|
164
164
|
agent_id: string;
|
|
165
165
|
env?: Record<string, string> | undefined;
|
|
@@ -171,10 +171,10 @@ export declare const CreateStatelessRunSchema: z.ZodObject<{
|
|
|
171
171
|
temperature?: number | undefined;
|
|
172
172
|
tokenBudget?: number | undefined;
|
|
173
173
|
maxResponseTokens?: number | undefined;
|
|
174
|
-
systemPrompt?: string | undefined;
|
|
175
174
|
streamIdleTimeoutMs?: number | undefined;
|
|
176
175
|
maxRequestRichContentBytes?: number | undefined;
|
|
177
176
|
permissionMode?: "plan" | "auto" | undefined;
|
|
177
|
+
systemPrompt?: string | undefined;
|
|
178
178
|
};
|
|
179
179
|
agent_id: string;
|
|
180
180
|
env?: Record<string, string> | undefined;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import type { CaseResult } from './types.js';
|
|
2
|
+
/** A host-owned outcome, with enough identity to check pairing and cite its trace. */
|
|
3
|
+
export interface HarnessTrial {
|
|
4
|
+
taskId: string;
|
|
5
|
+
trial: number;
|
|
6
|
+
/** Environment/seed/model/limits fingerprint. Equal within a pair, fresh in confirmation. */
|
|
7
|
+
conditions: string;
|
|
8
|
+
trajectoryId: string;
|
|
9
|
+
result: CaseResult;
|
|
10
|
+
}
|
|
11
|
+
export interface HarnessAttribution {
|
|
12
|
+
taskId: string;
|
|
13
|
+
effect: 'improvement' | 'regression' | 'unresolved';
|
|
14
|
+
reason: string;
|
|
15
|
+
baselineTrajectories: readonly string[];
|
|
16
|
+
candidateTrajectories: readonly string[];
|
|
17
|
+
}
|
|
18
|
+
export interface HarnessVerificationBatch {
|
|
19
|
+
baselineRevision: string;
|
|
20
|
+
candidateRevision: string;
|
|
21
|
+
baseline: readonly HarnessTrial[];
|
|
22
|
+
candidate: readonly HarnessTrial[];
|
|
23
|
+
/** Supplied by an independent trace reviewer, never inferred from scores alone. */
|
|
24
|
+
attributions: readonly HarnessAttribution[];
|
|
25
|
+
}
|
|
26
|
+
export type HarnessBehaviorStatus = 'recovered' | 'stable-success' | 'regressed' | 'still-failing' | 'mixed' | 'inconclusive';
|
|
27
|
+
export interface HarnessTaskComparison {
|
|
28
|
+
taskId: string;
|
|
29
|
+
status: HarnessBehaviorStatus;
|
|
30
|
+
baselinePasses: number;
|
|
31
|
+
candidatePasses: number;
|
|
32
|
+
trials: number;
|
|
33
|
+
}
|
|
34
|
+
export interface HarnessComparison {
|
|
35
|
+
tasks: readonly HarnessTaskComparison[];
|
|
36
|
+
/** Paired trial pass-rate difference, not held-out pass@1 or causal evidence. */
|
|
37
|
+
passRateDelta: number | null;
|
|
38
|
+
positiveEvidence: readonly string[];
|
|
39
|
+
regressions: readonly string[];
|
|
40
|
+
unresolved: readonly string[];
|
|
41
|
+
/** Actual recorded consumption from BOTH sides, including unsuccessful runs. */
|
|
42
|
+
usage: {
|
|
43
|
+
rollouts: number;
|
|
44
|
+
tokens: number;
|
|
45
|
+
costUsd: number;
|
|
46
|
+
durationMs: number;
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
/** HarnessLens Table 6 comparison plus explicit, trace-linked attribution. No model calls. */
|
|
50
|
+
export declare function compareHarnessTrials(batch: HarnessVerificationBatch): HarnessComparison;
|
|
51
|
+
export interface HarnessReview {
|
|
52
|
+
decision: 'accept' | 'reject' | 'inconclusive';
|
|
53
|
+
reason: string;
|
|
54
|
+
verification: HarnessComparison;
|
|
55
|
+
confirmation?: HarnessComparison;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Evaluate recorded evidence; never applies a patch. Requires at least five tasks,
|
|
59
|
+
* exactly two trials per task, and a same-size confirmation with at most two reused tasks.
|
|
60
|
+
* Hosts own isolated execution, task selection, scoring and independent trace review.
|
|
61
|
+
*/
|
|
62
|
+
export declare function reviewHarnessCandidate(verification: HarnessVerificationBatch, confirmation?: HarnessVerificationBatch): HarnessReview;
|
|
63
|
+
//# sourceMappingURL=harness-verification.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harness-verification.d.ts","sourceRoot":"","sources":["../../src/eval/harness-verification.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAE5C,sFAAsF;AACtF,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,6FAA6F;IAC7F,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,UAAU,CAAA;CAClB;AAED,MAAM,WAAW,kBAAkB;IAClC,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,aAAa,GAAG,YAAY,GAAG,YAAY,CAAA;IACnD,MAAM,EAAE,MAAM,CAAA;IACd,oBAAoB,EAAE,SAAS,MAAM,EAAE,CAAA;IACvC,qBAAqB,EAAE,SAAS,MAAM,EAAE,CAAA;CACxC;AAED,MAAM,WAAW,wBAAwB;IACxC,gBAAgB,EAAE,MAAM,CAAA;IACxB,iBAAiB,EAAE,MAAM,CAAA;IACzB,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAA;IACjC,SAAS,EAAE,SAAS,YAAY,EAAE,CAAA;IAClC,mFAAmF;IACnF,YAAY,EAAE,SAAS,kBAAkB,EAAE,CAAA;CAC3C;AAED,MAAM,MAAM,qBAAqB,GAC9B,WAAW,GACX,gBAAgB,GAChB,WAAW,GACX,eAAe,GACf,OAAO,GACP,cAAc,CAAA;AAEjB,MAAM,WAAW,qBAAqB;IACrC,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,qBAAqB,CAAA;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,MAAM,CAAA;IACvB,MAAM,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,iBAAiB;IACjC,KAAK,EAAE,SAAS,qBAAqB,EAAE,CAAA;IACvC,iFAAiF;IACjF,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAA;IACnC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAA;IAC9B,UAAU,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7B,gFAAgF;IAChF,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;CAChF;AA2CD,8FAA8F;AAC9F,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,wBAAwB,GAAG,iBAAiB,CA8FvF;AAED,MAAM,WAAW,aAAa;IAC7B,QAAQ,EAAE,QAAQ,GAAG,QAAQ,GAAG,cAAc,CAAA;IAC9C,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,iBAAiB,CAAA;IAC/B,YAAY,CAAC,EAAE,iBAAiB,CAAA;CAChC;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CACrC,YAAY,EAAE,wBAAwB,EACtC,YAAY,CAAC,EAAE,wBAAwB,GACrC,aAAa,CA4Df"}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
function nonempty(value) {
|
|
2
|
+
return typeof value === 'string' && value.trim().length > 0;
|
|
3
|
+
}
|
|
4
|
+
function key(trial) {
|
|
5
|
+
return JSON.stringify([trial.taskId, trial.trial]);
|
|
6
|
+
}
|
|
7
|
+
function indexTrials(trials) {
|
|
8
|
+
const result = new Map();
|
|
9
|
+
const traces = new Set();
|
|
10
|
+
for (const trial of trials) {
|
|
11
|
+
if (!nonempty(trial.taskId) ||
|
|
12
|
+
!nonempty(trial.conditions) ||
|
|
13
|
+
!nonempty(trial.trajectoryId) ||
|
|
14
|
+
!Number.isSafeInteger(trial.trial) ||
|
|
15
|
+
trial.trial < 0)
|
|
16
|
+
throw new Error('A harness trial needs a task, nonnegative trial index, conditions and trajectory ID.');
|
|
17
|
+
if (result.has(key(trial)) || traces.has(trial.trajectoryId))
|
|
18
|
+
throw new Error('Duplicate harness trial or trajectory ID.');
|
|
19
|
+
result.set(key(trial), trial);
|
|
20
|
+
traces.add(trial.trajectoryId);
|
|
21
|
+
}
|
|
22
|
+
return result;
|
|
23
|
+
}
|
|
24
|
+
function measured(trial) {
|
|
25
|
+
const { result } = trial;
|
|
26
|
+
const scores = Object.values(result.scores);
|
|
27
|
+
return (!result.run.error &&
|
|
28
|
+
(result.status === 'passed' || result.status === 'failed') &&
|
|
29
|
+
scores.length > 0 &&
|
|
30
|
+
scores.every((s) => !s.unavailable && Number.isFinite(s.score) && s.score >= 0 && s.score <= 1));
|
|
31
|
+
}
|
|
32
|
+
/** HarnessLens Table 6 comparison plus explicit, trace-linked attribution. No model calls. */
|
|
33
|
+
export function compareHarnessTrials(batch) {
|
|
34
|
+
if (!nonempty(batch.baselineRevision) ||
|
|
35
|
+
!nonempty(batch.candidateRevision) ||
|
|
36
|
+
batch.baselineRevision === batch.candidateRevision)
|
|
37
|
+
throw new Error('Distinct baseline and candidate revisions are required.');
|
|
38
|
+
const baseline = indexTrials(batch.baseline);
|
|
39
|
+
const candidate = indexTrials(batch.candidate);
|
|
40
|
+
if (!baseline.size || baseline.size !== candidate.size)
|
|
41
|
+
throw new Error('Complete paired trials are required.');
|
|
42
|
+
const baselineTraces = new Set(batch.baseline.map((t) => t.trajectoryId));
|
|
43
|
+
if (batch.candidate.some((t) => baselineTraces.has(t.trajectoryId)))
|
|
44
|
+
throw new Error('Baseline and candidate must have distinct trajectories.');
|
|
45
|
+
const groups = new Map();
|
|
46
|
+
for (const [id, before] of baseline) {
|
|
47
|
+
const after = candidate.get(id);
|
|
48
|
+
if (!after || before.conditions !== after.conditions)
|
|
49
|
+
throw new Error('Harness trial conditions do not match.');
|
|
50
|
+
const group = groups.get(before.taskId) ?? [];
|
|
51
|
+
group.push(before);
|
|
52
|
+
groups.set(before.taskId, group);
|
|
53
|
+
}
|
|
54
|
+
const tasks = [];
|
|
55
|
+
for (const [taskId, before] of groups) {
|
|
56
|
+
if (new Set(before.map((t) => t.conditions)).size !== before.length)
|
|
57
|
+
throw new Error('Repeated trials need distinct controlled conditions.');
|
|
58
|
+
const after = before.map((t) => candidate.get(key(t)));
|
|
59
|
+
const b = before.filter((t) => t.result.status === 'passed').length;
|
|
60
|
+
const c = after.filter((t) => t.result.status === 'passed').length;
|
|
61
|
+
const status = ![...before, ...after].every(measured)
|
|
62
|
+
? 'inconclusive'
|
|
63
|
+
: b === 0 && c > 0
|
|
64
|
+
? 'recovered'
|
|
65
|
+
: b > 0 && c === after.length
|
|
66
|
+
? 'stable-success'
|
|
67
|
+
: b > 0 && c === 0
|
|
68
|
+
? 'regressed'
|
|
69
|
+
: b === 0 && c === 0
|
|
70
|
+
? 'still-failing'
|
|
71
|
+
: 'mixed';
|
|
72
|
+
tasks.push({ taskId, status, baselinePasses: b, candidatePasses: c, trials: before.length });
|
|
73
|
+
}
|
|
74
|
+
const positive = new Set();
|
|
75
|
+
const regressions = new Set();
|
|
76
|
+
const unresolved = new Set(tasks.filter((t) => t.status === 'inconclusive' || t.status === 'mixed').map((t) => t.taskId));
|
|
77
|
+
// Conservative Namzu policy: even an unattributed observed regression blocks promotion.
|
|
78
|
+
for (const task of tasks)
|
|
79
|
+
if (task.status === 'regressed')
|
|
80
|
+
regressions.add(task.taskId);
|
|
81
|
+
for (const evidence of batch.attributions) {
|
|
82
|
+
const task = tasks.find((t) => t.taskId === evidence.taskId);
|
|
83
|
+
const cites = (ids, trials) => ids.length > 0 &&
|
|
84
|
+
ids.every((id) => trials.some((t) => t.taskId === evidence.taskId && t.trajectoryId === id));
|
|
85
|
+
if (!task ||
|
|
86
|
+
!nonempty(evidence.reason) ||
|
|
87
|
+
!cites(evidence.baselineTrajectories, batch.baseline) ||
|
|
88
|
+
!cites(evidence.candidateTrajectories, batch.candidate) ||
|
|
89
|
+
!['improvement', 'regression', 'unresolved'].includes(evidence.effect))
|
|
90
|
+
throw new Error('Attribution must explain and cite both sides of an existing task.');
|
|
91
|
+
if (evidence.effect === 'regression')
|
|
92
|
+
regressions.add(task.taskId);
|
|
93
|
+
else if (evidence.effect === 'unresolved')
|
|
94
|
+
unresolved.add(task.taskId);
|
|
95
|
+
else if (task.status === 'recovered' ||
|
|
96
|
+
(task.status === 'stable-success' && task.candidatePasses > task.baselinePasses))
|
|
97
|
+
positive.add(task.taskId);
|
|
98
|
+
}
|
|
99
|
+
const usage = { rollouts: baseline.size * 2, tokens: 0, costUsd: 0, durationMs: 0 };
|
|
100
|
+
for (const trial of [...batch.baseline, ...batch.candidate]) {
|
|
101
|
+
const values = [
|
|
102
|
+
trial.result.run.totalTokens,
|
|
103
|
+
trial.result.run.totalCostUsd,
|
|
104
|
+
trial.result.run.durationMs,
|
|
105
|
+
];
|
|
106
|
+
if (values.some((n) => !Number.isFinite(n) || n < 0))
|
|
107
|
+
throw new Error('Invalid recorded harness usage.');
|
|
108
|
+
usage.tokens += trial.result.run.totalTokens;
|
|
109
|
+
usage.costUsd += trial.result.run.totalCostUsd;
|
|
110
|
+
usage.durationMs += trial.result.run.durationMs;
|
|
111
|
+
}
|
|
112
|
+
return {
|
|
113
|
+
tasks,
|
|
114
|
+
positiveEvidence: [...positive],
|
|
115
|
+
regressions: [...regressions],
|
|
116
|
+
unresolved: [...unresolved],
|
|
117
|
+
usage,
|
|
118
|
+
passRateDelta: tasks.some((t) => t.status === 'inconclusive')
|
|
119
|
+
? null
|
|
120
|
+
: tasks.reduce((sum, t) => sum + t.candidatePasses - t.baselinePasses, 0) / baseline.size,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Evaluate recorded evidence; never applies a patch. Requires at least five tasks,
|
|
125
|
+
* exactly two trials per task, and a same-size confirmation with at most two reused tasks.
|
|
126
|
+
* Hosts own isolated execution, task selection, scoring and independent trace review.
|
|
127
|
+
*/
|
|
128
|
+
export function reviewHarnessCandidate(verification, confirmation) {
|
|
129
|
+
const first = compareHarnessTrials(verification);
|
|
130
|
+
const second = confirmation ? compareHarnessTrials(confirmation) : undefined;
|
|
131
|
+
const result = (decision, reason) => ({
|
|
132
|
+
decision,
|
|
133
|
+
reason,
|
|
134
|
+
verification: first,
|
|
135
|
+
...(second ? { confirmation: second } : {}),
|
|
136
|
+
});
|
|
137
|
+
const sufficient = (report) => report.tasks.length >= 5 && report.tasks.every((t) => t.trials === 2);
|
|
138
|
+
if (!sufficient(first))
|
|
139
|
+
return result('inconclusive', 'Verification requires at least five distinct tasks and two paired trials per task.');
|
|
140
|
+
if (first.regressions.length || second?.regressions.length)
|
|
141
|
+
return result('reject', 'Observed or attributed regression blocks promotion.');
|
|
142
|
+
if (first.unresolved.length || second?.unresolved.length)
|
|
143
|
+
return result('inconclusive', 'Incomplete or ambiguous behavioral evidence.');
|
|
144
|
+
if (!first.positiveEvidence.length)
|
|
145
|
+
return result('reject', 'A metric increase or preservation alone is not attributable improvement.');
|
|
146
|
+
if (!confirmation || !second)
|
|
147
|
+
return result('inconclusive', 'A complete fresh confirmation round is required.');
|
|
148
|
+
if (verification.baselineRevision !== confirmation.baselineRevision ||
|
|
149
|
+
verification.candidateRevision !== confirmation.candidateRevision)
|
|
150
|
+
return result('inconclusive', 'Revisions changed between verification and confirmation.');
|
|
151
|
+
const firstTasks = new Set(first.tasks.map((t) => t.taskId));
|
|
152
|
+
if (!sufficient(second) ||
|
|
153
|
+
second.tasks.length !== first.tasks.length ||
|
|
154
|
+
second.tasks.filter((t) => firstTasks.has(t.taskId)).length > 2)
|
|
155
|
+
return result('inconclusive', 'Confirmation must use the same batch size with at most two reused tasks.');
|
|
156
|
+
const conditions = new Set(verification.baseline.map((t) => t.conditions));
|
|
157
|
+
const traces = new Set([...verification.baseline, ...verification.candidate].map((t) => t.trajectoryId));
|
|
158
|
+
if (confirmation.baseline.some((t) => conditions.has(t.conditions)) ||
|
|
159
|
+
[...confirmation.baseline, ...confirmation.candidate].some((t) => traces.has(t.trajectoryId)))
|
|
160
|
+
return result('inconclusive', 'Confirmation requires fresh conditions and trajectories.');
|
|
161
|
+
if (!second.positiveEvidence.length || second.passRateDelta === null || second.passRateDelta <= 0)
|
|
162
|
+
return result('reject', 'Confirmation needs attributable improvement and a positive primary-metric difference.');
|
|
163
|
+
return result('accept', 'Both rounds support improvement without observed regression; confirmation improves the paired pass rate.');
|
|
164
|
+
}
|
|
165
|
+
//# sourceMappingURL=harness-verification.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harness-verification.js","sourceRoot":"","sources":["../../src/eval/harness-verification.ts"],"names":[],"mappings":"AAwDA,SAAS,QAAQ,CAAC,KAAa;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAA;AAC5D,CAAC;AAED,SAAS,GAAG,CAAC,KAAmB;IAC/B,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;AACnD,CAAC;AAED,SAAS,WAAW,CAAC,MAA+B;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAA;IAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAA;IAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IACC,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;YACvB,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC;YAC3B,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC;YAC7B,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC;YAClC,KAAK,CAAC,KAAK,GAAG,CAAC;YAEf,MAAM,IAAI,KAAK,CACd,sFAAsF,CACtF,CAAA;QACF,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;QAC7D,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;QAC7B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;IAC/B,CAAC;IACD,OAAO,MAAM,CAAA;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,KAAmB;IACpC,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAA;IACxB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAC3C,OAAO,CACN,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK;QACjB,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC;QAC1D,MAAM,CAAC,MAAM,GAAG,CAAC;QACjB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAC/F,CAAA;AACF,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,oBAAoB,CAAC,KAA+B;IACnE,IACC,CAAC,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC;QACjC,CAAC,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC;QAClC,KAAK,CAAC,gBAAgB,KAAK,KAAK,CAAC,iBAAiB;QAElD,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAA;IAC3E,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAC5C,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAC9C,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI;QACrD,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;IACxD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;IACzE,IAAI,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAA;IAC3E,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAA;IAChD,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAC/B,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU;YACnD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;QAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;QAC7C,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAClB,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IACjC,CAAC;IACD,MAAM,KAAK,GAA4B,EAAE,CAAA;IACzC,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvC,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;YAClE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAA;QACxE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAiB,CAAC,CAAA;QACtE,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;QACnE,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;QAClE,MAAM,MAAM,GAA0B,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;YAC3E,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;gBACjB,CAAC,CAAC,WAAW;gBACb,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM;oBAC5B,CAAC,CAAC,gBAAgB;oBAClB,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;wBACjB,CAAC,CAAC,WAAW;wBACb,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;4BACnB,CAAC,CAAC,eAAe;4BACjB,CAAC,CAAC,OAAO,CAAA;QACd,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC7F,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAClC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IACrC,MAAM,UAAU,GAAG,IAAI,GAAG,CACzB,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,IAAI,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAC7F,CAAA;IACD,wFAAwF;IACxF,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW;YAAE,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACvF,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAA;QAC5D,MAAM,KAAK,GAAG,CAAC,GAAsB,EAAE,MAA+B,EAAE,EAAE,CACzE,GAAG,CAAC,MAAM,GAAG,CAAC;YACd,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC,YAAY,KAAK,EAAE,CAAC,CAAC,CAAA;QAC7F,IACC,CAAC,IAAI;YACL,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC1B,CAAC,KAAK,CAAC,QAAQ,CAAC,oBAAoB,EAAE,KAAK,CAAC,QAAQ,CAAC;YACrD,CAAC,KAAK,CAAC,QAAQ,CAAC,qBAAqB,EAAE,KAAK,CAAC,SAAS,CAAC;YACvD,CAAC,CAAC,aAAa,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;YAEtE,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAA;QACrF,IAAI,QAAQ,CAAC,MAAM,KAAK,YAAY;YAAE,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;aAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,YAAY;YAAE,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;aACjE,IACJ,IAAI,CAAC,MAAM,KAAK,WAAW;YAC3B,CAAC,IAAI,CAAC,MAAM,KAAK,gBAAgB,IAAI,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC;YAEhF,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC;IACD,MAAM,KAAK,GAAG,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAA;IACnF,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAG;YACd,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW;YAC5B,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY;YAC7B,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU;SAC3B,CAAA;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;QACnD,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAA;QAC5C,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAA;QAC9C,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAA;IAChD,CAAC;IACD,OAAO;QACN,KAAK;QACL,gBAAgB,EAAE,CAAC,GAAG,QAAQ,CAAC;QAC/B,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;QAC7B,UAAU,EAAE,CAAC,GAAG,UAAU,CAAC;QAC3B,KAAK;QACL,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC;YAC5D,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI;KAC1F,CAAA;AACF,CAAC;AASD;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CACrC,YAAsC,EACtC,YAAuC;IAEvC,MAAM,KAAK,GAAG,oBAAoB,CAAC,YAAY,CAAC,CAAA;IAChD,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAC5E,MAAM,MAAM,GAAG,CAAC,QAAmC,EAAE,MAAc,EAAiB,EAAE,CAAC,CAAC;QACvF,QAAQ;QACR,MAAM;QACN,YAAY,EAAE,KAAK;QACnB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3C,CAAC,CAAA;IACF,MAAM,UAAU,GAAG,CAAC,MAAyB,EAAE,EAAE,CAChD,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAA;IACtE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QACrB,OAAO,MAAM,CACZ,cAAc,EACd,oFAAoF,CACpF,CAAA;IACF,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,MAAM,EAAE,WAAW,CAAC,MAAM;QACzD,OAAO,MAAM,CAAC,QAAQ,EAAE,qDAAqD,CAAC,CAAA;IAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,MAAM,EAAE,UAAU,CAAC,MAAM;QACvD,OAAO,MAAM,CAAC,cAAc,EAAE,8CAA8C,CAAC,CAAA;IAC9E,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,MAAM;QACjC,OAAO,MAAM,CACZ,QAAQ,EACR,0EAA0E,CAC1E,CAAA;IACF,IAAI,CAAC,YAAY,IAAI,CAAC,MAAM;QAC3B,OAAO,MAAM,CAAC,cAAc,EAAE,kDAAkD,CAAC,CAAA;IAClF,IACC,YAAY,CAAC,gBAAgB,KAAK,YAAY,CAAC,gBAAgB;QAC/D,YAAY,CAAC,iBAAiB,KAAK,YAAY,CAAC,iBAAiB;QAEjE,OAAO,MAAM,CAAC,cAAc,EAAE,0DAA0D,CAAC,CAAA;IAC1F,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;IAC5D,IACC,CAAC,UAAU,CAAC,MAAM,CAAC;QACnB,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,KAAK,CAAC,MAAM;QAC1C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC;QAE/D,OAAO,MAAM,CACZ,cAAc,EACd,0EAA0E,CAC1E,CAAA;IACF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,GAAG,CACrB,CAAC,GAAG,YAAY,CAAC,QAAQ,EAAE,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAChF,CAAA;IACD,IACC,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAC/D,CAAC,GAAG,YAAY,CAAC,QAAQ,EAAE,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QAE7F,OAAO,MAAM,CAAC,cAAc,EAAE,0DAA0D,CAAC,CAAA;IAC1F,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,MAAM,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI,IAAI,MAAM,CAAC,aAAa,IAAI,CAAC;QAChG,OAAO,MAAM,CACZ,QAAQ,EACR,uFAAuF,CACvF,CAAA;IACF,OAAO,MAAM,CACZ,QAAQ,EACR,0GAA0G,CAC1G,CAAA;AACF,CAAC"}
|
package/dist/eval/index.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export { evalRunFromQuery, evalRunFromRun } from './from-run.js';
|
|
2
|
+
export { compareHarnessTrials, reviewHarnessCandidate } from './harness-verification.js';
|
|
3
|
+
export type { HarnessTrial, HarnessAttribution, HarnessVerificationBatch, HarnessBehaviorStatus, HarnessTaskComparison, HarnessComparison, HarnessReview, } from './harness-verification.js';
|
|
2
4
|
export { formatReport, runExperiment } from './experiment.js';
|
|
3
5
|
export type { ExperimentConfig } from './experiment.js';
|
|
4
6
|
export { judgeScorer } from './judge.js';
|
package/dist/eval/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC7D,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AACxC,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EACN,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AACrE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACxD,YAAY,EACX,UAAU,EACV,UAAU,EACV,QAAQ,EACR,OAAO,EACP,gBAAgB,EAChB,KAAK,EACL,MAAM,GACN,MAAM,YAAY,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AACxF,YAAY,EACX,YAAY,EACZ,kBAAkB,EAClB,wBAAwB,EACxB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,aAAa,GACb,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC7D,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AACxC,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EACN,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AACrE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACxD,YAAY,EACX,UAAU,EACV,UAAU,EACV,QAAQ,EACR,OAAO,EACP,gBAAgB,EAChB,KAAK,EACL,MAAM,GACN,MAAM,YAAY,CAAA"}
|
package/dist/eval/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export { evalRunFromQuery, evalRunFromRun } from './from-run.js';
|
|
2
|
+
export { compareHarnessTrials, reviewHarnessCandidate } from './harness-verification.js';
|
|
2
3
|
export { formatReport, runExperiment } from './experiment.js';
|
|
3
4
|
export { judgeScorer } from './judge.js';
|
|
4
5
|
export { completionScorer, containsScorer, customScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
|
package/dist/eval/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,OAAO,EACN,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AAUxF,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,OAAO,EACN,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Serialize application data before sending it to the interpreter.
|
|
3
|
+
*
|
|
4
|
+
* JSON.stringify alone can allocate an arbitrarily large intermediate string.
|
|
5
|
+
* This writer accounts for each piece, refuses cycles and non-JSON values, and
|
|
6
|
+
* limits nesting. It never installs application objects in the guest realm.
|
|
7
|
+
* Undefined is supported only as a top-level result, represented out of band.
|
|
8
|
+
*/
|
|
9
|
+
export declare function encodeCodeValue(value: unknown, maxBytes: number): string | undefined;
|
|
10
|
+
//# sourceMappingURL=json-value.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-value.d.ts","sourceRoot":"","sources":["../../../src/execution/code-runtime/json-value.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAyEpF"}
|