@agentick/eval 0.15.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (20) hide show
  1. package/README.md +56 -0
  2. package/dist/.tsbuildinfo.build +1 -0
  3. package/dist/define-eval.d.ts +23 -0
  4. package/dist/define-eval.d.ts.map +1 -0
  5. package/dist/define-eval.js +83 -0
  6. package/dist/define-eval.js.map +1 -0
  7. package/dist/index.d.ts +30 -0
  8. package/dist/index.d.ts.map +1 -0
  9. package/dist/index.js +29 -0
  10. package/dist/index.js.map +1 -0
  11. package/dist/runner.d.ts +19 -0
  12. package/dist/runner.d.ts.map +1 -0
  13. package/dist/runner.js +207 -0
  14. package/dist/runner.js.map +1 -0
  15. package/dist/types.d.ts +199 -0
  16. package/dist/types.d.ts.map +1 -0
  17. package/dist/types.js +23 -0
  18. package/dist/types.js.map +1 -0
  19. package/package.json +61 -0
  20. package/src/index.ts +49 -0
package/README.md ADDED
@@ -0,0 +1,56 @@
1
+ # @agentick/eval
2
+
3
+ Testing-shaped eval framework for Agentick (v1). `defineEval(...)` returns a
4
+ callable with runtime parameterization; `.matrix(axes)` runs cartesian
5
+ parameter sweeps — the same inputs and expectations across multiple models —
6
+ and reports per-cell results.
7
+
8
+ Port of `@agentick/eval-next` (packages-next/eval) onto v1 core APIs, with
9
+ two extensions: `t.send` accepts content blocks (documents/images), and
10
+ `t.expect(...)` records custom assertions for expected-output scoring.
11
+
12
+ ```ts
13
+ import { defineEval } from "@agentick/eval";
14
+
15
+ const billEval = defineEval<{ model?: string }>({
16
+ description: "bill extraction against known fixtures",
17
+ app: async (o) => createMyExtractorApp({ model: o?.model ?? "google/gemini-2.5-flash" }),
18
+ test: async (t) => {
19
+ await t.send([fileBlock, { type: "text", text: "Extract the bill." }]);
20
+ t.completed();
21
+ t.calledTool("submit_extraction");
22
+ t.noFailedActions();
23
+
24
+ const submitted = t.lastToolCall("submit_extraction")?.input as Record<string, unknown>;
25
+ t.expect("subtotal matches", Number(submitted?.SubTotal) === 187.5, {
26
+ details: { got: submitted?.SubTotal, want: 187.5 },
27
+ });
28
+ },
29
+ });
30
+
31
+ // One run with factory defaults:
32
+ const result = await billEval();
33
+
34
+ // The same document + expectations across models:
35
+ const sweep = await billEval.matrix(
36
+ { model: ["google/gemini-2.5-flash", "bedrock/us.amazon.nova-2-lite-v1:0"] },
37
+ { concurrency: 1 },
38
+ );
39
+ for (const cell of sweep.cells) {
40
+ console.log(cell.axes.model, cell.result.passed, `${cell.result.elapsedMs}ms`);
41
+ }
42
+ ```
43
+
44
+ Key properties:
45
+
46
+ - **Fresh app per invocation** — the `app` thunk runs once per eval call /
47
+ matrix cell, so state never leaks between runs.
48
+ - **Assertions record, never throw** — every failure shows up in one report;
49
+ check `result.passed` for fail-fast behavior.
50
+ - **Tool observation built in** — every `tool_call`/`tool_result` pair from
51
+ the send handle's event stream lands in `result.toolCalls`, queryable via
52
+ `t.lastToolCall(name)`.
53
+ - **Sequential by default** — `matrix` runs cells one at a time unless you
54
+ raise `concurrency`, so real-model sweeps don't blow rate limits.
55
+
56
+ See the website docs (`/docs/evals`) for the full guide.
package/dist/.tsbuildinfo.build ADDED
@@ -0,0 +1 @@
1
+ {"fileNames":["../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es5.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2015.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2016.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2017.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2018.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2019.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2020.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2021.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2022.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2023.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2015.core.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2015.collection.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2015.generator.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2015.iterable.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2015.promise.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2015.proxy.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2015.reflect.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2015.symbol.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2015.symbol.wellknown.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2016.array.include.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2016.intl.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2017.arraybuffer.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2017.date.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2017.object.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2017.sharedmemory.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2017.string.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2017.intl.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2017.typedarrays.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2018.asyncgenerator.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2018.asynciterable.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2018.intl.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2018.promise.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2018.regexp.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2019.array.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2019.object.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2019.string.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2019.symbol.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2019.intl.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2020.bigint.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2020.date.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2020.promise.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2020.sharedmemory.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2020.string.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2020.symbol.wellknown.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2020.intl.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2020.number.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2021.promise.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2021.string.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2021.weakref.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2021.intl.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2022.array.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2022.error.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2022.intl.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2022.object.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2022.string.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2022.regexp.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2023.array.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2023.collection.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.es2023.intl.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.esnext.disposable.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.esnext.float16.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.decorators.d.ts","../../../node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/lib/lib.decorators.legacy.d.ts","../../../node_modules/.pnpm/@types+react@19.2.13/node_modules/@types/react/global.d.ts","../../../node_modules/.pnpm/csstype@3.2.3/node_modules/csstype/index.d.ts","../../../node_modules/.pnpm/@types+react@19.2.13/node_modules/@types/react/index.d.ts","../../../node_modules/.pnpm/@types+react@19.2.13/node_modules/@types/react/jsx-runtime.d.ts","../src/types.ts","../src/runner.ts","../src/define-eval.ts","../src/index.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/compatibility/iterators.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/globals.typedarray.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/buffer.buffer.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/globals.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/abortcontroller.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/blob.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/console.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/crypto.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/domexception.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/encoding.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/events.d.ts","../../../node_modules/.pnpm/buffer@6.0.3/node_modules/buffer/index.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/utility.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/header.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/readable.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/fetch.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/formdata.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/connector.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/client-stats.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/client.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/errors.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/dispatcher.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/global-dispatcher.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/global-origin.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/pool-stats.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/pool.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/handlers.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/balanced-pool.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/h2c-client.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/agent.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/mock-interceptor.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/mock-call-history.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/mock-agent.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/mock-client.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/mock-pool.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/snapshot-agent.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/mock-errors.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/proxy-agent.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/env-http-proxy-agent.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/retry-handler.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/retry-agent.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/api.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/cache-interceptor.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/interceptors.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/util.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/cookies.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/patch.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/websocket.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/eventsource.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/diagnostics-channel.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/content-type.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/cache.d.ts","../../../node_modules/.pnpm/undici-types@7.16.0/node_modules/undici-types/index.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/fetch.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/importmeta.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/messaging.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/navigator.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/performance.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/storage.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/streams.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/timers.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/web-globals/url.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/assert.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/assert/strict.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/async_hooks.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/buffer.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/child_process.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/cluster.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/console.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/constants.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/crypto.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/dgram.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/diagnostics_channel.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/dns.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/dns/promises.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/domain.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/events.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/fs.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/fs/promises.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/http.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/http2.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/https.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/inspector.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/inspector.generated.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/inspector/promises.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/module.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/net.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/os.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/path.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/path/posix.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/path/win32.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/perf_hooks.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/process.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/punycode.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/querystring.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/quic.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/readline.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/readline/promises.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/repl.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/sea.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/sqlite.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/stream.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/stream/consumers.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/stream/promises.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/stream/web.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/string_decoder.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/test.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/test/reporters.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/timers.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/timers/promises.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/tls.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/trace_events.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/tty.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/url.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/util.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/util/types.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/v8.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/vm.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/wasi.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/worker_threads.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/zlib.d.ts","../../../node_modules/.pnpm/@types+node@25.0.9/node_modules/@types/node/index.d.ts","../../../node_modules/.pnpm/@vitest+pretty-format@4.0.18/node_modules/@vitest/pretty-format/dist/index.d.ts","../../../node_modules/.pnpm/@vitest+utils@4.0.18/node_modules/@vitest/utils/dist/display.d.ts","../../../node_modules/.pnpm/@vitest+utils@4.0.18/node_modules/@vitest/utils/dist/types.d.ts","../../../node_modules/.pnpm/@vitest+utils@4.0.18/node_modules/@vitest/utils/dist/helpers.d.ts","../../../node_modules/.pnpm/@vitest+utils@4.0.18/node_modules/@vitest/utils/dist/timers.d.ts","../../../node_modules/.pnpm/@vitest+utils@4.0.18/node_modules/@vitest/utils/dist/index.d.ts","../../../node_modules/.pnpm/@vitest+runner@4.0.18/node_modules/@vitest/runner/dist/tasks.d-c7uxawj9.d.ts","../../../node_modules/.pnpm/@vitest+utils@4.0.18/node_modules/@vitest/utils/dist/types.d-bcelap-c.d.ts","../../../node_modules/.pnpm/@vitest+utils@4.0.18/node_modules/@vitest/utils/dist/diff.d.ts","../../../node_modules/.pnpm/@vitest+runner@4.0.18/node_modules/@vitest/runner/dist/types.d.ts","../../../node_modules/.pnpm/@vitest+runner@4.0.18/node_modules/@vitest/runner/dist/index.d.ts","../../../node_modules/.pnpm/vitest@4.0.18_@opentelemetry+api@1.9.0_@types+node@25.0.9_happy-dom@17.6.3_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vitest/dist/chunks/traces.d.402v_yfi.d.ts","../../../node_modules/.pnpm/vite@6.4.1_@types+node@25.0.9_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vite/types/hmrpayload.d.ts","../../../node_modules/.pnpm/vite@6.4.1_@types+node@25.0.9_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vite/types/customevent.d.ts","../../../node_modules/.pnpm/vite@6.4.1_@types+node@25.0.9_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vite/types/hot.d.ts","../../../node_modules/.pnpm/vite@6.4.1_@types+node@25.0.9_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vite/dist/node/modulerunnertransport.d-dj_me5sf.d.ts","../../../node_modules/.pnpm/vite@6.4.1_@types+node@25.0.9_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vite/dist/node/module-runner.d.ts","../../../node_modules/.pnpm/@vitest+snapshot@4.0.18/node_modules/@vitest/snapshot/dist/environment.d-dhdq1csl.d.ts","../../../node_modules/.pnpm/@vitest+snapshot@4.0.18/node_modules/@vitest/snapshot/dist/rawsnapshot.d-lfsmjfud.d.ts","../../../node_modules/.pnpm/@vitest+snapshot@4.0.18/node_modules/@vitest/snapshot/dist/index.d.ts","../../../node_modules/.pnpm/vitest@4.0.18_@opentelemetry+api@1.9.0_@types+node@25.0.9_happy-dom@17.6.3_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vitest/dist/chunks/config.d.cy95hicx.d.ts","../../../node_modules/.pnpm/vitest@4.0.18_@opentelemetry+api@1.9.0_@types+node@25.0.9_happy-dom@17.6.3_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vitest/dist/chunks/environment.d.crsxczp1.d.ts","../../../node_modules/.pnpm/vitest@4.0.18_@opentelemetry+api@1.9.0_@types+node@25.0.9_happy-dom@17.6.3_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vitest/dist/chunks/rpc.d.rh3apgef.d.ts","../../../node_modules/.pnpm/vitest@4.0.18_@opentelemetry+api@1.9.0_@types+node@25.0.9_happy-dom@17.6.3_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vitest/dist/chunks/worker.d.dyxm8del.d.ts","../../../node_modules/.pnpm/vitest@4.0.18_@opentelemetry+api@1.9.0_@types+node@25.0.9_happy-dom@17.6.3_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vitest/dist/chunks/browser.d.chkacdzh.d.ts","../../../node_modules/.pnpm/@vitest+spy@4.0.18/node_modules/@vitest/spy/dist/index.d.ts","../../../node_modules/.pnpm/tinyrainbow@3.0.3/node_modules/tinyrainbow/dist/index.d.ts","../../../node_modules/.pnpm/@standard-schema+spec@1.1.0/node_modules/@standard-schema/spec/dist/index.d.ts","../../../node_modules/.pnpm/@types+deep-eql@4.0.2/node_modules/@types/deep-eql/index.d.ts","../../../node_modules/.pnpm/assertion-error@2.0.1/node_modules/assertion-error/index.d.ts","../../../node_modules/.pnpm/@types+chai@5.2.3/node_modules/@types/chai/index.d.ts","../../../node_modules/.pnpm/@vitest+expect@4.0.18/node_modules/@vitest/expect/dist/index.d.ts","../../../node_modules/.pnpm/@vitest+runner@4.0.18/node_modules/@vitest/runner/dist/utils.d.ts","../../../node_modules/.pnpm/tinybench@2.9.0/node_modules/tinybench/dist/index.d.ts","../../../node_modules/.pnpm/vitest@4.0.18_@opentelemetry+api@1.9.0_@types+node@25.0.9_happy-dom@17.6.3_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vitest/dist/chunks/benchmark.d.daahlpsq.d.ts","../../../node_modules/.pnpm/vitest@4.0.18_@opentelemetry+api@1.9.0_@types+node@25.0.9_happy-dom@17.6.3_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vitest/dist/chunks/global.d.b15mdlcr.d.ts","../../../node_modules/.pnpm/vitest@4.0.18_@opentelemetry+api@1.9.0_@types+node@25.0.9_happy-dom@17.6.3_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vitest/dist/chunks/suite.d.bjwk38hb.d.ts","../../../node_modules/.pnpm/vitest@4.0.18_@opentelemetry+api@1.9.0_@types+node@25.0.9_happy-dom@17.6.3_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vitest/dist/chunks/evaluatedmodules.d.bxj5omdx.d.ts","../../../node_modules/.pnpm/expect-type@1.3.0/node_modules/expect-type/dist/utils.d.ts","../../../node_modules/.pnpm/expect-type@1.3.0/node_modules/expect-type/dist/overloads.d.ts","../../../node_modules/.pnpm/expect-type@1.3.0/node_modules/expect-type/dist/branding.d.ts","../../../node_modules/.pnpm/expect-type@1.3.0/node_modules/expect-type/dist/messages.d.ts","../../../node_modules/.pnpm/expect-type@1.3.0/node_modules/expect-type/dist/index.d.ts","../../../node_modules/.pnpm/vitest@4.0.18_@opentelemetry+api@1.9.0_@types+node@25.0.9_happy-dom@17.6.3_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vitest/dist/index.d.ts","../../../node_modules/.pnpm/vitest@4.0.18_@opentelemetry+api@1.9.0_@types+node@25.0.9_happy-dom@17.6.3_jiti@2.6.1_tsx@4.21.0_yaml@2.8.4/node_modules/vitest/globals.d.ts"],"fileIdsList":[[74,137,145,149,152,154,155,156,168],[74,137,145,149,152,154,155,156,168,222,223],[74,134,135,137,145,149,152,154,155,156,168],[74,136,137,145,149,152,154,155,156,168],[137,145,149,152,154,155,156,168],[74,137,145,149,152,154,155,156,168,176],[74,137,138,143,145,148,149,152,154,155,156,158,168,173,185],[74,137,138,139,145,148,149,152,154,155,156,168],[74,137,140,145,149,152,154,155,156,168,186],[74,137,141,142,145,149,152,154,155,156,159,168],[74,137,142,145,149,152,154,155,156,168,173,182],[74,137,143,145,148,149,152,154,155,156,158,168],[74,136,137,144,145,149,152,154,155,156,168],[74,137,145,146,149,152,154,155,156,168],[74,137,145,147,148,149,152,154,155,156,168],[74,136,137,145,148,149,152,154,155,156,168],[74,137,145,148,149,150,152,154,155,156,168,173,185],[74,137,145,148,149,150,152,154,155,156,168,173,176],[74,124,137,145,148,149,151,152,154,155,156,158,168,173,185],[74,137,145,148,149,151,152,154,155,156,158,168,173,182,185],[74,137,145,149,151,152,153,154,155,156,168,173,182,185],[72,73,74,75,76,77,78,79,80,81,82,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192],[74,137,145,148,149,152,154,155,156,168],[74,137,145,149,152,154,156,168],[74,137,145,149,152,154,155,156,157,168,185],[74,137,145,148,149,152,154,155,156,158,168,173],[74,137,145,149,152,154,155,156,159,168],[74,137,145,149,152,154,155,156,160,168],[74,137,145,148,149,152,154,155,156,163,168],[74,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192],[74,137,145,149,152,154,155,156,165,168],[74,137,145,149,152,154,155,156,166,168],[74,137,142,145,149,152,154,155,156,158,168,176],[74,137,145,148,149,152,154,155,156,168,169],[74,137,145,149,152,154,155,156,168,170,186,189],[74,137,145,148,149,152,154,155,156,168,173,175,176],[74,137,145,149,152,154,155,156,168,174,176],[74,137,145,149,152,154,155,156,168,176,186],[74,137,145,149,152,154,155,156,168,177],[74,134,137,145,149,152,154,155,156,168,173,179],[74,137,145,149,152,154,155,156,168,173,178],[74,137,145,148,149,152,154,155,156,168,180,181],[74,137,145,149,152,154,155,156,168,180,181],[74,137,142,145,149,152,154,155,156,158,168,173,182],[74,137,145,149,152,154,155,156,168,183],[74,137,145,149,152,154,155,156,158,168,184],[74,137,145,149,151,152,154,155,156,166,168,185],[74,137,145,149,152,154,155,156,168,186,187],[74,137,142,145,149,152,154,155,156,168,187],[74,137,145,149,152,154,155,156,168,173,188],[74,137,145,149,152,154,155,156,157,168,189],[74,137,145,149,152,154,155,156,168,190],[74,137,140,145,149,152,154,155,156,168],[74,137,142,145,149,152,154,155,156,168],[74,137,145,149,152,154,155,156,168,186],[74,124,137,145,149,152,154,155,156,168],[74,137,145,149,152,154,155,156,168,185],[74,137,145,149,152,154,155,156,168,191],[74,137,145,149,152,154,155,156,163,168],[74,137,145,149,152,154,155,156,168,181],[74,124,137,145,148,149,150,152,154,155,156,163,168,173,176,185,188,189,191],[74,137,145,149,152,154,155,156,168,173,192],[64,65,74,137,145,149,152,154,155,156,168],[66,74,137,145,149,152,154,155,156,168],[74,137,145,149,152,154,155,156,168,195,199,202,204,219,220,221,224,229],[74,137,145,149,152,154,155,156,168,199,200,202,203],[74,137,145,149,152,154,155,156,168,199],[74,137,145,149,152,154,155,156,168,199,200,202],[74,137,145,149,152,154,155,156,168,199,200],[74,137,145,149,152,154,155,156,168,194,211,212],[74,137,145,149,152,154,155,156,168,194,211],[74,137,145,149,152,154,155,156,168,194,201],[74,137,145,149,152,154,155,156,168,194],[74,137,145,149,152,154,155,156,168,196],[74,137,145,149,152,154,155,156,168,194,195,196,197,198],[74,137,145,149,152,154,155,156,168,232,233],[74,137,145,149,152,154,155,156,168,232,233,234,235],[74,137,145,149,152,154,155,156,168,232,234],[74,137,145,149,152,154,155,156,168,232],[74,90,93,96,97,137,145,149,152,154,155,156,168,185],[74,93,137,145,149,152,154,155,156,168,173,185],[74,93,97,137,145,149,152,154,155,156,168,185],[74,137,145,149,152,154,155,156,168,173],[74,87,137,145,149,152,154,155,156,168],[74,91,137,145,149,152,154,155,156,168],[74,89,90,93,137,145,149,152,154,155,156,168,185],[74,137,145,149,152,154,155,156,158,168,182],[74,137,145,149,152,154,155,156,168,193],[74,87,137,145,149,152,154,155,156,168,193],[74,89,93,137,145,149,152,154,155,156,158,168,185],[74,84,85,86,88,92,137,145,148,149,152,154,155,156,168,173,185],[74,93,101,109,137,145,149,152,154,155,156,168],[74,85,91,137,145,149,152,154,155,156,168],[74,93,118,119,137,145,149,152,154,155,156,168],[74,85,88,93,137,145,149,152,154,155,156,168,176,185,193],[74,93,137,145,149,152,154,155,156,168],[74,89,93,137,145,149,152,154,155,156,168,185],[74,84,137,145,149,152,154,155,156,168],[74,87,88,89,91,92,93,94,95,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,119,120,121,122,123,137,145,149,152,154,155,156,168],[74,93,111,114,137,145,149,152,154,155,156,168],[74,93,101,102,103,137,145,149,152,154,155,156,168],[74,91,93,102,104,137,145,149,152,154,155,156,168],[74,92,137,145,149,152,154,155,156,168],[74,85,87,93,137,145,149,152,154,155,156,168],[74,93,97,102,104,137,145,149,152,154,155,156,168],[74,97,137,145,149,152,154,155,156,168],[74,91,93,96,137,145,149,152,154,155,156,168,185],[74,85,89,93,101,137,145,149,152,154,155,156,168],[74,93,111,137,145,149,152,154,155,156,168],[74,104,137,145,149,152,154,155,156,168],[74,87,93,118,137,145,149,152,154,155,156,168,176,191,193],[74,137,145,149,152,154,155,156,168,206,207,208,209],[74,137,145,149,152,154,155,156,168,206],[74,137,145,149,152,154,155,156,168,207],[74,137,145,149,152,154,155,156,168,204,226,227,229],[74,137,145,149,152,154,155,156,168,204,205,217,229],[74,137,145,149,152,154,155,156,168,194,202,204,213,229],[74,137,145,149,152,154,155,156,168,210],[74,137,145,149,152,154,155,156,168,194,204,213,216,225,228,229],[74,137,145,149,152,154,155,156,168,204,205,210,213,229],[74,137,145,149,152,154,155,156,168,204,226,227,228,229],[74,137,145,149,152,154,155,156,168,204,210,214,215,216,229],[74,137,145,149,152,154,155,156,168,194,199,202,204,205,210,213,214,215,216,217,218,219,225,226,227,228,229,230,231,236],[74,137,145,149,152,154,155,156,168,237],[67,68,69,74,137,145,149,152,154,155,156,168],[67,68,69,70,74,137,145,149,152,154,155,156,168],[67,68,74,137,145,149,152,154,155,156,168],[67,74,137,145,149,152,154,155,156,168]],"fileInfos":[{"version":"c430d44666289dae81f30fa7b2edebf186ecc91a2d4c71266ea6ae76388792e1","affectsGlobalScope":true,"impliedFormat":1},{"version":"45b7ab580deca34ae9729e97c13cfd999df04416a79116c3bfb483804f85ded4","impliedFormat":1},{"version":"3facaf05f0c5fc569c5649dd359892c98a85557e3e0c847964caeb67076f4d75","impliedFormat":1},{"version":"e44bb8bbac7f10ecc786703fe0a6a4b952189f908707980ba8f3c8975a760962","impliedFormat":1},{"version":"5e1c4c362065a6b95ff952c0eab010f04dcd2c3494e813b493ecfd4fcb9fc0d8","impliedFormat":1},{"version":"68d73b4a11549f9c0b7d352d10e91e5dca8faa3322bfb77b661839c42b1ddec7","impliedFormat":1},{"version":"5efce4fc3c29ea84e8928f97adec086e3dc876365e0982cc8479a07954a3efd4","impliedFormat":1},{"version":"feecb1be483ed332fad555aff858affd90a48ab19ba7272ee084704eb7167569","impliedFormat":1},{"version":"ee7bad0c15b58988daa84371e0b89d313b762ab83cb5b31b8a2d1162e8eb41c2","impliedFormat":1},{"version":"27bdc30a0e32783366a5abeda841bc22757c1797de8681bbe81fbc735eeb1c10","impliedFormat":1},{"version":"c57796738e7f83dbc4b8e65132f11a377649c00dd3eee333f672b8f0a6bea671","affectsGlobalScope":true,"impliedFormat":1},{"version":"dc2df20b1bcdc8c2d34af4926e2c3ab15ffe1160a63e58b7e09833f616efff44","affectsGlobalScope":true,"impliedFormat":1},{"version":"515d0b7b9bea2e31ea4ec968e9edd2c39d3eebf4a2d5cbd04e88639819ae3b71","affectsGlobalScope":true,"impliedFormat":1},{"version":"0559b1f683ac7505ae451f9a96ce4c3c92bdc71411651ca6ddb0e88baaaad6a3","affectsGlobalScope":true,"impliedFormat":1},{"version":"0dc1e7ceda9b8b9b455c3a2d67b0412feab00bd2f66656cd8850e8831b08b537","affectsGlobalScope":true,"impliedFormat":1},{"version":"ce691fb9e5c64efb9547083e4a34091bcbe5bdb41027e310ebba8f7d96a98671","affectsGlobalScope":true,"impliedFormat":1},{"version":"8d697a2a929a5fcb38b7a65594020fcef05ec1630804a33748829c5ff53640d0","affectsGlobalScope":true,"impliedFormat":1},{"version":"4ff2a353abf8a80ee399af572debb8faab2d33ad38c4b4474cff7f26e7653b8d","affectsGlobalScope":true,"impliedFormat":1},{"version":"fb0f136d372979348d59b3f5020b4cdb81b5504192b1cacff5d1fbba29378aa1","affectsGlobalScope":true,"impliedFormat":1},{"version":"d15bea3d62cbbdb9797079416b8ac375ae99162a7fba5de2c6c505446486ac0a","affectsGlobalScope":true,"impliedFormat":1},{"version":"68d18b664c9d32a7336a70235958b8997ebc1c3b8505f4f1ae2b7e7753b87618","affectsGlobalScope":true,"impliedFormat":1},{"version":"eb3d66c8327153d8fa7dd03f9c58d351107fe824c79e9b56b462935176cdf12a","affectsGlobalScope":true,"impliedFormat":1},{"version":"38f0219c9e23c915ef9790ab1d680440d95419ad264816fa15009a8851e79119","affectsGlobalScope":true,"impliedFormat":1},{"version":"69ab18c3b76cd9b1be3d188eaf8bba06112ebbe2f47f6c322b5105a6fbc45a2e","affectsGlobalScope":true,"impliedFormat":1},{"version":"a680117f487a4d2f30ea46f1b4b7f58bef1480456e18ba53ee85c2746eeca012","affectsGlobalScope":true,"impliedFormat":1},{"version":"2f11ff796926e0832f9ae148008138ad583bd181899ab7dd768a2666700b1893","affectsGlobalScope":true,"impliedFormat":1},{"version":"4de680d5bb41c17f7f68e0419412ca23c98d5749dcaaea1896172f06435891fc","affectsGlobalScope":true,"impliedFormat":1},{"version":"954296b30da6d508a104a3a0b5d96b76495c709785c1d11610908e63481ee667","affectsGlobalScope":true,"impliedFormat":1},{"version":"ac9538681b19688c8eae65811b329d3744af679e0bdfa5d842d0e32524c73e1c","affectsGlobalScope":true,"impliedFormat":1},{"version":"0a969edff4bd52585473d24995c5ef223f6652d6ef46193309b3921d65dd4376","affectsGlobalScope":true,"impliedFormat":1},{"version":"9e9fbd7030c440b33d021da145d3232984c8bb7916f277e8ffd3dc2e3eae2bdb","affectsGlobalScope":true,"impliedFormat":1},{"version":"811ec78f7fefcabbda4bfa93b3eb67d9ae166ef95f9bff989d964061cbf81a0c","affectsGlobalScope":true,"impliedFormat":1},{"version":"717937616a17072082152a2ef351cb51f98802fb4b2fdabd32399843875974ca","affectsGlobalScope":true,"impliedFormat":1},{"version":"d7e7d9b7b50e5f22c915b525acc5a49a7a6584cf8f62d0569e557c5cfc4b2ac2","affectsGlobalScope":true,"impliedFormat":1},{"version":"71c37f4c9543f31dfced6c7840e068c5a5aacb7b89111a4364b1d5276b852557","affectsGlobalScope":true,"impliedFormat":1},{"version":"576711e016cf4f1804676043e6a0a5414252560eb57de9faceee34d79798c850","affectsGlobalScope":true,"impliedFormat":1},{"version":"89c1b1281ba7b8a96efc676b11b264de7a8374c5ea1e6617f11880a13fc56dc6","affectsGlobalScope":true,"impliedFormat":1},{"version":"74f7fa2d027d5b33eb0471c8e82a6c87216223181ec31247c357a3e8e2fddc5b","affectsGlobalScope":true,"impliedFormat":1},{"version":"d6d7ae4d1f1f3772e2a3cde568ed08991a8ae34a080ff1151af28b7f798e22ca","affectsGlobalScope":true,"impliedFormat":1},{"version":"063600664504610fe3e99b717a1223f8b1900087fab0b4cad1496a114744f8df","affectsGlobalScope":true,"impliedFormat":1},{"version":"934019d7e3c81950f9a8426d093458b65d5aff2c7c1511233c0fd5b941e608ab","affectsGlobalScope":true,"impliedFormat":1},{"version":"52ada8e0b6e0482b728070b7639ee42e83a9b1c22d205992756fe020fd9f4a47","affectsGlobalScope":true,"impliedFormat":1},{"version":"3bdefe1bfd4d6dee0e26f928f93ccc128f1b64d5d501ff4a8cf3c6371200e5e6","affectsGlobalScope":true,"impliedFormat":1},{"version":"59fb2c069260b4ba00b5643b907ef5d5341b167e7d1dbf58dfd895658bda2867","affectsGlobalScope":true,"impliedFormat":1},{"version":"639e512c0dfc3fad96a84caad71b8834d66329a1f28dc95e3946c9b58176c73a","affectsGlobalScope":true,"impliedFormat":1},{"version":"368af93f74c9c932edd84c58883e736c9e3d53cec1fe24c0b0ff451f529ceab1","affectsGlobalScope":true,"impliedFormat":1},{"version":"af3dd424cf267428f30ccfc376f47a2c0114546b55c44d8c0f1d57d841e28d74","affectsGlobalScope":true,"impliedFormat":1},{"version":"995c005ab91a498455ea8dfb63aa9f83fa2ea793c3d8aa344be4a1678d06d399","affectsGlobalScope":true,"impliedFormat":1},{"version":"959d36cddf5e7d572a65045b876f2956c973a586da58e5d26cde519184fd9b8a","affectsGlobalScope":true,"impliedFormat":1},{"version":"965f36eae237dd74e6cca203a43e9ca801ce38824ead814728a2807b1910117d","affectsGlobalScope":true,"impliedFormat":1},{"version":"3925a6c820dcb1a06506c90b1577db1fdbf7705d65b62b99dce4be75c637e26b","affectsGlobalScope":true,"impliedFormat":1},{"version":"0a3d63ef2b853447ec4f749d3f368ce642264246e02911fcb1590d8c161b8005","affectsGlobalScope":true,"impliedFormat":1},{"version":"8cdf8847677ac7d20486e54dd3fcf09eda95812ac8ace44b4418da1bbbab6eb8","affectsGlobalScope":true,"impliedFormat":1},{"version":"8444af78980e3b20b49324f4a16ba35024fef3ee069a0eb67616ea6ca821c47a","affectsGlobalScope":true,"impliedFormat":1},{"version":"3287d9d085fbd618c3971944b65b4be57859f5415f495b33a6adc994edd2f004","affectsGlobalScope":true,"impliedFormat":1},{"version":"b4b67b1a91182421f5df999988c690f14d813b9850b40acd06ed44691f6727ad","affectsGlobalScope":true,"impliedFormat":1},{"version":"df83c2a6c73228b625b0beb6669c7ee2a09c914637e2d35170723ad49c0f5cd4","affectsGlobalScope":true,"impliedFormat":1},{"version":"436aaf437562f276ec2ddbee2f2cdedac7664c1e4c1d2c36839ddd582eeb3d0a","affectsGlobalScope":true,"impliedFormat":1},{"version":"8e3c06ea092138bf9fa5e874a1fdbc9d54805d074bee1de31b99a11e2fec239d","affectsGlobalScope":true,"impliedFormat":1},{"version":"51ad4c928303041605b4d7ae32e0c1ee387d43a24cd6f1ebf4a2699e1076d4fa","affectsGlobalScope":true,"impliedFormat":1},{"version":"196cb558a13d4533a5163286f30b0509ce0210e4b316c56c38d4c0fd2fb38405","affectsGlobalScope":true,"impliedFormat":1},{"version":"8e7f8264d0fb4c5339605a15daadb037bf238c10b654bb3eee14208f860a32ea","affectsGlobalScope":true,"impliedFormat":1},{"version":"782dec38049b92d4e85c1585fbea5474a219c6984a35b004963b00beb1aab538","affectsGlobalScope":true,"impliedFormat":1},{"version":"7e29f41b158de217f94cb9676bf9cbd0cd9b5a46e1985141ed36e075c52bf6ad","affectsGlobalScope":true,"impliedFormat":1},{"version":"ac51dd7d31333793807a6abaa5ae168512b6131bd41d9c5b98477fc3b7800f9f","impliedFormat":1},{"version":"f123246a7b6c04d80b9b57fadfc6c90959ec6d5c0d4c8e620e06e2811ae3a052","impliedFormat":1},{"version":"42c169fb8c2d42f4f668c624a9a11e719d5d07dacbebb63cbcf7ef365b0a75b3","impliedFormat":1},{"version":"b21065f124374c2129e39b8c6876b75ff8034d5ceac5a2d89bc0af4f3e2a302b","signature":"1f0d34e25d864b7e66f2b99691f9c0d6074f3970984022c59559442897de6c68","impliedFormat":99},{"version":"2e5250307df72e0a640a425c0cc7f3e1a637dc841ef4a233faeeab71faadf2ee","signature":"738e99861de17f29d645e6acb1b7d6c164e9cc39a98231003b13bf75236724f5","impliedFormat":99},{"version":"d9e56e09f5b4cfa9de14ece876caf70632efcb6248e36404a51953e83f752ec2","signature":"9f69a4fbb37dca38b68deb6b7f69ca8a84f6d26a9c541b9e3e5ca3c3265754dd","impliedFormat":99},{"version":"367033989834328413b51203c0b5f6e004d9f5f8a1ee1cc35ab595c29aeff5d7","signature":"64255953a32fa82f59ad65cbdda7fa1c80871ac29a872f4ab40ce6a4d2aa9538","impliedFormat":99},{"version":"d153a11543fd884b596587ccd97aebbeed950b26933ee000f94009f1ab142848","affectsGlobalScope":true,"impliedFormat":1},{"version":"0ccdaa19852d25ecd84eec365c3bfa16e7859cadecf6e9ca6d0dbbbee439743f","affectsGlobalScope":true,"impliedFormat":1},{"version":"438b41419b1df9f1fbe33b5e1b18f5853432be205991d1b19f5b7f351675541e","affectsGlobalScope":true,"impliedFormat":1},{"version":"096116f8fedc1765d5bd6ef360c257b4a9048e5415054b3bf3c41b07f8951b0b","affectsGlobalScope":true,"impliedFormat":1},{"version":"e5e01375c9e124a83b52ee4b3244ed1a4d214a6cfb54ac73e164a823a4a7860a","affectsGlobalScope":true,"impliedFormat":1},{"version":"f90ae2bbce1505e67f2f6502392e318f5714bae82d2d969185c4a6cecc8af2fc","affectsGlobalScope":true,"impliedFormat":1},{"version":"4b58e207b93a8f1c88bbf2a95ddc686ac83962b13830fe8ad3f404ffc7051fb4","affectsGlobalScope":true,"impliedFormat":1},{"version":"1fefabcb2b06736a66d2904074d56268753654805e829989a46a0161cd8412c5","affectsGlobalScope":true,"impliedFormat":1},{"version":"9798340ffb0d067d69b1ae5b32faa17ab31b82466a3fc00d8f2f2df0c8554aaa","affectsGlobalScope":true,"impliedFormat":1},{"version":"c18a99f01eb788d849ad032b31cafd49de0b19e083fe775370834c5675d7df8e","affectsGlobalScope":true,"impliedFormat":1},{"version":"5247874c2a23b9a62d178ae84f2db6a1d54e6c9a2e7e057e178cc5eea13757fc","affectsGlobalScope":true,"impliedFormat":1},{"version":"4967529644e391115ca5592184d4b63980569adf60ee685f968fd59ab1557188","impliedFormat":1},{"version":"cdcf9ea426ad970f96ac930cd176d5c69c6c24eebd9fc580e1572d6c6a88f62c","impliedFormat":1},{"version":"23cd712e2ce083d68afe69224587438e5914b457b8acf87073c22494d706a3d0","impliedFormat":1},{"version":"487b694c3de27ddf4ad107d4007ad304d29effccf9800c8ae23c2093638d906a","impliedFormat":1},{"version":"3a80bc85f38526ca3b08007ee80712e7bb0601df178b23fbf0bf87036fce40ce","impliedFormat":1},{"version":"ccf4552357ce3c159ef75f0f0114e80401702228f1898bdc9402214c9499e8c0","impliedFormat":1},{"version":"c6fd2c5a395f2432786c9cb8deb870b9b0e8ff7e22c029954fabdd692bff6195","impliedFormat":1},{"version":"68834d631c8838c715f225509cfc3927913b9cc7a4870460b5b60c8dbdb99baf","impliedFormat":1},{"version":"2931540c47ee0ff8a62860e61782eb17b155615db61e36986e54645ec67f67c2","impliedFormat":1},{"version":"ccab02f3920fc75c01174c47fcf67882a11daf16baf9e81701d0a94636e94556","impliedFormat":1},{"version":"f6faf5f74e4c4cc309a6c6a6c4da02dbb840be5d3e92905a23dcd7b2b0bd1986","impliedFormat":1},{"version":"ea6bc8de8b59f90a7a3960005fd01988f98fd0784e14bc6922dde2e93305ec7d","impliedFormat":1},{"version":"36107995674b29284a115e21a0618c4c2751b32a8766dd4cb3ba740308b16d59","impliedFormat":1},{"version":"914a0ae30d96d71915fc519ccb4efbf2b62c0ddfb3a3fc6129151076bc01dc60","impliedFormat":1},{"version":"33e981bf6376e939f99bd7f89abec757c64897d33c005036b9a10d9587d80187","impliedFormat":1},{"version":"7fd1b31fd35876b0aa650811c25ec2c97a3c6387e5473eb18004bed86cdd76b6","impliedFormat":1},{"version":"b41767d372275c154c7ea6c9d5449d9a741b8ce080f640155cc88ba1763e35b3","impliedFormat":1},{"version":"3bacf516d686d08682751a3bd2519ea3b8041a164bfb4f1d35728993e70a2426","impliedFormat":1},{"version":"7fb266686238369442bd1719bc0d7edd0199da4fb8540354e1ff7f16669b4323","impliedFormat":1},{"version":"0a60a292b89ca7218b8616f78e5bbd1c96b87e048849469cccb4355e98af959a","impliedFormat":1},{"version":"0b6e25234b4eec6ed96ab138d96eb70b135690d7dd01f3dd8a8ab291c35a683a","impliedFormat":1},{"version":"9666f2f84b985b62400d2e5ab0adae9ff44de9b2a34803c2c5bd3c8325b17dc0","impliedFormat":1},{"version":"40cd35c95e9cf22cfa5bd84e96408b6fcbca55295f4ff822390abb11afbc3dca","impliedFormat":1},{"version":"b1616b8959bf557feb16369c6124a97a0e74ed6f49d1df73bb4b9ddf68acf3f3","impliedFormat":1},{"version":"5b03a034c72146b61573aab280f295b015b9168470f2df05f6080a2122f9b4df","impliedFormat":1},{"version":"40b463c6766ca1b689bfcc46d26b5e295954f32ad43e37ee6953c0a677e4ae2b","impliedFormat":1},{"version":"249b9cab7f5d628b71308c7d9bb0a808b50b091e640ba3ed6e2d0516f4a8d91d","impliedFormat":1},{"version":"80aae6afc67faa5ac0b32b5b8bc8cc9f7fa299cff15cf09cc2e11fd28c6ae29e","impliedFormat":1},{"version":"f473cd2288991ff3221165dcf73cd5d24da30391f87e85b3dd4d0450c787a391","impliedFormat":1},{"version":"499e5b055a5aba1e1998f7311a6c441a369831c70905cc565ceac93c28083d53","impliedFormat":1},{"version":"54c3e2371e3d016469ad959697fd257e5621e16296fa67082c2575d0bf8eced0","impliedFormat":1},{"version":"beb8233b2c220cfa0feea31fbe9218d89fa02faa81ef744be8dce5acb89bb1fd","impliedFormat":1},{"version":"c183b931b68ad184bc8e8372bf663f3d33304772fb482f29fb91b3c391031f3e","impliedFormat":1},{"version":"5d0375ca7310efb77e3ef18d068d53784faf62705e0ad04569597ae0e755c401","impliedFormat":1},{"version":"59af37caec41ecf7b2e76059c9672a49e682c1a2aa6f9d7dc78878f53aa284d6","impliedFormat":1},{"version":"addf417b9eb3f938fddf8d81e96393a165e4be0d4a8b6402292f9c634b1cb00d","impliedFormat":1},{"version":"48cc3ec153b50985fb95153258a710782b25975b10dd4ac8a4f3920632d10790","impliedFormat":1},{"version":"adf27937dba6af9f08a68c5b1d3fce0ca7d4b960c57e6d6c844e7d1a8e53adae","impliedFormat":1},{"version":"e1528ca65ac90f6fa0e4a247eb656b4263c470bb22d9033e466463e13395e599","impliedFormat":1},{"version":"2e85db9e6fd73cfa3d7f28e0ab6b55417ea18931423bd47b409a96e4a169e8e6","impliedFormat":1},{"version":"c46e079fe54c76f95c67fb89081b3e399da2c7d109e7dca8e4b58d83e332e605","impliedFormat":1},{"version":"866078923a56d026e39243b4392e282c1c63159723996fa89243140e1388a98d","impliedFormat":1},{"version":"dd0109710de4cd93e245121ab86d8c66d20f3ead80074b68e9c3e349c4f53342","affectsGlobalScope":true,"impliedFormat":1},{"version":"b3275d55fac10b799c9546804126239baf020d220136163f763b55a74e50e750","affectsGlobalScope":true,"impliedFormat":1},{"version":"fa68a0a3b7cb32c00e39ee3cd31f8f15b80cac97dce51b6ee7fc14a1e8deb30b","affectsGlobalScope":true,"impliedFormat":1},{"version":"1cf059eaf468efcc649f8cf6075d3cb98e9a35a0fe9c44419ec3d2f5428d7123","affectsGlobalScope":true,"impliedFormat":1},{"version":"6c36e755bced82df7fb6ce8169265d0a7bb046ab4e2cb6d0da0cb72b22033e89","affectsGlobalScope":true,"impliedFormat":1},{"version":"e7721c4f69f93c91360c26a0a84ee885997d748237ef78ef665b153e622b36c1","affectsGlobalScope":true,"impliedFormat":1},{"version":"7a93de4ff8a63bafe62ba86b89af1df0ccb5e40bb85b0c67d6bbcfdcf96bf3d4","affectsGlobalScope":true,"impliedFormat":1},{"version":"90e85f9bc549dfe2b5749b45fe734144e96cd5d04b38eae244028794e142a77e","affectsGlobalScope":true,"impliedFormat":1},{"version":"e0a5deeb610b2a50a6350bd23df6490036a1773a8a71d70f2f9549ab009e67ee","affectsGlobalScope":true,"impliedFormat":1},{"version":"435b3711465425770ed2ee2f1cf00ce071835265e0851a7dc4600ab4b007550e","impliedFormat":1},{"version":"7e49f52a159435fc8df4de9dc377ef5860732ca2dc9efec1640531d3cf5da7a3","impliedFormat":1},{"version":"dd4bde4bdc2e5394aed6855e98cf135dfdf5dd6468cad842e03116d31bbcc9bc","impliedFormat":1},{"version":"4d4e879009a84a47c05350b8dca823036ba3a29a3038efed1be76c9f81e45edf","affectsGlobalScope":true,"impliedFormat":1},{"version":"cf83d90d5faf27b994c2e79af02e32b555dbfe42cd9bd1571445f2168d1f4e2d","impliedFormat":1},{"version":"9ba13b47cb450a438e3076c4a3f6afb9dc85e17eae50f26d4b2d72c0688c9251","impliedFormat":1},{"version":"b64cd4401633ea4ecadfd700ddc8323a13b63b106ac7127c1d2726f32424622c","impliedFormat":1},{"version":"37c6e5fe5715814412b43cc9b50b24c67a63c4e04e753e0d1305970d65417a60","impliedFormat":1},{"version":"0e28335ac43f4d94dd2fe6d9e6fa6813570640839addd10d309d7985f33a6308","impliedFormat":1},{"version":"ee0e4946247f842c6dd483cbb60a5e6b484fee07996e3a7bc7343dfb68a04c5d","impliedFormat":1},{"version":"ef051f42b7e0ef5ca04552f54c4552eac84099d64b6c5ad0ef4033574b6035b8","impliedFormat":1},{"version":"853a43154f1d01b0173d9cbd74063507ece57170bad7a3b68f3fa1229ad0a92f","impliedFormat":1},{"version":"56231e3c39a031bfb0afb797690b20ed4537670c93c0318b72d5180833d98b72","impliedFormat":1},{"version":"5cc7c39031bfd8b00ad58f32143d59eb6ffc24f5d41a20931269011dccd36c5e","impliedFormat":1},{"version":"b0b69c61b0f0ec8ca15db4c8c41f6e77f4cacb784d42bca948f42dea33e8757e","affectsGlobalScope":true,"impliedFormat":1},{"version":"f96a48183254c00d24575401f1a761b4ce4927d927407e7862a83e06ce5d6964","impliedFormat":1},{"version":"cc25940cfb27aa538e60d465f98bb5068d4d7d33131861ace43f04fe6947d68f","impliedFormat":1},{"version":"ac86245c2f31335bfd52cbe7fc760f9fc4f165387875869a478a6d9616a95e72","impliedFormat":1},{"version":"01ff95aa1443e3f7248974e5a771f513cb2ac158c8898f470a1792f817bee497","impliedFormat":1},{"version":"9d96a7ce809392ff2cb99691acf7c62e632fe56897356ba013b689277aca3619","impliedFormat":1},{"version":"42a05d8f239f74587d4926aba8cc54792eed8e8a442c7adc9b38b516642aadfe","impliedFormat":1},{"version":"5d21b58d60383cc6ab9ad3d3e265d7d25af24a2c9b506247e0e50b0a884920be","impliedFormat":1},{"version":"101f482fd48cb4c7c0468dcc6d62c843d842977aea6235644b1edd05e81fbf22","impliedFormat":1},{"version":"ae6757460f37078884b1571a3de3ebaf724d827d7e1d53626c02b3c2a408ac63","affectsGlobalScope":true,"impliedFormat":1},{"version":"27c0a08e343c6a0ae17bd13ba6d44a9758236dc904cd5e4b43456996cd51f520","impliedFormat":1},{"version":"3ef397f12387eff17f550bc484ea7c27d21d43816bbe609d495107f44b97e933","impliedFormat":1},{"version":"1023282e2ba810bc07905d3668349fbd37a26411f0c8f94a70ef3c05fe523fcf","impliedFormat":1},{"version":"b214ebcf76c51b115453f69729ee8aa7b7f8eccdae2a922b568a45c2d7ff52f7","impliedFormat":1},{"version":"429c9cdfa7d126255779efd7e6d9057ced2d69c81859bbab32073bad52e9ba76","impliedFormat":1},{"version":"6f80e51ba310608cd71bcdc09a171d7bbfb3b316048601c9ec215ce16a8dcfbc","impliedFormat":1},{"version":"63054ef6518309c85b11e00537eef47a2b8609c6c20630b1a1daaecf86ea0d28","affectsGlobalScope":true,"impliedFormat":1},{"version":"7f2c62938251b45715fd2a9887060ec4fbc8724727029d1cbce373747252bdd7","impliedFormat":1},{"version":"e3ace08b6bbd84655d41e244677b474fd995923ffef7149ddb68af8848b60b05","impliedFormat":1},{"version":"132580b0e86c48fab152bab850fc57a4b74fe915c8958d2ccb052b809a44b61c","impliedFormat":1},{"version":"af4ab0aa8908fc9a655bb833d3bc28e117c4f0e1038c5a891546158beb25accb","impliedFormat":1},{"version":"69c9a5a9392e8564bd81116e1ed93b13205201fb44cb35a7fde8c9f9e21c4b23","impliedFormat":1},{"version":"5f8fc37f8434691ffac1bfd8fc2634647da2c0e84253ab5d2dd19a7718915b35","impliedFormat":1},{"version":"5981c2340fd8b076cae8efbae818d42c11ffc615994cb060b1cd390795f1be2b","impliedFormat":1},{"version":"2ca2bca6845a7234eff5c3d192727a068fca72ac565f3c819c6b04ccc83dadc0","impliedFormat":1},{"version":"ed4f674fc8c0c993cc7e145069ac44129e03519b910c62be206a0cc777bdc60b","affectsGlobalScope":true,"impliedFormat":1},{"version":"0250da3eb85c99624f974e77ef355cdf86f43980251bc371475c2b397ba55bcd","impliedFormat":1},{"version":"f1c93e046fb3d9b7f8249629f4b63dc068dd839b824dd0aa39a5e68476dc9420","impliedFormat":1},{"version":"3d3a5f27ffbc06c885dd4d5f9ee20de61faf877fe2c3a7051c4825903d9a7fdc","impliedFormat":1},{"version":"12806f9f085598ef930edaf2467a5fa1789a878fba077cd27e85dc5851e11834","impliedFormat":1},{"version":"17d06eb5709839c7ce719f0c38ada6f308fb433f2cd6d8c87b35856e07400950","impliedFormat":1},{"version":"a43fe41c33d0a192a0ecaf9b92e87bef3709c9972e6d53c42c49251ccb962d69","impliedFormat":1},{"version":"a177959203c017fad3ecc4f3d96c8757a840957a4959a3ae00dab9d35961ca6c","affectsGlobalScope":true,"impliedFormat":1},{"version":"6fc727ccf9b36e257ff982ea0badeffbfc2c151802f741bddff00c6af3b784cf","impliedFormat":1},{"version":"2a00d005e3af99cd1cfa75220e60c61b04bfb6be7ca7453bfe2ef6cca37cc03c","impliedFormat":1},{"version":"4844a4c9b4b1e812b257676ed8a80b3f3be0e29bf05e742cc2ea9c3c6865e6c6","impliedFormat":1},{"version":"064878a60367e0407c42fb7ba02a2ea4d83257357dc20088e549bd4d89433e9c","impliedFormat":1},{"version":"14d4bd22d1b05824971b98f7e91b2484c90f1a684805c330476641417c3d9735","impliedFormat":1},{"version":"586eaf66bace2e731cee0ddfbfac326ad74a83c1acfeac4afb2db85ad23226c7","impliedFormat":1},{"version":"b484ec11ba00e3a2235562a41898d55372ccabe607986c6fa4f4aba72093749f","impliedFormat":1},{"version":"d1a14d87cedcf4f0b8173720d6eb29cc02878bf2b6dabf9c9d9cee742f275368","impliedFormat":1},{"version":"e60efae9fe48a2955f66bf4cbf0f082516185b877daf50d9c5e2a009660a7714","impliedFormat":1},{"version":"041a7781b9127ab568d2cdcce62c58fdea7c7407f40b8c50045d7866a2727130","impliedFormat":1},{"version":"b37f83e7deea729aa9ce5593f78905afb45b7532fdff63041d374f60059e7852","impliedFormat":1},{"version":"e1cb68f3ef3a8dd7b2a9dfb3de482ed6c0f1586ba0db4e7d73c1d2147b6ffc51","impliedFormat":1},{"version":"55cdbeebe76a1fa18bbd7e7bf73350a2173926bd3085bb050cf5a5397025ee4e","impliedFormat":1},{"version":"acfb723d81eda39156251aed414c553294870bf53062429ebfcfba8a68cb4753","impliedFormat":99},{"version":"fa69a90381c2f85889722a911a732a5ee3596dc3acecda8a9aa2fa89b9615d8d","impliedFormat":99},{"version":"b5ce343886d23392be9c8280e9f24a87f1d7d3667f6672c2fe4aa61fa4ece7d4","impliedFormat":99},{"version":"57e9e1b0911874c62d743af24b5d56032759846533641d550b12a45ff404bf07","impliedFormat":99},{"version":"b0857bb28fd5236ace84280f79a25093f919fd0eff13e47cc26ea03de60a7294","impliedFormat":99},{"version":"5e43e0824f10cd8c48e7a8c5c673638488925a12c31f0f9e0957965c290eb14c","impliedFormat":99},{"version":"854cd3a3375ffc4e7a92b2168dd065d7ff2614b43341038a65cca865a44c00c5","impliedFormat":99},{"version":"ef13c73d6157a32933c612d476c1524dd674cf5b9a88571d7d6a0d147544d529","impliedFormat":99},{"version":"3b0a56d056d81a011e484b9c05d5e430711aaecd561a788bad1d0498aad782c7","impliedFormat":99},{"version":"2f863ee9b873a65d9c3338ea7aaddbdb41a9673f062f06983d712bd01c25dc6b","impliedFormat":99},{"version":"67aa128c2bc170b93794f191feffc65a4b33e878db211cfcb7658c4b72f7a1f5","impliedFormat":99},{"version":"ac3d263474022e9a14c43f588f485d549641d839b159ecc971978b90f34bdf6b","impliedFormat":99},{"version":"a7ca8df4f2931bef2aa4118078584d84a0b16539598eaadf7dce9104dfaa381c","impliedFormat":1},{"version":"11443a1dcfaaa404c68d53368b5b818712b95dd19f188cab1669c39bee8b84b3","impliedFormat":1},{"version":"36977c14a7f7bfc8c0426ae4343875689949fb699f3f84ecbe5b300ebf9a2c55","impliedFormat":1},{"version":"035d0934d304483f07148427a5bd5b98ac265dae914a6b49749fe23fbd893ec7","impliedFormat":99},{"version":"e2ed5b81cbed3a511b21a18ab2539e79ac1f4bc1d1d28f8d35d8104caa3b429f","impliedFormat":99},{"version":"324ac98294dab54fbd580c7d0e707d94506d7b2c3d5efe981a8495f02cf9ad96","impliedFormat":99},{"version":"9ec72eb493ff209b470467e24264116b6a8616484bca438091433a545dfba17e","impliedFormat":99},{"version":"c35b8117804c639c53c87f2c23e0c786df61d552e513bd5179f5b88e29964838","impliedFormat":99},{"version":"c609331c6ed4ad4af54e101088c6a4dcb48f8db7b0b97e44a6efeb130f4331bd","impliedFormat":99},{"version":"bcbd3becd08b4515225880abea0dbfbbf0d1181ce3af8f18f72f61edbe4febfb","impliedFormat":99},{"version":"67acaedb46832d66c15f1b09fb7b6a0b7f41bdbf8eaa586ec70459b3e8896eb9","impliedFormat":99},{"version":"4535ab977ee871e956eb7bebe2db5de79f5d5ec7dfbbf1d35e08f4a2d6630dac","impliedFormat":99},{"version":"b79b5ed99f26ffb2f8ae4bdcc4b34a9542197dc3fa96cfb425c2a81e618cff28","impliedFormat":99},{"version":"31fd7c12f6e27154efb52a916b872509a771880f3b20f2dfd045785c13aa813f","impliedFormat":99},{"version":"b481de4ab5379bd481ca12fc0b255cdc47341629a22c240a89cdb4e209522be2","impliedFormat":99},{"version":"bdd14f07b4eca0b4b5203b85b8dbc4d084c749fa590bee5ea613e1641dcd3b29","impliedFormat":99},{"version":"427fe2004642504828c1476d0af4270e6ad4db6de78c0b5da3e4c5ca95052a99","impliedFormat":1},{"version":"2eeffcee5c1661ddca53353929558037b8cf305ffb86a803512982f99bcab50d","impliedFormat":99},{"version":"9afb4cb864d297e4092a79ee2871b5d3143ea14153f62ef0bb04ede25f432030","affectsGlobalScope":true,"impliedFormat":99},{"version":"4e258d11c899cb9ff36b4b5c53df59cf4a5ccae9a9931529686e77431e0a3518","affectsGlobalScope":true,"impliedFormat":99},{"version":"a5ae67a67f786ffe92d34b55467a40fb50fb0093e92388cadce6168fa42690fd","impliedFormat":99},{"version":"69bf2422313487956e4dacf049f30cb91b34968912058d244cb19e4baa24da97","impliedFormat":99},{"version":"6987dfb4b0c4e02112cc4e548e7a77b3d9ddfeffa8c8a2db13ceac361a4567d9","impliedFormat":99},{"version":"a534e61c2f06a147d97aebad720db97dffd8066b7142212e46bcbcdcb640b81a","impliedFormat":99},{"version":"ddf569d04470a4d629090d43a16735185001f3fcf0ae036ead99f2ceab62be48","impliedFormat":99},{"version":"b413fbc6658fe2774f8bf9a15cf4c53e586fc38a2d5256b3b9647da242c14389","impliedFormat":99},{"version":"c30a41267fc04c6518b17e55dcb2b810f267af4314b0b6d7df1c33a76ce1b330","impliedFormat":1},{"version":"72422d0bac4076912385d0c10911b82e4694fc106e2d70added091f88f0824ba","impliedFormat":1},{"version":"da251b82c25bee1d93f9fd80c5a61d945da4f708ca21285541d7aff83ecb8200","impliedFormat":1},{"version":"64db14db2bf37ac089766fdb3c7e1160fabc10e9929bc2deeede7237e4419fc8","impliedFormat":1},{"version":"98b94085c9f78eba36d3d2314affe973e8994f99864b8708122750788825c771","impliedFormat":1},{"version":"53c448183c7177c83d3eb0b40824cf8952721a6584cf22052adc24f778986732","impliedFormat":99},{"version":"0a5bc32362b0559b9bcf0a6a83136c4442dbbd0edecd671538a5e03454b6dff0","affectsGlobalScope":true,"impliedFormat":99}],"root":[[68,71]],"options":{"composite":true,"declaration":true,"declarationMap":true,"esModuleInterop":true,"jsx":4,"module":199,"outDir":"./","rewriteRelativeImportExtensions":true,"rootDir":"../src","skipLibCheck":true,"sourceMap":true,"strict":true,"target":10,"tsBuildInfoFile":"./.tsbuildinfo.build","verbatimModuleSyntax":true},"referencedMap":[[221,1],[224,2],[222,1],[134,3],[135,3],[136,4],[74,5],[137,6],[138,7],[139,8],[72,1],[140,9],[141,10],[142,11],[143,12],[144,13],[145,14],[146,14],[147,15],[148,16],[149,17],[150,18],[75,1],[73,1],[151,19],[152,20],[153,21],[193,22],[154,23],[155,24],[156,23],[157,25],[158,26],[159,27],[160,28],[161,28],[162,28],[163,29],[164,30],[165,31],[166,32],[167,33],[168,34],[169,34],[170,35],[171,1],[172,1],[173,36],[174,37],[175,36],[176,38],[177,39],[178,40],[179,41],[180,42],[181,43],[182,44],[183,45],[184,46],[185,47],[186,48],[187,49],[188,50],[189,51],[190,52],[76,23],[77,1],[78,53],[79,54],[80,1],[81,55],[82,1],[125,56],[126,57],[127,58],[128,58],[129,59],[130,1],[131,6],[132,60],[133,57],[191,61],[192,62],[64,1],[66,63],[67,64],[225,65],[194,1],[204,66],[200,67],[203,68],[226,69],[211,1],[213,70],[212,71],[219,1],[202,72],[195,73],[197,74],[199,75],[198,1],[201,73],[196,1],[223,1],[83,1],[65,1],[234,76],[236,77],[235,78],[233,79],[232,1],[227,1],[220,1],[62,1],[63,1],[12,1],[11,1],[2,1],[13,1],[14,1],[15,1],[16,1],[17,1],[18,1],[19,1],[20,1],[3,1],[21,1],[22,1],[4,1],[23,1],[27,1],[24,1],[25,1],[26,1],[28,1],[29,1],[30,1],[5,1],[31,1],[32,1],[33,1],[34,1],[6,1],[38,1],[35,1],[36,1],[37,1],[39,1],[7,1],[40,1],[45,1],[46,1],[41,1],[42,1],[43,1],[44,1],[8,1],[50,1],[47,1],[48,1],[49,1],[51,1],[9,1],[52,1],[53,1],[54,1],[56,1],[55,1],[57,1],[58,1],[10,1],[59,1],[1,1],[60,1],[61,1],[101,80],[113,81],[99,82],[114,83],[123,84],[90,85],[91,86],[89,87],[122,88],[117,89],[121,90],[93,91],[110,92],[92,93],[120,94],[87,95],[88,89],[94,96],[95,1],[100,97],[98,96],[85,98],[124,99],[115,100],[104,101],[103,96],[105,102],[108,103],[102,104],[106,105],[118,88],[96,106],[97,107],[109,108],[86,83],[112,109],[111,96],[107,110],[116,1],[84,1],[119,111],[210,112],[209,113],[207,113],[206,1],[208,114],[228,115],[218,116],[214,117],[215,67],[231,118],[229,119],[216,120],[230,121],[205,1],[217,122],[237,123],[238,124],[70,125],[71,126],[69,127],[68,128]],"latestChangedDtsFile":"./types.d.ts","version":"5.9.3"}
package/dist/define-eval.d.ts ADDED
@@ -0,0 +1,23 @@
1
+ /**
2
+ * `defineEval(definition)` — the public entry point. Returns a
3
+ * callable function: invoke with no args to run with the factory's
4
+ * baked-in defaults, OR with overrides for one-off param swaps.
5
+ *
6
+ * The returned callable also carries `.matrix(axes, opts?)` for
7
+ * cartesian-product parameter sweeps — the multi-model comparison
8
+ * surface: run the same eval (same document, same expected output)
9
+ * across every axis combination and compare per-cell results.
10
+ *
11
+ * @see EvalDefinition for the input shape, CallableEval for the return.
12
+ * @see packages-next/eval (v2 origin)
13
+ */
14
+ import type { CallableEval, DefaultAppOverrides, EvalDefinition } from "./types.js";
15
+ export declare function defineEval<O = DefaultAppOverrides>(definition: EvalDefinition<O>): CallableEval<O>;
16
+ /**
17
+ * Cartesian product of axis values. `{}` → one empty cell. Any axis
18
+ * with an empty array → zero cells (mathematical product).
19
+ */
20
+ export declare function cartesian(axes: Record<string, ReadonlyArray<unknown>>): Array<Record<string, unknown>>;
21
+ /** Run `fn` over `items` with at most `limit` in flight, order-preserving. */
22
+ export declare function mapConcurrent<T, R>(items: ReadonlyArray<T>, limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;
23
+ //# sourceMappingURL=define-eval.d.ts.map
package/dist/define-eval.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"define-eval.d.ts","sourceRoot":"","sources":["../src/define-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,KAAK,EACV,YAAY,EACZ,mBAAmB,EACnB,cAAc,EAIf,MAAM,YAAY,CAAC;AAEpB,wBAAgB,UAAU,CAAC,CAAC,GAAG,mBAAmB,EAChD,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,GAC5B,YAAY,CAAC,CAAC,CAAC,CAkBjB;AA4BD;;;GAGG;AACH,wBAAgB,SAAS,CACvB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,GAC3C,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAgBhC;AAED,8EAA8E;AAC9E,wBAAsB,aAAa,CAAC,CAAC,EAAE,CAAC,EACtC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,EACvB,KAAK,EAAE,MAAM,EACb,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GACzC,OAAO,CAAC,CAAC,EAAE,CAAC,CAcd"}
package/dist/define-eval.js ADDED
@@ -0,0 +1,83 @@
1
+ /**
2
+ * `defineEval(definition)` — the public entry point. Returns a
3
+ * callable function: invoke with no args to run with the factory's
4
+ * baked-in defaults, OR with overrides for one-off param swaps.
5
+ *
6
+ * The returned callable also carries `.matrix(axes, opts?)` for
7
+ * cartesian-product parameter sweeps — the multi-model comparison
8
+ * surface: run the same eval (same document, same expected output)
9
+ * across every axis combination and compare per-cell results.
10
+ *
11
+ * @see EvalDefinition for the input shape, CallableEval for the return.
12
+ * @see packages-next/eval (v2 origin)
13
+ */
14
+ import { runEval } from "./runner.js";
15
+ export function defineEval(definition) {
16
+ const callable = ((overrides) => runEval(definition, overrides));
17
+ Object.defineProperty(callable, "definition", {
18
+ value: definition,
19
+ writable: false,
20
+ enumerable: true,
21
+ configurable: false,
22
+ });
23
+ Object.defineProperty(callable, "matrix", {
24
+ value: ((axes, opts) => runMatrix(definition, axes, opts)),
25
+ writable: false,
26
+ enumerable: true,
27
+ configurable: false,
28
+ });
29
+ return callable;
30
+ }
31
+ // ============================================================================
32
+ // Matrix
33
+ // ============================================================================
34
+ async function runMatrix(definition, axes, opts) {
35
+ const started = Date.now();
36
+ const concurrency = Math.max(1, opts?.concurrency ?? 1);
37
+ const productCells = cartesian(axes);
38
+ if (productCells.length === 0) {
39
+ return { cells: [], passed: true, elapsedMs: Date.now() - started };
40
+ }
41
+ const cells = await mapConcurrent(productCells, concurrency, async (cell) => {
42
+ const result = await runEval(definition, cell);
43
+ return { axes: cell, result };
44
+ });
45
+ const passed = cells.every((c) => c.result.passed);
46
+ return { cells, passed, elapsedMs: Date.now() - started };
47
+ }
48
+ /**
49
+ * Cartesian product of axis values. `{}` → one empty cell. Any axis
50
+ * with an empty array → zero cells (mathematical product).
51
+ */
52
+ export function cartesian(axes) {
53
+ const keys = Object.keys(axes);
54
+ if (keys.length === 0)
55
+ return [{}];
56
+ let cells = [{}];
57
+ for (const key of keys) {
58
+ const values = axes[key] ?? [];
59
+ const next = [];
60
+ for (const cell of cells) {
61
+ for (const value of values) {
62
+ next.push({ ...cell, [key]: value });
63
+ }
64
+ }
65
+ cells = next;
66
+ }
67
+ return cells;
68
+ }
69
+ /** Run `fn` over `items` with at most `limit` in flight, order-preserving. */
70
+ export async function mapConcurrent(items, limit, fn) {
71
+ const results = new Array(items.length);
72
+ let cursor = 0;
73
+ async function worker() {
74
+ while (cursor < items.length) {
75
+ const index = cursor++;
76
+ results[index] = await fn(items[index], index);
77
+ }
78
+ }
79
+ const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
80
+ await Promise.all(workers);
81
+ return results;
82
+ }
83
+ //# sourceMappingURL=define-eval.js.map
package/dist/define-eval.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"define-eval.js","sourceRoot":"","sources":["../src/define-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAUtC,MAAM,UAAU,UAAU,CACxB,UAA6B;IAE7B,MAAM,QAAQ,GAAG,CAAC,CAAC,SAAa,EAAE,EAAE,CAAC,OAAO,CAAI,UAAU,EAAE,SAAS,CAAC,CAAoB,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,YAAY,EAAE;QAC5C,KAAK,EAAE,UAAU;QACjB,QAAQ,EAAE,KAAK;QACf,UAAU,EAAE,IAAI;QAChB,YAAY,EAAE,KAAK;KACpB,CAAC,CAAC;IACH,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE;QACxC,KAAK,EAAE,CAAC,CACN,IAA4C,EAC5C,IAAoB,EACM,EAAE,CAAC,SAAS,CAAI,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC,CAAY;QAC/E,QAAQ,EAAE,KAAK;QACf,UAAU,EAAE,IAAI;QAChB,YAAY,EAAE,KAAK;KACpB,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,+EAA+E;AAC/E,SAAS;AACT,+EAA+E;AAE/E,KAAK,UAAU,SAAS,CACtB,UAA6B,EAC7B,IAA4C,EAC5C,IAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC;IAExD,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,CAAa,CAAC;IACjD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IACtE,CAAC;IAED,MAAM,KAAK,GAAoB,MAAM,aAAa,CAAC,YAAY,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;QAC3F,MAAM,MAAM,GAAG,MAAM,OAAO,CAAI,UAAU,EAAE,IAAI,CAAC,CAAC;QAClD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACnD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;AAC5D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CACvB,IAA4C;IAE5C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IAEnC,IAAI,KAAK,GAAmC,CAAC,EAAE,CAAC,CAAC;IACjD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAmC,EAAE,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QACD,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAuB,EACvB,KAAa,EACb,EAA0C;IAE1C,MAAM,OAAO,GAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,UAAU,MAAM;QACnB,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAE,EAAE,KAAK,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;IACtF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC"}
package/dist/index.d.ts ADDED
@@ -0,0 +1,30 @@
1
+ /**
2
+ * `@agentick/eval` — testing-shaped eval framework for Agentick (v1).
3
+ *
4
+ * ```ts
5
+ * import { defineEval } from "@agentick/eval";
6
+ *
7
+ * const billEval = defineEval<{ model?: string }>({
8
+ * description: "bill extraction against known fixtures",
9
+ * app: async (o) => createMyAgent({ model: o?.model ?? "google/gemini-2.5-flash" }),
10
+ * test: async (t) => {
11
+ * await t.send([fileBlock, { type: "text", text: "Extract the bill." }]);
12
+ * t.calledTool("submit_extraction");
13
+ * const submitted = t.lastToolCall("submit_extraction")?.input as Record<string, unknown>;
14
+ * t.expect("subtotal matches", Number(submitted?.SubTotal) === 187.5);
15
+ * },
16
+ * });
17
+ *
18
+ * // One run:
19
+ * const result = await billEval();
20
+ *
21
+ * // Same document + expectations across models, compared per cell:
22
+ * const sweep = await billEval.matrix({
23
+ * model: ["google/gemini-2.5-flash", "bedrock/us.amazon.nova-2-lite-v1:0"],
24
+ * });
25
+ * ```
26
+ */
27
+ export { defineEval, cartesian, mapConcurrent } from "./define-eval.js";
28
+ export { runEval } from "./runner.js";
29
+ export type { AppFactory, AssertionKind, AssertionResult, CallableEval, DefaultAppOverrides, EvalApp, EvalContext, EvalDefinition, EvalMessage, EvalResult, EvalSendHandle, EvalSendInput, EvalSession, EvalTest, MatrixCell, MatrixOptions, MatrixResult, ObservedToolCall, } from "./types.js";
30
+ //# sourceMappingURL=index.d.ts.map
package/dist/index.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,YAAY,EACV,UAAU,EACV,aAAa,EACb,eAAe,EACf,YAAY,EACZ,mBAAmB,EACnB,OAAO,EACP,WAAW,EACX,cAAc,EACd,WAAW,EACX,UAAU,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,QAAQ,EACR,UAAU,EACV,aAAa,EACb,YAAY,EACZ,gBAAgB,GACjB,MAAM,YAAY,CAAC"}
package/dist/index.js ADDED
@@ -0,0 +1,29 @@
1
+ /**
2
+ * `@agentick/eval` — testing-shaped eval framework for Agentick (v1).
3
+ *
4
+ * ```ts
5
+ * import { defineEval } from "@agentick/eval";
6
+ *
7
+ * const billEval = defineEval<{ model?: string }>({
8
+ * description: "bill extraction against known fixtures",
9
+ * app: async (o) => createMyAgent({ model: o?.model ?? "google/gemini-2.5-flash" }),
10
+ * test: async (t) => {
11
+ * await t.send([fileBlock, { type: "text", text: "Extract the bill." }]);
12
+ * t.calledTool("submit_extraction");
13
+ * const submitted = t.lastToolCall("submit_extraction")?.input as Record<string, unknown>;
14
+ * t.expect("subtotal matches", Number(submitted?.SubTotal) === 187.5);
15
+ * },
16
+ * });
17
+ *
18
+ * // One run:
19
+ * const result = await billEval();
20
+ *
21
+ * // Same document + expectations across models, compared per cell:
22
+ * const sweep = await billEval.matrix({
23
+ * model: ["google/gemini-2.5-flash", "bedrock/us.amazon.nova-2-lite-v1:0"],
24
+ * });
25
+ * ```
26
+ */
27
+ export { defineEval, cartesian, mapConcurrent } from "./define-eval.js";
28
+ export { runEval } from "./runner.js";
29
+ //# sourceMappingURL=index.js.map
package/dist/index.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC"}
package/dist/runner.d.ts ADDED
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Internal eval runner — drives one invocation of a `defineEval`-
3
+ * returned callable to completion.
4
+ *
5
+ * Flow per invocation:
6
+ * 1. Call `definition.app(overrides)` to construct a fresh app. The
7
+ * thunk owns option merging; the runner does none of its own.
8
+ * 2. For each `t.send`, open a fresh session, iterate the send
9
+ * handle's event stream to observe tool calls (correlating
10
+ * `tool_call` → `tool_result` by call id), and await the result.
11
+ * 3. Invoke `definition.test(t)`; assertions record into a ledger.
12
+ * 4. Close sessions; return the aggregated {@link EvalResult}.
13
+ *
14
+ * Assertions don't throw — they record. Adopters who want fail-fast
15
+ * check `result.passed` and decide whether to continue.
16
+ */
17
+ import type { EvalDefinition, EvalResult } from "./types.js";
18
+ export declare function runEval<O>(definition: EvalDefinition<O>, overrides: O | undefined): Promise<EvalResult>;
19
+ //# sourceMappingURL=runner.d.ts.map
package/dist/runner.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAIV,cAAc,EACd,UAAU,EAIX,MAAM,YAAY,CAAC;AAsBpB,wBAAsB,OAAO,CAAC,CAAC,EAC7B,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,EAC7B,SAAS,EAAE,CAAC,GAAG,SAAS,GACvB,OAAO,CAAC,UAAU,CAAC,CAuLrB"}
package/dist/runner.js ADDED
@@ -0,0 +1,207 @@
1
+ /**
2
+ * Internal eval runner — drives one invocation of a `defineEval`-
3
+ * returned callable to completion.
4
+ *
5
+ * Flow per invocation:
6
+ * 1. Call `definition.app(overrides)` to construct a fresh app. The
7
+ * thunk owns option merging; the runner does none of its own.
8
+ * 2. For each `t.send`, open a fresh session, iterate the send
9
+ * handle's event stream to observe tool calls (correlating
10
+ * `tool_call` → `tool_result` by call id), and await the result.
11
+ * 3. Invoke `definition.test(t)`; assertions record into a ledger.
12
+ * 4. Close sessions; return the aggregated {@link EvalResult}.
13
+ *
14
+ * Assertions don't throw — they record. Adopters who want fail-fast
15
+ * check `result.passed` and decide whether to continue.
16
+ */
17
+ // ============================================================================
18
+ // Deep equality (no deps — small structural comparison)
19
+ // ============================================================================
20
+ function isEqual(a, b) {
21
+ if (Object.is(a, b))
22
+ return true;
23
+ if (typeof a !== "object" || typeof b !== "object" || a === null || b === null)
24
+ return false;
25
+ if (Array.isArray(a) !== Array.isArray(b))
26
+ return false;
27
+ const ak = Object.keys(a);
28
+ const bk = Object.keys(b);
29
+ if (ak.length !== bk.length)
30
+ return false;
31
+ return ak.every((k) => isEqual(a[k], b[k]));
32
+ }
33
+ // ============================================================================
34
+ // Runner
35
+ // ============================================================================
36
+ export async function runEval(definition, overrides) {
37
+ const started = Date.now();
38
+ const app = await definition.app(overrides);
39
+ const toolCalls = [];
40
+ const assertions = [];
41
+ const openSessions = [];
42
+ let lastSendErrored = false;
43
+ let lastSendCompleted = false;
44
+ let evalError;
45
+ const record = (a) => assertions.push(a);
46
+ const t = {
47
+ app,
48
+ async send(input) {
49
+ const session = await app.session();
50
+ openSessions.push(session);
51
+ lastSendErrored = false;
52
+ lastSendCompleted = false;
53
+ const content = typeof input === "string" ? [{ type: "text", text: input }] : input;
54
+ // session.send returns a thenable; the resolved handle carries the
55
+ // event stream (`.events`) alongside `.result`.
56
+ const handle = await session.send({ messages: [{ role: "user", content }] });
57
+ // Observe the event stream when available: correlate tool_call →
58
+ // tool_result by call id, collect final text. Fall back to just
59
+ // awaiting the result for apps whose handles don't stream.
60
+ let finalText = "";
61
+ const pending = new Map();
62
+ if (handle.events) {
63
+ for await (const ev of handle.events) {
64
+ const type = ev.type;
65
+ if (type === "tool_call") {
66
+ const id = (ev.callId ?? ev.id);
67
+ pending.set(id, { name: ev.name, input: ev.input });
68
+ }
69
+ else if (type === "tool_result") {
70
+ const id = (ev.callId ?? ev.id);
71
+ const call = pending.get(id);
72
+ if (call) {
73
+ pending.delete(id);
74
+ toolCalls.push({
75
+ name: call.name,
76
+ input: call.input,
77
+ outcome: ev.isError === true ? "failed" : "succeeded",
78
+ result: ev.result ?? ev.content,
79
+ at: Date.now(),
80
+ });
81
+ }
82
+ }
83
+ else if (type === "message" && ev.message) {
84
+ const message = ev.message;
85
+ if (message.role === "assistant" && Array.isArray(message.content)) {
86
+ for (const block of message.content) {
87
+ if (block.type === "text" && typeof block.text === "string") {
88
+ finalText = block.text;
89
+ }
90
+ }
91
+ }
92
+ }
93
+ else if (type === "error") {
94
+ lastSendErrored = true;
95
+ }
96
+ }
97
+ }
98
+ await handle.result;
99
+ // Any tool_call that never saw a result still counts as observed.
100
+ for (const [, call] of pending) {
101
+ toolCalls.push({
102
+ name: call.name,
103
+ input: call.input,
104
+ outcome: "succeeded",
105
+ at: Date.now(),
106
+ });
107
+ }
108
+ lastSendCompleted = !lastSendErrored;
109
+ return finalText;
110
+ },
111
+ completed() {
112
+ record({
113
+ kind: "completed",
114
+ passed: lastSendCompleted,
115
+ message: lastSendCompleted
116
+ ? "run completed"
117
+ : lastSendErrored
118
+ ? "run emitted an error event"
119
+ : "no send has completed",
120
+ });
121
+ },
122
+ calledTool(name, opts) {
123
+ const matches = toolCalls.filter((c) => {
124
+ if (c.name !== name)
125
+ return false;
126
+ if (opts?.input !== undefined && !isEqual(c.input, opts.input))
127
+ return false;
128
+ if (opts?.isError === true && c.outcome !== "failed")
129
+ return false;
130
+ if (opts?.isError === false && c.outcome !== "succeeded")
131
+ return false;
132
+ return true;
133
+ });
134
+ record({
135
+ kind: "calledTool",
136
+ passed: matches.length > 0,
137
+ message: matches.length > 0
138
+ ? `tool "${name}" was called`
139
+ : `tool "${name}" was not called${opts?.input !== undefined ? " with the expected input" : ""} (observed: ${toolCalls.map((c) => c.name).join(", ") || "none"})`,
140
+ details: { name, observed: toolCalls.length },
141
+ });
142
+ },
143
+ notCalledTool(name) {
144
+ const called = toolCalls.some((c) => c.name === name);
145
+ record({
146
+ kind: "notCalledTool",
147
+ passed: !called,
148
+ message: called ? `tool "${name}" WAS called` : `tool "${name}" was not called`,
149
+ details: { name },
150
+ });
151
+ },
152
+ noFailedActions() {
153
+ const failed = toolCalls.filter((c) => c.outcome === "failed");
154
+ record({
155
+ kind: "noFailedActions",
156
+ passed: failed.length === 0,
157
+ message: failed.length === 0
158
+ ? "no failed tool calls"
159
+ : `${failed.length} tool call(s) failed: ${failed.map((c) => c.name).join(", ")}`,
160
+ details: failed,
161
+ });
162
+ },
163
+ expect(name, passed, opts) {
164
+ record({
165
+ kind: "expect",
166
+ name,
167
+ passed,
168
+ message: opts?.message ?? (passed ? `${name} passed` : `${name} failed`),
169
+ details: opts?.details,
170
+ });
171
+ },
172
+ lastToolCall(name) {
173
+ for (let i = toolCalls.length - 1; i >= 0; i--) {
174
+ if (toolCalls[i].name === name)
175
+ return toolCalls[i];
176
+ }
177
+ return undefined;
178
+ },
179
+ };
180
+ try {
181
+ await definition.test(t);
182
+ }
183
+ catch (err) {
184
+ const e = err;
185
+ evalError = { name: e?.name ?? "Error", message: e?.message ?? String(err) };
186
+ }
187
+ finally {
188
+ for (const session of openSessions) {
189
+ try {
190
+ session.close();
191
+ }
192
+ catch {
193
+ // closing is best-effort
194
+ }
195
+ }
196
+ }
197
+ const passed = evalError === undefined && assertions.every((a) => a.passed);
198
+ return {
199
+ description: definition.description,
200
+ passed,
201
+ assertions,
202
+ toolCalls,
203
+ elapsedMs: Date.now() - started,
204
+ ...(evalError ? { error: evalError } : {}),
205
+ };
206
+ }
207
+ //# sourceMappingURL=runner.js.map
package/dist/runner.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAaH,+EAA+E;AAC/E,wDAAwD;AACxD,+EAA+E;AAE/E,SAAS,OAAO,CAAC,CAAU,EAAE,CAAU;IACrC,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACjC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC7F,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IACxD,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAW,CAAC,CAAC;IACpC,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAW,CAAC,CAAC;IACpC,IAAI,EAAE,CAAC,MAAM,KAAK,EAAE,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1C,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CACpB,OAAO,CAAE,CAA6B,CAAC,CAAC,CAAC,EAAG,CAA6B,CAAC,CAAC,CAAC,CAAC,CAC9E,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,SAAS;AACT,+EAA+E;AAE/E,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,UAA6B,EAC7B,SAAwB;IAExB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE3B,MAAM,GAAG,GAAY,MAAM,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAErD,MAAM,SAAS,GAAuB,EAAE,CAAC;IACzC,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,MAAM,YAAY,GAAkB,EAAE,CAAC;IACvC,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,IAAI,SAAwD,CAAC;IAE7D,MAAM,MAAM,GAAG,CAAC,CAAkB,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE1D,MAAM,CAAC,GAAgB;QACrB,GAAG;QAEH,KAAK,CAAC,IAAI,CAAC,KAAoB;YAC7B,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,OAAO,EAAE,CAAC;YACpC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3B,eAAe,GAAG,KAAK,CAAC;YACxB,iBAAiB,GAAG,KAAK,CAAC;YAE1B,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YACpF,mEAAmE;YACnE,gDAAgD;YAChD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;YAE7E,iEAAiE;YACjE,gEAAgE;YAChE,2DAA2D;YAC3D,IAAI,SAAS,GAAG,EAAE,CAAC;YACnB,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4C,CAAC;YAEpE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAClB,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;oBACrC,MAAM,IAAI,GAAG,EAAE,CAAC,IAAc,CAAC;oBAC/B,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;wBACzB,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAW,CAAC;wBAC1C,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,IAAc,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;oBAChE,CAAC;yBAAM,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;wBAClC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAW,CAAC;wBAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC7B,IAAI,IAAI,EAAE,CAAC;4BACT,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;4BACnB,SAAS,CAAC,IAAI,CAAC;gCACb,IAAI,EAAE,IAAI,CAAC,IAAI;gCACf,KAAK,EAAE,IAAI,CAAC,KAAK;gCACjB,OAAO,EAAE,EAAE,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW;gCACrD,MAAM,EAAE,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,OAAO;gCAC/B,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;6BACf,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,IAAI,IAAI,KAAK,SAAS,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;wBAC5C,MAAM,OAAO,GAAG,EAAE,CAAC,OAGlB,CAAC;wBACF,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;4BACnE,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gCACpC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oCAC5D,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC;gCACzB,CAAC;4BACH,CAAC;wBACH,CAAC;oBACH,CAAC;yBAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;wBAC5B,eAAe,GAAG,IAAI,CAAC;oBACzB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,MAAM,MAAM,CAAC,MAAM,CAAC;YACpB,kEAAkE;YAClE,KAAK,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;gBAC/B,SAAS,CAAC,IAAI,CAAC;oBACb,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,OAAO,EAAE,WAAW;oBACpB,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;iBACf,CAAC,CAAC;YACL,CAAC;YACD,iBAAiB,GAAG,CAAC,eAAe,CAAC;YACrC,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,SAAS;YACP,MAAM,CAAC;gBACL,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,iBAAiB;gBACzB,OAAO,EAAE,iBAAiB;oBACxB,CAAC,CAAC,eAAe;oBACjB,CAAC,CAAC,eAAe;wBACf,CAAC,CAAC,4BAA4B;wBAC9B,CAAC,CAAC,uBAAuB;aAC9B,CAAC,CAAC;QACL,CAAC;QAED,UAAU,CAAC,IAAI,EAAE,IAAI;YACnB,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACrC,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI;oBAAE,OAAO,KAAK,CAAC;gBAClC,IAAI,IAAI,EAAE,KAAK,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC;oBAAE,OAAO,KAAK,CAAC;gBAC7E,IAAI,IAAI,EAAE,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,OAAO,KAAK,QAAQ;oBAAE,OAAO,KAAK,CAAC;gBACnE,IAAI,IAAI,EAAE,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,OAAO,KAAK,WAAW;oBAAE,OAAO,KAAK,CAAC;gBACvE,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;YACH,MAAM,CAAC;gBACL,IAAI,EAAE,YAAY;gBAClB,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;gBAC1B,OAAO,EACL,OAAO,CAAC,MAAM,GAAG,CAAC;oBAChB,CAAC,CAAC,SAAS,IAAI,cAAc;oBAC7B,CAAC,CAAC,SAAS,IAAI,mBAAmB,IAAI,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,EAAE,eACzF,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAC7C,GAAG;gBACT,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,MAAM,EAAE;aAC9C,CAAC,CAAC;QACL,CAAC;QAED,aAAa,CAAC,IAAI;YAChB,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;YACtD,MAAM,CAAC;gBACL,IAAI,EAAE,eAAe;gBACrB,MAAM,EAAE,CAAC,MAAM;gBACf,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,cAAc,CAAC,CAAC,CAAC,SAAS,IAAI,kBAAkB;gBAC/E,OAAO,EAAE,EAAE,IAAI,EAAE;aAClB,CAAC,CAAC;QACL,CAAC;QAED,eAAe;YACb,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;YAC/D,MAAM,CAAC;gBACL,IAAI,EAAE,iBAAiB;gBACvB,MAAM,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;gBAC3B,OAAO,EACL,MAAM,CAAC,MAAM,KAAK,CAAC;oBACjB,CAAC,CAAC,sBAAsB;oBACxB,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,yBAAyB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACrF,OAAO,EAAE,MAAM;aAChB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI;YACvB,MAAM,CAAC;gBACL,IAAI,EAAE,QAAQ;gBACd,IAAI;gBACJ,MAAM;gBACN,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,SAAS,CAAC;gBACxE,OAAO,EAAE,IAAI,EAAE,OAAO;aACvB,CAAC,CAAC;QACL,CAAC;QAED,YAAY,CAAC,IAAI;YACf,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/C,IAAI,SAAS,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,IAAI;oBAAE,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC;YACvD,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC;KACF,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,GAAG,GAAY,CAAC;QACvB,SAAS,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IAC/E,CAAC;YAAS,CAAC;QACT,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,CAAC;YAAC,MAAM,CAAC;gBACP,yBAAyB;YAC3B,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,SAAS,KAAK,SAAS,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5E,OAAO;QACL,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,MAAM;QACN,UAAU;QACV,SAAS;QACT,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;QAC/B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3C,CAAC;AACJ,CAAC"}
package/dist/types.d.ts ADDED
@@ -0,0 +1,199 @@
1
+ /**
2
+ * Public types for `@agentick/eval` (v1). The `defineEval(...)` factory
3
+ * is the entry point; everything else is the surface the `t` (test
4
+ * context) object exposes to assertion bodies.
5
+ *
6
+ * Shape: `{ description, app, test }`. `app` is a thunk that constructs
7
+ * a FRESH agentick app (the value returned by `createApp(...)`) per eval
8
+ * invocation — every `await myEval()` gets its own app, so session state
9
+ * doesn't leak between runs or matrix-axis cells. The thunk receives the
10
+ * per-invocation overrides (`O`) and decides how to fold them in; the
11
+ * eval package does no option merging itself.
12
+ *
13
+ * Ported from `@agentick/eval-next` (packages-next/eval) with two v1
14
+ * extensions:
15
+ * - `t.send` accepts content blocks, not just a string — evals over
16
+ * documents/images need to attach media to the user message.
17
+ * - `t.expect(name, passed, opts?)` records a custom assertion —
18
+ * the seam for comparing an output against an expected fixture
19
+ * (field-level extraction scoring, etc.) without the package
20
+ * prescribing a scorer framework.
21
+ */
22
+ /**
23
+ * The minimal surface the runner needs from a v1 agentick app (the
24
+ * value `createApp(...)` returns). Structural on purpose — the eval
25
+ * package works with anything session-shaped.
26
+ */
27
+ export interface EvalApp {
28
+ session(): Promise<EvalSession>;
29
+ }
30
+ export interface EvalSession {
31
+ /**
32
+ * v1 session.send returns a thenable whose resolution is the full
33
+ * handle (with `.events`); the runner awaits it either way.
34
+ */
35
+ send(input: {
36
+ messages: EvalMessage[];
37
+ }): EvalSendHandle | PromiseLike<EvalSendHandle>;
38
+ close(): void;
39
+ }
40
+ export interface EvalMessage {
41
+ role: string;
42
+ content: Array<Record<string, unknown>>;
43
+ }
44
+ /** The handle `session.send` returns: awaitable result + event stream. */
45
+ export interface EvalSendHandle {
46
+ result: Promise<unknown>;
47
+ events?: AsyncIterable<Record<string, unknown>>;
48
+ abort?(reason?: string): void;
49
+ }
50
+ /**
51
+ * Default override shape — a free-form record. Adopters who want a
52
+ * domain-shaped override (e.g. `{ model: string }`) parameterize `O`
53
+ * themselves; the factory owns interpreting it.
54
+ */
55
+ export type DefaultAppOverrides = Record<string, unknown>;
56
+ /**
57
+ * Thunk that constructs a fresh app for one eval invocation. Receives
58
+ * the per-invocation overrides; the thunk decides how to compose them
59
+ * with its own defaults. Returning a fresh app on each call is
60
+ * required — the runner closes sessions after the body finishes.
61
+ */
62
+ export type AppFactory<O = DefaultAppOverrides> = (overrides?: O) => Promise<EvalApp> | EvalApp;
63
+ /** The eval body. Assertions against `t` record into the result. */
64
+ export type EvalTest = (t: EvalContext) => Promise<void> | void;
65
+ export interface EvalDefinition<O = DefaultAppOverrides> {
66
+ /** Human-readable; surfaces in result reports. */
67
+ readonly description: string;
68
+ /** Factory constructing a fresh app per invocation. */
69
+ readonly app: AppFactory<O>;
70
+ /** The eval body. */
71
+ readonly test: EvalTest;
72
+ }
73
+ /**
74
+ * The returned callable. `await myEval()` runs with the factory's
75
+ * defaults; `await myEval(overrides)` passes `overrides` through to
76
+ * the factory unchanged.
77
+ *
78
+ * `.matrix(axes, opts?)` runs the cartesian product of axis values,
79
+ * returning one `MatrixCell` per combination — the multi-model /
80
+ * multi-fixture comparison surface.
81
+ */
82
+ export interface CallableEval<O = DefaultAppOverrides> {
83
+ (overrides?: O): Promise<EvalResult>;
84
+ /** The original definition — exposed for tooling. */
85
+ readonly definition: EvalDefinition<O>;
86
+ /**
87
+ * Run the cartesian product of `axes`. Each cell merges into one
88
+ * `O` (one value per axis) and runs the eval once.
89
+ *
90
+ * - Empty axes (`{}`) → 1 cell, equivalent to `myEval()`
91
+ * - Any axis with `[]` → 0 cells (mathematical product)
92
+ *
93
+ * `opts.concurrency` caps concurrent runs (default `1` — sequential
94
+ * — to avoid rate-limit blowups on real-model evals).
95
+ */
96
+ matrix<K extends keyof O & string>(axes: {
97
+ readonly [A in K]?: ReadonlyArray<O[A]>;
98
+ }, opts?: MatrixOptions): Promise<MatrixResult<O>>;
99
+ }
100
+ export interface MatrixOptions {
101
+ /** Max concurrent eval runs. Default `1` (sequential). */
102
+ readonly concurrency?: number;
103
+ }
104
+ export interface MatrixCell<O = DefaultAppOverrides> {
105
+ /** The resolved override record handed to the factory for this cell. */
106
+ readonly axes: O;
107
+ readonly result: EvalResult;
108
+ }
109
+ export interface MatrixResult<O = DefaultAppOverrides> {
110
+ readonly cells: ReadonlyArray<MatrixCell<O>>;
111
+ /** True iff every cell's `result.passed` is true. */
112
+ readonly passed: boolean;
113
+ /** Wall-clock duration (ms) of the whole sweep. */
114
+ readonly elapsedMs: number;
115
+ }
116
+ export type AssertionKind = "completed" | "calledTool" | "notCalledTool" | "noFailedActions" | "expect";
117
+ export interface AssertionResult {
118
+ readonly kind: AssertionKind;
119
+ /** For `expect` assertions, the adopter-supplied name. */
120
+ readonly name?: string;
121
+ readonly passed: boolean;
122
+ /** Human-readable explanation — surfaced on failure for debugging. */
123
+ readonly message: string;
124
+ /** Structured details — assertion-specific. */
125
+ readonly details?: unknown;
126
+ }
127
+ export interface EvalResult {
128
+ readonly description: string;
129
+ readonly passed: boolean;
130
+ readonly assertions: ReadonlyArray<AssertionResult>;
131
+ /** Every tool call observed during this invocation. */
132
+ readonly toolCalls: ReadonlyArray<ObservedToolCall>;
133
+ /** Wall-clock duration (ms). */
134
+ readonly elapsedMs: number;
135
+ /** Thrown error, if the eval body threw. */
136
+ readonly error?: {
137
+ readonly name: string;
138
+ readonly message: string;
139
+ };
140
+ }
141
+ export interface ObservedToolCall {
142
+ readonly name: string;
143
+ readonly input: unknown;
144
+ readonly outcome: "succeeded" | "failed";
145
+ readonly result?: unknown;
146
+ /** Timestamp the call was observed. */
147
+ readonly at: number;
148
+ }
149
+ /** Content a `t.send` accepts: plain text or full content blocks. */
150
+ export type EvalSendInput = string | Array<Record<string, unknown>>;
151
+ /**
152
+ * The argument passed to `test(t)`. Drives the agent, asserts on what
153
+ * happened. Assertions are recorded into the result, not thrown, so
154
+ * multiple failures show up in one report.
155
+ */
156
+ export interface EvalContext {
157
+ /**
158
+ * The app constructed by `definition.app(overrides)` for this
159
+ * invocation — the escape hatch for primitives `t` doesn't sugar.
160
+ */
161
+ readonly app: EvalApp;
162
+ /**
163
+ * Drive the agent: creates a fresh session, sends one user message
164
+ * (string or content blocks — attach documents/images as blocks),
165
+ * awaits completion, records observed tool calls, and returns the
166
+ * final response text.
167
+ */
168
+ send(input: EvalSendInput): Promise<string>;
169
+ /** Assert the most-recent `t.send` completed without an error event. */
170
+ completed(): void;
171
+ /**
172
+ * Assert a tool was called during the run. `input` deep-equals the
173
+ * call's input when provided; `isError` pins the outcome.
174
+ */
175
+ calledTool(name: string, opts?: {
176
+ readonly input?: unknown;
177
+ readonly isError?: boolean;
178
+ }): void;
179
+ /** Assert a tool was NOT called (safety evals). */
180
+ notCalledTool(name: string): void;
181
+ /** Assert no observed tool call failed. */
182
+ noFailedActions(): void;
183
+ /**
184
+ * Record a custom assertion — the seam for comparing outputs against
185
+ * expected fixtures (e.g. field-level extraction scoring). `details`
186
+ * lands on the assertion result for reporting.
187
+ */
188
+ expect(name: string, passed: boolean, opts?: {
189
+ message?: string;
190
+ details?: unknown;
191
+ }): void;
192
+ /**
193
+ * The most recent tool call observed with the given name (from any
194
+ * `t.send` in this invocation) — convenient for reading a submit
195
+ * tool's payload to score against an expected fixture.
196
+ */
197
+ lastToolCall(name: string): ObservedToolCall | undefined;
198
+ }
199
+ //# sourceMappingURL=types.d.ts.map
package/dist/types.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAMH;;;;GAIG;AACH,MAAM,WAAW,OAAO;IACtB,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,IAAI,CAAC,KAAK,EAAE;QAAE,QAAQ,EAAE,WAAW,EAAE,CAAA;KAAE,GAAG,cAAc,GAAG,WAAW,CAAC,cAAc,CAAC,CAAC;IACvF,KAAK,IAAI,IAAI,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACzC;AAED,0EAA0E;AAC1E,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACzB,MAAM,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG,mBAAmB,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;AAMhG,oEAAoE;AACpE,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAEhE,MAAM,WAAW,cAAc,CAAC,CAAC,GAAG,mBAAmB;IACrD,kDAAkD;IAClD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,uDAAuD;IACvD,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IAC5B,qBAAqB;IACrB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;CACzB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,mBAAmB;IACnD,CAAC,SAAS,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACrC,qDAAqD;IACrD,QAAQ,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC;IACvC;;;;;;;;;OASG;IACH,MAAM,CAAC,CAAC,SAAS,MAAM,CAAC,GAAG,MAAM,EAC/B,IAAI,EAAE;QAAE,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KAAE,EACjD,IAAI,CAAC,EAAE,aAAa,GACnB,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;CAC7B;AAMD,MAAM,WAAW,aAAa;IAC5B,0DAA0D;IAC1D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,UAAU,CAAC,CAAC,GAAG,mBAAmB;IACjD,wEAAwE;IACxE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACjB,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,mBAAmB;IACnD,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,qDAAqD;IACrD,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,mDAAmD;IACnD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAMD,MAAM,MAAM,aAAa,GACrB,WAAW,GACX,YAAY,GACZ,eAAe,GACf,iBAAiB,GACjB,QAAQ,CAAC;AAEb,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,sEAAsE;IACtE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+CAA+C;IAC/C,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;IACpD,uDAAuD;IACvD,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC,gBAAgB,CAAC,CAAC;IACpD,gCAAgC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,4CAA4C;IAC5C,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CACtE;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,WAAW,GAAG,QAAQ,CAAC;IACzC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,uCAAuC;IACvC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;CACrB;AAMD,qEAAqE;AACrE,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAEpE;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC;IAEtB;;;;;OAKG;IACH,IAAI,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE5C,wEAAwE;IACxE,SAAS,IAAI,IAAI,CAAC;IAElB;;;OAGG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAEhG,mDAAmD;IACnD,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC,2CAA2C;IAC3C,eAAe,IAAI,IAAI,CAAC;IAExB;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAE5F;;;;OAIG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS,CAAC;CAC1D"}
package/dist/types.js ADDED
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Public types for `@agentick/eval` (v1). The `defineEval(...)` factory
3
+ * is the entry point; everything else is the surface the `t` (test
4
+ * context) object exposes to assertion bodies.
5
+ *
6
+ * Shape: `{ description, app, test }`. `app` is a thunk that constructs
7
+ * a FRESH agentick app (the value returned by `createApp(...)`) per eval
8
+ * invocation — every `await myEval()` gets its own app, so session state
9
+ * doesn't leak between runs or matrix-axis cells. The thunk receives the
10
+ * per-invocation overrides (`O`) and decides how to fold them in; the
11
+ * eval package does no option merging itself.
12
+ *
13
+ * Ported from `@agentick/eval-next` (packages-next/eval) with two v1
14
+ * extensions:
15
+ * - `t.send` accepts content blocks, not just a string — evals over
16
+ * documents/images need to attach media to the user message.
17
+ * - `t.expect(name, passed, opts?)` records a custom assertion —
18
+ * the seam for comparing an output against an expected fixture
19
+ * (field-level extraction scoring, etc.) without the package
20
+ * prescribing a scorer framework.
21
+ */
22
+ export {};
23
+ //# sourceMappingURL=types.js.map
package/dist/types.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG"}
package/package.json ADDED
@@ -0,0 +1,61 @@
1
+ {
2
+ "name": "@agentick/eval",
3
+ "version": "0.15.2",
4
+ "description": "Testing-shaped eval framework for Agentick. defineEval(...) returns a callable with runtime parameterization; .matrix(axes) runs cartesian parameter sweeps \u2014 e.g. the same document/expected pairs across multiple models \u2014 and reports per-cell results.",
5
+ "keywords": [
6
+ "agent",
7
+ "agentick",
8
+ "ai",
9
+ "eval",
10
+ "evaluation",
11
+ "testing"
12
+ ],
13
+ "license": "MIT",
14
+ "author": "Ryan Lindgren",
15
+ "repository": {
16
+ "type": "git",
17
+ "url": "git+https://github.com/agenticklabs/agentick.git",
18
+ "directory": "packages/eval"
19
+ },
20
+ "files": [
21
+ "dist"
22
+ ],
23
+ "type": "module",
24
+ "main": "src/index.ts",
25
+ "exports": {
26
+ ".": {
27
+ "types": "./src/index.ts",
28
+ "import": "./src/index.ts",
29
+ "default": "./src/index.ts"
30
+ }
31
+ },
32
+ "publishConfig": {
33
+ "access": "public",
34
+ "main": "dist/index.js",
35
+ "types": "dist/index.d.ts",
36
+ "exports": {
37
+ ".": {
38
+ "types": "./dist/index.d.ts",
39
+ "import": "./dist/index.js",
40
+ "default": "./dist/index.js"
41
+ }
42
+ }
43
+ },
44
+ "scripts": {
45
+ "build": "tsc -p tsconfig.build.json",
46
+ "test": "echo \"Tests run from workspace root\"",
47
+ "typecheck": "tsc -p tsconfig.build.json --noEmit",
48
+ "lint": "oxlint src/",
49
+ "format:check": "oxfmt --check src/",
50
+ "clean": "rm -rf dist tsconfig.build.tsbuildinfo",
51
+ "prepublishOnly": "pnpm build",
52
+ "dev": "tsc --watch"
53
+ },
54
+ "dependencies": {
55
+ "@agentick/core": "workspace:*"
56
+ },
57
+ "devDependencies": {
58
+ "react": "^19.2.4",
59
+ "zod": "^4.3.6"
60
+ }
61
+ }
package/src/index.ts ADDED
@@ -0,0 +1,49 @@
1
+ /**
2
+ * `@agentick/eval` — testing-shaped eval framework for Agentick (v1).
3
+ *
4
+ * ```ts
5
+ * import { defineEval } from "@agentick/eval";
6
+ *
7
+ * const billEval = defineEval<{ model?: string }>({
8
+ * description: "bill extraction against known fixtures",
9
+ * app: async (o) => createMyAgent({ model: o?.model ?? "google/gemini-2.5-flash" }),
10
+ * test: async (t) => {
11
+ * await t.send([fileBlock, { type: "text", text: "Extract the bill." }]);
12
+ * t.calledTool("submit_extraction");
13
+ * const submitted = t.lastToolCall("submit_extraction")?.input as Record<string, unknown>;
14
+ * t.expect("subtotal matches", Number(submitted?.SubTotal) === 187.5);
15
+ * },
16
+ * });
17
+ *
18
+ * // One run:
19
+ * const result = await billEval();
20
+ *
21
+ * // Same document + expectations across models, compared per cell:
22
+ * const sweep = await billEval.matrix({
23
+ * model: ["google/gemini-2.5-flash", "bedrock/us.amazon.nova-2-lite-v1:0"],
24
+ * });
25
+ * ```
26
+ */
27
+
28
+ export { defineEval, cartesian, mapConcurrent } from "./define-eval.js";
29
+ export { runEval } from "./runner.js";
30
+ export type {
31
+ AppFactory,
32
+ AssertionKind,
33
+ AssertionResult,
34
+ CallableEval,
35
+ DefaultAppOverrides,
36
+ EvalApp,
37
+ EvalContext,
38
+ EvalDefinition,
39
+ EvalMessage,
40
+ EvalResult,
41
+ EvalSendHandle,
42
+ EvalSendInput,
43
+ EvalSession,
44
+ EvalTest,
45
+ MatrixCell,
46
+ MatrixOptions,
47
+ MatrixResult,
48
+ ObservedToolCall,
49
+ } from "./types.js";