@vitest-evals/harness-pi-ai 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -6
- package/dist/index.d.mts +48 -49
- package/dist/index.d.ts +48 -49
- package/dist/index.js +2 -8
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2 -8
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -17,7 +17,6 @@ import {
|
|
|
17
17
|
createJudge,
|
|
18
18
|
describeEval,
|
|
19
19
|
toolCalls,
|
|
20
|
-
type JudgeContext,
|
|
21
20
|
} from "vitest-evals";
|
|
22
21
|
|
|
23
22
|
const harness = piAiHarness({
|
|
@@ -79,16 +78,15 @@ If the agent already implements `run(input, runtime)`, you can omit `run` and
|
|
|
79
78
|
the harness will call that method automatically.
|
|
80
79
|
|
|
81
80
|
`agent` can be an object or a per-run factory. The factory receives the per-run
|
|
82
|
-
input
|
|
83
|
-
|
|
84
|
-
|
|
81
|
+
input before the adapter infers and instruments native agent tools. Use that
|
|
82
|
+
for scenario-specific instructions or tool closures without leaving the default
|
|
83
|
+
replay path:
|
|
85
84
|
|
|
86
85
|
```ts
|
|
87
86
|
const harness = piAiHarness({
|
|
88
|
-
agent: ({ input
|
|
87
|
+
agent: ({ input }) =>
|
|
89
88
|
createRefundAgent({
|
|
90
89
|
instructions: buildInstructions(input),
|
|
91
|
-
metadata: context.metadata,
|
|
92
90
|
}),
|
|
93
91
|
toolReplay: {
|
|
94
92
|
lookupInvoice: true,
|
package/dist/index.d.mts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import * as vitest_evals_judges from 'vitest-evals/judges';
|
|
2
|
-
import { NormalizedMessage, JsonValue,
|
|
2
|
+
import { NormalizedMessage, JsonValue, HarnessContext, HarnessRun, Harness } from 'vitest-evals/harness';
|
|
3
3
|
import { ReplayMode, ToolRecording, ToolReplayConfig } from 'vitest-evals/replay';
|
|
4
4
|
import { Api, Model, SimpleStreamOptions } from '@mariozechner/pi-ai';
|
|
5
5
|
|
|
6
6
|
type MaybePromise<T> = T | Promise<T>;
|
|
7
7
|
type JsonOutput<TValue> = [TValue] extends [JsonValue | undefined] ? TValue : undefined;
|
|
8
8
|
type ResultFieldOutput<TResult, TKey extends string> = TKey extends keyof TResult ? Record<string, never> extends Pick<TResult, TKey> ? JsonOutput<TResult[TKey]> | undefined : JsonOutput<TResult[TKey]> : undefined;
|
|
9
|
-
type AgentSource<TAgent, TInput = string
|
|
10
|
-
type AnyPiAiToolset<TInput = string
|
|
9
|
+
type AgentSource<TAgent, TInput = string> = TAgent | ((args: PiAiCreateAgentArgs<TInput>) => MaybePromise<TAgent>);
|
|
10
|
+
type AnyPiAiToolset<TInput = string> = Record<string, PiAiToolDefinition<any, any, TInput>>;
|
|
11
11
|
/**
|
|
12
12
|
* Configuration for adapting a Pi AI model into a judge harness.
|
|
13
13
|
*
|
|
@@ -56,10 +56,10 @@ interface PiAiJudgeHarnessConfig<TApi extends Api = Api> {
|
|
|
56
56
|
* ```
|
|
57
57
|
*/
|
|
58
58
|
declare function piAiJudgeHarness<TApi extends Api>(config: PiAiJudgeHarnessConfig<TApi>): vitest_evals_judges.JudgeHarness;
|
|
59
|
-
type InferredPiAiToolset<TInput = string
|
|
59
|
+
type InferredPiAiToolset<TInput = string> = Record<string, PiAiToolDefinition<Record<string, JsonValue>, JsonValue, TInput>>;
|
|
60
60
|
type PiAiResultOutput<TResult> = TResult extends HarnessRun<infer TOutput> ? TOutput : "output" extends keyof TResult ? ResultFieldOutput<TResult, "output"> : undefined;
|
|
61
|
-
type PiAiAgentResult<TAgent, TInput,
|
|
62
|
-
run: (input: TInput, runtime: PiAiRuntime<TTools, TInput
|
|
61
|
+
type PiAiAgentResult<TAgent, TInput, TTools extends PiAiToolset<TInput>> = TAgent extends {
|
|
62
|
+
run: (input: TInput, runtime: PiAiRuntime<TTools, TInput>) => MaybePromise<infer TResult>;
|
|
63
63
|
} ? Awaited<TResult> : unknown;
|
|
64
64
|
/** Replay mode alias used by the Pi AI harness package. */
|
|
65
65
|
type PiAiReplayMode = ReplayMode;
|
|
@@ -72,31 +72,30 @@ interface PiAiEventSink {
|
|
|
72
72
|
tool: (name: string, content: JsonValue, metadata?: Record<string, JsonValue>) => void;
|
|
73
73
|
}
|
|
74
74
|
/** Context passed to instrumented Pi AI tool executions. */
|
|
75
|
-
interface PiAiToolContext<TInput = string
|
|
75
|
+
interface PiAiToolContext<TInput = string> {
|
|
76
76
|
input: TInput;
|
|
77
|
-
metadata: HarnessContext<TMetadata>["metadata"];
|
|
78
77
|
signal?: AbortSignal;
|
|
79
|
-
setArtifact: HarnessContext
|
|
78
|
+
setArtifact: HarnessContext["setArtifact"];
|
|
80
79
|
}
|
|
81
80
|
/** Tool replay recording shape for Pi AI tools. */
|
|
82
81
|
type PiAiToolRecording<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue> = ToolRecording<TArgs, TResult>;
|
|
83
82
|
/** Replay configuration for a Pi AI tool. */
|
|
84
|
-
type PiAiToolReplayConfig<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string
|
|
83
|
+
type PiAiToolReplayConfig<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string> = ToolReplayConfig<TArgs, TResult, PiAiToolContext<TInput>>;
|
|
85
84
|
/** Replay policy for one Pi AI tool. */
|
|
86
|
-
type PiAiToolReplayPolicy<TInput = string
|
|
85
|
+
type PiAiToolReplayPolicy<TInput = string> = boolean | PiAiToolReplayConfig<Record<string, JsonValue>, JsonValue, TInput>;
|
|
87
86
|
/** Replay policy map keyed by Pi AI tool name. */
|
|
88
|
-
type PiAiToolReplayPolicies<TInput = string
|
|
87
|
+
type PiAiToolReplayPolicies<TInput = string> = Record<string, PiAiToolReplayPolicy<TInput>>;
|
|
89
88
|
/** Tool definition accepted by the Pi AI harness runtime. */
|
|
90
|
-
interface PiAiToolDefinition<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string
|
|
89
|
+
interface PiAiToolDefinition<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string> {
|
|
91
90
|
description?: string;
|
|
92
|
-
execute: (args: TArgs, context: PiAiToolContext<TInput
|
|
91
|
+
execute: (args: TArgs, context: PiAiToolContext<TInput>) => MaybePromise<TResult>;
|
|
93
92
|
}
|
|
94
93
|
/** Toolset shape accepted by the Pi AI harness. */
|
|
95
|
-
type PiAiToolset<TInput = string
|
|
96
|
-
type ToolArgs<TTool> = TTool extends PiAiToolDefinition<infer TArgs, infer _TResult, infer _TInput
|
|
97
|
-
type ToolResult<TTool> = TTool extends PiAiToolDefinition<infer _TArgs, infer TResult, infer _TInput
|
|
94
|
+
type PiAiToolset<TInput = string> = AnyPiAiToolset<TInput>;
|
|
95
|
+
type ToolArgs<TTool> = TTool extends PiAiToolDefinition<infer TArgs, infer _TResult, infer _TInput> ? TArgs : never;
|
|
96
|
+
type ToolResult<TTool> = TTool extends PiAiToolDefinition<infer _TArgs, infer TResult, infer _TInput> ? TResult : never;
|
|
98
97
|
/** Runtime object passed into Pi AI agent entrypoints. */
|
|
99
|
-
type PiAiRuntime<TTools extends PiAiToolset<TInput
|
|
98
|
+
type PiAiRuntime<TTools extends PiAiToolset<TInput>, TInput = string> = {
|
|
100
99
|
tools: {
|
|
101
100
|
[K in keyof TTools]: (args: ToolArgs<TTools[K]>) => Promise<ToolResult<TTools[K]>>;
|
|
102
101
|
};
|
|
@@ -104,68 +103,68 @@ type PiAiRuntime<TTools extends PiAiToolset<TInput, TMetadata>, TInput = string,
|
|
|
104
103
|
signal?: AbortSignal;
|
|
105
104
|
};
|
|
106
105
|
/** Arguments passed to custom Pi AI harness run callbacks. */
|
|
107
|
-
interface PiAiHarnessRunArgs<TAgent, TInput,
|
|
106
|
+
interface PiAiHarnessRunArgs<TAgent, TInput, TTools extends PiAiToolset<TInput>> {
|
|
108
107
|
agent: TAgent;
|
|
109
108
|
input: TInput;
|
|
110
|
-
context: HarnessContext
|
|
111
|
-
runtime: PiAiRuntime<TTools, TInput
|
|
109
|
+
context: HarnessContext;
|
|
110
|
+
runtime: PiAiRuntime<TTools, TInput>;
|
|
112
111
|
}
|
|
113
112
|
/** Arguments passed to Pi AI harness output selectors. */
|
|
114
|
-
interface PiAiHarnessResultArgs<TAgent, TInput,
|
|
113
|
+
interface PiAiHarnessResultArgs<TAgent, TInput, TResult, TTools extends PiAiToolset<TInput>> extends PiAiHarnessRunArgs<TAgent, TInput, TTools> {
|
|
115
114
|
result: TResult;
|
|
116
115
|
}
|
|
117
116
|
/** Arguments passed to per-run Pi AI agent factories. */
|
|
118
|
-
interface PiAiCreateAgentArgs<TInput = string
|
|
117
|
+
interface PiAiCreateAgentArgs<TInput = string> {
|
|
119
118
|
input: TInput;
|
|
120
|
-
context: HarnessContext
|
|
119
|
+
context: HarnessContext;
|
|
121
120
|
}
|
|
122
|
-
interface PiAiHarnessBaseOptions<TAgent, TInput = string,
|
|
123
|
-
agent: AgentSource<TAgent, TInput
|
|
124
|
-
toolReplay?: PiAiToolReplayPolicies<TInput
|
|
125
|
-
output?: PiAiHarnessOutputSelector<TAgent, TInput,
|
|
121
|
+
interface PiAiHarnessBaseOptions<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>, TOutput extends JsonValue | undefined = JsonValue | undefined> {
|
|
122
|
+
agent: AgentSource<TAgent, TInput>;
|
|
123
|
+
toolReplay?: PiAiToolReplayPolicies<TInput>;
|
|
124
|
+
output?: PiAiHarnessOutputSelector<TAgent, TInput, TResult, TTools, TOutput>;
|
|
126
125
|
name?: string;
|
|
127
126
|
}
|
|
128
|
-
interface PiAiHarnessWithToolsOptions<TAgent, TInput = string,
|
|
127
|
+
interface PiAiHarnessWithToolsOptions<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>, TOutput extends JsonValue | undefined = JsonValue | undefined> extends PiAiHarnessBaseOptions<TAgent, TInput, TResult, TTools, TOutput> {
|
|
129
128
|
tools: TTools;
|
|
130
|
-
run?: (args: PiAiHarnessRunArgs<TAgent, TInput,
|
|
129
|
+
run?: (args: PiAiHarnessRunArgs<TAgent, TInput, TTools>) => MaybePromise<TResult | HarnessRun<TOutput>>;
|
|
131
130
|
}
|
|
132
|
-
interface PiAiHarnessInferredToolsOptions<TAgent, TInput = string,
|
|
131
|
+
interface PiAiHarnessInferredToolsOptions<TAgent, TInput = string, TResult = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined> extends PiAiHarnessBaseOptions<TAgent, TInput, TResult, InferredPiAiToolset<TInput>, TOutput> {
|
|
133
132
|
tools?: undefined;
|
|
134
|
-
run?: (args: PiAiHarnessRunArgs<TAgent, TInput,
|
|
133
|
+
run?: (args: PiAiHarnessRunArgs<TAgent, TInput, InferredPiAiToolset<TInput>>) => MaybePromise<TResult | HarnessRun<TOutput>>;
|
|
135
134
|
}
|
|
136
|
-
type PiAiHarnessWithToolsOptionsWithOutput<TAgent, TInput = string,
|
|
137
|
-
output: PiAiHarnessOutputSelector<TAgent, TInput,
|
|
135
|
+
type PiAiHarnessWithToolsOptionsWithOutput<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>, TOutput extends JsonValue | undefined = JsonValue | undefined> = PiAiHarnessWithToolsOptions<TAgent, TInput, TResult, TTools, TOutput> & {
|
|
136
|
+
output: PiAiHarnessOutputSelector<TAgent, TInput, TResult, TTools, TOutput>;
|
|
138
137
|
};
|
|
139
|
-
type PiAiHarnessInferredToolsOptionsWithOutput<TAgent, TInput = string,
|
|
140
|
-
output: PiAiHarnessOutputSelector<TAgent, TInput,
|
|
138
|
+
type PiAiHarnessInferredToolsOptionsWithOutput<TAgent, TInput = string, TResult = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined> = PiAiHarnessInferredToolsOptions<TAgent, TInput, TResult, TOutput> & {
|
|
139
|
+
output: PiAiHarnessOutputSelector<TAgent, TInput, TResult, InferredPiAiToolset<TInput>, TOutput>;
|
|
141
140
|
};
|
|
142
|
-
type PiAiHarnessWithToolsRunOptionsWithoutOutput<TAgent, TInput = string,
|
|
141
|
+
type PiAiHarnessWithToolsRunOptionsWithoutOutput<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>> = PiAiHarnessBaseOptions<TAgent, TInput, TResult, TTools, JsonValue | undefined> & {
|
|
143
142
|
tools: TTools;
|
|
144
|
-
run: (args: PiAiHarnessRunArgs<TAgent, TInput,
|
|
143
|
+
run: (args: PiAiHarnessRunArgs<TAgent, TInput, TTools>) => MaybePromise<TResult>;
|
|
145
144
|
output?: never;
|
|
146
145
|
};
|
|
147
|
-
type PiAiHarnessWithToolsAgentOptionsWithoutOutput<TAgent, TInput = string,
|
|
146
|
+
type PiAiHarnessWithToolsAgentOptionsWithoutOutput<TAgent, TInput = string, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>> = PiAiHarnessBaseOptions<TAgent, TInput, unknown, TTools, JsonValue | undefined> & {
|
|
148
147
|
tools: TTools;
|
|
149
148
|
run?: never;
|
|
150
149
|
output?: never;
|
|
151
150
|
};
|
|
152
|
-
type PiAiHarnessInferredToolsRunOptionsWithoutOutput<TAgent, TInput = string,
|
|
151
|
+
type PiAiHarnessInferredToolsRunOptionsWithoutOutput<TAgent, TInput = string, TResult = unknown> = PiAiHarnessBaseOptions<TAgent, TInput, TResult, InferredPiAiToolset<TInput>, JsonValue | undefined> & {
|
|
153
152
|
tools?: undefined;
|
|
154
|
-
run: (args: PiAiHarnessRunArgs<TAgent, TInput,
|
|
153
|
+
run: (args: PiAiHarnessRunArgs<TAgent, TInput, InferredPiAiToolset<TInput>>) => MaybePromise<TResult>;
|
|
155
154
|
output?: never;
|
|
156
155
|
};
|
|
157
|
-
type PiAiHarnessInferredToolsAgentOptionsWithoutOutput<TAgent, TInput = string
|
|
156
|
+
type PiAiHarnessInferredToolsAgentOptionsWithoutOutput<TAgent, TInput = string> = PiAiHarnessBaseOptions<TAgent, TInput, unknown, InferredPiAiToolset<TInput>, JsonValue | undefined> & {
|
|
158
157
|
tools?: undefined;
|
|
159
158
|
run?: never;
|
|
160
159
|
output?: never;
|
|
161
160
|
};
|
|
162
|
-
type PiAiHarnessOutputSelector<TAgent, TInput = string,
|
|
161
|
+
type PiAiHarnessOutputSelector<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>, TOutput extends JsonValue | undefined = JsonValue | undefined> = (args: PiAiHarnessResultArgs<TAgent, TInput, TResult, TTools>) => MaybePromise<TOutput>;
|
|
163
162
|
/** Adapts a Pi agent runtime into a normalized vitest-evals harness. */
|
|
164
|
-
declare function piAiHarness<TAgent, TInput = string,
|
|
165
|
-
declare function piAiHarness<TAgent, TInput = string,
|
|
166
|
-
declare function piAiHarness<TAgent, TInput = string,
|
|
167
|
-
declare function piAiHarness<TAgent, TInput = string,
|
|
168
|
-
declare function piAiHarness<TAgent, TInput = string,
|
|
169
|
-
declare function piAiHarness<TAgent, TInput = string
|
|
163
|
+
declare function piAiHarness<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>, TOutput extends JsonValue | undefined = JsonValue | undefined>(options: PiAiHarnessWithToolsOptionsWithOutput<TAgent, TInput, TResult, TTools, TOutput>): Harness<TInput, TOutput>;
|
|
164
|
+
declare function piAiHarness<TAgent, TInput = string, TResult = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined>(options: PiAiHarnessInferredToolsOptionsWithOutput<TAgent, TInput, TResult, TOutput>): Harness<TInput, TOutput>;
|
|
165
|
+
declare function piAiHarness<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>>(options: PiAiHarnessWithToolsRunOptionsWithoutOutput<TAgent, TInput, TResult, TTools>): Harness<TInput, PiAiResultOutput<Awaited<TResult>>>;
|
|
166
|
+
declare function piAiHarness<TAgent, TInput = string, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>>(options: PiAiHarnessWithToolsAgentOptionsWithoutOutput<TAgent, TInput, TTools>): Harness<TInput, PiAiResultOutput<PiAiAgentResult<TAgent, TInput, TTools>>>;
|
|
167
|
+
declare function piAiHarness<TAgent, TInput = string, TResult = unknown>(options: PiAiHarnessInferredToolsRunOptionsWithoutOutput<TAgent, TInput, TResult>): Harness<TInput, PiAiResultOutput<Awaited<TResult>>>;
|
|
168
|
+
declare function piAiHarness<TAgent, TInput = string>(options: PiAiHarnessInferredToolsAgentOptionsWithoutOutput<TAgent, TInput>): Harness<TInput, PiAiResultOutput<PiAiAgentResult<TAgent, TInput, InferredPiAiToolset<TInput>>>>;
|
|
170
169
|
|
|
171
170
|
export { type PiAiCreateAgentArgs, type PiAiEventSink, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiJudgeHarnessConfig, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolReplayPolicies, type PiAiToolReplayPolicy, type PiAiToolset, piAiHarness, piAiJudgeHarness };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import * as vitest_evals_judges from 'vitest-evals/judges';
|
|
2
|
-
import { NormalizedMessage, JsonValue,
|
|
2
|
+
import { NormalizedMessage, JsonValue, HarnessContext, HarnessRun, Harness } from 'vitest-evals/harness';
|
|
3
3
|
import { ReplayMode, ToolRecording, ToolReplayConfig } from 'vitest-evals/replay';
|
|
4
4
|
import { Api, Model, SimpleStreamOptions } from '@mariozechner/pi-ai';
|
|
5
5
|
|
|
6
6
|
type MaybePromise<T> = T | Promise<T>;
|
|
7
7
|
type JsonOutput<TValue> = [TValue] extends [JsonValue | undefined] ? TValue : undefined;
|
|
8
8
|
type ResultFieldOutput<TResult, TKey extends string> = TKey extends keyof TResult ? Record<string, never> extends Pick<TResult, TKey> ? JsonOutput<TResult[TKey]> | undefined : JsonOutput<TResult[TKey]> : undefined;
|
|
9
|
-
type AgentSource<TAgent, TInput = string
|
|
10
|
-
type AnyPiAiToolset<TInput = string
|
|
9
|
+
type AgentSource<TAgent, TInput = string> = TAgent | ((args: PiAiCreateAgentArgs<TInput>) => MaybePromise<TAgent>);
|
|
10
|
+
type AnyPiAiToolset<TInput = string> = Record<string, PiAiToolDefinition<any, any, TInput>>;
|
|
11
11
|
/**
|
|
12
12
|
* Configuration for adapting a Pi AI model into a judge harness.
|
|
13
13
|
*
|
|
@@ -56,10 +56,10 @@ interface PiAiJudgeHarnessConfig<TApi extends Api = Api> {
|
|
|
56
56
|
* ```
|
|
57
57
|
*/
|
|
58
58
|
declare function piAiJudgeHarness<TApi extends Api>(config: PiAiJudgeHarnessConfig<TApi>): vitest_evals_judges.JudgeHarness;
|
|
59
|
-
type InferredPiAiToolset<TInput = string
|
|
59
|
+
type InferredPiAiToolset<TInput = string> = Record<string, PiAiToolDefinition<Record<string, JsonValue>, JsonValue, TInput>>;
|
|
60
60
|
type PiAiResultOutput<TResult> = TResult extends HarnessRun<infer TOutput> ? TOutput : "output" extends keyof TResult ? ResultFieldOutput<TResult, "output"> : undefined;
|
|
61
|
-
type PiAiAgentResult<TAgent, TInput,
|
|
62
|
-
run: (input: TInput, runtime: PiAiRuntime<TTools, TInput
|
|
61
|
+
type PiAiAgentResult<TAgent, TInput, TTools extends PiAiToolset<TInput>> = TAgent extends {
|
|
62
|
+
run: (input: TInput, runtime: PiAiRuntime<TTools, TInput>) => MaybePromise<infer TResult>;
|
|
63
63
|
} ? Awaited<TResult> : unknown;
|
|
64
64
|
/** Replay mode alias used by the Pi AI harness package. */
|
|
65
65
|
type PiAiReplayMode = ReplayMode;
|
|
@@ -72,31 +72,30 @@ interface PiAiEventSink {
|
|
|
72
72
|
tool: (name: string, content: JsonValue, metadata?: Record<string, JsonValue>) => void;
|
|
73
73
|
}
|
|
74
74
|
/** Context passed to instrumented Pi AI tool executions. */
|
|
75
|
-
interface PiAiToolContext<TInput = string
|
|
75
|
+
interface PiAiToolContext<TInput = string> {
|
|
76
76
|
input: TInput;
|
|
77
|
-
metadata: HarnessContext<TMetadata>["metadata"];
|
|
78
77
|
signal?: AbortSignal;
|
|
79
|
-
setArtifact: HarnessContext
|
|
78
|
+
setArtifact: HarnessContext["setArtifact"];
|
|
80
79
|
}
|
|
81
80
|
/** Tool replay recording shape for Pi AI tools. */
|
|
82
81
|
type PiAiToolRecording<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue> = ToolRecording<TArgs, TResult>;
|
|
83
82
|
/** Replay configuration for a Pi AI tool. */
|
|
84
|
-
type PiAiToolReplayConfig<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string
|
|
83
|
+
type PiAiToolReplayConfig<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string> = ToolReplayConfig<TArgs, TResult, PiAiToolContext<TInput>>;
|
|
85
84
|
/** Replay policy for one Pi AI tool. */
|
|
86
|
-
type PiAiToolReplayPolicy<TInput = string
|
|
85
|
+
type PiAiToolReplayPolicy<TInput = string> = boolean | PiAiToolReplayConfig<Record<string, JsonValue>, JsonValue, TInput>;
|
|
87
86
|
/** Replay policy map keyed by Pi AI tool name. */
|
|
88
|
-
type PiAiToolReplayPolicies<TInput = string
|
|
87
|
+
type PiAiToolReplayPolicies<TInput = string> = Record<string, PiAiToolReplayPolicy<TInput>>;
|
|
89
88
|
/** Tool definition accepted by the Pi AI harness runtime. */
|
|
90
|
-
interface PiAiToolDefinition<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string
|
|
89
|
+
interface PiAiToolDefinition<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string> {
|
|
91
90
|
description?: string;
|
|
92
|
-
execute: (args: TArgs, context: PiAiToolContext<TInput
|
|
91
|
+
execute: (args: TArgs, context: PiAiToolContext<TInput>) => MaybePromise<TResult>;
|
|
93
92
|
}
|
|
94
93
|
/** Toolset shape accepted by the Pi AI harness. */
|
|
95
|
-
type PiAiToolset<TInput = string
|
|
96
|
-
type ToolArgs<TTool> = TTool extends PiAiToolDefinition<infer TArgs, infer _TResult, infer _TInput
|
|
97
|
-
type ToolResult<TTool> = TTool extends PiAiToolDefinition<infer _TArgs, infer TResult, infer _TInput
|
|
94
|
+
type PiAiToolset<TInput = string> = AnyPiAiToolset<TInput>;
|
|
95
|
+
type ToolArgs<TTool> = TTool extends PiAiToolDefinition<infer TArgs, infer _TResult, infer _TInput> ? TArgs : never;
|
|
96
|
+
type ToolResult<TTool> = TTool extends PiAiToolDefinition<infer _TArgs, infer TResult, infer _TInput> ? TResult : never;
|
|
98
97
|
/** Runtime object passed into Pi AI agent entrypoints. */
|
|
99
|
-
type PiAiRuntime<TTools extends PiAiToolset<TInput
|
|
98
|
+
type PiAiRuntime<TTools extends PiAiToolset<TInput>, TInput = string> = {
|
|
100
99
|
tools: {
|
|
101
100
|
[K in keyof TTools]: (args: ToolArgs<TTools[K]>) => Promise<ToolResult<TTools[K]>>;
|
|
102
101
|
};
|
|
@@ -104,68 +103,68 @@ type PiAiRuntime<TTools extends PiAiToolset<TInput, TMetadata>, TInput = string,
|
|
|
104
103
|
signal?: AbortSignal;
|
|
105
104
|
};
|
|
106
105
|
/** Arguments passed to custom Pi AI harness run callbacks. */
|
|
107
|
-
interface PiAiHarnessRunArgs<TAgent, TInput,
|
|
106
|
+
interface PiAiHarnessRunArgs<TAgent, TInput, TTools extends PiAiToolset<TInput>> {
|
|
108
107
|
agent: TAgent;
|
|
109
108
|
input: TInput;
|
|
110
|
-
context: HarnessContext
|
|
111
|
-
runtime: PiAiRuntime<TTools, TInput
|
|
109
|
+
context: HarnessContext;
|
|
110
|
+
runtime: PiAiRuntime<TTools, TInput>;
|
|
112
111
|
}
|
|
113
112
|
/** Arguments passed to Pi AI harness output selectors. */
|
|
114
|
-
interface PiAiHarnessResultArgs<TAgent, TInput,
|
|
113
|
+
interface PiAiHarnessResultArgs<TAgent, TInput, TResult, TTools extends PiAiToolset<TInput>> extends PiAiHarnessRunArgs<TAgent, TInput, TTools> {
|
|
115
114
|
result: TResult;
|
|
116
115
|
}
|
|
117
116
|
/** Arguments passed to per-run Pi AI agent factories. */
|
|
118
|
-
interface PiAiCreateAgentArgs<TInput = string
|
|
117
|
+
interface PiAiCreateAgentArgs<TInput = string> {
|
|
119
118
|
input: TInput;
|
|
120
|
-
context: HarnessContext
|
|
119
|
+
context: HarnessContext;
|
|
121
120
|
}
|
|
122
|
-
interface PiAiHarnessBaseOptions<TAgent, TInput = string,
|
|
123
|
-
agent: AgentSource<TAgent, TInput
|
|
124
|
-
toolReplay?: PiAiToolReplayPolicies<TInput
|
|
125
|
-
output?: PiAiHarnessOutputSelector<TAgent, TInput,
|
|
121
|
+
interface PiAiHarnessBaseOptions<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>, TOutput extends JsonValue | undefined = JsonValue | undefined> {
|
|
122
|
+
agent: AgentSource<TAgent, TInput>;
|
|
123
|
+
toolReplay?: PiAiToolReplayPolicies<TInput>;
|
|
124
|
+
output?: PiAiHarnessOutputSelector<TAgent, TInput, TResult, TTools, TOutput>;
|
|
126
125
|
name?: string;
|
|
127
126
|
}
|
|
128
|
-
interface PiAiHarnessWithToolsOptions<TAgent, TInput = string,
|
|
127
|
+
interface PiAiHarnessWithToolsOptions<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>, TOutput extends JsonValue | undefined = JsonValue | undefined> extends PiAiHarnessBaseOptions<TAgent, TInput, TResult, TTools, TOutput> {
|
|
129
128
|
tools: TTools;
|
|
130
|
-
run?: (args: PiAiHarnessRunArgs<TAgent, TInput,
|
|
129
|
+
run?: (args: PiAiHarnessRunArgs<TAgent, TInput, TTools>) => MaybePromise<TResult | HarnessRun<TOutput>>;
|
|
131
130
|
}
|
|
132
|
-
interface PiAiHarnessInferredToolsOptions<TAgent, TInput = string,
|
|
131
|
+
interface PiAiHarnessInferredToolsOptions<TAgent, TInput = string, TResult = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined> extends PiAiHarnessBaseOptions<TAgent, TInput, TResult, InferredPiAiToolset<TInput>, TOutput> {
|
|
133
132
|
tools?: undefined;
|
|
134
|
-
run?: (args: PiAiHarnessRunArgs<TAgent, TInput,
|
|
133
|
+
run?: (args: PiAiHarnessRunArgs<TAgent, TInput, InferredPiAiToolset<TInput>>) => MaybePromise<TResult | HarnessRun<TOutput>>;
|
|
135
134
|
}
|
|
136
|
-
type PiAiHarnessWithToolsOptionsWithOutput<TAgent, TInput = string,
|
|
137
|
-
output: PiAiHarnessOutputSelector<TAgent, TInput,
|
|
135
|
+
type PiAiHarnessWithToolsOptionsWithOutput<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>, TOutput extends JsonValue | undefined = JsonValue | undefined> = PiAiHarnessWithToolsOptions<TAgent, TInput, TResult, TTools, TOutput> & {
|
|
136
|
+
output: PiAiHarnessOutputSelector<TAgent, TInput, TResult, TTools, TOutput>;
|
|
138
137
|
};
|
|
139
|
-
type PiAiHarnessInferredToolsOptionsWithOutput<TAgent, TInput = string,
|
|
140
|
-
output: PiAiHarnessOutputSelector<TAgent, TInput,
|
|
138
|
+
type PiAiHarnessInferredToolsOptionsWithOutput<TAgent, TInput = string, TResult = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined> = PiAiHarnessInferredToolsOptions<TAgent, TInput, TResult, TOutput> & {
|
|
139
|
+
output: PiAiHarnessOutputSelector<TAgent, TInput, TResult, InferredPiAiToolset<TInput>, TOutput>;
|
|
141
140
|
};
|
|
142
|
-
type PiAiHarnessWithToolsRunOptionsWithoutOutput<TAgent, TInput = string,
|
|
141
|
+
type PiAiHarnessWithToolsRunOptionsWithoutOutput<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>> = PiAiHarnessBaseOptions<TAgent, TInput, TResult, TTools, JsonValue | undefined> & {
|
|
143
142
|
tools: TTools;
|
|
144
|
-
run: (args: PiAiHarnessRunArgs<TAgent, TInput,
|
|
143
|
+
run: (args: PiAiHarnessRunArgs<TAgent, TInput, TTools>) => MaybePromise<TResult>;
|
|
145
144
|
output?: never;
|
|
146
145
|
};
|
|
147
|
-
type PiAiHarnessWithToolsAgentOptionsWithoutOutput<TAgent, TInput = string,
|
|
146
|
+
type PiAiHarnessWithToolsAgentOptionsWithoutOutput<TAgent, TInput = string, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>> = PiAiHarnessBaseOptions<TAgent, TInput, unknown, TTools, JsonValue | undefined> & {
|
|
148
147
|
tools: TTools;
|
|
149
148
|
run?: never;
|
|
150
149
|
output?: never;
|
|
151
150
|
};
|
|
152
|
-
type PiAiHarnessInferredToolsRunOptionsWithoutOutput<TAgent, TInput = string,
|
|
151
|
+
type PiAiHarnessInferredToolsRunOptionsWithoutOutput<TAgent, TInput = string, TResult = unknown> = PiAiHarnessBaseOptions<TAgent, TInput, TResult, InferredPiAiToolset<TInput>, JsonValue | undefined> & {
|
|
153
152
|
tools?: undefined;
|
|
154
|
-
run: (args: PiAiHarnessRunArgs<TAgent, TInput,
|
|
153
|
+
run: (args: PiAiHarnessRunArgs<TAgent, TInput, InferredPiAiToolset<TInput>>) => MaybePromise<TResult>;
|
|
155
154
|
output?: never;
|
|
156
155
|
};
|
|
157
|
-
type PiAiHarnessInferredToolsAgentOptionsWithoutOutput<TAgent, TInput = string
|
|
156
|
+
type PiAiHarnessInferredToolsAgentOptionsWithoutOutput<TAgent, TInput = string> = PiAiHarnessBaseOptions<TAgent, TInput, unknown, InferredPiAiToolset<TInput>, JsonValue | undefined> & {
|
|
158
157
|
tools?: undefined;
|
|
159
158
|
run?: never;
|
|
160
159
|
output?: never;
|
|
161
160
|
};
|
|
162
|
-
type PiAiHarnessOutputSelector<TAgent, TInput = string,
|
|
161
|
+
type PiAiHarnessOutputSelector<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>, TOutput extends JsonValue | undefined = JsonValue | undefined> = (args: PiAiHarnessResultArgs<TAgent, TInput, TResult, TTools>) => MaybePromise<TOutput>;
|
|
163
162
|
/** Adapts a Pi agent runtime into a normalized vitest-evals harness. */
|
|
164
|
-
declare function piAiHarness<TAgent, TInput = string,
|
|
165
|
-
declare function piAiHarness<TAgent, TInput = string,
|
|
166
|
-
declare function piAiHarness<TAgent, TInput = string,
|
|
167
|
-
declare function piAiHarness<TAgent, TInput = string,
|
|
168
|
-
declare function piAiHarness<TAgent, TInput = string,
|
|
169
|
-
declare function piAiHarness<TAgent, TInput = string
|
|
163
|
+
declare function piAiHarness<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>, TOutput extends JsonValue | undefined = JsonValue | undefined>(options: PiAiHarnessWithToolsOptionsWithOutput<TAgent, TInput, TResult, TTools, TOutput>): Harness<TInput, TOutput>;
|
|
164
|
+
declare function piAiHarness<TAgent, TInput = string, TResult = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined>(options: PiAiHarnessInferredToolsOptionsWithOutput<TAgent, TInput, TResult, TOutput>): Harness<TInput, TOutput>;
|
|
165
|
+
declare function piAiHarness<TAgent, TInput = string, TResult = unknown, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>>(options: PiAiHarnessWithToolsRunOptionsWithoutOutput<TAgent, TInput, TResult, TTools>): Harness<TInput, PiAiResultOutput<Awaited<TResult>>>;
|
|
166
|
+
declare function piAiHarness<TAgent, TInput = string, TTools extends PiAiToolset<TInput> = PiAiToolset<TInput>>(options: PiAiHarnessWithToolsAgentOptionsWithoutOutput<TAgent, TInput, TTools>): Harness<TInput, PiAiResultOutput<PiAiAgentResult<TAgent, TInput, TTools>>>;
|
|
167
|
+
declare function piAiHarness<TAgent, TInput = string, TResult = unknown>(options: PiAiHarnessInferredToolsRunOptionsWithoutOutput<TAgent, TInput, TResult>): Harness<TInput, PiAiResultOutput<Awaited<TResult>>>;
|
|
168
|
+
declare function piAiHarness<TAgent, TInput = string>(options: PiAiHarnessInferredToolsAgentOptionsWithoutOutput<TAgent, TInput>): Harness<TInput, PiAiResultOutput<PiAiAgentResult<TAgent, TInput, InferredPiAiToolset<TInput>>>>;
|
|
170
169
|
|
|
171
170
|
export { type PiAiCreateAgentArgs, type PiAiEventSink, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiJudgeHarnessConfig, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolReplayPolicies, type PiAiToolReplayPolicy, type PiAiToolset, piAiHarness, piAiJudgeHarness };
|
package/dist/index.js
CHANGED
|
@@ -94,9 +94,7 @@ function piAiHarness(options) {
|
|
|
94
94
|
content: (0, import_harness.normalizeContent)(input)
|
|
95
95
|
}
|
|
96
96
|
];
|
|
97
|
-
const inferredTools = resolveInferredToolSurfaces(
|
|
98
|
-
agent
|
|
99
|
-
);
|
|
97
|
+
const inferredTools = resolveInferredToolSurfaces(agent);
|
|
100
98
|
if (hasExplicitToolset(options)) {
|
|
101
99
|
return await executePiHarnessRun(
|
|
102
100
|
options,
|
|
@@ -442,7 +440,6 @@ async function withInstrumentedAgentTools(agent, toolsets, args, callback) {
|
|
|
442
440
|
const startedAt = /* @__PURE__ */ new Date();
|
|
443
441
|
const toolContext = {
|
|
444
442
|
input: args.input,
|
|
445
|
-
metadata: args.context.metadata,
|
|
446
443
|
signal: args.context.signal,
|
|
447
444
|
setArtifact: args.context.setArtifact
|
|
448
445
|
};
|
|
@@ -523,9 +520,7 @@ async function withInstrumentedAgentTools(agent, toolsets, args, callback) {
|
|
|
523
520
|
const resetResult = originalReset.apply(this, resetArgs);
|
|
524
521
|
if (isPromiseLike(resetResult)) {
|
|
525
522
|
return resetResult.finally(() => {
|
|
526
|
-
patchToolsets(
|
|
527
|
-
resolveInferredNativeToolsets(agent)
|
|
528
|
-
);
|
|
523
|
+
patchToolsets(resolveInferredNativeToolsets(agent));
|
|
529
524
|
});
|
|
530
525
|
}
|
|
531
526
|
patchToolsets(resolveInferredNativeToolsets(agent));
|
|
@@ -693,7 +688,6 @@ function createRuntime({
|
|
|
693
688
|
);
|
|
694
689
|
const toolContext = {
|
|
695
690
|
input,
|
|
696
|
-
metadata: context.metadata,
|
|
697
691
|
signal: context.signal,
|
|
698
692
|
setArtifact: context.setArtifact
|
|
699
693
|
};
|