@mappies/miniwat-eval 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +131 -0
- package/README.md +247 -2
- package/dist/index.cjs +3453 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +941 -0
- package/dist/index.d.ts +941 -0
- package/dist/index.js +3343 -0
- package/dist/index.js.map +1 -0
- package/dist/main.js +3582 -0
- package/dist/main.js.map +1 -0
- package/package.json +73 -5
- package/schemas/conversation.schema.json +102 -0
- package/schemas/eval-config.schema.json +141 -0
- package/schemas/report.schema.json +326 -0
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,941 @@
|
|
|
1
|
+
import { ITokenUsage, Message, ResponseMessage, TokenUsage, ToolResult, ToolUse } from "@mappies/miniwat-agent-core";
|
|
2
|
+
import { ClientOptions } from "configurapi-handler-ws";
|
|
3
|
+
import { Agent } from "@mappies/miniwat-agent";
|
|
4
|
+
interface ITurnExecution {
|
|
5
|
+
response: string;
|
|
6
|
+
tokenUsage: ITokenUsage | undefined;
|
|
7
|
+
toolUses: ToolUse[] | undefined;
|
|
8
|
+
toolResults: ToolResult[] | undefined;
|
|
9
|
+
statuses: string[];
|
|
10
|
+
durationMs: number;
|
|
11
|
+
}
|
|
12
|
+
interface IActiveTurn {
|
|
13
|
+
turn: number;
|
|
14
|
+
completion: Promise<ITurnExecution>;
|
|
15
|
+
}
|
|
16
|
+
declare class TurnExecution implements ITurnExecution {
|
|
17
|
+
response: string;
|
|
18
|
+
tokenUsage: ITokenUsage | undefined;
|
|
19
|
+
toolUses: ToolUse[] | undefined;
|
|
20
|
+
toolResults: ToolResult[] | undefined;
|
|
21
|
+
statuses: string[];
|
|
22
|
+
durationMs: number;
|
|
23
|
+
constructor(data?: Partial<ITurnExecution>);
|
|
24
|
+
}
|
|
25
|
+
declare class ActiveTurn implements IActiveTurn {
|
|
26
|
+
turn: number;
|
|
27
|
+
completion: Promise<TurnExecution>;
|
|
28
|
+
constructor(data?: Partial<IActiveTurn>);
|
|
29
|
+
}
|
|
30
|
+
export declare const ArgumentDifferenceKind: {
|
|
31
|
+
readonly Missing: "missing";
|
|
32
|
+
readonly Extra: "extra";
|
|
33
|
+
readonly Different: "different";
|
|
34
|
+
};
|
|
35
|
+
export type ArgumentDifferenceKind = typeof ArgumentDifferenceKind[keyof typeof ArgumentDifferenceKind];
|
|
36
|
+
interface IArgumentDifference {
|
|
37
|
+
path: string;
|
|
38
|
+
kind: ArgumentDifferenceKind;
|
|
39
|
+
expected: unknown;
|
|
40
|
+
observed: unknown;
|
|
41
|
+
}
|
|
42
|
+
declare class ArgumentDifference implements IArgumentDifference {
|
|
43
|
+
path: string;
|
|
44
|
+
kind: ArgumentDifferenceKind;
|
|
45
|
+
expected: unknown;
|
|
46
|
+
observed: unknown;
|
|
47
|
+
constructor(data?: Partial<IArgumentDifference>);
|
|
48
|
+
}
|
|
49
|
+
export declare const ErrorCategory: {
|
|
50
|
+
readonly Configuration: "configuration";
|
|
51
|
+
readonly Specification: "specification";
|
|
52
|
+
readonly Connection: "connection";
|
|
53
|
+
readonly Protocol: "protocol";
|
|
54
|
+
readonly Host: "host";
|
|
55
|
+
readonly Timeout: "timeout";
|
|
56
|
+
readonly Evaluator: "evaluator";
|
|
57
|
+
readonly Interrupted: "interrupted";
|
|
58
|
+
};
|
|
59
|
+
export type ErrorCategory = typeof ErrorCategory[keyof typeof ErrorCategory];
|
|
60
|
+
interface ISourceLocation {
|
|
61
|
+
file: string;
|
|
62
|
+
line: number;
|
|
63
|
+
column: number;
|
|
64
|
+
path: string;
|
|
65
|
+
}
|
|
66
|
+
interface IValidationIssue {
|
|
67
|
+
message: string;
|
|
68
|
+
location: ISourceLocation;
|
|
69
|
+
}
|
|
70
|
+
interface IEvalError {
|
|
71
|
+
category: ErrorCategory;
|
|
72
|
+
message: string;
|
|
73
|
+
issues: IValidationIssue[];
|
|
74
|
+
}
|
|
75
|
+
interface IAttemptExecution {
|
|
76
|
+
turns: ITurnExecution[];
|
|
77
|
+
turnsSent: number;
|
|
78
|
+
failure: IEvalError | undefined;
|
|
79
|
+
attemptErrors: IEvalError[];
|
|
80
|
+
closeError: IEvalError | undefined;
|
|
81
|
+
durationMs: number;
|
|
82
|
+
connectionAttempts: number;
|
|
83
|
+
}
|
|
84
|
+
declare class SourceLocation implements ISourceLocation {
|
|
85
|
+
file: string;
|
|
86
|
+
line: number;
|
|
87
|
+
column: number;
|
|
88
|
+
path: string;
|
|
89
|
+
constructor(data?: Partial<ISourceLocation>);
|
|
90
|
+
toString(): string;
|
|
91
|
+
}
|
|
92
|
+
declare class ValidationIssue implements IValidationIssue {
|
|
93
|
+
message: string;
|
|
94
|
+
location: SourceLocation;
|
|
95
|
+
constructor(data?: Partial<IValidationIssue>);
|
|
96
|
+
toString(): string;
|
|
97
|
+
}
|
|
98
|
+
declare class EvalError extends Error implements IEvalError {
|
|
99
|
+
category: ErrorCategory;
|
|
100
|
+
issues: ValidationIssue[];
|
|
101
|
+
constructor(data?: Partial<IEvalError>);
|
|
102
|
+
}
|
|
103
|
+
declare class AttemptExecution implements IAttemptExecution {
|
|
104
|
+
turns: TurnExecution[];
|
|
105
|
+
turnsSent: number;
|
|
106
|
+
failure: EvalError | undefined;
|
|
107
|
+
attemptErrors: EvalError[];
|
|
108
|
+
closeError: EvalError | undefined;
|
|
109
|
+
durationMs: number;
|
|
110
|
+
connectionAttempts: number;
|
|
111
|
+
constructor(data?: Partial<IAttemptExecution>);
|
|
112
|
+
private static toError;
|
|
113
|
+
}
|
|
114
|
+
export declare const ResultStatus: {
|
|
115
|
+
readonly Pass: "PASS";
|
|
116
|
+
readonly Fail: "FAIL";
|
|
117
|
+
readonly Error: "ERROR";
|
|
118
|
+
readonly Skipped: "SKIPPED";
|
|
119
|
+
readonly Flaky: "FLAKY";
|
|
120
|
+
readonly Interrupted: "INTERRUPTED";
|
|
121
|
+
};
|
|
122
|
+
export type ResultStatus = typeof ResultStatus[keyof typeof ResultStatus];
|
|
123
|
+
interface IEvaluationResponse {
|
|
124
|
+
score: number;
|
|
125
|
+
explanation: string;
|
|
126
|
+
evidence: string[];
|
|
127
|
+
}
|
|
128
|
+
interface ITokenUsageSummary {
|
|
129
|
+
total: ITokenUsage | undefined;
|
|
130
|
+
reported: number;
|
|
131
|
+
expected: number;
|
|
132
|
+
}
|
|
133
|
+
export declare const TestKind: {
|
|
134
|
+
readonly Response: "response";
|
|
135
|
+
readonly ToolUsed: "toolUsed";
|
|
136
|
+
readonly ToolNotUsed: "toolNotUsed";
|
|
137
|
+
readonly ToolsInOrder: "toolsInOrder";
|
|
138
|
+
readonly Conversation: "conversation";
|
|
139
|
+
};
|
|
140
|
+
export type TestKind = typeof TestKind[keyof typeof TestKind];
|
|
141
|
+
interface IAttemptTestResult {
|
|
142
|
+
kind: TestKind;
|
|
143
|
+
turn: number;
|
|
144
|
+
title: string;
|
|
145
|
+
status: ResultStatus;
|
|
146
|
+
location: ISourceLocation;
|
|
147
|
+
details: string[];
|
|
148
|
+
evaluatorUsage: ITokenUsageSummary;
|
|
149
|
+
evaluation: IEvaluationResponse | undefined;
|
|
150
|
+
}
|
|
151
|
+
interface IAttemptScenarioResult {
|
|
152
|
+
attempt: number;
|
|
153
|
+
status: ResultStatus;
|
|
154
|
+
execution: IAttemptExecution;
|
|
155
|
+
tests: IAttemptTestResult[];
|
|
156
|
+
}
|
|
157
|
+
declare class EvaluationResponse implements IEvaluationResponse {
|
|
158
|
+
score: number;
|
|
159
|
+
explanation: string;
|
|
160
|
+
evidence: string[];
|
|
161
|
+
constructor(data?: Partial<IEvaluationResponse>);
|
|
162
|
+
}
|
|
163
|
+
declare class TokenUsageSummary implements ITokenUsageSummary {
|
|
164
|
+
total: TokenUsage | undefined;
|
|
165
|
+
reported: number;
|
|
166
|
+
expected: number;
|
|
167
|
+
constructor(data?: Partial<ITokenUsageSummary>);
|
|
168
|
+
add(other: ITokenUsageSummary): TokenUsageSummary;
|
|
169
|
+
}
|
|
170
|
+
declare class AttemptTestResult implements IAttemptTestResult {
|
|
171
|
+
kind: TestKind;
|
|
172
|
+
turn: number;
|
|
173
|
+
title: string;
|
|
174
|
+
status: ResultStatus;
|
|
175
|
+
location: SourceLocation;
|
|
176
|
+
details: string[];
|
|
177
|
+
evaluatorUsage: TokenUsageSummary;
|
|
178
|
+
evaluation: EvaluationResponse | undefined;
|
|
179
|
+
constructor(data?: Partial<IAttemptTestResult>);
|
|
180
|
+
}
|
|
181
|
+
declare class AttemptScenarioResult implements IAttemptScenarioResult {
|
|
182
|
+
attempt: number;
|
|
183
|
+
status: ResultStatus;
|
|
184
|
+
execution: AttemptExecution;
|
|
185
|
+
tests: AttemptTestResult[];
|
|
186
|
+
constructor(data?: Partial<IAttemptScenarioResult>);
|
|
187
|
+
}
|
|
188
|
+
interface IConnectionConfig {
|
|
189
|
+
subprotocols: string[];
|
|
190
|
+
}
|
|
191
|
+
declare class ConnectionConfig implements IConnectionConfig {
|
|
192
|
+
subprotocols: string[];
|
|
193
|
+
constructor(data?: Partial<IConnectionConfig>);
|
|
194
|
+
}
|
|
195
|
+
interface IConversationExpectation {
|
|
196
|
+
criteria: string;
|
|
197
|
+
location: ISourceLocation;
|
|
198
|
+
}
|
|
199
|
+
declare class ConversationExpectation implements IConversationExpectation {
|
|
200
|
+
criteria: string;
|
|
201
|
+
location: SourceLocation;
|
|
202
|
+
constructor(data?: Partial<IConversationExpectation>);
|
|
203
|
+
}
|
|
204
|
+
interface IResponseExpectation {
|
|
205
|
+
criteria: string | undefined;
|
|
206
|
+
contains: string[];
|
|
207
|
+
notContains: string[];
|
|
208
|
+
location: ISourceLocation;
|
|
209
|
+
}
|
|
210
|
+
interface IToolExpectation {
|
|
211
|
+
name: string;
|
|
212
|
+
arguments: Record<string, unknown> | undefined;
|
|
213
|
+
status: boolean | undefined;
|
|
214
|
+
location: ISourceLocation;
|
|
215
|
+
}
|
|
216
|
+
interface ITurnExpectation {
|
|
217
|
+
response: IResponseExpectation | undefined;
|
|
218
|
+
toolUsed: IToolExpectation[];
|
|
219
|
+
toolNotUsed: IToolExpectation[];
|
|
220
|
+
toolsInOrder: IToolExpectation[];
|
|
221
|
+
}
|
|
222
|
+
interface ITurnSpec {
|
|
223
|
+
input: string;
|
|
224
|
+
expect: ITurnExpectation;
|
|
225
|
+
location: ISourceLocation;
|
|
226
|
+
}
|
|
227
|
+
interface IConversationSpec {
|
|
228
|
+
version: number;
|
|
229
|
+
name: string;
|
|
230
|
+
description: string;
|
|
231
|
+
turns: ITurnSpec[];
|
|
232
|
+
conversation: IConversationExpectation | undefined;
|
|
233
|
+
location: ISourceLocation;
|
|
234
|
+
secrets: string[];
|
|
235
|
+
}
|
|
236
|
+
declare class ResponseExpectation implements IResponseExpectation {
|
|
237
|
+
criteria: string | undefined;
|
|
238
|
+
contains: string[];
|
|
239
|
+
notContains: string[];
|
|
240
|
+
location: SourceLocation;
|
|
241
|
+
constructor(data?: Partial<IResponseExpectation>);
|
|
242
|
+
}
|
|
243
|
+
declare class ToolExpectation implements IToolExpectation {
|
|
244
|
+
name: string;
|
|
245
|
+
arguments: Record<string, unknown> | undefined;
|
|
246
|
+
status: boolean | undefined;
|
|
247
|
+
location: SourceLocation;
|
|
248
|
+
constructor(data?: Partial<IToolExpectation>);
|
|
249
|
+
}
|
|
250
|
+
declare class TurnExpectation implements ITurnExpectation {
|
|
251
|
+
response: ResponseExpectation | undefined;
|
|
252
|
+
toolUsed: ToolExpectation[];
|
|
253
|
+
toolNotUsed: ToolExpectation[];
|
|
254
|
+
toolsInOrder: ToolExpectation[];
|
|
255
|
+
constructor(data?: Partial<ITurnExpectation>);
|
|
256
|
+
}
|
|
257
|
+
declare class TurnSpec implements ITurnSpec {
|
|
258
|
+
input: string;
|
|
259
|
+
expect: TurnExpectation;
|
|
260
|
+
location: SourceLocation;
|
|
261
|
+
constructor(data?: Partial<ITurnSpec>);
|
|
262
|
+
}
|
|
263
|
+
declare class ConversationSpec implements IConversationSpec {
|
|
264
|
+
version: number;
|
|
265
|
+
name: string;
|
|
266
|
+
description: string;
|
|
267
|
+
turns: TurnSpec[];
|
|
268
|
+
conversation: ConversationExpectation | undefined;
|
|
269
|
+
location: SourceLocation;
|
|
270
|
+
secrets: string[];
|
|
271
|
+
constructor(data?: Partial<IConversationSpec>);
|
|
272
|
+
}
|
|
273
|
+
export declare const AgentEventType: {
|
|
274
|
+
readonly ResponseChunk: "responseChunk";
|
|
275
|
+
readonly ToolMessage: "toolMessage";
|
|
276
|
+
readonly ResponseMessage: "responseMessage";
|
|
277
|
+
readonly Status: "status";
|
|
278
|
+
readonly Error: "error";
|
|
279
|
+
readonly StreamEnd: "streamEnd";
|
|
280
|
+
};
|
|
281
|
+
export type AgentEventType = typeof AgentEventType[keyof typeof AgentEventType];
|
|
282
|
+
interface IErrorEvent {
|
|
283
|
+
type: typeof AgentEventType.Error;
|
|
284
|
+
error: IEvalError;
|
|
285
|
+
}
|
|
286
|
+
declare class ErrorEvent implements IErrorEvent {
|
|
287
|
+
readonly type: "error";
|
|
288
|
+
error: EvalError;
|
|
289
|
+
constructor(data?: Partial<IErrorEvent>);
|
|
290
|
+
}
|
|
291
|
+
export declare const EvaluatorProvider: {
|
|
292
|
+
readonly OpenAi: "openai";
|
|
293
|
+
readonly Bedrock: "bedrock";
|
|
294
|
+
};
|
|
295
|
+
export type EvaluatorProvider = typeof EvaluatorProvider[keyof typeof EvaluatorProvider];
|
|
296
|
+
interface IEvaluatorThresholds {
|
|
297
|
+
response: number;
|
|
298
|
+
conversation: number;
|
|
299
|
+
}
|
|
300
|
+
interface IEvaluatorConfig {
|
|
301
|
+
provider: EvaluatorProvider;
|
|
302
|
+
model: string;
|
|
303
|
+
apiKey: string | undefined;
|
|
304
|
+
region: string | undefined;
|
|
305
|
+
concurrency: number;
|
|
306
|
+
timeout: number;
|
|
307
|
+
retries: number;
|
|
308
|
+
maxToolResultChars: number;
|
|
309
|
+
thresholds: IEvaluatorThresholds;
|
|
310
|
+
}
|
|
311
|
+
export declare const FlakyPolicy: {
|
|
312
|
+
readonly Warn: "warn";
|
|
313
|
+
readonly Fail: "fail";
|
|
314
|
+
};
|
|
315
|
+
export type FlakyPolicy = typeof FlakyPolicy[keyof typeof FlakyPolicy];
|
|
316
|
+
interface IExecutionTimeouts {
|
|
317
|
+
connection: number;
|
|
318
|
+
turn: number;
|
|
319
|
+
conversation: number;
|
|
320
|
+
}
|
|
321
|
+
interface IExecutionConfig {
|
|
322
|
+
scenarioConcurrency: number;
|
|
323
|
+
repeat: number;
|
|
324
|
+
minPass: number;
|
|
325
|
+
continueOnFailure: boolean;
|
|
326
|
+
flakyPolicy: FlakyPolicy;
|
|
327
|
+
connectionRetries: number;
|
|
328
|
+
timeouts: IExecutionTimeouts;
|
|
329
|
+
}
|
|
330
|
+
export declare const ReportFormat: {
|
|
331
|
+
readonly Console: "console";
|
|
332
|
+
readonly Json: "json";
|
|
333
|
+
};
|
|
334
|
+
export type ReportFormat = typeof ReportFormat[keyof typeof ReportFormat];
|
|
335
|
+
interface IReportConfig {
|
|
336
|
+
directory: string;
|
|
337
|
+
formats: ReportFormat[];
|
|
338
|
+
}
|
|
339
|
+
interface IProtocolCapabilities {
|
|
340
|
+
toolUses: boolean;
|
|
341
|
+
toolResults: boolean;
|
|
342
|
+
tokenUsage: boolean;
|
|
343
|
+
}
|
|
344
|
+
export declare const RequestFormat: {
|
|
345
|
+
readonly Json: "json";
|
|
346
|
+
};
|
|
347
|
+
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
348
|
+
interface IRequestConfig {
|
|
349
|
+
format: RequestFormat;
|
|
350
|
+
value: object | undefined;
|
|
351
|
+
inputPath: string | undefined;
|
|
352
|
+
}
|
|
353
|
+
interface IProtocolConfig {
|
|
354
|
+
request: IRequestConfig;
|
|
355
|
+
capabilities: IProtocolCapabilities;
|
|
356
|
+
}
|
|
357
|
+
interface IWebSocketConfig {
|
|
358
|
+
endpoint: string;
|
|
359
|
+
connection: IConnectionConfig;
|
|
360
|
+
protocol: IProtocolConfig;
|
|
361
|
+
}
|
|
362
|
+
interface IEvalConfig {
|
|
363
|
+
version: number;
|
|
364
|
+
configPath: string;
|
|
365
|
+
websocket: IWebSocketConfig;
|
|
366
|
+
execution: IExecutionConfig;
|
|
367
|
+
evaluator: IEvaluatorConfig;
|
|
368
|
+
report: IReportConfig;
|
|
369
|
+
secrets: string[];
|
|
370
|
+
}
|
|
371
|
+
declare class EvaluatorThresholds implements IEvaluatorThresholds {
|
|
372
|
+
response: number;
|
|
373
|
+
conversation: number;
|
|
374
|
+
constructor(data?: Partial<IEvaluatorThresholds>);
|
|
375
|
+
}
|
|
376
|
+
declare class EvaluatorConfig implements IEvaluatorConfig {
|
|
377
|
+
provider: EvaluatorProvider;
|
|
378
|
+
model: string;
|
|
379
|
+
apiKey: string | undefined;
|
|
380
|
+
region: string | undefined;
|
|
381
|
+
concurrency: number;
|
|
382
|
+
timeout: number;
|
|
383
|
+
retries: number;
|
|
384
|
+
maxToolResultChars: number;
|
|
385
|
+
thresholds: EvaluatorThresholds;
|
|
386
|
+
constructor(data?: Partial<IEvaluatorConfig>);
|
|
387
|
+
}
|
|
388
|
+
declare class ExecutionTimeouts implements IExecutionTimeouts {
|
|
389
|
+
connection: number;
|
|
390
|
+
turn: number;
|
|
391
|
+
conversation: number;
|
|
392
|
+
constructor(data?: Partial<IExecutionTimeouts>);
|
|
393
|
+
}
|
|
394
|
+
declare class ExecutionConfig implements IExecutionConfig {
|
|
395
|
+
scenarioConcurrency: number;
|
|
396
|
+
repeat: number;
|
|
397
|
+
minPass: number;
|
|
398
|
+
continueOnFailure: boolean;
|
|
399
|
+
flakyPolicy: FlakyPolicy;
|
|
400
|
+
connectionRetries: number;
|
|
401
|
+
timeouts: ExecutionTimeouts;
|
|
402
|
+
constructor(data?: Partial<IExecutionConfig>);
|
|
403
|
+
}
|
|
404
|
+
declare class ReportConfig implements IReportConfig {
|
|
405
|
+
directory: string;
|
|
406
|
+
formats: ReportFormat[];
|
|
407
|
+
constructor(data?: Partial<IReportConfig>);
|
|
408
|
+
}
|
|
409
|
+
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
410
|
+
toolUses: boolean;
|
|
411
|
+
toolResults: boolean;
|
|
412
|
+
tokenUsage: boolean;
|
|
413
|
+
constructor(data?: Partial<IProtocolCapabilities>);
|
|
414
|
+
}
|
|
415
|
+
declare class RequestConfig implements IRequestConfig {
|
|
416
|
+
format: RequestFormat;
|
|
417
|
+
value: object | undefined;
|
|
418
|
+
inputPath: string | undefined;
|
|
419
|
+
constructor(data?: Partial<IRequestConfig>);
|
|
420
|
+
}
|
|
421
|
+
declare class ProtocolConfig implements IProtocolConfig {
|
|
422
|
+
request: RequestConfig;
|
|
423
|
+
capabilities: ProtocolCapabilities;
|
|
424
|
+
constructor(data?: Partial<IProtocolConfig>);
|
|
425
|
+
}
|
|
426
|
+
declare class WebSocketConfig implements IWebSocketConfig {
|
|
427
|
+
endpoint: string;
|
|
428
|
+
connection: ConnectionConfig;
|
|
429
|
+
protocol: ProtocolConfig;
|
|
430
|
+
constructor(data?: Partial<IWebSocketConfig>);
|
|
431
|
+
}
|
|
432
|
+
declare class EvalConfig implements IEvalConfig {
|
|
433
|
+
version: number;
|
|
434
|
+
configPath: string;
|
|
435
|
+
websocket: WebSocketConfig;
|
|
436
|
+
execution: ExecutionConfig;
|
|
437
|
+
evaluator: EvaluatorConfig;
|
|
438
|
+
report: ReportConfig;
|
|
439
|
+
secrets: string[];
|
|
440
|
+
constructor(data?: Partial<IEvalConfig>);
|
|
441
|
+
}
|
|
442
|
+
interface ITestResult {
|
|
443
|
+
kind: TestKind;
|
|
444
|
+
turn: number;
|
|
445
|
+
title: string;
|
|
446
|
+
location: ISourceLocation;
|
|
447
|
+
status: ResultStatus;
|
|
448
|
+
passed: number;
|
|
449
|
+
failed: number;
|
|
450
|
+
errors: number;
|
|
451
|
+
required: number;
|
|
452
|
+
history: IAttemptTestResult[];
|
|
453
|
+
}
|
|
454
|
+
interface IScenarioResult {
|
|
455
|
+
spec: IConversationSpec;
|
|
456
|
+
status: ResultStatus;
|
|
457
|
+
passed: number;
|
|
458
|
+
failed: number;
|
|
459
|
+
errors: number;
|
|
460
|
+
required: number;
|
|
461
|
+
attempts: IAttemptScenarioResult[];
|
|
462
|
+
tests: ITestResult[];
|
|
463
|
+
agentUsage: ITokenUsageSummary;
|
|
464
|
+
evaluatorUsage: ITokenUsageSummary;
|
|
465
|
+
}
|
|
466
|
+
export declare const RunStatus: {
|
|
467
|
+
readonly Completed: "COMPLETED";
|
|
468
|
+
readonly Interrupted: "INTERRUPTED";
|
|
469
|
+
};
|
|
470
|
+
export type RunStatus = typeof RunStatus[keyof typeof RunStatus];
|
|
471
|
+
interface IEvalRunResult {
|
|
472
|
+
status: RunStatus;
|
|
473
|
+
repeat: number;
|
|
474
|
+
minPass: number;
|
|
475
|
+
quick: boolean;
|
|
476
|
+
scenarios: IScenarioResult[];
|
|
477
|
+
agentUsage: ITokenUsageSummary;
|
|
478
|
+
evaluatorUsage: ITokenUsageSummary;
|
|
479
|
+
tokenUsageCapability: boolean;
|
|
480
|
+
startedAt: Date;
|
|
481
|
+
durationMs: number;
|
|
482
|
+
}
|
|
483
|
+
declare class TestResult implements ITestResult {
|
|
484
|
+
kind: TestKind;
|
|
485
|
+
turn: number;
|
|
486
|
+
title: string;
|
|
487
|
+
location: SourceLocation;
|
|
488
|
+
status: ResultStatus;
|
|
489
|
+
passed: number;
|
|
490
|
+
failed: number;
|
|
491
|
+
errors: number;
|
|
492
|
+
required: number;
|
|
493
|
+
history: AttemptTestResult[];
|
|
494
|
+
constructor(data?: Partial<ITestResult>);
|
|
495
|
+
}
|
|
496
|
+
declare class ScenarioResult implements IScenarioResult {
|
|
497
|
+
spec: ConversationSpec;
|
|
498
|
+
status: ResultStatus;
|
|
499
|
+
passed: number;
|
|
500
|
+
failed: number;
|
|
501
|
+
errors: number;
|
|
502
|
+
required: number;
|
|
503
|
+
attempts: AttemptScenarioResult[];
|
|
504
|
+
tests: TestResult[];
|
|
505
|
+
agentUsage: TokenUsageSummary;
|
|
506
|
+
evaluatorUsage: TokenUsageSummary;
|
|
507
|
+
constructor(data?: Partial<IScenarioResult>);
|
|
508
|
+
}
|
|
509
|
+
declare class EvalRunResult implements IEvalRunResult {
|
|
510
|
+
status: RunStatus;
|
|
511
|
+
repeat: number;
|
|
512
|
+
minPass: number;
|
|
513
|
+
quick: boolean;
|
|
514
|
+
scenarios: ScenarioResult[];
|
|
515
|
+
agentUsage: TokenUsageSummary;
|
|
516
|
+
evaluatorUsage: TokenUsageSummary;
|
|
517
|
+
tokenUsageCapability: boolean;
|
|
518
|
+
startedAt: Date;
|
|
519
|
+
durationMs: number;
|
|
520
|
+
constructor(data?: Partial<IEvalRunResult>);
|
|
521
|
+
}
|
|
522
|
+
interface IEvaluationOutcome {
|
|
523
|
+
response: IEvaluationResponse | undefined;
|
|
524
|
+
error: IEvalError | undefined;
|
|
525
|
+
calls: number;
|
|
526
|
+
usage: ITokenUsageSummary;
|
|
527
|
+
}
|
|
528
|
+
declare class EvaluationOutcome implements IEvaluationOutcome {
|
|
529
|
+
response: EvaluationResponse | undefined;
|
|
530
|
+
error: EvalError | undefined;
|
|
531
|
+
calls: number;
|
|
532
|
+
usage: TokenUsageSummary;
|
|
533
|
+
constructor(data?: Partial<IEvaluationOutcome>);
|
|
534
|
+
}
|
|
535
|
+
export declare const EvaluationScope: {
|
|
536
|
+
readonly Turn: "turn";
|
|
537
|
+
readonly Conversation: "conversation";
|
|
538
|
+
};
|
|
539
|
+
export type EvaluationScope = typeof EvaluationScope[keyof typeof EvaluationScope];
|
|
540
|
+
interface IEvaluationTranscriptEntry {
|
|
541
|
+
turn: number;
|
|
542
|
+
input: string;
|
|
543
|
+
response: string;
|
|
544
|
+
toolUses: ToolUse[] | undefined;
|
|
545
|
+
toolResults: ToolResult[] | undefined;
|
|
546
|
+
}
|
|
547
|
+
interface IEvaluationRequest {
|
|
548
|
+
scope: EvaluationScope;
|
|
549
|
+
expectation: string;
|
|
550
|
+
actualResponse: string;
|
|
551
|
+
transcript: IEvaluationTranscriptEntry[];
|
|
552
|
+
}
|
|
553
|
+
declare class EvaluationTranscriptEntry implements IEvaluationTranscriptEntry {
|
|
554
|
+
turn: number;
|
|
555
|
+
input: string;
|
|
556
|
+
response: string;
|
|
557
|
+
toolUses: ToolUse[] | undefined;
|
|
558
|
+
toolResults: ToolResult[] | undefined;
|
|
559
|
+
constructor(data?: Partial<IEvaluationTranscriptEntry>);
|
|
560
|
+
}
|
|
561
|
+
declare class EvaluationRequest implements IEvaluationRequest {
|
|
562
|
+
scope: EvaluationScope;
|
|
563
|
+
expectation: string;
|
|
564
|
+
actualResponse: string;
|
|
565
|
+
transcript: EvaluationTranscriptEntry[];
|
|
566
|
+
constructor(data?: Partial<IEvaluationRequest>);
|
|
567
|
+
}
|
|
568
|
+
interface IExecutionOverride {
|
|
569
|
+
repeat: number | undefined;
|
|
570
|
+
minPass: number | undefined;
|
|
571
|
+
quick: boolean;
|
|
572
|
+
scenarioConcurrency: number | undefined;
|
|
573
|
+
}
|
|
574
|
+
declare class ExecutionOverride implements IExecutionOverride {
|
|
575
|
+
repeat: number | undefined;
|
|
576
|
+
minPass: number | undefined;
|
|
577
|
+
quick: boolean;
|
|
578
|
+
scenarioConcurrency: number | undefined;
|
|
579
|
+
constructor(data?: Partial<IExecutionOverride>);
|
|
580
|
+
}
|
|
581
|
+
interface IReportOverride {
|
|
582
|
+
formats: string[] | undefined;
|
|
583
|
+
verbose: boolean;
|
|
584
|
+
}
|
|
585
|
+
declare class ReportOverride implements IReportOverride {
|
|
586
|
+
formats: string[] | undefined;
|
|
587
|
+
verbose: boolean;
|
|
588
|
+
constructor(data?: Partial<IReportOverride>);
|
|
589
|
+
}
|
|
590
|
+
interface IResponseChunkEvent {
|
|
591
|
+
type: typeof AgentEventType.ResponseChunk;
|
|
592
|
+
text: string;
|
|
593
|
+
}
|
|
594
|
+
declare class ResponseChunkEvent implements IResponseChunkEvent {
|
|
595
|
+
readonly type: "responseChunk";
|
|
596
|
+
text: string;
|
|
597
|
+
constructor(data?: Partial<IResponseChunkEvent>);
|
|
598
|
+
}
|
|
599
|
+
interface IResponseMessageEvent {
|
|
600
|
+
type: typeof AgentEventType.ResponseMessage;
|
|
601
|
+
response: ResponseMessage;
|
|
602
|
+
}
|
|
603
|
+
declare class ResponseMessageEvent implements IResponseMessageEvent {
|
|
604
|
+
readonly type: "responseMessage";
|
|
605
|
+
response: ResponseMessage;
|
|
606
|
+
constructor(data?: Partial<IResponseMessageEvent>);
|
|
607
|
+
}
|
|
608
|
+
interface ISemanticCheck {
|
|
609
|
+
test: IAttemptTestResult;
|
|
610
|
+
request: IEvaluationRequest;
|
|
611
|
+
}
|
|
612
|
+
declare class SemanticCheck implements ISemanticCheck {
|
|
613
|
+
test: AttemptTestResult;
|
|
614
|
+
request: EvaluationRequest;
|
|
615
|
+
constructor(data?: Partial<ISemanticCheck>);
|
|
616
|
+
}
|
|
617
|
+
interface IStatusEvent {
|
|
618
|
+
type: typeof AgentEventType.Status;
|
|
619
|
+
message: string;
|
|
620
|
+
}
|
|
621
|
+
declare class StatusEvent implements IStatusEvent {
|
|
622
|
+
readonly type: "status";
|
|
623
|
+
message: string;
|
|
624
|
+
constructor(data?: Partial<IStatusEvent>);
|
|
625
|
+
}
|
|
626
|
+
interface IStreamEndEvent {
|
|
627
|
+
type: typeof AgentEventType.StreamEnd;
|
|
628
|
+
}
|
|
629
|
+
declare class StreamEndEvent implements IStreamEndEvent {
|
|
630
|
+
readonly type: "streamEnd";
|
|
631
|
+
constructor(_data?: Partial<IStreamEndEvent>);
|
|
632
|
+
}
|
|
633
|
+
interface IToolMessageEvent {
|
|
634
|
+
type: typeof AgentEventType.ToolMessage;
|
|
635
|
+
message: Message;
|
|
636
|
+
}
|
|
637
|
+
declare class ToolMessageEvent implements IToolMessageEvent {
|
|
638
|
+
readonly type: "toolMessage";
|
|
639
|
+
message: Message;
|
|
640
|
+
constructor(data?: Partial<IToolMessageEvent>);
|
|
641
|
+
}
|
|
642
|
+
interface ITurnOutcome {
|
|
643
|
+
execution: ITurnExecution;
|
|
644
|
+
tests: IAttemptTestResult[];
|
|
645
|
+
checks: ISemanticCheck[];
|
|
646
|
+
}
|
|
647
|
+
declare class TurnOutcome implements ITurnOutcome {
|
|
648
|
+
execution: TurnExecution;
|
|
649
|
+
tests: AttemptTestResult[];
|
|
650
|
+
checks: SemanticCheck[];
|
|
651
|
+
constructor(data?: Partial<ITurnOutcome>);
|
|
652
|
+
}
|
|
653
|
+
interface IAgentConnection {
|
|
654
|
+
connect(): Promise<void>;
|
|
655
|
+
sendMessage(input: string): Promise<ITurnExecution>;
|
|
656
|
+
close(): Promise<void>;
|
|
657
|
+
}
|
|
658
|
+
interface IEvaluator {
|
|
659
|
+
evaluate(request: IEvaluationRequest, signal: AbortSignal): Promise<IEvaluationOutcome>;
|
|
660
|
+
}
|
|
661
|
+
export declare const CONFIG_FILE_NAME = "eval-config.yml";
|
|
662
|
+
declare class ConfigLocator {
|
|
663
|
+
private readonly cwd;
|
|
664
|
+
constructor(cwd?: string);
|
|
665
|
+
locate(target: string, explicitConfig: string): string;
|
|
666
|
+
private static searchUpward;
|
|
667
|
+
private static isProjectRoot;
|
|
668
|
+
private static candidate;
|
|
669
|
+
private static isFile;
|
|
670
|
+
}
|
|
671
|
+
export declare const DEFAULT_CONFIG: EvalConfig;
|
|
672
|
+
declare class EvalConfigLoader {
|
|
673
|
+
private readonly cwd;
|
|
674
|
+
private readonly documentLoader;
|
|
675
|
+
private readonly schemaValidator;
|
|
676
|
+
private readonly environmentResolver;
|
|
677
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string);
|
|
678
|
+
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
679
|
+
private static applyReport;
|
|
680
|
+
private validateSecretReferences;
|
|
681
|
+
private validateInputPath;
|
|
682
|
+
private build;
|
|
683
|
+
private applyRepetition;
|
|
684
|
+
private static secrets;
|
|
685
|
+
private static section;
|
|
686
|
+
private static child;
|
|
687
|
+
}
|
|
688
|
+
declare class ScenarioLoader {
|
|
689
|
+
private readonly cwd;
|
|
690
|
+
private readonly documentLoader;
|
|
691
|
+
private readonly schemaValidator;
|
|
692
|
+
private readonly conversationValidator;
|
|
693
|
+
private readonly environmentResolver;
|
|
694
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string);
|
|
695
|
+
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
696
|
+
discover(target: string, config: EvalConfig): string[];
|
|
697
|
+
private walk;
|
|
698
|
+
private build;
|
|
699
|
+
private buildTurn;
|
|
700
|
+
private buildResponse;
|
|
701
|
+
private buildTools;
|
|
702
|
+
}
|
|
703
|
+
type AgentEvent = ResponseChunkEvent | ToolMessageEvent | ResponseMessageEvent | StatusEvent | ErrorEvent | StreamEndEvent;
|
|
704
|
+
interface IWebSocketClient {
|
|
705
|
+
connect(): Promise<void>;
|
|
706
|
+
send<T>(request: unknown, options: {
|
|
707
|
+
timeoutMs: number;
|
|
708
|
+
onChunk: (payload: unknown) => void;
|
|
709
|
+
onError: (error: Error) => void;
|
|
710
|
+
}): Promise<T>;
|
|
711
|
+
close(): Promise<void>;
|
|
712
|
+
}
|
|
713
|
+
type WebSocketClientFactory = (endpoint: string, options: ClientOptions) => IWebSocketClient;
|
|
714
|
+
declare class WebSocketAgentConnection implements IAgentConnection {
|
|
715
|
+
private readonly config;
|
|
716
|
+
private readonly collector;
|
|
717
|
+
private readonly requestBuilder;
|
|
718
|
+
private readonly client;
|
|
719
|
+
private closed;
|
|
720
|
+
constructor(config: EvalConfig, createClient?: WebSocketClientFactory);
|
|
721
|
+
get attemptErrors(): EvalError[];
|
|
722
|
+
connect(): Promise<void>;
|
|
723
|
+
sendMessage(input: string): Promise<TurnExecution>;
|
|
724
|
+
close(): Promise<void>;
|
|
725
|
+
private assertOpen;
|
|
726
|
+
private closeQuietly;
|
|
727
|
+
private static withTimeout;
|
|
728
|
+
}
|
|
729
|
+
type AgentConnectionFactory = (config: EvalConfig) => WebSocketAgentConnection;
|
|
730
|
+
declare class ConversationRunner {
|
|
731
|
+
private readonly config;
|
|
732
|
+
private readonly evaluator;
|
|
733
|
+
private readonly createConnection;
|
|
734
|
+
constructor(config: EvalConfig, evaluator?: IEvaluator, createConnection?: AgentConnectionFactory);
|
|
735
|
+
run(spec: ConversationSpec, signal?: AbortSignal): Promise<AttemptScenarioResult>;
|
|
736
|
+
private semanticEvaluator;
|
|
737
|
+
private resolve;
|
|
738
|
+
private tests;
|
|
739
|
+
}
|
|
740
|
+
declare class AssertionRunner {
|
|
741
|
+
private readonly capabilities;
|
|
742
|
+
private readonly responses;
|
|
743
|
+
private readonly tools;
|
|
744
|
+
constructor(capabilities: ProtocolCapabilities);
|
|
745
|
+
run(turn: number, expectation: TurnExpectation, execution: TurnExecution): AttemptTestResult[];
|
|
746
|
+
semanticChecks(expectation: TurnExpectation, tests: AttemptTestResult[], execution: TurnExecution, transcript: EvaluationTranscriptEntry[]): SemanticCheck[];
|
|
747
|
+
skip(turn: number, expectation: TurnExpectation, reason: string, status?: ResultStatus): AttemptTestResult[];
|
|
748
|
+
private static listLocation;
|
|
749
|
+
}
|
|
750
|
+
declare class ResponseEvaluator {
|
|
751
|
+
evaluate(expectation: ResponseExpectation, turn: number, execution: TurnExecution): AttemptTestResult;
|
|
752
|
+
}
|
|
753
|
+
declare class ToolArgumentMatcher {
|
|
754
|
+
static argumentsOf(use: ToolUse): Record<string, unknown> | undefined;
|
|
755
|
+
matches(expected: Record<string, unknown>, observed: Record<string, unknown>): boolean;
|
|
756
|
+
differences(expected: Record<string, unknown>, observed: Record<string, unknown>): ArgumentDifference[];
|
|
757
|
+
private static compare;
|
|
758
|
+
private static keyPath;
|
|
759
|
+
}
|
|
760
|
+
declare class ToolUsageEvaluator {
|
|
761
|
+
private readonly matcher;
|
|
762
|
+
constructor(matcher?: ToolArgumentMatcher);
|
|
763
|
+
toolUsed(entries: ToolExpectation[], turn: number, execution: TurnExecution, capabilities: ProtocolCapabilities): AttemptTestResult[];
|
|
764
|
+
toolNotUsed(entries: ToolExpectation[], turn: number, execution: TurnExecution, capabilities: ProtocolCapabilities): AttemptTestResult[];
|
|
765
|
+
toolsInOrder(entries: ToolExpectation[], location: SourceLocation, turn: number, execution: TurnExecution, capabilities: ProtocolCapabilities): AttemptTestResult[];
|
|
766
|
+
private match;
|
|
767
|
+
private satisfiesInvocation;
|
|
768
|
+
private static satisfiesStatus;
|
|
769
|
+
private explainCandidate;
|
|
770
|
+
private differencesFor;
|
|
771
|
+
private static rank;
|
|
772
|
+
private static describe;
|
|
773
|
+
private static resultsById;
|
|
774
|
+
private static label;
|
|
775
|
+
private static result;
|
|
776
|
+
}
|
|
777
|
+
declare class TurnRunner {
|
|
778
|
+
private readonly connection;
|
|
779
|
+
private readonly assertions;
|
|
780
|
+
constructor(connection: IAgentConnection, assertions: AssertionRunner);
|
|
781
|
+
run(turnNumber: number, turn: TurnSpec, history: EvaluationTranscriptEntry[]): Promise<TurnOutcome>;
|
|
782
|
+
static transcriptEntry(turnNumber: number, turn: TurnSpec, execution: TurnExecution): EvaluationTranscriptEntry;
|
|
783
|
+
}
|
|
784
|
+
declare class ConversationEvaluator {
|
|
785
|
+
private readonly evaluator;
|
|
786
|
+
private readonly threshold;
|
|
787
|
+
constructor(evaluator: IEvaluator, threshold: number);
|
|
788
|
+
evaluate(expectation: ConversationExpectation, transcript: EvaluationTranscriptEntry[], allTurnsCompleted: boolean, signal?: AbortSignal): Promise<AttemptTestResult>;
|
|
789
|
+
}
|
|
790
|
+
export declare function applyScore(test: AttemptTestResult, outcome: EvaluationOutcome, threshold: number): AttemptTestResult;
|
|
791
|
+
type ProviderImporter = (packageName: string) => Promise<unknown>;
|
|
792
|
+
declare class EvaluatorAgentFactory {
|
|
793
|
+
private readonly config;
|
|
794
|
+
private readonly importProvider;
|
|
795
|
+
private model;
|
|
796
|
+
constructor(config: EvaluatorConfig, importProvider?: ProviderImporter);
|
|
797
|
+
init(): Promise<void>;
|
|
798
|
+
create(): Agent;
|
|
799
|
+
}
|
|
800
|
+
declare class EvaluatorOutputParser {
|
|
801
|
+
parse(text: string): EvaluationResponse;
|
|
802
|
+
private static invalid;
|
|
803
|
+
}
|
|
804
|
+
declare class EvaluatorPromptBuilder {
|
|
805
|
+
private readonly maxToolResultChars;
|
|
806
|
+
static readonly SYSTEM_PROMPT: string;
|
|
807
|
+
constructor(maxToolResultChars?: number);
|
|
808
|
+
build(request: EvaluationRequest): string;
|
|
809
|
+
private entry;
|
|
810
|
+
private result;
|
|
811
|
+
private limit;
|
|
812
|
+
}
|
|
813
|
+
declare class ConcurrencyLimiter {
|
|
814
|
+
readonly limit: number;
|
|
815
|
+
private running;
|
|
816
|
+
private readonly waiting;
|
|
817
|
+
constructor(limit: number);
|
|
818
|
+
get active(): number;
|
|
819
|
+
run<T>(task: () => Promise<T>, signal?: AbortSignal): Promise<T>;
|
|
820
|
+
private acquire;
|
|
821
|
+
}
|
|
822
|
+
declare class LlmEvaluator implements IEvaluator {
|
|
823
|
+
private readonly factory;
|
|
824
|
+
private readonly config;
|
|
825
|
+
private readonly limiter;
|
|
826
|
+
private readonly prompts;
|
|
827
|
+
private readonly parser;
|
|
828
|
+
constructor(factory: EvaluatorAgentFactory, config: EvaluatorConfig, limiter: ConcurrencyLimiter);
|
|
829
|
+
evaluate(request: EvaluationRequest, signal?: AbortSignal): Promise<EvaluationOutcome>;
|
|
830
|
+
private call;
|
|
831
|
+
private static failed;
|
|
832
|
+
private static usage;
|
|
833
|
+
}
|
|
834
|
+
declare class UnavailableEvaluator implements IEvaluator {
|
|
835
|
+
evaluate(): Promise<EvaluationOutcome>;
|
|
836
|
+
}
|
|
837
|
+
declare class ScenarioRunner {
|
|
838
|
+
private readonly config;
|
|
839
|
+
private readonly conversations;
|
|
840
|
+
private readonly results;
|
|
841
|
+
private readonly tokens;
|
|
842
|
+
constructor(config: EvalConfig, conversations: ConversationRunner);
|
|
843
|
+
run(spec: ConversationSpec, signal?: AbortSignal): Promise<ScenarioResult>;
|
|
844
|
+
}
|
|
845
|
+
type ScenarioRun = (spec: ConversationSpec, signal: AbortSignal) => Promise<ScenarioResult>;
|
|
846
|
+
declare class ScenarioScheduler {
|
|
847
|
+
private readonly config;
|
|
848
|
+
private readonly runScenario;
|
|
849
|
+
constructor(config: EvalConfig, runScenario: ScenarioRun);
|
|
850
|
+
run(specs: ConversationSpec[], signal: AbortSignal, onResult: (result: ScenarioResult, index: number) => void): Promise<ScenarioResult[]>;
|
|
851
|
+
private static notStarted;
|
|
852
|
+
}
|
|
853
|
+
type StatusCount = {
|
|
854
|
+
status: ResultStatus;
|
|
855
|
+
passed: number;
|
|
856
|
+
failed: number;
|
|
857
|
+
errors: number;
|
|
858
|
+
};
|
|
859
|
+
declare class ResultAggregator {
|
|
860
|
+
attemptStatus(attempt: AttemptScenarioResult): ResultStatus;
|
|
861
|
+
count(outcomes: ResultStatus[], minPass: number): StatusCount;
|
|
862
|
+
tests(attempts: AttemptScenarioResult[], minPass: number): TestResult[];
|
|
863
|
+
}
|
|
864
|
+
declare class TokenUsageAggregator {
|
|
865
|
+
agent(attempts: AttemptScenarioResult[], capability: boolean): TokenUsageSummary;
|
|
866
|
+
evaluator(attempts: AttemptScenarioResult[]): TokenUsageSummary;
|
|
867
|
+
sum(summaries: TokenUsageSummary[]): TokenUsageSummary;
|
|
868
|
+
}
|
|
869
|
+
export declare const INTERRUPTED_EXIT_CODE = 130;
|
|
870
|
+
declare class ExitCodeCalculator {
|
|
871
|
+
private readonly flakyPolicy;
|
|
872
|
+
constructor(flakyPolicy: FlakyPolicy);
|
|
873
|
+
count(run: EvalRunResult): number;
|
|
874
|
+
exitCode(run: EvalRunResult): number;
|
|
875
|
+
private scenarioCount;
|
|
876
|
+
private unsuccessful;
|
|
877
|
+
}
|
|
878
|
+
type RunHeader = {
|
|
879
|
+
quick: boolean;
|
|
880
|
+
repeat: number;
|
|
881
|
+
minPass: number;
|
|
882
|
+
scenarios: number;
|
|
883
|
+
concurrency: number;
|
|
884
|
+
};
|
|
885
|
+
declare class ConsoleReporter {
|
|
886
|
+
private readonly verbose;
|
|
887
|
+
private readonly colour;
|
|
888
|
+
constructor(verbose: boolean, colour: boolean);
|
|
889
|
+
header(run: RunHeader): string;
|
|
890
|
+
scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
|
|
891
|
+
summary(run: EvalRunResult, exitCode: number, reportPaths?: string[]): string;
|
|
892
|
+
static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
|
|
893
|
+
static finalLine(run: EvalRunResult, exitCode: number): string;
|
|
894
|
+
static duration(ms: number): string;
|
|
895
|
+
private problems;
|
|
896
|
+
private static testTitle;
|
|
897
|
+
private static tally;
|
|
898
|
+
private status;
|
|
899
|
+
private static counts;
|
|
900
|
+
private static usage;
|
|
901
|
+
private formatAttempt;
|
|
902
|
+
private formatTest;
|
|
903
|
+
private static formatTurn;
|
|
904
|
+
private static indent;
|
|
905
|
+
}
|
|
906
|
+
export declare const REDACTED = "[REDACTED]";
|
|
907
|
+
declare class SecretRedactor {
|
|
908
|
+
private readonly secrets;
|
|
909
|
+
constructor(secrets: string[]);
|
|
910
|
+
redact(text: string): string;
|
|
911
|
+
redactValue(value: unknown): unknown;
|
|
912
|
+
}
|
|
913
|
+
export declare const JSON_REPORT_FILE = "miniwat-eval-report.json";
|
|
914
|
+
export declare const JSON_REPORT_VERSION = 1;
|
|
915
|
+
declare class JsonReporter {
|
|
916
|
+
private readonly redactor;
|
|
917
|
+
private readonly cwd;
|
|
918
|
+
private readonly tokens;
|
|
919
|
+
constructor(redactor: SecretRedactor, cwd: string);
|
|
920
|
+
render(run: EvalRunResult, config: EvalConfig, unsuccessfulCount: number, exitCode: number): string;
|
|
921
|
+
private configuration;
|
|
922
|
+
private scenario;
|
|
923
|
+
private attempt;
|
|
924
|
+
private static turn;
|
|
925
|
+
private static test;
|
|
926
|
+
private static identity;
|
|
927
|
+
private static location;
|
|
928
|
+
private static counts;
|
|
929
|
+
private static tally;
|
|
930
|
+
private static usage;
|
|
931
|
+
private static tokenUsage;
|
|
932
|
+
private static error;
|
|
933
|
+
private path;
|
|
934
|
+
}
|
|
935
|
+
declare class ReportFileWriter {
|
|
936
|
+
private readonly directory;
|
|
937
|
+
constructor(directory: string);
|
|
938
|
+
path(fileName: string): string;
|
|
939
|
+
write(fileName: string, content: string): string;
|
|
940
|
+
}
|
|
941
|
+
export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
|