@mappies/miniwat-eval 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -11
- package/dist/index.cjs +743 -178
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +65 -48
- package/dist/index.d.ts +66 -48
- package/dist/index.js +743 -178
- package/dist/index.js.map +1 -1
- package/dist/main.js +699 -134
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +43 -4
- package/schemas/eval-config.schema.json +10 -0
- package/schemas/report.schema.json +12 -1
package/dist/index.d.cts
CHANGED
|
@@ -224,13 +224,38 @@ interface ITurnSpec {
|
|
|
224
224
|
expect: ITurnExpectation;
|
|
225
225
|
location: ISourceLocation;
|
|
226
226
|
}
|
|
227
|
+
interface IProtocolCapabilities {
|
|
228
|
+
toolUses: boolean;
|
|
229
|
+
toolResults: boolean;
|
|
230
|
+
tokenUsage: boolean;
|
|
231
|
+
}
|
|
232
|
+
export declare const RequestFormat: {
|
|
233
|
+
readonly Json: "json";
|
|
234
|
+
};
|
|
235
|
+
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
236
|
+
interface IRequestConfig {
|
|
237
|
+
format: RequestFormat;
|
|
238
|
+
value: object | undefined;
|
|
239
|
+
inputPath: string | undefined;
|
|
240
|
+
}
|
|
241
|
+
interface IProtocolConfig {
|
|
242
|
+
request: IRequestConfig;
|
|
243
|
+
capabilities: IProtocolCapabilities;
|
|
244
|
+
}
|
|
245
|
+
interface IWebSocketConfig {
|
|
246
|
+
endpoint: string;
|
|
247
|
+
connection: IConnectionConfig;
|
|
248
|
+
protocol: IProtocolConfig;
|
|
249
|
+
}
|
|
227
250
|
interface IConversationSpec {
|
|
228
251
|
version: number;
|
|
229
252
|
name: string;
|
|
253
|
+
runName: string;
|
|
230
254
|
description: string;
|
|
231
255
|
turns: ITurnSpec[];
|
|
232
256
|
conversation: IConversationExpectation | undefined;
|
|
233
257
|
location: ISourceLocation;
|
|
258
|
+
websocket: IWebSocketConfig | undefined;
|
|
234
259
|
secrets: string[];
|
|
235
260
|
}
|
|
236
261
|
declare class ResponseExpectation implements IResponseExpectation {
|
|
@@ -260,13 +285,38 @@ declare class TurnSpec implements ITurnSpec {
|
|
|
260
285
|
location: SourceLocation;
|
|
261
286
|
constructor(data?: Partial<ITurnSpec>);
|
|
262
287
|
}
|
|
288
|
+
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
289
|
+
toolUses: boolean;
|
|
290
|
+
toolResults: boolean;
|
|
291
|
+
tokenUsage: boolean;
|
|
292
|
+
constructor(data?: Partial<IProtocolCapabilities>);
|
|
293
|
+
}
|
|
294
|
+
declare class RequestConfig implements IRequestConfig {
|
|
295
|
+
format: RequestFormat;
|
|
296
|
+
value: object | undefined;
|
|
297
|
+
inputPath: string | undefined;
|
|
298
|
+
constructor(data?: Partial<IRequestConfig>);
|
|
299
|
+
}
|
|
300
|
+
declare class ProtocolConfig implements IProtocolConfig {
|
|
301
|
+
request: RequestConfig;
|
|
302
|
+
capabilities: ProtocolCapabilities;
|
|
303
|
+
constructor(data?: Partial<IProtocolConfig>);
|
|
304
|
+
}
|
|
305
|
+
declare class WebSocketConfig implements IWebSocketConfig {
|
|
306
|
+
endpoint: string;
|
|
307
|
+
connection: ConnectionConfig;
|
|
308
|
+
protocol: ProtocolConfig;
|
|
309
|
+
constructor(data?: Partial<IWebSocketConfig>);
|
|
310
|
+
}
|
|
263
311
|
declare class ConversationSpec implements IConversationSpec {
|
|
264
312
|
version: number;
|
|
265
313
|
name: string;
|
|
314
|
+
runName: string;
|
|
266
315
|
description: string;
|
|
267
316
|
turns: TurnSpec[];
|
|
268
317
|
conversation: ConversationExpectation | undefined;
|
|
269
318
|
location: SourceLocation;
|
|
319
|
+
websocket: WebSocketConfig | undefined;
|
|
270
320
|
secrets: string[];
|
|
271
321
|
constructor(data?: Partial<IConversationSpec>);
|
|
272
322
|
}
|
|
@@ -336,33 +386,12 @@ interface IReportConfig {
|
|
|
336
386
|
directory: string;
|
|
337
387
|
formats: ReportFormat[];
|
|
338
388
|
}
|
|
339
|
-
|
|
340
|
-
toolUses: boolean;
|
|
341
|
-
toolResults: boolean;
|
|
342
|
-
tokenUsage: boolean;
|
|
343
|
-
}
|
|
344
|
-
export declare const RequestFormat: {
|
|
345
|
-
readonly Json: "json";
|
|
346
|
-
};
|
|
347
|
-
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
348
|
-
interface IRequestConfig {
|
|
349
|
-
format: RequestFormat;
|
|
350
|
-
value: object | undefined;
|
|
351
|
-
inputPath: string | undefined;
|
|
352
|
-
}
|
|
353
|
-
interface IProtocolConfig {
|
|
354
|
-
request: IRequestConfig;
|
|
355
|
-
capabilities: IProtocolCapabilities;
|
|
356
|
-
}
|
|
357
|
-
interface IWebSocketConfig {
|
|
358
|
-
endpoint: string;
|
|
359
|
-
connection: IConnectionConfig;
|
|
360
|
-
protocol: IProtocolConfig;
|
|
361
|
-
}
|
|
389
|
+
type Variables = Record<string, string | string[]>;
|
|
362
390
|
interface IEvalConfig {
|
|
363
391
|
version: number;
|
|
364
392
|
configPath: string;
|
|
365
393
|
parametersPath: string;
|
|
394
|
+
variables: Variables;
|
|
366
395
|
websocket: IWebSocketConfig;
|
|
367
396
|
execution: IExecutionConfig;
|
|
368
397
|
evaluator: IEvaluatorConfig;
|
|
@@ -407,33 +436,11 @@ declare class ReportConfig implements IReportConfig {
|
|
|
407
436
|
formats: ReportFormat[];
|
|
408
437
|
constructor(data?: Partial<IReportConfig>);
|
|
409
438
|
}
|
|
410
|
-
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
411
|
-
toolUses: boolean;
|
|
412
|
-
toolResults: boolean;
|
|
413
|
-
tokenUsage: boolean;
|
|
414
|
-
constructor(data?: Partial<IProtocolCapabilities>);
|
|
415
|
-
}
|
|
416
|
-
declare class RequestConfig implements IRequestConfig {
|
|
417
|
-
format: RequestFormat;
|
|
418
|
-
value: object | undefined;
|
|
419
|
-
inputPath: string | undefined;
|
|
420
|
-
constructor(data?: Partial<IRequestConfig>);
|
|
421
|
-
}
|
|
422
|
-
declare class ProtocolConfig implements IProtocolConfig {
|
|
423
|
-
request: RequestConfig;
|
|
424
|
-
capabilities: ProtocolCapabilities;
|
|
425
|
-
constructor(data?: Partial<IProtocolConfig>);
|
|
426
|
-
}
|
|
427
|
-
declare class WebSocketConfig implements IWebSocketConfig {
|
|
428
|
-
endpoint: string;
|
|
429
|
-
connection: ConnectionConfig;
|
|
430
|
-
protocol: ProtocolConfig;
|
|
431
|
-
constructor(data?: Partial<IWebSocketConfig>);
|
|
432
|
-
}
|
|
433
439
|
declare class EvalConfig implements IEvalConfig {
|
|
434
440
|
version: number;
|
|
435
441
|
configPath: string;
|
|
436
442
|
parametersPath: string;
|
|
443
|
+
variables: Variables;
|
|
437
444
|
websocket: WebSocketConfig;
|
|
438
445
|
execution: ExecutionConfig;
|
|
439
446
|
evaluator: EvaluatorConfig;
|
|
@@ -675,17 +682,18 @@ declare class EvalConfigLoader {
|
|
|
675
682
|
private readonly cwd;
|
|
676
683
|
private readonly documentLoader;
|
|
677
684
|
private readonly schemaValidator;
|
|
685
|
+
private readonly variablesReader;
|
|
686
|
+
private readonly webSocketResolver;
|
|
678
687
|
private readonly environmentResolver;
|
|
679
688
|
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
680
689
|
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
690
|
+
static buildWebSocket(value: unknown): WebSocketConfig;
|
|
681
691
|
private static applyReport;
|
|
682
692
|
private validateSecretReferences;
|
|
683
|
-
private validateInputPath;
|
|
684
693
|
private build;
|
|
685
694
|
private applyRepetition;
|
|
686
695
|
private static secrets;
|
|
687
696
|
private static section;
|
|
688
|
-
private static child;
|
|
689
697
|
}
|
|
690
698
|
declare class ParametersLoader {
|
|
691
699
|
private readonly cwd;
|
|
@@ -701,9 +709,17 @@ declare class ScenarioLoader {
|
|
|
701
709
|
private readonly documentLoader;
|
|
702
710
|
private readonly schemaValidator;
|
|
703
711
|
private readonly conversationValidator;
|
|
712
|
+
private readonly variablesReader;
|
|
713
|
+
private readonly runWithReader;
|
|
714
|
+
private readonly webSocketResolver;
|
|
704
715
|
private readonly environmentResolver;
|
|
705
716
|
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
706
717
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
718
|
+
private resolveRun;
|
|
719
|
+
private static byRun;
|
|
720
|
+
private static shared;
|
|
721
|
+
private static variablesLocation;
|
|
722
|
+
private static unusedVariables;
|
|
707
723
|
discover(target: string, config: EvalConfig): string[];
|
|
708
724
|
private walk;
|
|
709
725
|
private build;
|
|
@@ -899,6 +915,7 @@ declare class ConsoleReporter {
|
|
|
899
915
|
constructor(verbose: boolean, colour: boolean);
|
|
900
916
|
header(run: RunHeader): string;
|
|
901
917
|
scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
|
|
918
|
+
static title(spec: ConversationSpec): string;
|
|
902
919
|
summary(run: EvalRunResult, exitCode: number, reportPaths?: string[]): string;
|
|
903
920
|
static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
|
|
904
921
|
static finalLine(run: EvalRunResult, exitCode: number): string;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ITokenUsage, Message, ResponseMessage, TokenUsage, ToolResult, ToolUse } from "@mappies/miniwat-agent-core";
|
|
2
|
+
import "yaml";
|
|
2
3
|
import { ClientOptions } from "configurapi-handler-ws";
|
|
3
4
|
import { Agent } from "@mappies/miniwat-agent";
|
|
4
5
|
interface ITurnExecution {
|
|
@@ -224,13 +225,38 @@ interface ITurnSpec {
|
|
|
224
225
|
expect: ITurnExpectation;
|
|
225
226
|
location: ISourceLocation;
|
|
226
227
|
}
|
|
228
|
+
interface IProtocolCapabilities {
|
|
229
|
+
toolUses: boolean;
|
|
230
|
+
toolResults: boolean;
|
|
231
|
+
tokenUsage: boolean;
|
|
232
|
+
}
|
|
233
|
+
export declare const RequestFormat: {
|
|
234
|
+
readonly Json: "json";
|
|
235
|
+
};
|
|
236
|
+
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
237
|
+
interface IRequestConfig {
|
|
238
|
+
format: RequestFormat;
|
|
239
|
+
value: object | undefined;
|
|
240
|
+
inputPath: string | undefined;
|
|
241
|
+
}
|
|
242
|
+
interface IProtocolConfig {
|
|
243
|
+
request: IRequestConfig;
|
|
244
|
+
capabilities: IProtocolCapabilities;
|
|
245
|
+
}
|
|
246
|
+
interface IWebSocketConfig {
|
|
247
|
+
endpoint: string;
|
|
248
|
+
connection: IConnectionConfig;
|
|
249
|
+
protocol: IProtocolConfig;
|
|
250
|
+
}
|
|
227
251
|
interface IConversationSpec {
|
|
228
252
|
version: number;
|
|
229
253
|
name: string;
|
|
254
|
+
runName: string;
|
|
230
255
|
description: string;
|
|
231
256
|
turns: ITurnSpec[];
|
|
232
257
|
conversation: IConversationExpectation | undefined;
|
|
233
258
|
location: ISourceLocation;
|
|
259
|
+
websocket: IWebSocketConfig | undefined;
|
|
234
260
|
secrets: string[];
|
|
235
261
|
}
|
|
236
262
|
declare class ResponseExpectation implements IResponseExpectation {
|
|
@@ -260,13 +286,38 @@ declare class TurnSpec implements ITurnSpec {
|
|
|
260
286
|
location: SourceLocation;
|
|
261
287
|
constructor(data?: Partial<ITurnSpec>);
|
|
262
288
|
}
|
|
289
|
+
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
290
|
+
toolUses: boolean;
|
|
291
|
+
toolResults: boolean;
|
|
292
|
+
tokenUsage: boolean;
|
|
293
|
+
constructor(data?: Partial<IProtocolCapabilities>);
|
|
294
|
+
}
|
|
295
|
+
declare class RequestConfig implements IRequestConfig {
|
|
296
|
+
format: RequestFormat;
|
|
297
|
+
value: object | undefined;
|
|
298
|
+
inputPath: string | undefined;
|
|
299
|
+
constructor(data?: Partial<IRequestConfig>);
|
|
300
|
+
}
|
|
301
|
+
declare class ProtocolConfig implements IProtocolConfig {
|
|
302
|
+
request: RequestConfig;
|
|
303
|
+
capabilities: ProtocolCapabilities;
|
|
304
|
+
constructor(data?: Partial<IProtocolConfig>);
|
|
305
|
+
}
|
|
306
|
+
declare class WebSocketConfig implements IWebSocketConfig {
|
|
307
|
+
endpoint: string;
|
|
308
|
+
connection: ConnectionConfig;
|
|
309
|
+
protocol: ProtocolConfig;
|
|
310
|
+
constructor(data?: Partial<IWebSocketConfig>);
|
|
311
|
+
}
|
|
263
312
|
declare class ConversationSpec implements IConversationSpec {
|
|
264
313
|
version: number;
|
|
265
314
|
name: string;
|
|
315
|
+
runName: string;
|
|
266
316
|
description: string;
|
|
267
317
|
turns: TurnSpec[];
|
|
268
318
|
conversation: ConversationExpectation | undefined;
|
|
269
319
|
location: SourceLocation;
|
|
320
|
+
websocket: WebSocketConfig | undefined;
|
|
270
321
|
secrets: string[];
|
|
271
322
|
constructor(data?: Partial<IConversationSpec>);
|
|
272
323
|
}
|
|
@@ -336,33 +387,12 @@ interface IReportConfig {
|
|
|
336
387
|
directory: string;
|
|
337
388
|
formats: ReportFormat[];
|
|
338
389
|
}
|
|
339
|
-
|
|
340
|
-
toolUses: boolean;
|
|
341
|
-
toolResults: boolean;
|
|
342
|
-
tokenUsage: boolean;
|
|
343
|
-
}
|
|
344
|
-
export declare const RequestFormat: {
|
|
345
|
-
readonly Json: "json";
|
|
346
|
-
};
|
|
347
|
-
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
348
|
-
interface IRequestConfig {
|
|
349
|
-
format: RequestFormat;
|
|
350
|
-
value: object | undefined;
|
|
351
|
-
inputPath: string | undefined;
|
|
352
|
-
}
|
|
353
|
-
interface IProtocolConfig {
|
|
354
|
-
request: IRequestConfig;
|
|
355
|
-
capabilities: IProtocolCapabilities;
|
|
356
|
-
}
|
|
357
|
-
interface IWebSocketConfig {
|
|
358
|
-
endpoint: string;
|
|
359
|
-
connection: IConnectionConfig;
|
|
360
|
-
protocol: IProtocolConfig;
|
|
361
|
-
}
|
|
390
|
+
type Variables = Record<string, string | string[]>;
|
|
362
391
|
interface IEvalConfig {
|
|
363
392
|
version: number;
|
|
364
393
|
configPath: string;
|
|
365
394
|
parametersPath: string;
|
|
395
|
+
variables: Variables;
|
|
366
396
|
websocket: IWebSocketConfig;
|
|
367
397
|
execution: IExecutionConfig;
|
|
368
398
|
evaluator: IEvaluatorConfig;
|
|
@@ -407,33 +437,11 @@ declare class ReportConfig implements IReportConfig {
|
|
|
407
437
|
formats: ReportFormat[];
|
|
408
438
|
constructor(data?: Partial<IReportConfig>);
|
|
409
439
|
}
|
|
410
|
-
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
411
|
-
toolUses: boolean;
|
|
412
|
-
toolResults: boolean;
|
|
413
|
-
tokenUsage: boolean;
|
|
414
|
-
constructor(data?: Partial<IProtocolCapabilities>);
|
|
415
|
-
}
|
|
416
|
-
declare class RequestConfig implements IRequestConfig {
|
|
417
|
-
format: RequestFormat;
|
|
418
|
-
value: object | undefined;
|
|
419
|
-
inputPath: string | undefined;
|
|
420
|
-
constructor(data?: Partial<IRequestConfig>);
|
|
421
|
-
}
|
|
422
|
-
declare class ProtocolConfig implements IProtocolConfig {
|
|
423
|
-
request: RequestConfig;
|
|
424
|
-
capabilities: ProtocolCapabilities;
|
|
425
|
-
constructor(data?: Partial<IProtocolConfig>);
|
|
426
|
-
}
|
|
427
|
-
declare class WebSocketConfig implements IWebSocketConfig {
|
|
428
|
-
endpoint: string;
|
|
429
|
-
connection: ConnectionConfig;
|
|
430
|
-
protocol: ProtocolConfig;
|
|
431
|
-
constructor(data?: Partial<IWebSocketConfig>);
|
|
432
|
-
}
|
|
433
440
|
declare class EvalConfig implements IEvalConfig {
|
|
434
441
|
version: number;
|
|
435
442
|
configPath: string;
|
|
436
443
|
parametersPath: string;
|
|
444
|
+
variables: Variables;
|
|
437
445
|
websocket: WebSocketConfig;
|
|
438
446
|
execution: ExecutionConfig;
|
|
439
447
|
evaluator: EvaluatorConfig;
|
|
@@ -675,17 +683,18 @@ declare class EvalConfigLoader {
|
|
|
675
683
|
private readonly cwd;
|
|
676
684
|
private readonly documentLoader;
|
|
677
685
|
private readonly schemaValidator;
|
|
686
|
+
private readonly variablesReader;
|
|
687
|
+
private readonly webSocketResolver;
|
|
678
688
|
private readonly environmentResolver;
|
|
679
689
|
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
680
690
|
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
691
|
+
static buildWebSocket(value: unknown): WebSocketConfig;
|
|
681
692
|
private static applyReport;
|
|
682
693
|
private validateSecretReferences;
|
|
683
|
-
private validateInputPath;
|
|
684
694
|
private build;
|
|
685
695
|
private applyRepetition;
|
|
686
696
|
private static secrets;
|
|
687
697
|
private static section;
|
|
688
|
-
private static child;
|
|
689
698
|
}
|
|
690
699
|
declare class ParametersLoader {
|
|
691
700
|
private readonly cwd;
|
|
@@ -701,9 +710,17 @@ declare class ScenarioLoader {
|
|
|
701
710
|
private readonly documentLoader;
|
|
702
711
|
private readonly schemaValidator;
|
|
703
712
|
private readonly conversationValidator;
|
|
713
|
+
private readonly variablesReader;
|
|
714
|
+
private readonly runWithReader;
|
|
715
|
+
private readonly webSocketResolver;
|
|
704
716
|
private readonly environmentResolver;
|
|
705
717
|
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
706
718
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
719
|
+
private resolveRun;
|
|
720
|
+
private static byRun;
|
|
721
|
+
private static shared;
|
|
722
|
+
private static variablesLocation;
|
|
723
|
+
private static unusedVariables;
|
|
707
724
|
discover(target: string, config: EvalConfig): string[];
|
|
708
725
|
private walk;
|
|
709
726
|
private build;
|
|
@@ -899,6 +916,7 @@ declare class ConsoleReporter {
|
|
|
899
916
|
constructor(verbose: boolean, colour: boolean);
|
|
900
917
|
header(run: RunHeader): string;
|
|
901
918
|
scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
|
|
919
|
+
static title(spec: ConversationSpec): string;
|
|
902
920
|
summary(run: EvalRunResult, exitCode: number, reportPaths?: string[]): string;
|
|
903
921
|
static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
|
|
904
922
|
static finalLine(run: EvalRunResult, exitCode: number): string;
|