@mappies/miniwat-eval 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -11
- package/dist/index.cjs +389 -173
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +58 -48
- package/dist/index.d.ts +58 -48
- package/dist/index.js +389 -173
- package/dist/index.js.map +1 -1
- package/dist/main.js +345 -129
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +9 -4
- package/schemas/eval-config.schema.json +5 -0
- package/schemas/report.schema.json +11 -1
package/dist/index.d.cts
CHANGED
|
@@ -224,6 +224,29 @@ interface ITurnSpec {
|
|
|
224
224
|
expect: ITurnExpectation;
|
|
225
225
|
location: ISourceLocation;
|
|
226
226
|
}
|
|
227
|
+
interface IProtocolCapabilities {
|
|
228
|
+
toolUses: boolean;
|
|
229
|
+
toolResults: boolean;
|
|
230
|
+
tokenUsage: boolean;
|
|
231
|
+
}
|
|
232
|
+
export declare const RequestFormat: {
|
|
233
|
+
readonly Json: "json";
|
|
234
|
+
};
|
|
235
|
+
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
236
|
+
interface IRequestConfig {
|
|
237
|
+
format: RequestFormat;
|
|
238
|
+
value: object | undefined;
|
|
239
|
+
inputPath: string | undefined;
|
|
240
|
+
}
|
|
241
|
+
interface IProtocolConfig {
|
|
242
|
+
request: IRequestConfig;
|
|
243
|
+
capabilities: IProtocolCapabilities;
|
|
244
|
+
}
|
|
245
|
+
interface IWebSocketConfig {
|
|
246
|
+
endpoint: string;
|
|
247
|
+
connection: IConnectionConfig;
|
|
248
|
+
protocol: IProtocolConfig;
|
|
249
|
+
}
|
|
227
250
|
interface IConversationSpec {
|
|
228
251
|
version: number;
|
|
229
252
|
name: string;
|
|
@@ -231,6 +254,7 @@ interface IConversationSpec {
|
|
|
231
254
|
turns: ITurnSpec[];
|
|
232
255
|
conversation: IConversationExpectation | undefined;
|
|
233
256
|
location: ISourceLocation;
|
|
257
|
+
websocket: IWebSocketConfig | undefined;
|
|
234
258
|
secrets: string[];
|
|
235
259
|
}
|
|
236
260
|
declare class ResponseExpectation implements IResponseExpectation {
|
|
@@ -260,6 +284,29 @@ declare class TurnSpec implements ITurnSpec {
|
|
|
260
284
|
location: SourceLocation;
|
|
261
285
|
constructor(data?: Partial<ITurnSpec>);
|
|
262
286
|
}
|
|
287
|
+
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
288
|
+
toolUses: boolean;
|
|
289
|
+
toolResults: boolean;
|
|
290
|
+
tokenUsage: boolean;
|
|
291
|
+
constructor(data?: Partial<IProtocolCapabilities>);
|
|
292
|
+
}
|
|
293
|
+
declare class RequestConfig implements IRequestConfig {
|
|
294
|
+
format: RequestFormat;
|
|
295
|
+
value: object | undefined;
|
|
296
|
+
inputPath: string | undefined;
|
|
297
|
+
constructor(data?: Partial<IRequestConfig>);
|
|
298
|
+
}
|
|
299
|
+
declare class ProtocolConfig implements IProtocolConfig {
|
|
300
|
+
request: RequestConfig;
|
|
301
|
+
capabilities: ProtocolCapabilities;
|
|
302
|
+
constructor(data?: Partial<IProtocolConfig>);
|
|
303
|
+
}
|
|
304
|
+
declare class WebSocketConfig implements IWebSocketConfig {
|
|
305
|
+
endpoint: string;
|
|
306
|
+
connection: ConnectionConfig;
|
|
307
|
+
protocol: ProtocolConfig;
|
|
308
|
+
constructor(data?: Partial<IWebSocketConfig>);
|
|
309
|
+
}
|
|
263
310
|
declare class ConversationSpec implements IConversationSpec {
|
|
264
311
|
version: number;
|
|
265
312
|
name: string;
|
|
@@ -267,6 +314,7 @@ declare class ConversationSpec implements IConversationSpec {
|
|
|
267
314
|
turns: TurnSpec[];
|
|
268
315
|
conversation: ConversationExpectation | undefined;
|
|
269
316
|
location: SourceLocation;
|
|
317
|
+
websocket: WebSocketConfig | undefined;
|
|
270
318
|
secrets: string[];
|
|
271
319
|
constructor(data?: Partial<IConversationSpec>);
|
|
272
320
|
}
|
|
@@ -336,33 +384,11 @@ interface IReportConfig {
|
|
|
336
384
|
directory: string;
|
|
337
385
|
formats: ReportFormat[];
|
|
338
386
|
}
|
|
339
|
-
interface IProtocolCapabilities {
|
|
340
|
-
toolUses: boolean;
|
|
341
|
-
toolResults: boolean;
|
|
342
|
-
tokenUsage: boolean;
|
|
343
|
-
}
|
|
344
|
-
export declare const RequestFormat: {
|
|
345
|
-
readonly Json: "json";
|
|
346
|
-
};
|
|
347
|
-
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
348
|
-
interface IRequestConfig {
|
|
349
|
-
format: RequestFormat;
|
|
350
|
-
value: object | undefined;
|
|
351
|
-
inputPath: string | undefined;
|
|
352
|
-
}
|
|
353
|
-
interface IProtocolConfig {
|
|
354
|
-
request: IRequestConfig;
|
|
355
|
-
capabilities: IProtocolCapabilities;
|
|
356
|
-
}
|
|
357
|
-
interface IWebSocketConfig {
|
|
358
|
-
endpoint: string;
|
|
359
|
-
connection: IConnectionConfig;
|
|
360
|
-
protocol: IProtocolConfig;
|
|
361
|
-
}
|
|
362
387
|
interface IEvalConfig {
|
|
363
388
|
version: number;
|
|
364
389
|
configPath: string;
|
|
365
390
|
parametersPath: string;
|
|
391
|
+
variables: Record<string, string>;
|
|
366
392
|
websocket: IWebSocketConfig;
|
|
367
393
|
execution: IExecutionConfig;
|
|
368
394
|
evaluator: IEvaluatorConfig;
|
|
@@ -407,33 +433,11 @@ declare class ReportConfig implements IReportConfig {
|
|
|
407
433
|
formats: ReportFormat[];
|
|
408
434
|
constructor(data?: Partial<IReportConfig>);
|
|
409
435
|
}
|
|
410
|
-
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
411
|
-
toolUses: boolean;
|
|
412
|
-
toolResults: boolean;
|
|
413
|
-
tokenUsage: boolean;
|
|
414
|
-
constructor(data?: Partial<IProtocolCapabilities>);
|
|
415
|
-
}
|
|
416
|
-
declare class RequestConfig implements IRequestConfig {
|
|
417
|
-
format: RequestFormat;
|
|
418
|
-
value: object | undefined;
|
|
419
|
-
inputPath: string | undefined;
|
|
420
|
-
constructor(data?: Partial<IRequestConfig>);
|
|
421
|
-
}
|
|
422
|
-
declare class ProtocolConfig implements IProtocolConfig {
|
|
423
|
-
request: RequestConfig;
|
|
424
|
-
capabilities: ProtocolCapabilities;
|
|
425
|
-
constructor(data?: Partial<IProtocolConfig>);
|
|
426
|
-
}
|
|
427
|
-
declare class WebSocketConfig implements IWebSocketConfig {
|
|
428
|
-
endpoint: string;
|
|
429
|
-
connection: ConnectionConfig;
|
|
430
|
-
protocol: ProtocolConfig;
|
|
431
|
-
constructor(data?: Partial<IWebSocketConfig>);
|
|
432
|
-
}
|
|
433
436
|
declare class EvalConfig implements IEvalConfig {
|
|
434
437
|
version: number;
|
|
435
438
|
configPath: string;
|
|
436
439
|
parametersPath: string;
|
|
440
|
+
variables: Record<string, string>;
|
|
437
441
|
websocket: WebSocketConfig;
|
|
438
442
|
execution: ExecutionConfig;
|
|
439
443
|
evaluator: EvaluatorConfig;
|
|
@@ -675,17 +679,18 @@ declare class EvalConfigLoader {
|
|
|
675
679
|
private readonly cwd;
|
|
676
680
|
private readonly documentLoader;
|
|
677
681
|
private readonly schemaValidator;
|
|
682
|
+
private readonly variablesReader;
|
|
683
|
+
private readonly webSocketResolver;
|
|
678
684
|
private readonly environmentResolver;
|
|
679
685
|
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
680
686
|
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
687
|
+
static buildWebSocket(value: unknown): WebSocketConfig;
|
|
681
688
|
private static applyReport;
|
|
682
689
|
private validateSecretReferences;
|
|
683
|
-
private validateInputPath;
|
|
684
690
|
private build;
|
|
685
691
|
private applyRepetition;
|
|
686
692
|
private static secrets;
|
|
687
693
|
private static section;
|
|
688
|
-
private static child;
|
|
689
694
|
}
|
|
690
695
|
declare class ParametersLoader {
|
|
691
696
|
private readonly cwd;
|
|
@@ -701,9 +706,14 @@ declare class ScenarioLoader {
|
|
|
701
706
|
private readonly documentLoader;
|
|
702
707
|
private readonly schemaValidator;
|
|
703
708
|
private readonly conversationValidator;
|
|
709
|
+
private readonly variablesReader;
|
|
710
|
+
private readonly webSocketResolver;
|
|
704
711
|
private readonly environmentResolver;
|
|
705
712
|
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
706
713
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
714
|
+
private static shared;
|
|
715
|
+
private static variablesLocation;
|
|
716
|
+
private static unusedVariables;
|
|
707
717
|
discover(target: string, config: EvalConfig): string[];
|
|
708
718
|
private walk;
|
|
709
719
|
private build;
|
package/dist/index.d.ts
CHANGED
|
@@ -224,6 +224,29 @@ interface ITurnSpec {
|
|
|
224
224
|
expect: ITurnExpectation;
|
|
225
225
|
location: ISourceLocation;
|
|
226
226
|
}
|
|
227
|
+
interface IProtocolCapabilities {
|
|
228
|
+
toolUses: boolean;
|
|
229
|
+
toolResults: boolean;
|
|
230
|
+
tokenUsage: boolean;
|
|
231
|
+
}
|
|
232
|
+
export declare const RequestFormat: {
|
|
233
|
+
readonly Json: "json";
|
|
234
|
+
};
|
|
235
|
+
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
236
|
+
interface IRequestConfig {
|
|
237
|
+
format: RequestFormat;
|
|
238
|
+
value: object | undefined;
|
|
239
|
+
inputPath: string | undefined;
|
|
240
|
+
}
|
|
241
|
+
interface IProtocolConfig {
|
|
242
|
+
request: IRequestConfig;
|
|
243
|
+
capabilities: IProtocolCapabilities;
|
|
244
|
+
}
|
|
245
|
+
interface IWebSocketConfig {
|
|
246
|
+
endpoint: string;
|
|
247
|
+
connection: IConnectionConfig;
|
|
248
|
+
protocol: IProtocolConfig;
|
|
249
|
+
}
|
|
227
250
|
interface IConversationSpec {
|
|
228
251
|
version: number;
|
|
229
252
|
name: string;
|
|
@@ -231,6 +254,7 @@ interface IConversationSpec {
|
|
|
231
254
|
turns: ITurnSpec[];
|
|
232
255
|
conversation: IConversationExpectation | undefined;
|
|
233
256
|
location: ISourceLocation;
|
|
257
|
+
websocket: IWebSocketConfig | undefined;
|
|
234
258
|
secrets: string[];
|
|
235
259
|
}
|
|
236
260
|
declare class ResponseExpectation implements IResponseExpectation {
|
|
@@ -260,6 +284,29 @@ declare class TurnSpec implements ITurnSpec {
|
|
|
260
284
|
location: SourceLocation;
|
|
261
285
|
constructor(data?: Partial<ITurnSpec>);
|
|
262
286
|
}
|
|
287
|
+
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
288
|
+
toolUses: boolean;
|
|
289
|
+
toolResults: boolean;
|
|
290
|
+
tokenUsage: boolean;
|
|
291
|
+
constructor(data?: Partial<IProtocolCapabilities>);
|
|
292
|
+
}
|
|
293
|
+
declare class RequestConfig implements IRequestConfig {
|
|
294
|
+
format: RequestFormat;
|
|
295
|
+
value: object | undefined;
|
|
296
|
+
inputPath: string | undefined;
|
|
297
|
+
constructor(data?: Partial<IRequestConfig>);
|
|
298
|
+
}
|
|
299
|
+
declare class ProtocolConfig implements IProtocolConfig {
|
|
300
|
+
request: RequestConfig;
|
|
301
|
+
capabilities: ProtocolCapabilities;
|
|
302
|
+
constructor(data?: Partial<IProtocolConfig>);
|
|
303
|
+
}
|
|
304
|
+
declare class WebSocketConfig implements IWebSocketConfig {
|
|
305
|
+
endpoint: string;
|
|
306
|
+
connection: ConnectionConfig;
|
|
307
|
+
protocol: ProtocolConfig;
|
|
308
|
+
constructor(data?: Partial<IWebSocketConfig>);
|
|
309
|
+
}
|
|
263
310
|
declare class ConversationSpec implements IConversationSpec {
|
|
264
311
|
version: number;
|
|
265
312
|
name: string;
|
|
@@ -267,6 +314,7 @@ declare class ConversationSpec implements IConversationSpec {
|
|
|
267
314
|
turns: TurnSpec[];
|
|
268
315
|
conversation: ConversationExpectation | undefined;
|
|
269
316
|
location: SourceLocation;
|
|
317
|
+
websocket: WebSocketConfig | undefined;
|
|
270
318
|
secrets: string[];
|
|
271
319
|
constructor(data?: Partial<IConversationSpec>);
|
|
272
320
|
}
|
|
@@ -336,33 +384,11 @@ interface IReportConfig {
|
|
|
336
384
|
directory: string;
|
|
337
385
|
formats: ReportFormat[];
|
|
338
386
|
}
|
|
339
|
-
interface IProtocolCapabilities {
|
|
340
|
-
toolUses: boolean;
|
|
341
|
-
toolResults: boolean;
|
|
342
|
-
tokenUsage: boolean;
|
|
343
|
-
}
|
|
344
|
-
export declare const RequestFormat: {
|
|
345
|
-
readonly Json: "json";
|
|
346
|
-
};
|
|
347
|
-
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
348
|
-
interface IRequestConfig {
|
|
349
|
-
format: RequestFormat;
|
|
350
|
-
value: object | undefined;
|
|
351
|
-
inputPath: string | undefined;
|
|
352
|
-
}
|
|
353
|
-
interface IProtocolConfig {
|
|
354
|
-
request: IRequestConfig;
|
|
355
|
-
capabilities: IProtocolCapabilities;
|
|
356
|
-
}
|
|
357
|
-
interface IWebSocketConfig {
|
|
358
|
-
endpoint: string;
|
|
359
|
-
connection: IConnectionConfig;
|
|
360
|
-
protocol: IProtocolConfig;
|
|
361
|
-
}
|
|
362
387
|
interface IEvalConfig {
|
|
363
388
|
version: number;
|
|
364
389
|
configPath: string;
|
|
365
390
|
parametersPath: string;
|
|
391
|
+
variables: Record<string, string>;
|
|
366
392
|
websocket: IWebSocketConfig;
|
|
367
393
|
execution: IExecutionConfig;
|
|
368
394
|
evaluator: IEvaluatorConfig;
|
|
@@ -407,33 +433,11 @@ declare class ReportConfig implements IReportConfig {
|
|
|
407
433
|
formats: ReportFormat[];
|
|
408
434
|
constructor(data?: Partial<IReportConfig>);
|
|
409
435
|
}
|
|
410
|
-
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
411
|
-
toolUses: boolean;
|
|
412
|
-
toolResults: boolean;
|
|
413
|
-
tokenUsage: boolean;
|
|
414
|
-
constructor(data?: Partial<IProtocolCapabilities>);
|
|
415
|
-
}
|
|
416
|
-
declare class RequestConfig implements IRequestConfig {
|
|
417
|
-
format: RequestFormat;
|
|
418
|
-
value: object | undefined;
|
|
419
|
-
inputPath: string | undefined;
|
|
420
|
-
constructor(data?: Partial<IRequestConfig>);
|
|
421
|
-
}
|
|
422
|
-
declare class ProtocolConfig implements IProtocolConfig {
|
|
423
|
-
request: RequestConfig;
|
|
424
|
-
capabilities: ProtocolCapabilities;
|
|
425
|
-
constructor(data?: Partial<IProtocolConfig>);
|
|
426
|
-
}
|
|
427
|
-
declare class WebSocketConfig implements IWebSocketConfig {
|
|
428
|
-
endpoint: string;
|
|
429
|
-
connection: ConnectionConfig;
|
|
430
|
-
protocol: ProtocolConfig;
|
|
431
|
-
constructor(data?: Partial<IWebSocketConfig>);
|
|
432
|
-
}
|
|
433
436
|
declare class EvalConfig implements IEvalConfig {
|
|
434
437
|
version: number;
|
|
435
438
|
configPath: string;
|
|
436
439
|
parametersPath: string;
|
|
440
|
+
variables: Record<string, string>;
|
|
437
441
|
websocket: WebSocketConfig;
|
|
438
442
|
execution: ExecutionConfig;
|
|
439
443
|
evaluator: EvaluatorConfig;
|
|
@@ -675,17 +679,18 @@ declare class EvalConfigLoader {
|
|
|
675
679
|
private readonly cwd;
|
|
676
680
|
private readonly documentLoader;
|
|
677
681
|
private readonly schemaValidator;
|
|
682
|
+
private readonly variablesReader;
|
|
683
|
+
private readonly webSocketResolver;
|
|
678
684
|
private readonly environmentResolver;
|
|
679
685
|
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
680
686
|
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
687
|
+
static buildWebSocket(value: unknown): WebSocketConfig;
|
|
681
688
|
private static applyReport;
|
|
682
689
|
private validateSecretReferences;
|
|
683
|
-
private validateInputPath;
|
|
684
690
|
private build;
|
|
685
691
|
private applyRepetition;
|
|
686
692
|
private static secrets;
|
|
687
693
|
private static section;
|
|
688
|
-
private static child;
|
|
689
694
|
}
|
|
690
695
|
declare class ParametersLoader {
|
|
691
696
|
private readonly cwd;
|
|
@@ -701,9 +706,14 @@ declare class ScenarioLoader {
|
|
|
701
706
|
private readonly documentLoader;
|
|
702
707
|
private readonly schemaValidator;
|
|
703
708
|
private readonly conversationValidator;
|
|
709
|
+
private readonly variablesReader;
|
|
710
|
+
private readonly webSocketResolver;
|
|
704
711
|
private readonly environmentResolver;
|
|
705
712
|
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
706
713
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
714
|
+
private static shared;
|
|
715
|
+
private static variablesLocation;
|
|
716
|
+
private static unusedVariables;
|
|
707
717
|
discover(target: string, config: EvalConfig): string[];
|
|
708
718
|
private walk;
|
|
709
719
|
private build;
|