@mappies/miniwat-eval 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -224,6 +224,29 @@ interface ITurnSpec {
224
224
  expect: ITurnExpectation;
225
225
  location: ISourceLocation;
226
226
  }
227
+ interface IProtocolCapabilities {
228
+ toolUses: boolean;
229
+ toolResults: boolean;
230
+ tokenUsage: boolean;
231
+ }
232
+ export declare const RequestFormat: {
233
+ readonly Json: "json";
234
+ };
235
+ export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
236
+ interface IRequestConfig {
237
+ format: RequestFormat;
238
+ value: object | undefined;
239
+ inputPath: string | undefined;
240
+ }
241
+ interface IProtocolConfig {
242
+ request: IRequestConfig;
243
+ capabilities: IProtocolCapabilities;
244
+ }
245
+ interface IWebSocketConfig {
246
+ endpoint: string;
247
+ connection: IConnectionConfig;
248
+ protocol: IProtocolConfig;
249
+ }
227
250
  interface IConversationSpec {
228
251
  version: number;
229
252
  name: string;
@@ -231,6 +254,7 @@ interface IConversationSpec {
231
254
  turns: ITurnSpec[];
232
255
  conversation: IConversationExpectation | undefined;
233
256
  location: ISourceLocation;
257
+ websocket: IWebSocketConfig | undefined;
234
258
  secrets: string[];
235
259
  }
236
260
  declare class ResponseExpectation implements IResponseExpectation {
@@ -260,6 +284,29 @@ declare class TurnSpec implements ITurnSpec {
260
284
  location: SourceLocation;
261
285
  constructor(data?: Partial<ITurnSpec>);
262
286
  }
287
+ declare class ProtocolCapabilities implements IProtocolCapabilities {
288
+ toolUses: boolean;
289
+ toolResults: boolean;
290
+ tokenUsage: boolean;
291
+ constructor(data?: Partial<IProtocolCapabilities>);
292
+ }
293
+ declare class RequestConfig implements IRequestConfig {
294
+ format: RequestFormat;
295
+ value: object | undefined;
296
+ inputPath: string | undefined;
297
+ constructor(data?: Partial<IRequestConfig>);
298
+ }
299
+ declare class ProtocolConfig implements IProtocolConfig {
300
+ request: RequestConfig;
301
+ capabilities: ProtocolCapabilities;
302
+ constructor(data?: Partial<IProtocolConfig>);
303
+ }
304
+ declare class WebSocketConfig implements IWebSocketConfig {
305
+ endpoint: string;
306
+ connection: ConnectionConfig;
307
+ protocol: ProtocolConfig;
308
+ constructor(data?: Partial<IWebSocketConfig>);
309
+ }
263
310
  declare class ConversationSpec implements IConversationSpec {
264
311
  version: number;
265
312
  name: string;
@@ -267,6 +314,7 @@ declare class ConversationSpec implements IConversationSpec {
267
314
  turns: TurnSpec[];
268
315
  conversation: ConversationExpectation | undefined;
269
316
  location: SourceLocation;
317
+ websocket: WebSocketConfig | undefined;
270
318
  secrets: string[];
271
319
  constructor(data?: Partial<IConversationSpec>);
272
320
  }
@@ -336,33 +384,11 @@ interface IReportConfig {
336
384
  directory: string;
337
385
  formats: ReportFormat[];
338
386
  }
339
- interface IProtocolCapabilities {
340
- toolUses: boolean;
341
- toolResults: boolean;
342
- tokenUsage: boolean;
343
- }
344
- export declare const RequestFormat: {
345
- readonly Json: "json";
346
- };
347
- export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
348
- interface IRequestConfig {
349
- format: RequestFormat;
350
- value: object | undefined;
351
- inputPath: string | undefined;
352
- }
353
- interface IProtocolConfig {
354
- request: IRequestConfig;
355
- capabilities: IProtocolCapabilities;
356
- }
357
- interface IWebSocketConfig {
358
- endpoint: string;
359
- connection: IConnectionConfig;
360
- protocol: IProtocolConfig;
361
- }
362
387
  interface IEvalConfig {
363
388
  version: number;
364
389
  configPath: string;
365
390
  parametersPath: string;
391
+ variables: Record<string, string>;
366
392
  websocket: IWebSocketConfig;
367
393
  execution: IExecutionConfig;
368
394
  evaluator: IEvaluatorConfig;
@@ -407,33 +433,11 @@ declare class ReportConfig implements IReportConfig {
407
433
  formats: ReportFormat[];
408
434
  constructor(data?: Partial<IReportConfig>);
409
435
  }
410
- declare class ProtocolCapabilities implements IProtocolCapabilities {
411
- toolUses: boolean;
412
- toolResults: boolean;
413
- tokenUsage: boolean;
414
- constructor(data?: Partial<IProtocolCapabilities>);
415
- }
416
- declare class RequestConfig implements IRequestConfig {
417
- format: RequestFormat;
418
- value: object | undefined;
419
- inputPath: string | undefined;
420
- constructor(data?: Partial<IRequestConfig>);
421
- }
422
- declare class ProtocolConfig implements IProtocolConfig {
423
- request: RequestConfig;
424
- capabilities: ProtocolCapabilities;
425
- constructor(data?: Partial<IProtocolConfig>);
426
- }
427
- declare class WebSocketConfig implements IWebSocketConfig {
428
- endpoint: string;
429
- connection: ConnectionConfig;
430
- protocol: ProtocolConfig;
431
- constructor(data?: Partial<IWebSocketConfig>);
432
- }
433
436
  declare class EvalConfig implements IEvalConfig {
434
437
  version: number;
435
438
  configPath: string;
436
439
  parametersPath: string;
440
+ variables: Record<string, string>;
437
441
  websocket: WebSocketConfig;
438
442
  execution: ExecutionConfig;
439
443
  evaluator: EvaluatorConfig;
@@ -675,17 +679,18 @@ declare class EvalConfigLoader {
675
679
  private readonly cwd;
676
680
  private readonly documentLoader;
677
681
  private readonly schemaValidator;
682
+ private readonly variablesReader;
683
+ private readonly webSocketResolver;
678
684
  private readonly environmentResolver;
679
685
  constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
680
686
  load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
687
+ static buildWebSocket(value: unknown): WebSocketConfig;
681
688
  private static applyReport;
682
689
  private validateSecretReferences;
683
- private validateInputPath;
684
690
  private build;
685
691
  private applyRepetition;
686
692
  private static secrets;
687
693
  private static section;
688
- private static child;
689
694
  }
690
695
  declare class ParametersLoader {
691
696
  private readonly cwd;
@@ -701,9 +706,14 @@ declare class ScenarioLoader {
701
706
  private readonly documentLoader;
702
707
  private readonly schemaValidator;
703
708
  private readonly conversationValidator;
709
+ private readonly variablesReader;
710
+ private readonly webSocketResolver;
704
711
  private readonly environmentResolver;
705
712
  constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
706
713
  load(target: string, config: EvalConfig): ConversationSpec[];
714
+ private static shared;
715
+ private static variablesLocation;
716
+ private static unusedVariables;
707
717
  discover(target: string, config: EvalConfig): string[];
708
718
  private walk;
709
719
  private build;
package/dist/index.d.ts CHANGED
@@ -224,6 +224,29 @@ interface ITurnSpec {
224
224
  expect: ITurnExpectation;
225
225
  location: ISourceLocation;
226
226
  }
227
+ interface IProtocolCapabilities {
228
+ toolUses: boolean;
229
+ toolResults: boolean;
230
+ tokenUsage: boolean;
231
+ }
232
+ export declare const RequestFormat: {
233
+ readonly Json: "json";
234
+ };
235
+ export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
236
+ interface IRequestConfig {
237
+ format: RequestFormat;
238
+ value: object | undefined;
239
+ inputPath: string | undefined;
240
+ }
241
+ interface IProtocolConfig {
242
+ request: IRequestConfig;
243
+ capabilities: IProtocolCapabilities;
244
+ }
245
+ interface IWebSocketConfig {
246
+ endpoint: string;
247
+ connection: IConnectionConfig;
248
+ protocol: IProtocolConfig;
249
+ }
227
250
  interface IConversationSpec {
228
251
  version: number;
229
252
  name: string;
@@ -231,6 +254,7 @@ interface IConversationSpec {
231
254
  turns: ITurnSpec[];
232
255
  conversation: IConversationExpectation | undefined;
233
256
  location: ISourceLocation;
257
+ websocket: IWebSocketConfig | undefined;
234
258
  secrets: string[];
235
259
  }
236
260
  declare class ResponseExpectation implements IResponseExpectation {
@@ -260,6 +284,29 @@ declare class TurnSpec implements ITurnSpec {
260
284
  location: SourceLocation;
261
285
  constructor(data?: Partial<ITurnSpec>);
262
286
  }
287
+ declare class ProtocolCapabilities implements IProtocolCapabilities {
288
+ toolUses: boolean;
289
+ toolResults: boolean;
290
+ tokenUsage: boolean;
291
+ constructor(data?: Partial<IProtocolCapabilities>);
292
+ }
293
+ declare class RequestConfig implements IRequestConfig {
294
+ format: RequestFormat;
295
+ value: object | undefined;
296
+ inputPath: string | undefined;
297
+ constructor(data?: Partial<IRequestConfig>);
298
+ }
299
+ declare class ProtocolConfig implements IProtocolConfig {
300
+ request: RequestConfig;
301
+ capabilities: ProtocolCapabilities;
302
+ constructor(data?: Partial<IProtocolConfig>);
303
+ }
304
+ declare class WebSocketConfig implements IWebSocketConfig {
305
+ endpoint: string;
306
+ connection: ConnectionConfig;
307
+ protocol: ProtocolConfig;
308
+ constructor(data?: Partial<IWebSocketConfig>);
309
+ }
263
310
  declare class ConversationSpec implements IConversationSpec {
264
311
  version: number;
265
312
  name: string;
@@ -267,6 +314,7 @@ declare class ConversationSpec implements IConversationSpec {
267
314
  turns: TurnSpec[];
268
315
  conversation: ConversationExpectation | undefined;
269
316
  location: SourceLocation;
317
+ websocket: WebSocketConfig | undefined;
270
318
  secrets: string[];
271
319
  constructor(data?: Partial<IConversationSpec>);
272
320
  }
@@ -336,33 +384,11 @@ interface IReportConfig {
336
384
  directory: string;
337
385
  formats: ReportFormat[];
338
386
  }
339
- interface IProtocolCapabilities {
340
- toolUses: boolean;
341
- toolResults: boolean;
342
- tokenUsage: boolean;
343
- }
344
- export declare const RequestFormat: {
345
- readonly Json: "json";
346
- };
347
- export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
348
- interface IRequestConfig {
349
- format: RequestFormat;
350
- value: object | undefined;
351
- inputPath: string | undefined;
352
- }
353
- interface IProtocolConfig {
354
- request: IRequestConfig;
355
- capabilities: IProtocolCapabilities;
356
- }
357
- interface IWebSocketConfig {
358
- endpoint: string;
359
- connection: IConnectionConfig;
360
- protocol: IProtocolConfig;
361
- }
362
387
  interface IEvalConfig {
363
388
  version: number;
364
389
  configPath: string;
365
390
  parametersPath: string;
391
+ variables: Record<string, string>;
366
392
  websocket: IWebSocketConfig;
367
393
  execution: IExecutionConfig;
368
394
  evaluator: IEvaluatorConfig;
@@ -407,33 +433,11 @@ declare class ReportConfig implements IReportConfig {
407
433
  formats: ReportFormat[];
408
434
  constructor(data?: Partial<IReportConfig>);
409
435
  }
410
- declare class ProtocolCapabilities implements IProtocolCapabilities {
411
- toolUses: boolean;
412
- toolResults: boolean;
413
- tokenUsage: boolean;
414
- constructor(data?: Partial<IProtocolCapabilities>);
415
- }
416
- declare class RequestConfig implements IRequestConfig {
417
- format: RequestFormat;
418
- value: object | undefined;
419
- inputPath: string | undefined;
420
- constructor(data?: Partial<IRequestConfig>);
421
- }
422
- declare class ProtocolConfig implements IProtocolConfig {
423
- request: RequestConfig;
424
- capabilities: ProtocolCapabilities;
425
- constructor(data?: Partial<IProtocolConfig>);
426
- }
427
- declare class WebSocketConfig implements IWebSocketConfig {
428
- endpoint: string;
429
- connection: ConnectionConfig;
430
- protocol: ProtocolConfig;
431
- constructor(data?: Partial<IWebSocketConfig>);
432
- }
433
436
  declare class EvalConfig implements IEvalConfig {
434
437
  version: number;
435
438
  configPath: string;
436
439
  parametersPath: string;
440
+ variables: Record<string, string>;
437
441
  websocket: WebSocketConfig;
438
442
  execution: ExecutionConfig;
439
443
  evaluator: EvaluatorConfig;
@@ -675,17 +679,18 @@ declare class EvalConfigLoader {
675
679
  private readonly cwd;
676
680
  private readonly documentLoader;
677
681
  private readonly schemaValidator;
682
+ private readonly variablesReader;
683
+ private readonly webSocketResolver;
678
684
  private readonly environmentResolver;
679
685
  constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
680
686
  load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
687
+ static buildWebSocket(value: unknown): WebSocketConfig;
681
688
  private static applyReport;
682
689
  private validateSecretReferences;
683
- private validateInputPath;
684
690
  private build;
685
691
  private applyRepetition;
686
692
  private static secrets;
687
693
  private static section;
688
- private static child;
689
694
  }
690
695
  declare class ParametersLoader {
691
696
  private readonly cwd;
@@ -701,9 +706,14 @@ declare class ScenarioLoader {
701
706
  private readonly documentLoader;
702
707
  private readonly schemaValidator;
703
708
  private readonly conversationValidator;
709
+ private readonly variablesReader;
710
+ private readonly webSocketResolver;
704
711
  private readonly environmentResolver;
705
712
  constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
706
713
  load(target: string, config: EvalConfig): ConversationSpec[];
714
+ private static shared;
715
+ private static variablesLocation;
716
+ private static unusedVariables;
707
717
  discover(target: string, config: EvalConfig): string[];
708
718
  private walk;
709
719
  private build;