@mappies/miniwat-eval 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -224,13 +224,38 @@ interface ITurnSpec {
224
224
  expect: ITurnExpectation;
225
225
  location: ISourceLocation;
226
226
  }
227
+ interface IProtocolCapabilities {
228
+ toolUses: boolean;
229
+ toolResults: boolean;
230
+ tokenUsage: boolean;
231
+ }
232
+ export declare const RequestFormat: {
233
+ readonly Json: "json";
234
+ };
235
+ export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
236
+ interface IRequestConfig {
237
+ format: RequestFormat;
238
+ value: object | undefined;
239
+ inputPath: string | undefined;
240
+ }
241
+ interface IProtocolConfig {
242
+ request: IRequestConfig;
243
+ capabilities: IProtocolCapabilities;
244
+ }
245
+ interface IWebSocketConfig {
246
+ endpoint: string;
247
+ connection: IConnectionConfig;
248
+ protocol: IProtocolConfig;
249
+ }
227
250
  interface IConversationSpec {
228
251
  version: number;
229
252
  name: string;
253
+ runName: string;
230
254
  description: string;
231
255
  turns: ITurnSpec[];
232
256
  conversation: IConversationExpectation | undefined;
233
257
  location: ISourceLocation;
258
+ websocket: IWebSocketConfig | undefined;
234
259
  secrets: string[];
235
260
  }
236
261
  declare class ResponseExpectation implements IResponseExpectation {
@@ -260,13 +285,38 @@ declare class TurnSpec implements ITurnSpec {
260
285
  location: SourceLocation;
261
286
  constructor(data?: Partial<ITurnSpec>);
262
287
  }
288
+ declare class ProtocolCapabilities implements IProtocolCapabilities {
289
+ toolUses: boolean;
290
+ toolResults: boolean;
291
+ tokenUsage: boolean;
292
+ constructor(data?: Partial<IProtocolCapabilities>);
293
+ }
294
+ declare class RequestConfig implements IRequestConfig {
295
+ format: RequestFormat;
296
+ value: object | undefined;
297
+ inputPath: string | undefined;
298
+ constructor(data?: Partial<IRequestConfig>);
299
+ }
300
+ declare class ProtocolConfig implements IProtocolConfig {
301
+ request: RequestConfig;
302
+ capabilities: ProtocolCapabilities;
303
+ constructor(data?: Partial<IProtocolConfig>);
304
+ }
305
+ declare class WebSocketConfig implements IWebSocketConfig {
306
+ endpoint: string;
307
+ connection: ConnectionConfig;
308
+ protocol: ProtocolConfig;
309
+ constructor(data?: Partial<IWebSocketConfig>);
310
+ }
263
311
  declare class ConversationSpec implements IConversationSpec {
264
312
  version: number;
265
313
  name: string;
314
+ runName: string;
266
315
  description: string;
267
316
  turns: TurnSpec[];
268
317
  conversation: ConversationExpectation | undefined;
269
318
  location: SourceLocation;
319
+ websocket: WebSocketConfig | undefined;
270
320
  secrets: string[];
271
321
  constructor(data?: Partial<IConversationSpec>);
272
322
  }
@@ -336,33 +386,12 @@ interface IReportConfig {
336
386
  directory: string;
337
387
  formats: ReportFormat[];
338
388
  }
339
- interface IProtocolCapabilities {
340
- toolUses: boolean;
341
- toolResults: boolean;
342
- tokenUsage: boolean;
343
- }
344
- export declare const RequestFormat: {
345
- readonly Json: "json";
346
- };
347
- export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
348
- interface IRequestConfig {
349
- format: RequestFormat;
350
- value: object | undefined;
351
- inputPath: string | undefined;
352
- }
353
- interface IProtocolConfig {
354
- request: IRequestConfig;
355
- capabilities: IProtocolCapabilities;
356
- }
357
- interface IWebSocketConfig {
358
- endpoint: string;
359
- connection: IConnectionConfig;
360
- protocol: IProtocolConfig;
361
- }
389
+ type Variables = Record<string, string | string[]>;
362
390
  interface IEvalConfig {
363
391
  version: number;
364
392
  configPath: string;
365
393
  parametersPath: string;
394
+ variables: Variables;
366
395
  websocket: IWebSocketConfig;
367
396
  execution: IExecutionConfig;
368
397
  evaluator: IEvaluatorConfig;
@@ -407,33 +436,11 @@ declare class ReportConfig implements IReportConfig {
407
436
  formats: ReportFormat[];
408
437
  constructor(data?: Partial<IReportConfig>);
409
438
  }
410
- declare class ProtocolCapabilities implements IProtocolCapabilities {
411
- toolUses: boolean;
412
- toolResults: boolean;
413
- tokenUsage: boolean;
414
- constructor(data?: Partial<IProtocolCapabilities>);
415
- }
416
- declare class RequestConfig implements IRequestConfig {
417
- format: RequestFormat;
418
- value: object | undefined;
419
- inputPath: string | undefined;
420
- constructor(data?: Partial<IRequestConfig>);
421
- }
422
- declare class ProtocolConfig implements IProtocolConfig {
423
- request: RequestConfig;
424
- capabilities: ProtocolCapabilities;
425
- constructor(data?: Partial<IProtocolConfig>);
426
- }
427
- declare class WebSocketConfig implements IWebSocketConfig {
428
- endpoint: string;
429
- connection: ConnectionConfig;
430
- protocol: ProtocolConfig;
431
- constructor(data?: Partial<IWebSocketConfig>);
432
- }
433
439
  declare class EvalConfig implements IEvalConfig {
434
440
  version: number;
435
441
  configPath: string;
436
442
  parametersPath: string;
443
+ variables: Variables;
437
444
  websocket: WebSocketConfig;
438
445
  execution: ExecutionConfig;
439
446
  evaluator: EvaluatorConfig;
@@ -675,17 +682,18 @@ declare class EvalConfigLoader {
675
682
  private readonly cwd;
676
683
  private readonly documentLoader;
677
684
  private readonly schemaValidator;
685
+ private readonly variablesReader;
686
+ private readonly webSocketResolver;
678
687
  private readonly environmentResolver;
679
688
  constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
680
689
  load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
690
+ static buildWebSocket(value: unknown): WebSocketConfig;
681
691
  private static applyReport;
682
692
  private validateSecretReferences;
683
- private validateInputPath;
684
693
  private build;
685
694
  private applyRepetition;
686
695
  private static secrets;
687
696
  private static section;
688
- private static child;
689
697
  }
690
698
  declare class ParametersLoader {
691
699
  private readonly cwd;
@@ -701,9 +709,17 @@ declare class ScenarioLoader {
701
709
  private readonly documentLoader;
702
710
  private readonly schemaValidator;
703
711
  private readonly conversationValidator;
712
+ private readonly variablesReader;
713
+ private readonly runWithReader;
714
+ private readonly webSocketResolver;
704
715
  private readonly environmentResolver;
705
716
  constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
706
717
  load(target: string, config: EvalConfig): ConversationSpec[];
718
+ private resolveRun;
719
+ private static byRun;
720
+ private static shared;
721
+ private static variablesLocation;
722
+ private static unusedVariables;
707
723
  discover(target: string, config: EvalConfig): string[];
708
724
  private walk;
709
725
  private build;
@@ -899,6 +915,7 @@ declare class ConsoleReporter {
899
915
  constructor(verbose: boolean, colour: boolean);
900
916
  header(run: RunHeader): string;
901
917
  scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
918
+ static title(spec: ConversationSpec): string;
902
919
  summary(run: EvalRunResult, exitCode: number, reportPaths?: string[]): string;
903
920
  static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
904
921
  static finalLine(run: EvalRunResult, exitCode: number): string;
package/dist/index.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { ITokenUsage, Message, ResponseMessage, TokenUsage, ToolResult, ToolUse } from "@mappies/miniwat-agent-core";
2
+ import "yaml";
2
3
  import { ClientOptions } from "configurapi-handler-ws";
3
4
  import { Agent } from "@mappies/miniwat-agent";
4
5
  interface ITurnExecution {
@@ -224,13 +225,38 @@ interface ITurnSpec {
224
225
  expect: ITurnExpectation;
225
226
  location: ISourceLocation;
226
227
  }
228
+ interface IProtocolCapabilities {
229
+ toolUses: boolean;
230
+ toolResults: boolean;
231
+ tokenUsage: boolean;
232
+ }
233
+ export declare const RequestFormat: {
234
+ readonly Json: "json";
235
+ };
236
+ export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
237
+ interface IRequestConfig {
238
+ format: RequestFormat;
239
+ value: object | undefined;
240
+ inputPath: string | undefined;
241
+ }
242
+ interface IProtocolConfig {
243
+ request: IRequestConfig;
244
+ capabilities: IProtocolCapabilities;
245
+ }
246
+ interface IWebSocketConfig {
247
+ endpoint: string;
248
+ connection: IConnectionConfig;
249
+ protocol: IProtocolConfig;
250
+ }
227
251
  interface IConversationSpec {
228
252
  version: number;
229
253
  name: string;
254
+ runName: string;
230
255
  description: string;
231
256
  turns: ITurnSpec[];
232
257
  conversation: IConversationExpectation | undefined;
233
258
  location: ISourceLocation;
259
+ websocket: IWebSocketConfig | undefined;
234
260
  secrets: string[];
235
261
  }
236
262
  declare class ResponseExpectation implements IResponseExpectation {
@@ -260,13 +286,38 @@ declare class TurnSpec implements ITurnSpec {
260
286
  location: SourceLocation;
261
287
  constructor(data?: Partial<ITurnSpec>);
262
288
  }
289
+ declare class ProtocolCapabilities implements IProtocolCapabilities {
290
+ toolUses: boolean;
291
+ toolResults: boolean;
292
+ tokenUsage: boolean;
293
+ constructor(data?: Partial<IProtocolCapabilities>);
294
+ }
295
+ declare class RequestConfig implements IRequestConfig {
296
+ format: RequestFormat;
297
+ value: object | undefined;
298
+ inputPath: string | undefined;
299
+ constructor(data?: Partial<IRequestConfig>);
300
+ }
301
+ declare class ProtocolConfig implements IProtocolConfig {
302
+ request: RequestConfig;
303
+ capabilities: ProtocolCapabilities;
304
+ constructor(data?: Partial<IProtocolConfig>);
305
+ }
306
+ declare class WebSocketConfig implements IWebSocketConfig {
307
+ endpoint: string;
308
+ connection: ConnectionConfig;
309
+ protocol: ProtocolConfig;
310
+ constructor(data?: Partial<IWebSocketConfig>);
311
+ }
263
312
  declare class ConversationSpec implements IConversationSpec {
264
313
  version: number;
265
314
  name: string;
315
+ runName: string;
266
316
  description: string;
267
317
  turns: TurnSpec[];
268
318
  conversation: ConversationExpectation | undefined;
269
319
  location: SourceLocation;
320
+ websocket: WebSocketConfig | undefined;
270
321
  secrets: string[];
271
322
  constructor(data?: Partial<IConversationSpec>);
272
323
  }
@@ -336,33 +387,12 @@ interface IReportConfig {
336
387
  directory: string;
337
388
  formats: ReportFormat[];
338
389
  }
339
- interface IProtocolCapabilities {
340
- toolUses: boolean;
341
- toolResults: boolean;
342
- tokenUsage: boolean;
343
- }
344
- export declare const RequestFormat: {
345
- readonly Json: "json";
346
- };
347
- export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
348
- interface IRequestConfig {
349
- format: RequestFormat;
350
- value: object | undefined;
351
- inputPath: string | undefined;
352
- }
353
- interface IProtocolConfig {
354
- request: IRequestConfig;
355
- capabilities: IProtocolCapabilities;
356
- }
357
- interface IWebSocketConfig {
358
- endpoint: string;
359
- connection: IConnectionConfig;
360
- protocol: IProtocolConfig;
361
- }
390
+ type Variables = Record<string, string | string[]>;
362
391
  interface IEvalConfig {
363
392
  version: number;
364
393
  configPath: string;
365
394
  parametersPath: string;
395
+ variables: Variables;
366
396
  websocket: IWebSocketConfig;
367
397
  execution: IExecutionConfig;
368
398
  evaluator: IEvaluatorConfig;
@@ -407,33 +437,11 @@ declare class ReportConfig implements IReportConfig {
407
437
  formats: ReportFormat[];
408
438
  constructor(data?: Partial<IReportConfig>);
409
439
  }
410
- declare class ProtocolCapabilities implements IProtocolCapabilities {
411
- toolUses: boolean;
412
- toolResults: boolean;
413
- tokenUsage: boolean;
414
- constructor(data?: Partial<IProtocolCapabilities>);
415
- }
416
- declare class RequestConfig implements IRequestConfig {
417
- format: RequestFormat;
418
- value: object | undefined;
419
- inputPath: string | undefined;
420
- constructor(data?: Partial<IRequestConfig>);
421
- }
422
- declare class ProtocolConfig implements IProtocolConfig {
423
- request: RequestConfig;
424
- capabilities: ProtocolCapabilities;
425
- constructor(data?: Partial<IProtocolConfig>);
426
- }
427
- declare class WebSocketConfig implements IWebSocketConfig {
428
- endpoint: string;
429
- connection: ConnectionConfig;
430
- protocol: ProtocolConfig;
431
- constructor(data?: Partial<IWebSocketConfig>);
432
- }
433
440
  declare class EvalConfig implements IEvalConfig {
434
441
  version: number;
435
442
  configPath: string;
436
443
  parametersPath: string;
444
+ variables: Variables;
437
445
  websocket: WebSocketConfig;
438
446
  execution: ExecutionConfig;
439
447
  evaluator: EvaluatorConfig;
@@ -675,17 +683,18 @@ declare class EvalConfigLoader {
675
683
  private readonly cwd;
676
684
  private readonly documentLoader;
677
685
  private readonly schemaValidator;
686
+ private readonly variablesReader;
687
+ private readonly webSocketResolver;
678
688
  private readonly environmentResolver;
679
689
  constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
680
690
  load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
691
+ static buildWebSocket(value: unknown): WebSocketConfig;
681
692
  private static applyReport;
682
693
  private validateSecretReferences;
683
- private validateInputPath;
684
694
  private build;
685
695
  private applyRepetition;
686
696
  private static secrets;
687
697
  private static section;
688
- private static child;
689
698
  }
690
699
  declare class ParametersLoader {
691
700
  private readonly cwd;
@@ -701,9 +710,17 @@ declare class ScenarioLoader {
701
710
  private readonly documentLoader;
702
711
  private readonly schemaValidator;
703
712
  private readonly conversationValidator;
713
+ private readonly variablesReader;
714
+ private readonly runWithReader;
715
+ private readonly webSocketResolver;
704
716
  private readonly environmentResolver;
705
717
  constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
706
718
  load(target: string, config: EvalConfig): ConversationSpec[];
719
+ private resolveRun;
720
+ private static byRun;
721
+ private static shared;
722
+ private static variablesLocation;
723
+ private static unusedVariables;
707
724
  discover(target: string, config: EvalConfig): string[];
708
725
  private walk;
709
726
  private build;
@@ -899,6 +916,7 @@ declare class ConsoleReporter {
899
916
  constructor(verbose: boolean, colour: boolean);
900
917
  header(run: RunHeader): string;
901
918
  scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
919
+ static title(spec: ConversationSpec): string;
902
920
  summary(run: EvalRunResult, exitCode: number, reportPaths?: string[]): string;
903
921
  static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
904
922
  static finalLine(run: EvalRunResult, exitCode: number): string;