@roleplay-sh/cli 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. package/.env.example +14 -0
  2. package/CHANGELOG.md +33 -7
  3. package/CONTRIBUTING.md +1 -1
  4. package/README.md +13 -1
  5. package/RELEASE.md +10 -3
  6. package/SECURITY.md +1 -3
  7. package/dist/cli.js +3438 -2682
  8. package/dist/cli.js.map +1 -1
  9. package/dist/index.d.ts +85 -43
  10. package/dist/index.js +462 -60
  11. package/dist/index.js.map +1 -1
  12. package/package.json +1 -1
package/dist/index.d.ts CHANGED
@@ -37,11 +37,11 @@ declare const reportSchema: z.ZodObject<{
37
37
  endedAt: z.ZodEffects<z.ZodString, string, string>;
38
38
  rawJudgeOutput: z.ZodOptional<z.ZodUnknown>;
39
39
  }, "strict", z.ZodTypeAny, {
40
+ summary: string;
40
41
  status: "passed" | "warning" | "failed";
41
42
  runId: string;
42
43
  scenario: string;
43
44
  score: number;
44
- summary: string;
45
45
  criteria: {
46
46
  criterion: string;
47
47
  result: "passed" | "failed" | "unclear";
@@ -57,11 +57,11 @@ declare const reportSchema: z.ZodObject<{
57
57
  endedAt: string;
58
58
  rawJudgeOutput?: unknown;
59
59
  }, {
60
+ summary: string;
60
61
  status: "passed" | "warning" | "failed";
61
62
  runId: string;
62
63
  scenario: string;
63
64
  score: number;
64
- summary: string;
65
65
  criteria: {
66
66
  criterion: string;
67
67
  result: "passed" | "failed" | "unclear";
@@ -199,14 +199,35 @@ declare const scenarioSchema: z.ZodObject<{
199
199
  hiddenContext: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
200
200
  successCriteria: z.ZodArray<z.ZodString, "many">;
201
201
  failureCriteria: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
202
- judge: z.ZodDefault<z.ZodObject<{
203
- type: z.ZodDefault<z.ZodEnum<["mock"]>>;
204
- rubric: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
202
+ attacker: z.ZodOptional<z.ZodObject<{
203
+ provider: z.ZodDefault<z.ZodEnum<["mock", "openai", "anthropic", "google", "openai-compatible"]>>;
204
+ model: z.ZodOptional<z.ZodString>;
205
+ baseUrl: z.ZodOptional<z.ZodString>;
205
206
  }, "strip", z.ZodTypeAny, {
206
- type: "mock";
207
+ provider: "mock" | "openai" | "anthropic" | "google" | "openai-compatible";
208
+ model?: string | undefined;
209
+ baseUrl?: string | undefined;
210
+ }, {
211
+ provider?: "mock" | "openai" | "anthropic" | "google" | "openai-compatible" | undefined;
212
+ model?: string | undefined;
213
+ baseUrl?: string | undefined;
214
+ }>>;
215
+ judge: z.ZodDefault<z.ZodObject<Omit<{
216
+ provider: z.ZodDefault<z.ZodEnum<["mock", "openai", "anthropic", "google", "openai-compatible"]>>;
217
+ model: z.ZodOptional<z.ZodString>;
218
+ baseUrl: z.ZodOptional<z.ZodString>;
219
+ } & {
220
+ type: z.ZodDefault<z.ZodEnum<["mock", "openai", "anthropic", "google", "openai-compatible"]>>;
221
+ rubric: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
222
+ }, "provider">, "strip", z.ZodTypeAny, {
223
+ type: "mock" | "openai" | "anthropic" | "google" | "openai-compatible";
224
+ model?: string | undefined;
225
+ baseUrl?: string | undefined;
207
226
  rubric?: Record<string, number> | undefined;
208
227
  }, {
209
- type?: "mock" | undefined;
228
+ type?: "mock" | "openai" | "anthropic" | "google" | "openai-compatible" | undefined;
229
+ model?: string | undefined;
230
+ baseUrl?: string | undefined;
210
231
  rubric?: Record<string, number> | undefined;
211
232
  }>>;
212
233
  output: z.ZodOptional<z.ZodObject<{
@@ -257,12 +278,19 @@ declare const scenarioSchema: z.ZodObject<{
257
278
  successCriteria: string[];
258
279
  failureCriteria: string[];
259
280
  judge: {
260
- type: "mock";
281
+ type: "mock" | "openai" | "anthropic" | "google" | "openai-compatible";
282
+ model?: string | undefined;
283
+ baseUrl?: string | undefined;
261
284
  rubric?: Record<string, number> | undefined;
262
285
  };
263
286
  output?: {
264
287
  expectations: string[];
265
288
  } | undefined;
289
+ attacker?: {
290
+ provider: "mock" | "openai" | "anthropic" | "google" | "openai-compatible";
291
+ model?: string | undefined;
292
+ baseUrl?: string | undefined;
293
+ } | undefined;
266
294
  }, {
267
295
  name: string;
268
296
  target: {
@@ -306,8 +334,15 @@ declare const scenarioSchema: z.ZodObject<{
306
334
  } | undefined;
307
335
  hiddenContext?: string[] | undefined;
308
336
  failureCriteria?: string[] | undefined;
337
+ attacker?: {
338
+ provider?: "mock" | "openai" | "anthropic" | "google" | "openai-compatible" | undefined;
339
+ model?: string | undefined;
340
+ baseUrl?: string | undefined;
341
+ } | undefined;
309
342
  judge?: {
310
- type?: "mock" | undefined;
343
+ type?: "mock" | "openai" | "anthropic" | "google" | "openai-compatible" | undefined;
344
+ model?: string | undefined;
345
+ baseUrl?: string | undefined;
311
346
  rubric?: Record<string, number> | undefined;
312
347
  } | undefined;
313
348
  }>;
@@ -327,15 +362,15 @@ declare const transcriptSchema: z.ZodEffects<z.ZodObject<{
327
362
  timestamp: z.ZodEffects<z.ZodEffects<z.ZodString, string, string>, string, string>;
328
363
  raw: z.ZodOptional<z.ZodUnknown>;
329
364
  }, "strict", z.ZodTypeAny, {
365
+ content: string;
330
366
  turn: number;
331
367
  role: "user" | "agent";
332
- content: string;
333
368
  timestamp: string;
334
369
  raw?: unknown;
335
370
  }, {
371
+ content: string;
336
372
  turn: number;
337
373
  role: "user" | "agent";
338
- content: string;
339
374
  timestamp: string;
340
375
  raw?: unknown;
341
376
  }>, "many">;
@@ -344,9 +379,9 @@ declare const transcriptSchema: z.ZodEffects<z.ZodObject<{
344
379
  startedAt: string;
345
380
  scenarioName: string;
346
381
  turns: {
382
+ content: string;
347
383
  turn: number;
348
384
  role: "user" | "agent";
349
- content: string;
350
385
  timestamp: string;
351
386
  raw?: unknown;
352
387
  }[];
@@ -356,9 +391,9 @@ declare const transcriptSchema: z.ZodEffects<z.ZodObject<{
356
391
  startedAt: string;
357
392
  scenarioName: string;
358
393
  turns: {
394
+ content: string;
359
395
  turn: number;
360
396
  role: "user" | "agent";
361
- content: string;
362
397
  timestamp: string;
363
398
  raw?: unknown;
364
399
  }[];
@@ -368,9 +403,9 @@ declare const transcriptSchema: z.ZodEffects<z.ZodObject<{
368
403
  startedAt: string;
369
404
  scenarioName: string;
370
405
  turns: {
406
+ content: string;
371
407
  turn: number;
372
408
  role: "user" | "agent";
373
- content: string;
374
409
  timestamp: string;
375
410
  raw?: unknown;
376
411
  }[];
@@ -380,9 +415,9 @@ declare const transcriptSchema: z.ZodEffects<z.ZodObject<{
380
415
  startedAt: string;
381
416
  scenarioName: string;
382
417
  turns: {
418
+ content: string;
383
419
  turn: number;
384
420
  role: "user" | "agent";
385
- content: string;
386
421
  timestamp: string;
387
422
  raw?: unknown;
388
423
  }[];
@@ -400,12 +435,19 @@ interface RunPaths {
400
435
  metadataPath: string;
401
436
  }
402
437
 
438
+ type LlmProviderName = 'mock' | 'openai' | 'anthropic' | 'google' | 'openai-compatible';
439
+
403
440
  interface RunOptions {
404
441
  scenarioRef: string;
405
442
  maxTurns?: number;
406
443
  outDir?: string;
407
444
  yes?: boolean;
408
445
  metadata?: Record<string, unknown>;
446
+ attackerProvider?: LlmProviderName;
447
+ judgeProvider?: LlmProviderName;
448
+ attackerModel?: string;
449
+ judgeModel?: string;
450
+ llmBaseUrl?: string;
409
451
  }
410
452
  interface RunResult {
411
453
  runId: string;
@@ -465,11 +507,11 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
465
507
  endedAt: z.ZodEffects<z.ZodString, string, string>;
466
508
  rawJudgeOutput: z.ZodOptional<z.ZodUnknown>;
467
509
  }, "strict", z.ZodTypeAny, {
510
+ summary: string;
468
511
  status: "passed" | "warning" | "failed";
469
512
  runId: string;
470
513
  scenario: string;
471
514
  score: number;
472
- summary: string;
473
515
  criteria: {
474
516
  criterion: string;
475
517
  result: "passed" | "failed" | "unclear";
@@ -485,11 +527,11 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
485
527
  endedAt: string;
486
528
  rawJudgeOutput?: unknown;
487
529
  }, {
530
+ summary: string;
488
531
  status: "passed" | "warning" | "failed";
489
532
  runId: string;
490
533
  scenario: string;
491
534
  score: number;
492
- summary: string;
493
535
  criteria: {
494
536
  criterion: string;
495
537
  result: "passed" | "failed" | "unclear";
@@ -517,15 +559,15 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
517
559
  timestamp: z.ZodEffects<z.ZodEffects<z.ZodString, string, string>, string, string>;
518
560
  raw: z.ZodOptional<z.ZodUnknown>;
519
561
  }, "strict", z.ZodTypeAny, {
562
+ content: string;
520
563
  turn: number;
521
564
  role: "user" | "agent";
522
- content: string;
523
565
  timestamp: string;
524
566
  raw?: unknown;
525
567
  }, {
568
+ content: string;
526
569
  turn: number;
527
570
  role: "user" | "agent";
528
- content: string;
529
571
  timestamp: string;
530
572
  raw?: unknown;
531
573
  }>, "many">;
@@ -534,9 +576,9 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
534
576
  startedAt: string;
535
577
  scenarioName: string;
536
578
  turns: {
579
+ content: string;
537
580
  turn: number;
538
581
  role: "user" | "agent";
539
- content: string;
540
582
  timestamp: string;
541
583
  raw?: unknown;
542
584
  }[];
@@ -546,9 +588,9 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
546
588
  startedAt: string;
547
589
  scenarioName: string;
548
590
  turns: {
591
+ content: string;
549
592
  turn: number;
550
593
  role: "user" | "agent";
551
- content: string;
552
594
  timestamp: string;
553
595
  raw?: unknown;
554
596
  }[];
@@ -558,9 +600,9 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
558
600
  startedAt: string;
559
601
  scenarioName: string;
560
602
  turns: {
603
+ content: string;
561
604
  turn: number;
562
605
  role: "user" | "agent";
563
- content: string;
564
606
  timestamp: string;
565
607
  raw?: unknown;
566
608
  }[];
@@ -570,9 +612,9 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
570
612
  startedAt: string;
571
613
  scenarioName: string;
572
614
  turns: {
615
+ content: string;
573
616
  turn: number;
574
617
  role: "user" | "agent";
575
- content: string;
576
618
  timestamp: string;
577
619
  raw?: unknown;
578
620
  }[];
@@ -582,11 +624,11 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
582
624
  metadata: z.ZodOptional<z.ZodUnknown>;
583
625
  }, "strict", z.ZodTypeAny, {
584
626
  report: {
627
+ summary: string;
585
628
  status: "passed" | "warning" | "failed";
586
629
  runId: string;
587
630
  scenario: string;
588
631
  score: number;
589
- summary: string;
590
632
  criteria: {
591
633
  criterion: string;
592
634
  result: "passed" | "failed" | "unclear";
@@ -607,9 +649,9 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
607
649
  startedAt: string;
608
650
  scenarioName: string;
609
651
  turns: {
652
+ content: string;
610
653
  turn: number;
611
654
  role: "user" | "agent";
612
- content: string;
613
655
  timestamp: string;
614
656
  raw?: unknown;
615
657
  }[];
@@ -619,11 +661,11 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
619
661
  metadata?: unknown;
620
662
  }, {
621
663
  report: {
664
+ summary: string;
622
665
  status: "passed" | "warning" | "failed";
623
666
  runId: string;
624
667
  scenario: string;
625
668
  score: number;
626
- summary: string;
627
669
  criteria: {
628
670
  criterion: string;
629
671
  result: "passed" | "failed" | "unclear";
@@ -644,9 +686,9 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
644
686
  startedAt: string;
645
687
  scenarioName: string;
646
688
  turns: {
689
+ content: string;
647
690
  turn: number;
648
691
  role: "user" | "agent";
649
- content: string;
650
692
  timestamp: string;
651
693
  raw?: unknown;
652
694
  }[];
@@ -658,11 +700,11 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
658
700
  }, "strict", z.ZodTypeAny, {
659
701
  run: {
660
702
  report: {
703
+ summary: string;
661
704
  status: "passed" | "warning" | "failed";
662
705
  runId: string;
663
706
  scenario: string;
664
707
  score: number;
665
- summary: string;
666
708
  criteria: {
667
709
  criterion: string;
668
710
  result: "passed" | "failed" | "unclear";
@@ -683,9 +725,9 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
683
725
  startedAt: string;
684
726
  scenarioName: string;
685
727
  turns: {
728
+ content: string;
686
729
  turn: number;
687
730
  role: "user" | "agent";
688
- content: string;
689
731
  timestamp: string;
690
732
  raw?: unknown;
691
733
  }[];
@@ -697,21 +739,21 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
697
739
  mode: "sanitized_findings" | "full_transcript_opt_in";
698
740
  projectId: string;
699
741
  source: "ci" | "local" | "scheduled";
742
+ attackPackId?: string | undefined;
743
+ attackPackScenario?: string | undefined;
700
744
  branch?: string | undefined;
701
745
  commit?: string | undefined;
702
746
  buildUrl?: string | undefined;
703
747
  environment?: string | undefined;
704
748
  targetAgent?: string | undefined;
705
- attackPackId?: string | undefined;
706
- attackPackScenario?: string | undefined;
707
749
  }, {
708
750
  run: {
709
751
  report: {
752
+ summary: string;
710
753
  status: "passed" | "warning" | "failed";
711
754
  runId: string;
712
755
  scenario: string;
713
756
  score: number;
714
- summary: string;
715
757
  criteria: {
716
758
  criterion: string;
717
759
  result: "passed" | "failed" | "unclear";
@@ -732,9 +774,9 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
732
774
  startedAt: string;
733
775
  scenarioName: string;
734
776
  turns: {
777
+ content: string;
735
778
  turn: number;
736
779
  role: "user" | "agent";
737
- content: string;
738
780
  timestamp: string;
739
781
  raw?: unknown;
740
782
  }[];
@@ -745,22 +787,22 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
745
787
  };
746
788
  projectId: string;
747
789
  mode?: "sanitized_findings" | "full_transcript_opt_in" | undefined;
790
+ attackPackId?: string | undefined;
791
+ attackPackScenario?: string | undefined;
748
792
  source?: "ci" | "local" | "scheduled" | undefined;
749
793
  branch?: string | undefined;
750
794
  commit?: string | undefined;
751
795
  buildUrl?: string | undefined;
752
796
  environment?: string | undefined;
753
797
  targetAgent?: string | undefined;
754
- attackPackId?: string | undefined;
755
- attackPackScenario?: string | undefined;
756
798
  }>, {
757
799
  run: {
758
800
  report: {
801
+ summary: string;
759
802
  status: "passed" | "warning" | "failed";
760
803
  runId: string;
761
804
  scenario: string;
762
805
  score: number;
763
- summary: string;
764
806
  criteria: {
765
807
  criterion: string;
766
808
  result: "passed" | "failed" | "unclear";
@@ -781,9 +823,9 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
781
823
  startedAt: string;
782
824
  scenarioName: string;
783
825
  turns: {
826
+ content: string;
784
827
  turn: number;
785
828
  role: "user" | "agent";
786
- content: string;
787
829
  timestamp: string;
788
830
  raw?: unknown;
789
831
  }[];
@@ -795,21 +837,21 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
795
837
  mode: "sanitized_findings" | "full_transcript_opt_in";
796
838
  projectId: string;
797
839
  source: "ci" | "local" | "scheduled";
840
+ attackPackId?: string | undefined;
841
+ attackPackScenario?: string | undefined;
798
842
  branch?: string | undefined;
799
843
  commit?: string | undefined;
800
844
  buildUrl?: string | undefined;
801
845
  environment?: string | undefined;
802
846
  targetAgent?: string | undefined;
803
- attackPackId?: string | undefined;
804
- attackPackScenario?: string | undefined;
805
847
  }, {
806
848
  run: {
807
849
  report: {
850
+ summary: string;
808
851
  status: "passed" | "warning" | "failed";
809
852
  runId: string;
810
853
  scenario: string;
811
854
  score: number;
812
- summary: string;
813
855
  criteria: {
814
856
  criterion: string;
815
857
  result: "passed" | "failed" | "unclear";
@@ -830,9 +872,9 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
830
872
  startedAt: string;
831
873
  scenarioName: string;
832
874
  turns: {
875
+ content: string;
833
876
  turn: number;
834
877
  role: "user" | "agent";
835
- content: string;
836
878
  timestamp: string;
837
879
  raw?: unknown;
838
880
  }[];
@@ -843,14 +885,14 @@ declare const cloudUploadSchema: z.ZodEffects<z.ZodObject<{
843
885
  };
844
886
  projectId: string;
845
887
  mode?: "sanitized_findings" | "full_transcript_opt_in" | undefined;
888
+ attackPackId?: string | undefined;
889
+ attackPackScenario?: string | undefined;
846
890
  source?: "ci" | "local" | "scheduled" | undefined;
847
891
  branch?: string | undefined;
848
892
  commit?: string | undefined;
849
893
  buildUrl?: string | undefined;
850
894
  environment?: string | undefined;
851
895
  targetAgent?: string | undefined;
852
- attackPackId?: string | undefined;
853
- attackPackScenario?: string | undefined;
854
896
  }>;
855
897
  type CloudUpload = z.infer<typeof cloudUploadSchema>;
856
898