@fabricorg/experiments-api-protocol 0.1.6 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -274,7 +274,9 @@ var TestRunArtifact = z.object({
274
274
  });
275
275
  var CreateTestRunInput = z.object({
276
276
  externalId: z.string().min(1).max(300).optional(),
277
- source: z.enum(["bdd", "live-suite", "junit", "cli", "other"]),
277
+ // 'eval-run' rows are written by the API itself when an eval run concludes
278
+ // (Quality Center evidence for the evals suite) — never via testRuns.create.
279
+ source: z.enum(["bdd", "live-suite", "junit", "cli", "other", "eval-run", "databricks-mlflow"]),
278
280
  suite: z.string().min(1).max(240),
279
281
  status: TestRunStatus,
280
282
  startedAt: z.string().datetime().nullable().optional(),
@@ -313,6 +315,346 @@ var HarnessOperatorDefinition = z.object({
313
315
  var HarnessOperatorListResponse = z.array(HarnessOperatorDefinition);
314
316
  var HarnessJobInput = z.record(z.unknown()).default({});
315
317
  var HarnessJobResponse = z.record(z.unknown());
318
+ var DatasetSummary = z.object({
319
+ id: z.string(),
320
+ name: z.string(),
321
+ description: z.string().optional(),
322
+ latestVersion: z.number().int(),
323
+ exampleCount: z.number().int(),
324
+ createdAtIso: z.string().datetime(),
325
+ updatedAtIso: z.string().datetime()
326
+ });
327
+ var DatasetListResponse = z.object({ datasets: z.array(DatasetSummary) });
328
+ var CreateDatasetInput = z.object({
329
+ id: z.string().min(2).max(64).regex(/^[a-z][a-z0-9-]*$/),
330
+ name: z.string().min(1).max(120),
331
+ description: z.string().max(2e3).optional()
332
+ });
333
+ var DatasetExampleWire = z.object({
334
+ exampleId: z.string().uuid(),
335
+ input: z.unknown(),
336
+ expectedOutput: z.unknown().optional(),
337
+ metadata: z.record(z.unknown()).default({}),
338
+ split: z.enum(["train", "test", "validation"]).default("test")
339
+ });
340
+ var AddExamplesInput = z.object({
341
+ examples: z.array(DatasetExampleWire).min(1).max(1e3)
342
+ });
343
+ var AddExamplesResponse = z.object({ added: z.number().int() });
344
+ var ListExamplesInput = z.object({
345
+ version: z.coerce.number().int().min(1).optional(),
346
+ split: z.enum(["train", "test", "validation"]).optional(),
347
+ limit: z.coerce.number().int().min(1).max(500).default(100),
348
+ offset: z.coerce.number().int().min(0).default(0)
349
+ });
350
+ var ListExamplesResponse = z.object({
351
+ examples: z.array(DatasetExampleWire),
352
+ total: z.number().int()
353
+ });
354
+ var CreateDatasetVersionInput = z.object({ note: z.string().max(500).optional() });
355
+ var DatasetVersionWire = z.object({
356
+ datasetId: z.string(),
357
+ version: z.number().int(),
358
+ exampleCount: z.number().int(),
359
+ contentHash: z.string(),
360
+ createdAtIso: z.string().datetime(),
361
+ note: z.string().optional()
362
+ });
363
+ var DatasetVersionListResponse = z.object({
364
+ versions: z.array(DatasetVersionWire)
365
+ });
366
+ var EvalRunTargetWire = z.discriminatedUnion("kind", [
367
+ z.object({ kind: z.literal("dataset"), datasetId: z.string(), version: z.number().int().min(1) }),
368
+ z.object({
369
+ kind: z.literal("experiment"),
370
+ experimentId: z.string(),
371
+ metricEventName: z.string().optional()
372
+ })
373
+ ]);
374
+ var CreateEvalRunInput = z.object({
375
+ target: EvalRunTargetWire,
376
+ evaluatorNames: z.array(z.string().min(1)).min(1).max(10),
377
+ maxJudgeTokens: z.number().int().min(0).optional(),
378
+ /**
379
+ * Quality Center policy for binary evaluators: the run's companion
380
+ * fx.test_runs evidence row is `failed` when any binary evaluator's pass
381
+ * rate lands below this threshold, even if the run itself succeeded.
382
+ * Defaults server-side to 0.9.
383
+ */
384
+ passThreshold: z.number().min(0).max(1).optional()
385
+ });
386
+ var EvalRunGenerationWire = z.object({
387
+ promptId: z.string(),
388
+ promptVersion: z.number().int().min(1),
389
+ datasetId: z.string(),
390
+ datasetVersion: z.number().int().min(1),
391
+ maxExamples: z.number().int().min(1)
392
+ });
393
+ var EvalRunSummary = z.object({
394
+ runId: z.string(),
395
+ target: EvalRunTargetWire,
396
+ generation: EvalRunGenerationWire.optional(),
397
+ evaluatorNames: z.array(z.string()),
398
+ status: z.enum(["pending", "running", "succeeded", "failed", "cancelled"]),
399
+ scoredExamples: z.number().int(),
400
+ totalExamples: z.number().int(),
401
+ tokensSpent: z.number().int(),
402
+ costUsd: z.number().nullable(),
403
+ passThreshold: z.number(),
404
+ createdAtIso: z.string().datetime(),
405
+ startedAtIso: z.string().datetime().optional(),
406
+ finishedAtIso: z.string().datetime().optional(),
407
+ error: z.string().optional()
408
+ });
409
+ var EvalRunListInput = z.object({
410
+ limit: z.coerce.number().int().min(1).max(200).default(50),
411
+ offset: z.coerce.number().int().min(0).default(0),
412
+ status: z.enum(["pending", "running", "succeeded", "failed", "cancelled"]).optional()
413
+ });
414
+ var EvalRunListResponse = z.object({ runs: z.array(EvalRunSummary) });
415
+ var EvaluatorAggregateWire = z.object({
416
+ evaluatorName: z.string(),
417
+ evaluatorVersion: z.string(),
418
+ scored: z.number().int(),
419
+ unscored: z.number().int(),
420
+ meanScore: z.number().nullable(),
421
+ passRate: z.object({ rate: z.number(), low: z.number(), high: z.number() }).nullable(),
422
+ labelCounts: z.record(z.number().int()),
423
+ totalTokens: z.number().int(),
424
+ meanLatencyMs: z.number().nullable()
425
+ });
426
+ var EvalRunDetail = z.object({
427
+ run: EvalRunSummary,
428
+ aggregates: z.array(EvaluatorAggregateWire)
429
+ });
430
+ var EvalScoreWire = z.object({
431
+ itemId: z.string(),
432
+ evaluatorName: z.string(),
433
+ evaluatorVersion: z.string(),
434
+ label: z.string().nullable(),
435
+ score: z.number().nullable(),
436
+ explanation: z.string().nullable(),
437
+ latencyMs: z.number()
438
+ });
439
+ var EvalScoreListInput = z.object({
440
+ evaluatorName: z.string().optional(),
441
+ limit: z.coerce.number().int().min(1).max(500).default(100),
442
+ offset: z.coerce.number().int().min(0).default(0)
443
+ });
444
+ var EvalScoreListResponse = z.object({
445
+ scores: z.array(EvalScoreWire),
446
+ total: z.number().int()
447
+ });
448
+ var PromptMessageWire = z.object({
449
+ role: z.enum(["system", "user", "assistant"]),
450
+ content: z.string().min(1).max(1e5)
451
+ });
452
+ var PromptTemplateWire = z.object({
453
+ messages: z.array(PromptMessageWire).min(1).max(50),
454
+ params: z.object({
455
+ model: z.string().max(256).optional(),
456
+ temperature: z.number().min(0).max(2).optional(),
457
+ maxTokens: z.number().int().min(1).max(2e5).optional(),
458
+ topP: z.number().min(0).max(1).optional()
459
+ }).default({})
460
+ });
461
+ var PromptSummary = z.object({
462
+ id: z.string(),
463
+ name: z.string(),
464
+ description: z.string().optional(),
465
+ latestVersion: z.number().int(),
466
+ createdAtIso: z.string().datetime(),
467
+ updatedAtIso: z.string().datetime()
468
+ });
469
+ var PromptListResponse = z.object({ prompts: z.array(PromptSummary) });
470
+ var CreatePromptInput = z.object({
471
+ id: z.string().min(2).max(64).regex(/^[a-z][a-z0-9-]*$/),
472
+ name: z.string().min(1).max(120),
473
+ description: z.string().max(2e3).optional()
474
+ });
475
+ var PromptVersionWire = z.object({
476
+ promptId: z.string(),
477
+ version: z.number().int(),
478
+ template: PromptTemplateWire,
479
+ contentHash: z.string(),
480
+ variables: z.array(z.string()),
481
+ createdAtIso: z.string().datetime(),
482
+ note: z.string().optional()
483
+ });
484
+ var CreatePromptVersionInput = z.object({
485
+ template: PromptTemplateWire,
486
+ note: z.string().max(500).optional()
487
+ });
488
+ var PromptVersionListResponse = z.object({
489
+ versions: z.array(PromptVersionWire)
490
+ });
491
+ var PlaygroundInvokeInput = z.object({
492
+ /** Registry reference (promptId + optional version, defaulting to latest)... */
493
+ promptId: z.string().optional(),
494
+ version: z.number().int().min(1).optional(),
495
+ /** ...or an inline template (draft mode). Exactly one source is required. */
496
+ template: PromptTemplateWire.optional(),
497
+ variables: z.record(z.string()).default({})
498
+ });
499
+ var PlaygroundInvokeResponse = z.object({
500
+ messages: z.array(PromptMessageWire),
501
+ missingVariables: z.array(z.string()),
502
+ text: z.string(),
503
+ inputTokens: z.number().int(),
504
+ outputTokens: z.number().int(),
505
+ latencyMs: z.number(),
506
+ modelEndpoint: z.string(),
507
+ /** Manifest cost of this invocation; null when the endpoint is unpriced. */
508
+ costUsd: z.number().nullable()
509
+ });
510
+ var PlaygroundReplayInput = z.object({
511
+ promptId: z.string(),
512
+ version: z.number().int().min(1),
513
+ datasetId: z.string(),
514
+ datasetVersion: z.number().int().min(1),
515
+ evaluatorNames: z.array(z.string().min(1)).min(1).max(10),
516
+ /** Shared budget: generation AND judge tokens both count against it. */
517
+ maxJudgeTokens: z.number().int().min(0).optional(),
518
+ /** Cap on examples replayed through the model. */
519
+ maxExamples: z.number().int().min(1).max(200).default(50)
520
+ });
521
+ var VariantEvalWire = z.object({
522
+ variantKey: z.string(),
523
+ evaluatorName: z.string(),
524
+ evaluatorVersion: z.string(),
525
+ scored: z.number().int(),
526
+ unscored: z.number().int(),
527
+ meanScore: z.number().nullable(),
528
+ passCount: z.number().int(),
529
+ binary: z.boolean()
530
+ });
531
+ var VariantEvalListResponse = z.object({ variants: z.array(VariantEvalWire) });
532
+ var SpanKindWire = z.enum([
533
+ "LLM",
534
+ "CHAIN",
535
+ "RETRIEVER",
536
+ "EMBEDDING",
537
+ "TOOL",
538
+ "AGENT",
539
+ "RERANKER",
540
+ "GUARDRAIL",
541
+ "EVALUATOR",
542
+ "UNKNOWN"
543
+ ]);
544
+ var TraceSummaryWire = z.object({
545
+ traceId: z.string(),
546
+ rootSpanName: z.string(),
547
+ rootSpanKind: SpanKindWire,
548
+ sessionId: z.string().nullable(),
549
+ experimentId: z.string().nullable(),
550
+ variantKey: z.string().nullable(),
551
+ startedAtIso: z.string().datetime(),
552
+ durationMs: z.number(),
553
+ spanCount: z.number().int(),
554
+ errorCount: z.number().int(),
555
+ totalInputTokens: z.number().int(),
556
+ totalOutputTokens: z.number().int()
557
+ });
558
+ var TraceListInput = z.object({
559
+ experimentId: z.string().optional(),
560
+ variantKey: z.string().optional(),
561
+ sessionId: z.string().optional(),
562
+ kind: SpanKindWire.optional(),
563
+ errorsOnly: z.coerce.boolean().optional(),
564
+ fromIso: z.string().datetime().optional(),
565
+ untilIso: z.string().datetime().optional(),
566
+ limit: z.coerce.number().int().min(1).max(200).default(50),
567
+ offset: z.coerce.number().int().min(0).default(0)
568
+ });
569
+ var TraceListResponse = z.object({
570
+ traces: z.array(TraceSummaryWire),
571
+ total: z.number().int()
572
+ });
573
+ var SessionTraceWire = TraceSummaryWire.extend({
574
+ rootInputValue: z.string().nullable(),
575
+ rootOutputValue: z.string().nullable()
576
+ });
577
+ var SessionDetailResponse = z.object({
578
+ sessionId: z.string(),
579
+ /** Conversation order: ascending by trace start time. */
580
+ traces: z.array(SessionTraceWire)
581
+ });
582
+ var SpanWire = z.object({
583
+ traceId: z.string(),
584
+ spanId: z.string(),
585
+ parentSpanId: z.string().nullable(),
586
+ name: z.string(),
587
+ kind: SpanKindWire,
588
+ startedAtIso: z.string().datetime(),
589
+ durationMs: z.number(),
590
+ status: z.enum(["UNSET", "OK", "ERROR"]),
591
+ statusMessage: z.string().nullable(),
592
+ sessionId: z.string().nullable(),
593
+ experimentId: z.string().nullable(),
594
+ variantKey: z.string().nullable(),
595
+ inputValue: z.string().nullable(),
596
+ outputValue: z.string().nullable(),
597
+ inputTokens: z.number().int().nullable(),
598
+ outputTokens: z.number().int().nullable(),
599
+ modelName: z.string().nullable(),
600
+ /** Ingest-time dollar cost from the model-cost manifest; null when unpriceable. */
601
+ costUsd: z.number().nullable(),
602
+ attributes: z.record(z.unknown())
603
+ });
604
+ var SpanAnnotationWire = z.object({
605
+ spanId: z.string(),
606
+ annotator: z.enum(["human", "llm"]),
607
+ /** Evaluator name for llm annotations; user id for human ones. */
608
+ annotatorId: z.string(),
609
+ label: z.string().nullable(),
610
+ score: z.number().nullable(),
611
+ explanation: z.string().nullable(),
612
+ createdAtIso: z.string().datetime()
613
+ });
614
+ var TraceDetail = z.object({
615
+ traceId: z.string(),
616
+ spans: z.array(SpanWire),
617
+ annotations: z.array(SpanAnnotationWire)
618
+ });
619
+ var AnnotateSpanInput = z.object({
620
+ spanId: z.string(),
621
+ label: z.string().max(64).optional(),
622
+ score: z.number().min(0).max(1).optional(),
623
+ explanation: z.string().max(1e4).optional()
624
+ });
625
+ var SpanAnalyticsInput = z.object({
626
+ experimentId: z.string().optional(),
627
+ variantKey: z.string().optional(),
628
+ groupBy: z.enum(["kind", "model"]).default("kind"),
629
+ fromIso: z.string().datetime().optional(),
630
+ untilIso: z.string().datetime().optional()
631
+ });
632
+ var SpanAnalyticsResponse = z.object({
633
+ groups: z.array(
634
+ z.object({
635
+ groupKey: z.string(),
636
+ spanCount: z.number().int(),
637
+ errorCount: z.number().int(),
638
+ meanDurationMs: z.number().nullable(),
639
+ p95DurationMs: z.number().nullable(),
640
+ totalInputTokens: z.number().int(),
641
+ totalOutputTokens: z.number().int(),
642
+ totalCostUsd: z.number()
643
+ })
644
+ ),
645
+ traceCount: z.number().int(),
646
+ /** Cost over the full filtered span set — NOT restricted by groupBy model. */
647
+ totalCostUsd: z.number()
648
+ });
649
+ var IngestTokenSummary = z.object({
650
+ tokenId: z.string(),
651
+ label: z.string(),
652
+ createdAtIso: z.string().datetime(),
653
+ revokedAtIso: z.string().datetime().nullable()
654
+ });
655
+ var IngestTokenListResponse = z.object({ tokens: z.array(IngestTokenSummary) });
656
+ var CreateIngestTokenInput = z.object({ label: z.string().min(1).max(120) });
657
+ var CreatedIngestToken = IngestTokenSummary.extend({ token: z.string() });
316
658
  var endpoints = {
317
659
  "tenants.me": {
318
660
  method: "GET",
@@ -626,6 +968,232 @@ var endpoints = {
626
968
  mutates: false,
627
969
  requiresRole: "viewer"
628
970
  },
971
+ "datasets.list": {
972
+ method: "GET",
973
+ path: "/v1/orgs/:orgId/datasets",
974
+ params: Empty,
975
+ response: DatasetListResponse,
976
+ mutates: false,
977
+ requiresRole: "viewer"
978
+ },
979
+ "datasets.create": {
980
+ method: "POST",
981
+ path: "/v1/orgs/:orgId/datasets",
982
+ params: CreateDatasetInput,
983
+ response: DatasetSummary,
984
+ mutates: true,
985
+ requiresRole: "experimenter"
986
+ },
987
+ "datasets.get": {
988
+ method: "GET",
989
+ path: "/v1/orgs/:orgId/datasets/:datasetId",
990
+ params: Empty,
991
+ response: DatasetSummary,
992
+ mutates: false,
993
+ requiresRole: "viewer"
994
+ },
995
+ "datasets.addExamples": {
996
+ method: "POST",
997
+ path: "/v1/orgs/:orgId/datasets/:datasetId/examples",
998
+ params: AddExamplesInput,
999
+ response: AddExamplesResponse,
1000
+ mutates: true,
1001
+ requiresRole: "experimenter"
1002
+ },
1003
+ "datasets.listExamples": {
1004
+ method: "GET",
1005
+ path: "/v1/orgs/:orgId/datasets/:datasetId/examples",
1006
+ params: ListExamplesInput,
1007
+ response: ListExamplesResponse,
1008
+ mutates: false,
1009
+ requiresRole: "viewer"
1010
+ },
1011
+ "datasets.createVersion": {
1012
+ method: "POST",
1013
+ path: "/v1/orgs/:orgId/datasets/:datasetId/versions",
1014
+ params: CreateDatasetVersionInput,
1015
+ response: DatasetVersionWire,
1016
+ mutates: true,
1017
+ requiresRole: "experimenter"
1018
+ },
1019
+ "datasets.listVersions": {
1020
+ method: "GET",
1021
+ path: "/v1/orgs/:orgId/datasets/:datasetId/versions",
1022
+ params: Empty,
1023
+ response: DatasetVersionListResponse,
1024
+ mutates: false,
1025
+ requiresRole: "viewer"
1026
+ },
1027
+ "traces.list": {
1028
+ method: "GET",
1029
+ path: "/v1/orgs/:orgId/traces",
1030
+ params: TraceListInput,
1031
+ response: TraceListResponse,
1032
+ mutates: false,
1033
+ requiresRole: "viewer"
1034
+ },
1035
+ "traces.get": {
1036
+ method: "GET",
1037
+ path: "/v1/orgs/:orgId/traces/:traceId",
1038
+ params: Empty,
1039
+ response: TraceDetail,
1040
+ mutates: false,
1041
+ requiresRole: "viewer"
1042
+ },
1043
+ "traces.annotate": {
1044
+ method: "POST",
1045
+ path: "/v1/orgs/:orgId/traces/:traceId/annotations",
1046
+ params: AnnotateSpanInput,
1047
+ response: SpanAnnotationWire,
1048
+ mutates: true,
1049
+ requiresRole: "experimenter"
1050
+ },
1051
+ "traces.analytics": {
1052
+ method: "GET",
1053
+ path: "/v1/orgs/:orgId/traces/analytics",
1054
+ params: SpanAnalyticsInput,
1055
+ response: SpanAnalyticsResponse,
1056
+ mutates: false,
1057
+ requiresRole: "viewer"
1058
+ },
1059
+ "traces.session": {
1060
+ method: "GET",
1061
+ path: "/v1/orgs/:orgId/traces/sessions/:sessionId",
1062
+ params: Empty,
1063
+ response: SessionDetailResponse,
1064
+ mutates: false,
1065
+ requiresRole: "viewer"
1066
+ },
1067
+ "evals.createRun": {
1068
+ method: "POST",
1069
+ path: "/v1/orgs/:orgId/evals/runs",
1070
+ params: CreateEvalRunInput,
1071
+ response: EvalRunSummary,
1072
+ mutates: true,
1073
+ requiresRole: "experimenter"
1074
+ },
1075
+ "evals.listRuns": {
1076
+ method: "GET",
1077
+ path: "/v1/orgs/:orgId/evals/runs",
1078
+ params: EvalRunListInput,
1079
+ response: EvalRunListResponse,
1080
+ mutates: false,
1081
+ requiresRole: "viewer"
1082
+ },
1083
+ "evals.getRun": {
1084
+ method: "GET",
1085
+ path: "/v1/orgs/:orgId/evals/runs/:runId",
1086
+ params: Empty,
1087
+ response: EvalRunDetail,
1088
+ mutates: false,
1089
+ requiresRole: "viewer"
1090
+ },
1091
+ "evals.listScores": {
1092
+ method: "GET",
1093
+ path: "/v1/orgs/:orgId/evals/runs/:runId/scores",
1094
+ params: EvalScoreListInput,
1095
+ response: EvalScoreListResponse,
1096
+ mutates: false,
1097
+ requiresRole: "viewer"
1098
+ },
1099
+ "evals.cancelRun": {
1100
+ method: "POST",
1101
+ path: "/v1/orgs/:orgId/evals/runs/:runId/cancel",
1102
+ params: Empty,
1103
+ response: EvalRunSummary,
1104
+ mutates: true,
1105
+ requiresRole: "experimenter"
1106
+ },
1107
+ "evals.byVariant": {
1108
+ method: "GET",
1109
+ path: "/v1/orgs/:orgId/experiments/:experimentId/evals",
1110
+ params: Empty,
1111
+ response: VariantEvalListResponse,
1112
+ mutates: false,
1113
+ requiresRole: "viewer"
1114
+ },
1115
+ "ingestTokens.list": {
1116
+ method: "GET",
1117
+ path: "/v1/orgs/:orgId/ingest-tokens",
1118
+ params: Empty,
1119
+ response: IngestTokenListResponse,
1120
+ mutates: false,
1121
+ requiresRole: "admin"
1122
+ },
1123
+ "ingestTokens.create": {
1124
+ method: "POST",
1125
+ path: "/v1/orgs/:orgId/ingest-tokens",
1126
+ params: CreateIngestTokenInput,
1127
+ response: CreatedIngestToken,
1128
+ mutates: true,
1129
+ requiresRole: "admin"
1130
+ },
1131
+ "ingestTokens.revoke": {
1132
+ method: "POST",
1133
+ path: "/v1/orgs/:orgId/ingest-tokens/:tokenId/revoke",
1134
+ params: Empty,
1135
+ response: IngestTokenSummary,
1136
+ mutates: true,
1137
+ requiresRole: "admin"
1138
+ },
1139
+ "prompts.list": {
1140
+ method: "GET",
1141
+ path: "/v1/orgs/:orgId/prompts",
1142
+ params: Empty,
1143
+ response: PromptListResponse,
1144
+ mutates: false,
1145
+ requiresRole: "viewer"
1146
+ },
1147
+ "prompts.create": {
1148
+ method: "POST",
1149
+ path: "/v1/orgs/:orgId/prompts",
1150
+ params: CreatePromptInput,
1151
+ response: PromptSummary,
1152
+ mutates: true,
1153
+ requiresRole: "experimenter"
1154
+ },
1155
+ "prompts.get": {
1156
+ method: "GET",
1157
+ path: "/v1/orgs/:orgId/prompts/:promptId",
1158
+ params: Empty,
1159
+ response: PromptSummary,
1160
+ mutates: false,
1161
+ requiresRole: "viewer"
1162
+ },
1163
+ "prompts.createVersion": {
1164
+ method: "POST",
1165
+ path: "/v1/orgs/:orgId/prompts/:promptId/versions",
1166
+ params: CreatePromptVersionInput,
1167
+ response: PromptVersionWire,
1168
+ mutates: true,
1169
+ requiresRole: "experimenter"
1170
+ },
1171
+ "prompts.listVersions": {
1172
+ method: "GET",
1173
+ path: "/v1/orgs/:orgId/prompts/:promptId/versions",
1174
+ params: Empty,
1175
+ response: PromptVersionListResponse,
1176
+ mutates: false,
1177
+ requiresRole: "viewer"
1178
+ },
1179
+ "playground.invoke": {
1180
+ method: "POST",
1181
+ path: "/v1/orgs/:orgId/playground/invoke",
1182
+ params: PlaygroundInvokeInput,
1183
+ response: PlaygroundInvokeResponse,
1184
+ mutates: false,
1185
+ requiresRole: "experimenter"
1186
+ },
1187
+ // 202-style: replay only creates a pending eval run (durable via the
1188
+ // outbox); generation + judging happen in the executor, off the request.
1189
+ "playground.replay": {
1190
+ method: "POST",
1191
+ path: "/v1/orgs/:orgId/playground/replay",
1192
+ params: PlaygroundReplayInput,
1193
+ response: EvalRunSummary,
1194
+ mutates: true,
1195
+ requiresRole: "experimenter"
1196
+ },
629
1197
  "harness.definitions": {
630
1198
  method: "GET",
631
1199
  path: "/v1/orgs/:orgId/harness/admin/agents",
@@ -664,6 +1232,6 @@ var ApiError = z.object({
664
1232
  details: z.record(z.unknown()).optional()
665
1233
  });
666
1234
 
667
- export { AggregateInput, AggregateResponse, AggregateRow, ApiError, ApiKeyListResponse, ApiKeySummary, AuditEvent, AuditExport, AuditExportListResponse, AuditListResponse, CreateApiKeyInput, CreateApiKeyResponse, CreateAuditExportInput, CreateExperimentInput, CreateExperimentResponse, CreateTestRunInput, EdgePushResult, EnvironmentListResponse, ExperimentListResponse, ExperimentSummary, FeatureListResponse, FeatureRuleListResponse, GovernanceSettings, GovernanceStatusResponse, HarnessJobInput, HarnessJobResponse, HarnessOperatorDefinition, HarnessOperatorListResponse, InvokeActionInput, InvokeActionResponse, ManifestKey, ManifestKeysResponse, PreviewSignInput, PreviewSignResponse, ProjectListResponse, PromoteEnvironmentInput, PromoteEnvironmentResponse, PropertyListResponse, PublicationListResponse, PublishManifestResponse, QualityGateResult, SamlConfig, TenantSelf, TestCaseResult, TestCaseStatus, TestRun, TestRunArtifact, TestRunListInput, TestRunListResponse, TestRunStatus, TestRunSummary, UpdateGovernanceSettingsInput, buildPath, endpoints };
1235
+ export { AddExamplesInput, AddExamplesResponse, AggregateInput, AggregateResponse, AggregateRow, AnnotateSpanInput, ApiError, ApiKeyListResponse, ApiKeySummary, AuditEvent, AuditExport, AuditExportListResponse, AuditListResponse, CreateApiKeyInput, CreateApiKeyResponse, CreateAuditExportInput, CreateDatasetInput, CreateDatasetVersionInput, CreateEvalRunInput, CreateExperimentInput, CreateExperimentResponse, CreateIngestTokenInput, CreatePromptInput, CreatePromptVersionInput, CreateTestRunInput, CreatedIngestToken, DatasetExampleWire, DatasetListResponse, DatasetSummary, DatasetVersionListResponse, DatasetVersionWire, EdgePushResult, EnvironmentListResponse, EvalRunDetail, EvalRunGenerationWire, EvalRunListInput, EvalRunListResponse, EvalRunSummary, EvalRunTargetWire, EvalScoreListInput, EvalScoreListResponse, EvalScoreWire, EvaluatorAggregateWire, ExperimentListResponse, ExperimentSummary, FeatureListResponse, FeatureRuleListResponse, GovernanceSettings, GovernanceStatusResponse, HarnessJobInput, HarnessJobResponse, HarnessOperatorDefinition, HarnessOperatorListResponse, IngestTokenListResponse, IngestTokenSummary, InvokeActionInput, InvokeActionResponse, ListExamplesInput, ListExamplesResponse, ManifestKey, ManifestKeysResponse, PlaygroundInvokeInput, PlaygroundInvokeResponse, PlaygroundReplayInput, PreviewSignInput, PreviewSignResponse, ProjectListResponse, PromoteEnvironmentInput, PromoteEnvironmentResponse, PromptListResponse, PromptMessageWire, PromptSummary, PromptTemplateWire, PromptVersionListResponse, PromptVersionWire, PropertyListResponse, PublicationListResponse, PublishManifestResponse, QualityGateResult, SamlConfig, SessionDetailResponse, SessionTraceWire, SpanAnalyticsInput, SpanAnalyticsResponse, SpanAnnotationWire, SpanKindWire, SpanWire, TenantSelf, TestCaseResult, TestCaseStatus, TestRun, TestRunArtifact, TestRunListInput, TestRunListResponse, TestRunStatus, TestRunSummary, TraceDetail, TraceListInput, TraceListResponse, TraceSummaryWire, UpdateGovernanceSettingsInput, VariantEvalListResponse, VariantEvalWire, buildPath, endpoints };
668
1236
  //# sourceMappingURL=index.js.map
669
1237
  //# sourceMappingURL=index.js.map