@fabricorg/experiments-api-protocol 0.1.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +620 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +7321 -2243
- package/dist/index.d.ts +7321 -2243
- package/dist/index.js +570 -2
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -274,7 +274,9 @@ var TestRunArtifact = z.object({
|
|
|
274
274
|
});
|
|
275
275
|
var CreateTestRunInput = z.object({
|
|
276
276
|
externalId: z.string().min(1).max(300).optional(),
|
|
277
|
-
|
|
277
|
+
// 'eval-run' rows are written by the API itself when an eval run concludes
|
|
278
|
+
// (Quality Center evidence for the evals suite) — never via testRuns.create.
|
|
279
|
+
source: z.enum(["bdd", "live-suite", "junit", "cli", "other", "eval-run", "databricks-mlflow"]),
|
|
278
280
|
suite: z.string().min(1).max(240),
|
|
279
281
|
status: TestRunStatus,
|
|
280
282
|
startedAt: z.string().datetime().nullable().optional(),
|
|
@@ -313,6 +315,346 @@ var HarnessOperatorDefinition = z.object({
|
|
|
313
315
|
var HarnessOperatorListResponse = z.array(HarnessOperatorDefinition);
|
|
314
316
|
var HarnessJobInput = z.record(z.unknown()).default({});
|
|
315
317
|
var HarnessJobResponse = z.record(z.unknown());
|
|
318
|
+
var DatasetSummary = z.object({
|
|
319
|
+
id: z.string(),
|
|
320
|
+
name: z.string(),
|
|
321
|
+
description: z.string().optional(),
|
|
322
|
+
latestVersion: z.number().int(),
|
|
323
|
+
exampleCount: z.number().int(),
|
|
324
|
+
createdAtIso: z.string().datetime(),
|
|
325
|
+
updatedAtIso: z.string().datetime()
|
|
326
|
+
});
|
|
327
|
+
var DatasetListResponse = z.object({ datasets: z.array(DatasetSummary) });
|
|
328
|
+
var CreateDatasetInput = z.object({
|
|
329
|
+
id: z.string().min(2).max(64).regex(/^[a-z][a-z0-9-]*$/),
|
|
330
|
+
name: z.string().min(1).max(120),
|
|
331
|
+
description: z.string().max(2e3).optional()
|
|
332
|
+
});
|
|
333
|
+
var DatasetExampleWire = z.object({
|
|
334
|
+
exampleId: z.string().uuid(),
|
|
335
|
+
input: z.unknown(),
|
|
336
|
+
expectedOutput: z.unknown().optional(),
|
|
337
|
+
metadata: z.record(z.unknown()).default({}),
|
|
338
|
+
split: z.enum(["train", "test", "validation"]).default("test")
|
|
339
|
+
});
|
|
340
|
+
var AddExamplesInput = z.object({
|
|
341
|
+
examples: z.array(DatasetExampleWire).min(1).max(1e3)
|
|
342
|
+
});
|
|
343
|
+
var AddExamplesResponse = z.object({ added: z.number().int() });
|
|
344
|
+
var ListExamplesInput = z.object({
|
|
345
|
+
version: z.coerce.number().int().min(1).optional(),
|
|
346
|
+
split: z.enum(["train", "test", "validation"]).optional(),
|
|
347
|
+
limit: z.coerce.number().int().min(1).max(500).default(100),
|
|
348
|
+
offset: z.coerce.number().int().min(0).default(0)
|
|
349
|
+
});
|
|
350
|
+
var ListExamplesResponse = z.object({
|
|
351
|
+
examples: z.array(DatasetExampleWire),
|
|
352
|
+
total: z.number().int()
|
|
353
|
+
});
|
|
354
|
+
var CreateDatasetVersionInput = z.object({ note: z.string().max(500).optional() });
|
|
355
|
+
var DatasetVersionWire = z.object({
|
|
356
|
+
datasetId: z.string(),
|
|
357
|
+
version: z.number().int(),
|
|
358
|
+
exampleCount: z.number().int(),
|
|
359
|
+
contentHash: z.string(),
|
|
360
|
+
createdAtIso: z.string().datetime(),
|
|
361
|
+
note: z.string().optional()
|
|
362
|
+
});
|
|
363
|
+
var DatasetVersionListResponse = z.object({
|
|
364
|
+
versions: z.array(DatasetVersionWire)
|
|
365
|
+
});
|
|
366
|
+
var EvalRunTargetWire = z.discriminatedUnion("kind", [
|
|
367
|
+
z.object({ kind: z.literal("dataset"), datasetId: z.string(), version: z.number().int().min(1) }),
|
|
368
|
+
z.object({
|
|
369
|
+
kind: z.literal("experiment"),
|
|
370
|
+
experimentId: z.string(),
|
|
371
|
+
metricEventName: z.string().optional()
|
|
372
|
+
})
|
|
373
|
+
]);
|
|
374
|
+
var CreateEvalRunInput = z.object({
|
|
375
|
+
target: EvalRunTargetWire,
|
|
376
|
+
evaluatorNames: z.array(z.string().min(1)).min(1).max(10),
|
|
377
|
+
maxJudgeTokens: z.number().int().min(0).optional(),
|
|
378
|
+
/**
|
|
379
|
+
* Quality Center policy for binary evaluators: the run's companion
|
|
380
|
+
* fx.test_runs evidence row is `failed` when any binary evaluator's pass
|
|
381
|
+
* rate lands below this threshold, even if the run itself succeeded.
|
|
382
|
+
* Defaults server-side to 0.9.
|
|
383
|
+
*/
|
|
384
|
+
passThreshold: z.number().min(0).max(1).optional()
|
|
385
|
+
});
|
|
386
|
+
var EvalRunGenerationWire = z.object({
|
|
387
|
+
promptId: z.string(),
|
|
388
|
+
promptVersion: z.number().int().min(1),
|
|
389
|
+
datasetId: z.string(),
|
|
390
|
+
datasetVersion: z.number().int().min(1),
|
|
391
|
+
maxExamples: z.number().int().min(1)
|
|
392
|
+
});
|
|
393
|
+
var EvalRunSummary = z.object({
|
|
394
|
+
runId: z.string(),
|
|
395
|
+
target: EvalRunTargetWire,
|
|
396
|
+
generation: EvalRunGenerationWire.optional(),
|
|
397
|
+
evaluatorNames: z.array(z.string()),
|
|
398
|
+
status: z.enum(["pending", "running", "succeeded", "failed", "cancelled"]),
|
|
399
|
+
scoredExamples: z.number().int(),
|
|
400
|
+
totalExamples: z.number().int(),
|
|
401
|
+
tokensSpent: z.number().int(),
|
|
402
|
+
costUsd: z.number().nullable(),
|
|
403
|
+
passThreshold: z.number(),
|
|
404
|
+
createdAtIso: z.string().datetime(),
|
|
405
|
+
startedAtIso: z.string().datetime().optional(),
|
|
406
|
+
finishedAtIso: z.string().datetime().optional(),
|
|
407
|
+
error: z.string().optional()
|
|
408
|
+
});
|
|
409
|
+
var EvalRunListInput = z.object({
|
|
410
|
+
limit: z.coerce.number().int().min(1).max(200).default(50),
|
|
411
|
+
offset: z.coerce.number().int().min(0).default(0),
|
|
412
|
+
status: z.enum(["pending", "running", "succeeded", "failed", "cancelled"]).optional()
|
|
413
|
+
});
|
|
414
|
+
var EvalRunListResponse = z.object({ runs: z.array(EvalRunSummary) });
|
|
415
|
+
var EvaluatorAggregateWire = z.object({
|
|
416
|
+
evaluatorName: z.string(),
|
|
417
|
+
evaluatorVersion: z.string(),
|
|
418
|
+
scored: z.number().int(),
|
|
419
|
+
unscored: z.number().int(),
|
|
420
|
+
meanScore: z.number().nullable(),
|
|
421
|
+
passRate: z.object({ rate: z.number(), low: z.number(), high: z.number() }).nullable(),
|
|
422
|
+
labelCounts: z.record(z.number().int()),
|
|
423
|
+
totalTokens: z.number().int(),
|
|
424
|
+
meanLatencyMs: z.number().nullable()
|
|
425
|
+
});
|
|
426
|
+
var EvalRunDetail = z.object({
|
|
427
|
+
run: EvalRunSummary,
|
|
428
|
+
aggregates: z.array(EvaluatorAggregateWire)
|
|
429
|
+
});
|
|
430
|
+
var EvalScoreWire = z.object({
|
|
431
|
+
itemId: z.string(),
|
|
432
|
+
evaluatorName: z.string(),
|
|
433
|
+
evaluatorVersion: z.string(),
|
|
434
|
+
label: z.string().nullable(),
|
|
435
|
+
score: z.number().nullable(),
|
|
436
|
+
explanation: z.string().nullable(),
|
|
437
|
+
latencyMs: z.number()
|
|
438
|
+
});
|
|
439
|
+
var EvalScoreListInput = z.object({
|
|
440
|
+
evaluatorName: z.string().optional(),
|
|
441
|
+
limit: z.coerce.number().int().min(1).max(500).default(100),
|
|
442
|
+
offset: z.coerce.number().int().min(0).default(0)
|
|
443
|
+
});
|
|
444
|
+
var EvalScoreListResponse = z.object({
|
|
445
|
+
scores: z.array(EvalScoreWire),
|
|
446
|
+
total: z.number().int()
|
|
447
|
+
});
|
|
448
|
+
var PromptMessageWire = z.object({
|
|
449
|
+
role: z.enum(["system", "user", "assistant"]),
|
|
450
|
+
content: z.string().min(1).max(1e5)
|
|
451
|
+
});
|
|
452
|
+
var PromptTemplateWire = z.object({
|
|
453
|
+
messages: z.array(PromptMessageWire).min(1).max(50),
|
|
454
|
+
params: z.object({
|
|
455
|
+
model: z.string().max(256).optional(),
|
|
456
|
+
temperature: z.number().min(0).max(2).optional(),
|
|
457
|
+
maxTokens: z.number().int().min(1).max(2e5).optional(),
|
|
458
|
+
topP: z.number().min(0).max(1).optional()
|
|
459
|
+
}).default({})
|
|
460
|
+
});
|
|
461
|
+
var PromptSummary = z.object({
|
|
462
|
+
id: z.string(),
|
|
463
|
+
name: z.string(),
|
|
464
|
+
description: z.string().optional(),
|
|
465
|
+
latestVersion: z.number().int(),
|
|
466
|
+
createdAtIso: z.string().datetime(),
|
|
467
|
+
updatedAtIso: z.string().datetime()
|
|
468
|
+
});
|
|
469
|
+
var PromptListResponse = z.object({ prompts: z.array(PromptSummary) });
|
|
470
|
+
var CreatePromptInput = z.object({
|
|
471
|
+
id: z.string().min(2).max(64).regex(/^[a-z][a-z0-9-]*$/),
|
|
472
|
+
name: z.string().min(1).max(120),
|
|
473
|
+
description: z.string().max(2e3).optional()
|
|
474
|
+
});
|
|
475
|
+
var PromptVersionWire = z.object({
|
|
476
|
+
promptId: z.string(),
|
|
477
|
+
version: z.number().int(),
|
|
478
|
+
template: PromptTemplateWire,
|
|
479
|
+
contentHash: z.string(),
|
|
480
|
+
variables: z.array(z.string()),
|
|
481
|
+
createdAtIso: z.string().datetime(),
|
|
482
|
+
note: z.string().optional()
|
|
483
|
+
});
|
|
484
|
+
var CreatePromptVersionInput = z.object({
|
|
485
|
+
template: PromptTemplateWire,
|
|
486
|
+
note: z.string().max(500).optional()
|
|
487
|
+
});
|
|
488
|
+
var PromptVersionListResponse = z.object({
|
|
489
|
+
versions: z.array(PromptVersionWire)
|
|
490
|
+
});
|
|
491
|
+
var PlaygroundInvokeInput = z.object({
|
|
492
|
+
/** Registry reference (promptId + optional version, defaulting to latest)... */
|
|
493
|
+
promptId: z.string().optional(),
|
|
494
|
+
version: z.number().int().min(1).optional(),
|
|
495
|
+
/** ...or an inline template (draft mode). Exactly one source is required. */
|
|
496
|
+
template: PromptTemplateWire.optional(),
|
|
497
|
+
variables: z.record(z.string()).default({})
|
|
498
|
+
});
|
|
499
|
+
var PlaygroundInvokeResponse = z.object({
|
|
500
|
+
messages: z.array(PromptMessageWire),
|
|
501
|
+
missingVariables: z.array(z.string()),
|
|
502
|
+
text: z.string(),
|
|
503
|
+
inputTokens: z.number().int(),
|
|
504
|
+
outputTokens: z.number().int(),
|
|
505
|
+
latencyMs: z.number(),
|
|
506
|
+
modelEndpoint: z.string(),
|
|
507
|
+
/** Manifest cost of this invocation; null when the endpoint is unpriced. */
|
|
508
|
+
costUsd: z.number().nullable()
|
|
509
|
+
});
|
|
510
|
+
var PlaygroundReplayInput = z.object({
|
|
511
|
+
promptId: z.string(),
|
|
512
|
+
version: z.number().int().min(1),
|
|
513
|
+
datasetId: z.string(),
|
|
514
|
+
datasetVersion: z.number().int().min(1),
|
|
515
|
+
evaluatorNames: z.array(z.string().min(1)).min(1).max(10),
|
|
516
|
+
/** Shared budget: generation AND judge tokens both count against it. */
|
|
517
|
+
maxJudgeTokens: z.number().int().min(0).optional(),
|
|
518
|
+
/** Cap on examples replayed through the model. */
|
|
519
|
+
maxExamples: z.number().int().min(1).max(200).default(50)
|
|
520
|
+
});
|
|
521
|
+
var VariantEvalWire = z.object({
|
|
522
|
+
variantKey: z.string(),
|
|
523
|
+
evaluatorName: z.string(),
|
|
524
|
+
evaluatorVersion: z.string(),
|
|
525
|
+
scored: z.number().int(),
|
|
526
|
+
unscored: z.number().int(),
|
|
527
|
+
meanScore: z.number().nullable(),
|
|
528
|
+
passCount: z.number().int(),
|
|
529
|
+
binary: z.boolean()
|
|
530
|
+
});
|
|
531
|
+
var VariantEvalListResponse = z.object({ variants: z.array(VariantEvalWire) });
|
|
532
|
+
var SpanKindWire = z.enum([
|
|
533
|
+
"LLM",
|
|
534
|
+
"CHAIN",
|
|
535
|
+
"RETRIEVER",
|
|
536
|
+
"EMBEDDING",
|
|
537
|
+
"TOOL",
|
|
538
|
+
"AGENT",
|
|
539
|
+
"RERANKER",
|
|
540
|
+
"GUARDRAIL",
|
|
541
|
+
"EVALUATOR",
|
|
542
|
+
"UNKNOWN"
|
|
543
|
+
]);
|
|
544
|
+
var TraceSummaryWire = z.object({
|
|
545
|
+
traceId: z.string(),
|
|
546
|
+
rootSpanName: z.string(),
|
|
547
|
+
rootSpanKind: SpanKindWire,
|
|
548
|
+
sessionId: z.string().nullable(),
|
|
549
|
+
experimentId: z.string().nullable(),
|
|
550
|
+
variantKey: z.string().nullable(),
|
|
551
|
+
startedAtIso: z.string().datetime(),
|
|
552
|
+
durationMs: z.number(),
|
|
553
|
+
spanCount: z.number().int(),
|
|
554
|
+
errorCount: z.number().int(),
|
|
555
|
+
totalInputTokens: z.number().int(),
|
|
556
|
+
totalOutputTokens: z.number().int()
|
|
557
|
+
});
|
|
558
|
+
var TraceListInput = z.object({
|
|
559
|
+
experimentId: z.string().optional(),
|
|
560
|
+
variantKey: z.string().optional(),
|
|
561
|
+
sessionId: z.string().optional(),
|
|
562
|
+
kind: SpanKindWire.optional(),
|
|
563
|
+
errorsOnly: z.coerce.boolean().optional(),
|
|
564
|
+
fromIso: z.string().datetime().optional(),
|
|
565
|
+
untilIso: z.string().datetime().optional(),
|
|
566
|
+
limit: z.coerce.number().int().min(1).max(200).default(50),
|
|
567
|
+
offset: z.coerce.number().int().min(0).default(0)
|
|
568
|
+
});
|
|
569
|
+
var TraceListResponse = z.object({
|
|
570
|
+
traces: z.array(TraceSummaryWire),
|
|
571
|
+
total: z.number().int()
|
|
572
|
+
});
|
|
573
|
+
var SessionTraceWire = TraceSummaryWire.extend({
|
|
574
|
+
rootInputValue: z.string().nullable(),
|
|
575
|
+
rootOutputValue: z.string().nullable()
|
|
576
|
+
});
|
|
577
|
+
var SessionDetailResponse = z.object({
|
|
578
|
+
sessionId: z.string(),
|
|
579
|
+
/** Conversation order: ascending by trace start time. */
|
|
580
|
+
traces: z.array(SessionTraceWire)
|
|
581
|
+
});
|
|
582
|
+
var SpanWire = z.object({
|
|
583
|
+
traceId: z.string(),
|
|
584
|
+
spanId: z.string(),
|
|
585
|
+
parentSpanId: z.string().nullable(),
|
|
586
|
+
name: z.string(),
|
|
587
|
+
kind: SpanKindWire,
|
|
588
|
+
startedAtIso: z.string().datetime(),
|
|
589
|
+
durationMs: z.number(),
|
|
590
|
+
status: z.enum(["UNSET", "OK", "ERROR"]),
|
|
591
|
+
statusMessage: z.string().nullable(),
|
|
592
|
+
sessionId: z.string().nullable(),
|
|
593
|
+
experimentId: z.string().nullable(),
|
|
594
|
+
variantKey: z.string().nullable(),
|
|
595
|
+
inputValue: z.string().nullable(),
|
|
596
|
+
outputValue: z.string().nullable(),
|
|
597
|
+
inputTokens: z.number().int().nullable(),
|
|
598
|
+
outputTokens: z.number().int().nullable(),
|
|
599
|
+
modelName: z.string().nullable(),
|
|
600
|
+
/** Ingest-time dollar cost from the model-cost manifest; null when unpriceable. */
|
|
601
|
+
costUsd: z.number().nullable(),
|
|
602
|
+
attributes: z.record(z.unknown())
|
|
603
|
+
});
|
|
604
|
+
var SpanAnnotationWire = z.object({
|
|
605
|
+
spanId: z.string(),
|
|
606
|
+
annotator: z.enum(["human", "llm"]),
|
|
607
|
+
/** Evaluator name for llm annotations; user id for human ones. */
|
|
608
|
+
annotatorId: z.string(),
|
|
609
|
+
label: z.string().nullable(),
|
|
610
|
+
score: z.number().nullable(),
|
|
611
|
+
explanation: z.string().nullable(),
|
|
612
|
+
createdAtIso: z.string().datetime()
|
|
613
|
+
});
|
|
614
|
+
var TraceDetail = z.object({
|
|
615
|
+
traceId: z.string(),
|
|
616
|
+
spans: z.array(SpanWire),
|
|
617
|
+
annotations: z.array(SpanAnnotationWire)
|
|
618
|
+
});
|
|
619
|
+
var AnnotateSpanInput = z.object({
|
|
620
|
+
spanId: z.string(),
|
|
621
|
+
label: z.string().max(64).optional(),
|
|
622
|
+
score: z.number().min(0).max(1).optional(),
|
|
623
|
+
explanation: z.string().max(1e4).optional()
|
|
624
|
+
});
|
|
625
|
+
var SpanAnalyticsInput = z.object({
|
|
626
|
+
experimentId: z.string().optional(),
|
|
627
|
+
variantKey: z.string().optional(),
|
|
628
|
+
groupBy: z.enum(["kind", "model"]).default("kind"),
|
|
629
|
+
fromIso: z.string().datetime().optional(),
|
|
630
|
+
untilIso: z.string().datetime().optional()
|
|
631
|
+
});
|
|
632
|
+
var SpanAnalyticsResponse = z.object({
|
|
633
|
+
groups: z.array(
|
|
634
|
+
z.object({
|
|
635
|
+
groupKey: z.string(),
|
|
636
|
+
spanCount: z.number().int(),
|
|
637
|
+
errorCount: z.number().int(),
|
|
638
|
+
meanDurationMs: z.number().nullable(),
|
|
639
|
+
p95DurationMs: z.number().nullable(),
|
|
640
|
+
totalInputTokens: z.number().int(),
|
|
641
|
+
totalOutputTokens: z.number().int(),
|
|
642
|
+
totalCostUsd: z.number()
|
|
643
|
+
})
|
|
644
|
+
),
|
|
645
|
+
traceCount: z.number().int(),
|
|
646
|
+
/** Cost over the full filtered span set — NOT restricted by groupBy model. */
|
|
647
|
+
totalCostUsd: z.number()
|
|
648
|
+
});
|
|
649
|
+
var IngestTokenSummary = z.object({
|
|
650
|
+
tokenId: z.string(),
|
|
651
|
+
label: z.string(),
|
|
652
|
+
createdAtIso: z.string().datetime(),
|
|
653
|
+
revokedAtIso: z.string().datetime().nullable()
|
|
654
|
+
});
|
|
655
|
+
var IngestTokenListResponse = z.object({ tokens: z.array(IngestTokenSummary) });
|
|
656
|
+
var CreateIngestTokenInput = z.object({ label: z.string().min(1).max(120) });
|
|
657
|
+
var CreatedIngestToken = IngestTokenSummary.extend({ token: z.string() });
|
|
316
658
|
var endpoints = {
|
|
317
659
|
"tenants.me": {
|
|
318
660
|
method: "GET",
|
|
@@ -626,6 +968,232 @@ var endpoints = {
|
|
|
626
968
|
mutates: false,
|
|
627
969
|
requiresRole: "viewer"
|
|
628
970
|
},
|
|
971
|
+
"datasets.list": {
|
|
972
|
+
method: "GET",
|
|
973
|
+
path: "/v1/orgs/:orgId/datasets",
|
|
974
|
+
params: Empty,
|
|
975
|
+
response: DatasetListResponse,
|
|
976
|
+
mutates: false,
|
|
977
|
+
requiresRole: "viewer"
|
|
978
|
+
},
|
|
979
|
+
"datasets.create": {
|
|
980
|
+
method: "POST",
|
|
981
|
+
path: "/v1/orgs/:orgId/datasets",
|
|
982
|
+
params: CreateDatasetInput,
|
|
983
|
+
response: DatasetSummary,
|
|
984
|
+
mutates: true,
|
|
985
|
+
requiresRole: "experimenter"
|
|
986
|
+
},
|
|
987
|
+
"datasets.get": {
|
|
988
|
+
method: "GET",
|
|
989
|
+
path: "/v1/orgs/:orgId/datasets/:datasetId",
|
|
990
|
+
params: Empty,
|
|
991
|
+
response: DatasetSummary,
|
|
992
|
+
mutates: false,
|
|
993
|
+
requiresRole: "viewer"
|
|
994
|
+
},
|
|
995
|
+
"datasets.addExamples": {
|
|
996
|
+
method: "POST",
|
|
997
|
+
path: "/v1/orgs/:orgId/datasets/:datasetId/examples",
|
|
998
|
+
params: AddExamplesInput,
|
|
999
|
+
response: AddExamplesResponse,
|
|
1000
|
+
mutates: true,
|
|
1001
|
+
requiresRole: "experimenter"
|
|
1002
|
+
},
|
|
1003
|
+
"datasets.listExamples": {
|
|
1004
|
+
method: "GET",
|
|
1005
|
+
path: "/v1/orgs/:orgId/datasets/:datasetId/examples",
|
|
1006
|
+
params: ListExamplesInput,
|
|
1007
|
+
response: ListExamplesResponse,
|
|
1008
|
+
mutates: false,
|
|
1009
|
+
requiresRole: "viewer"
|
|
1010
|
+
},
|
|
1011
|
+
"datasets.createVersion": {
|
|
1012
|
+
method: "POST",
|
|
1013
|
+
path: "/v1/orgs/:orgId/datasets/:datasetId/versions",
|
|
1014
|
+
params: CreateDatasetVersionInput,
|
|
1015
|
+
response: DatasetVersionWire,
|
|
1016
|
+
mutates: true,
|
|
1017
|
+
requiresRole: "experimenter"
|
|
1018
|
+
},
|
|
1019
|
+
"datasets.listVersions": {
|
|
1020
|
+
method: "GET",
|
|
1021
|
+
path: "/v1/orgs/:orgId/datasets/:datasetId/versions",
|
|
1022
|
+
params: Empty,
|
|
1023
|
+
response: DatasetVersionListResponse,
|
|
1024
|
+
mutates: false,
|
|
1025
|
+
requiresRole: "viewer"
|
|
1026
|
+
},
|
|
1027
|
+
"traces.list": {
|
|
1028
|
+
method: "GET",
|
|
1029
|
+
path: "/v1/orgs/:orgId/traces",
|
|
1030
|
+
params: TraceListInput,
|
|
1031
|
+
response: TraceListResponse,
|
|
1032
|
+
mutates: false,
|
|
1033
|
+
requiresRole: "viewer"
|
|
1034
|
+
},
|
|
1035
|
+
"traces.get": {
|
|
1036
|
+
method: "GET",
|
|
1037
|
+
path: "/v1/orgs/:orgId/traces/:traceId",
|
|
1038
|
+
params: Empty,
|
|
1039
|
+
response: TraceDetail,
|
|
1040
|
+
mutates: false,
|
|
1041
|
+
requiresRole: "viewer"
|
|
1042
|
+
},
|
|
1043
|
+
"traces.annotate": {
|
|
1044
|
+
method: "POST",
|
|
1045
|
+
path: "/v1/orgs/:orgId/traces/:traceId/annotations",
|
|
1046
|
+
params: AnnotateSpanInput,
|
|
1047
|
+
response: SpanAnnotationWire,
|
|
1048
|
+
mutates: true,
|
|
1049
|
+
requiresRole: "experimenter"
|
|
1050
|
+
},
|
|
1051
|
+
"traces.analytics": {
|
|
1052
|
+
method: "GET",
|
|
1053
|
+
path: "/v1/orgs/:orgId/traces/analytics",
|
|
1054
|
+
params: SpanAnalyticsInput,
|
|
1055
|
+
response: SpanAnalyticsResponse,
|
|
1056
|
+
mutates: false,
|
|
1057
|
+
requiresRole: "viewer"
|
|
1058
|
+
},
|
|
1059
|
+
"traces.session": {
|
|
1060
|
+
method: "GET",
|
|
1061
|
+
path: "/v1/orgs/:orgId/traces/sessions/:sessionId",
|
|
1062
|
+
params: Empty,
|
|
1063
|
+
response: SessionDetailResponse,
|
|
1064
|
+
mutates: false,
|
|
1065
|
+
requiresRole: "viewer"
|
|
1066
|
+
},
|
|
1067
|
+
"evals.createRun": {
|
|
1068
|
+
method: "POST",
|
|
1069
|
+
path: "/v1/orgs/:orgId/evals/runs",
|
|
1070
|
+
params: CreateEvalRunInput,
|
|
1071
|
+
response: EvalRunSummary,
|
|
1072
|
+
mutates: true,
|
|
1073
|
+
requiresRole: "experimenter"
|
|
1074
|
+
},
|
|
1075
|
+
"evals.listRuns": {
|
|
1076
|
+
method: "GET",
|
|
1077
|
+
path: "/v1/orgs/:orgId/evals/runs",
|
|
1078
|
+
params: EvalRunListInput,
|
|
1079
|
+
response: EvalRunListResponse,
|
|
1080
|
+
mutates: false,
|
|
1081
|
+
requiresRole: "viewer"
|
|
1082
|
+
},
|
|
1083
|
+
"evals.getRun": {
|
|
1084
|
+
method: "GET",
|
|
1085
|
+
path: "/v1/orgs/:orgId/evals/runs/:runId",
|
|
1086
|
+
params: Empty,
|
|
1087
|
+
response: EvalRunDetail,
|
|
1088
|
+
mutates: false,
|
|
1089
|
+
requiresRole: "viewer"
|
|
1090
|
+
},
|
|
1091
|
+
"evals.listScores": {
|
|
1092
|
+
method: "GET",
|
|
1093
|
+
path: "/v1/orgs/:orgId/evals/runs/:runId/scores",
|
|
1094
|
+
params: EvalScoreListInput,
|
|
1095
|
+
response: EvalScoreListResponse,
|
|
1096
|
+
mutates: false,
|
|
1097
|
+
requiresRole: "viewer"
|
|
1098
|
+
},
|
|
1099
|
+
"evals.cancelRun": {
|
|
1100
|
+
method: "POST",
|
|
1101
|
+
path: "/v1/orgs/:orgId/evals/runs/:runId/cancel",
|
|
1102
|
+
params: Empty,
|
|
1103
|
+
response: EvalRunSummary,
|
|
1104
|
+
mutates: true,
|
|
1105
|
+
requiresRole: "experimenter"
|
|
1106
|
+
},
|
|
1107
|
+
"evals.byVariant": {
|
|
1108
|
+
method: "GET",
|
|
1109
|
+
path: "/v1/orgs/:orgId/experiments/:experimentId/evals",
|
|
1110
|
+
params: Empty,
|
|
1111
|
+
response: VariantEvalListResponse,
|
|
1112
|
+
mutates: false,
|
|
1113
|
+
requiresRole: "viewer"
|
|
1114
|
+
},
|
|
1115
|
+
"ingestTokens.list": {
|
|
1116
|
+
method: "GET",
|
|
1117
|
+
path: "/v1/orgs/:orgId/ingest-tokens",
|
|
1118
|
+
params: Empty,
|
|
1119
|
+
response: IngestTokenListResponse,
|
|
1120
|
+
mutates: false,
|
|
1121
|
+
requiresRole: "admin"
|
|
1122
|
+
},
|
|
1123
|
+
"ingestTokens.create": {
|
|
1124
|
+
method: "POST",
|
|
1125
|
+
path: "/v1/orgs/:orgId/ingest-tokens",
|
|
1126
|
+
params: CreateIngestTokenInput,
|
|
1127
|
+
response: CreatedIngestToken,
|
|
1128
|
+
mutates: true,
|
|
1129
|
+
requiresRole: "admin"
|
|
1130
|
+
},
|
|
1131
|
+
"ingestTokens.revoke": {
|
|
1132
|
+
method: "POST",
|
|
1133
|
+
path: "/v1/orgs/:orgId/ingest-tokens/:tokenId/revoke",
|
|
1134
|
+
params: Empty,
|
|
1135
|
+
response: IngestTokenSummary,
|
|
1136
|
+
mutates: true,
|
|
1137
|
+
requiresRole: "admin"
|
|
1138
|
+
},
|
|
1139
|
+
"prompts.list": {
|
|
1140
|
+
method: "GET",
|
|
1141
|
+
path: "/v1/orgs/:orgId/prompts",
|
|
1142
|
+
params: Empty,
|
|
1143
|
+
response: PromptListResponse,
|
|
1144
|
+
mutates: false,
|
|
1145
|
+
requiresRole: "viewer"
|
|
1146
|
+
},
|
|
1147
|
+
"prompts.create": {
|
|
1148
|
+
method: "POST",
|
|
1149
|
+
path: "/v1/orgs/:orgId/prompts",
|
|
1150
|
+
params: CreatePromptInput,
|
|
1151
|
+
response: PromptSummary,
|
|
1152
|
+
mutates: true,
|
|
1153
|
+
requiresRole: "experimenter"
|
|
1154
|
+
},
|
|
1155
|
+
"prompts.get": {
|
|
1156
|
+
method: "GET",
|
|
1157
|
+
path: "/v1/orgs/:orgId/prompts/:promptId",
|
|
1158
|
+
params: Empty,
|
|
1159
|
+
response: PromptSummary,
|
|
1160
|
+
mutates: false,
|
|
1161
|
+
requiresRole: "viewer"
|
|
1162
|
+
},
|
|
1163
|
+
"prompts.createVersion": {
|
|
1164
|
+
method: "POST",
|
|
1165
|
+
path: "/v1/orgs/:orgId/prompts/:promptId/versions",
|
|
1166
|
+
params: CreatePromptVersionInput,
|
|
1167
|
+
response: PromptVersionWire,
|
|
1168
|
+
mutates: true,
|
|
1169
|
+
requiresRole: "experimenter"
|
|
1170
|
+
},
|
|
1171
|
+
"prompts.listVersions": {
|
|
1172
|
+
method: "GET",
|
|
1173
|
+
path: "/v1/orgs/:orgId/prompts/:promptId/versions",
|
|
1174
|
+
params: Empty,
|
|
1175
|
+
response: PromptVersionListResponse,
|
|
1176
|
+
mutates: false,
|
|
1177
|
+
requiresRole: "viewer"
|
|
1178
|
+
},
|
|
1179
|
+
"playground.invoke": {
|
|
1180
|
+
method: "POST",
|
|
1181
|
+
path: "/v1/orgs/:orgId/playground/invoke",
|
|
1182
|
+
params: PlaygroundInvokeInput,
|
|
1183
|
+
response: PlaygroundInvokeResponse,
|
|
1184
|
+
mutates: false,
|
|
1185
|
+
requiresRole: "experimenter"
|
|
1186
|
+
},
|
|
1187
|
+
// 202-style: replay only creates a pending eval run (durable via the
|
|
1188
|
+
// outbox); generation + judging happen in the executor, off the request.
|
|
1189
|
+
"playground.replay": {
|
|
1190
|
+
method: "POST",
|
|
1191
|
+
path: "/v1/orgs/:orgId/playground/replay",
|
|
1192
|
+
params: PlaygroundReplayInput,
|
|
1193
|
+
response: EvalRunSummary,
|
|
1194
|
+
mutates: true,
|
|
1195
|
+
requiresRole: "experimenter"
|
|
1196
|
+
},
|
|
629
1197
|
"harness.definitions": {
|
|
630
1198
|
method: "GET",
|
|
631
1199
|
path: "/v1/orgs/:orgId/harness/admin/agents",
|
|
@@ -664,6 +1232,6 @@ var ApiError = z.object({
|
|
|
664
1232
|
details: z.record(z.unknown()).optional()
|
|
665
1233
|
});
|
|
666
1234
|
|
|
667
|
-
export { AggregateInput, AggregateResponse, AggregateRow, ApiError, ApiKeyListResponse, ApiKeySummary, AuditEvent, AuditExport, AuditExportListResponse, AuditListResponse, CreateApiKeyInput, CreateApiKeyResponse, CreateAuditExportInput, CreateExperimentInput, CreateExperimentResponse, CreateTestRunInput, EdgePushResult, EnvironmentListResponse, ExperimentListResponse, ExperimentSummary, FeatureListResponse, FeatureRuleListResponse, GovernanceSettings, GovernanceStatusResponse, HarnessJobInput, HarnessJobResponse, HarnessOperatorDefinition, HarnessOperatorListResponse, InvokeActionInput, InvokeActionResponse, ManifestKey, ManifestKeysResponse, PreviewSignInput, PreviewSignResponse, ProjectListResponse, PromoteEnvironmentInput, PromoteEnvironmentResponse, PropertyListResponse, PublicationListResponse, PublishManifestResponse, QualityGateResult, SamlConfig, TenantSelf, TestCaseResult, TestCaseStatus, TestRun, TestRunArtifact, TestRunListInput, TestRunListResponse, TestRunStatus, TestRunSummary, UpdateGovernanceSettingsInput, buildPath, endpoints };
|
|
1235
|
+
export { AddExamplesInput, AddExamplesResponse, AggregateInput, AggregateResponse, AggregateRow, AnnotateSpanInput, ApiError, ApiKeyListResponse, ApiKeySummary, AuditEvent, AuditExport, AuditExportListResponse, AuditListResponse, CreateApiKeyInput, CreateApiKeyResponse, CreateAuditExportInput, CreateDatasetInput, CreateDatasetVersionInput, CreateEvalRunInput, CreateExperimentInput, CreateExperimentResponse, CreateIngestTokenInput, CreatePromptInput, CreatePromptVersionInput, CreateTestRunInput, CreatedIngestToken, DatasetExampleWire, DatasetListResponse, DatasetSummary, DatasetVersionListResponse, DatasetVersionWire, EdgePushResult, EnvironmentListResponse, EvalRunDetail, EvalRunGenerationWire, EvalRunListInput, EvalRunListResponse, EvalRunSummary, EvalRunTargetWire, EvalScoreListInput, EvalScoreListResponse, EvalScoreWire, EvaluatorAggregateWire, ExperimentListResponse, ExperimentSummary, FeatureListResponse, FeatureRuleListResponse, GovernanceSettings, GovernanceStatusResponse, HarnessJobInput, HarnessJobResponse, HarnessOperatorDefinition, HarnessOperatorListResponse, IngestTokenListResponse, IngestTokenSummary, InvokeActionInput, InvokeActionResponse, ListExamplesInput, ListExamplesResponse, ManifestKey, ManifestKeysResponse, PlaygroundInvokeInput, PlaygroundInvokeResponse, PlaygroundReplayInput, PreviewSignInput, PreviewSignResponse, ProjectListResponse, PromoteEnvironmentInput, PromoteEnvironmentResponse, PromptListResponse, PromptMessageWire, PromptSummary, PromptTemplateWire, PromptVersionListResponse, PromptVersionWire, PropertyListResponse, PublicationListResponse, PublishManifestResponse, QualityGateResult, SamlConfig, SessionDetailResponse, SessionTraceWire, SpanAnalyticsInput, SpanAnalyticsResponse, SpanAnnotationWire, SpanKindWire, SpanWire, TenantSelf, TestCaseResult, TestCaseStatus, TestRun, TestRunArtifact, TestRunListInput, TestRunListResponse, TestRunStatus, TestRunSummary, TraceDetail, TraceListInput, TraceListResponse, TraceSummaryWire, UpdateGovernanceSettingsInput, VariantEvalListResponse, VariantEvalWire, buildPath, endpoints };
|
|
668
1236
|
//# sourceMappingURL=index.js.map
|
|
669
1237
|
//# sourceMappingURL=index.js.map
|