@fabricorg/experiments-api-client 0.1.7 → 0.1.9

package/dist/index.d.cts

@@ -6,6 +6,8 @@ interface ApiClientOptions {
     apiKey?: string;
     /** Upstream OAuth bearer (for example, a Databricks Apps gateway token). */
     bearerToken?: string;
+    /** Additional request headers, primarily for server-side session forwarding. */
+    headers?: Record<string, string>;
     /** Override fetch (for tests / SSR). Defaults to globalThis.fetch. */
     fetchImpl?: typeof fetch;
     /** Default org id used when an endpoint's path includes :orgId and no override is supplied. */
@@ -944,6 +946,465 @@ declare class ApiClient {
             certificate: string | null;
         }>;
     };
+    datasets: {
+        list: (orgId?: string) => Promise<{
+            datasets: {
+                id: string;
+                name: string;
+                latestVersion: number;
+                exampleCount: number;
+                createdAtIso: string;
+                updatedAtIso: string;
+                description?: string | undefined;
+            }[];
+        }>;
+        create: (params: EndpointParams<"datasets.create">, orgId?: string) => Promise<{
+            id: string;
+            name: string;
+            latestVersion: number;
+            exampleCount: number;
+            createdAtIso: string;
+            updatedAtIso: string;
+            description?: string | undefined;
+        }>;
+        get: (datasetId: string, orgId?: string) => Promise<{
+            id: string;
+            name: string;
+            latestVersion: number;
+            exampleCount: number;
+            createdAtIso: string;
+            updatedAtIso: string;
+            description?: string | undefined;
+        }>;
+        addExamples: (datasetId: string, params: EndpointParams<"datasets.addExamples">, orgId?: string) => Promise<{
+            added: number;
+        }>;
+        listExamples: (datasetId: string, params: EndpointParams<"datasets.listExamples">, orgId?: string) => Promise<{
+            total: number;
+            examples: {
+                exampleId: string;
+                metadata: Record<string, unknown>;
+                split: "validation" | "train" | "test";
+                input?: unknown;
+                expectedOutput?: unknown;
+            }[];
+        }>;
+        createVersion: (datasetId: string, params: EndpointParams<"datasets.createVersion">, orgId?: string) => Promise<{
+            contentHash: string;
+            exampleCount: number;
+            createdAtIso: string;
+            version: number;
+            datasetId: string;
+            note?: string | undefined;
+        }>;
+        listVersions: (datasetId: string, orgId?: string) => Promise<{
+            versions: {
+                contentHash: string;
+                exampleCount: number;
+                createdAtIso: string;
+                version: number;
+                datasetId: string;
+                note?: string | undefined;
+            }[];
+        }>;
+    };
+    evals: {
+        createRun: (params: EndpointParams<"evals.createRun">, orgId?: string) => Promise<{
+            status: "running" | "failed" | "pending" | "succeeded" | "cancelled";
+            runId: string;
+            createdAtIso: string;
+            target: {
+                kind: "dataset";
+                version: number;
+                datasetId: string;
+            } | {
+                experimentId: string;
+                kind: "experiment";
+                metricEventName?: string | undefined;
+            };
+            evaluatorNames: string[];
+            passThreshold: number;
+            scoredExamples: number;
+            totalExamples: number;
+            tokensSpent: number;
+            costUsd: number | null;
+            error?: string | undefined;
+            generation?: {
+                datasetId: string;
+                promptId: string;
+                promptVersion: number;
+                datasetVersion: number;
+                maxExamples: number;
+            } | undefined;
+            startedAtIso?: string | undefined;
+            finishedAtIso?: string | undefined;
+        }>;
+        listRuns: (params: EndpointParams<"evals.listRuns">, orgId?: string) => Promise<{
+            runs: {
+                status: "running" | "failed" | "pending" | "succeeded" | "cancelled";
+                runId: string;
+                createdAtIso: string;
+                target: {
+                    kind: "dataset";
+                    version: number;
+                    datasetId: string;
+                } | {
+                    experimentId: string;
+                    kind: "experiment";
+                    metricEventName?: string | undefined;
+                };
+                evaluatorNames: string[];
+                passThreshold: number;
+                scoredExamples: number;
+                totalExamples: number;
+                tokensSpent: number;
+                costUsd: number | null;
+                error?: string | undefined;
+                generation?: {
+                    datasetId: string;
+                    promptId: string;
+                    promptVersion: number;
+                    datasetVersion: number;
+                    maxExamples: number;
+                } | undefined;
+                startedAtIso?: string | undefined;
+                finishedAtIso?: string | undefined;
+            }[];
+        }>;
+        getRun: (runId: string, orgId?: string) => Promise<{
+            run: {
+                status: "running" | "failed" | "pending" | "succeeded" | "cancelled";
+                runId: string;
+                createdAtIso: string;
+                target: {
+                    kind: "dataset";
+                    version: number;
+                    datasetId: string;
+                } | {
+                    experimentId: string;
+                    kind: "experiment";
+                    metricEventName?: string | undefined;
+                };
+                evaluatorNames: string[];
+                passThreshold: number;
+                scoredExamples: number;
+                totalExamples: number;
+                tokensSpent: number;
+                costUsd: number | null;
+                error?: string | undefined;
+                generation?: {
+                    datasetId: string;
+                    promptId: string;
+                    promptVersion: number;
+                    datasetVersion: number;
+                    maxExamples: number;
+                } | undefined;
+                startedAtIso?: string | undefined;
+                finishedAtIso?: string | undefined;
+            };
+            aggregates: {
+                evaluatorName: string;
+                evaluatorVersion: string;
+                scored: number;
+                unscored: number;
+                meanScore: number | null;
+                passRate: {
+                    rate: number;
+                    low: number;
+                    high: number;
+                } | null;
+                labelCounts: Record<string, number>;
+                totalTokens: number;
+                meanLatencyMs: number | null;
+            }[];
+        }>;
+        listScores: (runId: string, params: EndpointParams<"evals.listScores">, orgId?: string) => Promise<{
+            total: number;
+            scores: {
+                evaluatorName: string;
+                evaluatorVersion: string;
+                itemId: string;
+                label: string | null;
+                score: number | null;
+                explanation: string | null;
+                latencyMs: number;
+            }[];
+        }>;
+        cancelRun: (runId: string, orgId?: string) => Promise<{
+            status: "running" | "failed" | "pending" | "succeeded" | "cancelled";
+            runId: string;
+            createdAtIso: string;
+            target: {
+                kind: "dataset";
+                version: number;
+                datasetId: string;
+            } | {
+                experimentId: string;
+                kind: "experiment";
+                metricEventName?: string | undefined;
+            };
+            evaluatorNames: string[];
+            passThreshold: number;
+            scoredExamples: number;
+            totalExamples: number;
+            tokensSpent: number;
+            costUsd: number | null;
+            error?: string | undefined;
+            generation?: {
+                datasetId: string;
+                promptId: string;
+                promptVersion: number;
+                datasetVersion: number;
+                maxExamples: number;
+            } | undefined;
+            startedAtIso?: string | undefined;
+            finishedAtIso?: string | undefined;
+        }>;
+        byVariant: (experimentId: string, orgId?: string) => Promise<{
+            variants: {
+                variantKey: string;
+                evaluatorName: string;
+                evaluatorVersion: string;
+                scored: number;
+                unscored: number;
+                meanScore: number | null;
+                passCount: number;
+                binary: boolean;
+            }[];
+        }>;
+    };
+    traces: {
+        list: (params: EndpointParams<"traces.list">, orgId?: string) => Promise<{
+            total: number;
+            traces: {
+                experimentId: string | null;
+                variantKey: string | null;
+                durationMs: number;
+                startedAtIso: string;
+                traceId: string;
+                rootSpanName: string;
+                rootSpanKind: "LLM" | "CHAIN" | "RETRIEVER" | "EMBEDDING" | "TOOL" | "AGENT" | "RERANKER" | "GUARDRAIL" | "EVALUATOR" | "UNKNOWN";
+                sessionId: string | null;
+                spanCount: number;
+                errorCount: number;
+                totalInputTokens: number;
+                totalOutputTokens: number;
+            }[];
+        }>;
+        get: (traceId: string, orgId?: string) => Promise<{
+            traceId: string;
+            spans: {
+                status: "UNSET" | "OK" | "ERROR";
+                name: string;
+                experimentId: string | null;
+                variantKey: string | null;
+                durationMs: number;
+                kind: "LLM" | "CHAIN" | "RETRIEVER" | "EMBEDDING" | "TOOL" | "AGENT" | "RERANKER" | "GUARDRAIL" | "EVALUATOR" | "UNKNOWN";
+                costUsd: number | null;
+                startedAtIso: string;
+                inputTokens: number | null;
+                outputTokens: number | null;
+                traceId: string;
+                sessionId: string | null;
+                spanId: string;
+                parentSpanId: string | null;
+                statusMessage: string | null;
+                inputValue: string | null;
+                outputValue: string | null;
+                modelName: string | null;
+                attributes: Record<string, unknown>;
+            }[];
+            annotations: {
+                createdAtIso: string;
+                label: string | null;
+                score: number | null;
+                explanation: string | null;
+                spanId: string;
+                annotator: "human" | "llm";
+                annotatorId: string;
+            }[];
+        }>;
+        annotate: (traceId: string, params: EndpointParams<"traces.annotate">, orgId?: string) => Promise<{
+            createdAtIso: string;
+            label: string | null;
+            score: number | null;
+            explanation: string | null;
+            spanId: string;
+            annotator: "human" | "llm";
+            annotatorId: string;
+        }>;
+        analytics: (params: EndpointParams<"traces.analytics">, orgId?: string) => Promise<{
+            groups: {
+                spanCount: number;
+                errorCount: number;
+                totalInputTokens: number;
+                totalOutputTokens: number;
+                groupKey: string;
+                meanDurationMs: number | null;
+                p95DurationMs: number | null;
+                totalCostUsd: number;
+            }[];
+            totalCostUsd: number;
+            traceCount: number;
+        }>;
+        session: (sessionId: string, orgId?: string) => Promise<{
+            sessionId: string;
+            traces: {
+                experimentId: string | null;
+                variantKey: string | null;
+                durationMs: number;
+                startedAtIso: string;
+                traceId: string;
+                rootSpanName: string;
+                rootSpanKind: "LLM" | "CHAIN" | "RETRIEVER" | "EMBEDDING" | "TOOL" | "AGENT" | "RERANKER" | "GUARDRAIL" | "EVALUATOR" | "UNKNOWN";
+                sessionId: string | null;
+                spanCount: number;
+                errorCount: number;
+                totalInputTokens: number;
+                totalOutputTokens: number;
+                rootInputValue: string | null;
+                rootOutputValue: string | null;
+            }[];
+        }>;
+    };
+    ingestTokens: {
+        list: (orgId?: string) => Promise<{
+            tokens: {
+                createdAtIso: string;
+                label: string;
+                tokenId: string;
+                revokedAtIso: string | null;
+            }[];
+        }>;
+        create: (params: EndpointParams<"ingestTokens.create">, orgId?: string) => Promise<{
+            token: string;
+            createdAtIso: string;
+            label: string;
+            tokenId: string;
+            revokedAtIso: string | null;
+        }>;
+        revoke: (tokenId: string, orgId?: string) => Promise<{
+            createdAtIso: string;
+            label: string;
+            tokenId: string;
+            revokedAtIso: string | null;
+        }>;
+    };
+    prompts: {
+        list: (orgId?: string) => Promise<{
+            prompts: {
+                id: string;
+                name: string;
+                latestVersion: number;
+                createdAtIso: string;
+                updatedAtIso: string;
+                description?: string | undefined;
+            }[];
+        }>;
+        create: (params: EndpointParams<"prompts.create">, orgId?: string) => Promise<{
+            id: string;
+            name: string;
+            latestVersion: number;
+            createdAtIso: string;
+            updatedAtIso: string;
+            description?: string | undefined;
+        }>;
+        get: (promptId: string, orgId?: string) => Promise<{
+            id: string;
+            name: string;
+            latestVersion: number;
+            createdAtIso: string;
+            updatedAtIso: string;
+            description?: string | undefined;
+        }>;
+        createVersion: (promptId: string, params: EndpointParams<"prompts.createVersion">, orgId?: string) => Promise<{
+            contentHash: string;
+            createdAtIso: string;
+            version: number;
+            promptId: string;
+            template: {
+                params: {
+                    model?: string | undefined;
+                    temperature?: number | undefined;
+                    maxTokens?: number | undefined;
+                    topP?: number | undefined;
+                };
+                messages: {
+                    role: "user" | "system" | "assistant";
+                    content: string;
+                }[];
+            };
+            variables: string[];
+            note?: string | undefined;
+        }>;
+        listVersions: (promptId: string, orgId?: string) => Promise<{
+            versions: {
+                contentHash: string;
+                createdAtIso: string;
+                version: number;
+                promptId: string;
+                template: {
+                    params: {
+                        model?: string | undefined;
+                        temperature?: number | undefined;
+                        maxTokens?: number | undefined;
+                        topP?: number | undefined;
+                    };
+                    messages: {
+                        role: "user" | "system" | "assistant";
+                        content: string;
+                    }[];
+                };
+                variables: string[];
+                note?: string | undefined;
+            }[];
+        }>;
+    };
+    playground: {
+        invoke: (params: EndpointParams<"playground.invoke">, orgId?: string) => Promise<{
+            costUsd: number | null;
+            latencyMs: number;
+            messages: {
+                role: "user" | "system" | "assistant";
+                content: string;
+            }[];
+            missingVariables: string[];
+            text: string;
+            inputTokens: number;
+            outputTokens: number;
+            modelEndpoint: string;
+        }>;
+        replay: (params: EndpointParams<"playground.replay">, orgId?: string) => Promise<{
+            status: "running" | "failed" | "pending" | "succeeded" | "cancelled";
+            runId: string;
+            createdAtIso: string;
+            target: {
+                kind: "dataset";
+                version: number;
+                datasetId: string;
+            } | {
+                experimentId: string;
+                kind: "experiment";
+                metricEventName?: string | undefined;
+            };
+            evaluatorNames: string[];
+            passThreshold: number;
+            scoredExamples: number;
+            totalExamples: number;
+            tokensSpent: number;
+            costUsd: number | null;
+            error?: string | undefined;
+            generation?: {
+                datasetId: string;
+                promptId: string;
+                promptVersion: number;
+                datasetVersion: number;
+                maxExamples: number;
+            } | undefined;
+            startedAtIso?: string | undefined;
+            finishedAtIso?: string | undefined;
+        }>;
+    };
     testRuns: {
         list: (params?: EndpointParams<"testRuns.list">, orgId?: string) => Promise<{
             runs: {
@@ -953,7 +1414,7 @@ declare class ApiClient {
                 updatedAt: string;
                 suite: string;
                 durationMs: number;
-                source: "bdd" | "live-suite" | "junit" | "cli" | "other";
+                source: "bdd" | "live-suite" | "junit" | "cli" | "other" | "eval-run";
                 summary: {
                     passed: number;
                     failed: number;
@@ -982,7 +1443,7 @@ declare class ApiClient {
             updatedAt: string;
             suite: string;
             durationMs: number;
-            source: "bdd" | "live-suite" | "junit" | "cli" | "other";
+            source: "bdd" | "live-suite" | "junit" | "cli" | "other" | "eval-run";
             summary: {
                 passed: number;
                 failed: number;
@@ -1022,7 +1483,7 @@ declare class ApiClient {
             updatedAt: string;
             suite: string;
             durationMs: number;
-            source: "bdd" | "live-suite" | "junit" | "cli" | "other";
+            source: "bdd" | "live-suite" | "junit" | "cli" | "other" | "eval-run";
             summary: {
                 passed: number;
                 failed: number;
@@ -1056,6 +1517,15 @@ declare class ApiClient {
             startedAt?: string | null | undefined;
         }>;
     };
+    harness: {
+        definitions: (orgId?: string) => Promise<{
+            name: "quality-status" | "srm-watcher" | "analyst";
+            kind: "job";
+            description: string;
+            mutates: boolean;
+        }[]>;
+        invokeJob: (jobName: string, params?: EndpointParams<"harness.invokeJob">, orgId?: string) => Promise<Record<string, unknown>>;
+    };
 }
 
 export { ApiClient, type ApiClientOptions, ApiError, type CallPathParams };