orchestrated 0.1.10 → 0.1.12

package/index.d.ts

@@ -13,7 +13,7 @@ import { ClientOptions } from 'openai';
 import { default as default_2 } from 'openai';
 import type { Metadata } from 'openai/resources';
 import OpenAI from 'openai';
-import type { z } from 'zod';
+import { z } from 'zod';
 
 /**
  * Async task function type
@@ -180,16 +180,6 @@ export declare const Behavioral: ((args: unknown) => Promise<Score>) & {
     definition?: SerializableScorerDefinition;
 };
 
-/**
- * Bundle reference with fingerprint
- */
-declare interface BundleReference {
-    /** S3 URI to the bundle file (e.g., s3://bucket/fingerprint/handlers.bundle.js) */
-    file: string;
-    /** Bundle fingerprint from Bun's build output */
-    fingerprint: string;
-}
-
 /**
  * Clear all tracked evaluations (for testing)
  */
@@ -270,6 +260,11 @@ declare type DataSourceDefinition = {
             method: "GET" | "POST";
         };
         options: Record<string, any>;
+        /**
+         * JSON Schema describing the arguments this dataset accepts
+         * Generated from the Zod schema for the dataset options
+         */
+        args?: Record<string, any>;
     };
 } | {
     type: "static";
@@ -694,17 +689,6 @@ export declare function getRunningEvaluationCount(): number;
  */
 export declare function getState(): Readonly<EvalState>;
 
-/**
- * Reference to a custom handler function stored in globalThis.handlers
- */
-declare interface HandlerReference {
-    type: "custom_function";
-    /** Handler name (derived from scorer name, e.g., "ExecutionHandler") */
-    reference: string;
-    /** Bundle reference with fingerprint */
-    bundle?: BundleReference;
-}
-
 /**
  * Defines High-Resolution Time.
  *
@@ -719,32 +703,6 @@ declare interface HandlerReference {
  */
 declare type HrTime = [number, number];
 
-/**
- * API Gateway data source configuration
- */
-declare interface HTTPDataSourceOptions {
-    /**
-     * Base URL of the API Gateway endpoint
-     * @example "https://abc123.execute-api.ap-southeast-2.amazonaws.com/prod"
-     */
-    baseUrl: string;
-    /**
-     * Cognito access token for Bearer authentication
-     * If not provided, will use state.accessToken (from ORCHESTRATED_ACCESS_TOKEN environment variable)
-     */
-    accessToken?: string;
-    /**
-     * API Key for API Key authentication
-     * Use this instead of accessToken for programmatic access
-     */
-    apiKey?: string;
-    /**
-     * Request timeout in milliseconds
-     * @default 30000
-     */
-    timeout?: number;
-}
-
 /**
  * Initialize global state with optional overrides
  *
@@ -786,48 +744,26 @@ export declare function initState(partial?: PartialEvalState, skipAuth?: boolean
  * });
  * ```
  */
-export declare function interactions(options?: Partial<Omit<InteractionsDatasetOptions, "baseUrl">>): DataSourceDefinition;
+export declare function interactions(options?: Partial<InteractionsDatasetOptions>): DataSourceDefinition;
 
 /**
- * Options for fetching the interactions dataset
+ * TypeScript type inferred from Zod schema
  */
-export declare interface InteractionsDatasetOptions extends HTTPDataSourceOptions {
-    /**
-     * Tenant ID to filter interactions
-     */
-    tenantId: string;
-    /**
-     * Service name to filter interactions
-     */
-    serviceName: string;
-    /**
-     * Environment to filter interactions
-     * @example "production", "staging", "development"
-     */
-    environment: string;
-    /**
-     * Optional group name for organizing datasets in the same namespace
-     * If not provided, defaults to "interactions"
-     * This is used to build the source_type namespace (e.g., "live.interactions")
-     * @example "sessions", "user-queries"
-     */
-    group_name?: string;
-    /**
-     * Optional month filter (date string)
-     * @example "01/2026"
-     */
-    month?: string;
-    /**
-     * Optional period filter (date string)
-     * @example "01/01/2026"
-     */
-    startDate?: string;
-    /**
-     * Optional period filter (date string)
-     * @example "01/01/2026"
-     */
-    endDate?: string;
-}
+export declare type InteractionsDatasetOptions = z.infer<typeof InteractionsDatasetOptionsSchema>;
+
+/**
+ * Zod schema for interactions dataset options
+ */
+declare const InteractionsDatasetOptionsSchema: z.ZodObject<{
+    tenantId: z.ZodString;
+    serviceName: z.ZodString;
+    environment: z.ZodString;
+    group_name: z.ZodOptional<z.ZodString>;
+    month: z.ZodOptional<z.ZodString>;
+    startDate: z.ZodOptional<z.ZodString>;
+    endDate: z.ZodOptional<z.ZodString>;
+    timeout: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
 
 export declare const iso: {
     writeln: (text: string) => void;
@@ -1033,28 +969,60 @@ declare interface ScorerMetadata {
 export declare type ScorerName = keyof typeof SCORERS;
 
 /**
- * Reference to a scorer in evaluation definitions
- */
-declare type ScorerReference = {
-    type: "internal";
-    name: string;
-} | {
-    type: "custom_scorer";
-    slug: string;
-    fingerprint?: string;
-};
-
-/**
- * Scorer reference definition (from definitions.json)
- */
-declare type ScorerReferenceDefinition = {
-    type: "internal";
-    name: string;
-} | {
-    type: "custom_scorer";
-    slug: string;
-    fingerprint?: string;
-};
+ * Scorer reference or full definition
+ * Can be either a reference (internal or custom_scorer by slug/fingerprint)
+ * or a complete definition (prompt or custom_scorer with all fields)
+ */
+declare type ScorerReferenceDefinition = ScorerReferenceOrDefinition;
+
+declare type ScorerReferenceOrDefinition = z.infer<typeof ScorerReferenceOrDefinitionSchema>;
+
+/**
+ * Union schema for scorer references or full definitions
+ */
+declare const ScorerReferenceOrDefinitionSchema: z.ZodUnion<readonly [z.ZodDiscriminatedUnion<[z.ZodObject<{
+    type: z.ZodLiteral<"internal">;
+    name: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"custom_scorer">;
+    slug: z.ZodString;
+    fingerprint: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>], "type">, z.ZodDiscriminatedUnion<[z.ZodObject<{
+    name: z.ZodString;
+    slug: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    schema: z.ZodObject<{
+        type: z.ZodLiteral<"zod">;
+        definition: z.ZodString;
+    }, z.core.$strip>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    fingerprint: z.ZodOptional<z.ZodString>;
+    type: z.ZodLiteral<"prompt">;
+    promptTemplate: z.ZodString;
+    choiceScores: z.ZodRecord<z.ZodString, z.ZodNumber>;
+    model: z.ZodOptional<z.ZodString>;
+    useCoT: z.ZodOptional<z.ZodBoolean>;
+    temperature: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>, z.ZodObject<{
+    name: z.ZodString;
+    slug: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    schema: z.ZodObject<{
+        type: z.ZodLiteral<"zod">;
+        definition: z.ZodString;
+    }, z.core.$strip>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    fingerprint: z.ZodOptional<z.ZodString>;
+    type: z.ZodLiteral<"custom_scorer">;
+    handler: z.ZodObject<{
+        type: z.ZodLiteral<"custom_function">;
+        reference: z.ZodString;
+        bundle: z.ZodOptional<z.ZodObject<{
+            file: z.ZodString;
+            fingerprint: z.ZodString;
+        }, z.core.$strip>>;
+    }, z.core.$strip>;
+}, z.core.$strip>], "type">]>;
 
 export declare class ScorerRegistry {
     /**
@@ -1094,113 +1062,139 @@ declare type ScoreWithArgs = Score & {
     args?: Record<string, any>;
 };
 
-/**
- * Serializable custom scorer with handler function
- */
-export declare interface SerializableCustomScorer extends SerializableScorerBase {
-    /** Type of scorer */
-    type: "custom_scorer";
-    /** Reference to the custom handler function */
-    handler: HandlerReference;
-}
-
-/**
- * Serializable data source configuration
- */
-declare type SerializableDataSource = {
-    /** Dataset type - references a named dataset */
-    type: "dataset";
-    /** Dataset configuration */
-    config: {
-        /** Dataset name (e.g., "orchestrated.dataset.interactions") */
-        name: string;
-        /** HTTP endpoint configuration */
-        source: {
-            baseUrl: string;
-            path: string;
-            method: "GET" | "POST";
-        };
-        /** Dataset options */
-        options: Record<string, any>;
-    };
-} | {
-    /** Static type - inline or executed data */
-    type: "static";
-    /** Static data array */
-    data: unknown[];
-};
-
-/**
- * Serializable evaluation definition
- */
-export declare interface SerializableEvaluation {
-    /** Evaluation name */
-    name: string;
-    /** URL-safe identifier */
-    slug: string;
-    /** Data source configuration */
-    data: SerializableDataSource;
-    /** Optional task reference */
-    task?: TaskReference;
-    /** Scorer references */
-    scores: ScorerReference[];
-    /** Optional metadata */
-    metadata?: Record<string, any>;
-    /** SHA256 fingerprint of the evaluation definition */
-    fingerprint?: string;
-    /** Evaluation options (including __schedule) */
-    options?: {
-        __schedule?: string;
-        [key: string]: any;
-    };
-}
-
-/**
- * Serializable representation of a Zod schema
- * For now, we store the schema as a string that can be re-parsed
- */
-declare interface SerializableSchema {
-    /** Type of schema (for future extensibility) */
-    type: "zod";
-    /** Schema definition (stringified or encoded) */
-    definition: string;
-}
-
-/**
- * Base fields for all serializable scorers
- */
-declare interface SerializableScorerBase {
-    /** Scorer name */
-    name: string;
-    /** URL-safe identifier */
-    slug?: string;
-    /** Human-readable description */
-    description?: string;
-    /** Serialized schema */
-    schema: SerializableSchema;
-    /** Optional metadata (e.g., thresholds, config) */
-    metadata?: Record<string, any>;
-    /** SHA256 fingerprint of the scorer definition */
-    fingerprint?: string;
-}
-
-/**
- * Serializable scorer definition created via createTypedScorer
- */
-export declare interface SerializableScorerDefinition extends SerializableScorerBase {
-    /** Type of scorer */
-    type: "prompt";
-    /** LLM prompt template */
-    promptTemplate: string;
-    /** Choice to score mapping */
-    choiceScores: Record<string, number>;
-    /** Model to use (default: gpt-4o) */
-    model?: string;
-    /** Use Chain of Thought */
-    useCoT?: boolean;
-    /** Temperature for LLM */
-    temperature?: number;
-}
+export declare type SerializableCustomScorer = z.infer<typeof SerializableCustomScorerSchema>;
+
+/**
+ * Custom scorer with handler schema
+ */
+declare const SerializableCustomScorerSchema: z.ZodObject<{
+    name: z.ZodString;
+    slug: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    schema: z.ZodObject<{
+        type: z.ZodLiteral<"zod">;
+        definition: z.ZodString;
+    }, z.core.$strip>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    fingerprint: z.ZodOptional<z.ZodString>;
+    type: z.ZodLiteral<"custom_scorer">;
+    handler: z.ZodObject<{
+        type: z.ZodLiteral<"custom_function">;
+        reference: z.ZodString;
+        bundle: z.ZodOptional<z.ZodObject<{
+            file: z.ZodString;
+            fingerprint: z.ZodString;
+        }, z.core.$strip>>;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+
+export declare type SerializableEvaluation = z.infer<typeof SerializableEvaluationSchema>;
+
+/**
+ * Serializable evaluation schema
+ */
+declare const SerializableEvaluationSchema: z.ZodObject<{
+    name: z.ZodString;
+    slug: z.ZodString;
+    data: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        type: z.ZodLiteral<"dataset">;
+        config: z.ZodObject<{
+            name: z.ZodString;
+            source: z.ZodObject<{
+                baseUrl: z.ZodString;
+                path: z.ZodString;
+                method: z.ZodEnum<{
+                    POST: "POST";
+                    GET: "GET";
+                }>;
+            }, z.core.$strip>;
+            options: z.ZodRecord<z.ZodString, z.ZodAny>;
+            group_name: z.ZodOptional<z.ZodString>;
+            args: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+        }, z.core.$strip>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"static">;
+        data: z.ZodArray<z.ZodAny>;
+        group_name: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>], "type">;
+    task: z.ZodOptional<z.ZodObject<{
+        type: z.ZodLiteral<"custom_function">;
+        reference: z.ZodString;
+        bundle: z.ZodOptional<z.ZodObject<{
+            file: z.ZodString;
+            fingerprint: z.ZodString;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    scores: z.ZodArray<z.ZodUnion<readonly [z.ZodDiscriminatedUnion<[z.ZodObject<{
+        type: z.ZodLiteral<"internal">;
+        name: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"custom_scorer">;
+        slug: z.ZodString;
+        fingerprint: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>], "type">, z.ZodDiscriminatedUnion<[z.ZodObject<{
+        name: z.ZodString;
+        slug: z.ZodOptional<z.ZodString>;
+        description: z.ZodOptional<z.ZodString>;
+        schema: z.ZodObject<{
+            type: z.ZodLiteral<"zod">;
+            definition: z.ZodString;
+        }, z.core.$strip>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+        fingerprint: z.ZodOptional<z.ZodString>;
+        type: z.ZodLiteral<"prompt">;
+        promptTemplate: z.ZodString;
+        choiceScores: z.ZodRecord<z.ZodString, z.ZodNumber>;
+        model: z.ZodOptional<z.ZodString>;
+        useCoT: z.ZodOptional<z.ZodBoolean>;
+        temperature: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{
+        name: z.ZodString;
+        slug: z.ZodOptional<z.ZodString>;
+        description: z.ZodOptional<z.ZodString>;
+        schema: z.ZodObject<{
+            type: z.ZodLiteral<"zod">;
+            definition: z.ZodString;
+        }, z.core.$strip>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+        fingerprint: z.ZodOptional<z.ZodString>;
+        type: z.ZodLiteral<"custom_scorer">;
+        handler: z.ZodObject<{
+            type: z.ZodLiteral<"custom_function">;
+            reference: z.ZodString;
+            bundle: z.ZodOptional<z.ZodObject<{
+                file: z.ZodString;
+                fingerprint: z.ZodString;
+            }, z.core.$strip>>;
+        }, z.core.$strip>;
+    }, z.core.$strip>], "type">]>>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    fingerprint: z.ZodOptional<z.ZodString>;
+    options: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+}, z.core.$strip>;
+
+export declare type SerializableScorerDefinition = z.infer<typeof SerializableScorerDefinitionSchema>;
+
+/**
+ * Prompt scorer definition schema
+ */
+declare const SerializableScorerDefinitionSchema: z.ZodObject<{
+    name: z.ZodString;
+    slug: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    schema: z.ZodObject<{
+        type: z.ZodLiteral<"zod">;
+        definition: z.ZodString;
+    }, z.core.$strip>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    fingerprint: z.ZodOptional<z.ZodString>;
+    type: z.ZodLiteral<"prompt">;
+    promptTemplate: z.ZodString;
+    choiceScores: z.ZodRecord<z.ZodString, z.ZodNumber>;
+    model: z.ZodOptional<z.ZodString>;
+    useCoT: z.ZodOptional<z.ZodBoolean>;
+    temperature: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
 
 /**
  * Creates a dataset definition for the sessions dataset
@@ -1222,48 +1216,26 @@ export declare interface SerializableScorerDefinition extends SerializableScorer
  * });
  * ```
  */
-export declare function sessions(options?: Partial<Omit<SessionsDatasetOptions, "baseUrl">>): DataSourceDefinition;
+export declare function sessions(options?: Partial<SessionsDatasetOptions>): DataSourceDefinition;
 
 /**
- * Options for fetching the sessions dataset
+ * TypeScript type inferred from Zod schema
  */
-export declare interface SessionsDatasetOptions extends HTTPDataSourceOptions {
-    /**
-     * Tenant ID to filter sessions
-     */
-    tenantId: string;
-    /**
-     * Service name to filter sessions
-     */
-    serviceName: string;
-    /**
-     * Environment to filter sessions
-     * @example "production", "staging", "development"
-     */
-    environment: string;
-    /**
-     * Optional group name for organizing datasets in the same namespace
-     * If not provided, defaults to "sessions"
-     * This is used to build the source_type namespace (e.g., "live.sessions")
-     * @example "user-sessions", "support-sessions"
-     */
-    group_name?: string;
-    /**
-     * Optional month filter (date string)
-     * @example "01/2026"
-     */
-    month?: string;
-    /**
-     * Optional period filter (date string)
-     * @example "01/01/2026"
-     */
-    startDate?: string;
-    /**
-     * Optional period filter (date string)
-     * @example "01/01/2026"
-     */
-    endDate?: string;
-}
+export declare type SessionsDatasetOptions = z.infer<typeof SessionsDatasetOptionsSchema>;
+
+/**
+ * Zod schema for sessions dataset options
+ */
+declare const SessionsDatasetOptionsSchema: z.ZodObject<{
+    tenantId: z.ZodString;
+    serviceName: z.ZodString;
+    environment: z.ZodString;
+    group_name: z.ZodOptional<z.ZodString>;
+    month: z.ZodOptional<z.ZodString>;
+    startDate: z.ZodOptional<z.ZodString>;
+    endDate: z.ZodOptional<z.ZodString>;
+    timeout: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
 
 /**
  * An interface that represents a span. A span represents a single operation
@@ -1489,15 +1461,6 @@ declare type TaskContext<Expected, Ctx extends BaseCtx = BaseCtx> = {
  */
 export declare type TaskFunction<Input, Output, Expected, Ctx extends BaseCtx = BaseCtx> = SyncTaskFunction<Input, Output, Expected, Ctx> | AsyncTaskFunction<Input, Output, Expected, Ctx>;
 
-/**
- * Reference to a task handler function
- */
-declare interface TaskReference {
-    type: "custom_function";
-    reference: string;
-    bundle?: BundleReference;
-}
-
 /**
  * Task reference definition (from definitions.json)
  */