opik 1.11.0 → 1.11.2

package/dist/index.d.cts

@@ -1121,6 +1121,32 @@ interface ExperimentBatchUpdate {
     mergeTags?: boolean;
 }
 
+/**
+ * @example
+ *     {
+ *         datasetName: "dataset_name",
+ *         prompts: [{
+ *                 model: "model",
+ *                 messages: [{
+ *                         role: "role",
+ *                         content: {
+ *                             "key": "value"
+ *                         }
+ *                     }]
+ *             }],
+ *         datasetId: "dataset_id"
+ *     }
+ */
+interface ExperimentExecutionRequest {
+    datasetName: string;
+    datasetVersionId?: string;
+    prompts: PromptVariant[];
+    projectName?: string;
+    datasetId: string;
+    versionHash?: string;
+    promptVersions?: PromptVersionLink[];
+}
+
 /**
  * @example
  *     {
@@ -1918,6 +1944,7 @@ type PromptWriteTemplateStructure = (typeof PromptWriteTemplateStructure)[keyof
 declare const PromptWriteType: {
     readonly Mustache: "mustache";
     readonly Jinja2: "jinja2";
+    readonly Python: "python";
 };
 type PromptWriteType = (typeof PromptWriteType)[keyof typeof PromptWriteType];
 
@@ -2398,6 +2425,8 @@ interface SpanSearchStreamRequestPublic {
     lastRetrievedId?: string;
     /** Truncate image included in either input, output or metadata */
     truncate?: boolean;
+    /** Fields to exclude from the response */
+    exclude?: SpanSearchStreamRequestPublicExcludeItem[];
     /** Filter spans created from this time (ISO-8601 format). */
     fromTime?: Date;
     /** Filter spans created up to this time (ISO-8601 format). If not provided, defaults to current time. Must be after 'from_time'. */
@@ -2452,6 +2481,33 @@ declare const GetSpansByProjectRequestType: {
 };
 type GetSpansByProjectRequestType = (typeof GetSpansByProjectRequestType)[keyof typeof GetSpansByProjectRequestType];
 
+/** Fields to exclude from the response */
+declare const SpanSearchStreamRequestPublicExcludeItem: {
+    readonly Name: "name";
+    readonly Type: "type";
+    readonly StartTime: "start_time";
+    readonly EndTime: "end_time";
+    readonly Input: "input";
+    readonly Output: "output";
+    readonly Metadata: "metadata";
+    readonly Model: "model";
+    readonly Provider: "provider";
+    readonly Tags: "tags";
+    readonly Usage: "usage";
+    readonly ErrorInfo: "error_info";
+    readonly CreatedAt: "created_at";
+    readonly CreatedBy: "created_by";
+    readonly LastUpdatedBy: "last_updated_by";
+    readonly FeedbackScores: "feedback_scores";
+    readonly Comments: "comments";
+    readonly TotalEstimatedCost: "total_estimated_cost";
+    readonly TotalEstimatedCostVersion: "total_estimated_cost_version";
+    readonly Duration: "duration";
+    readonly Ttft: "ttft";
+    readonly Source: "source";
+};
+type SpanSearchStreamRequestPublicExcludeItem = (typeof SpanSearchStreamRequestPublicExcludeItem)[keyof typeof SpanSearchStreamRequestPublicExcludeItem];
+
 declare const SpanSearchStreamRequestPublicType: {
     readonly General: "general";
     readonly Tool: "tool";
@@ -4409,6 +4465,11 @@ interface ExecutionPolicyWrite {
     passThreshold?: number;
 }
 
+interface ExperimentExecutionResponse {
+    experiments?: ExperimentInfo[];
+    totalItems?: number;
+}
+
 interface ExperimentGroupAggregationsResponse {
     content?: Record<string, GroupContentWithAggregations>;
 }
@@ -4418,6 +4479,11 @@ interface ExperimentGroupResponse {
     details?: GroupDetails;
 }
 
+interface ExperimentInfo {
+    experimentId?: string;
+    promptIndex?: number;
+}
+
 interface ExperimentItem {
     id?: string;
     experimentId: string;
@@ -5365,7 +5431,10 @@ interface ManualEvaluationResponse {
     rulesApplied?: number;
 }
 
-type Message = Record<string, unknown>;
+interface Message {
+    role: string;
+    content: JsonNode;
+}
 
 interface MultipartUploadPart {
     eTag: string;
@@ -5764,6 +5833,13 @@ interface PromptTokensDetails {
     cachedTokens?: number;
 }
 
+interface PromptVariant {
+    model: string;
+    messages: Message[];
+    configs?: Record<string, JsonNode>;
+    promptVersions?: PromptVersionLink[];
+}
+
 interface PromptVersionDetail {
     /** version unique identifier, generated if absent */
     id?: string;
@@ -5790,9 +5866,17 @@ type PromptVersionDetailTemplateStructure = (typeof PromptVersionDetailTemplateS
 declare const PromptVersionDetailType: {
     readonly Mustache: "mustache";
     readonly Jinja2: "jinja2";
+    readonly Python: "python";
 };
 type PromptVersionDetailType = (typeof PromptVersionDetailType)[keyof typeof PromptVersionDetailType];
 
+interface PromptVersionLink {
+    id: string;
+    commit?: string;
+    promptId?: string;
+    promptName?: string;
+}
+
 interface PromptVersionLinkPublic {
     promptVersionId?: string;
     commit?: string;
@@ -5837,6 +5921,7 @@ type PromptVersionPublicTemplateStructure = (typeof PromptVersionPublicTemplateS
 declare const PromptVersionPublicType: {
     readonly Mustache: "mustache";
     readonly Jinja2: "jinja2";
+    readonly Python: "python";
 };
 type PromptVersionPublicType = (typeof PromptVersionPublicType)[keyof typeof PromptVersionPublicType];
 
@@ -8623,6 +8708,29 @@ declare class ExperimentsClient {
      */
     deleteExperimentsById(request: DeleteIdsHolder, requestOptions?: ExperimentsClient.RequestOptions): HttpResponsePromise<void>;
     private __deleteExperimentsById;
+    /**
+     * Creates experiments for each prompt variant and asynchronously processes all dataset items
+     *
+     * @param {OpikApi.ExperimentExecutionRequest} request
+     * @param {ExperimentsClient.RequestOptions} requestOptions - Request-specific configuration.
+     *
+     * @example
+     *     await client.experiments.executeExperiment({
+     *         datasetName: "dataset_name",
+     *         prompts: [{
+     *                 model: "model",
+     *                 messages: [{
+     *                         role: "role",
+     *                         content: {
+     *                             "key": "value"
+     *                         }
+     *                     }]
+     *             }],
+     *         datasetId: "dataset_id"
+     *     })
+     */
+    executeExperiment(request: ExperimentExecutionRequest, requestOptions?: ExperimentsClient.RequestOptions): HttpResponsePromise<ExperimentExecutionResponse>;
+    private __executeExperiment;
     /**
      * Record experiment items in bulk with traces, spans, and feedback scores. Maximum request size is 4MB.
      *
@@ -11495,6 +11603,8 @@ declare const PromptType: {
     readonly MUSTACHE: "mustache";
     /** Jinja2 template syntax with {% %} blocks and {{ }} variables */
     readonly JINJA2: "jinja2";
+    /** Python template syntax with {variable} placeholders */
+    readonly PYTHON: "python";
 };
 type PromptType = (typeof PromptType)[keyof typeof PromptType];
 /**
@@ -11669,8 +11779,8 @@ declare class PromptVersion {
  * Base data interface for all prompt types
  */
 interface BasePromptData {
-    promptId: string;
-    versionId: string;
+    promptId?: string;
+    versionId?: string;
     name: string;
     commit?: string;
     metadata?: JsonNode;
@@ -11679,18 +11789,21 @@ interface BasePromptData {
     description?: string;
     tags?: string[];
     templateStructure?: PromptTemplateStructure;
+    synced?: boolean;
 }
 /**
  * Abstract base class for all prompt types.
  * Provides common functionality for versioning, property updates, and deletion.
  */
 declare abstract class BasePrompt {
-    readonly id: string;
-    readonly versionId: string;
+    readonly id: string | undefined;
+    readonly versionId: string | undefined;
     readonly commit: string | undefined;
     readonly type: PromptType;
     readonly changeDescription: string | undefined;
     readonly templateStructure: PromptTemplateStructure;
+    /** Whether the prompt has been successfully synced with the backend. */
+    readonly synced: boolean;
     protected _name: string;
     protected _description: string | undefined;
     protected _tags: string[];
@@ -11751,6 +11864,11 @@ declare abstract class BasePrompt {
      * @returns Promise resolving to the API response or null if not found
      */
     protected retrieveVersionByCommit(commit: string): Promise<PromptVersionDetail | null>;
+    /**
+     * Throws an error if the prompt has not been synced with the backend.
+     * Call syncWithBackend() first to sync.
+     */
+    protected requireSynced(operation: string): void;
     /**
      * Get a specific version by commit hash.
      * Returns a new instance of the appropriate prompt type.
@@ -11766,6 +11884,15 @@ declare abstract class BasePrompt {
      * @returns Promise resolving to a new prompt instance with the restored version
      */
     abstract useVersion(version: PromptVersion): Promise<BasePrompt>;
+    /**
+     * Synchronize the prompt with the backend.
+     *
+     * Creates or updates the prompt on the Opik server. If the sync fails,
+     * a warning is logged and the prompt continues to work locally.
+     *
+     * @returns Promise resolving to a new synced instance, or the same instance if sync fails
+     */
+    abstract syncWithBackend(): Promise<BasePrompt>;
 }
 
 interface PromptData extends BasePromptData {
@@ -11871,6 +11998,15 @@ declare class Prompt extends BasePrompt {
      * }
      * ```
      */
+    /**
+     * Synchronize the prompt with the backend.
+     *
+     * Creates or updates the prompt on the Opik server. If the sync fails,
+     * a warning is logged and the same (unsynced) instance is returned.
+     *
+     * @returns Promise resolving to a new synced Prompt instance, or this instance if sync fails
+     */
+    syncWithBackend(): Promise<Prompt>;
     getVersion(commit: string): Promise<Prompt | null>;
 }
 
@@ -12563,6 +12699,15 @@ declare class ChatPrompt extends BasePrompt {
      * }
      * ```
      */
+    /**
+     * Synchronize the chat prompt with the backend.
+     *
+     * Creates or updates the chat prompt on the Opik server. If the sync fails,
+     * a warning is logged and the same (unsynced) instance is returned.
+     *
+     * @returns Promise resolving to a new synced ChatPrompt instance, or this instance if sync fails
+     */
+    syncWithBackend(): Promise<ChatPrompt>;
     getVersion(commit: string): Promise<ChatPrompt | null>;
 }
 
@@ -13188,6 +13333,7 @@ declare class OpikClient {
         filterString?: string;
         maxResults?: number;
         truncate?: boolean;
+        exclude?: string[];
         waitForAtLeast?: number;
         waitForTimeout?: number;
     }) => Promise<SpanPublic[]>;

package/dist/index.d.ts

@@ -1121,6 +1121,32 @@ interface ExperimentBatchUpdate {
     mergeTags?: boolean;
 }
 
+/**
+ * @example
+ *     {
+ *         datasetName: "dataset_name",
+ *         prompts: [{
+ *                 model: "model",
+ *                 messages: [{
+ *                         role: "role",
+ *                         content: {
+ *                             "key": "value"
+ *                         }
+ *                     }]
+ *             }],
+ *         datasetId: "dataset_id"
+ *     }
+ */
+interface ExperimentExecutionRequest {
+    datasetName: string;
+    datasetVersionId?: string;
+    prompts: PromptVariant[];
+    projectName?: string;
+    datasetId: string;
+    versionHash?: string;
+    promptVersions?: PromptVersionLink[];
+}
+
 /**
  * @example
  *     {
@@ -1918,6 +1944,7 @@ type PromptWriteTemplateStructure = (typeof PromptWriteTemplateStructure)[keyof
 declare const PromptWriteType: {
     readonly Mustache: "mustache";
     readonly Jinja2: "jinja2";
+    readonly Python: "python";
 };
 type PromptWriteType = (typeof PromptWriteType)[keyof typeof PromptWriteType];
 
@@ -2398,6 +2425,8 @@ interface SpanSearchStreamRequestPublic {
     lastRetrievedId?: string;
     /** Truncate image included in either input, output or metadata */
     truncate?: boolean;
+    /** Fields to exclude from the response */
+    exclude?: SpanSearchStreamRequestPublicExcludeItem[];
     /** Filter spans created from this time (ISO-8601 format). */
     fromTime?: Date;
     /** Filter spans created up to this time (ISO-8601 format). If not provided, defaults to current time. Must be after 'from_time'. */
@@ -2452,6 +2481,33 @@ declare const GetSpansByProjectRequestType: {
 };
 type GetSpansByProjectRequestType = (typeof GetSpansByProjectRequestType)[keyof typeof GetSpansByProjectRequestType];
 
+/** Fields to exclude from the response */
+declare const SpanSearchStreamRequestPublicExcludeItem: {
+    readonly Name: "name";
+    readonly Type: "type";
+    readonly StartTime: "start_time";
+    readonly EndTime: "end_time";
+    readonly Input: "input";
+    readonly Output: "output";
+    readonly Metadata: "metadata";
+    readonly Model: "model";
+    readonly Provider: "provider";
+    readonly Tags: "tags";
+    readonly Usage: "usage";
+    readonly ErrorInfo: "error_info";
+    readonly CreatedAt: "created_at";
+    readonly CreatedBy: "created_by";
+    readonly LastUpdatedBy: "last_updated_by";
+    readonly FeedbackScores: "feedback_scores";
+    readonly Comments: "comments";
+    readonly TotalEstimatedCost: "total_estimated_cost";
+    readonly TotalEstimatedCostVersion: "total_estimated_cost_version";
+    readonly Duration: "duration";
+    readonly Ttft: "ttft";
+    readonly Source: "source";
+};
+type SpanSearchStreamRequestPublicExcludeItem = (typeof SpanSearchStreamRequestPublicExcludeItem)[keyof typeof SpanSearchStreamRequestPublicExcludeItem];
+
 declare const SpanSearchStreamRequestPublicType: {
     readonly General: "general";
     readonly Tool: "tool";
@@ -4409,6 +4465,11 @@ interface ExecutionPolicyWrite {
     passThreshold?: number;
 }
 
+interface ExperimentExecutionResponse {
+    experiments?: ExperimentInfo[];
+    totalItems?: number;
+}
+
 interface ExperimentGroupAggregationsResponse {
     content?: Record<string, GroupContentWithAggregations>;
 }
@@ -4418,6 +4479,11 @@ interface ExperimentGroupResponse {
     details?: GroupDetails;
 }
 
+interface ExperimentInfo {
+    experimentId?: string;
+    promptIndex?: number;
+}
+
 interface ExperimentItem {
     id?: string;
     experimentId: string;
@@ -5365,7 +5431,10 @@ interface ManualEvaluationResponse {
     rulesApplied?: number;
 }
 
-type Message = Record<string, unknown>;
+interface Message {
+    role: string;
+    content: JsonNode;
+}
 
 interface MultipartUploadPart {
     eTag: string;
@@ -5764,6 +5833,13 @@ interface PromptTokensDetails {
     cachedTokens?: number;
 }
 
+interface PromptVariant {
+    model: string;
+    messages: Message[];
+    configs?: Record<string, JsonNode>;
+    promptVersions?: PromptVersionLink[];
+}
+
 interface PromptVersionDetail {
     /** version unique identifier, generated if absent */
     id?: string;
@@ -5790,9 +5866,17 @@ type PromptVersionDetailTemplateStructure = (typeof PromptVersionDetailTemplateS
 declare const PromptVersionDetailType: {
     readonly Mustache: "mustache";
     readonly Jinja2: "jinja2";
+    readonly Python: "python";
 };
 type PromptVersionDetailType = (typeof PromptVersionDetailType)[keyof typeof PromptVersionDetailType];
 
+interface PromptVersionLink {
+    id: string;
+    commit?: string;
+    promptId?: string;
+    promptName?: string;
+}
+
 interface PromptVersionLinkPublic {
     promptVersionId?: string;
     commit?: string;
@@ -5837,6 +5921,7 @@ type PromptVersionPublicTemplateStructure = (typeof PromptVersionPublicTemplateS
 declare const PromptVersionPublicType: {
     readonly Mustache: "mustache";
     readonly Jinja2: "jinja2";
+    readonly Python: "python";
 };
 type PromptVersionPublicType = (typeof PromptVersionPublicType)[keyof typeof PromptVersionPublicType];
 
@@ -8623,6 +8708,29 @@ declare class ExperimentsClient {
      */
     deleteExperimentsById(request: DeleteIdsHolder, requestOptions?: ExperimentsClient.RequestOptions): HttpResponsePromise<void>;
     private __deleteExperimentsById;
+    /**
+     * Creates experiments for each prompt variant and asynchronously processes all dataset items
+     *
+     * @param {OpikApi.ExperimentExecutionRequest} request
+     * @param {ExperimentsClient.RequestOptions} requestOptions - Request-specific configuration.
+     *
+     * @example
+     *     await client.experiments.executeExperiment({
+     *         datasetName: "dataset_name",
+     *         prompts: [{
+     *                 model: "model",
+     *                 messages: [{
+     *                         role: "role",
+     *                         content: {
+     *                             "key": "value"
+     *                         }
+     *                     }]
+     *             }],
+     *         datasetId: "dataset_id"
+     *     })
+     */
+    executeExperiment(request: ExperimentExecutionRequest, requestOptions?: ExperimentsClient.RequestOptions): HttpResponsePromise<ExperimentExecutionResponse>;
+    private __executeExperiment;
     /**
      * Record experiment items in bulk with traces, spans, and feedback scores. Maximum request size is 4MB.
      *
@@ -11495,6 +11603,8 @@ declare const PromptType: {
     readonly MUSTACHE: "mustache";
     /** Jinja2 template syntax with {% %} blocks and {{ }} variables */
     readonly JINJA2: "jinja2";
+    /** Python template syntax with {variable} placeholders */
+    readonly PYTHON: "python";
 };
 type PromptType = (typeof PromptType)[keyof typeof PromptType];
 /**
@@ -11669,8 +11779,8 @@ declare class PromptVersion {
  * Base data interface for all prompt types
  */
 interface BasePromptData {
-    promptId: string;
-    versionId: string;
+    promptId?: string;
+    versionId?: string;
     name: string;
     commit?: string;
     metadata?: JsonNode;
@@ -11679,18 +11789,21 @@ interface BasePromptData {
     description?: string;
     tags?: string[];
     templateStructure?: PromptTemplateStructure;
+    synced?: boolean;
 }
 /**
  * Abstract base class for all prompt types.
  * Provides common functionality for versioning, property updates, and deletion.
  */
 declare abstract class BasePrompt {
-    readonly id: string;
-    readonly versionId: string;
+    readonly id: string | undefined;
+    readonly versionId: string | undefined;
     readonly commit: string | undefined;
     readonly type: PromptType;
     readonly changeDescription: string | undefined;
     readonly templateStructure: PromptTemplateStructure;
+    /** Whether the prompt has been successfully synced with the backend. */
+    readonly synced: boolean;
     protected _name: string;
     protected _description: string | undefined;
     protected _tags: string[];
@@ -11751,6 +11864,11 @@ declare abstract class BasePrompt {
      * @returns Promise resolving to the API response or null if not found
      */
     protected retrieveVersionByCommit(commit: string): Promise<PromptVersionDetail | null>;
+    /**
+     * Throws an error if the prompt has not been synced with the backend.
+     * Call syncWithBackend() first to sync.
+     */
+    protected requireSynced(operation: string): void;
     /**
      * Get a specific version by commit hash.
      * Returns a new instance of the appropriate prompt type.
@@ -11766,6 +11884,15 @@ declare abstract class BasePrompt {
      * @returns Promise resolving to a new prompt instance with the restored version
      */
     abstract useVersion(version: PromptVersion): Promise<BasePrompt>;
+    /**
+     * Synchronize the prompt with the backend.
+     *
+     * Creates or updates the prompt on the Opik server. If the sync fails,
+     * a warning is logged and the prompt continues to work locally.
+     *
+     * @returns Promise resolving to a new synced instance, or the same instance if sync fails
+     */
+    abstract syncWithBackend(): Promise<BasePrompt>;
 }
 
 interface PromptData extends BasePromptData {
@@ -11871,6 +11998,15 @@ declare class Prompt extends BasePrompt {
      * }
      * ```
      */
+    /**
+     * Synchronize the prompt with the backend.
+     *
+     * Creates or updates the prompt on the Opik server. If the sync fails,
+     * a warning is logged and the same (unsynced) instance is returned.
+     *
+     * @returns Promise resolving to a new synced Prompt instance, or this instance if sync fails
+     */
+    syncWithBackend(): Promise<Prompt>;
     getVersion(commit: string): Promise<Prompt | null>;
 }
 
@@ -12563,6 +12699,15 @@ declare class ChatPrompt extends BasePrompt {
      * }
      * ```
      */
+    /**
+     * Synchronize the chat prompt with the backend.
+     *
+     * Creates or updates the chat prompt on the Opik server. If the sync fails,
+     * a warning is logged and the same (unsynced) instance is returned.
+     *
+     * @returns Promise resolving to a new synced ChatPrompt instance, or this instance if sync fails
+     */
+    syncWithBackend(): Promise<ChatPrompt>;
     getVersion(commit: string): Promise<ChatPrompt | null>;
 }
 
@@ -13188,6 +13333,7 @@ declare class OpikClient {
         filterString?: string;
         maxResults?: number;
         truncate?: boolean;
+        exclude?: string[];
         waitForAtLeast?: number;
         waitForTimeout?: number;
     }) => Promise<SpanPublic[]>;