opik 1.11.13 → 2.0.0

package/dist/index.d.cts

@@ -12681,22 +12681,50 @@ type ItemResult = {
     hasAssertions: boolean;
     runsPassed: number;
     runsTotal: number;
+    /** Configured runsPerItem from the execution policy. */
+    configuredRunsPerItem: number;
     passThreshold: number;
     testResults: EvaluationTestResult[];
 };
 /**
  * Result of a test suite run.
- */
-type TestSuiteResult = {
-    allItemsPassed: boolean;
-    itemsPassed: number;
-    itemsTotal: number;
-    passRate: number | undefined;
-    itemResults: Map<string, ItemResult>;
-    experimentId: string;
-    experimentName?: string;
-    experimentUrl?: string;
-};
+ *
+ * Contains pass/fail status for each item based on execution policy,
+ * as well as overall suite pass/fail status.
+ */
+declare class TestSuiteResult {
+    readonly allItemsPassed: boolean;
+    readonly itemsPassed: number;
+    readonly itemsTotal: number;
+    readonly passRate: number | undefined;
+    readonly itemResults: Map<string, ItemResult>;
+    readonly experimentId: string;
+    readonly experimentName?: string;
+    readonly experimentUrl?: string;
+    readonly suiteName?: string;
+    readonly totalTime?: number;
+    constructor(data: {
+        allItemsPassed: boolean;
+        itemsPassed: number;
+        itemsTotal: number;
+        passRate: number | undefined;
+        itemResults: Map<string, ItemResult>;
+        experimentId: string;
+        experimentName?: string;
+        experimentUrl?: string;
+        suiteName?: string;
+        totalTime?: number;
+    });
+    /**
+     * Convert the result to a structured report dictionary.
+     *
+     * The returned object mirrors the structure produced by the Python SDK's
+     * `to_report_dict()` method (with camelCase keys per TypeScript conventions).
+     */
+    toReportDict(): Record<string, unknown>;
+    /** Alias for {@link toReportDict}. */
+    toDict(): Record<string, unknown>;
+}
 
 /**
  * Builds a TestSuiteResult from an EvaluationResult and execution policies.
@@ -12711,7 +12739,10 @@ type TestSuiteResult = {
  * - allItemsPassed = itemsPassed === itemsTotal
  * - passRate = itemsPassed / itemsWithAssertions (undefined if none have assertions)
  */
-declare function buildSuiteResult(evalResult: EvaluationResult): TestSuiteResult;
+declare function buildSuiteResult(evalResult: EvaluationResult, options?: {
+    suiteName?: string;
+    totalTime?: number;
+}): TestSuiteResult;
 
 interface EvaluateTestSuiteOptions<T = Record<string, unknown>> {
     /** The dataset to evaluate against */
@@ -15948,4 +15979,4 @@ declare class ConfigMismatchError extends OpikError {
 
 declare function activateRunner(): void;
 
-export { AgentTaskCompletionJudge, AgentToolCorrectnessJudge, type AllProviderOptions, AnnotationQueuePublicScope as AnnotationQueueScope, AnswerRelevance, type AnthropicProviderOptions, BaseLLMJudgeMetric, BaseMetric, BaseSuiteEvaluator, ChatPrompt, ComplianceRiskJudge, type Config, ConfigMismatchError, ConfigNotFoundError, Contains, type CreateTestSuiteOptions, DEFAULT_EXECUTION_POLICY, Dataset, type DatasetPublic, DatasetVersion, DatasetVersionNotFoundError, type DatasetVersionPublic, DemographicBiasJudge, DialogueHelpfulnessJudge, type ErrorInfo, type EvaluateOptions, type EvaluatePromptOptions, type EvaluateTestSuiteOptions, type EvaluationError, type EvaluationResult, type EvaluationScoreResult, type EvaluationTask, type EvaluationTestCase, type EvaluationTestResult, ExactMatch, type ExecutionPolicy, type FeedbackScoreData, type FewShotExampleAnswerRelevanceNoContext, type FewShotExampleAnswerRelevanceWithContext, type FewShotExampleHallucination, type FewShotExampleModeration, type FilterExpression, GEval, GEvalPreset, GenderBiasJudge, type GoogleProviderOptions, Hallucination, IsJson, type ItemResult, LLMJudge, type LLMJudgeConfig, type LLMJudgeModelSettings, type LLMJudgeOptions, type LLMJudgeResponseFormat, ModelConfigurationError, ModelError, ModelGenerationError, Moderation, type OpenAIProviderOptions, OpikClient as Opik, type OpikAssistantMessage, OpikBaseModel, type OpikConfig, type OpikMessage, OpikQueryLanguage, SpanType as OpikSpanType, type OpikSystemMessage, type OpikToolMessage, type OpikUserMessage, type Param, PoliticalBiasJudge, Prompt, PromptType, PromptUncertaintyJudge, type ProviderOptionsForModel, QARelevanceJudge, RegexMatch, RegionalBiasJudge, type RegistryEntry, ReligiousBiasJudge, ResponseSchema, type RunTestsOptions, SYSTEM_PROMPT, type ScoringKeyMappingType, Span, SpanType, SummarizationCoherenceJudge, SummarizationConsistencyJudge, type SupportedModelId, TestSuite, type TestSuiteItem, type TestSuiteResult, ThreadsAnnotationQueue, Trace, TracesAnnotationQueue, USER_PROMPT_TEMPLATE, type UpdateTestSuiteItem, type UpdateTestSuiteOptions, Usefulness, VercelAIChatModel, activateRunner, agentConfigContext, buildSuiteResult, createModel, createModelFromInstance, deserializeEvaluators, detectProvider, disableLogger, evaluate, evaluatePrompt, evaluateTestSuite, flushAll, generateId, getTrackContext, logger, resolveEvaluators, resolveExecutionPolicy, resolveItemExecutionPolicy, resolveModel, runTests, serializeEvaluators, setLoggerLevel, track, validateEvaluators, validateExecutionPolicy };
+export { AgentTaskCompletionJudge, AgentToolCorrectnessJudge, type AllProviderOptions, AnnotationQueuePublicScope as AnnotationQueueScope, AnswerRelevance, type AnthropicProviderOptions, BaseLLMJudgeMetric, BaseMetric, BaseSuiteEvaluator, ChatPrompt, ComplianceRiskJudge, type Config, ConfigMismatchError, ConfigNotFoundError, Contains, type CreateTestSuiteOptions, DEFAULT_EXECUTION_POLICY, Dataset, type DatasetPublic, DatasetVersion, DatasetVersionNotFoundError, type DatasetVersionPublic, DemographicBiasJudge, DialogueHelpfulnessJudge, type ErrorInfo, type EvaluateOptions, type EvaluatePromptOptions, type EvaluateTestSuiteOptions, type EvaluationError, type EvaluationResult, type EvaluationScoreResult, type EvaluationTask, type EvaluationTestCase, type EvaluationTestResult, ExactMatch, type ExecutionPolicy, type FeedbackScoreData, type FewShotExampleAnswerRelevanceNoContext, type FewShotExampleAnswerRelevanceWithContext, type FewShotExampleHallucination, type FewShotExampleModeration, type FilterExpression, GEval, GEvalPreset, GenderBiasJudge, type GoogleProviderOptions, Hallucination, IsJson, type ItemResult, LLMJudge, type LLMJudgeConfig, type LLMJudgeModelSettings, type LLMJudgeOptions, type LLMJudgeResponseFormat, ModelConfigurationError, ModelError, ModelGenerationError, Moderation, type OpenAIProviderOptions, OpikClient as Opik, type OpikAssistantMessage, OpikBaseModel, type OpikConfig, type OpikMessage, OpikQueryLanguage, SpanType as OpikSpanType, type OpikSystemMessage, type OpikToolMessage, type OpikUserMessage, type Param, PoliticalBiasJudge, Prompt, PromptType, PromptUncertaintyJudge, type ProviderOptionsForModel, QARelevanceJudge, RegexMatch, RegionalBiasJudge, type RegistryEntry, ReligiousBiasJudge, ResponseSchema, type RunTestsOptions, SYSTEM_PROMPT, type ScoringKeyMappingType, Span, SpanType, SummarizationCoherenceJudge, SummarizationConsistencyJudge, type SupportedModelId, TestSuite, type TestSuiteItem, TestSuiteResult, ThreadsAnnotationQueue, Trace, TracesAnnotationQueue, USER_PROMPT_TEMPLATE, type UpdateTestSuiteItem, type UpdateTestSuiteOptions, Usefulness, VercelAIChatModel, activateRunner, agentConfigContext, buildSuiteResult, createModel, createModelFromInstance, deserializeEvaluators, detectProvider, disableLogger, evaluate, evaluatePrompt, evaluateTestSuite, flushAll, generateId, getTrackContext, logger, resolveEvaluators, resolveExecutionPolicy, resolveItemExecutionPolicy, resolveModel, runTests, serializeEvaluators, setLoggerLevel, track, validateEvaluators, validateExecutionPolicy };

package/dist/index.d.ts

@@ -12681,22 +12681,50 @@ type ItemResult = {
     hasAssertions: boolean;
     runsPassed: number;
     runsTotal: number;
+    /** Configured runsPerItem from the execution policy. */
+    configuredRunsPerItem: number;
     passThreshold: number;
     testResults: EvaluationTestResult[];
 };
 /**
  * Result of a test suite run.
- */
-type TestSuiteResult = {
-    allItemsPassed: boolean;
-    itemsPassed: number;
-    itemsTotal: number;
-    passRate: number | undefined;
-    itemResults: Map<string, ItemResult>;
-    experimentId: string;
-    experimentName?: string;
-    experimentUrl?: string;
-};
+ *
+ * Contains pass/fail status for each item based on execution policy,
+ * as well as overall suite pass/fail status.
+ */
+declare class TestSuiteResult {
+    readonly allItemsPassed: boolean;
+    readonly itemsPassed: number;
+    readonly itemsTotal: number;
+    readonly passRate: number | undefined;
+    readonly itemResults: Map<string, ItemResult>;
+    readonly experimentId: string;
+    readonly experimentName?: string;
+    readonly experimentUrl?: string;
+    readonly suiteName?: string;
+    readonly totalTime?: number;
+    constructor(data: {
+        allItemsPassed: boolean;
+        itemsPassed: number;
+        itemsTotal: number;
+        passRate: number | undefined;
+        itemResults: Map<string, ItemResult>;
+        experimentId: string;
+        experimentName?: string;
+        experimentUrl?: string;
+        suiteName?: string;
+        totalTime?: number;
+    });
+    /**
+     * Convert the result to a structured report dictionary.
+     *
+     * The returned object mirrors the structure produced by the Python SDK's
+     * `to_report_dict()` method (with camelCase keys per TypeScript conventions).
+     */
+    toReportDict(): Record<string, unknown>;
+    /** Alias for {@link toReportDict}. */
+    toDict(): Record<string, unknown>;
+}
 
 /**
  * Builds a TestSuiteResult from an EvaluationResult and execution policies.
@@ -12711,7 +12739,10 @@ type TestSuiteResult = {
  * - allItemsPassed = itemsPassed === itemsTotal
  * - passRate = itemsPassed / itemsWithAssertions (undefined if none have assertions)
  */
-declare function buildSuiteResult(evalResult: EvaluationResult): TestSuiteResult;
+declare function buildSuiteResult(evalResult: EvaluationResult, options?: {
+    suiteName?: string;
+    totalTime?: number;
+}): TestSuiteResult;
 
 interface EvaluateTestSuiteOptions<T = Record<string, unknown>> {
     /** The dataset to evaluate against */
@@ -15948,4 +15979,4 @@ declare class ConfigMismatchError extends OpikError {
 
 declare function activateRunner(): void;
 
-export { AgentTaskCompletionJudge, AgentToolCorrectnessJudge, type AllProviderOptions, AnnotationQueuePublicScope as AnnotationQueueScope, AnswerRelevance, type AnthropicProviderOptions, BaseLLMJudgeMetric, BaseMetric, BaseSuiteEvaluator, ChatPrompt, ComplianceRiskJudge, type Config, ConfigMismatchError, ConfigNotFoundError, Contains, type CreateTestSuiteOptions, DEFAULT_EXECUTION_POLICY, Dataset, type DatasetPublic, DatasetVersion, DatasetVersionNotFoundError, type DatasetVersionPublic, DemographicBiasJudge, DialogueHelpfulnessJudge, type ErrorInfo, type EvaluateOptions, type EvaluatePromptOptions, type EvaluateTestSuiteOptions, type EvaluationError, type EvaluationResult, type EvaluationScoreResult, type EvaluationTask, type EvaluationTestCase, type EvaluationTestResult, ExactMatch, type ExecutionPolicy, type FeedbackScoreData, type FewShotExampleAnswerRelevanceNoContext, type FewShotExampleAnswerRelevanceWithContext, type FewShotExampleHallucination, type FewShotExampleModeration, type FilterExpression, GEval, GEvalPreset, GenderBiasJudge, type GoogleProviderOptions, Hallucination, IsJson, type ItemResult, LLMJudge, type LLMJudgeConfig, type LLMJudgeModelSettings, type LLMJudgeOptions, type LLMJudgeResponseFormat, ModelConfigurationError, ModelError, ModelGenerationError, Moderation, type OpenAIProviderOptions, OpikClient as Opik, type OpikAssistantMessage, OpikBaseModel, type OpikConfig, type OpikMessage, OpikQueryLanguage, SpanType as OpikSpanType, type OpikSystemMessage, type OpikToolMessage, type OpikUserMessage, type Param, PoliticalBiasJudge, Prompt, PromptType, PromptUncertaintyJudge, type ProviderOptionsForModel, QARelevanceJudge, RegexMatch, RegionalBiasJudge, type RegistryEntry, ReligiousBiasJudge, ResponseSchema, type RunTestsOptions, SYSTEM_PROMPT, type ScoringKeyMappingType, Span, SpanType, SummarizationCoherenceJudge, SummarizationConsistencyJudge, type SupportedModelId, TestSuite, type TestSuiteItem, type TestSuiteResult, ThreadsAnnotationQueue, Trace, TracesAnnotationQueue, USER_PROMPT_TEMPLATE, type UpdateTestSuiteItem, type UpdateTestSuiteOptions, Usefulness, VercelAIChatModel, activateRunner, agentConfigContext, buildSuiteResult, createModel, createModelFromInstance, deserializeEvaluators, detectProvider, disableLogger, evaluate, evaluatePrompt, evaluateTestSuite, flushAll, generateId, getTrackContext, logger, resolveEvaluators, resolveExecutionPolicy, resolveItemExecutionPolicy, resolveModel, runTests, serializeEvaluators, setLoggerLevel, track, validateEvaluators, validateExecutionPolicy };
+export { AgentTaskCompletionJudge, AgentToolCorrectnessJudge, type AllProviderOptions, AnnotationQueuePublicScope as AnnotationQueueScope, AnswerRelevance, type AnthropicProviderOptions, BaseLLMJudgeMetric, BaseMetric, BaseSuiteEvaluator, ChatPrompt, ComplianceRiskJudge, type Config, ConfigMismatchError, ConfigNotFoundError, Contains, type CreateTestSuiteOptions, DEFAULT_EXECUTION_POLICY, Dataset, type DatasetPublic, DatasetVersion, DatasetVersionNotFoundError, type DatasetVersionPublic, DemographicBiasJudge, DialogueHelpfulnessJudge, type ErrorInfo, type EvaluateOptions, type EvaluatePromptOptions, type EvaluateTestSuiteOptions, type EvaluationError, type EvaluationResult, type EvaluationScoreResult, type EvaluationTask, type EvaluationTestCase, type EvaluationTestResult, ExactMatch, type ExecutionPolicy, type FeedbackScoreData, type FewShotExampleAnswerRelevanceNoContext, type FewShotExampleAnswerRelevanceWithContext, type FewShotExampleHallucination, type FewShotExampleModeration, type FilterExpression, GEval, GEvalPreset, GenderBiasJudge, type GoogleProviderOptions, Hallucination, IsJson, type ItemResult, LLMJudge, type LLMJudgeConfig, type LLMJudgeModelSettings, type LLMJudgeOptions, type LLMJudgeResponseFormat, ModelConfigurationError, ModelError, ModelGenerationError, Moderation, type OpenAIProviderOptions, OpikClient as Opik, type OpikAssistantMessage, OpikBaseModel, type OpikConfig, type OpikMessage, OpikQueryLanguage, SpanType as OpikSpanType, type OpikSystemMessage, type OpikToolMessage, type OpikUserMessage, type Param, PoliticalBiasJudge, Prompt, PromptType, PromptUncertaintyJudge, type ProviderOptionsForModel, QARelevanceJudge, RegexMatch, RegionalBiasJudge, type RegistryEntry, ReligiousBiasJudge, ResponseSchema, type RunTestsOptions, SYSTEM_PROMPT, type ScoringKeyMappingType, Span, SpanType, SummarizationCoherenceJudge, SummarizationConsistencyJudge, type SupportedModelId, TestSuite, type TestSuiteItem, TestSuiteResult, ThreadsAnnotationQueue, Trace, TracesAnnotationQueue, USER_PROMPT_TEMPLATE, type UpdateTestSuiteItem, type UpdateTestSuiteOptions, Usefulness, VercelAIChatModel, activateRunner, agentConfigContext, buildSuiteResult, createModel, createModelFromInstance, deserializeEvaluators, detectProvider, disableLogger, evaluate, evaluatePrompt, evaluateTestSuite, flushAll, generateId, getTrackContext, logger, resolveEvaluators, resolveExecutionPolicy, resolveItemExecutionPolicy, resolveModel, runTests, serializeEvaluators, setLoggerLevel, track, validateEvaluators, validateExecutionPolicy };

package/dist/index.js

@@ -1 +1 @@
-export{na as AgentTaskCompletionJudge,ma as AgentToolCorrectnessJudge,aa as AnswerRelevance,Y as BaseLLMJudgeMetric,x as BaseMetric,y as BaseSuiteEvaluator,m as ChatPrompt,pa as ComplianceRiskJudge,i as ConfigMismatchError,h as ConfigNotFoundError,V as Contains,v as DEFAULT_EXECUTION_POLICY,j as Dataset,f as DatasetVersion,g as DatasetVersionNotFoundError,ha as DemographicBiasJudge,fa as DialogueHelpfulnessJudge,U as ExactMatch,ba as GEval,ca as GEvalPreset,ja as GenderBiasJudge,$ as Hallucination,X as IsJson,L as LLMJudge,C as ModelConfigurationError,A as ModelError,B as ModelGenerationError,Z as Moderation,ua as Opik,z as OpikBaseModel,n as OpikQueryLanguage,d as OpikSpanType,ia as PoliticalBiasJudge,l as Prompt,k as PromptType,oa as PromptUncertaintyJudge,ga as QARelevanceJudge,W as RegexMatch,la as RegionalBiasJudge,ka as ReligiousBiasJudge,K as ResponseSchema,I as SYSTEM_PROMPT,ea as SummarizationCoherenceJudge,da as SummarizationConsistencyJudge,ta as TestSuite,p as ThreadsAnnotationQueue,o as TracesAnnotationQueue,J as USER_PROMPT_TEMPLATE,_ as Usefulness,E as VercelAIChatModel,s as activateRunner,q as agentConfigContext,w as buildSuiteResult,F as createModel,G as createModelFromInstance,N as deserializeEvaluators,D as detectProvider,c as disableLogger,S as evaluate,T as evaluatePrompt,Q as evaluateTestSuite,r as flushAll,e as generateId,t as getTrackContext,a as logger,qa as resolveEvaluators,O as resolveExecutionPolicy,P as resolveItemExecutionPolicy,H as resolveModel,R as runTests,M as serializeEvaluators,b as setLoggerLevel,u as track,ra as validateEvaluators,sa as validateExecutionPolicy,va as z}from'./chunk-MHMIE52N.js';
+export{oa as AgentTaskCompletionJudge,na as AgentToolCorrectnessJudge,ba as AnswerRelevance,Z as BaseLLMJudgeMetric,y as BaseMetric,z as BaseSuiteEvaluator,m as ChatPrompt,qa as ComplianceRiskJudge,i as ConfigMismatchError,h as ConfigNotFoundError,W as Contains,v as DEFAULT_EXECUTION_POLICY,j as Dataset,f as DatasetVersion,g as DatasetVersionNotFoundError,ia as DemographicBiasJudge,ga as DialogueHelpfulnessJudge,V as ExactMatch,ca as GEval,da as GEvalPreset,ka as GenderBiasJudge,aa as Hallucination,Y as IsJson,M as LLMJudge,D as ModelConfigurationError,B as ModelError,C as ModelGenerationError,_ as Moderation,va as Opik,A as OpikBaseModel,n as OpikQueryLanguage,d as OpikSpanType,ja as PoliticalBiasJudge,l as Prompt,k as PromptType,pa as PromptUncertaintyJudge,ha as QARelevanceJudge,X as RegexMatch,ma as RegionalBiasJudge,la as ReligiousBiasJudge,L as ResponseSchema,J as SYSTEM_PROMPT,fa as SummarizationCoherenceJudge,ea as SummarizationConsistencyJudge,ua as TestSuite,w as TestSuiteResult,p as ThreadsAnnotationQueue,o as TracesAnnotationQueue,K as USER_PROMPT_TEMPLATE,$ as Usefulness,F as VercelAIChatModel,s as activateRunner,q as agentConfigContext,x as buildSuiteResult,G as createModel,H as createModelFromInstance,O as deserializeEvaluators,E as detectProvider,c as disableLogger,T as evaluate,U as evaluatePrompt,R as evaluateTestSuite,r as flushAll,e as generateId,t as getTrackContext,a as logger,ra as resolveEvaluators,P as resolveExecutionPolicy,Q as resolveItemExecutionPolicy,I as resolveModel,S as runTests,N as serializeEvaluators,b as setLoggerLevel,u as track,sa as validateEvaluators,ta as validateExecutionPolicy,wa as z}from'./chunk-2AOEXUQ4.js';

package/dist/suite-DE3AOLJG.js

@@ -0,0 +1 @@
+export{v as DEFAULT_EXECUTION_POLICY,ua as TestSuite,w as TestSuiteResult,x as buildSuiteResult,O as deserializeEvaluators,R as evaluateTestSuite,P as resolveExecutionPolicy,Q as resolveItemExecutionPolicy,S as runTests,N as serializeEvaluators}from'./chunk-2AOEXUQ4.js';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "opik",
   "description": "Opik TypeScript and JavaScript SDK",
-  "version": "1.11.13",
+  "version": "2.0.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/comet-ml/opik.git",

package/dist/suite-6XFUV2Y7.js

@@ -1 +0,0 @@
-export{v as DEFAULT_EXECUTION_POLICY,ta as TestSuite,w as buildSuiteResult,N as deserializeEvaluators,Q as evaluateTestSuite,O as resolveExecutionPolicy,P as resolveItemExecutionPolicy,R as runTests,M as serializeEvaluators}from'./chunk-MHMIE52N.js';