opik 2.0.18 → 2.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-2HP64V53.js +404 -0
- package/dist/index.cjs +36 -36
- package/dist/index.d.cts +57 -9
- package/dist/index.d.ts +57 -9
- package/dist/index.js +1 -1
- package/dist/{suite-HGQSCN74.js → suite-PNRRBE3L.js} +1 -1
- package/package.json +1 -1
- package/dist/chunk-62D43JIB.js +0 -404
package/dist/index.d.cts
CHANGED
|
@@ -12257,6 +12257,15 @@ declare class Trace {
|
|
|
12257
12257
|
update: (updates: TraceUpdateData) => this;
|
|
12258
12258
|
}
|
|
12259
12259
|
|
|
12260
|
+
interface OpikApiClientTempOptions extends OpikApiClient.Options {
|
|
12261
|
+
requestOptions?: RequestOptions;
|
|
12262
|
+
}
|
|
12263
|
+
declare class OpikApiClientTemp extends OpikApiClient {
|
|
12264
|
+
requestOptions: RequestOptions;
|
|
12265
|
+
constructor(options?: OpikApiClientTempOptions);
|
|
12266
|
+
setHeaders: (headers: Record<string, string>) => void;
|
|
12267
|
+
}
|
|
12268
|
+
|
|
12260
12269
|
interface BatchQueueOptions {
|
|
12261
12270
|
delay?: number;
|
|
12262
12271
|
enableCreateBatch?: boolean;
|
|
@@ -12285,13 +12294,24 @@ declare abstract class BatchQueue<EntityData = object, EntityId = string> {
|
|
|
12285
12294
|
flush: () => Promise<void>;
|
|
12286
12295
|
}
|
|
12287
12296
|
|
|
12288
|
-
|
|
12289
|
-
|
|
12290
|
-
|
|
12291
|
-
|
|
12292
|
-
|
|
12293
|
-
|
|
12294
|
-
|
|
12297
|
+
type AssertionResultId = {
|
|
12298
|
+
entityId: string;
|
|
12299
|
+
name: string;
|
|
12300
|
+
};
|
|
12301
|
+
declare class AssertionResultsBatchQueue extends BatchQueue<AssertionResultBatchItem, AssertionResultId> {
|
|
12302
|
+
private readonly api;
|
|
12303
|
+
private readonly entityType;
|
|
12304
|
+
private useLegacyFallback;
|
|
12305
|
+
constructor(api: OpikApiClientTemp, delay?: number, entityType?: AssertionResultBatchEntityType);
|
|
12306
|
+
protected getId(entity: AssertionResultBatchItem): {
|
|
12307
|
+
entityId: string;
|
|
12308
|
+
name: string;
|
|
12309
|
+
};
|
|
12310
|
+
protected createEntities(assertionResults: AssertionResultBatchItem[]): Promise<void>;
|
|
12311
|
+
private writeViaLegacyFeedbackScores;
|
|
12312
|
+
protected getEntity(): Promise<AssertionResultBatchItem | undefined>;
|
|
12313
|
+
protected updateEntity(): Promise<void>;
|
|
12314
|
+
protected deleteEntities(): Promise<void>;
|
|
12295
12315
|
}
|
|
12296
12316
|
|
|
12297
12317
|
type SpanUpdate = Partial<SavedSpan> & {
|
|
@@ -12726,7 +12746,7 @@ type EvaluationScoreResult = {
|
|
|
12726
12746
|
reason?: string;
|
|
12727
12747
|
/** Whether the scoring failed */
|
|
12728
12748
|
scoringFailed?: boolean;
|
|
12729
|
-
/** Optional category name for grouping scores
|
|
12749
|
+
/** Optional category name for grouping scores */
|
|
12730
12750
|
categoryName?: string;
|
|
12731
12751
|
};
|
|
12732
12752
|
/**
|
|
@@ -13124,6 +13144,14 @@ declare abstract class BaseSuiteEvaluator extends BaseMetric {
|
|
|
13124
13144
|
readonly validationSchema: z.ZodObject<{}, "passthrough", z.ZodTypeAny, z.objectOutputType<{}, z.ZodTypeAny, "passthrough">, z.objectInputType<{}, z.ZodTypeAny, "passthrough">>;
|
|
13125
13145
|
protected constructor(name: string, trackMetric?: boolean);
|
|
13126
13146
|
abstract toConfig(): LLMJudgeConfig;
|
|
13147
|
+
/**
|
|
13148
|
+
* Score the given input and return one or more assertion results.
|
|
13149
|
+
*
|
|
13150
|
+
* Each returned `EvaluationScoreResult.value` MUST be `0` (failed) or `1`
|
|
13151
|
+
* (passed) — suite evaluator outputs are routed to the assertion-results
|
|
13152
|
+
* endpoint where they are persisted as `passed | failed`. Non-binary values
|
|
13153
|
+
* are coerced to `failed` and logged as a warning.
|
|
13154
|
+
*/
|
|
13127
13155
|
abstract score(input: unknown): EvaluationScoreResult | EvaluationScoreResult[] | Promise<EvaluationScoreResult> | Promise<EvaluationScoreResult[]>;
|
|
13128
13156
|
}
|
|
13129
13157
|
|
|
@@ -13499,6 +13527,7 @@ declare class OpikClient {
|
|
|
13499
13527
|
traceBatchQueue: TraceBatchQueue;
|
|
13500
13528
|
spanFeedbackScoresBatchQueue: SpanFeedbackScoresBatchQueue;
|
|
13501
13529
|
traceFeedbackScoresBatchQueue: TraceFeedbackScoresBatchQueue;
|
|
13530
|
+
traceAssertionResultsBatchQueue: AssertionResultsBatchQueue;
|
|
13502
13531
|
datasetBatchQueue: DatasetBatchQueue;
|
|
13503
13532
|
private lastProjectNameLogged;
|
|
13504
13533
|
constructor(explicitConfig?: Partial<ConstructorOpikConfig>);
|
|
@@ -16179,4 +16208,23 @@ declare class ConfigMismatchError extends OpikError {
|
|
|
16179
16208
|
|
|
16180
16209
|
declare function activateRunner(): void;
|
|
16181
16210
|
|
|
16182
|
-
|
|
16211
|
+
/**
|
|
16212
|
+
* HTTP header keys carrying Opik distributed trace context across service
|
|
16213
|
+
* boundaries. They are intentionally lowercase to match the canonical form
|
|
16214
|
+
* `Headers#get` returns and the way `node:http` exposes incoming headers.
|
|
16215
|
+
*/
|
|
16216
|
+
declare const OPIK_TRACE_ID_HEADER = "opik_trace_id";
|
|
16217
|
+
declare const OPIK_PARENT_SPAN_ID_HEADER = "opik_parent_span_id";
|
|
16218
|
+
interface DistributedTraceHeaders {
|
|
16219
|
+
[OPIK_TRACE_ID_HEADER]: string;
|
|
16220
|
+
[OPIK_PARENT_SPAN_ID_HEADER]: string;
|
|
16221
|
+
}
|
|
16222
|
+
/**
|
|
16223
|
+
* Returns the Opik distributed-trace HTTP headers describing the currently
|
|
16224
|
+
* active trace and span. Intended to be called from inside a function
|
|
16225
|
+
* wrapped with `track()` (or any code running within a `trackStorage`
|
|
16226
|
+
* context); returns `null` when called outside an active trace context.
|
|
16227
|
+
*/
|
|
16228
|
+
declare function getDistributedTraceHeaders(): DistributedTraceHeaders | null;
|
|
16229
|
+
|
|
16230
|
+
export { AgentTaskCompletionJudge, AgentToolCorrectnessJudge, type AllProviderOptions, AnnotationQueuePublicScope as AnnotationQueueScope, AnswerRelevance, type AnthropicProviderOptions, BaseLLMJudgeMetric, BaseMetric, BaseSuiteEvaluator, ChatPrompt, ComplianceRiskJudge, type Config, ConfigMismatchError, ConfigNotFoundError, Contains, type CreateTestSuiteOptions, DEFAULT_EXECUTION_POLICY, Dataset, type DatasetPublic, DatasetVersion, DatasetVersionNotFoundError, type DatasetVersionPublic, DemographicBiasJudge, DialogueHelpfulnessJudge, type DistributedTraceHeaders, type ErrorInfo, type EvaluateOptions, type EvaluatePromptOptions, type EvaluateTestSuiteOptions, type EvaluationError, type EvaluationResult, type EvaluationScoreResult, type EvaluationTask, type EvaluationTestCase, type EvaluationTestResult, ExactMatch, type ExecutionPolicy, type FeedbackScoreData, type FewShotExampleAnswerRelevanceNoContext, type FewShotExampleAnswerRelevanceWithContext, type FewShotExampleHallucination, type FewShotExampleModeration, type FilterExpression, GEval, GEvalPreset, GenderBiasJudge, type GoogleProviderOptions, Hallucination, IsJson, type ItemResult, LLMJudge, type LLMJudgeConfig, type LLMJudgeModelSettings, type LLMJudgeOptions, type LLMJudgeResponseFormat, ModelConfigurationError, ModelError, ModelGenerationError, Moderation, OPIK_PARENT_SPAN_ID_HEADER, OPIK_TRACE_ID_HEADER, type OpenAIProviderOptions, OpikClient as Opik, type OpikAssistantMessage, OpikBaseModel, type OpikConfig, type OpikMessage, OpikQueryLanguage, SpanType as OpikSpanType, type OpikSystemMessage, type OpikToolMessage, type OpikUserMessage, type Param, PoliticalBiasJudge, Prompt, PromptType, PromptUncertaintyJudge, type ProviderOptionsForModel, QARelevanceJudge, type RawTestSuiteItem, RegexMatch, RegionalBiasJudge, type RegistryEntry, ReligiousBiasJudge, ResponseSchema, type RunTestsOptions, SYSTEM_PROMPT, type ScoringKeyMappingType, Span, SpanType, SummarizationCoherenceJudge, SummarizationConsistencyJudge, type SupportedModelId, TestSuite, type TestSuiteItem, TestSuiteResult, ThreadsAnnotationQueue, Trace, TracesAnnotationQueue, USER_PROMPT_TEMPLATE, type UpdateTestSuiteItem, type UpdateTestSuiteOptions, Usefulness, VercelAIChatModel, activateRunner, agentConfigContext, buildSuiteResult, createModel, createModelFromInstance, deserializeEvaluators, detectProvider, disableLogger, evaluate, evaluatePrompt, evaluateTestSuite, flushAll, generateId, getDistributedTraceHeaders, getGlobalClient, getTrackContext, logger, resetGlobalClient, resolveEvaluators, resolveExecutionPolicy, resolveItemExecutionPolicy, resolveModel, runTests, serializeEvaluators, setGlobalClient, setLoggerLevel, track, validateEvaluators, validateExecutionPolicy };
|
package/dist/index.d.ts
CHANGED
|
@@ -12257,6 +12257,15 @@ declare class Trace {
|
|
|
12257
12257
|
update: (updates: TraceUpdateData) => this;
|
|
12258
12258
|
}
|
|
12259
12259
|
|
|
12260
|
+
interface OpikApiClientTempOptions extends OpikApiClient.Options {
|
|
12261
|
+
requestOptions?: RequestOptions;
|
|
12262
|
+
}
|
|
12263
|
+
declare class OpikApiClientTemp extends OpikApiClient {
|
|
12264
|
+
requestOptions: RequestOptions;
|
|
12265
|
+
constructor(options?: OpikApiClientTempOptions);
|
|
12266
|
+
setHeaders: (headers: Record<string, string>) => void;
|
|
12267
|
+
}
|
|
12268
|
+
|
|
12260
12269
|
interface BatchQueueOptions {
|
|
12261
12270
|
delay?: number;
|
|
12262
12271
|
enableCreateBatch?: boolean;
|
|
@@ -12285,13 +12294,24 @@ declare abstract class BatchQueue<EntityData = object, EntityId = string> {
|
|
|
12285
12294
|
flush: () => Promise<void>;
|
|
12286
12295
|
}
|
|
12287
12296
|
|
|
12288
|
-
|
|
12289
|
-
|
|
12290
|
-
|
|
12291
|
-
|
|
12292
|
-
|
|
12293
|
-
|
|
12294
|
-
|
|
12297
|
+
type AssertionResultId = {
|
|
12298
|
+
entityId: string;
|
|
12299
|
+
name: string;
|
|
12300
|
+
};
|
|
12301
|
+
declare class AssertionResultsBatchQueue extends BatchQueue<AssertionResultBatchItem, AssertionResultId> {
|
|
12302
|
+
private readonly api;
|
|
12303
|
+
private readonly entityType;
|
|
12304
|
+
private useLegacyFallback;
|
|
12305
|
+
constructor(api: OpikApiClientTemp, delay?: number, entityType?: AssertionResultBatchEntityType);
|
|
12306
|
+
protected getId(entity: AssertionResultBatchItem): {
|
|
12307
|
+
entityId: string;
|
|
12308
|
+
name: string;
|
|
12309
|
+
};
|
|
12310
|
+
protected createEntities(assertionResults: AssertionResultBatchItem[]): Promise<void>;
|
|
12311
|
+
private writeViaLegacyFeedbackScores;
|
|
12312
|
+
protected getEntity(): Promise<AssertionResultBatchItem | undefined>;
|
|
12313
|
+
protected updateEntity(): Promise<void>;
|
|
12314
|
+
protected deleteEntities(): Promise<void>;
|
|
12295
12315
|
}
|
|
12296
12316
|
|
|
12297
12317
|
type SpanUpdate = Partial<SavedSpan> & {
|
|
@@ -12726,7 +12746,7 @@ type EvaluationScoreResult = {
|
|
|
12726
12746
|
reason?: string;
|
|
12727
12747
|
/** Whether the scoring failed */
|
|
12728
12748
|
scoringFailed?: boolean;
|
|
12729
|
-
/** Optional category name for grouping scores
|
|
12749
|
+
/** Optional category name for grouping scores */
|
|
12730
12750
|
categoryName?: string;
|
|
12731
12751
|
};
|
|
12732
12752
|
/**
|
|
@@ -13124,6 +13144,14 @@ declare abstract class BaseSuiteEvaluator extends BaseMetric {
|
|
|
13124
13144
|
readonly validationSchema: z.ZodObject<{}, "passthrough", z.ZodTypeAny, z.objectOutputType<{}, z.ZodTypeAny, "passthrough">, z.objectInputType<{}, z.ZodTypeAny, "passthrough">>;
|
|
13125
13145
|
protected constructor(name: string, trackMetric?: boolean);
|
|
13126
13146
|
abstract toConfig(): LLMJudgeConfig;
|
|
13147
|
+
/**
|
|
13148
|
+
* Score the given input and return one or more assertion results.
|
|
13149
|
+
*
|
|
13150
|
+
* Each returned `EvaluationScoreResult.value` MUST be `0` (failed) or `1`
|
|
13151
|
+
* (passed) — suite evaluator outputs are routed to the assertion-results
|
|
13152
|
+
* endpoint where they are persisted as `passed | failed`. Non-binary values
|
|
13153
|
+
* are coerced to `failed` and logged as a warning.
|
|
13154
|
+
*/
|
|
13127
13155
|
abstract score(input: unknown): EvaluationScoreResult | EvaluationScoreResult[] | Promise<EvaluationScoreResult> | Promise<EvaluationScoreResult[]>;
|
|
13128
13156
|
}
|
|
13129
13157
|
|
|
@@ -13499,6 +13527,7 @@ declare class OpikClient {
|
|
|
13499
13527
|
traceBatchQueue: TraceBatchQueue;
|
|
13500
13528
|
spanFeedbackScoresBatchQueue: SpanFeedbackScoresBatchQueue;
|
|
13501
13529
|
traceFeedbackScoresBatchQueue: TraceFeedbackScoresBatchQueue;
|
|
13530
|
+
traceAssertionResultsBatchQueue: AssertionResultsBatchQueue;
|
|
13502
13531
|
datasetBatchQueue: DatasetBatchQueue;
|
|
13503
13532
|
private lastProjectNameLogged;
|
|
13504
13533
|
constructor(explicitConfig?: Partial<ConstructorOpikConfig>);
|
|
@@ -16179,4 +16208,23 @@ declare class ConfigMismatchError extends OpikError {
|
|
|
16179
16208
|
|
|
16180
16209
|
declare function activateRunner(): void;
|
|
16181
16210
|
|
|
16182
|
-
|
|
16211
|
+
/**
|
|
16212
|
+
* HTTP header keys carrying Opik distributed trace context across service
|
|
16213
|
+
* boundaries. They are intentionally lowercase to match the canonical form
|
|
16214
|
+
* `Headers#get` returns and the way `node:http` exposes incoming headers.
|
|
16215
|
+
*/
|
|
16216
|
+
declare const OPIK_TRACE_ID_HEADER = "opik_trace_id";
|
|
16217
|
+
declare const OPIK_PARENT_SPAN_ID_HEADER = "opik_parent_span_id";
|
|
16218
|
+
interface DistributedTraceHeaders {
|
|
16219
|
+
[OPIK_TRACE_ID_HEADER]: string;
|
|
16220
|
+
[OPIK_PARENT_SPAN_ID_HEADER]: string;
|
|
16221
|
+
}
|
|
16222
|
+
/**
|
|
16223
|
+
* Returns the Opik distributed-trace HTTP headers describing the currently
|
|
16224
|
+
* active trace and span. Intended to be called from inside a function
|
|
16225
|
+
* wrapped with `track()` (or any code running within a `trackStorage`
|
|
16226
|
+
* context); returns `null` when called outside an active trace context.
|
|
16227
|
+
*/
|
|
16228
|
+
declare function getDistributedTraceHeaders(): DistributedTraceHeaders | null;
|
|
16229
|
+
|
|
16230
|
+
export { AgentTaskCompletionJudge, AgentToolCorrectnessJudge, type AllProviderOptions, AnnotationQueuePublicScope as AnnotationQueueScope, AnswerRelevance, type AnthropicProviderOptions, BaseLLMJudgeMetric, BaseMetric, BaseSuiteEvaluator, ChatPrompt, ComplianceRiskJudge, type Config, ConfigMismatchError, ConfigNotFoundError, Contains, type CreateTestSuiteOptions, DEFAULT_EXECUTION_POLICY, Dataset, type DatasetPublic, DatasetVersion, DatasetVersionNotFoundError, type DatasetVersionPublic, DemographicBiasJudge, DialogueHelpfulnessJudge, type DistributedTraceHeaders, type ErrorInfo, type EvaluateOptions, type EvaluatePromptOptions, type EvaluateTestSuiteOptions, type EvaluationError, type EvaluationResult, type EvaluationScoreResult, type EvaluationTask, type EvaluationTestCase, type EvaluationTestResult, ExactMatch, type ExecutionPolicy, type FeedbackScoreData, type FewShotExampleAnswerRelevanceNoContext, type FewShotExampleAnswerRelevanceWithContext, type FewShotExampleHallucination, type FewShotExampleModeration, type FilterExpression, GEval, GEvalPreset, GenderBiasJudge, type GoogleProviderOptions, Hallucination, IsJson, type ItemResult, LLMJudge, type LLMJudgeConfig, type LLMJudgeModelSettings, type LLMJudgeOptions, type LLMJudgeResponseFormat, ModelConfigurationError, ModelError, ModelGenerationError, Moderation, OPIK_PARENT_SPAN_ID_HEADER, OPIK_TRACE_ID_HEADER, type OpenAIProviderOptions, OpikClient as Opik, type OpikAssistantMessage, OpikBaseModel, type OpikConfig, type OpikMessage, OpikQueryLanguage, SpanType as OpikSpanType, type OpikSystemMessage, type OpikToolMessage, type OpikUserMessage, type Param, PoliticalBiasJudge, Prompt, PromptType, PromptUncertaintyJudge, type ProviderOptionsForModel, QARelevanceJudge, type RawTestSuiteItem, RegexMatch, RegionalBiasJudge, type RegistryEntry, ReligiousBiasJudge, ResponseSchema, type RunTestsOptions, SYSTEM_PROMPT, type ScoringKeyMappingType, Span, SpanType, SummarizationCoherenceJudge, SummarizationConsistencyJudge, type SupportedModelId, TestSuite, type TestSuiteItem, TestSuiteResult, ThreadsAnnotationQueue, Trace, TracesAnnotationQueue, USER_PROMPT_TEMPLATE, type UpdateTestSuiteItem, type UpdateTestSuiteOptions, Usefulness, VercelAIChatModel, activateRunner, agentConfigContext, buildSuiteResult, createModel, createModelFromInstance, deserializeEvaluators, detectProvider, disableLogger, evaluate, evaluatePrompt, evaluateTestSuite, flushAll, generateId, getDistributedTraceHeaders, getGlobalClient, getTrackContext, logger, resetGlobalClient, resolveEvaluators, resolveExecutionPolicy, resolveItemExecutionPolicy, resolveModel, runTests, serializeEvaluators, setGlobalClient, setLoggerLevel, track, validateEvaluators, validateExecutionPolicy };
|
package/dist/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {Ea}from'./chunk-2HP64V53.js';export{ra as AgentTaskCompletionJudge,qa as AgentToolCorrectnessJudge,ea as AnswerRelevance,aa as BaseLLMJudgeMetric,B as BaseMetric,C as BaseSuiteEvaluator,p as ChatPrompt,ta as ComplianceRiskJudge,i as ConfigMismatchError,h as ConfigNotFoundError,Z as Contains,y as DEFAULT_EXECUTION_POLICY,j as Dataset,f as DatasetVersion,g as DatasetVersionNotFoundError,la as DemographicBiasJudge,ja as DialogueHelpfulnessJudge,Y as ExactMatch,fa as GEval,ga as GEvalPreset,na as GenderBiasJudge,da as Hallucination,$ as IsJson,P as LLMJudge,G as ModelConfigurationError,E as ModelError,F as ModelGenerationError,ba as Moderation,Ba as OPIK_PARENT_SPAN_ID_HEADER,Aa as OPIK_TRACE_ID_HEADER,za as Opik,D as OpikBaseModel,q as OpikQueryLanguage,d as OpikSpanType,ma as PoliticalBiasJudge,o as Prompt,k as PromptType,sa as PromptUncertaintyJudge,ka as QARelevanceJudge,_ as RegexMatch,pa as RegionalBiasJudge,oa as ReligiousBiasJudge,O as ResponseSchema,M as SYSTEM_PROMPT,ia as SummarizationCoherenceJudge,ha as SummarizationConsistencyJudge,xa as TestSuite,z as TestSuiteResult,s as ThreadsAnnotationQueue,r as TracesAnnotationQueue,N as USER_PROMPT_TEMPLATE,ca as Usefulness,I as VercelAIChatModel,v as activateRunner,t as agentConfigContext,A as buildSuiteResult,J as createModel,K as createModelFromInstance,R as deserializeEvaluators,H as detectProvider,c as disableLogger,W as evaluate,X as evaluatePrompt,U as evaluateTestSuite,u as flushAll,e as generateId,Ca as getDistributedTraceHeaders,l as getGlobalClient,w as getTrackContext,a as logger,n as resetGlobalClient,ua as resolveEvaluators,S as resolveExecutionPolicy,T as resolveItemExecutionPolicy,L as resolveModel,V as runTests,Q as serializeEvaluators,m as setGlobalClient,b as setLoggerLevel,x as track,va as validateEvaluators,wa as validateExecutionPolicy,Da as z}from'./chunk-2HP64V53.js';Ea();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import {ya}from'./chunk-
|
|
1
|
+
import {ya}from'./chunk-2HP64V53.js';export{y as DEFAULT_EXECUTION_POLICY,xa as TestSuite,z as TestSuiteResult,A as buildSuiteResult,R as deserializeEvaluators,U as evaluateTestSuite,S as resolveExecutionPolicy,T as resolveItemExecutionPolicy,V as runTests,Q as serializeEvaluators}from'./chunk-2HP64V53.js';ya();
|