@anvia/core 0.22.0 → 0.24.0

package/dist/agent/index.js

@@ -1,21 +1,21 @@
 import {
   AgentBuilder
-} from "../chunk-4HWN7734.js";
-import "../chunk-YEBIVZM2.js";
-import "../chunk-VXONXP2G.js";
+} from "../chunk-JI5YZWNT.js";
+import "../chunk-DALYP4PX.js";
+import "../chunk-BALBBVI3.js";
 import "../chunk-YK4WAAS4.js";
 import "../chunk-XUUY2L2D.js";
-import "../chunk-2NDQHV7V.js";
-import "../chunk-4BGN6PYF.js";
+import "../chunk-HYUYZEAJ.js";
+import "../chunk-MELYDKWQ.js";
 import "../chunk-2ODTMRHP.js";
 import "../chunk-UQX6EXWG.js";
 import "../chunk-MRQLEK2B.js";
+import "../chunk-MNWK2USR.js";
+import "../chunk-OIMLU4SF.js";
 import "../chunk-Q25IWLBT.js";
 import "../chunk-MC3CCKEB.js";
 import "../chunk-ADH7NNCS.js";
 import "../chunk-WQKHFADH.js";
-import "../chunk-MNWK2USR.js";
-import "../chunk-OIMLU4SF.js";
 import "../chunk-CWUJUSOS.js";
 export {
   AgentBuilder

package/dist/{chunk-VXONXP2G.js → chunk-BALBBVI3.js}

@@ -4,7 +4,7 @@ import {
 import {
   parseToolArgs,
   toolResultContentToText
-} from "./chunk-2NDQHV7V.js";
+} from "./chunk-HYUYZEAJ.js";
 import {
   runControl,
   toolCallControl
@@ -15,6 +15,9 @@ import {
   createMemoryCompactionSummary,
   cumulativeCompactedMessageCount
 } from "./chunk-MRQLEK2B.js";
+import {
+  mapWithConcurrency
+} from "./chunk-OIMLU4SF.js";
 import {
   CompletionRequestBuilder
 } from "./chunk-Q25IWLBT.js";
@@ -30,9 +33,6 @@ import {
   getAssistantGenerationMetadata,
   textFromAssistantContent
 } from "./chunk-ADH7NNCS.js";
-import {
-  mapWithConcurrency
-} from "./chunk-OIMLU4SF.js";
 import {
   appendGuardrailPolicies,
   hasEnforcedOutputGuardrails,
@@ -2729,4 +2729,4 @@ export {
   extractRagText,
   PromptRequest
 };
-//# sourceMappingURL=chunk-VXONXP2G.js.map
+//# sourceMappingURL=chunk-BALBBVI3.js.map

package/dist/{chunk-YEBIVZM2.js → chunk-DALYP4PX.js}

@@ -1,15 +1,15 @@
 import {
   PromptRequest
-} from "./chunk-VXONXP2G.js";
+} from "./chunk-BALBBVI3.js";
 import {
   isSkillTool
 } from "./chunk-YK4WAAS4.js";
 import {
   ToolSet
-} from "./chunk-2NDQHV7V.js";
+} from "./chunk-HYUYZEAJ.js";
 import {
   createTool
-} from "./chunk-4BGN6PYF.js";
+} from "./chunk-MELYDKWQ.js";
 import {
   isStreamingCompletionModel
 } from "./chunk-MC3CCKEB.js";
@@ -222,4 +222,4 @@ export {
   Agent,
   AgentSession
 };
-//# sourceMappingURL=chunk-YEBIVZM2.js.map
+//# sourceMappingURL=chunk-DALYP4PX.js.map

package/dist/{chunk-2NDQHV7V.js → chunk-HYUYZEAJ.js}

@@ -1,14 +1,14 @@
 import {
   InMemoryVectorStore,
   createTool
-} from "./chunk-4BGN6PYF.js";
+} from "./chunk-MELYDKWQ.js";
+import {
+  embedDocuments
+} from "./chunk-MNWK2USR.js";
 import {
   isToolResultContentArray,
   serializeToolResultOutput
 } from "./chunk-ADH7NNCS.js";
-import {
-  embedDocuments
-} from "./chunk-MNWK2USR.js";
 
 // src/tool/errors.ts
 var ToolCallError = class extends Error {
@@ -220,4 +220,4 @@ export {
   createMiddleware,
   createThinkTool
 };
-//# sourceMappingURL=chunk-2NDQHV7V.js.map
+//# sourceMappingURL=chunk-HYUYZEAJ.js.map

package/dist/{chunk-4HWN7734.js → chunk-JI5YZWNT.js}

@@ -1,10 +1,10 @@
 import {
   Agent,
   normalizeAgentId
-} from "./chunk-YEBIVZM2.js";
+} from "./chunk-DALYP4PX.js";
 import {
   ToolSet
-} from "./chunk-2NDQHV7V.js";
+} from "./chunk-HYUYZEAJ.js";
 import {
   resolveMemoryOptions
 } from "./chunk-UQX6EXWG.js";
@@ -217,4 +217,4 @@ var AgentBuilder = class {
 export {
   AgentBuilder
 };
-//# sourceMappingURL=chunk-4HWN7734.js.map
+//# sourceMappingURL=chunk-JI5YZWNT.js.map

package/dist/{chunk-4BGN6PYF.js → chunk-MELYDKWQ.js}

@@ -1,10 +1,10 @@
-import {
-  toProviderJsonSchema
-} from "./chunk-WQKHFADH.js";
 import {
   cosineSimilarity,
   embedText
 } from "./chunk-MNWK2USR.js";
+import {
+  toProviderJsonSchema
+} from "./chunk-WQKHFADH.js";
 
 // src/vector-store/index.ts
 import { z } from "zod";
@@ -334,4 +334,4 @@ export {
   InMemoryVectorIndex,
   createVectorSearchTool
 };
-//# sourceMappingURL=chunk-4BGN6PYF.js.map
+//# sourceMappingURL=chunk-MELYDKWQ.js.map

package/dist/{chunk-BCA4VZ5W.js → chunk-NCPN4SJR.js}

@@ -6,7 +6,7 @@ import {
 } from "./chunk-YK4WAAS4.js";
 import {
   createTool
-} from "./chunk-4BGN6PYF.js";
+} from "./chunk-MELYDKWQ.js";
 
 // src/skills/instructions.ts
 function skillInstructions(skills) {
@@ -395,4 +395,4 @@ export {
   SkillValidationError,
   skill
 };
-//# sourceMappingURL=chunk-BCA4VZ5W.js.map
+//# sourceMappingURL=chunk-NCPN4SJR.js.map

package/dist/{chunk-KYEBWN3E.js → chunk-YLMDBQO4.js}

@@ -1,12 +1,12 @@
 import {
   AgentBuilder
-} from "./chunk-4HWN7734.js";
+} from "./chunk-JI5YZWNT.js";
 import {
   extractRagText
-} from "./chunk-VXONXP2G.js";
+} from "./chunk-BALBBVI3.js";
 import {
   createTool
-} from "./chunk-4BGN6PYF.js";
+} from "./chunk-MELYDKWQ.js";
 import {
   CompletionRequestBuilder
 } from "./chunk-Q25IWLBT.js";
@@ -138,4 +138,4 @@ export {
   Extractor,
   ExtractorBuilder
 };
-//# sourceMappingURL=chunk-KYEBWN3E.js.map
+//# sourceMappingURL=chunk-YLMDBQO4.js.map

package/dist/evals/index.d.ts

@@ -1,15 +1,15 @@
+import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
+import { l as JsonValue, C as CompletionModel, M as Message } from '../types-G12i56s2.js';
 import { A as Agent } from '../agent-x3YTyv5i.js';
-import { l as JsonValue, M as Message, C as CompletionModel } from '../types-G12i56s2.js';
 import { P as PromptResponse } from '../index-BxSbmOMr.js';
-import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
 import { E as EmbeddingModel } from '../types-BCTRUGex.js';
+import 'zod';
 import '../guardrails/index.js';
 import '../types-D9JpeAPF.js';
 import '../types-BrLbKnEz.js';
 import '../tool-Cohdv5GC.js';
 import '../middleware-Dp3d_pxi.js';
 import '../types-O-abTIin.js';
-import 'zod';
 
 type EvalOutcome<Score = unknown> = {
     outcome: "pass";
@@ -45,14 +45,47 @@ declare const EvalOutcome: {
 };
 
 type EvalMetadata = Record<string, JsonValue | undefined>;
+type EvalRunOptions = {
+    id?: string | undefined;
+    datasetName?: string | undefined;
+    datasetVersion?: string | undefined;
+    metadata?: EvalMetadata | undefined;
+};
+type EvalRunContext = {
+    id: string;
+    startedAt: string;
+    datasetName?: string | undefined;
+    datasetVersion?: string | undefined;
+    metadata?: EvalMetadata | undefined;
+};
 type EvalCase<Input, Expected = unknown> = {
     id: string;
     input: Input;
     expected?: Expected | undefined;
+    context?: string[] | undefined;
+    retrievalContext?: string[] | undefined;
+    metadata?: EvalMetadata | undefined;
+};
+type EvalTurn = {
+    role: "user" | "assistant";
+    content: string;
     metadata?: EvalMetadata | undefined;
 };
+type EvalTraceRef = {
+    traceId: string;
+    observationId?: string | undefined;
+    responseId?: string | undefined;
+};
 type EvalTarget<Input, Output, Expected = unknown> = (input: Input, testCase: EvalCase<Input, Expected>) => Output | Promise<Output>;
 type EvalOutcomeStatus = "pass" | "fail" | "invalid";
+type EvalScoreProjection = {
+    outcome: EvalOutcomeStatus;
+    value: number | string;
+    numericValue?: number | undefined;
+    categoricalValue?: string | undefined;
+    label: string;
+    explanation?: string | undefined;
+};
 type EvalMetricArgs<Input, Output, Expected = unknown> = {
     suiteName: string;
     case: EvalCase<Input, Expected>;
@@ -79,35 +112,144 @@ type EvalCaseResult<Input, Output, Expected = unknown> = {
 };
 type EvalSuiteResult<Input, Output, Expected = unknown> = {
     name: string;
+    run: EvalRunContext & {
+        completedAt: string;
+    };
     results: Array<EvalCaseResult<Input, Output, Expected>>;
     passed: number;
     failed: number;
     invalid: number;
     durationMs: number;
+    reporterErrors: unknown[];
 };
 type EvalReportArgs<Input, Output, Score = unknown, Expected = unknown> = {
+    run?: EvalRunContext | undefined;
     suiteName: string;
     case: EvalCase<Input, Expected>;
     output?: Output | undefined;
     targetError?: unknown;
+    trace?: EvalTraceRef | undefined;
     metric: EvalMetric<Input, Output, Score, Expected>;
     outcome: EvalOutcome<Score>;
 };
+type EvalRunStartArgs = {
+    run: EvalRunContext;
+    suiteName: string;
+    caseCount: number;
+    metricNames: string[];
+};
+type EvalRunEndArgs = EvalRunStartArgs & {
+    status: "completed" | "failed";
+    completedAt: string;
+    durationMs: number;
+    passed?: number | undefined;
+    failed?: number | undefined;
+    invalid?: number | undefined;
+    error?: unknown;
+};
+type EvalTraceSelectorArgs<Input, Output, Expected = unknown> = {
+    suiteName: string;
+    case: EvalCase<Input, Expected>;
+    output?: Output | undefined;
+    targetError?: unknown;
+};
+type EvalTraceSelector<Input, Output, Expected = unknown> = (args: EvalTraceSelectorArgs<Input, Output, Expected>) => EvalTraceRef | undefined | Promise<EvalTraceRef | undefined>;
 type EvalReporter<Input = unknown, Output = unknown, Expected = unknown> = {
+    onRunStart?(args: EvalRunStartArgs): void | Promise<void>;
     report(args: EvalReportArgs<Input, Output, unknown, Expected>): void | Promise<void>;
+    onRunEnd?(args: EvalRunEndArgs): void | Promise<void>;
 };
 type RunEvalSuiteOptions<Input, Output, Expected = unknown> = {
     name: string;
+    run?: EvalRunOptions | undefined;
     cases: Array<EvalCase<Input, Expected>>;
     target: EvalTarget<Input, Output, Expected>;
     metrics: Array<EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>>>;
     concurrency?: number | undefined;
+    trace?: EvalTraceSelector<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>> | undefined;
     reporters?: Array<EvalReporter<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>>> | undefined;
     failOnReporterError?: boolean | undefined;
 };
 type ValueSelector<Input, Output, Expected, Value> = (args: EvalMetricArgs<Input, Output, Expected>) => Value | Promise<Value>;
 type SelectorOrValue<Input, Output, Expected, Value> = Value | ValueSelector<Input, Output, Expected, Value>;
 
+type LlmEvalOptions<Input, Output, Expected = unknown> = {
+    name?: string | undefined;
+    model: CompletionModel;
+    threshold?: number | undefined;
+    strictMode?: boolean | undefined;
+    includeReason?: boolean | undefined;
+    retries?: number | undefined;
+    input?: ValueSelector<Input, Output, Expected, string> | undefined;
+    actual?: ValueSelector<Input, Output, Expected, string> | undefined;
+};
+type AnswerRelevancyOptions<Input, Output, Expected = unknown> = LlmEvalOptions<Input, Output, Expected>;
+declare function answerRelevancy<Input, Output, Expected = unknown>(options: AnswerRelevancyOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+type PromptAlignmentOptions<Input, Output, Expected = unknown> = LlmEvalOptions<Input, Output, Expected> & {
+    promptInstructions: string[];
+};
+declare function promptAlignment<Input, Output, Expected = unknown>(options: PromptAlignmentOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+type JsonCorrectnessOptions<Input, Output, SchemaOutput, Expected = unknown> = {
+    name?: string | undefined;
+    schema: ZodSchema<SchemaOutput>;
+    model?: CompletionModel | undefined;
+    threshold?: number | undefined;
+    strictMode?: boolean | undefined;
+    includeReason?: boolean | undefined;
+    retries?: number | undefined;
+    actual?: ValueSelector<Input, Output, Expected, string> | undefined;
+};
+declare function jsonCorrectness<Input, Output, SchemaOutput, Expected = unknown>(options: JsonCorrectnessOptions<Input, Output, SchemaOutput, Expected>): EvalMetric<Input, Output, number, Expected>;
+type HallucinationOptions<Input, Output, Expected = unknown> = LlmEvalOptions<Input, Output, Expected> & {
+    context?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;
+};
+declare function hallucination<Input, Output, Expected = unknown>(options: HallucinationOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+type FaithfulnessOptions<Input, Output, Expected = unknown> = LlmEvalOptions<Input, Output, Expected> & {
+    retrievalContext?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;
+    truthsExtractionLimit?: number | undefined;
+    penalizeAmbiguousClaims?: boolean | undefined;
+};
+declare function faithfulness<Input, Output, Expected = unknown>(options: FaithfulnessOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+type SummarizationOptions<Input, Output, Expected = unknown> = LlmEvalOptions<Input, Output, Expected> & {
+    assessmentQuestions?: string[] | undefined;
+    questionCount?: number | undefined;
+    truthsExtractionLimit?: number | undefined;
+};
+declare function summarization<Input, Output, Expected = unknown>(options: SummarizationOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+type GEvalParameter = "input" | "actualOutput" | "expectedOutput" | "context" | "retrievalContext" | "metadata";
+type GEvalRubric = {
+    scoreRange: readonly [number, number];
+    expectedOutcome: string;
+};
+type GEvalOptions<Input, Output, Expected = unknown> = Omit<LlmEvalOptions<Input, Output, Expected>, "name"> & {
+    name: string;
+    evaluationParams: GEvalParameter[];
+    criteria?: string | undefined;
+    evaluationSteps?: string[] | undefined;
+    rubric?: GEvalRubric[] | undefined;
+    expected?: ValueSelector<Input, Output, Expected, unknown> | undefined;
+    context?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;
+    retrievalContext?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;
+};
+declare function gEval<Input, Output, Expected = unknown>(options: GEvalOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+type ConversationEvalOptions<Input, Output, Expected = unknown> = {
+    name?: string | undefined;
+    model: CompletionModel;
+    threshold?: number | undefined;
+    strictMode?: boolean | undefined;
+    includeReason?: boolean | undefined;
+    retries?: number | undefined;
+    concurrency?: number | undefined;
+    turns?: ValueSelector<Input, Output, Expected, ConversationSource> | undefined;
+};
+type TurnRelevancyOptions<Input, Output, Expected = unknown> = ConversationEvalOptions<Input, Output, Expected> & {
+    windowSize?: number | undefined;
+};
+declare function turnRelevancy<Input, Output, Expected = unknown>(options: TurnRelevancyOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+type KnowledgeRetentionOptions<Input, Output, Expected = unknown> = ConversationEvalOptions<Input, Output, Expected>;
+declare function knowledgeRetention<Input, Output, Expected = unknown>(options: KnowledgeRetentionOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+type ConversationSource = EvalTurn[] | Message[];
+
 type AgentEvalTargetOptions<Input, Output = PromptResponse> = {
     prompt?: ((input: Input, testCase: EvalCase<Input>) => string | Message) | undefined;
     output?: ((response: PromptResponse, testCase: EvalCase<Input>) => Output) | undefined;
@@ -162,6 +304,14 @@ type LlmScoreOptions<Input, Output, Expected = unknown> = {
 };
 declare function llmScore<Input, Output, Expected = unknown>(options: LlmScoreOptions<Input, Output, Expected>): EvalMetric<Input, Output, LlmScoreMetricScore, Expected>;
 
+declare function projectEvalOutcome(outcome: EvalOutcome, dataType: EvalMetric<unknown, unknown>["dataType"]): EvalScoreProjection;
+declare function resolveEvalTraceRef(args: {
+    output?: unknown;
+    input?: unknown;
+    metadata?: EvalMetadata | undefined;
+}): EvalTraceRef | undefined;
+declare function defaultEvalTraceSelector<Input, Output, Expected>(args: EvalTraceSelectorArgs<Input, Output, Expected>): EvalTraceRef | undefined;
+
 declare function runEvalSuite<Input, Output, Expected = unknown>(options: RunEvalSuiteOptions<Input, Output, Expected>): Promise<EvalSuiteResult<Input, Output, Expected>>;
 
-export { type AgentEvalTargetOptions, type ContainsOptions, type EvalCase, type EvalCaseResult, type EvalMetadata, type EvalMetric, type EvalMetricArgs, type EvalMetricResult, EvalOutcome, type EvalOutcomeStatus, type EvalReportArgs, type EvalReporter, type EvalSuiteResult, type EvalTarget, type ExactMatchOptions, type LlmJudgeOptions, type LlmScoreMetricScore, type LlmScoreOptions, type RunEvalSuiteOptions, type SelectorOrValue, type SemanticSimilarityOptions, type ValueSelector, agentEvalTarget, contains, defineMetric, exactMatch, llmJudge, llmScore, runEvalSuite, semanticSimilarity };
+export { type AgentEvalTargetOptions, type AnswerRelevancyOptions, type ContainsOptions, type EvalCase, type EvalCaseResult, type EvalMetadata, type EvalMetric, type EvalMetricArgs, type EvalMetricResult, EvalOutcome, type EvalOutcomeStatus, type EvalReportArgs, type EvalReporter, type EvalRunContext, type EvalRunEndArgs, type EvalRunOptions, type EvalRunStartArgs, type EvalScoreProjection, type EvalSuiteResult, type EvalTarget, type EvalTraceRef, type EvalTraceSelector, type EvalTraceSelectorArgs, type EvalTurn, type ExactMatchOptions, type FaithfulnessOptions, type GEvalOptions, type GEvalParameter, type GEvalRubric, type HallucinationOptions, type JsonCorrectnessOptions, type KnowledgeRetentionOptions, type LlmJudgeOptions, type LlmScoreMetricScore, type LlmScoreOptions, type PromptAlignmentOptions, type RunEvalSuiteOptions, type SelectorOrValue, type SemanticSimilarityOptions, type SummarizationOptions, type TurnRelevancyOptions, type ValueSelector, agentEvalTarget, answerRelevancy, contains, defaultEvalTraceSelector, defineMetric, exactMatch, faithfulness, gEval, hallucination, jsonCorrectness, knowledgeRetention, llmJudge, llmScore, projectEvalOutcome, promptAlignment, resolveEvalTraceRef, runEvalSuite, semanticSimilarity, summarization, turnRelevancy };