@anvia/core 0.5.0 → 0.6.0

package/dist/completion/index.js

@@ -7,11 +7,15 @@ import {
   Usage,
   UserContent,
   assertCompletionRequestSupported,
+  createCompletion,
+  createCompletionStream,
+  createParsedCompletion,
   formatDocument,
   normalizeDocuments,
   reasoningDisplayText,
   textFromAssistantContent
-} from "../chunk-6GJDBBDC.js";
+} from "../chunk-A4GUJSEK.js";
+import "../chunk-WQKHFADH.js";
 export {
   AssistantContent,
   CompletionCapabilityError,
@@ -21,6 +25,9 @@ export {
   Usage,
   UserContent,
   assertCompletionRequestSupported,
+  createCompletion,
+  createCompletionStream,
+  createParsedCompletion,
   formatDocument,
   normalizeDocuments,
   reasoningDisplayText,

package/dist/create-completion-Bog3ni7b.d.ts

@@ -0,0 +1,36 @@
+import { Z as ZodSchema } from './zod-schema-C7F4clpm.js';
+import { M as Message, D as Document, g as ToolDefinition, m as ToolChoice, J as JsonObject, e as JsonValue, A as AssistantContent, U as Usage, c as CompletionResponse, C as CompletionModel, n as StreamingCompletionModel, o as CompletionStreamEvent } from './types-Dmoq-Fhz.js';
+
+type CreateCompletionInput = string | Message | Message[];
+type CreateCompletionBaseOptions = {
+    input?: CreateCompletionInput | undefined;
+    messages?: Message[] | undefined;
+    instructions?: string | undefined;
+    documents?: Document[] | undefined;
+    tools?: ToolDefinition[] | undefined;
+    temperature?: number | undefined;
+    maxTokens?: number | undefined;
+    toolChoice?: ToolChoice | undefined;
+    outputSchema?: JsonObject | undefined;
+    params?: JsonValue | undefined;
+};
+type CreateCompletionOptions = CreateCompletionBaseOptions;
+type CreateCompletionStreamOptions = CreateCompletionBaseOptions;
+type CreateParsedCompletionOptions<T> = Omit<CreateCompletionBaseOptions, "outputSchema"> & {
+    schema: ZodSchema<T>;
+};
+type CreateCompletionResult<RawResponse = unknown> = {
+    text: string;
+    content: AssistantContent[];
+    usage: Usage;
+    response: CompletionResponse<RawResponse>;
+};
+type CreateParsedCompletionResult<T, RawResponse = unknown> = CreateCompletionResult<RawResponse> & {
+    data: T;
+};
+type RawResponseOf<Model> = Model extends CompletionModel<infer RawResponse> ? RawResponse : unknown;
+declare function createCompletion<Model extends CompletionModel>(model: Model, options: CreateCompletionOptions): Promise<CreateCompletionResult<RawResponseOf<Model>>>;
+declare function createCompletionStream<Model extends StreamingCompletionModel>(model: Model, options: CreateCompletionStreamOptions): AsyncIterable<CompletionStreamEvent<RawResponseOf<Model>>>;
+declare function createParsedCompletion<T, Model extends CompletionModel>(model: Model, options: CreateParsedCompletionOptions<T>): Promise<CreateParsedCompletionResult<T, RawResponseOf<Model>>>;
+
+export { type CreateCompletionBaseOptions as C, type CreateCompletionInput as a, type CreateCompletionOptions as b, type CreateCompletionResult as c, type CreateCompletionStreamOptions as d, type CreateParsedCompletionOptions as e, type CreateParsedCompletionResult as f, createCompletion as g, createCompletionStream as h, createParsedCompletion as i };

package/dist/evals/index.d.ts

@@ -1,11 +1,11 @@
-import { P as PromptResponse, e as Agent } from '../agent-CWGuTdMJ.js';
-import { e as JsonValue, M as Message, C as CompletionModel } from '../types-C54aNoCd.js';
-import { E as EmbeddingModel } from '../types-IB2e9u5M.js';
+import { P as PromptResponse, e as Agent } from '../agent-BURoB-me.js';
+import { e as JsonValue, M as Message, C as CompletionModel } from '../types-Dmoq-Fhz.js';
 import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
+import { E as EmbeddingModel } from '../types-IB2e9u5M.js';
 import '../memory/index.js';
 import '../observability/index.js';
-import '../tool-C3ciF-VG.js';
-import '../middleware-qfm1EFKK.js';
+import '../tool-Dz-lHZon.js';
+import '../middleware-C9QIms85.js';
 import '../vector-store/index.js';
 import 'zod';
 

package/dist/evals/index.js

@@ -1,14 +1,15 @@
 import {
   ExtractorBuilder
-} from "../chunk-Z7HY4WU7.js";
-import "../chunk-365W6FJD.js";
-import "../chunk-ORLEOXD7.js";
+} from "../chunk-OXYZDHNI.js";
+import "../chunk-6NWNL2KM.js";
+import "../chunk-IXK3TEHJ.js";
 import "../chunk-XUUY2L2D.js";
 import "../chunk-XXT2UCAR.js";
 import "../chunk-YK4WAAS4.js";
-import "../chunk-S7EOE6EC.js";
-import "../chunk-P425B6GR.js";
-import "../chunk-6GJDBBDC.js";
+import "../chunk-AER5FGPK.js";
+import "../chunk-I47EHI45.js";
+import "../chunk-A4GUJSEK.js";
+import "../chunk-WQKHFADH.js";
 import {
   cosineSimilarity,
   embedText

package/dist/evals/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/evals/agent-target.ts","../../src/evals/metrics.ts","../../src/evals/format.ts","../../src/evals/outcome.ts","../../src/evals/selectors.ts","../../src/evals/runner.ts"],"sourcesContent":["import type { Agent } from \"../agent/agent\";\nimport type { PromptResponse } from \"../agent/request\";\nimport type { Message } from \"../completion\";\nimport type { EvalCase, EvalTarget } from \"./types\";\n\nexport type AgentEvalTargetOptions<Input, Output = PromptResponse> = {\n  prompt?: ((input: Input, testCase: EvalCase<Input>) => string | Message) | undefined;\n  output?: ((response: PromptResponse, testCase: EvalCase<Input>) => Output) | undefined;\n};\n\nexport function agentEvalTarget<Input>(\n  agent: Agent,\n  options?: AgentEvalTargetOptions<Input, PromptResponse>,\n): EvalTarget<Input, PromptResponse>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output>,\n): EvalTarget<Input, Output>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output | PromptResponse> = {},\n): EvalTarget<Input, Output | PromptResponse> {\n  return async (input, testCase) => {\n    const prompt = options.prompt?.(input, testCase) ?? String(input);\n    const response = await agent.prompt(prompt).send();\n    return options.output === undefined ? response : options.output(response, testCase);\n  };\n}\n","import { z } from \"zod\";\nimport type { CompletionModel } from \"../completion\";\nimport { cosineSimilarity, type EmbeddingModel, embedText } from \"../embeddings\";\nimport { ExtractorBuilder } from \"../extractor\";\nimport type { ZodSchema } from \"../schema\";\nimport { errorMessage, formatValue, stableComparable } from \"./format\";\nimport { EvalOutcome } from \"./outcome\";\nimport { resolveActual, resolveActualText, resolveExpected, resolveJudgePrompt } from \"./selectors\";\nimport type { EvalMetric, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport type ExactMatchOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, unknown> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, unknown> | undefined;\n};\n\nexport function exactMatch<Input, Output, Expected = unknown>(\n  options: ExactMatchOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"exact_match\",\n    async evaluate(args) {\n      const actual = await resolveActual(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for exact match.\");\n      }\n      const passed = stableComparable(actual) === stableComparable(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });\n    },\n  };\n}\n\nexport type ContainsOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string | RegExp> | undefined;\n};\n\nexport function contains<Input, Output, Expected = unknown>(\n  options: ContainsOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"contains\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for contains.\");\n      }\n      if (typeof expected !== \"string\" && !(expected instanceof RegExp)) {\n        return EvalOutcome.invalid(\"Contains expected value must be a string or RegExp.\");\n      }\n      const passed = expected instanceof RegExp ? expected.test(actual) : actual.includes(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Output did not contain ${String(expected)}.` });\n    },\n  };\n}\n\nexport type SemanticSimilarityOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: EmbeddingModel;\n  threshold: number;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string> | undefined;\n};\n\nexport function semanticSimilarity<Input, Output, Expected = unknown>(\n  options: SemanticSimilarityOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, number, Expected> {\n  return {\n    name: options.name ?? \"semantic_similarity\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for semantic similarity.\");\n      }\n      if (typeof expected !== \"string\") {\n        return EvalOutcome.invalid(\"Semantic similarity expected value must be a string.\");\n      }\n      const [actualEmbedding, expectedEmbedding] = await Promise.all([\n        embedText(options.model, actual),\n        embedText(options.model, expected),\n      ]);\n      const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);\n      return score >= options.threshold\n        ? EvalOutcome.pass(score)\n        : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });\n    },\n  };\n}\n\nexport type LlmJudgeOptions<Input, Output, SchemaOutput, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  schema: ZodSchema<SchemaOutput>;\n  passes(value: SchemaOutput): boolean;\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmJudge<Input, Output, SchemaOutput, Expected = unknown>(\n  options: LlmJudgeOptions<Input, Output, SchemaOutput, Expected>,\n): EvalMetric<Input, Output, SchemaOutput, Expected> {\n  const extractor = new ExtractorBuilder(options.model, options.schema)\n    .instructions(\n      options.instructions ??\n        \"Judge the eval case by the requested schema. Submit the judgment using the schema.\",\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_judge\",\n    async evaluate(args) {\n      try {\n        const judgment = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        return options.passes(judgment) ? EvalOutcome.pass(judgment) : EvalOutcome.fail(judgment);\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n\nexport type LlmScoreMetricScore = {\n  score: number;\n  feedback: string;\n};\n\nexport type LlmScoreOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  threshold: number;\n  criteria: string | string[];\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmScore<Input, Output, Expected = unknown>(\n  options: LlmScoreOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, LlmScoreMetricScore, Expected> {\n  const criteria = Array.isArray(options.criteria) ? options.criteria.join(\"\\n\") : options.criteria;\n  const extractor = new ExtractorBuilder(\n    options.model,\n    z.object({\n      score: z.number(),\n      feedback: z.string(),\n    }),\n  )\n    .instructions(\n      options.instructions ??\n        `Score the eval case against these criteria:\\n${criteria}\\n\\nReturn a score between 0 and 1 and brief feedback.`,\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_score\",\n    async evaluate(args) {\n      try {\n        const score = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        if (score.score < 0 || score.score > 1) {\n          return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {\n            score,\n          });\n        }\n        return score.score >= options.threshold\n          ? EvalOutcome.pass(score, { comment: score.feedback })\n          : EvalOutcome.fail(score, { comment: score.feedback });\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n","export function defaultOutputValue(output: unknown): unknown {\n  if (\n    typeof output === \"object\" &&\n    output !== null &&\n    \"output\" in output &&\n    typeof (output as { output?: unknown }).output === \"string\"\n  ) {\n    return (output as { output: string }).output;\n  }\n  return output;\n}\n\nexport function stableComparable(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  return JSON.stringify(value);\n}\n\nexport function formatValue(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  try {\n    return JSON.stringify(value);\n  } catch {\n    return String(value);\n  }\n}\n\nexport function errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n","import type { EvalMetadata } from \"./types\";\n\nexport type EvalOutcome<Score = unknown> =\n  | {\n      outcome: \"pass\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"fail\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"invalid\";\n      reason: string;\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    };\n\nexport const EvalOutcome = {\n  pass<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    return {\n      outcome: \"pass\",\n      ...(score === undefined ? {} : { score }),\n      ...(options.comment === undefined ? {} : { comment: options.comment }),\n      ...(options.metadata === undefined ? {} : { metadata: options.metadata }),\n    };\n  },\n\n  fail<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    return {\n      outcome: \"fail\",\n      ...(score === undefined ? {} : { score }),\n      ...(options.comment === undefined ? {} : { comment: options.comment }),\n      ...(options.metadata === undefined ? {} : { metadata: options.metadata }),\n    };\n  },\n\n  invalid<Score = never>(\n    reason: string,\n    options: {\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    } = {},\n  ): EvalOutcome<Score> {\n    return {\n      outcome: \"invalid\",\n      reason,\n      ...(options.score === undefined ? {} : { score: options.score }),\n      ...(options.comment === undefined ? {} : { comment: options.comment }),\n      ...(options.metadata === undefined ? {} : { metadata: options.metadata }),\n    };\n  },\n};\n","import { defaultOutputValue, formatValue } from \"./format\";\nimport type { EvalMetricArgs, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport async function resolveActual<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, unknown> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<unknown> {\n  return selector === undefined ? defaultOutputValue(args.output) : selector(args);\n}\n\nexport async function resolveActualText<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  const value = selector === undefined ? defaultOutputValue(args.output) : await selector(args);\n  return typeof value === \"string\" ? value : JSON.stringify(value);\n}\n\nexport async function resolveExpected<Input, Output, Expected, Value>(\n  selectorOrValue: SelectorOrValue<Input, Output, Expected, Value> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<Value | Expected | undefined> {\n  if (selectorOrValue === undefined) {\n    return args.case.expected;\n  }\n  return typeof selectorOrValue === \"function\"\n    ? (selectorOrValue as ValueSelector<Input, Output, Expected, Value>)(args)\n    : selectorOrValue;\n}\n\nexport async function resolveJudgePrompt<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  if (selector !== undefined) {\n    return selector(args);\n  }\n  return [\n    `Suite: ${args.suiteName}`,\n    `Case: ${args.case.id}`,\n    `Input: ${formatValue(args.case.input)}`,\n    `Expected: ${formatValue(args.case.expected)}`,\n    `Output: ${formatValue(defaultOutputValue(args.output))}`,\n  ].join(\"\\n\\n\");\n}\n","import { mapWithConcurrency } from \"../internal/concurrency\";\nimport { errorMessage } from \"./format\";\nimport { EvalOutcome, type EvalOutcome as EvalOutcomeType } from \"./outcome\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalMetric,\n  EvalMetricResult,\n  EvalReporter,\n  EvalSuiteResult,\n  RunEvalSuiteOptions,\n} from \"./types\";\n\nexport async function runEvalSuite<Input, Output, Expected = unknown>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n): Promise<EvalSuiteResult<Input, Output, Expected>> {\n  const startedAt = Date.now();\n  const results = await mapWithConcurrency(\n    options.cases,\n    Math.max(1, Math.trunc(options.concurrency ?? 1)),\n    (testCase) => runEvalCase(options, testCase),\n  );\n  const counts = countOutcomes(results);\n  return {\n    name: options.name,\n    results,\n    ...counts,\n    durationMs: Date.now() - startedAt,\n  };\n}\n\nasync function runEvalCase<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n  testCase: EvalCase<Input, Expected>,\n): Promise<EvalCaseResult<Input, Output, Expected>> {\n  let output: Output | undefined;\n  let targetError: unknown;\n  try {\n    output = await options.target(testCase.input, testCase);\n  } catch (error) {\n    targetError = error;\n  }\n\n  const metrics: EvalMetricResult[] = [];\n  for (const metric of options.metrics) {\n    const outcome =\n      targetError === undefined\n        ? await safeEvaluate(options.name, testCase, output as Output, metric)\n        : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`);\n    const reporterErrors = await reportOutcome({\n      suiteName: options.name,\n      testCase,\n      output,\n      targetError,\n      metric,\n      outcome,\n      reporters: options.reporters ?? [],\n      failOnReporterError: options.failOnReporterError === true,\n    });\n    metrics.push({ metricName: metric.name, outcome, reporterErrors });\n  }\n\n  return {\n    case: testCase,\n    ...(output === undefined ? {} : { output }),\n    ...(targetError === undefined ? {} : { targetError }),\n    metrics,\n  };\n}\n\nasync function safeEvaluate<Input, Output, Expected>(\n  suiteName: string,\n  testCase: EvalCase<Input, Expected>,\n  output: Output,\n  metric: EvalMetric<Input, Output, unknown, Expected>,\n): Promise<EvalOutcomeType> {\n  try {\n    return await metric.evaluate({ suiteName, case: testCase, output });\n  } catch (error) {\n    return EvalOutcome.invalid(errorMessage(error));\n  }\n}\n\nasync function reportOutcome<Input, Output, Expected>(args: {\n  suiteName: string;\n  testCase: EvalCase<Input, Expected>;\n  output: Output | undefined;\n  targetError: unknown;\n  metric: EvalMetric<Input, Output, unknown, Expected>;\n  outcome: EvalOutcomeType;\n  reporters: Array<EvalReporter<Input, Output, Expected>>;\n  failOnReporterError: boolean;\n}): Promise<unknown[]> {\n  const errors: unknown[] = [];\n  for (const reporter of args.reporters) {\n    try {\n      await reporter.report({\n        suiteName: args.suiteName,\n        case: args.testCase,\n        output: args.output,\n        targetError: args.targetError,\n        metric: args.metric,\n        outcome: args.outcome,\n      });\n    } catch (error) {\n      if (args.failOnReporterError) {\n        throw error;\n      }\n      errors.push(error);\n    }\n  }\n  return errors;\n}\n\nfunction countOutcomes(results: Array<EvalCaseResult<unknown, unknown, unknown>>): {\n  passed: number;\n  failed: number;\n  invalid: number;\n} {\n  let passed = 0;\n  let failed = 0;\n  let invalid = 0;\n  for (const result of results) {\n    for (const metric of result.metrics) {\n      if (metric.outcome.outcome === \"pass\") passed += 1;\n      if (metric.outcome.outcome === \"fail\") failed += 1;\n      if (metric.outcome.outcome === \"invalid\") invalid += 1;\n    }\n  }\n  return { passed, failed, invalid };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAkBO,SAAS,gBACd,OACA,UAAkE,CAAC,GACvB;AAC5C,SAAO,OAAO,OAAO,aAAa;AAChC,UAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,KAAK,OAAO,KAAK;AAChE,UAAM,WAAW,MAAM,MAAM,OAAO,MAAM,EAAE,KAAK;AACjD,WAAO,QAAQ,WAAW,SAAY,WAAW,QAAQ,OAAO,UAAU,QAAQ;AAAA,EACpF;AACF;;;AC3BA,SAAS,SAAS;;;ACAX,SAAS,mBAAmB,QAA0B;AAC3D,MACE,OAAO,WAAW,YAClB,WAAW,QACX,YAAY,UACZ,OAAQ,OAAgC,WAAW,UACnD;AACA,WAAQ,OAA8B;AAAA,EACxC;AACA,SAAO;AACT;AAEO,SAAS,iBAAiB,OAAwB;AACvD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,SAAO,KAAK,UAAU,KAAK;AAC7B;AAEO,SAAS,YAAY,OAAwB;AAClD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,MAAI;AACF,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B,QAAQ;AACN,WAAO,OAAO,KAAK;AAAA,EACrB;AACF;AAEO,SAAS,aAAa,OAAwB;AACnD,SAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC9D;;;ACTO,IAAM,cAAc;AAAA,EACzB,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,WAAO;AAAA,MACL,SAAS;AAAA,MACT,GAAI,UAAU,SAAY,CAAC,IAAI,EAAE,MAAM;AAAA,MACvC,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,MACpE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AAAA,EAEA,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,WAAO;AAAA,MACL,SAAS;AAAA,MACT,GAAI,UAAU,SAAY,CAAC,IAAI,EAAE,MAAM;AAAA,MACvC,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,MACpE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AAAA,EAEA,QACE,QACA,UAII,CAAC,GACe;AACpB,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,MACA,GAAI,QAAQ,UAAU,SAAY,CAAC,IAAI,EAAE,OAAO,QAAQ,MAAM;AAAA,MAC9D,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,MACpE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AACF;;;AC7DA,eAAsB,cACpB,UACA,MACkB;AAClB,SAAO,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,SAAS,IAAI;AACjF;AAEA,eAAsB,kBACpB,UACA,MACiB;AACjB,QAAM,QAAQ,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,MAAM,SAAS,IAAI;AAC5F,SAAO,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK;AACjE;AAEA,eAAsB,gBACpB,iBACA,MACuC;AACvC,MAAI,oBAAoB,QAAW;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AACA,SAAO,OAAO,oBAAoB,aAC7B,gBAAkE,IAAI,IACvE;AACN;AAEA,eAAsB,mBACpB,UACA,MACiB;AACjB,MAAI,aAAa,QAAW;AAC1B,WAAO,SAAS,IAAI;AAAA,EACtB;AACA,SAAO;AAAA,IACL,UAAU,KAAK,SAAS;AAAA,IACxB,SAAS,KAAK,KAAK,EAAE;AAAA,IACrB,UAAU,YAAY,KAAK,KAAK,KAAK,CAAC;AAAA,IACtC,aAAa,YAAY,KAAK,KAAK,QAAQ,CAAC;AAAA,IAC5C,WAAW,YAAY,mBAAmB,KAAK,MAAM,CAAC,CAAC;AAAA,EACzD,EAAE,KAAK,MAAM;AACf;;;AH5BO,SAAS,WACd,UAAsD,CAAC,GACT;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,cAAc,QAAQ,QAAQ,IAAI;AACvD,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,6CAA6C;AAAA,MAC1E;AACA,YAAM,SAAS,iBAAiB,MAAM,MAAM,iBAAiB,QAAQ;AACrE,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,YAAY,YAAY,QAAQ,CAAC,IAAI,CAAC;AAAA,IAC/E;AAAA,EACF;AACF;AAQO,SAAS,SACd,UAAoD,CAAC,GACP;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,0CAA0C;AAAA,MACvE;AACA,UAAI,OAAO,aAAa,YAAY,EAAE,oBAAoB,SAAS;AACjE,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,YAAM,SAAS,oBAAoB,SAAS,SAAS,KAAK,MAAM,IAAI,OAAO,SAAS,QAAQ;AAC5F,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,0BAA0B,OAAO,QAAQ,CAAC,IAAI,CAAC;AAAA,IACxF;AAAA,EACF;AACF;AAUO,SAAS,mBACd,SAC6C;AAC7C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,UAAI,OAAO,aAAa,UAAU;AAChC,eAAO,YAAY,QAAQ,sDAAsD;AAAA,MACnF;AACA,YAAM,CAAC,iBAAiB,iBAAiB,IAAI,MAAM,QAAQ,IAAI;AAAA,QAC7D,UAAU,QAAQ,OAAO,MAAM;AAAA,QAC/B,UAAU,QAAQ,OAAO,QAAQ;AAAA,MACnC,CAAC;AACD,YAAM,QAAQ,iBAAiB,gBAAgB,QAAQ,kBAAkB,MAAM;AAC/E,aAAO,SAAS,QAAQ,YACpB,YAAY,KAAK,KAAK,IACtB,YAAY,KAAK,OAAO,EAAE,SAAS,8BAA8B,QAAQ,SAAS,IAAI,CAAC;AAAA,IAC7F;AAAA,EACF;AACF;AAYO,SAAS,SACd,SACmD;AACnD,QAAM,YAAY,IAAI,iBAAiB,QAAQ,OAAO,QAAQ,MAAM,EACjE;AAAA,IACC,QAAQ,gBACN;AAAA,EACJ,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,WAAW,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACvF,eAAO,QAAQ,OAAO,QAAQ,IAAI,YAAY,KAAK,QAAQ,IAAI,YAAY,KAAK,QAAQ;AAAA,MAC1F,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;AAiBO,SAAS,SACd,SAC0D;AAC1D,QAAM,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAAI,QAAQ,SAAS,KAAK,IAAI,IAAI,QAAQ;AACzF,QAAM,YAAY,IAAI;AAAA,IACpB,QAAQ;AAAA,IACR,EAAE,OAAO;AAAA,MACP,OAAO,EAAE,OAAO;AAAA,MAChB,UAAU,EAAE,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,EACG;AAAA,IACC,QAAQ,gBACN;AAAA,EAAgD,QAAQ;AAAA;AAAA;AAAA,EAC5D,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,QAAQ,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACpF,YAAI,MAAM,QAAQ,KAAK,MAAM,QAAQ,GAAG;AACtC,iBAAO,YAAY,QAAQ,SAAS,MAAM,KAAK,gCAAgC;AAAA,YAC7E;AAAA,UACF,CAAC;AAAA,QACH;AACA,eAAO,MAAM,SAAS,QAAQ,YAC1B,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC,IACnD,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC;AAAA,MACzD,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;;;AIzKA,eAAsB,aACpB,SACmD;AACnD,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,UAAU,MAAM;AAAA,IACpB,QAAQ;AAAA,IACR,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,eAAe,CAAC,CAAC;AAAA,IAChD,CAAC,aAAa,YAAY,SAAS,QAAQ;AAAA,EAC7C;AACA,QAAM,SAAS,cAAc,OAAO;AACpC,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd;AAAA,IACA,GAAG;AAAA,IACH,YAAY,KAAK,IAAI,IAAI;AAAA,EAC3B;AACF;AAEA,eAAe,YACb,SACA,UACkD;AAClD,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,QAAQ,OAAO,SAAS,OAAO,QAAQ;AAAA,EACxD,SAAS,OAAO;AACd,kBAAc;AAAA,EAChB;AAEA,QAAM,UAA8B,CAAC;AACrC,aAAW,UAAU,QAAQ,SAAS;AACpC,UAAM,UACJ,gBAAgB,SACZ,MAAM,aAAa,QAAQ,MAAM,UAAU,QAAkB,MAAM,IACnE,YAAY,QAAQ,kBAAkB,aAAa,WAAW,CAAC,EAAE;AACvE,UAAM,iBAAiB,MAAM,cAAc;AAAA,MACzC,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW,QAAQ,aAAa,CAAC;AAAA,MACjC,qBAAqB,QAAQ,wBAAwB;AAAA,IACvD,CAAC;AACD,YAAQ,KAAK,EAAE,YAAY,OAAO,MAAM,SAAS,eAAe,CAAC;AAAA,EACnE;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,GAAI,WAAW,SAAY,CAAC,IAAI,EAAE,OAAO;AAAA,IACzC,GAAI,gBAAgB,SAAY,CAAC,IAAI,EAAE,YAAY;AAAA,IACnD;AAAA,EACF;AACF;AAEA,eAAe,aACb,WACA,UACA,QACA,QAC0B;AAC1B,MAAI;AACF,WAAO,MAAM,OAAO,SAAS,EAAE,WAAW,MAAM,UAAU,OAAO,CAAC;AAAA,EACpE,SAAS,OAAO;AACd,WAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,EAChD;AACF;AAEA,eAAe,cAAuC,MAS/B;AACrB,QAAM,SAAoB,CAAC;AAC3B,aAAW,YAAY,KAAK,WAAW;AACrC,QAAI;AACF,YAAM,SAAS,OAAO;AAAA,QACpB,WAAW,KAAK;AAAA,QAChB,MAAM,KAAK;AAAA,QACX,QAAQ,KAAK;AAAA,QACb,aAAa,KAAK;AAAA,QAClB,QAAQ,KAAK;AAAA,QACb,SAAS,KAAK;AAAA,MAChB,CAAC;AAAA,IACH,SAAS,OAAO;AACd,UAAI,KAAK,qBAAqB;AAC5B,cAAM;AAAA,MACR;AACA,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,cAAc,SAIrB;AACA,MAAI,SAAS;AACb,MAAI,SAAS;AACb,MAAI,UAAU;AACd,aAAW,UAAU,SAAS;AAC5B,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,UAAW,YAAW;AAAA,IACvD;AAAA,EACF;AACA,SAAO,EAAE,QAAQ,QAAQ,QAAQ;AACnC;","names":[]}
+{"version":3,"sources":["../../src/evals/agent-target.ts","../../src/evals/metrics.ts","../../src/evals/format.ts","../../src/evals/outcome.ts","../../src/evals/selectors.ts","../../src/evals/runner.ts"],"sourcesContent":["import type { Agent } from \"../agent/agent\";\nimport type { PromptResponse } from \"../agent/request\";\nimport type { Message } from \"../completion\";\nimport type { EvalCase, EvalTarget } from \"./types\";\n\nexport type AgentEvalTargetOptions<Input, Output = PromptResponse> = {\n  prompt?: ((input: Input, testCase: EvalCase<Input>) => string | Message) | undefined;\n  output?: ((response: PromptResponse, testCase: EvalCase<Input>) => Output) | undefined;\n};\n\nexport function agentEvalTarget<Input>(\n  agent: Agent,\n  options?: AgentEvalTargetOptions<Input, PromptResponse>,\n): EvalTarget<Input, PromptResponse>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output>,\n): EvalTarget<Input, Output>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output | PromptResponse> = {},\n): EvalTarget<Input, Output | PromptResponse> {\n  return async (input, testCase) => {\n    const prompt = options.prompt?.(input, testCase) ?? String(input);\n    const response = await agent.prompt(prompt).send();\n    return options.output === undefined ? response : options.output(response, testCase);\n  };\n}\n","import { z } from \"zod\";\nimport type { CompletionModel } from \"../completion\";\nimport { cosineSimilarity, type EmbeddingModel, embedText } from \"../embeddings\";\nimport { ExtractorBuilder } from \"../extractor\";\nimport type { ZodSchema } from \"../schema\";\nimport { errorMessage, formatValue, stableComparable } from \"./format\";\nimport { EvalOutcome } from \"./outcome\";\nimport { resolveActual, resolveActualText, resolveExpected, resolveJudgePrompt } from \"./selectors\";\nimport type { EvalMetric, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport type ExactMatchOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, unknown> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, unknown> | undefined;\n};\n\nexport function exactMatch<Input, Output, Expected = unknown>(\n  options: ExactMatchOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"exact_match\",\n    async evaluate(args) {\n      const actual = await resolveActual(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for exact match.\");\n      }\n      const passed = stableComparable(actual) === stableComparable(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });\n    },\n  };\n}\n\nexport type ContainsOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string | RegExp> | undefined;\n};\n\nexport function contains<Input, Output, Expected = unknown>(\n  options: ContainsOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"contains\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for contains.\");\n      }\n      if (typeof expected !== \"string\" && !(expected instanceof RegExp)) {\n        return EvalOutcome.invalid(\"Contains expected value must be a string or RegExp.\");\n      }\n      const passed = expected instanceof RegExp ? expected.test(actual) : actual.includes(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Output did not contain ${String(expected)}.` });\n    },\n  };\n}\n\nexport type SemanticSimilarityOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: EmbeddingModel;\n  threshold: number;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string> | undefined;\n};\n\nexport function semanticSimilarity<Input, Output, Expected = unknown>(\n  options: SemanticSimilarityOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, number, Expected> {\n  return {\n    name: options.name ?? \"semantic_similarity\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for semantic similarity.\");\n      }\n      if (typeof expected !== \"string\") {\n        return EvalOutcome.invalid(\"Semantic similarity expected value must be a string.\");\n      }\n      const [actualEmbedding, expectedEmbedding] = await Promise.all([\n        embedText(options.model, actual),\n        embedText(options.model, expected),\n      ]);\n      const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);\n      return score >= options.threshold\n        ? EvalOutcome.pass(score)\n        : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });\n    },\n  };\n}\n\nexport type LlmJudgeOptions<Input, Output, SchemaOutput, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  schema: ZodSchema<SchemaOutput>;\n  passes(value: SchemaOutput): boolean;\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmJudge<Input, Output, SchemaOutput, Expected = unknown>(\n  options: LlmJudgeOptions<Input, Output, SchemaOutput, Expected>,\n): EvalMetric<Input, Output, SchemaOutput, Expected> {\n  const extractor = new ExtractorBuilder(options.model, options.schema)\n    .instructions(\n      options.instructions ??\n        \"Judge the eval case by the requested schema. Submit the judgment using the schema.\",\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_judge\",\n    async evaluate(args) {\n      try {\n        const judgment = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        return options.passes(judgment) ? EvalOutcome.pass(judgment) : EvalOutcome.fail(judgment);\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n\nexport type LlmScoreMetricScore = {\n  score: number;\n  feedback: string;\n};\n\nexport type LlmScoreOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  threshold: number;\n  criteria: string | string[];\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmScore<Input, Output, Expected = unknown>(\n  options: LlmScoreOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, LlmScoreMetricScore, Expected> {\n  const criteria = Array.isArray(options.criteria) ? options.criteria.join(\"\\n\") : options.criteria;\n  const extractor = new ExtractorBuilder(\n    options.model,\n    z.object({\n      score: z.number(),\n      feedback: z.string(),\n    }),\n  )\n    .instructions(\n      options.instructions ??\n        `Score the eval case against these criteria:\\n${criteria}\\n\\nReturn a score between 0 and 1 and brief feedback.`,\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_score\",\n    async evaluate(args) {\n      try {\n        const score = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        if (score.score < 0 || score.score > 1) {\n          return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {\n            score,\n          });\n        }\n        return score.score >= options.threshold\n          ? EvalOutcome.pass(score, { comment: score.feedback })\n          : EvalOutcome.fail(score, { comment: score.feedback });\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n","export function defaultOutputValue(output: unknown): unknown {\n  if (\n    typeof output === \"object\" &&\n    output !== null &&\n    \"output\" in output &&\n    typeof (output as { output?: unknown }).output === \"string\"\n  ) {\n    return (output as { output: string }).output;\n  }\n  return output;\n}\n\nexport function stableComparable(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  return JSON.stringify(value);\n}\n\nexport function formatValue(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  try {\n    return JSON.stringify(value);\n  } catch {\n    return String(value);\n  }\n}\n\nexport function errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n","import type { EvalMetadata } from \"./types\";\n\nexport type EvalOutcome<Score = unknown> =\n  | {\n      outcome: \"pass\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"fail\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"invalid\";\n      reason: string;\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    };\n\nexport const EvalOutcome = {\n  pass<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    return {\n      outcome: \"pass\",\n      ...(score === undefined ? {} : { score }),\n      ...(options.comment === undefined ? {} : { comment: options.comment }),\n      ...(options.metadata === undefined ? {} : { metadata: options.metadata }),\n    };\n  },\n\n  fail<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    return {\n      outcome: \"fail\",\n      ...(score === undefined ? {} : { score }),\n      ...(options.comment === undefined ? {} : { comment: options.comment }),\n      ...(options.metadata === undefined ? {} : { metadata: options.metadata }),\n    };\n  },\n\n  invalid<Score = never>(\n    reason: string,\n    options: {\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    } = {},\n  ): EvalOutcome<Score> {\n    return {\n      outcome: \"invalid\",\n      reason,\n      ...(options.score === undefined ? {} : { score: options.score }),\n      ...(options.comment === undefined ? {} : { comment: options.comment }),\n      ...(options.metadata === undefined ? {} : { metadata: options.metadata }),\n    };\n  },\n};\n","import { defaultOutputValue, formatValue } from \"./format\";\nimport type { EvalMetricArgs, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport async function resolveActual<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, unknown> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<unknown> {\n  return selector === undefined ? defaultOutputValue(args.output) : selector(args);\n}\n\nexport async function resolveActualText<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  const value = selector === undefined ? defaultOutputValue(args.output) : await selector(args);\n  return typeof value === \"string\" ? value : JSON.stringify(value);\n}\n\nexport async function resolveExpected<Input, Output, Expected, Value>(\n  selectorOrValue: SelectorOrValue<Input, Output, Expected, Value> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<Value | Expected | undefined> {\n  if (selectorOrValue === undefined) {\n    return args.case.expected;\n  }\n  return typeof selectorOrValue === \"function\"\n    ? (selectorOrValue as ValueSelector<Input, Output, Expected, Value>)(args)\n    : selectorOrValue;\n}\n\nexport async function resolveJudgePrompt<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  if (selector !== undefined) {\n    return selector(args);\n  }\n  return [\n    `Suite: ${args.suiteName}`,\n    `Case: ${args.case.id}`,\n    `Input: ${formatValue(args.case.input)}`,\n    `Expected: ${formatValue(args.case.expected)}`,\n    `Output: ${formatValue(defaultOutputValue(args.output))}`,\n  ].join(\"\\n\\n\");\n}\n","import { mapWithConcurrency } from \"../internal/concurrency\";\nimport { errorMessage } from \"./format\";\nimport { EvalOutcome, type EvalOutcome as EvalOutcomeType } from \"./outcome\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalMetric,\n  EvalMetricResult,\n  EvalReporter,\n  EvalSuiteResult,\n  RunEvalSuiteOptions,\n} from \"./types\";\n\nexport async function runEvalSuite<Input, Output, Expected = unknown>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n): Promise<EvalSuiteResult<Input, Output, Expected>> {\n  const startedAt = Date.now();\n  const results = await mapWithConcurrency(\n    options.cases,\n    Math.max(1, Math.trunc(options.concurrency ?? 1)),\n    (testCase) => runEvalCase(options, testCase),\n  );\n  const counts = countOutcomes(results);\n  return {\n    name: options.name,\n    results,\n    ...counts,\n    durationMs: Date.now() - startedAt,\n  };\n}\n\nasync function runEvalCase<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n  testCase: EvalCase<Input, Expected>,\n): Promise<EvalCaseResult<Input, Output, Expected>> {\n  let output: Output | undefined;\n  let targetError: unknown;\n  try {\n    output = await options.target(testCase.input, testCase);\n  } catch (error) {\n    targetError = error;\n  }\n\n  const metrics: EvalMetricResult[] = [];\n  for (const metric of options.metrics) {\n    const outcome =\n      targetError === undefined\n        ? await safeEvaluate(options.name, testCase, output as Output, metric)\n        : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`);\n    const reporterErrors = await reportOutcome({\n      suiteName: options.name,\n      testCase,\n      output,\n      targetError,\n      metric,\n      outcome,\n      reporters: options.reporters ?? [],\n      failOnReporterError: options.failOnReporterError === true,\n    });\n    metrics.push({ metricName: metric.name, outcome, reporterErrors });\n  }\n\n  return {\n    case: testCase,\n    ...(output === undefined ? {} : { output }),\n    ...(targetError === undefined ? {} : { targetError }),\n    metrics,\n  };\n}\n\nasync function safeEvaluate<Input, Output, Expected>(\n  suiteName: string,\n  testCase: EvalCase<Input, Expected>,\n  output: Output,\n  metric: EvalMetric<Input, Output, unknown, Expected>,\n): Promise<EvalOutcomeType> {\n  try {\n    return await metric.evaluate({ suiteName, case: testCase, output });\n  } catch (error) {\n    return EvalOutcome.invalid(errorMessage(error));\n  }\n}\n\nasync function reportOutcome<Input, Output, Expected>(args: {\n  suiteName: string;\n  testCase: EvalCase<Input, Expected>;\n  output: Output | undefined;\n  targetError: unknown;\n  metric: EvalMetric<Input, Output, unknown, Expected>;\n  outcome: EvalOutcomeType;\n  reporters: Array<EvalReporter<Input, Output, Expected>>;\n  failOnReporterError: boolean;\n}): Promise<unknown[]> {\n  const errors: unknown[] = [];\n  for (const reporter of args.reporters) {\n    try {\n      await reporter.report({\n        suiteName: args.suiteName,\n        case: args.testCase,\n        output: args.output,\n        targetError: args.targetError,\n        metric: args.metric,\n        outcome: args.outcome,\n      });\n    } catch (error) {\n      if (args.failOnReporterError) {\n        throw error;\n      }\n      errors.push(error);\n    }\n  }\n  return errors;\n}\n\nfunction countOutcomes(results: Array<EvalCaseResult<unknown, unknown, unknown>>): {\n  passed: number;\n  failed: number;\n  invalid: number;\n} {\n  let passed = 0;\n  let failed = 0;\n  let invalid = 0;\n  for (const result of results) {\n    for (const metric of result.metrics) {\n      if (metric.outcome.outcome === \"pass\") passed += 1;\n      if (metric.outcome.outcome === \"fail\") failed += 1;\n      if (metric.outcome.outcome === \"invalid\") invalid += 1;\n    }\n  }\n  return { passed, failed, invalid };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAkBO,SAAS,gBACd,OACA,UAAkE,CAAC,GACvB;AAC5C,SAAO,OAAO,OAAO,aAAa;AAChC,UAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,KAAK,OAAO,KAAK;AAChE,UAAM,WAAW,MAAM,MAAM,OAAO,MAAM,EAAE,KAAK;AACjD,WAAO,QAAQ,WAAW,SAAY,WAAW,QAAQ,OAAO,UAAU,QAAQ;AAAA,EACpF;AACF;;;AC3BA,SAAS,SAAS;;;ACAX,SAAS,mBAAmB,QAA0B;AAC3D,MACE,OAAO,WAAW,YAClB,WAAW,QACX,YAAY,UACZ,OAAQ,OAAgC,WAAW,UACnD;AACA,WAAQ,OAA8B;AAAA,EACxC;AACA,SAAO;AACT;AAEO,SAAS,iBAAiB,OAAwB;AACvD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,SAAO,KAAK,UAAU,KAAK;AAC7B;AAEO,SAAS,YAAY,OAAwB;AAClD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,MAAI;AACF,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B,QAAQ;AACN,WAAO,OAAO,KAAK;AAAA,EACrB;AACF;AAEO,SAAS,aAAa,OAAwB;AACnD,SAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC9D;;;ACTO,IAAM,cAAc;AAAA,EACzB,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,WAAO;AAAA,MACL,SAAS;AAAA,MACT,GAAI,UAAU,SAAY,CAAC,IAAI,EAAE,MAAM;AAAA,MACvC,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,MACpE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AAAA,EAEA,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,WAAO;AAAA,MACL,SAAS;AAAA,MACT,GAAI,UAAU,SAAY,CAAC,IAAI,EAAE,MAAM;AAAA,MACvC,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,MACpE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AAAA,EAEA,QACE,QACA,UAII,CAAC,GACe;AACpB,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,MACA,GAAI,QAAQ,UAAU,SAAY,CAAC,IAAI,EAAE,OAAO,QAAQ,MAAM;AAAA,MAC9D,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,MACpE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AACF;;;AC7DA,eAAsB,cACpB,UACA,MACkB;AAClB,SAAO,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,SAAS,IAAI;AACjF;AAEA,eAAsB,kBACpB,UACA,MACiB;AACjB,QAAM,QAAQ,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,MAAM,SAAS,IAAI;AAC5F,SAAO,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK;AACjE;AAEA,eAAsB,gBACpB,iBACA,MACuC;AACvC,MAAI,oBAAoB,QAAW;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AACA,SAAO,OAAO,oBAAoB,aAC7B,gBAAkE,IAAI,IACvE;AACN;AAEA,eAAsB,mBACpB,UACA,MACiB;AACjB,MAAI,aAAa,QAAW;AAC1B,WAAO,SAAS,IAAI;AAAA,EACtB;AACA,SAAO;AAAA,IACL,UAAU,KAAK,SAAS;AAAA,IACxB,SAAS,KAAK,KAAK,EAAE;AAAA,IACrB,UAAU,YAAY,KAAK,KAAK,KAAK,CAAC;AAAA,IACtC,aAAa,YAAY,KAAK,KAAK,QAAQ,CAAC;AAAA,IAC5C,WAAW,YAAY,mBAAmB,KAAK,MAAM,CAAC,CAAC;AAAA,EACzD,EAAE,KAAK,MAAM;AACf;;;AH5BO,SAAS,WACd,UAAsD,CAAC,GACT;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,cAAc,QAAQ,QAAQ,IAAI;AACvD,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,6CAA6C;AAAA,MAC1E;AACA,YAAM,SAAS,iBAAiB,MAAM,MAAM,iBAAiB,QAAQ;AACrE,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,YAAY,YAAY,QAAQ,CAAC,IAAI,CAAC;AAAA,IAC/E;AAAA,EACF;AACF;AAQO,SAAS,SACd,UAAoD,CAAC,GACP;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,0CAA0C;AAAA,MACvE;AACA,UAAI,OAAO,aAAa,YAAY,EAAE,oBAAoB,SAAS;AACjE,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,YAAM,SAAS,oBAAoB,SAAS,SAAS,KAAK,MAAM,IAAI,OAAO,SAAS,QAAQ;AAC5F,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,0BAA0B,OAAO,QAAQ,CAAC,IAAI,CAAC;AAAA,IACxF;AAAA,EACF;AACF;AAUO,SAAS,mBACd,SAC6C;AAC7C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,UAAI,OAAO,aAAa,UAAU;AAChC,eAAO,YAAY,QAAQ,sDAAsD;AAAA,MACnF;AACA,YAAM,CAAC,iBAAiB,iBAAiB,IAAI,MAAM,QAAQ,IAAI;AAAA,QAC7D,UAAU,QAAQ,OAAO,MAAM;AAAA,QAC/B,UAAU,QAAQ,OAAO,QAAQ;AAAA,MACnC,CAAC;AACD,YAAM,QAAQ,iBAAiB,gBAAgB,QAAQ,kBAAkB,MAAM;AAC/E,aAAO,SAAS,QAAQ,YACpB,YAAY,KAAK,KAAK,IACtB,YAAY,KAAK,OAAO,EAAE,SAAS,8BAA8B,QAAQ,SAAS,IAAI,CAAC;AAAA,IAC7F;AAAA,EACF;AACF;AAYO,SAAS,SACd,SACmD;AACnD,QAAM,YAAY,IAAI,iBAAiB,QAAQ,OAAO,QAAQ,MAAM,EACjE;AAAA,IACC,QAAQ,gBACN;AAAA,EACJ,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,WAAW,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACvF,eAAO,QAAQ,OAAO,QAAQ,IAAI,YAAY,KAAK,QAAQ,IAAI,YAAY,KAAK,QAAQ;AAAA,MAC1F,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;AAiBO,SAAS,SACd,SAC0D;AAC1D,QAAM,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAAI,QAAQ,SAAS,KAAK,IAAI,IAAI,QAAQ;AACzF,QAAM,YAAY,IAAI;AAAA,IACpB,QAAQ;AAAA,IACR,EAAE,OAAO;AAAA,MACP,OAAO,EAAE,OAAO;AAAA,MAChB,UAAU,EAAE,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,EACG;AAAA,IACC,QAAQ,gBACN;AAAA,EAAgD,QAAQ;AAAA;AAAA;AAAA,EAC5D,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,QAAQ,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACpF,YAAI,MAAM,QAAQ,KAAK,MAAM,QAAQ,GAAG;AACtC,iBAAO,YAAY,QAAQ,SAAS,MAAM,KAAK,gCAAgC;AAAA,YAC7E;AAAA,UACF,CAAC;AAAA,QACH;AACA,eAAO,MAAM,SAAS,QAAQ,YAC1B,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC,IACnD,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC;AAAA,MACzD,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;;;AIzKA,eAAsB,aACpB,SACmD;AACnD,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,UAAU,MAAM;AAAA,IACpB,QAAQ;AAAA,IACR,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,eAAe,CAAC,CAAC;AAAA,IAChD,CAAC,aAAa,YAAY,SAAS,QAAQ;AAAA,EAC7C;AACA,QAAM,SAAS,cAAc,OAAO;AACpC,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd;AAAA,IACA,GAAG;AAAA,IACH,YAAY,KAAK,IAAI,IAAI;AAAA,EAC3B;AACF;AAEA,eAAe,YACb,SACA,UACkD;AAClD,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,QAAQ,OAAO,SAAS,OAAO,QAAQ;AAAA,EACxD,SAAS,OAAO;AACd,kBAAc;AAAA,EAChB;AAEA,QAAM,UAA8B,CAAC;AACrC,aAAW,UAAU,QAAQ,SAAS;AACpC,UAAM,UACJ,gBAAgB,SACZ,MAAM,aAAa,QAAQ,MAAM,UAAU,QAAkB,MAAM,IACnE,YAAY,QAAQ,kBAAkB,aAAa,WAAW,CAAC,EAAE;AACvE,UAAM,iBAAiB,MAAM,cAAc;AAAA,MACzC,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW,QAAQ,aAAa,CAAC;AAAA,MACjC,qBAAqB,QAAQ,wBAAwB;AAAA,IACvD,CAAC;AACD,YAAQ,KAAK,EAAE,YAAY,OAAO,MAAM,SAAS,eAAe,CAAC;AAAA,EACnE;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,GAAI,WAAW,SAAY,CAAC,IAAI,EAAE,OAAO;AAAA,IACzC,GAAI,gBAAgB,SAAY,CAAC,IAAI,EAAE,YAAY;AAAA,IACnD;AAAA,EACF;AACF;AAEA,eAAe,aACb,WACA,UACA,QACA,QAC0B;AAC1B,MAAI;AACF,WAAO,MAAM,OAAO,SAAS,EAAE,WAAW,MAAM,UAAU,OAAO,CAAC;AAAA,EACpE,SAAS,OAAO;AACd,WAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,EAChD;AACF;AAEA,eAAe,cAAuC,MAS/B;AACrB,QAAM,SAAoB,CAAC;AAC3B,aAAW,YAAY,KAAK,WAAW;AACrC,QAAI;AACF,YAAM,SAAS,OAAO;AAAA,QACpB,WAAW,KAAK;AAAA,QAChB,MAAM,KAAK;AAAA,QACX,QAAQ,KAAK;AAAA,QACb,aAAa,KAAK;AAAA,QAClB,QAAQ,KAAK;AAAA,QACb,SAAS,KAAK;AAAA,MAChB,CAAC;AAAA,IACH,SAAS,OAAO;AACd,UAAI,KAAK,qBAAqB;AAC5B,cAAM;AAAA,MACR;AACA,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,cAAc,SAIrB;AACA,MAAI,SAAS;AACb,MAAI,SAAS;AACb,MAAI,UAAU;AACd,aAAW,UAAU,SAAS;AAC5B,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,UAAW,YAAW;AAAA,IACvD;AAAA,EACF;AACA,SAAO,EAAE,QAAQ,QAAQ,QAAQ;AACnC;","names":[]}

package/dist/extractor/index.d.ts

@@ -1,10 +1,10 @@
-import { e as Agent } from '../agent-CWGuTdMJ.js';
-import { U as Usage, M as Message, C as CompletionModel, e as JsonValue, m as ToolChoice } from '../types-C54aNoCd.js';
+import { e as Agent } from '../agent-BURoB-me.js';
 import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
+import { U as Usage, M as Message, C as CompletionModel, e as JsonValue, m as ToolChoice } from '../types-Dmoq-Fhz.js';
 import '../memory/index.js';
 import '../observability/index.js';
-import '../tool-C3ciF-VG.js';
-import '../middleware-qfm1EFKK.js';
+import '../tool-Dz-lHZon.js';
+import '../middleware-C9QIms85.js';
 import '../types-IB2e9u5M.js';
 import '../vector-store/index.js';
 import 'zod';

package/dist/extractor/index.js

@@ -2,15 +2,16 @@ import {
   ExtractionError,
   Extractor,
   ExtractorBuilder
-} from "../chunk-Z7HY4WU7.js";
-import "../chunk-365W6FJD.js";
-import "../chunk-ORLEOXD7.js";
+} from "../chunk-OXYZDHNI.js";
+import "../chunk-6NWNL2KM.js";
+import "../chunk-IXK3TEHJ.js";
 import "../chunk-XUUY2L2D.js";
 import "../chunk-XXT2UCAR.js";
 import "../chunk-YK4WAAS4.js";
-import "../chunk-S7EOE6EC.js";
-import "../chunk-P425B6GR.js";
-import "../chunk-6GJDBBDC.js";
+import "../chunk-AER5FGPK.js";
+import "../chunk-I47EHI45.js";
+import "../chunk-A4GUJSEK.js";
+import "../chunk-WQKHFADH.js";
 import "../chunk-EFGX3EX5.js";
 import "../chunk-OIMLU4SF.js";
 export {

package/dist/image-generation/index.d.ts

@@ -1,4 +1,4 @@
-import { e as JsonValue } from '../types-C54aNoCd.js';
+import { e as JsonValue } from '../types-Dmoq-Fhz.js';
 
 type ImageGenerationRequest = {
     prompt: string;

package/dist/index.d.ts

@@ -1,14 +1,15 @@
 export { AgentBuilder, MaxTurnsError, PromptCancelledError } from './agent/index.js';
-export { A as AgentChildStreamEvent, a as AgentStreamEvent, P as PromptResponse, c as cancelPrompt, b as createHook, r as requestToolApproval, d as runControl, s as skipTool, t as toolCallControl } from './agent-CWGuTdMJ.js';
-export { A as AssistantContent, a as AssistantMessage, C as CompletionModel, b as CompletionRequest, c as CompletionResponse, D as Document, I as ImageContent, J as JsonObject, d as JsonPrimitive, e as JsonValue, M as Message, S as SystemMessage, T as Text, f as ToolCall, g as ToolDefinition, h as ToolMessage, i as ToolResult, j as ToolResultContent, U as Usage, k as UserContent, l as UserMessage } from './types-C54aNoCd.js';
+export { A as AgentChildStreamEvent, a as AgentStreamEvent, P as PromptResponse, c as cancelPrompt, b as createHook, r as requestToolApproval, d as runControl, s as skipTool, t as toolCallControl } from './agent-BURoB-me.js';
+export { C as CreateCompletionBaseOptions, a as CreateCompletionInput, b as CreateCompletionOptions, c as CreateCompletionResult, d as CreateCompletionStreamOptions, e as CreateParsedCompletionOptions, f as CreateParsedCompletionResult, g as createCompletion, h as createCompletionStream, i as createParsedCompletion } from './create-completion-Bog3ni7b.js';
+export { A as AssistantContent, a as AssistantMessage, C as CompletionModel, b as CompletionRequest, c as CompletionResponse, D as Document, I as ImageContent, J as JsonObject, d as JsonPrimitive, e as JsonValue, M as Message, S as SystemMessage, T as Text, f as ToolCall, g as ToolDefinition, h as ToolMessage, i as ToolResult, j as ToolResultContent, U as Usage, k as UserContent, l as UserMessage } from './types-Dmoq-Fhz.js';
 export { MemoryStore } from './memory/index.js';
 export { Z as ZodSchema } from './zod-schema-C7F4clpm.js';
 export { loadSkills, skill } from './skills/index.js';
-export { S as SkillValidationError } from './types-T9rlOIUc.js';
-export { C as CreateToolOptions, c as createThinkTool, a as createTool } from './think-tool-ByArsrxe.js';
-export { A as AnyTool, T as Tool, a as ToolApprovalContext, b as ToolApprovalPolicy, c as ToolCallContext, d as ToolCallStreamEvent } from './tool-C3ciF-VG.js';
-export { A as AgentMiddleware, C as CompletionRequestMiddlewareArgs, a as CompletionRequestMiddlewareResult, b as CompletionResponseMiddlewareArgs, c as CompletionResponseMiddlewareResult, T as ToolInputMiddlewareArgs, d as ToolInputMiddlewareResult, e as ToolMiddleware, f as ToolOutputMiddlewareArgs, g as ToolOutputMiddlewareResult, h as ToolResultMiddlewareArgs, i as createMiddleware, j as createToolMiddleware } from './middleware-qfm1EFKK.js';
-import './types-UhcgW8Fo.js';
+export { S as SkillValidationError } from './types-DUivppau.js';
+export { C as CreateToolOptions, c as createThinkTool, a as createTool } from './think-tool-DLTCqqJ6.js';
+export { A as AnyTool, T as Tool, a as ToolApprovalContext, b as ToolApprovalPolicy, c as ToolCallContext, d as ToolCallStreamEvent } from './tool-Dz-lHZon.js';
+export { A as AgentMiddleware, C as CompletionRequestMiddlewareArgs, a as CompletionRequestMiddlewareResult, b as CompletionResponseMiddlewareArgs, c as CompletionResponseMiddlewareResult, T as ToolInputMiddlewareArgs, d as ToolInputMiddlewareResult, e as ToolMiddleware, f as ToolOutputMiddlewareArgs, g as ToolOutputMiddlewareResult, h as ToolResultMiddlewareArgs, i as createMiddleware, j as createToolMiddleware } from './middleware-C9QIms85.js';
+import './types-BfHZz1ZG.js';
 import '@modelcontextprotocol/sdk/client/sse.js';
 import '@modelcontextprotocol/sdk/client/stdio.js';
 import '@modelcontextprotocol/sdk/client/streamableHttp.js';

package/dist/index.js

@@ -2,10 +2,10 @@ import {
   SkillValidationError,
   loadSkills,
   skill
-} from "./chunk-DUP7FMAF.js";
+} from "./chunk-SHHSMV7O.js";
 import {
   AgentBuilder
-} from "./chunk-365W6FJD.js";
+} from "./chunk-6NWNL2KM.js";
 import {
   MaxTurnsError,
   PromptCancelledError,
@@ -15,7 +15,7 @@ import {
   runControl,
   skipTool,
   toolCallControl
-} from "./chunk-ORLEOXD7.js";
+} from "./chunk-IXK3TEHJ.js";
 import "./chunk-XUUY2L2D.js";
 import "./chunk-XXT2UCAR.js";
 import "./chunk-YK4WAAS4.js";
@@ -23,16 +23,20 @@ import {
   createMiddleware,
   createThinkTool,
   createToolMiddleware
-} from "./chunk-S7EOE6EC.js";
+} from "./chunk-AER5FGPK.js";
 import {
   createTool
-} from "./chunk-P425B6GR.js";
+} from "./chunk-I47EHI45.js";
 import {
   AssistantContent,
   Message,
   Usage,
-  UserContent
-} from "./chunk-6GJDBBDC.js";
+  UserContent,
+  createCompletion,
+  createCompletionStream,
+  createParsedCompletion
+} from "./chunk-A4GUJSEK.js";
+import "./chunk-WQKHFADH.js";
 import "./chunk-EFGX3EX5.js";
 import "./chunk-OIMLU4SF.js";
 export {
@@ -45,8 +49,11 @@ export {
   Usage,
   UserContent,
   cancelPrompt,
+  createCompletion,
+  createCompletionStream,
   createHook,
   createMiddleware,
+  createParsedCompletion,
   createThinkTool,
   createTool,
   createToolMiddleware,

package/dist/internal/agent.d.ts

@@ -1,8 +1,8 @@
-export { e as Agent, j as AgentEventAppendInput, k as AgentEventRecord, h as AgentEventStore, l as AgentEventStoreInclude, i as AgentEventStoreOptions, I as AgentEventStoreRegistration, J as AgentOptions, K as AgentSession, L as AgentToolOptions, M as DEFAULT_MAX_TURNS, D as DynamicContextOptions, N as DynamicContextRegistration, f as DynamicToolOptions, O as DynamicToolRegistration } from '../agent-CWGuTdMJ.js';
-import '../types-C54aNoCd.js';
+export { e as Agent, j as AgentEventAppendInput, k as AgentEventRecord, h as AgentEventStore, l as AgentEventStoreInclude, i as AgentEventStoreOptions, I as AgentEventStoreRegistration, J as AgentOptions, K as AgentSession, L as AgentToolOptions, M as DEFAULT_MAX_TURNS, D as DynamicContextOptions, N as DynamicContextRegistration, f as DynamicToolOptions, O as DynamicToolRegistration } from '../agent-BURoB-me.js';
+import '../types-Dmoq-Fhz.js';
 import '../memory/index.js';
 import '../observability/index.js';
-import '../tool-C3ciF-VG.js';
-import '../middleware-qfm1EFKK.js';
+import '../tool-Dz-lHZon.js';
+import '../middleware-C9QIms85.js';
 import '../types-IB2e9u5M.js';
 import '../vector-store/index.js';

package/dist/internal/agent.js

@@ -2,12 +2,13 @@ import {
   Agent,
   AgentSession,
   DEFAULT_MAX_TURNS
-} from "../chunk-ORLEOXD7.js";
+} from "../chunk-IXK3TEHJ.js";
 import "../chunk-XUUY2L2D.js";
 import "../chunk-YK4WAAS4.js";
-import "../chunk-S7EOE6EC.js";
-import "../chunk-P425B6GR.js";
-import "../chunk-6GJDBBDC.js";
+import "../chunk-AER5FGPK.js";
+import "../chunk-I47EHI45.js";
+import "../chunk-A4GUJSEK.js";
+import "../chunk-WQKHFADH.js";
 import "../chunk-EFGX3EX5.js";
 import "../chunk-OIMLU4SF.js";
 export {

package/dist/loaders/index.d.ts

@@ -1,4 +1,4 @@
-import { D as Document } from '../types-C54aNoCd.js';
+import { D as Document } from '../types-Dmoq-Fhz.js';
 
 type LoaderResult<T> = {
     ok: true;

package/dist/mcp/index.d.ts

@@ -1,10 +1,10 @@
-import { M as McpConnection, a as McpServer, b as McpStdioOptions, c as McpHttpOptions, d as McpSseOptions } from '../types-UhcgW8Fo.js';
-export { e as McpClient, f as McpToolCallContent, g as McpToolCallResult, h as McpToolDefinition } from '../types-UhcgW8Fo.js';
+import { M as McpConnection, a as McpServer, b as McpStdioOptions, c as McpHttpOptions, d as McpSseOptions } from '../types-BfHZz1ZG.js';
+export { e as McpClient, f as McpToolCallContent, g as McpToolCallResult, h as McpToolDefinition } from '../types-BfHZz1ZG.js';
 import '@modelcontextprotocol/sdk/client/sse.js';
 import '@modelcontextprotocol/sdk/client/stdio.js';
 import '@modelcontextprotocol/sdk/client/streamableHttp.js';
-import '../types-C54aNoCd.js';
-import '../tool-C3ciF-VG.js';
+import '../types-Dmoq-Fhz.js';
+import '../tool-Dz-lHZon.js';
 
 declare function connectMcp(connection: McpConnection): Promise<McpServer>;
 

package/dist/memory/index.d.ts

@@ -1,4 +1,4 @@
-import { J as JsonObject, M as Message } from '../types-C54aNoCd.js';
+import { J as JsonObject, M as Message } from '../types-Dmoq-Fhz.js';
 
 type MemorySavePolicy = "message" | "turn" | "run";
 type MemoryContext = {

package/dist/{middleware-qfm1EFKK.d.ts → middleware-C9QIms85.d.ts}

@@ -1,7 +1,7 @@
-import { g as ToolDefinition, b as CompletionRequest, c as CompletionResponse, e as JsonValue, j as ToolResultContent } from './types-C54aNoCd.js';
+import { g as ToolDefinition, b as CompletionRequest, c as CompletionResponse, e as JsonValue, j as ToolResultContent } from './types-Dmoq-Fhz.js';
 import { V as VectorMetadata, E as EmbeddingModel, b as EmbeddedDocument } from './types-IB2e9u5M.js';
 import { VectorSearchIndex } from './vector-store/index.js';
-import { A as AnyTool, c as ToolCallContext, N as NormalizedToolOutput } from './tool-C3ciF-VG.js';
+import { A as AnyTool, c as ToolCallContext, N as NormalizedToolOutput } from './tool-Dz-lHZon.js';
 
 declare class ToolSet {
     private readonly tools;

package/dist/observability/index.d.ts

@@ -1,5 +1,5 @@
-import { M as Message, b as CompletionRequest, J as JsonObject, o as CompletionModelCapabilities, c as CompletionResponse, f as ToolCall, g as ToolDefinition, j as ToolResultContent, U as Usage } from '../types-C54aNoCd.js';
-import { d as ToolCallStreamEvent } from '../tool-C3ciF-VG.js';
+import { M as Message, b as CompletionRequest, J as JsonObject, q as CompletionModelCapabilities, c as CompletionResponse, f as ToolCall, g as ToolDefinition, j as ToolResultContent, U as Usage } from '../types-Dmoq-Fhz.js';
+import { d as ToolCallStreamEvent } from '../tool-Dz-lHZon.js';
 
 type AgentTraceInfo = {
     traceId?: string | undefined;

package/dist/pipeline/index.d.ts

@@ -1,10 +1,10 @@
-import { e as Agent } from '../agent-CWGuTdMJ.js';
-import { J as JsonObject, C as CompletionModel } from '../types-C54aNoCd.js';
+import { e as Agent } from '../agent-BURoB-me.js';
+import { J as JsonObject, C as CompletionModel } from '../types-Dmoq-Fhz.js';
 import { Extractor } from '../extractor/index.js';
 import '../memory/index.js';
 import '../observability/index.js';
-import '../tool-C3ciF-VG.js';
-import '../middleware-qfm1EFKK.js';
+import '../tool-Dz-lHZon.js';
+import '../middleware-C9QIms85.js';
 import '../types-IB2e9u5M.js';
 import '../vector-store/index.js';
 import '../zod-schema-C7F4clpm.js';

package/dist/skills/index.d.ts

@@ -1,7 +1,7 @@
-import { b as SkillLoader, a as SkillSet } from '../types-T9rlOIUc.js';
-export { c as Skill, S as SkillValidationError, d as SkillValidationIssue } from '../types-T9rlOIUc.js';
-import '../tool-C3ciF-VG.js';
-import '../types-C54aNoCd.js';
+import { b as SkillLoader, a as SkillSet } from '../types-DUivppau.js';
+export { c as Skill, S as SkillValidationError, d as SkillValidationIssue } from '../types-DUivppau.js';
+import '../tool-Dz-lHZon.js';
+import '../types-Dmoq-Fhz.js';
 
 declare function loadSkills(loaders: SkillLoader | SkillLoader[]): Promise<SkillSet>;
 

package/dist/skills/index.js

@@ -2,10 +2,11 @@ import {
   SkillValidationError,
   loadSkills,
   skill
-} from "../chunk-DUP7FMAF.js";
+} from "../chunk-SHHSMV7O.js";
 import "../chunk-YK4WAAS4.js";
-import "../chunk-S7EOE6EC.js";
-import "../chunk-P425B6GR.js";
+import "../chunk-AER5FGPK.js";
+import "../chunk-I47EHI45.js";
+import "../chunk-WQKHFADH.js";
 import "../chunk-EFGX3EX5.js";
 import "../chunk-OIMLU4SF.js";
 export {

package/dist/{think-tool-ByArsrxe.d.ts → think-tool-DLTCqqJ6.d.ts}

@@ -1,6 +1,6 @@
 import { z } from 'zod';
 import { Z as ZodSchema } from './zod-schema-C7F4clpm.js';
-import { b as ToolApprovalPolicy, c as ToolCallContext, T as Tool } from './tool-C3ciF-VG.js';
+import { b as ToolApprovalPolicy, c as ToolCallContext, T as Tool } from './tool-Dz-lHZon.js';
 
 type CreateToolOptions<InputSchema extends ZodSchema, OutputSchema extends ZodSchema | undefined = undefined, Output = unknown> = {
     name: string;

package/dist/tool/index.d.ts

@@ -1,9 +1,9 @@
-export { b as CreateThinkToolOptions, C as CreateToolOptions, c as createThinkTool, a as createTool } from '../think-tool-ByArsrxe.js';
-export { A as AgentMiddleware, C as CompletionRequestMiddlewareArgs, a as CompletionRequestMiddlewareResult, b as CompletionResponseMiddlewareArgs, c as CompletionResponseMiddlewareResult, D as DynamicToolIndex, E as EmbedToolsOptions, T as ToolInputMiddlewareArgs, d as ToolInputMiddlewareResult, e as ToolMiddleware, f as ToolOutputMiddlewareArgs, g as ToolOutputMiddlewareResult, h as ToolResultMiddlewareArgs, k as ToolSearchDocument, l as ToolSet, i as createMiddleware, m as createToolIndex, j as createToolMiddleware, n as embedTools, o as isDynamicToolIndex } from '../middleware-qfm1EFKK.js';
-export { A as AnyTool, N as NormalizedToolOutput, T as Tool, a as ToolApprovalContext, b as ToolApprovalPolicy, e as ToolApprovalRunContext, c as ToolCallContext, d as ToolCallStreamEvent, f as ToolOutput, i as isToolResultContentArray, n as normalizeToolResultOutput, p as parseToolArgs, s as serializeToolOutput, t as toolResultContentToText } from '../tool-C3ciF-VG.js';
+export { b as CreateThinkToolOptions, C as CreateToolOptions, c as createThinkTool, a as createTool } from '../think-tool-DLTCqqJ6.js';
+export { A as AgentMiddleware, C as CompletionRequestMiddlewareArgs, a as CompletionRequestMiddlewareResult, b as CompletionResponseMiddlewareArgs, c as CompletionResponseMiddlewareResult, D as DynamicToolIndex, E as EmbedToolsOptions, T as ToolInputMiddlewareArgs, d as ToolInputMiddlewareResult, e as ToolMiddleware, f as ToolOutputMiddlewareArgs, g as ToolOutputMiddlewareResult, h as ToolResultMiddlewareArgs, k as ToolSearchDocument, l as ToolSet, i as createMiddleware, m as createToolIndex, j as createToolMiddleware, n as embedTools, o as isDynamicToolIndex } from '../middleware-C9QIms85.js';
+export { A as AnyTool, N as NormalizedToolOutput, T as Tool, a as ToolApprovalContext, b as ToolApprovalPolicy, e as ToolApprovalRunContext, c as ToolCallContext, d as ToolCallStreamEvent, f as ToolOutput, i as isToolResultContentArray, n as normalizeToolResultOutput, p as parseToolArgs, s as serializeToolOutput, t as toolResultContentToText } from '../tool-Dz-lHZon.js';
 import 'zod';
 import '../zod-schema-C7F4clpm.js';
-import '../types-C54aNoCd.js';
+import '../types-Dmoq-Fhz.js';
 import '../types-IB2e9u5M.js';
 import '../vector-store/index.js';
 

package/dist/tool/index.js

@@ -15,10 +15,11 @@ import {
   parseToolArgs,
   serializeToolOutput,
   toolResultContentToText
-} from "../chunk-S7EOE6EC.js";
+} from "../chunk-AER5FGPK.js";
 import {
   createTool
-} from "../chunk-P425B6GR.js";
+} from "../chunk-I47EHI45.js";
+import "../chunk-WQKHFADH.js";
 import "../chunk-EFGX3EX5.js";
 import "../chunk-OIMLU4SF.js";
 export {

package/dist/{tool-C3ciF-VG.d.ts → tool-Dz-lHZon.d.ts}

@@ -1,4 +1,4 @@
-import { J as JsonObject, g as ToolDefinition, j as ToolResultContent, e as JsonValue } from './types-C54aNoCd.js';
+import { J as JsonObject, g as ToolDefinition, j as ToolResultContent, e as JsonValue } from './types-Dmoq-Fhz.js';
 
 type ToolApprovalRunContext = {
     agentId: string;

package/dist/transcription/index.d.ts

@@ -1,4 +1,4 @@
-import { e as JsonValue } from '../types-C54aNoCd.js';
+import { e as JsonValue } from '../types-Dmoq-Fhz.js';
 
 type TranscriptionRequest = {
     data: Uint8Array;

package/dist/{types-UhcgW8Fo.d.ts → types-BfHZz1ZG.d.ts}

@@ -1,8 +1,8 @@
 import { SSEClientTransportOptions } from '@modelcontextprotocol/sdk/client/sse.js';
 import { StdioServerParameters } from '@modelcontextprotocol/sdk/client/stdio.js';
 import { StreamableHTTPClientTransportOptions } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
-import { J as JsonObject } from './types-C54aNoCd.js';
-import { A as AnyTool } from './tool-C3ciF-VG.js';
+import { J as JsonObject } from './types-Dmoq-Fhz.js';
+import { A as AnyTool } from './tool-Dz-lHZon.js';
 
 type McpToolDefinition = {
     name: string;

package/dist/{types-T9rlOIUc.d.ts → types-DUivppau.d.ts}

@@ -1,4 +1,4 @@
-import { A as AnyTool } from './tool-C3ciF-VG.js';
+import { A as AnyTool } from './tool-Dz-lHZon.js';
 
 type Skill = {
     readonly name: string;

package/dist/{types-C54aNoCd.d.ts → types-Dmoq-Fhz.d.ts}

@@ -250,4 +250,4 @@ declare function assertCompletionRequestSupported(model: CompletionModel, reques
 }): void;
 declare function textFromAssistantContent(content: AssistantContent[]): string;
 
-export { AssistantContent as A, type CompletionModel as C, type Document as D, type ImageContent as I, type JsonObject as J, Message as M, type Reasoning as R, type SystemMessage as S, type Text as T, Usage as U, type AssistantMessage as a, type CompletionRequest as b, type CompletionResponse as c, type JsonPrimitive as d, type JsonValue as e, type ToolCall as f, type ToolDefinition as g, type ToolMessage as h, type ToolResult as i, type ToolResultContent as j, UserContent as k, type UserMessage as l, type ToolChoice as m, CompletionCapabilityError as n, type CompletionModelCapabilities as o, type CompletionStreamEvent as p, type DocumentContent as q, type ImageDetail as r, type ReasoningContent as s, type ReasoningContentType as t, type StreamingCompletionModel as u, ToolContent as v, type ToolFunction as w, assertCompletionRequestSupported as x, reasoningDisplayText as y, textFromAssistantContent as z };
+export { AssistantContent as A, type CompletionModel as C, type Document as D, type ImageContent as I, type JsonObject as J, Message as M, type Reasoning as R, type SystemMessage as S, type Text as T, Usage as U, type AssistantMessage as a, type CompletionRequest as b, type CompletionResponse as c, type JsonPrimitive as d, type JsonValue as e, type ToolCall as f, type ToolDefinition as g, type ToolMessage as h, type ToolResult as i, type ToolResultContent as j, UserContent as k, type UserMessage as l, type ToolChoice as m, type StreamingCompletionModel as n, type CompletionStreamEvent as o, CompletionCapabilityError as p, type CompletionModelCapabilities as q, type DocumentContent as r, type ImageDetail as s, type ReasoningContent as t, type ReasoningContentType as u, ToolContent as v, type ToolFunction as w, assertCompletionRequestSupported as x, reasoningDisplayText as y, textFromAssistantContent as z };

package/dist/vector-store/index.d.ts

@@ -1,6 +1,6 @@
 import { d as VectorMetadataValue, V as VectorMetadata, E as EmbeddingModel, b as EmbeddedDocument, c as Embedding } from '../types-IB2e9u5M.js';
-import { T as Tool } from '../tool-C3ciF-VG.js';
-import '../types-C54aNoCd.js';
+import { T as Tool } from '../tool-Dz-lHZon.js';
+import '../types-Dmoq-Fhz.js';
 
 type VectorFilter = {
     type: "eq";

package/dist/vector-store/index.js

@@ -3,7 +3,8 @@ import {
   InMemoryVectorStore,
   createVectorSearchTool,
   vectorFilter
-} from "../chunk-P425B6GR.js";
+} from "../chunk-I47EHI45.js";
+import "../chunk-WQKHFADH.js";
 import "../chunk-EFGX3EX5.js";
 import "../chunk-OIMLU4SF.js";
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anvia/core",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Core runtime primitives for context-aware Anvia agents.",
   "author": "anvia",
   "maintainer": "Indra Zulfi",