@anvia/core 0.11.2 → 0.12.0

package/dist/{create-completion-BEbtU5pX.d.ts → create-completion-Y6YsYbQB.d.ts}

@@ -1,5 +1,5 @@
 import { Z as ZodSchema } from './zod-schema-C7F4clpm.js';
-import { M as Message, D as Document, h as ToolDefinition, n as ToolChoice, J as JsonObject, e as JsonValue, A as AssistantContent, U as Usage, c as CompletionResponse, C as CompletionModel, o as StreamingCompletionModel, p as CompletionStreamEvent } from './types-C8loR4il.js';
+import { M as Message, D as Document, h as ToolDefinition, n as ToolChoice, J as JsonObject, e as JsonValue, A as AssistantContent, U as Usage, c as CompletionResponse, C as CompletionModel, o as StreamingCompletionModel, p as CompletionStreamEvent } from './types-q9G7Bvwl.js';
 
 type CreateCompletionInput = string | Message | Message[];
 type CreateCompletionBaseOptions = {
@@ -28,7 +28,7 @@ type CreateCompletionResult<RawResponse = unknown> = {
 type CreateParsedCompletionResult<T, RawResponse = unknown> = CreateCompletionResult<RawResponse> & {
     data: T;
 };
-type RawResponseOf<Model> = Model extends CompletionModel<infer RawResponse> ? RawResponse : unknown;
+type RawResponseOf<Model> = Model extends CompletionModel<infer RawResponse, infer _ModelName> ? RawResponse : unknown;
 declare function createCompletion<Model extends CompletionModel>(model: Model, options: CreateCompletionOptions): Promise<CreateCompletionResult<RawResponseOf<Model>>>;
 declare function createCompletionStream<Model extends StreamingCompletionModel>(model: Model, options: CreateCompletionStreamOptions): AsyncIterable<CompletionStreamEvent<RawResponseOf<Model>>>;
 declare function createParsedCompletion<T, Model extends CompletionModel>(model: Model, options: CreateParsedCompletionOptions<T>): Promise<CreateParsedCompletionResult<T, RawResponseOf<Model>>>;

package/dist/evals/index.d.ts

@@ -1,13 +1,14 @@
-import { A as Agent } from '../agent-Bq2KtSaw.js';
-import { e as JsonValue, M as Message, C as CompletionModel } from '../types-C8loR4il.js';
-import { P as PromptResponse } from '../index-CTNzKidE.js';
+import { A as Agent } from '../agent-BWF8_O8k.js';
+import { e as JsonValue, M as Message, C as CompletionModel } from '../types-q9G7Bvwl.js';
+import { P as PromptResponse } from '../index-DsWWqP0Q.js';
 import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
 import { E as EmbeddingModel } from '../types-DHHZztGO.js';
-import '../types-CCxnnJjU.js';
-import '../types-CNfjQ5rB.js';
-import '../tool-BLypI31e.js';
-import '../middleware-CG7l0rR3.js';
-import '../types-KKjICcrA.js';
+import '../guardrails/index.js';
+import '../types-CGML74sE.js';
+import '../types-CqG6d7Ce.js';
+import '../tool-DGELXD2z.js';
+import '../middleware-BTR0eRtx.js';
+import '../types-9VPKg7ih.js';
 import 'zod';
 
 type EvalOutcome<Score = unknown> = {

package/dist/evals/index.js

@@ -1,14 +1,21 @@
 import {
   ExtractorBuilder
-} from "../chunk-R75SGHCI.js";
-import "../chunk-62RUYGMM.js";
-import "../chunk-N6REB3DH.js";
-import "../chunk-SISUMRUX.js";
+} from "../chunk-MIJUAWHU.js";
+import "../chunk-TIH4QM55.js";
+import "../chunk-4AJJIIAQ.js";
+import "../chunk-KTTIPWE7.js";
 import "../chunk-YK4WAAS4.js";
 import "../chunk-XUUY2L2D.js";
-import "../chunk-X3SWV6W6.js";
-import "../chunk-5EDHXYH5.js";
+import "../chunk-R7F2JF7J.js";
+import "../chunk-PGFYN5ZX.js";
+import "../chunk-NPZDYOE6.js";
+import {
+  compact
+} from "../chunk-MMHG7WAM.js";
 import "../chunk-M6RPFHG7.js";
+import "../chunk-SRBW3LD7.js";
+import "../chunk-5LFGEMIW.js";
+import "../chunk-WQKHFADH.js";
 import {
   cosineSimilarity,
   embedText
@@ -16,13 +23,7 @@ import {
 import {
   mapWithConcurrency
 } from "../chunk-OIMLU4SF.js";
-import "../chunk-5G3VME2D.js";
-import "../chunk-WJO6NVE2.js";
-import "../chunk-WQKHFADH.js";
-import "../chunk-NPZDYOE6.js";
-import {
-  compact
-} from "../chunk-MMHG7WAM.js";
+import "../chunk-YRERPIA5.js";
 
 // src/evals/agent-target.ts
 function agentEvalTarget(agent, options = {}) {

package/dist/evals/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/evals/agent-target.ts","../../src/evals/metric.ts","../../src/evals/metrics.ts","../../src/evals/format.ts","../../src/evals/outcome.ts","../../src/evals/selectors.ts","../../src/evals/runner.ts"],"sourcesContent":["import type { Agent } from \"../agent/agent\";\nimport type { Message } from \"../completion\";\nimport type { PromptResponse } from \"../request/types\";\nimport type { EvalCase, EvalTarget } from \"./types\";\n\nexport type AgentEvalTargetOptions<Input, Output = PromptResponse> = {\n  prompt?: ((input: Input, testCase: EvalCase<Input>) => string | Message) | undefined;\n  output?: ((response: PromptResponse, testCase: EvalCase<Input>) => Output) | undefined;\n};\n\nexport function agentEvalTarget<Input>(\n  agent: Agent,\n  options?: AgentEvalTargetOptions<Input, PromptResponse>,\n): EvalTarget<Input, PromptResponse>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output>,\n): EvalTarget<Input, Output>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output | PromptResponse> = {},\n): EvalTarget<Input, Output | PromptResponse> {\n  return async (input, testCase) => {\n    const prompt = options.prompt?.(input, testCase) ?? String(input);\n    const response = await agent.prompt(prompt).send();\n    return options.output === undefined ? response : options.output(response, testCase);\n  };\n}\n","import type { EvalMetric } from \"./types\";\n\nexport function defineMetric<Input, Output, Score, Expected>(\n  metric: EvalMetric<Input, Output, Score, Expected>,\n): EvalMetric<Input, Output, Score, Expected> {\n  return metric;\n}\n","import { z } from \"zod\";\nimport type { CompletionModel } from \"../completion\";\nimport { cosineSimilarity, type EmbeddingModel, embedText } from \"../embeddings\";\nimport { ExtractorBuilder } from \"../extractor\";\nimport type { ZodSchema } from \"../schema\";\nimport { errorMessage, formatValue, stableComparable } from \"./format\";\nimport { EvalOutcome } from \"./outcome\";\nimport { resolveActual, resolveActualText, resolveExpected, resolveJudgePrompt } from \"./selectors\";\nimport type { EvalMetric, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport type ExactMatchOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, unknown> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, unknown> | undefined;\n};\n\nexport function exactMatch<Input, Output, Expected = unknown>(\n  options: ExactMatchOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"exact_match\",\n    async evaluate(args) {\n      const actual = await resolveActual(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for exact match.\");\n      }\n      const passed = stableComparable(actual) === stableComparable(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });\n    },\n  };\n}\n\nexport type ContainsOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string | RegExp> | undefined;\n};\n\nexport function contains<Input, Output, Expected = unknown>(\n  options: ContainsOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"contains\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for contains.\");\n      }\n      if (typeof expected !== \"string\" && !(expected instanceof RegExp)) {\n        return EvalOutcome.invalid(\"Contains expected value must be a string or RegExp.\");\n      }\n      const passed = expected instanceof RegExp ? expected.test(actual) : actual.includes(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Output did not contain ${String(expected)}.` });\n    },\n  };\n}\n\nexport type SemanticSimilarityOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: EmbeddingModel;\n  threshold: number;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string> | undefined;\n};\n\nexport function semanticSimilarity<Input, Output, Expected = unknown>(\n  options: SemanticSimilarityOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, number, Expected> {\n  return {\n    name: options.name ?? \"semantic_similarity\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for semantic similarity.\");\n      }\n      if (typeof expected !== \"string\") {\n        return EvalOutcome.invalid(\"Semantic similarity expected value must be a string.\");\n      }\n      const [actualEmbedding, expectedEmbedding] = await Promise.all([\n        embedText(options.model, actual),\n        embedText(options.model, expected),\n      ]);\n      const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);\n      return score >= options.threshold\n        ? EvalOutcome.pass(score)\n        : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });\n    },\n  };\n}\n\nexport type LlmJudgeOptions<Input, Output, SchemaOutput, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  schema: ZodSchema<SchemaOutput>;\n  passes(value: SchemaOutput): boolean;\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmJudge<Input, Output, SchemaOutput, Expected = unknown>(\n  options: LlmJudgeOptions<Input, Output, SchemaOutput, Expected>,\n): EvalMetric<Input, Output, SchemaOutput, Expected> {\n  const extractor = new ExtractorBuilder(options.model, options.schema)\n    .instructions(\n      options.instructions ??\n        \"Judge the eval case by the requested schema. Submit the judgment using the schema.\",\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_judge\",\n    async evaluate(args) {\n      try {\n        const judgment = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        return options.passes(judgment) ? EvalOutcome.pass(judgment) : EvalOutcome.fail(judgment);\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n\nexport type LlmScoreMetricScore = {\n  score: number;\n  feedback: string;\n};\n\nexport type LlmScoreOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  threshold: number;\n  criteria: string | string[];\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmScore<Input, Output, Expected = unknown>(\n  options: LlmScoreOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, LlmScoreMetricScore, Expected> {\n  const criteria = Array.isArray(options.criteria) ? options.criteria.join(\"\\n\") : options.criteria;\n  const extractor = new ExtractorBuilder(\n    options.model,\n    z.object({\n      score: z.number(),\n      feedback: z.string(),\n    }),\n  )\n    .instructions(\n      options.instructions ??\n        `Score the eval case against these criteria:\\n${criteria}\\n\\nReturn a score between 0 and 1 and brief feedback.`,\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_score\",\n    async evaluate(args) {\n      try {\n        const score = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        if (score.score < 0 || score.score > 1) {\n          return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {\n            score,\n          });\n        }\n        return score.score >= options.threshold\n          ? EvalOutcome.pass(score, { comment: score.feedback })\n          : EvalOutcome.fail(score, { comment: score.feedback });\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n","export function defaultOutputValue(output: unknown): unknown {\n  if (\n    typeof output === \"object\" &&\n    output !== null &&\n    \"output\" in output &&\n    typeof (output as { output?: unknown }).output === \"string\"\n  ) {\n    return (output as { output: string }).output;\n  }\n  return output;\n}\n\nexport function stableComparable(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  return JSON.stringify(value);\n}\n\nexport function formatValue(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  try {\n    return JSON.stringify(value);\n  } catch {\n    return String(value);\n  }\n}\n\nexport function errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n","import { compact } from \"../internal/compact\";\nimport type { EvalMetadata } from \"./types\";\n\nexport type EvalOutcome<Score = unknown> =\n  | {\n      outcome: \"pass\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"fail\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"invalid\";\n      reason: string;\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    };\n\nexport const EvalOutcome = {\n  pass<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    return compact({\n      outcome: \"pass\" as const,\n      score,\n      comment: options.comment,\n      metadata: options.metadata,\n    }) as EvalOutcome<Score>;\n  },\n\n  fail<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    return compact({\n      outcome: \"fail\" as const,\n      score,\n      comment: options.comment,\n      metadata: options.metadata,\n    }) as EvalOutcome<Score>;\n  },\n\n  invalid<Score = never>(\n    reason: string,\n    options: {\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    } = {},\n  ): EvalOutcome<Score> {\n    return compact({\n      outcome: \"invalid\" as const,\n      reason,\n      score: options.score,\n      comment: options.comment,\n      metadata: options.metadata,\n    }) as EvalOutcome<Score>;\n  },\n};\n","import { defaultOutputValue, formatValue } from \"./format\";\nimport type { EvalMetricArgs, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport async function resolveActual<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, unknown> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<unknown> {\n  return selector === undefined ? defaultOutputValue(args.output) : selector(args);\n}\n\nexport async function resolveActualText<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  const value = selector === undefined ? defaultOutputValue(args.output) : await selector(args);\n  return typeof value === \"string\" ? value : JSON.stringify(value);\n}\n\nexport async function resolveExpected<Input, Output, Expected, Value>(\n  selectorOrValue: SelectorOrValue<Input, Output, Expected, Value> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<Value | Expected | undefined> {\n  if (selectorOrValue === undefined) {\n    return args.case.expected;\n  }\n  return typeof selectorOrValue === \"function\"\n    ? (selectorOrValue as ValueSelector<Input, Output, Expected, Value>)(args)\n    : selectorOrValue;\n}\n\nexport async function resolveJudgePrompt<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  if (selector !== undefined) {\n    return selector(args);\n  }\n  return [\n    `Suite: ${args.suiteName}`,\n    `Case: ${args.case.id}`,\n    `Input: ${formatValue(args.case.input)}`,\n    `Expected: ${formatValue(args.case.expected)}`,\n    `Output: ${formatValue(defaultOutputValue(args.output))}`,\n  ].join(\"\\n\\n\");\n}\n","import { compact } from \"../internal/compact\";\nimport { mapWithConcurrency } from \"../internal/concurrency\";\nimport { errorMessage } from \"./format\";\nimport { EvalOutcome, type EvalOutcome as EvalOutcomeType } from \"./outcome\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalMetric,\n  EvalMetricResult,\n  EvalReporter,\n  EvalSuiteResult,\n  RunEvalSuiteOptions,\n} from \"./types\";\n\nexport async function runEvalSuite<Input, Output, Expected = unknown>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n): Promise<EvalSuiteResult<Input, Output, Expected>> {\n  const startedAt = Date.now();\n  const results = await mapWithConcurrency(\n    options.cases,\n    Math.max(1, Math.trunc(options.concurrency ?? 1)),\n    (testCase) => runEvalCase(options, testCase),\n  );\n  const counts = countOutcomes(results);\n  return {\n    name: options.name,\n    results,\n    ...counts,\n    durationMs: Date.now() - startedAt,\n  };\n}\n\nasync function runEvalCase<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n  testCase: EvalCase<Input, Expected>,\n): Promise<EvalCaseResult<Input, Output, Expected>> {\n  let output: Output | undefined;\n  let targetError: unknown;\n  try {\n    output = await options.target(testCase.input, testCase);\n  } catch (error) {\n    targetError = error;\n  }\n\n  const metrics: EvalMetricResult[] = [];\n  for (const metric of options.metrics) {\n    const outcome =\n      targetError === undefined\n        ? await safeEvaluate(options.name, testCase, output as Output, metric)\n        : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`);\n    const reporterErrors = await reportOutcome({\n      suiteName: options.name,\n      testCase,\n      output,\n      targetError,\n      metric,\n      outcome,\n      reporters: options.reporters ?? [],\n      failOnReporterError: options.failOnReporterError === true,\n    });\n    metrics.push({ metricName: metric.name, outcome, reporterErrors });\n  }\n\n  return compact({\n    case: testCase,\n    output,\n    targetError,\n    metrics,\n  }) as EvalCaseResult<Input, Output, Expected>;\n}\n\nasync function safeEvaluate<Input, Output, Expected>(\n  suiteName: string,\n  testCase: EvalCase<Input, Expected>,\n  output: Output,\n  metric: EvalMetric<Input, Output, unknown, Expected>,\n): Promise<EvalOutcomeType> {\n  try {\n    return await metric.evaluate({ suiteName, case: testCase, output });\n  } catch (error) {\n    return EvalOutcome.invalid(errorMessage(error));\n  }\n}\n\nasync function reportOutcome<Input, Output, Expected>(args: {\n  suiteName: string;\n  testCase: EvalCase<Input, Expected>;\n  output: Output | undefined;\n  targetError: unknown;\n  metric: EvalMetric<Input, Output, unknown, Expected>;\n  outcome: EvalOutcomeType;\n  reporters: Array<EvalReporter<Input, Output, Expected>>;\n  failOnReporterError: boolean;\n}): Promise<unknown[]> {\n  const errors: unknown[] = [];\n  for (const reporter of args.reporters) {\n    try {\n      await reporter.report({\n        suiteName: args.suiteName,\n        case: args.testCase,\n        output: args.output,\n        targetError: args.targetError,\n        metric: args.metric,\n        outcome: args.outcome,\n      });\n    } catch (error) {\n      if (args.failOnReporterError) {\n        throw error;\n      }\n      errors.push(error);\n    }\n  }\n  return errors;\n}\n\nfunction countOutcomes(results: Array<EvalCaseResult<unknown, unknown, unknown>>): {\n  passed: number;\n  failed: number;\n  invalid: number;\n} {\n  let passed = 0;\n  let failed = 0;\n  let invalid = 0;\n  for (const result of results) {\n    for (const metric of result.metrics) {\n      if (metric.outcome.outcome === \"pass\") passed += 1;\n      if (metric.outcome.outcome === \"fail\") failed += 1;\n      if (metric.outcome.outcome === \"invalid\") invalid += 1;\n    }\n  }\n  return { passed, failed, invalid };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAkBO,SAAS,gBACd,OACA,UAAkE,CAAC,GACvB;AAC5C,SAAO,OAAO,OAAO,aAAa;AAChC,UAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,KAAK,OAAO,KAAK;AAChE,UAAM,WAAW,MAAM,MAAM,OAAO,MAAM,EAAE,KAAK;AACjD,WAAO,QAAQ,WAAW,SAAY,WAAW,QAAQ,OAAO,UAAU,QAAQ;AAAA,EACpF;AACF;;;ACzBO,SAAS,aACd,QAC4C;AAC5C,SAAO;AACT;;;ACNA,SAAS,SAAS;;;ACAX,SAAS,mBAAmB,QAA0B;AAC3D,MACE,OAAO,WAAW,YAClB,WAAW,QACX,YAAY,UACZ,OAAQ,OAAgC,WAAW,UACnD;AACA,WAAQ,OAA8B;AAAA,EACxC;AACA,SAAO;AACT;AAEO,SAAS,iBAAiB,OAAwB;AACvD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,SAAO,KAAK,UAAU,KAAK;AAC7B;AAEO,SAAS,YAAY,OAAwB;AAClD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,MAAI;AACF,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B,QAAQ;AACN,WAAO,OAAO,KAAK;AAAA,EACrB;AACF;AAEO,SAAS,aAAa,OAAwB;AACnD,SAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC9D;;;ACRO,IAAM,cAAc;AAAA,EACzB,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,WAAO,QAAQ;AAAA,MACb,SAAS;AAAA,MACT;AAAA,MACA,SAAS,QAAQ;AAAA,MACjB,UAAU,QAAQ;AAAA,IACpB,CAAC;AAAA,EACH;AAAA,EAEA,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,WAAO,QAAQ;AAAA,MACb,SAAS;AAAA,MACT;AAAA,MACA,SAAS,QAAQ;AAAA,MACjB,UAAU,QAAQ;AAAA,IACpB,CAAC;AAAA,EACH;AAAA,EAEA,QACE,QACA,UAII,CAAC,GACe;AACpB,WAAO,QAAQ;AAAA,MACb,SAAS;AAAA,MACT;AAAA,MACA,OAAO,QAAQ;AAAA,MACf,SAAS,QAAQ;AAAA,MACjB,UAAU,QAAQ;AAAA,IACpB,CAAC;AAAA,EACH;AACF;;;AC9DA,eAAsB,cACpB,UACA,MACkB;AAClB,SAAO,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,SAAS,IAAI;AACjF;AAEA,eAAsB,kBACpB,UACA,MACiB;AACjB,QAAM,QAAQ,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,MAAM,SAAS,IAAI;AAC5F,SAAO,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK;AACjE;AAEA,eAAsB,gBACpB,iBACA,MACuC;AACvC,MAAI,oBAAoB,QAAW;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AACA,SAAO,OAAO,oBAAoB,aAC7B,gBAAkE,IAAI,IACvE;AACN;AAEA,eAAsB,mBACpB,UACA,MACiB;AACjB,MAAI,aAAa,QAAW;AAC1B,WAAO,SAAS,IAAI;AAAA,EACtB;AACA,SAAO;AAAA,IACL,UAAU,KAAK,SAAS;AAAA,IACxB,SAAS,KAAK,KAAK,EAAE;AAAA,IACrB,UAAU,YAAY,KAAK,KAAK,KAAK,CAAC;AAAA,IACtC,aAAa,YAAY,KAAK,KAAK,QAAQ,CAAC;AAAA,IAC5C,WAAW,YAAY,mBAAmB,KAAK,MAAM,CAAC,CAAC;AAAA,EACzD,EAAE,KAAK,MAAM;AACf;;;AH5BO,SAAS,WACd,UAAsD,CAAC,GACT;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,cAAc,QAAQ,QAAQ,IAAI;AACvD,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,6CAA6C;AAAA,MAC1E;AACA,YAAM,SAAS,iBAAiB,MAAM,MAAM,iBAAiB,QAAQ;AACrE,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,YAAY,YAAY,QAAQ,CAAC,IAAI,CAAC;AAAA,IAC/E;AAAA,EACF;AACF;AAQO,SAAS,SACd,UAAoD,CAAC,GACP;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,0CAA0C;AAAA,MACvE;AACA,UAAI,OAAO,aAAa,YAAY,EAAE,oBAAoB,SAAS;AACjE,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,YAAM,SAAS,oBAAoB,SAAS,SAAS,KAAK,MAAM,IAAI,OAAO,SAAS,QAAQ;AAC5F,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,0BAA0B,OAAO,QAAQ,CAAC,IAAI,CAAC;AAAA,IACxF;AAAA,EACF;AACF;AAUO,SAAS,mBACd,SAC6C;AAC7C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,UAAI,OAAO,aAAa,UAAU;AAChC,eAAO,YAAY,QAAQ,sDAAsD;AAAA,MACnF;AACA,YAAM,CAAC,iBAAiB,iBAAiB,IAAI,MAAM,QAAQ,IAAI;AAAA,QAC7D,UAAU,QAAQ,OAAO,MAAM;AAAA,QAC/B,UAAU,QAAQ,OAAO,QAAQ;AAAA,MACnC,CAAC;AACD,YAAM,QAAQ,iBAAiB,gBAAgB,QAAQ,kBAAkB,MAAM;AAC/E,aAAO,SAAS,QAAQ,YACpB,YAAY,KAAK,KAAK,IACtB,YAAY,KAAK,OAAO,EAAE,SAAS,8BAA8B,QAAQ,SAAS,IAAI,CAAC;AAAA,IAC7F;AAAA,EACF;AACF;AAYO,SAAS,SACd,SACmD;AACnD,QAAM,YAAY,IAAI,iBAAiB,QAAQ,OAAO,QAAQ,MAAM,EACjE;AAAA,IACC,QAAQ,gBACN;AAAA,EACJ,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,WAAW,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACvF,eAAO,QAAQ,OAAO,QAAQ,IAAI,YAAY,KAAK,QAAQ,IAAI,YAAY,KAAK,QAAQ;AAAA,MAC1F,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;AAiBO,SAAS,SACd,SAC0D;AAC1D,QAAM,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAAI,QAAQ,SAAS,KAAK,IAAI,IAAI,QAAQ;AACzF,QAAM,YAAY,IAAI;AAAA,IACpB,QAAQ;AAAA,IACR,EAAE,OAAO;AAAA,MACP,OAAO,EAAE,OAAO;AAAA,MAChB,UAAU,EAAE,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,EACG;AAAA,IACC,QAAQ,gBACN;AAAA,EAAgD,QAAQ;AAAA;AAAA;AAAA,EAC5D,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,QAAQ,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACpF,YAAI,MAAM,QAAQ,KAAK,MAAM,QAAQ,GAAG;AACtC,iBAAO,YAAY,QAAQ,SAAS,MAAM,KAAK,gCAAgC;AAAA,YAC7E;AAAA,UACF,CAAC;AAAA,QACH;AACA,eAAO,MAAM,SAAS,QAAQ,YAC1B,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC,IACnD,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC;AAAA,MACzD,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;;;AIxKA,eAAsB,aACpB,SACmD;AACnD,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,UAAU,MAAM;AAAA,IACpB,QAAQ;AAAA,IACR,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,eAAe,CAAC,CAAC;AAAA,IAChD,CAAC,aAAa,YAAY,SAAS,QAAQ;AAAA,EAC7C;AACA,QAAM,SAAS,cAAc,OAAO;AACpC,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd;AAAA,IACA,GAAG;AAAA,IACH,YAAY,KAAK,IAAI,IAAI;AAAA,EAC3B;AACF;AAEA,eAAe,YACb,SACA,UACkD;AAClD,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,QAAQ,OAAO,SAAS,OAAO,QAAQ;AAAA,EACxD,SAAS,OAAO;AACd,kBAAc;AAAA,EAChB;AAEA,QAAM,UAA8B,CAAC;AACrC,aAAW,UAAU,QAAQ,SAAS;AACpC,UAAM,UACJ,gBAAgB,SACZ,MAAM,aAAa,QAAQ,MAAM,UAAU,QAAkB,MAAM,IACnE,YAAY,QAAQ,kBAAkB,aAAa,WAAW,CAAC,EAAE;AACvE,UAAM,iBAAiB,MAAM,cAAc;AAAA,MACzC,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW,QAAQ,aAAa,CAAC;AAAA,MACjC,qBAAqB,QAAQ,wBAAwB;AAAA,IACvD,CAAC;AACD,YAAQ,KAAK,EAAE,YAAY,OAAO,MAAM,SAAS,eAAe,CAAC;AAAA,EACnE;AAEA,SAAO,QAAQ;AAAA,IACb,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACH;AAEA,eAAe,aACb,WACA,UACA,QACA,QAC0B;AAC1B,MAAI;AACF,WAAO,MAAM,OAAO,SAAS,EAAE,WAAW,MAAM,UAAU,OAAO,CAAC;AAAA,EACpE,SAAS,OAAO;AACd,WAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,EAChD;AACF;AAEA,eAAe,cAAuC,MAS/B;AACrB,QAAM,SAAoB,CAAC;AAC3B,aAAW,YAAY,KAAK,WAAW;AACrC,QAAI;AACF,YAAM,SAAS,OAAO;AAAA,QACpB,WAAW,KAAK;AAAA,QAChB,MAAM,KAAK;AAAA,QACX,QAAQ,KAAK;AAAA,QACb,aAAa,KAAK;AAAA,QAClB,QAAQ,KAAK;AAAA,QACb,SAAS,KAAK;AAAA,MAChB,CAAC;AAAA,IACH,SAAS,OAAO;AACd,UAAI,KAAK,qBAAqB;AAC5B,cAAM;AAAA,MACR;AACA,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,cAAc,SAIrB;AACA,MAAI,SAAS;AACb,MAAI,SAAS;AACb,MAAI,UAAU;AACd,aAAW,UAAU,SAAS;AAC5B,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,UAAW,YAAW;AAAA,IACvD;AAAA,EACF;AACA,SAAO,EAAE,QAAQ,QAAQ,QAAQ;AACnC;","names":[]}
+{"version":3,"sources":["../../src/evals/agent-target.ts","../../src/evals/metric.ts","../../src/evals/metrics.ts","../../src/evals/format.ts","../../src/evals/outcome.ts","../../src/evals/selectors.ts","../../src/evals/runner.ts"],"sourcesContent":["import type { Agent } from \"../agent/agent\";\nimport type { Message } from \"../completion\";\nimport type { PromptResponse } from \"../request/types\";\nimport type { EvalCase, EvalTarget } from \"./types\";\n\nexport type AgentEvalTargetOptions<Input, Output = PromptResponse> = {\n  prompt?: ((input: Input, testCase: EvalCase<Input>) => string | Message) | undefined;\n  output?: ((response: PromptResponse, testCase: EvalCase<Input>) => Output) | undefined;\n};\n\nexport function agentEvalTarget<Input>(\n  agent: Agent,\n  options?: AgentEvalTargetOptions<Input, PromptResponse>,\n): EvalTarget<Input, PromptResponse>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output>,\n): EvalTarget<Input, Output>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output | PromptResponse> = {},\n): EvalTarget<Input, Output | PromptResponse> {\n  return async (input, testCase) => {\n    const prompt = options.prompt?.(input, testCase) ?? String(input);\n    const response = await agent.prompt(prompt).send();\n    return options.output === undefined ? response : options.output(response, testCase);\n  };\n}\n","import type { EvalMetric } from \"./types\";\n\nexport function defineMetric<Input, Output, Score, Expected>(\n  metric: EvalMetric<Input, Output, Score, Expected>,\n): EvalMetric<Input, Output, Score, Expected> {\n  return metric;\n}\n","import { z } from \"zod\";\nimport type { CompletionModel } from \"../completion\";\nimport { cosineSimilarity, type EmbeddingModel, embedText } from \"../embeddings\";\nimport { ExtractorBuilder } from \"../extractor\";\nimport type { ZodSchema } from \"../schema\";\nimport { errorMessage, formatValue, stableComparable } from \"./format\";\nimport { EvalOutcome } from \"./outcome\";\nimport { resolveActual, resolveActualText, resolveExpected, resolveJudgePrompt } from \"./selectors\";\nimport type { EvalMetric, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport type ExactMatchOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, unknown> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, unknown> | undefined;\n};\n\nexport function exactMatch<Input, Output, Expected = unknown>(\n  options: ExactMatchOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"exact_match\",\n    async evaluate(args) {\n      const actual = await resolveActual(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for exact match.\");\n      }\n      const passed = stableComparable(actual) === stableComparable(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });\n    },\n  };\n}\n\nexport type ContainsOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string | RegExp> | undefined;\n};\n\nexport function contains<Input, Output, Expected = unknown>(\n  options: ContainsOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"contains\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for contains.\");\n      }\n      if (typeof expected !== \"string\" && !(expected instanceof RegExp)) {\n        return EvalOutcome.invalid(\"Contains expected value must be a string or RegExp.\");\n      }\n      const passed = expected instanceof RegExp ? expected.test(actual) : actual.includes(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Output did not contain ${String(expected)}.` });\n    },\n  };\n}\n\nexport type SemanticSimilarityOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: EmbeddingModel;\n  threshold: number;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string> | undefined;\n};\n\nexport function semanticSimilarity<Input, Output, Expected = unknown>(\n  options: SemanticSimilarityOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, number, Expected> {\n  return {\n    name: options.name ?? \"semantic_similarity\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for semantic similarity.\");\n      }\n      if (typeof expected !== \"string\") {\n        return EvalOutcome.invalid(\"Semantic similarity expected value must be a string.\");\n      }\n      const [actualEmbedding, expectedEmbedding] = await Promise.all([\n        embedText(options.model, actual),\n        embedText(options.model, expected),\n      ]);\n      const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);\n      return score >= options.threshold\n        ? EvalOutcome.pass(score)\n        : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });\n    },\n  };\n}\n\nexport type LlmJudgeOptions<Input, Output, SchemaOutput, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  schema: ZodSchema<SchemaOutput>;\n  passes(value: SchemaOutput): boolean;\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmJudge<Input, Output, SchemaOutput, Expected = unknown>(\n  options: LlmJudgeOptions<Input, Output, SchemaOutput, Expected>,\n): EvalMetric<Input, Output, SchemaOutput, Expected> {\n  const extractor = new ExtractorBuilder(options.model, options.schema)\n    .instructions(\n      options.instructions ??\n        \"Judge the eval case by the requested schema. Submit the judgment using the schema.\",\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_judge\",\n    async evaluate(args) {\n      try {\n        const judgment = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        return options.passes(judgment) ? EvalOutcome.pass(judgment) : EvalOutcome.fail(judgment);\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n\nexport type LlmScoreMetricScore = {\n  score: number;\n  feedback: string;\n};\n\nexport type LlmScoreOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  threshold: number;\n  criteria: string | string[];\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmScore<Input, Output, Expected = unknown>(\n  options: LlmScoreOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, LlmScoreMetricScore, Expected> {\n  const criteria = Array.isArray(options.criteria) ? options.criteria.join(\"\\n\") : options.criteria;\n  const extractor = new ExtractorBuilder(\n    options.model,\n    z.object({\n      score: z.number(),\n      feedback: z.string(),\n    }),\n  )\n    .instructions(\n      options.instructions ??\n        `Score the eval case against these criteria:\\n${criteria}\\n\\nReturn a score between 0 and 1 and brief feedback.`,\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_score\",\n    async evaluate(args) {\n      try {\n        const score = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        if (score.score < 0 || score.score > 1) {\n          return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {\n            score,\n          });\n        }\n        return score.score >= options.threshold\n          ? EvalOutcome.pass(score, { comment: score.feedback })\n          : EvalOutcome.fail(score, { comment: score.feedback });\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n","export function defaultOutputValue(output: unknown): unknown {\n  if (\n    typeof output === \"object\" &&\n    output !== null &&\n    \"output\" in output &&\n    typeof (output as { output?: unknown }).output === \"string\"\n  ) {\n    return (output as { output: string }).output;\n  }\n  return output;\n}\n\nexport function stableComparable(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  return JSON.stringify(value);\n}\n\nexport function formatValue(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  try {\n    return JSON.stringify(value);\n  } catch {\n    return String(value);\n  }\n}\n\nexport function errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n","import { compact } from \"../internal/compact\";\nimport type { EvalMetadata } from \"./types\";\n\nexport type EvalOutcome<Score = unknown> =\n  | {\n      outcome: \"pass\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"fail\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"invalid\";\n      reason: string;\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    };\n\nexport const EvalOutcome = {\n  pass<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    return compact({\n      outcome: \"pass\" as const,\n      score,\n      comment: options.comment,\n      metadata: options.metadata,\n    }) as EvalOutcome<Score>;\n  },\n\n  fail<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    return compact({\n      outcome: \"fail\" as const,\n      score,\n      comment: options.comment,\n      metadata: options.metadata,\n    }) as EvalOutcome<Score>;\n  },\n\n  invalid<Score = never>(\n    reason: string,\n    options: {\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    } = {},\n  ): EvalOutcome<Score> {\n    return compact({\n      outcome: \"invalid\" as const,\n      reason,\n      score: options.score,\n      comment: options.comment,\n      metadata: options.metadata,\n    }) as EvalOutcome<Score>;\n  },\n};\n","import { defaultOutputValue, formatValue } from \"./format\";\nimport type { EvalMetricArgs, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport async function resolveActual<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, unknown> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<unknown> {\n  return selector === undefined ? defaultOutputValue(args.output) : selector(args);\n}\n\nexport async function resolveActualText<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  const value = selector === undefined ? defaultOutputValue(args.output) : await selector(args);\n  return typeof value === \"string\" ? value : JSON.stringify(value);\n}\n\nexport async function resolveExpected<Input, Output, Expected, Value>(\n  selectorOrValue: SelectorOrValue<Input, Output, Expected, Value> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<Value | Expected | undefined> {\n  if (selectorOrValue === undefined) {\n    return args.case.expected;\n  }\n  return typeof selectorOrValue === \"function\"\n    ? (selectorOrValue as ValueSelector<Input, Output, Expected, Value>)(args)\n    : selectorOrValue;\n}\n\nexport async function resolveJudgePrompt<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  if (selector !== undefined) {\n    return selector(args);\n  }\n  return [\n    `Suite: ${args.suiteName}`,\n    `Case: ${args.case.id}`,\n    `Input: ${formatValue(args.case.input)}`,\n    `Expected: ${formatValue(args.case.expected)}`,\n    `Output: ${formatValue(defaultOutputValue(args.output))}`,\n  ].join(\"\\n\\n\");\n}\n","import { compact } from \"../internal/compact\";\nimport { mapWithConcurrency } from \"../internal/concurrency\";\nimport { errorMessage } from \"./format\";\nimport { EvalOutcome, type EvalOutcome as EvalOutcomeType } from \"./outcome\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalMetric,\n  EvalMetricResult,\n  EvalReporter,\n  EvalSuiteResult,\n  RunEvalSuiteOptions,\n} from \"./types\";\n\nexport async function runEvalSuite<Input, Output, Expected = unknown>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n): Promise<EvalSuiteResult<Input, Output, Expected>> {\n  const startedAt = Date.now();\n  const results = await mapWithConcurrency(\n    options.cases,\n    Math.max(1, Math.trunc(options.concurrency ?? 1)),\n    (testCase) => runEvalCase(options, testCase),\n  );\n  const counts = countOutcomes(results);\n  return {\n    name: options.name,\n    results,\n    ...counts,\n    durationMs: Date.now() - startedAt,\n  };\n}\n\nasync function runEvalCase<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n  testCase: EvalCase<Input, Expected>,\n): Promise<EvalCaseResult<Input, Output, Expected>> {\n  let output: Output | undefined;\n  let targetError: unknown;\n  try {\n    output = await options.target(testCase.input, testCase);\n  } catch (error) {\n    targetError = error;\n  }\n\n  const metrics: EvalMetricResult[] = [];\n  for (const metric of options.metrics) {\n    const outcome =\n      targetError === undefined\n        ? await safeEvaluate(options.name, testCase, output as Output, metric)\n        : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`);\n    const reporterErrors = await reportOutcome({\n      suiteName: options.name,\n      testCase,\n      output,\n      targetError,\n      metric,\n      outcome,\n      reporters: options.reporters ?? [],\n      failOnReporterError: options.failOnReporterError === true,\n    });\n    metrics.push({ metricName: metric.name, outcome, reporterErrors });\n  }\n\n  return compact({\n    case: testCase,\n    output,\n    targetError,\n    metrics,\n  }) as EvalCaseResult<Input, Output, Expected>;\n}\n\nasync function safeEvaluate<Input, Output, Expected>(\n  suiteName: string,\n  testCase: EvalCase<Input, Expected>,\n  output: Output,\n  metric: EvalMetric<Input, Output, unknown, Expected>,\n): Promise<EvalOutcomeType> {\n  try {\n    return await metric.evaluate({ suiteName, case: testCase, output });\n  } catch (error) {\n    return EvalOutcome.invalid(errorMessage(error));\n  }\n}\n\nasync function reportOutcome<Input, Output, Expected>(args: {\n  suiteName: string;\n  testCase: EvalCase<Input, Expected>;\n  output: Output | undefined;\n  targetError: unknown;\n  metric: EvalMetric<Input, Output, unknown, Expected>;\n  outcome: EvalOutcomeType;\n  reporters: Array<EvalReporter<Input, Output, Expected>>;\n  failOnReporterError: boolean;\n}): Promise<unknown[]> {\n  const errors: unknown[] = [];\n  for (const reporter of args.reporters) {\n    try {\n      await reporter.report({\n        suiteName: args.suiteName,\n        case: args.testCase,\n        output: args.output,\n        targetError: args.targetError,\n        metric: args.metric,\n        outcome: args.outcome,\n      });\n    } catch (error) {\n      if (args.failOnReporterError) {\n        throw error;\n      }\n      errors.push(error);\n    }\n  }\n  return errors;\n}\n\nfunction countOutcomes(results: Array<EvalCaseResult<unknown, unknown, unknown>>): {\n  passed: number;\n  failed: number;\n  invalid: number;\n} {\n  let passed = 0;\n  let failed = 0;\n  let invalid = 0;\n  for (const result of results) {\n    for (const metric of result.metrics) {\n      if (metric.outcome.outcome === \"pass\") passed += 1;\n      if (metric.outcome.outcome === \"fail\") failed += 1;\n      if (metric.outcome.outcome === \"invalid\") invalid += 1;\n    }\n  }\n  return { passed, failed, invalid };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkBO,SAAS,gBACd,OACA,UAAkE,CAAC,GACvB;AAC5C,SAAO,OAAO,OAAO,aAAa;AAChC,UAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,KAAK,OAAO,KAAK;AAChE,UAAM,WAAW,MAAM,MAAM,OAAO,MAAM,EAAE,KAAK;AACjD,WAAO,QAAQ,WAAW,SAAY,WAAW,QAAQ,OAAO,UAAU,QAAQ;AAAA,EACpF;AACF;;;ACzBO,SAAS,aACd,QAC4C;AAC5C,SAAO;AACT;;;ACNA,SAAS,SAAS;;;ACAX,SAAS,mBAAmB,QAA0B;AAC3D,MACE,OAAO,WAAW,YAClB,WAAW,QACX,YAAY,UACZ,OAAQ,OAAgC,WAAW,UACnD;AACA,WAAQ,OAA8B;AAAA,EACxC;AACA,SAAO;AACT;AAEO,SAAS,iBAAiB,OAAwB;AACvD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,SAAO,KAAK,UAAU,KAAK;AAC7B;AAEO,SAAS,YAAY,OAAwB;AAClD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,MAAI;AACF,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B,QAAQ;AACN,WAAO,OAAO,KAAK;AAAA,EACrB;AACF;AAEO,SAAS,aAAa,OAAwB;AACnD,SAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC9D;;;ACRO,IAAM,cAAc;AAAA,EACzB,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,WAAO,QAAQ;AAAA,MACb,SAAS;AAAA,MACT;AAAA,MACA,SAAS,QAAQ;AAAA,MACjB,UAAU,QAAQ;AAAA,IACpB,CAAC;AAAA,EACH;AAAA,EAEA,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,WAAO,QAAQ;AAAA,MACb,SAAS;AAAA,MACT;AAAA,MACA,SAAS,QAAQ;AAAA,MACjB,UAAU,QAAQ;AAAA,IACpB,CAAC;AAAA,EACH;AAAA,EAEA,QACE,QACA,UAII,CAAC,GACe;AACpB,WAAO,QAAQ;AAAA,MACb,SAAS;AAAA,MACT;AAAA,MACA,OAAO,QAAQ;AAAA,MACf,SAAS,QAAQ;AAAA,MACjB,UAAU,QAAQ;AAAA,IACpB,CAAC;AAAA,EACH;AACF;;;AC9DA,eAAsB,cACpB,UACA,MACkB;AAClB,SAAO,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,SAAS,IAAI;AACjF;AAEA,eAAsB,kBACpB,UACA,MACiB;AACjB,QAAM,QAAQ,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,MAAM,SAAS,IAAI;AAC5F,SAAO,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK;AACjE;AAEA,eAAsB,gBACpB,iBACA,MACuC;AACvC,MAAI,oBAAoB,QAAW;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AACA,SAAO,OAAO,oBAAoB,aAC7B,gBAAkE,IAAI,IACvE;AACN;AAEA,eAAsB,mBACpB,UACA,MACiB;AACjB,MAAI,aAAa,QAAW;AAC1B,WAAO,SAAS,IAAI;AAAA,EACtB;AACA,SAAO;AAAA,IACL,UAAU,KAAK,SAAS;AAAA,IACxB,SAAS,KAAK,KAAK,EAAE;AAAA,IACrB,UAAU,YAAY,KAAK,KAAK,KAAK,CAAC;AAAA,IACtC,aAAa,YAAY,KAAK,KAAK,QAAQ,CAAC;AAAA,IAC5C,WAAW,YAAY,mBAAmB,KAAK,MAAM,CAAC,CAAC;AAAA,EACzD,EAAE,KAAK,MAAM;AACf;;;AH5BO,SAAS,WACd,UAAsD,CAAC,GACT;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,cAAc,QAAQ,QAAQ,IAAI;AACvD,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,6CAA6C;AAAA,MAC1E;AACA,YAAM,SAAS,iBAAiB,MAAM,MAAM,iBAAiB,QAAQ;AACrE,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,YAAY,YAAY,QAAQ,CAAC,IAAI,CAAC;AAAA,IAC/E;AAAA,EACF;AACF;AAQO,SAAS,SACd,UAAoD,CAAC,GACP;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,0CAA0C;AAAA,MACvE;AACA,UAAI,OAAO,aAAa,YAAY,EAAE,oBAAoB,SAAS;AACjE,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,YAAM,SAAS,oBAAoB,SAAS,SAAS,KAAK,MAAM,IAAI,OAAO,SAAS,QAAQ;AAC5F,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,0BAA0B,OAAO,QAAQ,CAAC,IAAI,CAAC;AAAA,IACxF;AAAA,EACF;AACF;AAUO,SAAS,mBACd,SAC6C;AAC7C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,UAAI,OAAO,aAAa,UAAU;AAChC,eAAO,YAAY,QAAQ,sDAAsD;AAAA,MACnF;AACA,YAAM,CAAC,iBAAiB,iBAAiB,IAAI,MAAM,QAAQ,IAAI;AAAA,QAC7D,UAAU,QAAQ,OAAO,MAAM;AAAA,QAC/B,UAAU,QAAQ,OAAO,QAAQ;AAAA,MACnC,CAAC;AACD,YAAM,QAAQ,iBAAiB,gBAAgB,QAAQ,kBAAkB,MAAM;AAC/E,aAAO,SAAS,QAAQ,YACpB,YAAY,KAAK,KAAK,IACtB,YAAY,KAAK,OAAO,EAAE,SAAS,8BAA8B,QAAQ,SAAS,IAAI,CAAC;AAAA,IAC7F;AAAA,EACF;AACF;AAYO,SAAS,SACd,SACmD;AACnD,QAAM,YAAY,IAAI,iBAAiB,QAAQ,OAAO,QAAQ,MAAM,EACjE;AAAA,IACC,QAAQ,gBACN;AAAA,EACJ,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,WAAW,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACvF,eAAO,QAAQ,OAAO,QAAQ,IAAI,YAAY,KAAK,QAAQ,IAAI,YAAY,KAAK,QAAQ;AAAA,MAC1F,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;AAiBO,SAAS,SACd,SAC0D;AAC1D,QAAM,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAAI,QAAQ,SAAS,KAAK,IAAI,IAAI,QAAQ;AACzF,QAAM,YAAY,IAAI;AAAA,IACpB,QAAQ;AAAA,IACR,EAAE,OAAO;AAAA,MACP,OAAO,EAAE,OAAO;AAAA,MAChB,UAAU,EAAE,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,EACG;AAAA,IACC,QAAQ,gBACN;AAAA,EAAgD,QAAQ;AAAA;AAAA;AAAA,EAC5D,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,QAAQ,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACpF,YAAI,MAAM,QAAQ,KAAK,MAAM,QAAQ,GAAG;AACtC,iBAAO,YAAY,QAAQ,SAAS,MAAM,KAAK,gCAAgC;AAAA,YAC7E;AAAA,UACF,CAAC;AAAA,QACH;AACA,eAAO,MAAM,SAAS,QAAQ,YAC1B,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC,IACnD,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC;AAAA,MACzD,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;;;AIxKA,eAAsB,aACpB,SACmD;AACnD,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,UAAU,MAAM;AAAA,IACpB,QAAQ;AAAA,IACR,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,eAAe,CAAC,CAAC;AAAA,IAChD,CAAC,aAAa,YAAY,SAAS,QAAQ;AAAA,EAC7C;AACA,QAAM,SAAS,cAAc,OAAO;AACpC,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd;AAAA,IACA,GAAG;AAAA,IACH,YAAY,KAAK,IAAI,IAAI;AAAA,EAC3B;AACF;AAEA,eAAe,YACb,SACA,UACkD;AAClD,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,QAAQ,OAAO,SAAS,OAAO,QAAQ;AAAA,EACxD,SAAS,OAAO;AACd,kBAAc;AAAA,EAChB;AAEA,QAAM,UAA8B,CAAC;AACrC,aAAW,UAAU,QAAQ,SAAS;AACpC,UAAM,UACJ,gBAAgB,SACZ,MAAM,aAAa,QAAQ,MAAM,UAAU,QAAkB,MAAM,IACnE,YAAY,QAAQ,kBAAkB,aAAa,WAAW,CAAC,EAAE;AACvE,UAAM,iBAAiB,MAAM,cAAc;AAAA,MACzC,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW,QAAQ,aAAa,CAAC;AAAA,MACjC,qBAAqB,QAAQ,wBAAwB;AAAA,IACvD,CAAC;AACD,YAAQ,KAAK,EAAE,YAAY,OAAO,MAAM,SAAS,eAAe,CAAC;AAAA,EACnE;AAEA,SAAO,QAAQ;AAAA,IACb,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACH;AAEA,eAAe,aACb,WACA,UACA,QACA,QAC0B;AAC1B,MAAI;AACF,WAAO,MAAM,OAAO,SAAS,EAAE,WAAW,MAAM,UAAU,OAAO,CAAC;AAAA,EACpE,SAAS,OAAO;AACd,WAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,EAChD;AACF;AAEA,eAAe,cAAuC,MAS/B;AACrB,QAAM,SAAoB,CAAC;AAC3B,aAAW,YAAY,KAAK,WAAW;AACrC,QAAI;AACF,YAAM,SAAS,OAAO;AAAA,QACpB,WAAW,KAAK;AAAA,QAChB,MAAM,KAAK;AAAA,QACX,QAAQ,KAAK;AAAA,QACb,aAAa,KAAK;AAAA,QAClB,QAAQ,KAAK;AAAA,QACb,SAAS,KAAK;AAAA,MAChB,CAAC;AAAA,IACH,SAAS,OAAO;AACd,UAAI,KAAK,qBAAqB;AAC5B,cAAM;AAAA,MACR;AACA,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,cAAc,SAIrB;AACA,MAAI,SAAS;AACb,MAAI,SAAS;AACb,MAAI,UAAU;AACd,aAAW,UAAU,SAAS;AAC5B,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,UAAW,YAAW;AAAA,IACvD;AAAA,EACF;AACA,SAAO,EAAE,QAAQ,QAAQ,QAAQ;AACnC;","names":[]}

package/dist/extractor/index.d.ts

@@ -1,13 +1,14 @@
-import { A as Agent } from '../agent-Bq2KtSaw.js';
+import { A as Agent } from '../agent-BWF8_O8k.js';
 import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
-import { U as Usage, M as Message, C as CompletionModel, e as JsonValue, n as ToolChoice } from '../types-C8loR4il.js';
-import '../types-CCxnnJjU.js';
-import '../types-CNfjQ5rB.js';
-import '../index-CTNzKidE.js';
-import '../tool-BLypI31e.js';
-import '../middleware-CG7l0rR3.js';
+import { U as Usage, M as Message, C as CompletionModel, e as JsonValue, n as ToolChoice } from '../types-q9G7Bvwl.js';
+import '../guardrails/index.js';
+import '../types-CGML74sE.js';
+import '../types-CqG6d7Ce.js';
+import '../index-DsWWqP0Q.js';
+import '../tool-DGELXD2z.js';
+import '../middleware-BTR0eRtx.js';
 import '../types-DHHZztGO.js';
-import '../types-KKjICcrA.js';
+import '../types-9VPKg7ih.js';
 import 'zod';
 
 type ExtractionResponse<T> = {

package/dist/extractor/index.js

@@ -2,22 +2,23 @@ import {
   ExtractionError,
   Extractor,
   ExtractorBuilder
-} from "../chunk-R75SGHCI.js";
-import "../chunk-62RUYGMM.js";
-import "../chunk-N6REB3DH.js";
-import "../chunk-SISUMRUX.js";
+} from "../chunk-MIJUAWHU.js";
+import "../chunk-TIH4QM55.js";
+import "../chunk-4AJJIIAQ.js";
+import "../chunk-KTTIPWE7.js";
 import "../chunk-YK4WAAS4.js";
 import "../chunk-XUUY2L2D.js";
-import "../chunk-X3SWV6W6.js";
-import "../chunk-5EDHXYH5.js";
+import "../chunk-R7F2JF7J.js";
+import "../chunk-PGFYN5ZX.js";
+import "../chunk-NPZDYOE6.js";
+import "../chunk-MMHG7WAM.js";
 import "../chunk-M6RPFHG7.js";
+import "../chunk-SRBW3LD7.js";
+import "../chunk-5LFGEMIW.js";
+import "../chunk-WQKHFADH.js";
 import "../chunk-JS5MGVNM.js";
 import "../chunk-OIMLU4SF.js";
-import "../chunk-5G3VME2D.js";
-import "../chunk-WJO6NVE2.js";
-import "../chunk-WQKHFADH.js";
-import "../chunk-NPZDYOE6.js";
-import "../chunk-MMHG7WAM.js";
+import "../chunk-YRERPIA5.js";
 export {
   ExtractionError,
   Extractor,

package/dist/guardrails/index.d.ts

@@ -0,0 +1,211 @@
+import { J as JsonObject, e as JsonValue, k as ToolResultContent, M as Message, U as Usage } from '../types-q9G7Bvwl.js';
+
+type MaybePromise<T> = T | Promise<T>;
+type GuardrailMode = "enforce" | "observe";
+type GuardrailBoundary = "input" | "tool" | "tool_result" | "output";
+type GuardrailRunContext = {
+    agentId: string;
+    runId: string;
+    sessionId?: string | undefined;
+    metadata?: JsonObject | undefined;
+};
+type GuardrailDecisionRecord = {
+    policyId: string;
+    guardrailId: string;
+    boundary: GuardrailBoundary;
+    mode: GuardrailMode;
+    action: GuardrailActionName;
+    applied: boolean;
+    reason?: string | undefined;
+    message?: string | undefined;
+    metadata?: JsonObject | undefined;
+    latencyMs: number;
+};
+type GuardrailActionName = "allow" | "block" | "rewrite" | "request_approval" | "error";
+type GuardrailActionBase = {
+    reason?: string | undefined;
+    metadata?: JsonObject | undefined;
+};
+type GuardrailAllow = GuardrailActionBase & {
+    action: "allow";
+};
+type GuardrailBlock = GuardrailActionBase & {
+    action: "block";
+    reason: string;
+    message?: string | undefined;
+};
+type InputGuardrailRewrite = GuardrailActionBase & {
+    action: "rewrite";
+    prompt?: Message | undefined;
+    inputText?: string | undefined;
+};
+type ToolGuardrailRewrite = GuardrailActionBase & {
+    action: "rewrite";
+    args: JsonValue | string;
+};
+type ToolResultGuardrailRewrite = GuardrailActionBase & {
+    action: "rewrite";
+    result?: string | undefined;
+    structuredResult?: ToolResultContent[] | undefined;
+};
+type OutputGuardrailRewrite = GuardrailActionBase & {
+    action: "rewrite";
+    outputText: string;
+};
+type ToolGuardrailApprovalRequest = GuardrailActionBase & {
+    action: "request_approval";
+    reason?: string | undefined;
+    rejectMessage?: string | undefined;
+};
+type InputGuardrailResult = GuardrailAllow | GuardrailBlock | InputGuardrailRewrite;
+type ToolGuardrailResult = GuardrailAllow | GuardrailBlock | ToolGuardrailRewrite | ToolGuardrailApprovalRequest;
+type ToolResultGuardrailResult = GuardrailAllow | GuardrailBlock | ToolResultGuardrailRewrite;
+type OutputGuardrailResult = GuardrailAllow | GuardrailBlock | OutputGuardrailRewrite;
+type GuardrailCommonActions = {
+    allow(options?: GuardrailActionBase): GuardrailAllow;
+    block(options: Omit<GuardrailBlock, "action">): GuardrailBlock;
+};
+type InputGuardrailActions = GuardrailCommonActions & {
+    rewrite(options: Omit<InputGuardrailRewrite, "action">): InputGuardrailRewrite;
+};
+type ToolGuardrailActions = GuardrailCommonActions & {
+    rewrite(options: Omit<ToolGuardrailRewrite, "action">): ToolGuardrailRewrite;
+    requestApproval(options?: Omit<ToolGuardrailApprovalRequest, "action">): ToolGuardrailApprovalRequest;
+};
+type ToolResultGuardrailActions = GuardrailCommonActions & {
+    rewrite(options: Omit<ToolResultGuardrailRewrite, "action">): ToolResultGuardrailRewrite;
+};
+type OutputGuardrailActions = GuardrailCommonActions & {
+    rewrite(options: Omit<OutputGuardrailRewrite, "action">): OutputGuardrailRewrite;
+};
+type InputGuardrailContext = {
+    prompt: Message;
+    history: Message[];
+    inputText: string;
+    run: GuardrailRunContext;
+};
+type ToolGuardrailContext<Args = unknown> = {
+    toolName: string;
+    args: Args;
+    rawArgs: string;
+    toolCallId?: string | undefined;
+    internalCallId: string;
+    turn: number;
+    run: GuardrailRunContext;
+};
+type ToolResultGuardrailContext<Args = unknown> = {
+    toolName: string;
+    args: Args;
+    rawArgs: string;
+    result: string;
+    structuredResult?: ToolResultContent[] | undefined;
+    toolCallId?: string | undefined;
+    internalCallId: string;
+    turn: number;
+    run: GuardrailRunContext;
+};
+type OutputGuardrailContext = {
+    outputText: string;
+    messages: Message[];
+    usage: Usage;
+    run: GuardrailRunContext;
+};
+type InputGuardrail = {
+    id: string;
+    check(context: InputGuardrailContext, actions: InputGuardrailActions): MaybePromise<InputGuardrailResult | undefined>;
+};
+type ToolGuardrail<Args = unknown> = {
+    id: string;
+    tool?: string | string[] | undefined;
+    check(context: ToolGuardrailContext<Args>, actions: ToolGuardrailActions): MaybePromise<ToolGuardrailResult | undefined>;
+};
+type ToolResultGuardrail<Args = unknown> = {
+    id: string;
+    tool?: string | string[] | undefined;
+    check(context: ToolResultGuardrailContext<Args>, actions: ToolResultGuardrailActions): MaybePromise<ToolResultGuardrailResult | undefined>;
+};
+type OutputGuardrail = {
+    id: string;
+    check(context: OutputGuardrailContext, actions: OutputGuardrailActions): MaybePromise<OutputGuardrailResult | undefined>;
+};
+type GuardrailPolicy = {
+    id: string;
+    mode: GuardrailMode;
+    input: InputGuardrail[];
+    tools: ToolGuardrail[];
+    toolResults: ToolResultGuardrail[];
+    output: OutputGuardrail[];
+};
+type GuardrailPolicyOptions = {
+    id: string;
+    mode?: GuardrailMode | undefined;
+    input?: InputGuardrail[] | undefined;
+    tools?: ToolGuardrail[] | undefined;
+    toolResults?: ToolResultGuardrail[] | undefined;
+    output?: OutputGuardrail[] | undefined;
+};
+type GuardrailPolicyInput = GuardrailPolicy | GuardrailPolicy[];
+type InputGuardrailRunResult = {
+    prompt: Message;
+    inputText: string;
+    blocked: boolean;
+    message?: string | undefined;
+    decisions: GuardrailDecisionRecord[];
+};
+type ToolGuardrailRunResult = {
+    rawArgs: string;
+    blocked: boolean;
+    message?: string | undefined;
+    approval?: {
+        reason?: string | undefined;
+        rejectMessage?: string | undefined;
+    } | undefined;
+    decisions: GuardrailDecisionRecord[];
+};
+type ToolResultGuardrailRunResult = {
+    result: string;
+    structuredResult?: ToolResultContent[] | undefined;
+    blocked: boolean;
+    message?: string | undefined;
+    decisions: GuardrailDecisionRecord[];
+};
+type OutputGuardrailRunResult = {
+    outputText: string;
+    blocked: boolean;
+    message?: string | undefined;
+    decisions: GuardrailDecisionRecord[];
+};
+declare function defineGuardrailPolicy(options: GuardrailPolicyOptions): GuardrailPolicy;
+declare function defineInputGuardrail(guardrail: InputGuardrail): InputGuardrail;
+declare function defineToolGuardrail<Args = unknown>(guardrail: ToolGuardrail<Args>): ToolGuardrail<Args>;
+declare function defineToolResultGuardrail<Args = unknown>(guardrail: ToolResultGuardrail<Args>): ToolResultGuardrail<Args>;
+declare function defineOutputGuardrail(guardrail: OutputGuardrail): OutputGuardrail;
+declare function allow(options?: GuardrailActionBase): GuardrailAllow;
+declare function block(options: Omit<GuardrailBlock, "action">): GuardrailBlock;
+declare const guardrails: {
+    blockText: typeof blockText;
+    redactText: typeof redactText;
+};
+declare function normalizeGuardrailPolicies(policies: GuardrailPolicyInput | undefined): GuardrailPolicy[];
+declare function appendGuardrailPolicies(current: GuardrailPolicy[], next: GuardrailPolicyInput): GuardrailPolicy[];
+declare function hasEnforcedOutputGuardrails(policies: GuardrailPolicy[]): boolean;
+declare function runInputGuardrails(policies: GuardrailPolicy[], context: InputGuardrailContext): Promise<InputGuardrailRunResult>;
+declare function runToolGuardrails<Args>(policies: GuardrailPolicy[], toolGuardrails: ToolGuardrail<Args>[], context: ToolGuardrailContext<Args>): Promise<ToolGuardrailRunResult>;
+declare function runToolResultGuardrails<Args>(policies: GuardrailPolicy[], toolGuardrails: ToolResultGuardrail<Args>[], context: ToolResultGuardrailContext<Args>): Promise<ToolResultGuardrailRunResult>;
+declare function runOutputGuardrails(policies: GuardrailPolicy[], context: OutputGuardrailContext): Promise<OutputGuardrailRunResult>;
+type TextPatternBoundary = "input" | "output" | "tool_result";
+type TextPatternGuardrailFor<Boundary extends TextPatternBoundary> = Boundary extends "input" ? InputGuardrail : Boundary extends "output" ? OutputGuardrail : ToolResultGuardrail;
+type TextPatternGuardrailOptions<Boundary extends TextPatternBoundary = TextPatternBoundary> = {
+    id: string;
+    boundary: Boundary;
+    patterns: Array<string | RegExp>;
+    reason: string;
+    message?: string | undefined;
+};
+type TextPatternRedactOptions<Boundary extends TextPatternBoundary = TextPatternBoundary> = TextPatternGuardrailOptions<Boundary> & {
+    replacement?: string | undefined;
+};
+declare function blockText<Boundary extends TextPatternBoundary>(options: TextPatternGuardrailOptions<Boundary>): TextPatternGuardrailFor<Boundary>;
+declare function redactText<Boundary extends TextPatternBoundary>(options: TextPatternRedactOptions<Boundary>): TextPatternGuardrailFor<Boundary>;
+
+export { type GuardrailActionBase, type GuardrailActionName, type GuardrailAllow, type GuardrailBlock, type GuardrailBoundary, type GuardrailCommonActions, type GuardrailDecisionRecord, type GuardrailMode, type GuardrailPolicy, type GuardrailPolicyInput, type GuardrailPolicyOptions, type GuardrailRunContext, type InputGuardrail, type InputGuardrailActions, type InputGuardrailContext, type InputGuardrailResult, type InputGuardrailRewrite, type InputGuardrailRunResult, type OutputGuardrail, type OutputGuardrailActions, type OutputGuardrailContext, type OutputGuardrailResult, type OutputGuardrailRewrite, type OutputGuardrailRunResult, type ToolGuardrail, type ToolGuardrailActions, type ToolGuardrailApprovalRequest, type ToolGuardrailContext, type ToolGuardrailResult, type ToolGuardrailRewrite, type ToolGuardrailRunResult, type ToolResultGuardrail, type ToolResultGuardrailActions, type ToolResultGuardrailContext, type ToolResultGuardrailResult, type ToolResultGuardrailRewrite, type ToolResultGuardrailRunResult, allow, appendGuardrailPolicies, block, defineGuardrailPolicy, defineInputGuardrail, defineOutputGuardrail, defineToolGuardrail, defineToolResultGuardrail, guardrails, hasEnforcedOutputGuardrails, normalizeGuardrailPolicies, runInputGuardrails, runOutputGuardrails, runToolGuardrails, runToolResultGuardrails };

package/dist/guardrails/index.js

@@ -0,0 +1,35 @@
+import {
+  allow,
+  appendGuardrailPolicies,
+  block,
+  defineGuardrailPolicy,
+  defineInputGuardrail,
+  defineOutputGuardrail,
+  defineToolGuardrail,
+  defineToolResultGuardrail,
+  guardrails,
+  hasEnforcedOutputGuardrails,
+  normalizeGuardrailPolicies,
+  runInputGuardrails,
+  runOutputGuardrails,
+  runToolGuardrails,
+  runToolResultGuardrails
+} from "../chunk-YRERPIA5.js";
+export {
+  allow,
+  appendGuardrailPolicies,
+  block,
+  defineGuardrailPolicy,
+  defineInputGuardrail,
+  defineOutputGuardrail,
+  defineToolGuardrail,
+  defineToolResultGuardrail,
+  guardrails,
+  hasEnforcedOutputGuardrails,
+  normalizeGuardrailPolicies,
+  runInputGuardrails,
+  runOutputGuardrails,
+  runToolGuardrails,
+  runToolResultGuardrails
+};
+//# sourceMappingURL=index.js.map

package/dist/guardrails/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/hooks/index.d.ts

@@ -1,6 +1,6 @@
-import { H as HookAction, P as PromptHook, T as ToolApprovalRequestOptions, a as ToolCallHookAction, R as RunControl, b as ToolCallControl } from '../types-CCxnnJjU.js';
-export { C as CompletionCallHookArgs, c as CompletionErrorHookArgs, d as CompletionResponseHookArgs, e as HookResult, f as RunEndHookArgs, g as RunErrorHookArgs, h as RunStartHookArgs, i as ToolCallHookArgs, j as ToolCallHookResult, k as ToolErrorHookArgs, l as ToolHookArgs, m as ToolResultHookArgs, n as TurnEndHookArgs, o as TurnStartHookArgs } from '../types-CCxnnJjU.js';
-import '../types-C8loR4il.js';
+import { H as HookAction, P as PromptHook, T as ToolApprovalRequestOptions, a as ToolCallHookAction, R as RunControl, b as ToolCallControl } from '../types-CGML74sE.js';
+export { C as CompletionCallHookArgs, c as CompletionErrorHookArgs, d as CompletionResponseHookArgs, e as HookResult, f as RunEndHookArgs, g as RunErrorHookArgs, h as RunStartHookArgs, i as ToolCallHookArgs, j as ToolCallHookResult, k as ToolErrorHookArgs, l as ToolHookArgs, m as ToolResultHookArgs, n as TurnEndHookArgs, o as TurnStartHookArgs } from '../types-CGML74sE.js';
+import '../types-q9G7Bvwl.js';
 
 declare function createHook<RawResponse = unknown>(hook: PromptHook<RawResponse>): PromptHook<RawResponse>;
 declare function cancelPrompt(reason: string): HookAction;

package/dist/image-generation/index.d.ts

@@ -1,4 +1,4 @@
-import { e as JsonValue } from '../types-C8loR4il.js';
+import { e as JsonValue } from '../types-q9G7Bvwl.js';
 
 type ImageGenerationRequest = {
     prompt: string;
@@ -16,9 +16,9 @@ type ImageGenerationResponse<RawResponse = unknown> = {
     mediaType?: string | undefined;
     rawResponse: RawResponse;
 };
-interface ImageGenerationModel<RawResponse = unknown> {
+interface ImageGenerationModel<RawResponse = unknown, ModelName extends string = string> {
     readonly provider?: string | undefined;
-    readonly defaultModel?: string | undefined;
+    readonly defaultModel?: ModelName | undefined;
     imageGeneration(request: ImageGenerationRequest): Promise<ImageGenerationResponse<RawResponse>>;
 }
 

package/dist/{index-CTNzKidE.d.ts → index-DsWWqP0Q.d.ts}

@@ -1,11 +1,13 @@
-import { x as ReasoningContentType, f as ToolCall, U as Usage, M as Message, k as ToolResultContent, c as CompletionResponse, b as CompletionRequest, J as JsonObject, t as CompletionModelCapabilities, h as ToolDefinition, e as JsonValue } from './types-C8loR4il.js';
-import { g as ToolCallStreamEvent } from './tool-BLypI31e.js';
+import { x as ReasoningContentType, f as ToolCall, U as Usage, M as Message, k as ToolResultContent, c as CompletionResponse, b as CompletionRequest, J as JsonObject, t as CompletionModelCapabilities, h as ToolDefinition, e as JsonValue } from './types-q9G7Bvwl.js';
+import { GuardrailDecisionRecord } from './guardrails/index.js';
+import { g as ToolCallStreamEvent } from './tool-DGELXD2z.js';
 
 type PromptResponse = {
     output: string;
     usage: Usage;
     messages: Message[];
     trace?: AgentTraceInfo | undefined;
+    guardrails?: GuardrailDecisionRecord[] | undefined;
 };
 type AgentDeltaEvent = {
     type: "text_delta";
@@ -53,6 +55,10 @@ type AgentChildStreamEvent<RawResponse = unknown> = {
     type: "turn_end";
     turn: number;
     response: CompletionResponse<RawResponse>;
+} | {
+    type: "guardrail_decision";
+    turn?: number | undefined;
+    decision: GuardrailDecisionRecord;
 } | {
     type: "final";
     runId: string;
@@ -60,6 +66,7 @@ type AgentChildStreamEvent<RawResponse = unknown> = {
     usage: Usage;
     messages: Message[];
     trace?: AgentTraceInfo | undefined;
+    guardrails?: GuardrailDecisionRecord[] | undefined;
 } | {
     type: "error";
     error: unknown;

package/dist/index.d.ts

@@ -1,22 +1,23 @@
 export { AgentBuilder } from './agent/index.js';
-export { C as CreateCompletionBaseOptions, a as CreateCompletionInput, b as CreateCompletionOptions, c as CreateCompletionResult, d as CreateCompletionStreamOptions, e as CreateParsedCompletionOptions, f as CreateParsedCompletionResult, g as createCompletion, h as createCompletionStream, i as createParsedCompletion } from './create-completion-BEbtU5pX.js';
-export { A as AssistantContent, a as AssistantMessage, C as CompletionModel, b as CompletionRequest, c as CompletionResponse, D as Document, I as ImageContent, J as JsonObject, d as JsonPrimitive, e as JsonValue, M as Message, S as SystemMessage, T as Text, f as ToolCall, g as ToolContent, h as ToolDefinition, i as ToolMessage, j as ToolResult, k as ToolResultContent, U as Usage, l as UserContent, m as UserMessage } from './types-C8loR4il.js';
+export { C as CreateCompletionBaseOptions, a as CreateCompletionInput, b as CreateCompletionOptions, c as CreateCompletionResult, d as CreateCompletionStreamOptions, e as CreateParsedCompletionOptions, f as CreateParsedCompletionResult, g as createCompletion, h as createCompletionStream, i as createParsedCompletion } from './create-completion-Y6YsYbQB.js';
+export { A as AssistantContent, a as AssistantMessage, C as CompletionModel, b as CompletionRequest, c as CompletionResponse, D as Document, I as ImageContent, J as JsonObject, d as JsonPrimitive, e as JsonValue, M as Message, S as SystemMessage, T as Text, f as ToolCall, g as ToolContent, h as ToolDefinition, i as ToolMessage, j as ToolResult, k as ToolResultContent, U as Usage, l as UserContent, m as UserMessage } from './types-q9G7Bvwl.js';
 export { cancelPrompt, createHook, requestToolApproval, runControl, skipTool, toolCallControl } from './hooks/index.js';
-export { M as MemoryStore } from './types-CNfjQ5rB.js';
+export { GuardrailBoundary, GuardrailDecisionRecord, GuardrailMode, GuardrailPolicy, GuardrailPolicyInput, GuardrailPolicyOptions, InputGuardrail, InputGuardrailActions, InputGuardrailContext, OutputGuardrail, OutputGuardrailActions, OutputGuardrailContext, ToolGuardrail, ToolGuardrailActions, ToolGuardrailContext, ToolResultGuardrail, ToolResultGuardrailActions, ToolResultGuardrailContext, allow, block, defineGuardrailPolicy, defineInputGuardrail, defineOutputGuardrail, defineToolGuardrail, defineToolResultGuardrail, guardrails } from './guardrails/index.js';
+export { M as MemoryStore } from './types-CqG6d7Ce.js';
 export { MaxTurnsError, PromptCancelledError, ToolApprovalRequiredError } from './request/index.js';
-export { A as AgentChildStreamEvent, a as AgentStreamEvent, P as PromptResponse } from './index-CTNzKidE.js';
+export { A as AgentChildStreamEvent, a as AgentStreamEvent, P as PromptResponse } from './index-DsWWqP0Q.js';
 export { Z as ZodSchema } from './zod-schema-C7F4clpm.js';
 export { loadSkills, skill } from './skills/index.js';
-export { S as SkillValidationError } from './types-D-YTJdLj.js';
-export { C as CreateToolOptions, c as createThinkTool, a as createTool } from './think-tool-B8Rqw2i_.js';
-export { A as AnyTool, T as Tool, a as ToolApprovalContext, b as ToolApprovalDecision, c as ToolApprovalPolicy, d as ToolApprovalRequest, e as ToolApprovalsOptions, f as ToolCallContext, g as ToolCallStreamEvent } from './tool-BLypI31e.js';
-export { A as AgentMiddleware, C as CompletionRequestMiddlewareArgs, a as CompletionRequestMiddlewareResult, b as CompletionResponseMiddlewareArgs, c as CompletionResponseMiddlewareResult, T as ToolInputMiddlewareArgs, d as ToolInputMiddlewareResult, e as ToolMiddleware, f as ToolOutputMiddlewareArgs, g as ToolOutputMiddlewareResult, h as ToolResultMiddlewareArgs, i as createMiddleware, j as createToolMiddleware } from './middleware-CG7l0rR3.js';
+export { S as SkillValidationError } from './types-Bx_ZtjPi.js';
+export { C as CreateToolOptions, c as createThinkTool, a as createTool } from './think-tool-Bn3HHu_z.js';
+export { A as AnyTool, T as Tool, a as ToolApprovalContext, b as ToolApprovalDecision, c as ToolApprovalPolicy, d as ToolApprovalRequest, e as ToolApprovalsOptions, f as ToolCallContext, g as ToolCallStreamEvent } from './tool-DGELXD2z.js';
+export { A as AgentMiddleware, C as CompletionRequestMiddlewareArgs, a as CompletionRequestMiddlewareResult, b as CompletionResponseMiddlewareArgs, c as CompletionResponseMiddlewareResult, T as ToolInputMiddlewareArgs, d as ToolInputMiddlewareResult, e as ToolMiddleware, f as ToolOutputMiddlewareArgs, g as ToolOutputMiddlewareResult, h as ToolResultMiddlewareArgs, i as createMiddleware, j as createToolMiddleware } from './middleware-BTR0eRtx.js';
 export { UIError, UIMessage, UIMessagePart, UIMessageRole, UIStreamEvent, UIStreamRequest, coreMessagesToUIMessages, uiMessagesToCoreMessages } from './ui/index.js';
-import './agent-Bq2KtSaw.js';
-import './types-CCxnnJjU.js';
-import './types-KKjICcrA.js';
+import './agent-BWF8_O8k.js';
+import './types-CGML74sE.js';
+import './types-9VPKg7ih.js';
 import './types-DHHZztGO.js';
-import './types-CIwm9p4w.js';
+import './types-Dc83Qzch.js';
 import '@modelcontextprotocol/sdk/client/sse.js';
 import '@modelcontextprotocol/sdk/client/stdio.js';
 import '@modelcontextprotocol/sdk/client/streamableHttp.js';