@anvia/core 0.17.0 → 0.18.0

package/dist/chunk-ZNTIUOKL.js

@@ -0,0 +1,33 @@
+import {
+  assertNonnegativeInteger,
+  assertPositiveInteger
+} from "./chunk-TGLNXVII.js";
+
+// src/memory/options.ts
+function resolveMemoryOptions(options = {}) {
+  const resolved = {
+    savePolicy: options.savePolicy ?? "message"
+  };
+  if (options.compaction !== void 0) {
+    assertPositiveInteger(options.compaction.maxMessages, "compaction.maxMessages");
+    const keepRecentUserTurns = options.compaction.keepRecentUserTurns ?? 4;
+    const conflictRetries = options.compaction.conflictRetries ?? 1;
+    assertPositiveInteger(keepRecentUserTurns, "compaction.keepRecentUserTurns");
+    assertNonnegativeInteger(conflictRetries, "compaction.conflictRetries");
+    if (typeof options.compaction.compactor !== "function") {
+      throw new TypeError("compaction.compactor must be a function.");
+    }
+    resolved.compaction = {
+      maxMessages: options.compaction.maxMessages,
+      keepRecentUserTurns,
+      compactor: options.compaction.compactor,
+      conflictRetries
+    };
+  }
+  return resolved;
+}
+
+export {
+  resolveMemoryOptions
+};
+//# sourceMappingURL=chunk-ZNTIUOKL.js.map

package/dist/chunk-ZNTIUOKL.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/memory/options.ts"],"sourcesContent":["import { assertNonnegativeInteger, assertPositiveInteger } from \"./assert\";\nimport type { MemoryOptions, ResolvedMemoryOptions } from \"./types\";\n\nexport function resolveMemoryOptions(options: MemoryOptions = {}): ResolvedMemoryOptions {\n  const resolved: ResolvedMemoryOptions = {\n    savePolicy: options.savePolicy ?? \"message\",\n  };\n  if (options.compaction !== undefined) {\n    assertPositiveInteger(options.compaction.maxMessages, \"compaction.maxMessages\");\n    const keepRecentUserTurns = options.compaction.keepRecentUserTurns ?? 4;\n    const conflictRetries = options.compaction.conflictRetries ?? 1;\n    assertPositiveInteger(keepRecentUserTurns, \"compaction.keepRecentUserTurns\");\n    assertNonnegativeInteger(conflictRetries, \"compaction.conflictRetries\");\n    if (typeof options.compaction.compactor !== \"function\") {\n      throw new TypeError(\"compaction.compactor must be a function.\");\n    }\n    resolved.compaction = {\n      maxMessages: options.compaction.maxMessages,\n      keepRecentUserTurns,\n      compactor: options.compaction.compactor,\n      conflictRetries,\n    };\n  }\n  return resolved;\n}\n"],"mappings":";;;;;;AAGO,SAAS,qBAAqB,UAAyB,CAAC,GAA0B;AACvF,QAAM,WAAkC;AAAA,IACtC,YAAY,QAAQ,cAAc;AAAA,EACpC;AACA,MAAI,QAAQ,eAAe,QAAW;AACpC,0BAAsB,QAAQ,WAAW,aAAa,wBAAwB;AAC9E,UAAM,sBAAsB,QAAQ,WAAW,uBAAuB;AACtE,UAAM,kBAAkB,QAAQ,WAAW,mBAAmB;AAC9D,0BAAsB,qBAAqB,gCAAgC;AAC3E,6BAAyB,iBAAiB,4BAA4B;AACtE,QAAI,OAAO,QAAQ,WAAW,cAAc,YAAY;AACtD,YAAM,IAAI,UAAU,0CAA0C;AAAA,IAChE;AACA,aAAS,aAAa;AAAA,MACpB,aAAa,QAAQ,WAAW;AAAA,MAChC;AAAA,MACA,WAAW,QAAQ,WAAW;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;","names":[]}

package/dist/completion/index.js

@@ -1,12 +1,14 @@
 import {
   CompletionRequestBuilder,
+  formatDocument,
+  normalizeDocuments
+} from "../chunk-MTGNA4OS.js";
+import {
   createCompletion,
   createCompletionStream,
   createParsedCompletion,
-  formatDocument,
-  isStreamingCompletionModel,
-  normalizeDocuments
-} from "../chunk-GCED5ZNM.js";
+  isStreamingCompletionModel
+} from "../chunk-F56AMXMQ.js";
 import {
   AssistantContent,
   CompletionCapabilityError,

package/dist/errors-Bf661V89.d.ts

@@ -0,0 +1,17 @@
+import { C as CompletionModel, M as Message, U as Usage } from './types-XjIh933x.js';
+import { S as SummaryMemoryCompactorOptions, e as MemoryCompactor } from './types-DBHzPsbx.js';
+
+declare function createSummaryMemoryCompactor(model: CompletionModel, options?: SummaryMemoryCompactorOptions): MemoryCompactor;
+declare function isMemoryCompactionSummary(message: Message): boolean;
+
+declare class MemoryCompactionError extends Error {
+    readonly usage: Usage | undefined;
+    constructor(message: string, options?: ErrorOptions & {
+        usage?: Usage | undefined;
+    });
+}
+declare class MemoryCompactionConflictError extends MemoryCompactionError {
+    constructor(attempts: number, usage?: Usage);
+}
+
+export { MemoryCompactionConflictError as M, MemoryCompactionError as a, createSummaryMemoryCompactor as c, isMemoryCompactionSummary as i };

package/dist/evals/index.d.ts

@@ -1,11 +1,11 @@
-import { A as Agent } from '../agent-Cg0UigNK.js';
+import { A as Agent } from '../agent-CBz3CRxa.js';
 import { i as JsonValue, M as Message, C as CompletionModel } from '../types-XjIh933x.js';
 import { P as PromptResponse } from '../index-XTUEMWhU.js';
 import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
 import { E as EmbeddingModel } from '../types-DHHZztGO.js';
 import '../guardrails/index.js';
 import '../types-BCA8p0sb.js';
-import '../types-CBmhLT_r.js';
+import '../types-DBHzPsbx.js';
 import '../tool-Dhwg__1L.js';
 import '../middleware-Cp0tm6zE.js';
 import '../types-DMGvEFRf.js';

package/dist/evals/index.js

@@ -1,15 +1,20 @@
 import {
   ExtractorBuilder
-} from "../chunk-54S4I3SX.js";
-import "../chunk-RWAOHGRL.js";
-import "../chunk-BQCTJSNL.js";
-import "../chunk-OPAEENGD.js";
+} from "../chunk-3YJXLOGO.js";
+import "../chunk-DPTZ2MK4.js";
+import "../chunk-QABIBFKE.js";
+import "../chunk-7LCKIULT.js";
 import "../chunk-YK4WAAS4.js";
 import "../chunk-XUUY2L2D.js";
-import "../chunk-SWUIBS4N.js";
-import "../chunk-F3C5653H.js";
+import "../chunk-JNJBKSLA.js";
+import "../chunk-I7TM7TKX.js";
 import "../chunk-2ODTMRHP.js";
-import "../chunk-M6RPFHG7.js";
+import "../chunk-ZNTIUOKL.js";
+import "../chunk-TGLNXVII.js";
+import "../chunk-MTGNA4OS.js";
+import "../chunk-F56AMXMQ.js";
+import "../chunk-PJDL5UT2.js";
+import "../chunk-WQKHFADH.js";
 import {
   cosineSimilarity,
   embedText
@@ -17,9 +22,6 @@ import {
 import {
   mapWithConcurrency
 } from "../chunk-OIMLU4SF.js";
-import "../chunk-GCED5ZNM.js";
-import "../chunk-PJDL5UT2.js";
-import "../chunk-WQKHFADH.js";
 import "../chunk-CWUJUSOS.js";
 
 // src/evals/agent-target.ts

package/dist/evals/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/evals/agent-target.ts","../../src/evals/metric.ts","../../src/evals/metrics.ts","../../src/evals/format.ts","../../src/evals/outcome.ts","../../src/evals/selectors.ts","../../src/evals/runner.ts"],"sourcesContent":["import type { Agent } from \"../agent/agent\";\nimport type { Message } from \"../completion\";\nimport type { PromptResponse } from \"../request/types\";\nimport type { EvalCase, EvalTarget } from \"./types\";\n\nexport type AgentEvalTargetOptions<Input, Output = PromptResponse> = {\n  prompt?: ((input: Input, testCase: EvalCase<Input>) => string | Message) | undefined;\n  output?: ((response: PromptResponse, testCase: EvalCase<Input>) => Output) | undefined;\n};\n\nexport function agentEvalTarget<Input>(\n  agent: Agent,\n  options?: AgentEvalTargetOptions<Input, PromptResponse>,\n): EvalTarget<Input, PromptResponse>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output>,\n): EvalTarget<Input, Output>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output | PromptResponse> = {},\n): EvalTarget<Input, Output | PromptResponse> {\n  return async (input, testCase) => {\n    const prompt = options.prompt?.(input, testCase) ?? String(input);\n    const response = await agent.prompt(prompt).send();\n    return options.output === undefined ? response : options.output(response, testCase);\n  };\n}\n","import type { EvalMetric } from \"./types\";\n\nexport function defineMetric<Input, Output, Score, Expected>(\n  metric: EvalMetric<Input, Output, Score, Expected>,\n): EvalMetric<Input, Output, Score, Expected> {\n  return metric;\n}\n","import { z } from \"zod\";\nimport type { CompletionModel } from \"../completion\";\nimport { cosineSimilarity, type EmbeddingModel, embedText } from \"../embeddings\";\nimport { ExtractorBuilder } from \"../extractor\";\nimport type { ZodSchema } from \"../schema\";\nimport { errorMessage, formatValue, stableComparable } from \"./format\";\nimport { EvalOutcome } from \"./outcome\";\nimport { resolveActual, resolveActualText, resolveExpected, resolveJudgePrompt } from \"./selectors\";\nimport type { EvalMetric, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport type ExactMatchOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, unknown> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, unknown> | undefined;\n};\n\nexport function exactMatch<Input, Output, Expected = unknown>(\n  options: ExactMatchOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"exact_match\",\n    async evaluate(args) {\n      const actual = await resolveActual(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for exact match.\");\n      }\n      const passed = stableComparable(actual) === stableComparable(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });\n    },\n  };\n}\n\nexport type ContainsOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string | RegExp> | undefined;\n};\n\nexport function contains<Input, Output, Expected = unknown>(\n  options: ContainsOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"contains\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for contains.\");\n      }\n      if (typeof expected !== \"string\" && !(expected instanceof RegExp)) {\n        return EvalOutcome.invalid(\"Contains expected value must be a string or RegExp.\");\n      }\n      const passed =\n        expected instanceof RegExp ? regexMatches(expected, actual) : actual.includes(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Output did not contain ${String(expected)}.` });\n    },\n  };\n}\n\nfunction regexMatches(pattern: RegExp, text: string): boolean {\n  pattern.lastIndex = 0;\n  const matched = pattern.test(text);\n  pattern.lastIndex = 0;\n  return matched;\n}\n\nexport type SemanticSimilarityOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: EmbeddingModel;\n  threshold: number;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string> | undefined;\n};\n\nexport function semanticSimilarity<Input, Output, Expected = unknown>(\n  options: SemanticSimilarityOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, number, Expected> {\n  return {\n    name: options.name ?? \"semantic_similarity\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for semantic similarity.\");\n      }\n      if (typeof expected !== \"string\") {\n        return EvalOutcome.invalid(\"Semantic similarity expected value must be a string.\");\n      }\n      const [actualEmbedding, expectedEmbedding] = await Promise.all([\n        embedText(options.model, actual),\n        embedText(options.model, expected),\n      ]);\n      const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);\n      return score >= options.threshold\n        ? EvalOutcome.pass(score)\n        : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });\n    },\n  };\n}\n\nexport type LlmJudgeOptions<Input, Output, SchemaOutput, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  schema: ZodSchema<SchemaOutput>;\n  passes(value: SchemaOutput): boolean;\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmJudge<Input, Output, SchemaOutput, Expected = unknown>(\n  options: LlmJudgeOptions<Input, Output, SchemaOutput, Expected>,\n): EvalMetric<Input, Output, SchemaOutput, Expected> {\n  const extractor = new ExtractorBuilder(options.model, options.schema)\n    .instructions(\n      options.instructions ??\n        \"Judge the eval case by the requested schema. Submit the judgment using the schema.\",\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_judge\",\n    async evaluate(args) {\n      try {\n        const judgment = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        return options.passes(judgment) ? EvalOutcome.pass(judgment) : EvalOutcome.fail(judgment);\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n\nexport type LlmScoreMetricScore = {\n  score: number;\n  feedback: string;\n};\n\nexport type LlmScoreOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  threshold: number;\n  criteria: string | string[];\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmScore<Input, Output, Expected = unknown>(\n  options: LlmScoreOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, LlmScoreMetricScore, Expected> {\n  const criteria = Array.isArray(options.criteria) ? options.criteria.join(\"\\n\") : options.criteria;\n  const extractor = new ExtractorBuilder(\n    options.model,\n    z.object({\n      score: z.number(),\n      feedback: z.string(),\n    }),\n  )\n    .instructions(\n      options.instructions ??\n        `Score the eval case against these criteria:\\n${criteria}\\n\\nReturn a score between 0 and 1 and brief feedback.`,\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_score\",\n    async evaluate(args) {\n      try {\n        const score = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        if (score.score < 0 || score.score > 1) {\n          return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {\n            score,\n          });\n        }\n        return score.score >= options.threshold\n          ? EvalOutcome.pass(score, { comment: score.feedback })\n          : EvalOutcome.fail(score, { comment: score.feedback });\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n","export function defaultOutputValue(output: unknown): unknown {\n  if (\n    typeof output === \"object\" &&\n    output !== null &&\n    \"output\" in output &&\n    typeof (output as { output?: unknown }).output === \"string\"\n  ) {\n    return (output as { output: string }).output;\n  }\n  return output;\n}\n\nexport function stableComparable(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  return JSON.stringify(value);\n}\n\nexport function formatValue(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  try {\n    return JSON.stringify(value);\n  } catch {\n    return String(value);\n  }\n}\n\nexport function errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n","import type { EvalMetadata } from \"./types\";\n\nexport type EvalOutcome<Score = unknown> =\n  | {\n      outcome: \"pass\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"fail\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"invalid\";\n      reason: string;\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    };\n\nexport const EvalOutcome = {\n  pass<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    const outcome: EvalOutcome<Score> = {\n      outcome: \"pass\" as const,\n    };\n    if (score !== undefined) {\n      outcome.score = score;\n    }\n    if (options.comment !== undefined) {\n      outcome.comment = options.comment;\n    }\n    if (options.metadata !== undefined) {\n      outcome.metadata = options.metadata;\n    }\n    return outcome;\n  },\n\n  fail<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    const outcome: EvalOutcome<Score> = {\n      outcome: \"fail\" as const,\n    };\n    if (score !== undefined) {\n      outcome.score = score;\n    }\n    if (options.comment !== undefined) {\n      outcome.comment = options.comment;\n    }\n    if (options.metadata !== undefined) {\n      outcome.metadata = options.metadata;\n    }\n    return outcome;\n  },\n\n  invalid<Score = never>(\n    reason: string,\n    options: {\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    } = {},\n  ): EvalOutcome<Score> {\n    const outcome: EvalOutcome<Score> = {\n      outcome: \"invalid\" as const,\n      reason,\n    };\n    if (options.score !== undefined) {\n      outcome.score = options.score;\n    }\n    if (options.comment !== undefined) {\n      outcome.comment = options.comment;\n    }\n    if (options.metadata !== undefined) {\n      outcome.metadata = options.metadata;\n    }\n    return outcome;\n  },\n};\n","import { defaultOutputValue, formatValue } from \"./format\";\nimport type { EvalMetricArgs, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport async function resolveActual<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, unknown> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<unknown> {\n  return selector === undefined ? defaultOutputValue(args.output) : selector(args);\n}\n\nexport async function resolveActualText<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  const value = selector === undefined ? defaultOutputValue(args.output) : await selector(args);\n  return typeof value === \"string\" ? value : JSON.stringify(value);\n}\n\nexport async function resolveExpected<Input, Output, Expected, Value>(\n  selectorOrValue: SelectorOrValue<Input, Output, Expected, Value> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<Value | Expected | undefined> {\n  if (selectorOrValue === undefined) {\n    return args.case.expected;\n  }\n  return typeof selectorOrValue === \"function\"\n    ? (selectorOrValue as ValueSelector<Input, Output, Expected, Value>)(args)\n    : selectorOrValue;\n}\n\nexport async function resolveJudgePrompt<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  if (selector !== undefined) {\n    return selector(args);\n  }\n  return [\n    `Suite: ${args.suiteName}`,\n    `Case: ${args.case.id}`,\n    `Input: ${formatValue(args.case.input)}`,\n    `Expected: ${formatValue(args.case.expected)}`,\n    `Output: ${formatValue(defaultOutputValue(args.output))}`,\n  ].join(\"\\n\\n\");\n}\n","import { mapWithConcurrency } from \"../internal/concurrency\";\nimport { errorMessage } from \"./format\";\nimport { EvalOutcome, type EvalOutcome as EvalOutcomeType } from \"./outcome\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalMetric,\n  EvalMetricResult,\n  EvalReporter,\n  EvalSuiteResult,\n  RunEvalSuiteOptions,\n} from \"./types\";\n\nexport async function runEvalSuite<Input, Output, Expected = unknown>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n): Promise<EvalSuiteResult<Input, Output, Expected>> {\n  const startedAt = Date.now();\n  const results = await mapWithConcurrency(\n    options.cases,\n    Math.max(1, Math.trunc(options.concurrency ?? 1)),\n    (testCase) => runEvalCase(options, testCase),\n  );\n  const counts = countOutcomes(results);\n  return {\n    name: options.name,\n    results,\n    ...counts,\n    durationMs: Date.now() - startedAt,\n  };\n}\n\nasync function runEvalCase<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n  testCase: EvalCase<Input, Expected>,\n): Promise<EvalCaseResult<Input, Output, Expected>> {\n  let output: Output | undefined;\n  let targetError: unknown;\n  try {\n    output = await options.target(testCase.input, testCase);\n  } catch (error) {\n    targetError = error;\n  }\n\n  const metrics: EvalMetricResult[] = [];\n  for (const metric of options.metrics) {\n    const outcome =\n      targetError === undefined\n        ? await safeEvaluate(options.name, testCase, output as Output, metric)\n        : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`);\n    const reporterErrors = await reportOutcome({\n      suiteName: options.name,\n      testCase,\n      output,\n      targetError,\n      metric,\n      outcome,\n      reporters: options.reporters ?? [],\n      failOnReporterError: options.failOnReporterError === true,\n    });\n    metrics.push({ metricName: metric.name, outcome, reporterErrors });\n  }\n\n  const result: EvalCaseResult<Input, Output, Expected> = {\n    case: testCase,\n    metrics,\n  };\n  if (output !== undefined) {\n    result.output = output;\n  }\n  if (targetError !== undefined) {\n    result.targetError = targetError;\n  }\n  return result;\n}\n\nasync function safeEvaluate<Input, Output, Expected>(\n  suiteName: string,\n  testCase: EvalCase<Input, Expected>,\n  output: Output,\n  metric: EvalMetric<Input, Output, unknown, Expected>,\n): Promise<EvalOutcomeType> {\n  try {\n    return await metric.evaluate({ suiteName, case: testCase, output });\n  } catch (error) {\n    return EvalOutcome.invalid(errorMessage(error));\n  }\n}\n\nasync function reportOutcome<Input, Output, Expected>(args: {\n  suiteName: string;\n  testCase: EvalCase<Input, Expected>;\n  output: Output | undefined;\n  targetError: unknown;\n  metric: EvalMetric<Input, Output, unknown, Expected>;\n  outcome: EvalOutcomeType;\n  reporters: Array<EvalReporter<Input, Output, Expected>>;\n  failOnReporterError: boolean;\n}): Promise<unknown[]> {\n  const errors: unknown[] = [];\n  for (const reporter of args.reporters) {\n    try {\n      await reporter.report({\n        suiteName: args.suiteName,\n        case: args.testCase,\n        output: args.output,\n        targetError: args.targetError,\n        metric: args.metric,\n        outcome: args.outcome,\n      });\n    } catch (error) {\n      if (args.failOnReporterError) {\n        throw error;\n      }\n      errors.push(error);\n    }\n  }\n  return errors;\n}\n\nfunction countOutcomes(results: Array<EvalCaseResult<unknown, unknown, unknown>>): {\n  passed: number;\n  failed: number;\n  invalid: number;\n} {\n  let passed = 0;\n  let failed = 0;\n  let invalid = 0;\n  for (const result of results) {\n    for (const metric of result.metrics) {\n      if (metric.outcome.outcome === \"pass\") passed += 1;\n      if (metric.outcome.outcome === \"fail\") failed += 1;\n      if (metric.outcome.outcome === \"invalid\") invalid += 1;\n    }\n  }\n  return { passed, failed, invalid };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAkBO,SAAS,gBACd,OACA,UAAkE,CAAC,GACvB;AAC5C,SAAO,OAAO,OAAO,aAAa;AAChC,UAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,KAAK,OAAO,KAAK;AAChE,UAAM,WAAW,MAAM,MAAM,OAAO,MAAM,EAAE,KAAK;AACjD,WAAO,QAAQ,WAAW,SAAY,WAAW,QAAQ,OAAO,UAAU,QAAQ;AAAA,EACpF;AACF;;;ACzBO,SAAS,aACd,QAC4C;AAC5C,SAAO;AACT;;;ACNA,SAAS,SAAS;;;ACAX,SAAS,mBAAmB,QAA0B;AAC3D,MACE,OAAO,WAAW,YAClB,WAAW,QACX,YAAY,UACZ,OAAQ,OAAgC,WAAW,UACnD;AACA,WAAQ,OAA8B;AAAA,EACxC;AACA,SAAO;AACT;AAEO,SAAS,iBAAiB,OAAwB;AACvD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,SAAO,KAAK,UAAU,KAAK;AAC7B;AAEO,SAAS,YAAY,OAAwB;AAClD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,MAAI;AACF,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B,QAAQ;AACN,WAAO,OAAO,KAAK;AAAA,EACrB;AACF;AAEO,SAAS,aAAa,OAAwB;AACnD,SAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC9D;;;ACTO,IAAM,cAAc;AAAA,EACzB,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,UAAM,UAA8B;AAAA,MAClC,SAAS;AAAA,IACX;AACA,QAAI,UAAU,QAAW;AACvB,cAAQ,QAAQ;AAAA,IAClB;AACA,QAAI,QAAQ,YAAY,QAAW;AACjC,cAAQ,UAAU,QAAQ;AAAA,IAC5B;AACA,QAAI,QAAQ,aAAa,QAAW;AAClC,cAAQ,WAAW,QAAQ;AAAA,IAC7B;AACA,WAAO;AAAA,EACT;AAAA,EAEA,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,UAAM,UAA8B;AAAA,MAClC,SAAS;AAAA,IACX;AACA,QAAI,UAAU,QAAW;AACvB,cAAQ,QAAQ;AAAA,IAClB;AACA,QAAI,QAAQ,YAAY,QAAW;AACjC,cAAQ,UAAU,QAAQ;AAAA,IAC5B;AACA,QAAI,QAAQ,aAAa,QAAW;AAClC,cAAQ,WAAW,QAAQ;AAAA,IAC7B;AACA,WAAO;AAAA,EACT;AAAA,EAEA,QACE,QACA,UAII,CAAC,GACe;AACpB,UAAM,UAA8B;AAAA,MAClC,SAAS;AAAA,MACT;AAAA,IACF;AACA,QAAI,QAAQ,UAAU,QAAW;AAC/B,cAAQ,QAAQ,QAAQ;AAAA,IAC1B;AACA,QAAI,QAAQ,YAAY,QAAW;AACjC,cAAQ,UAAU,QAAQ;AAAA,IAC5B;AACA,QAAI,QAAQ,aAAa,QAAW;AAClC,cAAQ,WAAW,QAAQ;AAAA,IAC7B;AACA,WAAO;AAAA,EACT;AACF;;;AClFA,eAAsB,cACpB,UACA,MACkB;AAClB,SAAO,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,SAAS,IAAI;AACjF;AAEA,eAAsB,kBACpB,UACA,MACiB;AACjB,QAAM,QAAQ,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,MAAM,SAAS,IAAI;AAC5F,SAAO,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK;AACjE;AAEA,eAAsB,gBACpB,iBACA,MACuC;AACvC,MAAI,oBAAoB,QAAW;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AACA,SAAO,OAAO,oBAAoB,aAC7B,gBAAkE,IAAI,IACvE;AACN;AAEA,eAAsB,mBACpB,UACA,MACiB;AACjB,MAAI,aAAa,QAAW;AAC1B,WAAO,SAAS,IAAI;AAAA,EACtB;AACA,SAAO;AAAA,IACL,UAAU,KAAK,SAAS;AAAA,IACxB,SAAS,KAAK,KAAK,EAAE;AAAA,IACrB,UAAU,YAAY,KAAK,KAAK,KAAK,CAAC;AAAA,IACtC,aAAa,YAAY,KAAK,KAAK,QAAQ,CAAC;AAAA,IAC5C,WAAW,YAAY,mBAAmB,KAAK,MAAM,CAAC,CAAC;AAAA,EACzD,EAAE,KAAK,MAAM;AACf;;;AH5BO,SAAS,WACd,UAAsD,CAAC,GACT;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,cAAc,QAAQ,QAAQ,IAAI;AACvD,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,6CAA6C;AAAA,MAC1E;AACA,YAAM,SAAS,iBAAiB,MAAM,MAAM,iBAAiB,QAAQ;AACrE,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,YAAY,YAAY,QAAQ,CAAC,IAAI,CAAC;AAAA,IAC/E;AAAA,EACF;AACF;AAQO,SAAS,SACd,UAAoD,CAAC,GACP;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,0CAA0C;AAAA,MACvE;AACA,UAAI,OAAO,aAAa,YAAY,EAAE,oBAAoB,SAAS;AACjE,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,YAAM,SACJ,oBAAoB,SAAS,aAAa,UAAU,MAAM,IAAI,OAAO,SAAS,QAAQ;AACxF,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,0BAA0B,OAAO,QAAQ,CAAC,IAAI,CAAC;AAAA,IACxF;AAAA,EACF;AACF;AAEA,SAAS,aAAa,SAAiB,MAAuB;AAC5D,UAAQ,YAAY;AACpB,QAAM,UAAU,QAAQ,KAAK,IAAI;AACjC,UAAQ,YAAY;AACpB,SAAO;AACT;AAUO,SAAS,mBACd,SAC6C;AAC7C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,UAAI,OAAO,aAAa,UAAU;AAChC,eAAO,YAAY,QAAQ,sDAAsD;AAAA,MACnF;AACA,YAAM,CAAC,iBAAiB,iBAAiB,IAAI,MAAM,QAAQ,IAAI;AAAA,QAC7D,UAAU,QAAQ,OAAO,MAAM;AAAA,QAC/B,UAAU,QAAQ,OAAO,QAAQ;AAAA,MACnC,CAAC;AACD,YAAM,QAAQ,iBAAiB,gBAAgB,QAAQ,kBAAkB,MAAM;AAC/E,aAAO,SAAS,QAAQ,YACpB,YAAY,KAAK,KAAK,IACtB,YAAY,KAAK,OAAO,EAAE,SAAS,8BAA8B,QAAQ,SAAS,IAAI,CAAC;AAAA,IAC7F;AAAA,EACF;AACF;AAYO,SAAS,SACd,SACmD;AACnD,QAAM,YAAY,IAAI,iBAAiB,QAAQ,OAAO,QAAQ,MAAM,EACjE;AAAA,IACC,QAAQ,gBACN;AAAA,EACJ,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,WAAW,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACvF,eAAO,QAAQ,OAAO,QAAQ,IAAI,YAAY,KAAK,QAAQ,IAAI,YAAY,KAAK,QAAQ;AAAA,MAC1F,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;AAiBO,SAAS,SACd,SAC0D;AAC1D,QAAM,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAAI,QAAQ,SAAS,KAAK,IAAI,IAAI,QAAQ;AACzF,QAAM,YAAY,IAAI;AAAA,IACpB,QAAQ;AAAA,IACR,EAAE,OAAO;AAAA,MACP,OAAO,EAAE,OAAO;AAAA,MAChB,UAAU,EAAE,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,EACG;AAAA,IACC,QAAQ,gBACN;AAAA,EAAgD,QAAQ;AAAA;AAAA;AAAA,EAC5D,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,QAAQ,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACpF,YAAI,MAAM,QAAQ,KAAK,MAAM,QAAQ,GAAG;AACtC,iBAAO,YAAY,QAAQ,SAAS,MAAM,KAAK,gCAAgC;AAAA,YAC7E;AAAA,UACF,CAAC;AAAA,QACH;AACA,eAAO,MAAM,SAAS,QAAQ,YAC1B,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC,IACnD,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC;AAAA,MACzD,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;;;AIjLA,eAAsB,aACpB,SACmD;AACnD,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,UAAU,MAAM;AAAA,IACpB,QAAQ;AAAA,IACR,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,eAAe,CAAC,CAAC;AAAA,IAChD,CAAC,aAAa,YAAY,SAAS,QAAQ;AAAA,EAC7C;AACA,QAAM,SAAS,cAAc,OAAO;AACpC,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd;AAAA,IACA,GAAG;AAAA,IACH,YAAY,KAAK,IAAI,IAAI;AAAA,EAC3B;AACF;AAEA,eAAe,YACb,SACA,UACkD;AAClD,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,QAAQ,OAAO,SAAS,OAAO,QAAQ;AAAA,EACxD,SAAS,OAAO;AACd,kBAAc;AAAA,EAChB;AAEA,QAAM,UAA8B,CAAC;AACrC,aAAW,UAAU,QAAQ,SAAS;AACpC,UAAM,UACJ,gBAAgB,SACZ,MAAM,aAAa,QAAQ,MAAM,UAAU,QAAkB,MAAM,IACnE,YAAY,QAAQ,kBAAkB,aAAa,WAAW,CAAC,EAAE;AACvE,UAAM,iBAAiB,MAAM,cAAc;AAAA,MACzC,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW,QAAQ,aAAa,CAAC;AAAA,MACjC,qBAAqB,QAAQ,wBAAwB;AAAA,IACvD,CAAC;AACD,YAAQ,KAAK,EAAE,YAAY,OAAO,MAAM,SAAS,eAAe,CAAC;AAAA,EACnE;AAEA,QAAM,SAAkD;AAAA,IACtD,MAAM;AAAA,IACN;AAAA,EACF;AACA,MAAI,WAAW,QAAW;AACxB,WAAO,SAAS;AAAA,EAClB;AACA,MAAI,gBAAgB,QAAW;AAC7B,WAAO,cAAc;AAAA,EACvB;AACA,SAAO;AACT;AAEA,eAAe,aACb,WACA,UACA,QACA,QAC0B;AAC1B,MAAI;AACF,WAAO,MAAM,OAAO,SAAS,EAAE,WAAW,MAAM,UAAU,OAAO,CAAC;AAAA,EACpE,SAAS,OAAO;AACd,WAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,EAChD;AACF;AAEA,eAAe,cAAuC,MAS/B;AACrB,QAAM,SAAoB,CAAC;AAC3B,aAAW,YAAY,KAAK,WAAW;AACrC,QAAI;AACF,YAAM,SAAS,OAAO;AAAA,QACpB,WAAW,KAAK;AAAA,QAChB,MAAM,KAAK;AAAA,QACX,QAAQ,KAAK;AAAA,QACb,aAAa,KAAK;AAAA,QAClB,QAAQ,KAAK;AAAA,QACb,SAAS,KAAK;AAAA,MAChB,CAAC;AAAA,IACH,SAAS,OAAO;AACd,UAAI,KAAK,qBAAqB;AAC5B,cAAM;AAAA,MACR;AACA,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,cAAc,SAIrB;AACA,MAAI,SAAS;AACb,MAAI,SAAS;AACb,MAAI,UAAU;AACd,aAAW,UAAU,SAAS;AAC5B,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,UAAW,YAAW;AAAA,IACvD;AAAA,EACF;AACA,SAAO,EAAE,QAAQ,QAAQ,QAAQ;AACnC;","names":[]}
+{"version":3,"sources":["../../src/evals/agent-target.ts","../../src/evals/metric.ts","../../src/evals/metrics.ts","../../src/evals/format.ts","../../src/evals/outcome.ts","../../src/evals/selectors.ts","../../src/evals/runner.ts"],"sourcesContent":["import type { Agent } from \"../agent/agent\";\nimport type { Message } from \"../completion\";\nimport type { PromptResponse } from \"../request/types\";\nimport type { EvalCase, EvalTarget } from \"./types\";\n\nexport type AgentEvalTargetOptions<Input, Output = PromptResponse> = {\n  prompt?: ((input: Input, testCase: EvalCase<Input>) => string | Message) | undefined;\n  output?: ((response: PromptResponse, testCase: EvalCase<Input>) => Output) | undefined;\n};\n\nexport function agentEvalTarget<Input>(\n  agent: Agent,\n  options?: AgentEvalTargetOptions<Input, PromptResponse>,\n): EvalTarget<Input, PromptResponse>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output>,\n): EvalTarget<Input, Output>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output | PromptResponse> = {},\n): EvalTarget<Input, Output | PromptResponse> {\n  return async (input, testCase) => {\n    const prompt = options.prompt?.(input, testCase) ?? String(input);\n    const response = await agent.prompt(prompt).send();\n    return options.output === undefined ? response : options.output(response, testCase);\n  };\n}\n","import type { EvalMetric } from \"./types\";\n\nexport function defineMetric<Input, Output, Score, Expected>(\n  metric: EvalMetric<Input, Output, Score, Expected>,\n): EvalMetric<Input, Output, Score, Expected> {\n  return metric;\n}\n","import { z } from \"zod\";\nimport type { CompletionModel } from \"../completion\";\nimport { cosineSimilarity, type EmbeddingModel, embedText } from \"../embeddings\";\nimport { ExtractorBuilder } from \"../extractor\";\nimport type { ZodSchema } from \"../schema\";\nimport { errorMessage, formatValue, stableComparable } from \"./format\";\nimport { EvalOutcome } from \"./outcome\";\nimport { resolveActual, resolveActualText, resolveExpected, resolveJudgePrompt } from \"./selectors\";\nimport type { EvalMetric, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport type ExactMatchOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, unknown> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, unknown> | undefined;\n};\n\nexport function exactMatch<Input, Output, Expected = unknown>(\n  options: ExactMatchOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"exact_match\",\n    async evaluate(args) {\n      const actual = await resolveActual(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for exact match.\");\n      }\n      const passed = stableComparable(actual) === stableComparable(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });\n    },\n  };\n}\n\nexport type ContainsOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string | RegExp> | undefined;\n};\n\nexport function contains<Input, Output, Expected = unknown>(\n  options: ContainsOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"contains\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for contains.\");\n      }\n      if (typeof expected !== \"string\" && !(expected instanceof RegExp)) {\n        return EvalOutcome.invalid(\"Contains expected value must be a string or RegExp.\");\n      }\n      const passed =\n        expected instanceof RegExp ? regexMatches(expected, actual) : actual.includes(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Output did not contain ${String(expected)}.` });\n    },\n  };\n}\n\nfunction regexMatches(pattern: RegExp, text: string): boolean {\n  pattern.lastIndex = 0;\n  const matched = pattern.test(text);\n  pattern.lastIndex = 0;\n  return matched;\n}\n\nexport type SemanticSimilarityOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: EmbeddingModel;\n  threshold: number;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string> | undefined;\n};\n\nexport function semanticSimilarity<Input, Output, Expected = unknown>(\n  options: SemanticSimilarityOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, number, Expected> {\n  return {\n    name: options.name ?? \"semantic_similarity\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for semantic similarity.\");\n      }\n      if (typeof expected !== \"string\") {\n        return EvalOutcome.invalid(\"Semantic similarity expected value must be a string.\");\n      }\n      const [actualEmbedding, expectedEmbedding] = await Promise.all([\n        embedText(options.model, actual),\n        embedText(options.model, expected),\n      ]);\n      const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);\n      return score >= options.threshold\n        ? EvalOutcome.pass(score)\n        : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });\n    },\n  };\n}\n\nexport type LlmJudgeOptions<Input, Output, SchemaOutput, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  schema: ZodSchema<SchemaOutput>;\n  passes(value: SchemaOutput): boolean;\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmJudge<Input, Output, SchemaOutput, Expected = unknown>(\n  options: LlmJudgeOptions<Input, Output, SchemaOutput, Expected>,\n): EvalMetric<Input, Output, SchemaOutput, Expected> {\n  const extractor = new ExtractorBuilder(options.model, options.schema)\n    .instructions(\n      options.instructions ??\n        \"Judge the eval case by the requested schema. Submit the judgment using the schema.\",\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_judge\",\n    async evaluate(args) {\n      try {\n        const judgment = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        return options.passes(judgment) ? EvalOutcome.pass(judgment) : EvalOutcome.fail(judgment);\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n\nexport type LlmScoreMetricScore = {\n  score: number;\n  feedback: string;\n};\n\nexport type LlmScoreOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  threshold: number;\n  criteria: string | string[];\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmScore<Input, Output, Expected = unknown>(\n  options: LlmScoreOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, LlmScoreMetricScore, Expected> {\n  const criteria = Array.isArray(options.criteria) ? options.criteria.join(\"\\n\") : options.criteria;\n  const extractor = new ExtractorBuilder(\n    options.model,\n    z.object({\n      score: z.number(),\n      feedback: z.string(),\n    }),\n  )\n    .instructions(\n      options.instructions ??\n        `Score the eval case against these criteria:\\n${criteria}\\n\\nReturn a score between 0 and 1 and brief feedback.`,\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_score\",\n    async evaluate(args) {\n      try {\n        const score = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        if (score.score < 0 || score.score > 1) {\n          return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {\n            score,\n          });\n        }\n        return score.score >= options.threshold\n          ? EvalOutcome.pass(score, { comment: score.feedback })\n          : EvalOutcome.fail(score, { comment: score.feedback });\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n","export function defaultOutputValue(output: unknown): unknown {\n  if (\n    typeof output === \"object\" &&\n    output !== null &&\n    \"output\" in output &&\n    typeof (output as { output?: unknown }).output === \"string\"\n  ) {\n    return (output as { output: string }).output;\n  }\n  return output;\n}\n\nexport function stableComparable(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  return JSON.stringify(value);\n}\n\nexport function formatValue(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  try {\n    return JSON.stringify(value);\n  } catch {\n    return String(value);\n  }\n}\n\nexport function errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n","import type { EvalMetadata } from \"./types\";\n\nexport type EvalOutcome<Score = unknown> =\n  | {\n      outcome: \"pass\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"fail\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"invalid\";\n      reason: string;\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    };\n\nexport const EvalOutcome = {\n  pass<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    const outcome: EvalOutcome<Score> = {\n      outcome: \"pass\" as const,\n    };\n    if (score !== undefined) {\n      outcome.score = score;\n    }\n    if (options.comment !== undefined) {\n      outcome.comment = options.comment;\n    }\n    if (options.metadata !== undefined) {\n      outcome.metadata = options.metadata;\n    }\n    return outcome;\n  },\n\n  fail<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    const outcome: EvalOutcome<Score> = {\n      outcome: \"fail\" as const,\n    };\n    if (score !== undefined) {\n      outcome.score = score;\n    }\n    if (options.comment !== undefined) {\n      outcome.comment = options.comment;\n    }\n    if (options.metadata !== undefined) {\n      outcome.metadata = options.metadata;\n    }\n    return outcome;\n  },\n\n  invalid<Score = never>(\n    reason: string,\n    options: {\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    } = {},\n  ): EvalOutcome<Score> {\n    const outcome: EvalOutcome<Score> = {\n      outcome: \"invalid\" as const,\n      reason,\n    };\n    if (options.score !== undefined) {\n      outcome.score = options.score;\n    }\n    if (options.comment !== undefined) {\n      outcome.comment = options.comment;\n    }\n    if (options.metadata !== undefined) {\n      outcome.metadata = options.metadata;\n    }\n    return outcome;\n  },\n};\n","import { defaultOutputValue, formatValue } from \"./format\";\nimport type { EvalMetricArgs, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport async function resolveActual<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, unknown> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<unknown> {\n  return selector === undefined ? defaultOutputValue(args.output) : selector(args);\n}\n\nexport async function resolveActualText<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  const value = selector === undefined ? defaultOutputValue(args.output) : await selector(args);\n  return typeof value === \"string\" ? value : JSON.stringify(value);\n}\n\nexport async function resolveExpected<Input, Output, Expected, Value>(\n  selectorOrValue: SelectorOrValue<Input, Output, Expected, Value> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<Value | Expected | undefined> {\n  if (selectorOrValue === undefined) {\n    return args.case.expected;\n  }\n  return typeof selectorOrValue === \"function\"\n    ? (selectorOrValue as ValueSelector<Input, Output, Expected, Value>)(args)\n    : selectorOrValue;\n}\n\nexport async function resolveJudgePrompt<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  if (selector !== undefined) {\n    return selector(args);\n  }\n  return [\n    `Suite: ${args.suiteName}`,\n    `Case: ${args.case.id}`,\n    `Input: ${formatValue(args.case.input)}`,\n    `Expected: ${formatValue(args.case.expected)}`,\n    `Output: ${formatValue(defaultOutputValue(args.output))}`,\n  ].join(\"\\n\\n\");\n}\n","import { mapWithConcurrency } from \"../internal/concurrency\";\nimport { errorMessage } from \"./format\";\nimport { EvalOutcome, type EvalOutcome as EvalOutcomeType } from \"./outcome\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalMetric,\n  EvalMetricResult,\n  EvalReporter,\n  EvalSuiteResult,\n  RunEvalSuiteOptions,\n} from \"./types\";\n\nexport async function runEvalSuite<Input, Output, Expected = unknown>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n): Promise<EvalSuiteResult<Input, Output, Expected>> {\n  const startedAt = Date.now();\n  const results = await mapWithConcurrency(\n    options.cases,\n    Math.max(1, Math.trunc(options.concurrency ?? 1)),\n    (testCase) => runEvalCase(options, testCase),\n  );\n  const counts = countOutcomes(results);\n  return {\n    name: options.name,\n    results,\n    ...counts,\n    durationMs: Date.now() - startedAt,\n  };\n}\n\nasync function runEvalCase<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n  testCase: EvalCase<Input, Expected>,\n): Promise<EvalCaseResult<Input, Output, Expected>> {\n  let output: Output | undefined;\n  let targetError: unknown;\n  try {\n    output = await options.target(testCase.input, testCase);\n  } catch (error) {\n    targetError = error;\n  }\n\n  const metrics: EvalMetricResult[] = [];\n  for (const metric of options.metrics) {\n    const outcome =\n      targetError === undefined\n        ? await safeEvaluate(options.name, testCase, output as Output, metric)\n        : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`);\n    const reporterErrors = await reportOutcome({\n      suiteName: options.name,\n      testCase,\n      output,\n      targetError,\n      metric,\n      outcome,\n      reporters: options.reporters ?? [],\n      failOnReporterError: options.failOnReporterError === true,\n    });\n    metrics.push({ metricName: metric.name, outcome, reporterErrors });\n  }\n\n  const result: EvalCaseResult<Input, Output, Expected> = {\n    case: testCase,\n    metrics,\n  };\n  if (output !== undefined) {\n    result.output = output;\n  }\n  if (targetError !== undefined) {\n    result.targetError = targetError;\n  }\n  return result;\n}\n\nasync function safeEvaluate<Input, Output, Expected>(\n  suiteName: string,\n  testCase: EvalCase<Input, Expected>,\n  output: Output,\n  metric: EvalMetric<Input, Output, unknown, Expected>,\n): Promise<EvalOutcomeType> {\n  try {\n    return await metric.evaluate({ suiteName, case: testCase, output });\n  } catch (error) {\n    return EvalOutcome.invalid(errorMessage(error));\n  }\n}\n\nasync function reportOutcome<Input, Output, Expected>(args: {\n  suiteName: string;\n  testCase: EvalCase<Input, Expected>;\n  output: Output | undefined;\n  targetError: unknown;\n  metric: EvalMetric<Input, Output, unknown, Expected>;\n  outcome: EvalOutcomeType;\n  reporters: Array<EvalReporter<Input, Output, Expected>>;\n  failOnReporterError: boolean;\n}): Promise<unknown[]> {\n  const errors: unknown[] = [];\n  for (const reporter of args.reporters) {\n    try {\n      await reporter.report({\n        suiteName: args.suiteName,\n        case: args.testCase,\n        output: args.output,\n        targetError: args.targetError,\n        metric: args.metric,\n        outcome: args.outcome,\n      });\n    } catch (error) {\n      if (args.failOnReporterError) {\n        throw error;\n      }\n      errors.push(error);\n    }\n  }\n  return errors;\n}\n\nfunction countOutcomes(results: Array<EvalCaseResult<unknown, unknown, unknown>>): {\n  passed: number;\n  failed: number;\n  invalid: number;\n} {\n  let passed = 0;\n  let failed = 0;\n  let invalid = 0;\n  for (const result of results) {\n    for (const metric of result.metrics) {\n      if (metric.outcome.outcome === \"pass\") passed += 1;\n      if (metric.outcome.outcome === \"fail\") failed += 1;\n      if (metric.outcome.outcome === \"invalid\") invalid += 1;\n    }\n  }\n  return { passed, failed, invalid };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAkBO,SAAS,gBACd,OACA,UAAkE,CAAC,GACvB;AAC5C,SAAO,OAAO,OAAO,aAAa;AAChC,UAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,KAAK,OAAO,KAAK;AAChE,UAAM,WAAW,MAAM,MAAM,OAAO,MAAM,EAAE,KAAK;AACjD,WAAO,QAAQ,WAAW,SAAY,WAAW,QAAQ,OAAO,UAAU,QAAQ;AAAA,EACpF;AACF;;;ACzBO,SAAS,aACd,QAC4C;AAC5C,SAAO;AACT;;;ACNA,SAAS,SAAS;;;ACAX,SAAS,mBAAmB,QAA0B;AAC3D,MACE,OAAO,WAAW,YAClB,WAAW,QACX,YAAY,UACZ,OAAQ,OAAgC,WAAW,UACnD;AACA,WAAQ,OAA8B;AAAA,EACxC;AACA,SAAO;AACT;AAEO,SAAS,iBAAiB,OAAwB;AACvD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,SAAO,KAAK,UAAU,KAAK;AAC7B;AAEO,SAAS,YAAY,OAAwB;AAClD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,MAAI;AACF,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B,QAAQ;AACN,WAAO,OAAO,KAAK;AAAA,EACrB;AACF;AAEO,SAAS,aAAa,OAAwB;AACnD,SAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC9D;;;ACTO,IAAM,cAAc;AAAA,EACzB,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,UAAM,UAA8B;AAAA,MAClC,SAAS;AAAA,IACX;AACA,QAAI,UAAU,QAAW;AACvB,cAAQ,QAAQ;AAAA,IAClB;AACA,QAAI,QAAQ,YAAY,QAAW;AACjC,cAAQ,UAAU,QAAQ;AAAA,IAC5B;AACA,QAAI,QAAQ,aAAa,QAAW;AAClC,cAAQ,WAAW,QAAQ;AAAA,IAC7B;AACA,WAAO;AAAA,EACT;AAAA,EAEA,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,UAAM,UAA8B;AAAA,MAClC,SAAS;AAAA,IACX;AACA,QAAI,UAAU,QAAW;AACvB,cAAQ,QAAQ;AAAA,IAClB;AACA,QAAI,QAAQ,YAAY,QAAW;AACjC,cAAQ,UAAU,QAAQ;AAAA,IAC5B;AACA,QAAI,QAAQ,aAAa,QAAW;AAClC,cAAQ,WAAW,QAAQ;AAAA,IAC7B;AACA,WAAO;AAAA,EACT;AAAA,EAEA,QACE,QACA,UAII,CAAC,GACe;AACpB,UAAM,UAA8B;AAAA,MAClC,SAAS;AAAA,MACT;AAAA,IACF;AACA,QAAI,QAAQ,UAAU,QAAW;AAC/B,cAAQ,QAAQ,QAAQ;AAAA,IAC1B;AACA,QAAI,QAAQ,YAAY,QAAW;AACjC,cAAQ,UAAU,QAAQ;AAAA,IAC5B;AACA,QAAI,QAAQ,aAAa,QAAW;AAClC,cAAQ,WAAW,QAAQ;AAAA,IAC7B;AACA,WAAO;AAAA,EACT;AACF;;;AClFA,eAAsB,cACpB,UACA,MACkB;AAClB,SAAO,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,SAAS,IAAI;AACjF;AAEA,eAAsB,kBACpB,UACA,MACiB;AACjB,QAAM,QAAQ,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,MAAM,SAAS,IAAI;AAC5F,SAAO,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK;AACjE;AAEA,eAAsB,gBACpB,iBACA,MACuC;AACvC,MAAI,oBAAoB,QAAW;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AACA,SAAO,OAAO,oBAAoB,aAC7B,gBAAkE,IAAI,IACvE;AACN;AAEA,eAAsB,mBACpB,UACA,MACiB;AACjB,MAAI,aAAa,QAAW;AAC1B,WAAO,SAAS,IAAI;AAAA,EACtB;AACA,SAAO;AAAA,IACL,UAAU,KAAK,SAAS;AAAA,IACxB,SAAS,KAAK,KAAK,EAAE;AAAA,IACrB,UAAU,YAAY,KAAK,KAAK,KAAK,CAAC;AAAA,IACtC,aAAa,YAAY,KAAK,KAAK,QAAQ,CAAC;AAAA,IAC5C,WAAW,YAAY,mBAAmB,KAAK,MAAM,CAAC,CAAC;AAAA,EACzD,EAAE,KAAK,MAAM;AACf;;;AH5BO,SAAS,WACd,UAAsD,CAAC,GACT;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,cAAc,QAAQ,QAAQ,IAAI;AACvD,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,6CAA6C;AAAA,MAC1E;AACA,YAAM,SAAS,iBAAiB,MAAM,MAAM,iBAAiB,QAAQ;AACrE,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,YAAY,YAAY,QAAQ,CAAC,IAAI,CAAC;AAAA,IAC/E;AAAA,EACF;AACF;AAQO,SAAS,SACd,UAAoD,CAAC,GACP;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,0CAA0C;AAAA,MACvE;AACA,UAAI,OAAO,aAAa,YAAY,EAAE,oBAAoB,SAAS;AACjE,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,YAAM,SACJ,oBAAoB,SAAS,aAAa,UAAU,MAAM,IAAI,OAAO,SAAS,QAAQ;AACxF,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,0BAA0B,OAAO,QAAQ,CAAC,IAAI,CAAC;AAAA,IACxF;AAAA,EACF;AACF;AAEA,SAAS,aAAa,SAAiB,MAAuB;AAC5D,UAAQ,YAAY;AACpB,QAAM,UAAU,QAAQ,KAAK,IAAI;AACjC,UAAQ,YAAY;AACpB,SAAO;AACT;AAUO,SAAS,mBACd,SAC6C;AAC7C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,UAAI,OAAO,aAAa,UAAU;AAChC,eAAO,YAAY,QAAQ,sDAAsD;AAAA,MACnF;AACA,YAAM,CAAC,iBAAiB,iBAAiB,IAAI,MAAM,QAAQ,IAAI;AAAA,QAC7D,UAAU,QAAQ,OAAO,MAAM;AAAA,QAC/B,UAAU,QAAQ,OAAO,QAAQ;AAAA,MACnC,CAAC;AACD,YAAM,QAAQ,iBAAiB,gBAAgB,QAAQ,kBAAkB,MAAM;AAC/E,aAAO,SAAS,QAAQ,YACpB,YAAY,KAAK,KAAK,IACtB,YAAY,KAAK,OAAO,EAAE,SAAS,8BAA8B,QAAQ,SAAS,IAAI,CAAC;AAAA,IAC7F;AAAA,EACF;AACF;AAYO,SAAS,SACd,SACmD;AACnD,QAAM,YAAY,IAAI,iBAAiB,QAAQ,OAAO,QAAQ,MAAM,EACjE;AAAA,IACC,QAAQ,gBACN;AAAA,EACJ,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,WAAW,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACvF,eAAO,QAAQ,OAAO,QAAQ,IAAI,YAAY,KAAK,QAAQ,IAAI,YAAY,KAAK,QAAQ;AAAA,MAC1F,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;AAiBO,SAAS,SACd,SAC0D;AAC1D,QAAM,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAAI,QAAQ,SAAS,KAAK,IAAI,IAAI,QAAQ;AACzF,QAAM,YAAY,IAAI;AAAA,IACpB,QAAQ;AAAA,IACR,EAAE,OAAO;AAAA,MACP,OAAO,EAAE,OAAO;AAAA,MAChB,UAAU,EAAE,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,EACG;AAAA,IACC,QAAQ,gBACN;AAAA,EAAgD,QAAQ;AAAA;AAAA;AAAA,EAC5D,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,QAAQ,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACpF,YAAI,MAAM,QAAQ,KAAK,MAAM,QAAQ,GAAG;AACtC,iBAAO,YAAY,QAAQ,SAAS,MAAM,KAAK,gCAAgC;AAAA,YAC7E;AAAA,UACF,CAAC;AAAA,QACH;AACA,eAAO,MAAM,SAAS,QAAQ,YAC1B,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC,IACnD,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC;AAAA,MACzD,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;;;AIjLA,eAAsB,aACpB,SACmD;AACnD,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,UAAU,MAAM;AAAA,IACpB,QAAQ;AAAA,IACR,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,eAAe,CAAC,CAAC;AAAA,IAChD,CAAC,aAAa,YAAY,SAAS,QAAQ;AAAA,EAC7C;AACA,QAAM,SAAS,cAAc,OAAO;AACpC,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd;AAAA,IACA,GAAG;AAAA,IACH,YAAY,KAAK,IAAI,IAAI;AAAA,EAC3B;AACF;AAEA,eAAe,YACb,SACA,UACkD;AAClD,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,QAAQ,OAAO,SAAS,OAAO,QAAQ;AAAA,EACxD,SAAS,OAAO;AACd,kBAAc;AAAA,EAChB;AAEA,QAAM,UAA8B,CAAC;AACrC,aAAW,UAAU,QAAQ,SAAS;AACpC,UAAM,UACJ,gBAAgB,SACZ,MAAM,aAAa,QAAQ,MAAM,UAAU,QAAkB,MAAM,IACnE,YAAY,QAAQ,kBAAkB,aAAa,WAAW,CAAC,EAAE;AACvE,UAAM,iBAAiB,MAAM,cAAc;AAAA,MACzC,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW,QAAQ,aAAa,CAAC;AAAA,MACjC,qBAAqB,QAAQ,wBAAwB;AAAA,IACvD,CAAC;AACD,YAAQ,KAAK,EAAE,YAAY,OAAO,MAAM,SAAS,eAAe,CAAC;AAAA,EACnE;AAEA,QAAM,SAAkD;AAAA,IACtD,MAAM;AAAA,IACN;AAAA,EACF;AACA,MAAI,WAAW,QAAW;AACxB,WAAO,SAAS;AAAA,EAClB;AACA,MAAI,gBAAgB,QAAW;AAC7B,WAAO,cAAc;AAAA,EACvB;AACA,SAAO;AACT;AAEA,eAAe,aACb,WACA,UACA,QACA,QAC0B;AAC1B,MAAI;AACF,WAAO,MAAM,OAAO,SAAS,EAAE,WAAW,MAAM,UAAU,OAAO,CAAC;AAAA,EACpE,SAAS,OAAO;AACd,WAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,EAChD;AACF;AAEA,eAAe,cAAuC,MAS/B;AACrB,QAAM,SAAoB,CAAC;AAC3B,aAAW,YAAY,KAAK,WAAW;AACrC,QAAI;AACF,YAAM,SAAS,OAAO;AAAA,QACpB,WAAW,KAAK;AAAA,QAChB,MAAM,KAAK;AAAA,QACX,QAAQ,KAAK;AAAA,QACb,aAAa,KAAK;AAAA,QAClB,QAAQ,KAAK;AAAA,QACb,SAAS,KAAK;AAAA,MAChB,CAAC;AAAA,IACH,SAAS,OAAO;AACd,UAAI,KAAK,qBAAqB;AAC5B,cAAM;AAAA,MACR;AACA,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,cAAc,SAIrB;AACA,MAAI,SAAS;AACb,MAAI,SAAS;AACb,MAAI,UAAU;AACd,aAAW,UAAU,SAAS;AAC5B,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,UAAW,YAAW;AAAA,IACvD;AAAA,EACF;AACA,SAAO,EAAE,QAAQ,QAAQ,QAAQ;AACnC;","names":[]}

package/dist/extractor/index.d.ts

@@ -1,9 +1,9 @@
-import { A as Agent } from '../agent-Cg0UigNK.js';
+import { A as Agent } from '../agent-CBz3CRxa.js';
 import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
 import { U as Usage, M as Message, C as CompletionModel, i as JsonValue, z as ToolChoice } from '../types-XjIh933x.js';
 import '../guardrails/index.js';
 import '../types-BCA8p0sb.js';
-import '../types-CBmhLT_r.js';
+import '../types-DBHzPsbx.js';
 import '../index-XTUEMWhU.js';
 import '../tool-Dhwg__1L.js';
 import '../middleware-Cp0tm6zE.js';

package/dist/extractor/index.js

@@ -2,21 +2,23 @@ import {
   ExtractionError,
   Extractor,
   ExtractorBuilder
-} from "../chunk-54S4I3SX.js";
-import "../chunk-RWAOHGRL.js";
-import "../chunk-BQCTJSNL.js";
-import "../chunk-OPAEENGD.js";
+} from "../chunk-3YJXLOGO.js";
+import "../chunk-DPTZ2MK4.js";
+import "../chunk-QABIBFKE.js";
+import "../chunk-7LCKIULT.js";
 import "../chunk-YK4WAAS4.js";
 import "../chunk-XUUY2L2D.js";
-import "../chunk-SWUIBS4N.js";
-import "../chunk-F3C5653H.js";
+import "../chunk-JNJBKSLA.js";
+import "../chunk-I7TM7TKX.js";
 import "../chunk-2ODTMRHP.js";
-import "../chunk-M6RPFHG7.js";
-import "../chunk-YBBD4HZ3.js";
-import "../chunk-OIMLU4SF.js";
-import "../chunk-GCED5ZNM.js";
+import "../chunk-ZNTIUOKL.js";
+import "../chunk-TGLNXVII.js";
+import "../chunk-MTGNA4OS.js";
+import "../chunk-F56AMXMQ.js";
 import "../chunk-PJDL5UT2.js";
 import "../chunk-WQKHFADH.js";
+import "../chunk-YBBD4HZ3.js";
+import "../chunk-OIMLU4SF.js";
 import "../chunk-CWUJUSOS.js";
 export {
   ExtractionError,

package/dist/index.d.ts

@@ -3,9 +3,10 @@ export { C as CreateCompletionBaseOptions, a as CreateCompletionInput, b as Crea
 export { A as AssistantContent, a as AssistantGenerationMetadata, b as AssistantMessage, c as AssistantMessageOptions, C as CompletionModel, d as CompletionRequest, e as CompletionResponse, f as CompletionSource, g as CompletionTool, D as Document, I as ImageContent, J as JsonObject, h as JsonPrimitive, i as JsonValue, M as Message, j as MessageOptions, P as ProviderTool, k as ProviderToolCall, S as SystemMessage, T as Text, l as ToolCall, m as ToolContent, n as ToolDefinition, o as ToolMessage, p as ToolResult, q as ToolResultContent, r as ToolResultMessageOptions, s as ToolResultOptions, U as Usage, t as UserContent, u as UserMessage, v as getAssistantGenerationMetadata, w as isProviderTool } from './types-XjIh933x.js';
 export { GuardrailBoundary, GuardrailDecisionRecord, GuardrailMode, GuardrailPolicy, GuardrailPolicyInput, GuardrailPolicyOptions, InputGuardrail, InputGuardrailActions, InputGuardrailContext, OutputGuardrail, OutputGuardrailActions, OutputGuardrailContext, allow, block, defineGuardrailPolicy, defineInputGuardrail, defineOutputGuardrail, guardrails } from './guardrails/index.js';
 export { cancelPrompt, createHook, requestToolApproval, runControl, skipTool, toolCallControl } from './hooks/index.js';
-export { M as MemoryConversation, a as MemoryConversationListOptions, b as MemoryConversationMessage, c as MemoryConversationSummary, d as MemoryInspector, e as MemoryStore } from './types-CBmhLT_r.js';
+export { M as MemoryCompactionConflictError, a as MemoryCompactionError, c as createSummaryMemoryCompactor, i as isMemoryCompactionSummary } from './errors-Bf661V89.js';
+export { M as MemoryCompactionCommitInput, a as MemoryCompactionCommitResult, b as MemoryCompactionOptions, c as MemoryCompactionSnapshot, d as MemoryCompactionStore, e as MemoryCompactor, f as MemoryCompactorInput, g as MemoryCompactorResult, h as MemoryConversation, i as MemoryConversationListOptions, j as MemoryConversationMessage, k as MemoryConversationSummary, l as MemoryInspector, m as MemoryStore, R as ResolvedMemoryCompactionOptions, S as SummaryMemoryCompactorOptions } from './types-DBHzPsbx.js';
 export { MaxTurnsError, PromptCancelledError, ToolApprovalRequiredError } from './request/index.js';
-export { C as CompletionRetryContext, a as CompletionRetryOptions } from './agent-Cg0UigNK.js';
+export { C as CompletionRetryContext, a as CompletionRetryOptions } from './agent-CBz3CRxa.js';
 export { A as AgentChildStreamEvent, a as AgentChildStreamEventWithToolCallDeltas, b as AgentChildStreamEventWithoutToolCallDeltas, c as AgentErrorStreamEvent, d as AgentStreamEvent, e as AgentStreamEventWithToolCallDeltas, f as AgentStreamEventWithoutToolCallDeltas, g as AgentStreamOptions, h as AgentToolCallDeltaEvent, P as PromptResponse } from './index-XTUEMWhU.js';
 export { Z as ZodSchema } from './zod-schema-C7F4clpm.js';
 export { loadSkills, skill } from './skills/index.js';

package/dist/index.js

@@ -2,31 +2,32 @@ import {
   SkillValidationError,
   loadSkills,
   skill
-} from "./chunk-77D7XBWL.js";
+} from "./chunk-DC2GJDPR.js";
 import {
   coreMessagesToUIMessages,
   uiMessagesToCoreMessages
 } from "./chunk-SWUH4G2I.js";
 import "./chunk-CQNNSZPG.js";
+import "./chunk-XWUC7CIT.js";
 import {
   AgentBuilder
-} from "./chunk-RWAOHGRL.js";
-import "./chunk-BQCTJSNL.js";
+} from "./chunk-DPTZ2MK4.js";
+import "./chunk-QABIBFKE.js";
 import {
   MaxTurnsError,
   PromptCancelledError,
   ToolApprovalRequiredError
-} from "./chunk-OPAEENGD.js";
+} from "./chunk-7LCKIULT.js";
 import "./chunk-YK4WAAS4.js";
 import "./chunk-XUUY2L2D.js";
 import {
   createMiddleware,
   createThinkTool,
   createToolMiddleware
-} from "./chunk-SWUIBS4N.js";
+} from "./chunk-JNJBKSLA.js";
 import {
   createTool
-} from "./chunk-F3C5653H.js";
+} from "./chunk-I7TM7TKX.js";
 import {
   cancelPrompt,
   createHook,
@@ -35,14 +36,19 @@ import {
   skipTool,
   toolCallControl
 } from "./chunk-2ODTMRHP.js";
-import "./chunk-M6RPFHG7.js";
-import "./chunk-YBBD4HZ3.js";
-import "./chunk-OIMLU4SF.js";
+import "./chunk-ZNTIUOKL.js";
+import {
+  MemoryCompactionConflictError,
+  MemoryCompactionError,
+  createSummaryMemoryCompactor,
+  isMemoryCompactionSummary
+} from "./chunk-TGLNXVII.js";
+import "./chunk-MTGNA4OS.js";
 import {
   createCompletion,
   createCompletionStream,
   createParsedCompletion
-} from "./chunk-GCED5ZNM.js";
+} from "./chunk-F56AMXMQ.js";
 import {
   AssistantContent,
   Message,
@@ -54,6 +60,8 @@ import {
   isProviderTool
 } from "./chunk-PJDL5UT2.js";
 import "./chunk-WQKHFADH.js";
+import "./chunk-YBBD4HZ3.js";
+import "./chunk-OIMLU4SF.js";
 import {
   allow,
   block,
@@ -66,6 +74,8 @@ export {
   AgentBuilder,
   AssistantContent,
   MaxTurnsError,
+  MemoryCompactionConflictError,
+  MemoryCompactionError,
   Message,
   PromptCancelledError,
   SkillValidationError,
@@ -82,6 +92,7 @@ export {
   createHook,
   createMiddleware,
   createParsedCompletion,
+  createSummaryMemoryCompactor,
   createThinkTool,
   createTool,
   createToolMiddleware,
@@ -91,6 +102,7 @@ export {
   getAssistantGenerationMetadata,
   guardrails,
   isJsonValue,
+  isMemoryCompactionSummary,
   isProviderTool,
   loadSkills,
   requestToolApproval,

package/dist/internal/agent.d.ts

@@ -1,8 +1,8 @@
-export { A as Agent, e as AgentEventAppendInput, f as AgentEventRecord, c as AgentEventStore, g as AgentEventStoreInclude, d as AgentEventStoreOptions, j as AgentEventStoreRegistration, k as AgentOptions, h as AgentSession, i as AgentToolOptions, l as DEFAULT_MAX_TURNS, D as DynamicContextOptions, m as DynamicContextRegistration, b as DynamicToolOptions, n as DynamicToolRegistration } from '../agent-Cg0UigNK.js';
+export { A as Agent, e as AgentEventAppendInput, f as AgentEventRecord, c as AgentEventStore, g as AgentEventStoreInclude, d as AgentEventStoreOptions, j as AgentEventStoreRegistration, k as AgentOptions, h as AgentSession, i as AgentToolOptions, l as DEFAULT_MAX_TURNS, D as DynamicContextOptions, m as DynamicContextRegistration, b as DynamicToolOptions, n as DynamicToolRegistration } from '../agent-CBz3CRxa.js';
 import '../types-XjIh933x.js';
 import '../guardrails/index.js';
 import '../types-BCA8p0sb.js';
-import '../types-CBmhLT_r.js';
+import '../types-DBHzPsbx.js';
 import '../index-XTUEMWhU.js';
 import '../tool-Dhwg__1L.js';
 import '../middleware-Cp0tm6zE.js';

package/dist/internal/agent.js

@@ -2,18 +2,20 @@ import {
   Agent,
   AgentSession,
   DEFAULT_MAX_TURNS
-} from "../chunk-BQCTJSNL.js";
-import "../chunk-OPAEENGD.js";
+} from "../chunk-QABIBFKE.js";
+import "../chunk-7LCKIULT.js";
 import "../chunk-YK4WAAS4.js";
 import "../chunk-XUUY2L2D.js";
-import "../chunk-SWUIBS4N.js";
-import "../chunk-F3C5653H.js";
+import "../chunk-JNJBKSLA.js";
+import "../chunk-I7TM7TKX.js";
 import "../chunk-2ODTMRHP.js";
-import "../chunk-YBBD4HZ3.js";
-import "../chunk-OIMLU4SF.js";
-import "../chunk-GCED5ZNM.js";
+import "../chunk-TGLNXVII.js";
+import "../chunk-MTGNA4OS.js";
+import "../chunk-F56AMXMQ.js";
 import "../chunk-PJDL5UT2.js";
 import "../chunk-WQKHFADH.js";
+import "../chunk-YBBD4HZ3.js";
+import "../chunk-OIMLU4SF.js";
 import "../chunk-CWUJUSOS.js";
 export {
   Agent,

package/dist/memory/index.d.ts

@@ -1,5 +1,6 @@
-import { f as MemoryOptions, R as ResolvedMemoryOptions } from '../types-CBmhLT_r.js';
-export { g as MemoryAppendInput, h as MemoryContext, M as MemoryConversation, a as MemoryConversationListOptions, b as MemoryConversationMessage, c as MemoryConversationSummary, i as MemoryErrorInput, d as MemoryInspector, j as MemoryRegistration, k as MemorySavePolicy, e as MemoryStore, S as SessionOptions } from '../types-CBmhLT_r.js';
+export { M as MemoryCompactionConflictError, a as MemoryCompactionError, c as createSummaryMemoryCompactor, i as isMemoryCompactionSummary } from '../errors-Bf661V89.js';
+import { n as MemoryOptions, o as ResolvedMemoryOptions } from '../types-DBHzPsbx.js';
+export { p as MemoryAppendInput, M as MemoryCompactionCommitInput, a as MemoryCompactionCommitResult, b as MemoryCompactionOptions, c as MemoryCompactionSnapshot, d as MemoryCompactionStore, e as MemoryCompactor, f as MemoryCompactorInput, g as MemoryCompactorResult, q as MemoryContext, h as MemoryConversation, i as MemoryConversationListOptions, j as MemoryConversationMessage, k as MemoryConversationSummary, r as MemoryErrorInput, l as MemoryInspector, s as MemoryRegistration, t as MemorySavePolicy, m as MemoryStore, R as ResolvedMemoryCompactionOptions, u as SessionOptions, S as SummaryMemoryCompactorOptions } from '../types-DBHzPsbx.js';
 import '../types-XjIh933x.js';
 
 declare function resolveMemoryOptions(options?: MemoryOptions): ResolvedMemoryOptions;

package/dist/memory/index.js

@@ -1,7 +1,21 @@
+import "../chunk-XWUC7CIT.js";
 import {
   resolveMemoryOptions
-} from "../chunk-M6RPFHG7.js";
+} from "../chunk-ZNTIUOKL.js";
+import {
+  MemoryCompactionConflictError,
+  MemoryCompactionError,
+  createSummaryMemoryCompactor,
+  isMemoryCompactionSummary
+} from "../chunk-TGLNXVII.js";
+import "../chunk-F56AMXMQ.js";
+import "../chunk-PJDL5UT2.js";
+import "../chunk-WQKHFADH.js";
 export {
+  MemoryCompactionConflictError,
+  MemoryCompactionError,
+  createSummaryMemoryCompactor,
+  isMemoryCompactionSummary,
   resolveMemoryOptions
 };
 //# sourceMappingURL=index.js.map

package/dist/pipeline/index.d.ts

@@ -1,10 +1,10 @@
 import { z } from 'zod';
-import { A as Agent } from '../agent-Cg0UigNK.js';
+import { A as Agent } from '../agent-CBz3CRxa.js';
 import { J as JsonObject, C as CompletionModel } from '../types-XjIh933x.js';
 import { Extractor } from '../extractor/index.js';
 import '../guardrails/index.js';
 import '../types-BCA8p0sb.js';
-import '../types-CBmhLT_r.js';
+import '../types-DBHzPsbx.js';
 import '../index-XTUEMWhU.js';
 import '../tool-Dhwg__1L.js';
 import '../middleware-Cp0tm6zE.js';

package/dist/request/index.d.ts

@@ -1,10 +1,10 @@
 import { M as Message } from '../types-XjIh933x.js';
 import { d as ToolApprovalRequest } from '../tool-Dhwg__1L.js';
-export { C as CompletionRetryContext, a as CompletionRetryOptions, P as PromptRequest } from '../agent-Cg0UigNK.js';
+export { C as CompletionRetryContext, a as CompletionRetryOptions, P as PromptRequest } from '../agent-CBz3CRxa.js';
 export { A as AgentChildStreamEvent, a as AgentChildStreamEventWithToolCallDeltas, b as AgentChildStreamEventWithoutToolCallDeltas, l as AgentDeltaEvent, c as AgentErrorStreamEvent, d as AgentStreamEvent, e as AgentStreamEventWithToolCallDeltas, f as AgentStreamEventWithoutToolCallDeltas, g as AgentStreamOptions, h as AgentToolCallDeltaEvent, P as PromptResponse } from '../index-XTUEMWhU.js';
 import '../guardrails/index.js';
 import '../types-BCA8p0sb.js';
-import '../types-CBmhLT_r.js';
+import '../types-DBHzPsbx.js';
 import '../middleware-Cp0tm6zE.js';
 import '../types-DHHZztGO.js';
 import '../types-DMGvEFRf.js';

package/dist/request/index.js

@@ -3,16 +3,18 @@ import {
   PromptCancelledError,
   PromptRequest,
   ToolApprovalRequiredError
-} from "../chunk-OPAEENGD.js";
+} from "../chunk-7LCKIULT.js";
 import "../chunk-XUUY2L2D.js";
-import "../chunk-SWUIBS4N.js";
-import "../chunk-F3C5653H.js";
+import "../chunk-JNJBKSLA.js";
+import "../chunk-I7TM7TKX.js";
 import "../chunk-2ODTMRHP.js";
-import "../chunk-YBBD4HZ3.js";
-import "../chunk-OIMLU4SF.js";
-import "../chunk-GCED5ZNM.js";
+import "../chunk-TGLNXVII.js";
+import "../chunk-MTGNA4OS.js";
+import "../chunk-F56AMXMQ.js";
 import "../chunk-PJDL5UT2.js";
 import "../chunk-WQKHFADH.js";
+import "../chunk-YBBD4HZ3.js";
+import "../chunk-OIMLU4SF.js";
 import "../chunk-CWUJUSOS.js";
 export {
   MaxTurnsError,

package/dist/skills/index.js

@@ -2,15 +2,15 @@ import {
   SkillValidationError,
   loadSkills,
   skill
-} from "../chunk-77D7XBWL.js";
+} from "../chunk-DC2GJDPR.js";
 import "../chunk-CQNNSZPG.js";
 import "../chunk-YK4WAAS4.js";
-import "../chunk-SWUIBS4N.js";
-import "../chunk-F3C5653H.js";
-import "../chunk-YBBD4HZ3.js";
-import "../chunk-OIMLU4SF.js";
+import "../chunk-JNJBKSLA.js";
+import "../chunk-I7TM7TKX.js";
 import "../chunk-PJDL5UT2.js";
 import "../chunk-WQKHFADH.js";
+import "../chunk-YBBD4HZ3.js";
+import "../chunk-OIMLU4SF.js";
 export {
   SkillValidationError,
   loadSkills,

package/dist/tool/index.js

@@ -13,17 +13,17 @@ import {
   normalizeToolResultOutput,
   parseToolArgs,
   toolResultContentToText
-} from "../chunk-SWUIBS4N.js";
+} from "../chunk-JNJBKSLA.js";
 import {
   createTool
-} from "../chunk-F3C5653H.js";
-import "../chunk-YBBD4HZ3.js";
-import "../chunk-OIMLU4SF.js";
+} from "../chunk-I7TM7TKX.js";
 import {
   isToolResultContentArray,
   serializeToolResultOutput
 } from "../chunk-PJDL5UT2.js";
 import "../chunk-WQKHFADH.js";
+import "../chunk-YBBD4HZ3.js";
+import "../chunk-OIMLU4SF.js";
 export {
   ToolCallError,
   ToolJsonError,