@anvia/core 0.4.0 → 0.4.1

package/dist/evals/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/evals/index.ts"],"sourcesContent":["import { z } from \"zod\";\nimport type { Agent } from \"../agent/agent\";\nimport type { PromptResponse } from \"../agent/request\";\nimport type { CompletionModel, JsonValue, Message } from \"../completion\";\nimport { cosineSimilarity, type EmbeddingModel, embedText } from \"../embeddings\";\nimport { ExtractorBuilder } from \"../extractor\";\nimport type { ZodSchema } from \"../schema\";\n\nexport type EvalMetadata = Record<string, JsonValue | undefined>;\n\nexport type EvalCase<Input, Expected = unknown> = {\n  id: string;\n  input: Input;\n  expected?: Expected | undefined;\n  metadata?: EvalMetadata | undefined;\n};\n\nexport type EvalTarget<Input, Output, Expected = unknown> = (\n  input: Input,\n  testCase: EvalCase<Input, Expected>,\n) => Output | Promise<Output>;\n\nexport type EvalOutcomeStatus = \"pass\" | \"fail\" | \"invalid\";\n\nexport type EvalOutcome<Score = unknown> =\n  | {\n      outcome: \"pass\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"fail\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"invalid\";\n      reason: string;\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    };\n\nexport const EvalOutcome = {\n  pass<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    return {\n      outcome: \"pass\",\n      ...(score === undefined ? {} : { score }),\n      ...(options.comment === undefined ? {} : { comment: options.comment }),\n      ...(options.metadata === undefined ? {} : { metadata: options.metadata }),\n    };\n  },\n\n  fail<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    return {\n      outcome: \"fail\",\n      ...(score === undefined ? {} : { score }),\n      ...(options.comment === undefined ? {} : { comment: options.comment }),\n      ...(options.metadata === undefined ? {} : { metadata: options.metadata }),\n    };\n  },\n\n  invalid<Score = never>(\n    reason: string,\n    options: {\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    } = {},\n  ): EvalOutcome<Score> {\n    return {\n      outcome: \"invalid\",\n      reason,\n      ...(options.score === undefined ? {} : { score: options.score }),\n      ...(options.comment === undefined ? {} : { comment: options.comment }),\n      ...(options.metadata === undefined ? {} : { metadata: options.metadata }),\n    };\n  },\n};\n\nexport type EvalMetricArgs<Input, Output, Expected = unknown> = {\n  suiteName: string;\n  case: EvalCase<Input, Expected>;\n  output: Output;\n};\n\nexport type EvalMetric<Input, Output, Score = unknown, Expected = unknown> = {\n  name: string;\n  evaluate(\n    args: EvalMetricArgs<Input, Output, Expected>,\n  ): EvalOutcome<Score> | Promise<EvalOutcome<Score>>;\n};\n\nexport type EvalMetricResult<Score = unknown> = {\n  metricName: string;\n  outcome: EvalOutcome<Score>;\n  reporterErrors: unknown[];\n};\n\nexport type EvalCaseResult<Input, Output, Expected = unknown> = {\n  case: EvalCase<Input, Expected>;\n  output?: Output | undefined;\n  targetError?: unknown;\n  metrics: EvalMetricResult[];\n};\n\nexport type EvalSuiteResult<Input, Output, Expected = unknown> = {\n  name: string;\n  results: Array<EvalCaseResult<Input, Output, Expected>>;\n  passed: number;\n  failed: number;\n  invalid: number;\n  durationMs: number;\n};\n\nexport type EvalReportArgs<Input, Output, Score = unknown, Expected = unknown> = {\n  suiteName: string;\n  case: EvalCase<Input, Expected>;\n  output?: Output | undefined;\n  targetError?: unknown;\n  metric: EvalMetric<Input, Output, Score, Expected>;\n  outcome: EvalOutcome<Score>;\n};\n\nexport type EvalReporter<Input = unknown, Output = unknown, Expected = unknown> = {\n  report(args: EvalReportArgs<Input, Output, unknown, Expected>): void | Promise<void>;\n};\n\nexport type RunEvalSuiteOptions<Input, Output, Expected = unknown> = {\n  name: string;\n  cases: Array<EvalCase<Input, Expected>>;\n  target: EvalTarget<Input, Output, Expected>;\n  metrics: Array<EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>>>;\n  concurrency?: number | undefined;\n  reporters?: Array<EvalReporter<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>>> | undefined;\n  failOnReporterError?: boolean | undefined;\n};\n\nexport async function runEvalSuite<Input, Output, Expected = unknown>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n): Promise<EvalSuiteResult<Input, Output, Expected>> {\n  const startedAt = Date.now();\n  const results = await mapWithConcurrency(\n    options.cases,\n    Math.max(1, Math.trunc(options.concurrency ?? 1)),\n    (testCase) => runEvalCase(options, testCase),\n  );\n  const counts = countOutcomes(results);\n  return {\n    name: options.name,\n    results,\n    ...counts,\n    durationMs: Date.now() - startedAt,\n  };\n}\n\nexport type ValueSelector<Input, Output, Expected, Value> = (\n  args: EvalMetricArgs<Input, Output, Expected>,\n) => Value | Promise<Value>;\n\nexport type SelectorOrValue<Input, Output, Expected, Value> =\n  | Value\n  | ValueSelector<Input, Output, Expected, Value>;\n\nexport type ExactMatchOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, unknown> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, unknown> | undefined;\n};\n\nexport function exactMatch<Input, Output, Expected = unknown>(\n  options: ExactMatchOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"exact_match\",\n    async evaluate(args) {\n      const actual = await resolveActual(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for exact match.\");\n      }\n      const passed = stableComparable(actual) === stableComparable(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });\n    },\n  };\n}\n\nexport type ContainsOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string | RegExp> | undefined;\n};\n\nexport function contains<Input, Output, Expected = unknown>(\n  options: ContainsOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"contains\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for contains.\");\n      }\n      if (typeof expected !== \"string\" && !(expected instanceof RegExp)) {\n        return EvalOutcome.invalid(\"Contains expected value must be a string or RegExp.\");\n      }\n      const passed = expected instanceof RegExp ? expected.test(actual) : actual.includes(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Output did not contain ${String(expected)}.` });\n    },\n  };\n}\n\nexport type SemanticSimilarityOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: EmbeddingModel;\n  threshold: number;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string> | undefined;\n};\n\nexport function semanticSimilarity<Input, Output, Expected = unknown>(\n  options: SemanticSimilarityOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, number, Expected> {\n  return {\n    name: options.name ?? \"semantic_similarity\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for semantic similarity.\");\n      }\n      if (typeof expected !== \"string\") {\n        return EvalOutcome.invalid(\"Semantic similarity expected value must be a string.\");\n      }\n      const [actualEmbedding, expectedEmbedding] = await Promise.all([\n        embedText(options.model, actual),\n        embedText(options.model, expected),\n      ]);\n      const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);\n      return score >= options.threshold\n        ? EvalOutcome.pass(score)\n        : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });\n    },\n  };\n}\n\nexport type LlmJudgeOptions<Input, Output, SchemaOutput, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  schema: ZodSchema<SchemaOutput>;\n  passes(value: SchemaOutput): boolean;\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmJudge<Input, Output, SchemaOutput, Expected = unknown>(\n  options: LlmJudgeOptions<Input, Output, SchemaOutput, Expected>,\n): EvalMetric<Input, Output, SchemaOutput, Expected> {\n  const extractor = new ExtractorBuilder(options.model, options.schema)\n    .instructions(\n      options.instructions ??\n        \"Judge the eval case by the requested schema. Submit the judgment using the schema.\",\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_judge\",\n    async evaluate(args) {\n      try {\n        const judgment = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        return options.passes(judgment) ? EvalOutcome.pass(judgment) : EvalOutcome.fail(judgment);\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n\nexport type LlmScoreMetricScore = {\n  score: number;\n  feedback: string;\n};\n\nexport type LlmScoreOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  threshold: number;\n  criteria: string | string[];\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmScore<Input, Output, Expected = unknown>(\n  options: LlmScoreOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, LlmScoreMetricScore, Expected> {\n  const criteria = Array.isArray(options.criteria) ? options.criteria.join(\"\\n\") : options.criteria;\n  const extractor = new ExtractorBuilder(\n    options.model,\n    z.object({\n      score: z.number(),\n      feedback: z.string(),\n    }),\n  )\n    .instructions(\n      options.instructions ??\n        `Score the eval case against these criteria:\\n${criteria}\\n\\nReturn a score between 0 and 1 and brief feedback.`,\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_score\",\n    async evaluate(args) {\n      try {\n        const score = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        if (score.score < 0 || score.score > 1) {\n          return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {\n            score,\n          });\n        }\n        return score.score >= options.threshold\n          ? EvalOutcome.pass(score, { comment: score.feedback })\n          : EvalOutcome.fail(score, { comment: score.feedback });\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n\nexport type AgentEvalTargetOptions<Input, Output = PromptResponse> = {\n  prompt?: ((input: Input, testCase: EvalCase<Input>) => string | Message) | undefined;\n  output?: ((response: PromptResponse, testCase: EvalCase<Input>) => Output) | undefined;\n};\n\nexport function agentEvalTarget<Input>(\n  agent: Agent,\n  options?: AgentEvalTargetOptions<Input, PromptResponse>,\n): EvalTarget<Input, PromptResponse>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output>,\n): EvalTarget<Input, Output>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output | PromptResponse> = {},\n): EvalTarget<Input, Output | PromptResponse> {\n  return async (input, testCase) => {\n    const prompt = options.prompt?.(input, testCase) ?? String(input);\n    const response = await agent.prompt(prompt).send();\n    return options.output === undefined ? response : options.output(response, testCase);\n  };\n}\n\nasync function runEvalCase<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n  testCase: EvalCase<Input, Expected>,\n): Promise<EvalCaseResult<Input, Output, Expected>> {\n  let output: Output | undefined;\n  let targetError: unknown;\n  try {\n    output = await options.target(testCase.input, testCase);\n  } catch (error) {\n    targetError = error;\n  }\n\n  const metrics: EvalMetricResult[] = [];\n  for (const metric of options.metrics) {\n    const outcome =\n      targetError === undefined\n        ? await safeEvaluate(options.name, testCase, output as Output, metric)\n        : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`);\n    const reporterErrors = await reportOutcome({\n      suiteName: options.name,\n      testCase,\n      output,\n      targetError,\n      metric,\n      outcome,\n      reporters: options.reporters ?? [],\n      failOnReporterError: options.failOnReporterError === true,\n    });\n    metrics.push({ metricName: metric.name, outcome, reporterErrors });\n  }\n\n  return {\n    case: testCase,\n    ...(output === undefined ? {} : { output }),\n    ...(targetError === undefined ? {} : { targetError }),\n    metrics,\n  };\n}\n\nasync function safeEvaluate<Input, Output, Expected>(\n  suiteName: string,\n  testCase: EvalCase<Input, Expected>,\n  output: Output,\n  metric: EvalMetric<Input, Output, unknown, Expected>,\n): Promise<EvalOutcome> {\n  try {\n    return await metric.evaluate({ suiteName, case: testCase, output });\n  } catch (error) {\n    return EvalOutcome.invalid(errorMessage(error));\n  }\n}\n\nasync function reportOutcome<Input, Output, Expected>(args: {\n  suiteName: string;\n  testCase: EvalCase<Input, Expected>;\n  output: Output | undefined;\n  targetError: unknown;\n  metric: EvalMetric<Input, Output, unknown, Expected>;\n  outcome: EvalOutcome;\n  reporters: Array<EvalReporter<Input, Output, Expected>>;\n  failOnReporterError: boolean;\n}): Promise<unknown[]> {\n  const errors: unknown[] = [];\n  for (const reporter of args.reporters) {\n    try {\n      await reporter.report({\n        suiteName: args.suiteName,\n        case: args.testCase,\n        output: args.output,\n        targetError: args.targetError,\n        metric: args.metric,\n        outcome: args.outcome,\n      });\n    } catch (error) {\n      if (args.failOnReporterError) {\n        throw error;\n      }\n      errors.push(error);\n    }\n  }\n  return errors;\n}\n\nfunction countOutcomes(results: Array<EvalCaseResult<unknown, unknown, unknown>>): {\n  passed: number;\n  failed: number;\n  invalid: number;\n} {\n  let passed = 0;\n  let failed = 0;\n  let invalid = 0;\n  for (const result of results) {\n    for (const metric of result.metrics) {\n      if (metric.outcome.outcome === \"pass\") passed += 1;\n      if (metric.outcome.outcome === \"fail\") failed += 1;\n      if (metric.outcome.outcome === \"invalid\") invalid += 1;\n    }\n  }\n  return { passed, failed, invalid };\n}\n\nasync function resolveActual<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, unknown> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<unknown> {\n  return selector === undefined ? defaultOutputValue(args.output) : selector(args);\n}\n\nasync function resolveActualText<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  const value = selector === undefined ? defaultOutputValue(args.output) : await selector(args);\n  return typeof value === \"string\" ? value : JSON.stringify(value);\n}\n\nasync function resolveExpected<Input, Output, Expected, Value>(\n  selectorOrValue: SelectorOrValue<Input, Output, Expected, Value> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<Value | Expected | undefined> {\n  if (selectorOrValue === undefined) {\n    return args.case.expected;\n  }\n  return typeof selectorOrValue === \"function\"\n    ? (selectorOrValue as ValueSelector<Input, Output, Expected, Value>)(args)\n    : selectorOrValue;\n}\n\nasync function resolveJudgePrompt<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  if (selector !== undefined) {\n    return selector(args);\n  }\n  return [\n    `Suite: ${args.suiteName}`,\n    `Case: ${args.case.id}`,\n    `Input: ${formatValue(args.case.input)}`,\n    `Expected: ${formatValue(args.case.expected)}`,\n    `Output: ${formatValue(defaultOutputValue(args.output))}`,\n  ].join(\"\\n\\n\");\n}\n\nfunction defaultOutputValue(output: unknown): unknown {\n  if (\n    typeof output === \"object\" &&\n    output !== null &&\n    \"output\" in output &&\n    typeof (output as { output?: unknown }).output === \"string\"\n  ) {\n    return (output as { output: string }).output;\n  }\n  return output;\n}\n\nfunction stableComparable(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  return JSON.stringify(value);\n}\n\nfunction formatValue(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  try {\n    return JSON.stringify(value);\n  } catch {\n    return String(value);\n  }\n}\n\nfunction errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n\nasync function mapWithConcurrency<Input, Output>(\n  inputs: Input[],\n  concurrency: number,\n  mapper: (input: Input) => Promise<Output>,\n): Promise<Output[]> {\n  const results = new Array<Output>(inputs.length);\n  let next = 0;\n\n  async function worker(): Promise<void> {\n    while (next < inputs.length) {\n      const index = next;\n      next += 1;\n      results[index] = await mapper(inputs[index] as Input);\n    }\n  }\n\n  await Promise.all(Array.from({ length: Math.min(concurrency, inputs.length) }, () => worker()));\n  return results;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAAA,SAAS,SAAS;AA6CX,IAAM,cAAc;AAAA,EACzB,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,WAAO;AAAA,MACL,SAAS;AAAA,MACT,GAAI,UAAU,SAAY,CAAC,IAAI,EAAE,MAAM;AAAA,MACvC,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,MACpE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AAAA,EAEA,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,WAAO;AAAA,MACL,SAAS;AAAA,MACT,GAAI,UAAU,SAAY,CAAC,IAAI,EAAE,MAAM;AAAA,MACvC,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,MACpE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AAAA,EAEA,QACE,QACA,UAII,CAAC,GACe;AACpB,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,MACA,GAAI,QAAQ,UAAU,SAAY,CAAC,IAAI,EAAE,OAAO,QAAQ,MAAM;AAAA,MAC9D,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,MACpE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AACF;AA4DA,eAAsB,aACpB,SACmD;AACnD,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,UAAU,MAAM;AAAA,IACpB,QAAQ;AAAA,IACR,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,eAAe,CAAC,CAAC;AAAA,IAChD,CAAC,aAAa,YAAY,SAAS,QAAQ;AAAA,EAC7C;AACA,QAAM,SAAS,cAAc,OAAO;AACpC,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd;AAAA,IACA,GAAG;AAAA,IACH,YAAY,KAAK,IAAI,IAAI;AAAA,EAC3B;AACF;AAgBO,SAAS,WACd,UAAsD,CAAC,GACT;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,cAAc,QAAQ,QAAQ,IAAI;AACvD,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,6CAA6C;AAAA,MAC1E;AACA,YAAM,SAAS,iBAAiB,MAAM,MAAM,iBAAiB,QAAQ;AACrE,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,YAAY,YAAY,QAAQ,CAAC,IAAI,CAAC;AAAA,IAC/E;AAAA,EACF;AACF;AAQO,SAAS,SACd,UAAoD,CAAC,GACP;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,0CAA0C;AAAA,MACvE;AACA,UAAI,OAAO,aAAa,YAAY,EAAE,oBAAoB,SAAS;AACjE,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,YAAM,SAAS,oBAAoB,SAAS,SAAS,KAAK,MAAM,IAAI,OAAO,SAAS,QAAQ;AAC5F,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,0BAA0B,OAAO,QAAQ,CAAC,IAAI,CAAC;AAAA,IACxF;AAAA,EACF;AACF;AAUO,SAAS,mBACd,SAC6C;AAC7C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,UAAI,OAAO,aAAa,UAAU;AAChC,eAAO,YAAY,QAAQ,sDAAsD;AAAA,MACnF;AACA,YAAM,CAAC,iBAAiB,iBAAiB,IAAI,MAAM,QAAQ,IAAI;AAAA,QAC7D,UAAU,QAAQ,OAAO,MAAM;AAAA,QAC/B,UAAU,QAAQ,OAAO,QAAQ;AAAA,MACnC,CAAC;AACD,YAAM,QAAQ,iBAAiB,gBAAgB,QAAQ,kBAAkB,MAAM;AAC/E,aAAO,SAAS,QAAQ,YACpB,YAAY,KAAK,KAAK,IACtB,YAAY,KAAK,OAAO,EAAE,SAAS,8BAA8B,QAAQ,SAAS,IAAI,CAAC;AAAA,IAC7F;AAAA,EACF;AACF;AAYO,SAAS,SACd,SACmD;AACnD,QAAM,YAAY,IAAI,iBAAiB,QAAQ,OAAO,QAAQ,MAAM,EACjE;AAAA,IACC,QAAQ,gBACN;AAAA,EACJ,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,WAAW,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACvF,eAAO,QAAQ,OAAO,QAAQ,IAAI,YAAY,KAAK,QAAQ,IAAI,YAAY,KAAK,QAAQ;AAAA,MAC1F,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;AAiBO,SAAS,SACd,SAC0D;AAC1D,QAAM,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAAI,QAAQ,SAAS,KAAK,IAAI,IAAI,QAAQ;AACzF,QAAM,YAAY,IAAI;AAAA,IACpB,QAAQ;AAAA,IACR,EAAE,OAAO;AAAA,MACP,OAAO,EAAE,OAAO;AAAA,MAChB,UAAU,EAAE,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,EACG;AAAA,IACC,QAAQ,gBACN;AAAA,EAAgD,QAAQ;AAAA;AAAA;AAAA,EAC5D,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,QAAQ,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACpF,YAAI,MAAM,QAAQ,KAAK,MAAM,QAAQ,GAAG;AACtC,iBAAO,YAAY,QAAQ,SAAS,MAAM,KAAK,gCAAgC;AAAA,YAC7E;AAAA,UACF,CAAC;AAAA,QACH;AACA,eAAO,MAAM,SAAS,QAAQ,YAC1B,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC,IACnD,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC;AAAA,MACzD,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;AAeO,SAAS,gBACd,OACA,UAAkE,CAAC,GACvB;AAC5C,SAAO,OAAO,OAAO,aAAa;AAChC,UAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,KAAK,OAAO,KAAK;AAChE,UAAM,WAAW,MAAM,MAAM,OAAO,MAAM,EAAE,KAAK;AACjD,WAAO,QAAQ,WAAW,SAAY,WAAW,QAAQ,OAAO,UAAU,QAAQ;AAAA,EACpF;AACF;AAEA,eAAe,YACb,SACA,UACkD;AAClD,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,QAAQ,OAAO,SAAS,OAAO,QAAQ;AAAA,EACxD,SAAS,OAAO;AACd,kBAAc;AAAA,EAChB;AAEA,QAAM,UAA8B,CAAC;AACrC,aAAW,UAAU,QAAQ,SAAS;AACpC,UAAM,UACJ,gBAAgB,SACZ,MAAM,aAAa,QAAQ,MAAM,UAAU,QAAkB,MAAM,IACnE,YAAY,QAAQ,kBAAkB,aAAa,WAAW,CAAC,EAAE;AACvE,UAAM,iBAAiB,MAAM,cAAc;AAAA,MACzC,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW,QAAQ,aAAa,CAAC;AAAA,MACjC,qBAAqB,QAAQ,wBAAwB;AAAA,IACvD,CAAC;AACD,YAAQ,KAAK,EAAE,YAAY,OAAO,MAAM,SAAS,eAAe,CAAC;AAAA,EACnE;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,GAAI,WAAW,SAAY,CAAC,IAAI,EAAE,OAAO;AAAA,IACzC,GAAI,gBAAgB,SAAY,CAAC,IAAI,EAAE,YAAY;AAAA,IACnD;AAAA,EACF;AACF;AAEA,eAAe,aACb,WACA,UACA,QACA,QACsB;AACtB,MAAI;AACF,WAAO,MAAM,OAAO,SAAS,EAAE,WAAW,MAAM,UAAU,OAAO,CAAC;AAAA,EACpE,SAAS,OAAO;AACd,WAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,EAChD;AACF;AAEA,eAAe,cAAuC,MAS/B;AACrB,QAAM,SAAoB,CAAC;AAC3B,aAAW,YAAY,KAAK,WAAW;AACrC,QAAI;AACF,YAAM,SAAS,OAAO;AAAA,QACpB,WAAW,KAAK;AAAA,QAChB,MAAM,KAAK;AAAA,QACX,QAAQ,KAAK;AAAA,QACb,aAAa,KAAK;AAAA,QAClB,QAAQ,KAAK;AAAA,QACb,SAAS,KAAK;AAAA,MAChB,CAAC;AAAA,IACH,SAAS,OAAO;AACd,UAAI,KAAK,qBAAqB;AAC5B,cAAM;AAAA,MACR;AACA,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,cAAc,SAIrB;AACA,MAAI,SAAS;AACb,MAAI,SAAS;AACb,MAAI,UAAU;AACd,aAAW,UAAU,SAAS;AAC5B,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,UAAW,YAAW;AAAA,IACvD;AAAA,EACF;AACA,SAAO,EAAE,QAAQ,QAAQ,QAAQ;AACnC;AAEA,eAAe,cACb,UACA,MACkB;AAClB,SAAO,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,SAAS,IAAI;AACjF;AAEA,eAAe,kBACb,UACA,MACiB;AACjB,QAAM,QAAQ,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,MAAM,SAAS,IAAI;AAC5F,SAAO,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK;AACjE;AAEA,eAAe,gBACb,iBACA,MACuC;AACvC,MAAI,oBAAoB,QAAW;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AACA,SAAO,OAAO,oBAAoB,aAC7B,gBAAkE,IAAI,IACvE;AACN;AAEA,eAAe,mBACb,UACA,MACiB;AACjB,MAAI,aAAa,QAAW;AAC1B,WAAO,SAAS,IAAI;AAAA,EACtB;AACA,SAAO;AAAA,IACL,UAAU,KAAK,SAAS;AAAA,IACxB,SAAS,KAAK,KAAK,EAAE;AAAA,IACrB,UAAU,YAAY,KAAK,KAAK,KAAK,CAAC;AAAA,IACtC,aAAa,YAAY,KAAK,KAAK,QAAQ,CAAC;AAAA,IAC5C,WAAW,YAAY,mBAAmB,KAAK,MAAM,CAAC,CAAC;AAAA,EACzD,EAAE,KAAK,MAAM;AACf;AAEA,SAAS,mBAAmB,QAA0B;AACpD,MACE,OAAO,WAAW,YAClB,WAAW,QACX,YAAY,UACZ,OAAQ,OAAgC,WAAW,UACnD;AACA,WAAQ,OAA8B;AAAA,EACxC;AACA,SAAO;AACT;AAEA,SAAS,iBAAiB,OAAwB;AAChD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,SAAO,KAAK,UAAU,KAAK;AAC7B;AAEA,SAAS,YAAY,OAAwB;AAC3C,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,MAAI;AACF,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B,QAAQ;AACN,WAAO,OAAO,KAAK;AAAA,EACrB;AACF;AAEA,SAAS,aAAa,OAAwB;AAC5C,SAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC9D;AAEA,eAAe,mBACb,QACA,aACA,QACmB;AACnB,QAAM,UAAU,IAAI,MAAc,OAAO,MAAM;AAC/C,MAAI,OAAO;AAEX,iBAAe,SAAwB;AACrC,WAAO,OAAO,OAAO,QAAQ;AAC3B,YAAM,QAAQ;AACd,cAAQ;AACR,cAAQ,KAAK,IAAI,MAAM,OAAO,OAAO,KAAK,CAAU;AAAA,IACtD;AAAA,EACF;AAEA,QAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,OAAO,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAC9F,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../../src/evals/agent-target.ts","../../src/evals/metrics.ts","../../src/evals/format.ts","../../src/evals/outcome.ts","../../src/evals/selectors.ts","../../src/evals/runner.ts"],"sourcesContent":["import type { Agent } from \"../agent/agent\";\nimport type { PromptResponse } from \"../agent/request\";\nimport type { Message } from \"../completion\";\nimport type { EvalCase, EvalTarget } from \"./types\";\n\nexport type AgentEvalTargetOptions<Input, Output = PromptResponse> = {\n  prompt?: ((input: Input, testCase: EvalCase<Input>) => string | Message) | undefined;\n  output?: ((response: PromptResponse, testCase: EvalCase<Input>) => Output) | undefined;\n};\n\nexport function agentEvalTarget<Input>(\n  agent: Agent,\n  options?: AgentEvalTargetOptions<Input, PromptResponse>,\n): EvalTarget<Input, PromptResponse>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output>,\n): EvalTarget<Input, Output>;\nexport function agentEvalTarget<Input, Output>(\n  agent: Agent,\n  options: AgentEvalTargetOptions<Input, Output | PromptResponse> = {},\n): EvalTarget<Input, Output | PromptResponse> {\n  return async (input, testCase) => {\n    const prompt = options.prompt?.(input, testCase) ?? String(input);\n    const response = await agent.prompt(prompt).send();\n    return options.output === undefined ? response : options.output(response, testCase);\n  };\n}\n","import { z } from \"zod\";\nimport type { CompletionModel } from \"../completion\";\nimport { cosineSimilarity, type EmbeddingModel, embedText } from \"../embeddings\";\nimport { ExtractorBuilder } from \"../extractor\";\nimport type { ZodSchema } from \"../schema\";\nimport { errorMessage, formatValue, stableComparable } from \"./format\";\nimport { EvalOutcome } from \"./outcome\";\nimport { resolveActual, resolveActualText, resolveExpected, resolveJudgePrompt } from \"./selectors\";\nimport type { EvalMetric, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport type ExactMatchOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, unknown> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, unknown> | undefined;\n};\n\nexport function exactMatch<Input, Output, Expected = unknown>(\n  options: ExactMatchOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"exact_match\",\n    async evaluate(args) {\n      const actual = await resolveActual(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for exact match.\");\n      }\n      const passed = stableComparable(actual) === stableComparable(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });\n    },\n  };\n}\n\nexport type ContainsOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string | RegExp> | undefined;\n};\n\nexport function contains<Input, Output, Expected = unknown>(\n  options: ContainsOptions<Input, Output, Expected> = {},\n): EvalMetric<Input, Output, boolean, Expected> {\n  return {\n    name: options.name ?? \"contains\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for contains.\");\n      }\n      if (typeof expected !== \"string\" && !(expected instanceof RegExp)) {\n        return EvalOutcome.invalid(\"Contains expected value must be a string or RegExp.\");\n      }\n      const passed = expected instanceof RegExp ? expected.test(actual) : actual.includes(expected);\n      return passed\n        ? EvalOutcome.pass(true)\n        : EvalOutcome.fail(false, { comment: `Output did not contain ${String(expected)}.` });\n    },\n  };\n}\n\nexport type SemanticSimilarityOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: EmbeddingModel;\n  threshold: number;\n  actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n  expected?: SelectorOrValue<Input, Output, Expected, string> | undefined;\n};\n\nexport function semanticSimilarity<Input, Output, Expected = unknown>(\n  options: SemanticSimilarityOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, number, Expected> {\n  return {\n    name: options.name ?? \"semantic_similarity\",\n    async evaluate(args) {\n      const actual = await resolveActualText(options.actual, args);\n      const expected = await resolveExpected(options.expected, args);\n      if (expected === undefined) {\n        return EvalOutcome.invalid(\"No expected value provided for semantic similarity.\");\n      }\n      if (typeof expected !== \"string\") {\n        return EvalOutcome.invalid(\"Semantic similarity expected value must be a string.\");\n      }\n      const [actualEmbedding, expectedEmbedding] = await Promise.all([\n        embedText(options.model, actual),\n        embedText(options.model, expected),\n      ]);\n      const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);\n      return score >= options.threshold\n        ? EvalOutcome.pass(score)\n        : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });\n    },\n  };\n}\n\nexport type LlmJudgeOptions<Input, Output, SchemaOutput, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  schema: ZodSchema<SchemaOutput>;\n  passes(value: SchemaOutput): boolean;\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmJudge<Input, Output, SchemaOutput, Expected = unknown>(\n  options: LlmJudgeOptions<Input, Output, SchemaOutput, Expected>,\n): EvalMetric<Input, Output, SchemaOutput, Expected> {\n  const extractor = new ExtractorBuilder(options.model, options.schema)\n    .instructions(\n      options.instructions ??\n        \"Judge the eval case by the requested schema. Submit the judgment using the schema.\",\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_judge\",\n    async evaluate(args) {\n      try {\n        const judgment = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        return options.passes(judgment) ? EvalOutcome.pass(judgment) : EvalOutcome.fail(judgment);\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n\nexport type LlmScoreMetricScore = {\n  score: number;\n  feedback: string;\n};\n\nexport type LlmScoreOptions<Input, Output, Expected = unknown> = {\n  name?: string | undefined;\n  model: CompletionModel;\n  threshold: number;\n  criteria: string | string[];\n  instructions?: string | undefined;\n  retries?: number | undefined;\n  prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmScore<Input, Output, Expected = unknown>(\n  options: LlmScoreOptions<Input, Output, Expected>,\n): EvalMetric<Input, Output, LlmScoreMetricScore, Expected> {\n  const criteria = Array.isArray(options.criteria) ? options.criteria.join(\"\\n\") : options.criteria;\n  const extractor = new ExtractorBuilder(\n    options.model,\n    z.object({\n      score: z.number(),\n      feedback: z.string(),\n    }),\n  )\n    .instructions(\n      options.instructions ??\n        `Score the eval case against these criteria:\\n${criteria}\\n\\nReturn a score between 0 and 1 and brief feedback.`,\n    )\n    .retries(options.retries ?? 0)\n    .build();\n\n  return {\n    name: options.name ?? \"llm_score\",\n    async evaluate(args) {\n      try {\n        const score = await extractor.extract(await resolveJudgePrompt(options.prompt, args));\n        if (score.score < 0 || score.score > 1) {\n          return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {\n            score,\n          });\n        }\n        return score.score >= options.threshold\n          ? EvalOutcome.pass(score, { comment: score.feedback })\n          : EvalOutcome.fail(score, { comment: score.feedback });\n      } catch (error) {\n        return EvalOutcome.invalid(errorMessage(error));\n      }\n    },\n  };\n}\n","export function defaultOutputValue(output: unknown): unknown {\n  if (\n    typeof output === \"object\" &&\n    output !== null &&\n    \"output\" in output &&\n    typeof (output as { output?: unknown }).output === \"string\"\n  ) {\n    return (output as { output: string }).output;\n  }\n  return output;\n}\n\nexport function stableComparable(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  return JSON.stringify(value);\n}\n\nexport function formatValue(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  try {\n    return JSON.stringify(value);\n  } catch {\n    return String(value);\n  }\n}\n\nexport function errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n","import type { EvalMetadata } from \"./types\";\n\nexport type EvalOutcome<Score = unknown> =\n  | {\n      outcome: \"pass\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"fail\";\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    }\n  | {\n      outcome: \"invalid\";\n      reason: string;\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    };\n\nexport const EvalOutcome = {\n  pass<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    return {\n      outcome: \"pass\",\n      ...(score === undefined ? {} : { score }),\n      ...(options.comment === undefined ? {} : { comment: options.comment }),\n      ...(options.metadata === undefined ? {} : { metadata: options.metadata }),\n    };\n  },\n\n  fail<Score>(\n    score?: Score,\n    options: { comment?: string | undefined; metadata?: EvalMetadata | undefined } = {},\n  ): EvalOutcome<Score> {\n    return {\n      outcome: \"fail\",\n      ...(score === undefined ? {} : { score }),\n      ...(options.comment === undefined ? {} : { comment: options.comment }),\n      ...(options.metadata === undefined ? {} : { metadata: options.metadata }),\n    };\n  },\n\n  invalid<Score = never>(\n    reason: string,\n    options: {\n      score?: Score | undefined;\n      comment?: string | undefined;\n      metadata?: EvalMetadata | undefined;\n    } = {},\n  ): EvalOutcome<Score> {\n    return {\n      outcome: \"invalid\",\n      reason,\n      ...(options.score === undefined ? {} : { score: options.score }),\n      ...(options.comment === undefined ? {} : { comment: options.comment }),\n      ...(options.metadata === undefined ? {} : { metadata: options.metadata }),\n    };\n  },\n};\n","import { defaultOutputValue, formatValue } from \"./format\";\nimport type { EvalMetricArgs, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport async function resolveActual<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, unknown> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<unknown> {\n  return selector === undefined ? defaultOutputValue(args.output) : selector(args);\n}\n\nexport async function resolveActualText<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  const value = selector === undefined ? defaultOutputValue(args.output) : await selector(args);\n  return typeof value === \"string\" ? value : JSON.stringify(value);\n}\n\nexport async function resolveExpected<Input, Output, Expected, Value>(\n  selectorOrValue: SelectorOrValue<Input, Output, Expected, Value> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<Value | Expected | undefined> {\n  if (selectorOrValue === undefined) {\n    return args.case.expected;\n  }\n  return typeof selectorOrValue === \"function\"\n    ? (selectorOrValue as ValueSelector<Input, Output, Expected, Value>)(args)\n    : selectorOrValue;\n}\n\nexport async function resolveJudgePrompt<Input, Output, Expected>(\n  selector: ValueSelector<Input, Output, Expected, string> | undefined,\n  args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n  if (selector !== undefined) {\n    return selector(args);\n  }\n  return [\n    `Suite: ${args.suiteName}`,\n    `Case: ${args.case.id}`,\n    `Input: ${formatValue(args.case.input)}`,\n    `Expected: ${formatValue(args.case.expected)}`,\n    `Output: ${formatValue(defaultOutputValue(args.output))}`,\n  ].join(\"\\n\\n\");\n}\n","import { mapWithConcurrency } from \"../internal/concurrency\";\nimport { errorMessage } from \"./format\";\nimport { EvalOutcome, type EvalOutcome as EvalOutcomeType } from \"./outcome\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalMetric,\n  EvalMetricResult,\n  EvalReporter,\n  EvalSuiteResult,\n  RunEvalSuiteOptions,\n} from \"./types\";\n\nexport async function runEvalSuite<Input, Output, Expected = unknown>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n): Promise<EvalSuiteResult<Input, Output, Expected>> {\n  const startedAt = Date.now();\n  const results = await mapWithConcurrency(\n    options.cases,\n    Math.max(1, Math.trunc(options.concurrency ?? 1)),\n    (testCase) => runEvalCase(options, testCase),\n  );\n  const counts = countOutcomes(results);\n  return {\n    name: options.name,\n    results,\n    ...counts,\n    durationMs: Date.now() - startedAt,\n  };\n}\n\nasync function runEvalCase<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n  testCase: EvalCase<Input, Expected>,\n): Promise<EvalCaseResult<Input, Output, Expected>> {\n  let output: Output | undefined;\n  let targetError: unknown;\n  try {\n    output = await options.target(testCase.input, testCase);\n  } catch (error) {\n    targetError = error;\n  }\n\n  const metrics: EvalMetricResult[] = [];\n  for (const metric of options.metrics) {\n    const outcome =\n      targetError === undefined\n        ? await safeEvaluate(options.name, testCase, output as Output, metric)\n        : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`);\n    const reporterErrors = await reportOutcome({\n      suiteName: options.name,\n      testCase,\n      output,\n      targetError,\n      metric,\n      outcome,\n      reporters: options.reporters ?? [],\n      failOnReporterError: options.failOnReporterError === true,\n    });\n    metrics.push({ metricName: metric.name, outcome, reporterErrors });\n  }\n\n  return {\n    case: testCase,\n    ...(output === undefined ? {} : { output }),\n    ...(targetError === undefined ? {} : { targetError }),\n    metrics,\n  };\n}\n\nasync function safeEvaluate<Input, Output, Expected>(\n  suiteName: string,\n  testCase: EvalCase<Input, Expected>,\n  output: Output,\n  metric: EvalMetric<Input, Output, unknown, Expected>,\n): Promise<EvalOutcomeType> {\n  try {\n    return await metric.evaluate({ suiteName, case: testCase, output });\n  } catch (error) {\n    return EvalOutcome.invalid(errorMessage(error));\n  }\n}\n\nasync function reportOutcome<Input, Output, Expected>(args: {\n  suiteName: string;\n  testCase: EvalCase<Input, Expected>;\n  output: Output | undefined;\n  targetError: unknown;\n  metric: EvalMetric<Input, Output, unknown, Expected>;\n  outcome: EvalOutcomeType;\n  reporters: Array<EvalReporter<Input, Output, Expected>>;\n  failOnReporterError: boolean;\n}): Promise<unknown[]> {\n  const errors: unknown[] = [];\n  for (const reporter of args.reporters) {\n    try {\n      await reporter.report({\n        suiteName: args.suiteName,\n        case: args.testCase,\n        output: args.output,\n        targetError: args.targetError,\n        metric: args.metric,\n        outcome: args.outcome,\n      });\n    } catch (error) {\n      if (args.failOnReporterError) {\n        throw error;\n      }\n      errors.push(error);\n    }\n  }\n  return errors;\n}\n\nfunction countOutcomes(results: Array<EvalCaseResult<unknown, unknown, unknown>>): {\n  passed: number;\n  failed: number;\n  invalid: number;\n} {\n  let passed = 0;\n  let failed = 0;\n  let invalid = 0;\n  for (const result of results) {\n    for (const metric of result.metrics) {\n      if (metric.outcome.outcome === \"pass\") passed += 1;\n      if (metric.outcome.outcome === \"fail\") failed += 1;\n      if (metric.outcome.outcome === \"invalid\") invalid += 1;\n    }\n  }\n  return { passed, failed, invalid };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAkBO,SAAS,gBACd,OACA,UAAkE,CAAC,GACvB;AAC5C,SAAO,OAAO,OAAO,aAAa;AAChC,UAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,KAAK,OAAO,KAAK;AAChE,UAAM,WAAW,MAAM,MAAM,OAAO,MAAM,EAAE,KAAK;AACjD,WAAO,QAAQ,WAAW,SAAY,WAAW,QAAQ,OAAO,UAAU,QAAQ;AAAA,EACpF;AACF;;;AC3BA,SAAS,SAAS;;;ACAX,SAAS,mBAAmB,QAA0B;AAC3D,MACE,OAAO,WAAW,YAClB,WAAW,QACX,YAAY,UACZ,OAAQ,OAAgC,WAAW,UACnD;AACA,WAAQ,OAA8B;AAAA,EACxC;AACA,SAAO;AACT;AAEO,SAAS,iBAAiB,OAAwB;AACvD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,SAAO,KAAK,UAAU,KAAK;AAC7B;AAEO,SAAS,YAAY,OAAwB;AAClD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,MAAI;AACF,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B,QAAQ;AACN,WAAO,OAAO,KAAK;AAAA,EACrB;AACF;AAEO,SAAS,aAAa,OAAwB;AACnD,SAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC9D;;;ACTO,IAAM,cAAc;AAAA,EACzB,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,WAAO;AAAA,MACL,SAAS;AAAA,MACT,GAAI,UAAU,SAAY,CAAC,IAAI,EAAE,MAAM;AAAA,MACvC,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,MACpE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AAAA,EAEA,KACE,OACA,UAAiF,CAAC,GAC9D;AACpB,WAAO;AAAA,MACL,SAAS;AAAA,MACT,GAAI,UAAU,SAAY,CAAC,IAAI,EAAE,MAAM;AAAA,MACvC,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,MACpE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AAAA,EAEA,QACE,QACA,UAII,CAAC,GACe;AACpB,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,MACA,GAAI,QAAQ,UAAU,SAAY,CAAC,IAAI,EAAE,OAAO,QAAQ,MAAM;AAAA,MAC9D,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,MACpE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AACF;;;AC7DA,eAAsB,cACpB,UACA,MACkB;AAClB,SAAO,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,SAAS,IAAI;AACjF;AAEA,eAAsB,kBACpB,UACA,MACiB;AACjB,QAAM,QAAQ,aAAa,SAAY,mBAAmB,KAAK,MAAM,IAAI,MAAM,SAAS,IAAI;AAC5F,SAAO,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK;AACjE;AAEA,eAAsB,gBACpB,iBACA,MACuC;AACvC,MAAI,oBAAoB,QAAW;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AACA,SAAO,OAAO,oBAAoB,aAC7B,gBAAkE,IAAI,IACvE;AACN;AAEA,eAAsB,mBACpB,UACA,MACiB;AACjB,MAAI,aAAa,QAAW;AAC1B,WAAO,SAAS,IAAI;AAAA,EACtB;AACA,SAAO;AAAA,IACL,UAAU,KAAK,SAAS;AAAA,IACxB,SAAS,KAAK,KAAK,EAAE;AAAA,IACrB,UAAU,YAAY,KAAK,KAAK,KAAK,CAAC;AAAA,IACtC,aAAa,YAAY,KAAK,KAAK,QAAQ,CAAC;AAAA,IAC5C,WAAW,YAAY,mBAAmB,KAAK,MAAM,CAAC,CAAC;AAAA,EACzD,EAAE,KAAK,MAAM;AACf;;;AH5BO,SAAS,WACd,UAAsD,CAAC,GACT;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,cAAc,QAAQ,QAAQ,IAAI;AACvD,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,6CAA6C;AAAA,MAC1E;AACA,YAAM,SAAS,iBAAiB,MAAM,MAAM,iBAAiB,QAAQ;AACrE,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,YAAY,YAAY,QAAQ,CAAC,IAAI,CAAC;AAAA,IAC/E;AAAA,EACF;AACF;AAQO,SAAS,SACd,UAAoD,CAAC,GACP;AAC9C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,0CAA0C;AAAA,MACvE;AACA,UAAI,OAAO,aAAa,YAAY,EAAE,oBAAoB,SAAS;AACjE,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,YAAM,SAAS,oBAAoB,SAAS,SAAS,KAAK,MAAM,IAAI,OAAO,SAAS,QAAQ;AAC5F,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,0BAA0B,OAAO,QAAQ,CAAC,IAAI,CAAC;AAAA,IACxF;AAAA,EACF;AACF;AAUO,SAAS,mBACd,SAC6C;AAC7C,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,UAAI,OAAO,aAAa,UAAU;AAChC,eAAO,YAAY,QAAQ,sDAAsD;AAAA,MACnF;AACA,YAAM,CAAC,iBAAiB,iBAAiB,IAAI,MAAM,QAAQ,IAAI;AAAA,QAC7D,UAAU,QAAQ,OAAO,MAAM;AAAA,QAC/B,UAAU,QAAQ,OAAO,QAAQ;AAAA,MACnC,CAAC;AACD,YAAM,QAAQ,iBAAiB,gBAAgB,QAAQ,kBAAkB,MAAM;AAC/E,aAAO,SAAS,QAAQ,YACpB,YAAY,KAAK,KAAK,IACtB,YAAY,KAAK,OAAO,EAAE,SAAS,8BAA8B,QAAQ,SAAS,IAAI,CAAC;AAAA,IAC7F;AAAA,EACF;AACF;AAYO,SAAS,SACd,SACmD;AACnD,QAAM,YAAY,IAAI,iBAAiB,QAAQ,OAAO,QAAQ,MAAM,EACjE;AAAA,IACC,QAAQ,gBACN;AAAA,EACJ,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,WAAW,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACvF,eAAO,QAAQ,OAAO,QAAQ,IAAI,YAAY,KAAK,QAAQ,IAAI,YAAY,KAAK,QAAQ;AAAA,MAC1F,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;AAiBO,SAAS,SACd,SAC0D;AAC1D,QAAM,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAAI,QAAQ,SAAS,KAAK,IAAI,IAAI,QAAQ;AACzF,QAAM,YAAY,IAAI;AAAA,IACpB,QAAQ;AAAA,IACR,EAAE,OAAO;AAAA,MACP,OAAO,EAAE,OAAO;AAAA,MAChB,UAAU,EAAE,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,EACG;AAAA,IACC,QAAQ,gBACN;AAAA,EAAgD,QAAQ;AAAA;AAAA;AAAA,EAC5D,EACC,QAAQ,QAAQ,WAAW,CAAC,EAC5B,MAAM;AAET,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,QAAQ,MAAM,UAAU,QAAQ,MAAM,mBAAmB,QAAQ,QAAQ,IAAI,CAAC;AACpF,YAAI,MAAM,QAAQ,KAAK,MAAM,QAAQ,GAAG;AACtC,iBAAO,YAAY,QAAQ,SAAS,MAAM,KAAK,gCAAgC;AAAA,YAC7E;AAAA,UACF,CAAC;AAAA,QACH;AACA,eAAO,MAAM,SAAS,QAAQ,YAC1B,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC,IACnD,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,SAAS,CAAC;AAAA,MACzD,SAAS,OAAO;AACd,eAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,MAChD;AAAA,IACF;AAAA,EACF;AACF;;;AIzKA,eAAsB,aACpB,SACmD;AACnD,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,UAAU,MAAM;AAAA,IACpB,QAAQ;AAAA,IACR,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,eAAe,CAAC,CAAC;AAAA,IAChD,CAAC,aAAa,YAAY,SAAS,QAAQ;AAAA,EAC7C;AACA,QAAM,SAAS,cAAc,OAAO;AACpC,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd;AAAA,IACA,GAAG;AAAA,IACH,YAAY,KAAK,IAAI,IAAI;AAAA,EAC3B;AACF;AAEA,eAAe,YACb,SACA,UACkD;AAClD,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,QAAQ,OAAO,SAAS,OAAO,QAAQ;AAAA,EACxD,SAAS,OAAO;AACd,kBAAc;AAAA,EAChB;AAEA,QAAM,UAA8B,CAAC;AACrC,aAAW,UAAU,QAAQ,SAAS;AACpC,UAAM,UACJ,gBAAgB,SACZ,MAAM,aAAa,QAAQ,MAAM,UAAU,QAAkB,MAAM,IACnE,YAAY,QAAQ,kBAAkB,aAAa,WAAW,CAAC,EAAE;AACvE,UAAM,iBAAiB,MAAM,cAAc;AAAA,MACzC,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW,QAAQ,aAAa,CAAC;AAAA,MACjC,qBAAqB,QAAQ,wBAAwB;AAAA,IACvD,CAAC;AACD,YAAQ,KAAK,EAAE,YAAY,OAAO,MAAM,SAAS,eAAe,CAAC;AAAA,EACnE;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,GAAI,WAAW,SAAY,CAAC,IAAI,EAAE,OAAO;AAAA,IACzC,GAAI,gBAAgB,SAAY,CAAC,IAAI,EAAE,YAAY;AAAA,IACnD;AAAA,EACF;AACF;AAEA,eAAe,aACb,WACA,UACA,QACA,QAC0B;AAC1B,MAAI;AACF,WAAO,MAAM,OAAO,SAAS,EAAE,WAAW,MAAM,UAAU,OAAO,CAAC;AAAA,EACpE,SAAS,OAAO;AACd,WAAO,YAAY,QAAQ,aAAa,KAAK,CAAC;AAAA,EAChD;AACF;AAEA,eAAe,cAAuC,MAS/B;AACrB,QAAM,SAAoB,CAAC;AAC3B,aAAW,YAAY,KAAK,WAAW;AACrC,QAAI;AACF,YAAM,SAAS,OAAO;AAAA,QACpB,WAAW,KAAK;AAAA,QAChB,MAAM,KAAK;AAAA,QACX,QAAQ,KAAK;AAAA,QACb,aAAa,KAAK;AAAA,QAClB,QAAQ,KAAK;AAAA,QACb,SAAS,KAAK;AAAA,MAChB,CAAC;AAAA,IACH,SAAS,OAAO;AACd,UAAI,KAAK,qBAAqB;AAC5B,cAAM;AAAA,MACR;AACA,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,cAAc,SAIrB;AACA,MAAI,SAAS;AACb,MAAI,SAAS;AACb,MAAI,UAAU;AACd,aAAW,UAAU,SAAS;AAC5B,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,OAAQ,WAAU;AACjD,UAAI,OAAO,QAAQ,YAAY,UAAW,YAAW;AAAA,IACvD;AAAA,EACF;AACA,SAAO,EAAE,QAAQ,QAAQ,QAAQ;AACnC;","names":[]}

package/dist/extractor/index.d.ts

@@ -1,11 +1,11 @@
-import { e as Agent } from '../agent-0UeJ9Rad.js';
+import { e as Agent } from '../agent-B-ls5y_g.js';
 import { U as Usage, M as Message, C as CompletionModel, e as JsonValue, m as ToolChoice } from '../types-C54aNoCd.js';
 import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
 import '../memory/index.js';
 import '../observability/index.js';
 import '../tool-C3ciF-VG.js';
-import '../middleware-BQ7fkEEe.js';
-import '../embeddings/index.js';
+import '../middleware-CGiEIaBx.js';
+import '../types-IB2e9u5M.js';
 import '../vector-store/index.js';
 import 'zod';
 

package/dist/extractor/index.js

@@ -2,16 +2,17 @@ import {
   ExtractionError,
   Extractor,
   ExtractorBuilder
-} from "../chunk-PP4VIN3Y.js";
-import "../chunk-6U6PZ5MD.js";
-import "../chunk-TP32W7XT.js";
+} from "../chunk-4PHDOEKY.js";
+import "../chunk-BD66WU2B.js";
+import "../chunk-TVIADATY.js";
 import "../chunk-XUUY2L2D.js";
 import "../chunk-XXT2UCAR.js";
 import "../chunk-YK4WAAS4.js";
-import "../chunk-I2HOMD3R.js";
-import "../chunk-65QV627O.js";
-import "../chunk-A7VDIZQN.js";
+import "../chunk-CY7SB5FG.js";
+import "../chunk-OIA4CN5V.js";
 import "../chunk-6GJDBBDC.js";
+import "../chunk-EFGX3EX5.js";
+import "../chunk-OIMLU4SF.js";
 export {
   ExtractionError,
   Extractor,

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 export { AgentBuilder, MaxTurnsError, PromptCancelledError } from './agent/index.js';
-export { A as AgentChildStreamEvent, a as AgentStreamEvent, P as PromptResponse, c as cancelPrompt, b as createHook, r as requestToolApproval, d as runControl, s as skipTool, t as toolCallControl } from './agent-0UeJ9Rad.js';
+export { A as AgentChildStreamEvent, a as AgentStreamEvent, P as PromptResponse, c as cancelPrompt, b as createHook, r as requestToolApproval, d as runControl, s as skipTool, t as toolCallControl } from './agent-B-ls5y_g.js';
 export { A as AssistantContent, a as AssistantMessage, C as CompletionModel, b as CompletionRequest, c as CompletionResponse, D as Document, I as ImageContent, J as JsonObject, d as JsonPrimitive, e as JsonValue, M as Message, S as SystemMessage, T as Text, f as ToolCall, g as ToolDefinition, h as ToolMessage, i as ToolResult, j as ToolResultContent, U as Usage, k as UserContent, l as UserMessage } from './types-C54aNoCd.js';
 export { MemoryStore } from './memory/index.js';
 export { Z as ZodSchema } from './zod-schema-C7F4clpm.js';
@@ -12,7 +12,7 @@ import '@modelcontextprotocol/sdk/client/sse.js';
 import '@modelcontextprotocol/sdk/client/stdio.js';
 import '@modelcontextprotocol/sdk/client/streamableHttp.js';
 import './observability/index.js';
-import './middleware-BQ7fkEEe.js';
-import './embeddings/index.js';
+import './middleware-CGiEIaBx.js';
+import './types-IB2e9u5M.js';
 import './vector-store/index.js';
 import 'zod';

package/dist/index.js

@@ -2,10 +2,10 @@ import {
   SkillValidationError,
   loadSkills,
   skill
-} from "./chunk-TILE6Z2N.js";
+} from "./chunk-L6JCKFAX.js";
 import {
   AgentBuilder
-} from "./chunk-6U6PZ5MD.js";
+} from "./chunk-BD66WU2B.js";
 import {
   MaxTurnsError,
   PromptCancelledError,
@@ -15,23 +15,24 @@ import {
   runControl,
   skipTool,
   toolCallControl
-} from "./chunk-TP32W7XT.js";
+} from "./chunk-TVIADATY.js";
 import "./chunk-XUUY2L2D.js";
 import "./chunk-XXT2UCAR.js";
 import "./chunk-YK4WAAS4.js";
 import {
   createThinkTool
-} from "./chunk-I2HOMD3R.js";
+} from "./chunk-CY7SB5FG.js";
 import {
   createTool
-} from "./chunk-65QV627O.js";
-import "./chunk-A7VDIZQN.js";
+} from "./chunk-OIA4CN5V.js";
 import {
   AssistantContent,
   Message,
   Usage,
   UserContent
 } from "./chunk-6GJDBBDC.js";
+import "./chunk-EFGX3EX5.js";
+import "./chunk-OIMLU4SF.js";
 export {
   AgentBuilder,
   AssistantContent,

package/dist/internal/agent.d.ts

@@ -1,8 +1,8 @@
-export { e as Agent, j as AgentEventAppendInput, k as AgentEventRecord, h as AgentEventStore, l as AgentEventStoreInclude, i as AgentEventStoreOptions, x as AgentEventStoreRegistration, y as AgentOptions, z as AgentSession, B as AgentToolOptions, E as DEFAULT_MAX_TURNS, D as DynamicContextOptions, F as DynamicContextRegistration, f as DynamicToolOptions, G as DynamicToolRegistration } from '../agent-0UeJ9Rad.js';
+export { e as Agent, j as AgentEventAppendInput, k as AgentEventRecord, h as AgentEventStore, l as AgentEventStoreInclude, i as AgentEventStoreOptions, x as AgentEventStoreRegistration, y as AgentOptions, z as AgentSession, B as AgentToolOptions, E as DEFAULT_MAX_TURNS, D as DynamicContextOptions, F as DynamicContextRegistration, f as DynamicToolOptions, G as DynamicToolRegistration } from '../agent-B-ls5y_g.js';
 import '../types-C54aNoCd.js';
 import '../memory/index.js';
 import '../observability/index.js';
 import '../tool-C3ciF-VG.js';
-import '../middleware-BQ7fkEEe.js';
-import '../embeddings/index.js';
+import '../middleware-CGiEIaBx.js';
+import '../types-IB2e9u5M.js';
 import '../vector-store/index.js';

package/dist/internal/agent.js

@@ -2,13 +2,14 @@ import {
   Agent,
   AgentSession,
   DEFAULT_MAX_TURNS
-} from "../chunk-TP32W7XT.js";
+} from "../chunk-TVIADATY.js";
 import "../chunk-XUUY2L2D.js";
 import "../chunk-YK4WAAS4.js";
-import "../chunk-I2HOMD3R.js";
-import "../chunk-65QV627O.js";
-import "../chunk-A7VDIZQN.js";
+import "../chunk-CY7SB5FG.js";
+import "../chunk-OIA4CN5V.js";
 import "../chunk-6GJDBBDC.js";
+import "../chunk-EFGX3EX5.js";
+import "../chunk-OIMLU4SF.js";
 export {
   Agent,
   AgentSession,

package/dist/{middleware-BQ7fkEEe.d.ts → middleware-CGiEIaBx.d.ts}

@@ -1,5 +1,5 @@
 import { g as ToolDefinition, j as ToolResultContent } from './types-C54aNoCd.js';
-import { VectorMetadata, EmbeddingModel, EmbeddedDocument } from './embeddings/index.js';
+import { V as VectorMetadata, E as EmbeddingModel, b as EmbeddedDocument } from './types-IB2e9u5M.js';
 import { VectorSearchIndex } from './vector-store/index.js';
 import { A as AnyTool, c as ToolCallContext, N as NormalizedToolOutput } from './tool-C3ciF-VG.js';
 

package/dist/pipeline/index.d.ts

@@ -1,11 +1,11 @@
-import { e as Agent } from '../agent-0UeJ9Rad.js';
+import { e as Agent } from '../agent-B-ls5y_g.js';
 import { J as JsonObject, C as CompletionModel } from '../types-C54aNoCd.js';
 import { Extractor } from '../extractor/index.js';
 import '../memory/index.js';
 import '../observability/index.js';
 import '../tool-C3ciF-VG.js';
-import '../middleware-BQ7fkEEe.js';
-import '../embeddings/index.js';
+import '../middleware-CGiEIaBx.js';
+import '../types-IB2e9u5M.js';
 import '../vector-store/index.js';
 import '../zod-schema-C7F4clpm.js';
 import 'zod';
@@ -87,6 +87,7 @@ type PipelineBuilderState = {
     nextNodeIndex: number;
     nextEdgeIndex: number;
 };
+
 /** Runnable pipeline returned by `PipelineBuilder.build()`. */
 declare class Pipeline<Input, Output> implements PipelineOp<Input, Awaited<Output>> {
     private readonly executor;
@@ -102,6 +103,7 @@ declare class Pipeline<Input, Output> implements PipelineOp<Input, Awaited<Outpu
     batch<I extends Iterable<Input>>(inputs: I, options: PipelineBatchOptions): Promise<Array<Awaited<Output>>>;
     graph(): PipelineGraph;
 }
+
 /** Builds a typed pipeline from an original input type to an inferred output type. */
 declare class PipelineBuilder<Input, Output = Input> {
     private readonly executor;

package/dist/pipeline/index.js

@@ -1,4 +1,134 @@
-// src/pipeline/index.ts
+import {
+  mapWithConcurrency
+} from "../chunk-OIMLU4SF.js";
+
+// src/pipeline/graph.ts
+function initialBuilderState(metadata) {
+  return {
+    graph: initialGraph(metadata),
+    terminalNodeId: "input",
+    terminalNodeIds: ["input"],
+    nextNodeIndex: 1,
+    nextEdgeIndex: 1
+  };
+}
+function initialGraph(metadata) {
+  const id = normalizeId(metadata.id ?? "pipeline");
+  return {
+    id,
+    ...metadata.name === void 0 ? {} : { name: metadata.name },
+    ...metadata.description === void 0 ? {} : { description: metadata.description },
+    ...metadata.metadata === void 0 ? {} : { metadata: metadata.metadata },
+    nodes: [{ id: "input", kind: "input", label: "Input" }],
+    edges: []
+  };
+}
+function appendNode(state, kind, label, options = {}) {
+  const node = graphNode(kind, label, state.nextNodeIndex, {
+    ...options,
+    existingIds: new Set(state.graph.nodes.map((item) => item.id))
+  });
+  return {
+    node,
+    state: appendGraphNode(state, node, activeTerminalNodeIds(state), [node.id])
+  };
+}
+function appendChildNode(state, parentId, kind, label, options = {}) {
+  const node = graphNode(kind, label, state.nextNodeIndex, {
+    ...options,
+    existingIds: new Set(state.graph.nodes.map((item) => item.id))
+  });
+  return {
+    node,
+    state: appendGraphNode(state, node, [parentId], activeTerminalNodeIds(state))
+  };
+}
+function activeTerminalNodeIds(state) {
+  return state.terminalNodeIds.length > 0 ? state.terminalNodeIds : [state.terminalNodeId];
+}
+function withTerminalNodes(state, terminalNodeIds) {
+  return {
+    ...state,
+    terminalNodeId: terminalNodeIds.at(-1) ?? state.terminalNodeId,
+    terminalNodeIds
+  };
+}
+function withOutputNode(state) {
+  const graph = cloneGraph(state.graph);
+  if (graph.nodes.some((node) => node.id === "output")) {
+    return graph;
+  }
+  graph.nodes.push({ id: "output", kind: "output", label: "Output" });
+  graph.edges.push(
+    ...activeTerminalNodeIds(state).map((sourceId, index) => ({
+      id: `edge_${state.nextEdgeIndex + index}`,
+      source: sourceId,
+      target: "output"
+    }))
+  );
+  return graph;
+}
+function nextStageLabel(state, prefix) {
+  return `${prefix} ${state.nextNodeIndex}`;
+}
+function cloneGraph(graph) {
+  return {
+    ...graph,
+    nodes: graph.nodes.map((node) => ({ ...node })),
+    edges: graph.edges.map((edge) => ({ ...edge }))
+  };
+}
+function appendGraphNode(state, node, sourceIds, terminalNodeIds) {
+  const edges = sourceIds.map((sourceId, index) => ({
+    id: `edge_${state.nextEdgeIndex + index}`,
+    source: sourceId,
+    target: node.id
+  }));
+  const terminalNodeId = terminalNodeIds.at(-1) ?? state.terminalNodeId;
+  return {
+    graph: {
+      ...state.graph,
+      nodes: [...state.graph.nodes, node],
+      edges: [...state.graph.edges, ...edges]
+    },
+    terminalNodeId,
+    terminalNodeIds,
+    nextNodeIndex: state.nextNodeIndex + 1,
+    nextEdgeIndex: state.nextEdgeIndex + edges.length
+  };
+}
+function graphNode(kind, label, index, options = {}) {
+  const id = uniqueGraphNodeId(
+    normalizeId(options.preferredId ?? `${kind}_${index}`),
+    options.existingIds ?? /* @__PURE__ */ new Set()
+  );
+  return {
+    id,
+    kind,
+    label,
+    ...options.description === void 0 ? {} : { description: options.description },
+    ...options.metadata === void 0 ? {} : { metadata: options.metadata },
+    ...options.agentId === void 0 ? {} : { agentId: options.agentId },
+    ...options.agentName === void 0 ? {} : { agentName: options.agentName },
+    ...options.pipelineId === void 0 ? {} : { pipelineId: options.pipelineId },
+    ...options.branchKey === void 0 ? {} : { branchKey: options.branchKey }
+  };
+}
+function normalizeId(value) {
+  const normalized = value.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, "_").replace(/^_+|_+$/g, "");
+  return normalized.length === 0 ? "pipeline" : normalized;
+}
+function uniqueGraphNodeId(baseId, existingIds) {
+  let id = baseId;
+  let suffix = 2;
+  while (existingIds.has(id)) {
+    id = `${baseId}_${suffix}`;
+    suffix += 1;
+  }
+  return id;
+}
+
+// src/pipeline/pipeline.ts
 var Pipeline = class {
   constructor(executor, pipelineGraph = initialGraph({})) {
     this.executor = executor;
@@ -26,6 +156,31 @@ var Pipeline = class {
     return cloneGraph(this.pipelineGraph);
   }
 };
+
+// src/pipeline/runtime.ts
+async function runNode(context, node, fn) {
+  const startedAt = Date.now();
+  await context.observer?.onEvent({ type: "stage_started", node });
+  try {
+    const output = await fn();
+    await context.observer?.onEvent({
+      type: "stage_completed",
+      node,
+      durationMs: Date.now() - startedAt
+    });
+    return output;
+  } catch (error) {
+    await context.observer?.onEvent({
+      type: "stage_failed",
+      node,
+      durationMs: Date.now() - startedAt,
+      error
+    });
+    throw error;
+  }
+}
+
+// src/pipeline/builder.ts
 var PipelineBuilder = class _PipelineBuilder {
   executor;
   state;
@@ -175,168 +330,9 @@ var PipelineBuilder = class _PipelineBuilder {
     return await this.executor(input, context);
   }
 };
-function initialBuilderState(metadata) {
-  return {
-    graph: initialGraph(metadata),
-    terminalNodeId: "input",
-    terminalNodeIds: ["input"],
-    nextNodeIndex: 1,
-    nextEdgeIndex: 1
-  };
-}
-function initialGraph(metadata) {
-  const id = normalizeId(metadata.id ?? "pipeline");
-  return {
-    id,
-    ...metadata.name === void 0 ? {} : { name: metadata.name },
-    ...metadata.description === void 0 ? {} : { description: metadata.description },
-    ...metadata.metadata === void 0 ? {} : { metadata: metadata.metadata },
-    nodes: [{ id: "input", kind: "input", label: "Input" }],
-    edges: []
-  };
-}
-function appendNode(state, kind, label, options = {}) {
-  const node = graphNode(kind, label, state.nextNodeIndex, {
-    ...options,
-    existingIds: new Set(state.graph.nodes.map((item) => item.id))
-  });
-  return {
-    node,
-    state: appendGraphNode(state, node, activeTerminalNodeIds(state), [node.id])
-  };
-}
-function appendChildNode(state, parentId, kind, label, options = {}) {
-  const node = graphNode(kind, label, state.nextNodeIndex, {
-    ...options,
-    existingIds: new Set(state.graph.nodes.map((item) => item.id))
-  });
-  return {
-    node,
-    state: appendGraphNode(state, node, [parentId], activeTerminalNodeIds(state))
-  };
-}
-function appendGraphNode(state, node, sourceIds, terminalNodeIds) {
-  const edges = sourceIds.map((sourceId, index) => ({
-    id: `edge_${state.nextEdgeIndex + index}`,
-    source: sourceId,
-    target: node.id
-  }));
-  const terminalNodeId = terminalNodeIds.at(-1) ?? state.terminalNodeId;
-  return {
-    graph: {
-      ...state.graph,
-      nodes: [...state.graph.nodes, node],
-      edges: [...state.graph.edges, ...edges]
-    },
-    terminalNodeId,
-    terminalNodeIds,
-    nextNodeIndex: state.nextNodeIndex + 1,
-    nextEdgeIndex: state.nextEdgeIndex + edges.length
-  };
-}
-function activeTerminalNodeIds(state) {
-  return state.terminalNodeIds.length > 0 ? state.terminalNodeIds : [state.terminalNodeId];
-}
-function withTerminalNodes(state, terminalNodeIds) {
-  return {
-    ...state,
-    terminalNodeId: terminalNodeIds.at(-1) ?? state.terminalNodeId,
-    terminalNodeIds
-  };
-}
-function graphNode(kind, label, index, options = {}) {
-  const id = uniqueGraphNodeId(
-    normalizeId(options.preferredId ?? `${kind}_${index}`),
-    options.existingIds ?? /* @__PURE__ */ new Set()
-  );
-  return {
-    id,
-    kind,
-    label,
-    ...options.description === void 0 ? {} : { description: options.description },
-    ...options.metadata === void 0 ? {} : { metadata: options.metadata },
-    ...options.agentId === void 0 ? {} : { agentId: options.agentId },
-    ...options.agentName === void 0 ? {} : { agentName: options.agentName },
-    ...options.pipelineId === void 0 ? {} : { pipelineId: options.pipelineId },
-    ...options.branchKey === void 0 ? {} : { branchKey: options.branchKey }
-  };
-}
-function withOutputNode(state) {
-  const graph = cloneGraph(state.graph);
-  if (graph.nodes.some((node) => node.id === "output")) {
-    return graph;
-  }
-  graph.nodes.push({ id: "output", kind: "output", label: "Output" });
-  graph.edges.push(
-    ...activeTerminalNodeIds(state).map((sourceId, index) => ({
-      id: `edge_${state.nextEdgeIndex + index}`,
-      source: sourceId,
-      target: "output"
-    }))
-  );
-  return graph;
-}
-async function runNode(context, node, fn) {
-  const startedAt = Date.now();
-  await context.observer?.onEvent({ type: "stage_started", node });
-  try {
-    const output = await fn();
-    await context.observer?.onEvent({
-      type: "stage_completed",
-      node,
-      durationMs: Date.now() - startedAt
-    });
-    return output;
-  } catch (error) {
-    await context.observer?.onEvent({
-      type: "stage_failed",
-      node,
-      durationMs: Date.now() - startedAt,
-      error
-    });
-    throw error;
-  }
-}
-function nextStageLabel(state, prefix) {
-  return `${prefix} ${state.nextNodeIndex}`;
-}
-function cloneGraph(graph) {
-  return {
-    ...graph,
-    nodes: graph.nodes.map((node) => ({ ...node })),
-    edges: graph.edges.map((edge) => ({ ...edge }))
-  };
-}
-function normalizeId(value) {
-  const normalized = value.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, "_").replace(/^_+|_+$/g, "");
-  return normalized.length === 0 ? "pipeline" : normalized;
-}
-function uniqueGraphNodeId(baseId, existingIds) {
-  let id = baseId;
-  let suffix = 2;
-  while (existingIds.has(id)) {
-    id = `${baseId}_${suffix}`;
-    suffix += 1;
-  }
-  return id;
-}
 function identity(input) {
   return input;
 }
-async function mapWithConcurrency(inputs, concurrency, fn) {
-  const limit = Math.max(1, Math.trunc(concurrency));
-  const results = new Array(inputs.length);
-  let nextIndex = 0;
-  async function worker() {
-    while (nextIndex < inputs.length) {
-      const index = nextIndex;
-      nextIndex += 1;
-      results[index] = await fn(inputs[index]);
-    }
-  }
-  await Promise.all(Array.from({ length: Math.min(limit, inputs.length) }, () => worker()));
-  return results;
-}
 export {
   Pipeline,
   PipelineBuilder