@anvia/core 1.0.0-rc.2 → 1.0.0-rc.3

package/dist/{dynamic-tools-KwEGRbHp.d.ts → dynamic-tools-AEBOTgu1.d.ts}

@@ -1,8 +1,8 @@
 import { b as RetrySetting } from './retry-CjvSlKGW.js';
-import { v as ToolDefinition } from './types-CQ_ioXWK.js';
+import { v as ToolDefinition } from './types-D9rKQ_as.js';
 import { V as VectorMetadata, E as EmbeddingModel, a as EmbeddedDocument } from './types-Cr4uiYo5.js';
-import { A as AnyTool } from './tool--Mz4v1eL.js';
-import { V as VectorFilter, a as VectorSearchResult, b as VectorInspectRequest, c as VectorInspectPage } from './types-Dld1TpWj.js';
+import { A as AnyTool } from './tool-r62gf3QY.js';
+import { V as VectorFilter, a as VectorSearchResult, b as VectorInspectRequest, c as VectorInspectPage } from './types-C3gv9EX6.js';
 
 type ToolSearchDocument<Metadata extends VectorMetadata = VectorMetadata> = {
     toolName: string;

package/dist/embeddings/index.js

@@ -10,7 +10,8 @@ import {
   embedTexts,
   euclideanDistance,
   manhattanDistance
-} from "../chunk-FMHLROPQ.js";
+} from "../chunk-GQL5KRAH.js";
+import "../chunk-3RM57ZT2.js";
 import "../chunk-FN7HLLAZ.js";
 export {
   angularDistance,

package/dist/evals/index.d.ts

@@ -1,15 +1,15 @@
-import { U as Usage, J as JsonObject, C as CompletionModel, M as Message } from '../types-CQ_ioXWK.js';
+import { U as Usage, J as JsonObject, C as CompletionModel, M as Message } from '../types-D9rKQ_as.js';
 import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
-import { b as AgentApprovalRequiredResult, m as AgentResponse, o as AgentRunOptions, n as AgentResult } from '../types-3x3mOLHU.js';
+import { a as AgentInteractionRequest, b as AgentInteractionResponse } from '../interactions-csssHYcN.js';
+import { t as AgentSuspendedResult, j as AgentResponse, l as AgentRunOptions, k as AgentResult } from '../types-CElwurHZ.js';
 import { E as EmbeddingModel } from '../types-Cr4uiYo5.js';
 import '../model-call-options-CZkSw_xN.js';
 import 'zod';
-import '../retry-CjvSlKGW.js';
-import '../guardrails/index.js';
 import '../type-utils-CtHVDRn_.js';
-import '../types-BVzM4RsC.js';
-import '../tool--Mz4v1eL.js';
-import '../middleware-C7PdmKF7.js';
+import '../retry-CjvSlKGW.js';
+import '../types-CIPaIbZo.js';
+import '../types-DbolvLtZ.js';
+import '../tool-r62gf3QY.js';
 
 type EvalOutcome<Score = unknown> = {
     outcome: "pass";
@@ -394,6 +394,14 @@ type AgentEvalTargetOptions<Input, AgentOutput = string, Output = AgentResponse<
         input: Input;
         testCase: EvalCase<Input, Expected>;
     }): AgentRunOptions<AgentOutput> | Promise<AgentRunOptions<AgentOutput>>;
+    interactions?: {
+        maxResponses?: number | undefined;
+        respond(args: {
+            interaction: AgentInteractionRequest;
+            testCase: EvalCase<Input, Expected>;
+            phase: number;
+        }): AgentInteractionResponse | Promise<AgentInteractionResponse>;
+    } | undefined;
 } & (IsExactly<Output, AgentResponse<AgentOutput>> extends true ? {
     output?(args: {
         response: AgentResponse<AgentOutput>;
@@ -405,9 +413,9 @@ type AgentEvalTargetOptions<Input, AgentOutput = string, Output = AgentResponse<
         testCase: EvalCase<Input, Expected>;
     }): Output | Promise<Output>;
 });
-declare class AgentEvalApprovalError extends Error {
-    readonly result: AgentApprovalRequiredResult;
-    constructor(result: AgentApprovalRequiredResult);
+declare class AgentEvalSuspensionError extends Error {
+    readonly result: AgentSuspendedResult;
+    constructor(result: AgentSuspendedResult, message?: string);
 }
 declare function agentEvalTarget<Input, AgentOutput = string, Output = AgentResponse<AgentOutput>, Expected = unknown>(options: AgentEvalTargetOptions<Input, AgentOutput, Output, Expected>): EvalTarget<Input, Output, Expected>;
 
@@ -660,4 +668,4 @@ declare function defineEvalSuite<const Cases extends readonly EvalCaseLike[], co
     target: Target;
 };
 
-export { type AbstentionCategory, type AbstentionOptions, AgentEvalApprovalError, type AgentEvalTargetOptions, type AnswerRelevancyOptions, type AnyEvalMetric, type ContainsAllOptions, type ContainsAnyOptions, type ContainsListOptions, type ContainsOptions, type DefaultEvalActual, type DefinedEvalSuite, type DoesNotMatchOptions, EvalAssertionError, type EvalCase, type EvalCaseRequirements, type EvalCaseResult, type EvalCasesExpected, type EvalCasesForMetrics, type EvalCasesInput, type EvalCostCalculatorArgs, type EvalCostOptions, type EvalCostSummary, type EvalDataType, type EvalExpectations, type EvalExpectedOutcomes, type EvalExpectedTotals, type EvalMetadata, type EvalMetric, type EvalMetricArgs, type EvalMetricDescriptor, type EvalMetricResult, type EvalMetricResultFor, type EvalMetricScore, EvalOutcome, type EvalOutcomeStatus, type EvalOutputFormat, type EvalOutputWriters, type EvalReportArgs, type EvalReporter, EvalReporterDispatchError, type EvalReporterErrorPolicy, type EvalRunContext, type EvalRunEndArgs, type EvalRunOptions, type EvalRunStartArgs, type EvalScoreDirection, type EvalScoreMap, type EvalScoreProjection, type EvalSuiteResult, type EvalTarget, type EvalTargetUsageSelector, type EvalTotals, type EvalTraceCarrier, type EvalTraceRef, type EvalTraceSelector, type EvalTraceSelectorArgs, type EvalTurn, type EvalUsageSummary, type ExactMatchOptions, type FaithfulnessOptions, type GEvalOptions, type GEvalParameter, type GEvalRubric, type HallucinationOptions, type JsonCorrectnessOptions, type KnowledgeRetentionOptions, type LlmJudgeOptions, type LlmScoreMetricScore, type LlmScoreOptions, type MatchesOptions, type MaxLengthOptions, type NotContainsOptions, type PrintEvalResultOptions, type PromptAlignmentOptions, type RequiredFieldsOptions, type RunEvalCliOptions, type RunEvalSuiteOptions, type SelectorOrValue, type SemanticSimilarityOptions, type SummarizationOptions, type TurnRelevancyOptions, type ValueSelector, abstention, agentEvalTarget, answerRelevancy, assertEvalOutcomes, assertEvalTotals, contains, containsAll, containsAny, defaultEvalTraceSelector, defineEvalCases, defineEvalSuite, defineMetric, doesNotMatch, evalExitCode, exactMatch, faithfulness, gEval, hallucination, jsonCorrectness, knowledgeRetention, llmJudge, llmScore, matches, maxLength, notContains, printEvalResult, projectEvalOutcome, promptAlignment, requiredFields, resolveEvalTraceRef, runEvalCli, runEvalSuite, selectPromptOutput, semanticSimilarity, summarization, turnRelevancy };
+export { type AbstentionCategory, type AbstentionOptions, AgentEvalSuspensionError, type AgentEvalTargetOptions, type AnswerRelevancyOptions, type AnyEvalMetric, type ContainsAllOptions, type ContainsAnyOptions, type ContainsListOptions, type ContainsOptions, type DefaultEvalActual, type DefinedEvalSuite, type DoesNotMatchOptions, EvalAssertionError, type EvalCase, type EvalCaseRequirements, type EvalCaseResult, type EvalCasesExpected, type EvalCasesForMetrics, type EvalCasesInput, type EvalCostCalculatorArgs, type EvalCostOptions, type EvalCostSummary, type EvalDataType, type EvalExpectations, type EvalExpectedOutcomes, type EvalExpectedTotals, type EvalMetadata, type EvalMetric, type EvalMetricArgs, type EvalMetricDescriptor, type EvalMetricResult, type EvalMetricResultFor, type EvalMetricScore, EvalOutcome, type EvalOutcomeStatus, type EvalOutputFormat, type EvalOutputWriters, type EvalReportArgs, type EvalReporter, EvalReporterDispatchError, type EvalReporterErrorPolicy, type EvalRunContext, type EvalRunEndArgs, type EvalRunOptions, type EvalRunStartArgs, type EvalScoreDirection, type EvalScoreMap, type EvalScoreProjection, type EvalSuiteResult, type EvalTarget, type EvalTargetUsageSelector, type EvalTotals, type EvalTraceCarrier, type EvalTraceRef, type EvalTraceSelector, type EvalTraceSelectorArgs, type EvalTurn, type EvalUsageSummary, type ExactMatchOptions, type FaithfulnessOptions, type GEvalOptions, type GEvalParameter, type GEvalRubric, type HallucinationOptions, type JsonCorrectnessOptions, type KnowledgeRetentionOptions, type LlmJudgeOptions, type LlmScoreMetricScore, type LlmScoreOptions, type MatchesOptions, type MaxLengthOptions, type NotContainsOptions, type PrintEvalResultOptions, type PromptAlignmentOptions, type RequiredFieldsOptions, type RunEvalCliOptions, type RunEvalSuiteOptions, type SelectorOrValue, type SemanticSimilarityOptions, type SummarizationOptions, type TurnRelevancyOptions, type ValueSelector, abstention, agentEvalTarget, answerRelevancy, assertEvalOutcomes, assertEvalTotals, contains, containsAll, containsAny, defaultEvalTraceSelector, defineEvalCases, defineEvalSuite, defineMetric, doesNotMatch, evalExitCode, exactMatch, faithfulness, gEval, hallucination, jsonCorrectness, knowledgeRetention, llmJudge, llmScore, matches, maxLength, notContains, printEvalResult, projectEvalOutcome, promptAlignment, requiredFields, resolveEvalTraceRef, runEvalCli, runEvalSuite, selectPromptOutput, semanticSimilarity, summarization, turnRelevancy };

package/dist/evals/index.js

@@ -1,28 +1,21 @@
 import {
-  AgentRunBlockedError,
-  cancelAgentApproval
-} from "../chunk-DBSCKTUL.js";
-import "../chunk-YK4WAAS4.js";
-import "../chunk-EINTGEGY.js";
-import "../chunk-CQNNSZPG.js";
-import "../chunk-D37WMSP6.js";
-import "../chunk-77AIYARB.js";
-import "../chunk-OJH6HWHX.js";
-import "../chunk-CWM7B2OO.js";
+  AgentRunBlockedError
+} from "../chunk-TRQ3XRFD.js";
 import {
   cosineSimilarity,
-  embedText,
+  embedText
+} from "../chunk-GQL5KRAH.js";
+import {
   mapWithConcurrency
-} from "../chunk-FMHLROPQ.js";
+} from "../chunk-3RM57ZT2.js";
 import {
   extract
-} from "../chunk-4WI3VGZK.js";
+} from "../chunk-ZCPOIDAQ.js";
 import "../chunk-MGNTXGTX.js";
 import {
   Usage
-} from "../chunk-KR3RCCHJ.js";
-import "../chunk-TIZYTPJO.js";
-import "../chunk-KIDGIJM7.js";
+} from "../chunk-D2ECOWN5.js";
+import "../chunk-PK5LKOFL.js";
 import "../chunk-FN7HLLAZ.js";
 
 // src/evals/advanced-metrics.ts
@@ -1122,26 +1115,61 @@ function contentText(content) {
 }
 
 // src/evals/agent-target.ts
-var AgentEvalApprovalError = class extends Error {
-  constructor(result) {
-    super("Agent eval targets cannot suspend for tool approval.");
+var AgentEvalSuspensionError = class extends Error {
+  constructor(result, message = "Agent eval target suspended without an interaction responder.") {
+    super(message);
     this.result = result;
-    this.name = "AgentEvalApprovalError";
+    this.name = "AgentEvalSuspensionError";
   }
   result;
 };
 function agentEvalTarget(options) {
   return async (input, testCase) => {
+    const maxResponses = options.interactions?.maxResponses ?? 10;
+    if (!Number.isSafeInteger(maxResponses) || maxResponses < 1) {
+      throw new TypeError("Agent eval interactions.maxResponses must be a positive integer.");
+    }
     const request = await options.request({ input, testCase });
-    const response = await options.agent.generate(request);
-    if (response.status === "approval_required") {
-      await cancelAgentApproval(response, "Agent eval targets cannot suspend for tool approval.");
-      throw new AgentEvalApprovalError(response);
+    const runSettings = agentRunSettings(request);
+    let response = await options.agent.generate(request);
+    let phase = 0;
+    while (response.status === "suspended") {
+      if (options.interactions === void 0) {
+        throw new AgentEvalSuspensionError(response);
+      }
+      if (phase >= maxResponses) {
+        throw new AgentEvalSuspensionError(
+          response,
+          `Agent eval target exceeded the interaction response limit of ${maxResponses}.`
+        );
+      }
+      phase += 1;
+      const interactionResponse = await options.interactions.respond({
+        interaction: response.interaction,
+        testCase,
+        phase
+      });
+      response = await options.agent.generate({
+        continuation: response.continuation,
+        response: interactionResponse,
+        ...runSettings
+      });
     }
     if (response.status === "blocked") throw new AgentRunBlockedError(response);
     return options.output === void 0 ? response : await options.output({ response, testCase });
   };
 }
+function agentRunSettings(request) {
+  const {
+    prompt: _prompt,
+    messages: _messages,
+    session: _session,
+    continuation: _continuation,
+    response: _response,
+    ...settings
+  } = request;
+  return settings;
+}
 
 // src/evals/reporting.ts
 function projectEvalOutcome(outcome, dataType, projectScore) {
@@ -1286,25 +1314,26 @@ async function runEvalSuite(options) {
     metrics: aggregates.metrics,
     cases: aggregates.cases,
     usage: aggregates.usage,
-    ...aggregates.cost === void 0 ? {} : { cost: aggregates.cost },
     durationMs: Date.now() - startedAtMs,
     reporterErrors
   };
+  if (aggregates.cost !== void 0) {
+    result.cost = aggregates.cost;
+  }
+  const runEndArgs = {
+    ...lifecycle,
+    status: "completed",
+    completedAt,
+    durationMs: result.durationMs,
+    metrics: result.metrics,
+    cases: result.cases,
+    usage: result.usage
+  };
+  if (result.cost !== void 0) {
+    runEndArgs.cost = result.cost;
+  }
   result.reporterErrors.push(
-    ...await notifyRunEnd(
-      reporters,
-      {
-        ...lifecycle,
-        status: "completed",
-        completedAt,
-        durationMs: result.durationMs,
-        metrics: result.metrics,
-        cases: result.cases,
-        usage: result.usage,
-        ...result.cost === void 0 ? {} : { cost: result.cost }
-      },
-      options.reporterErrorPolicy ?? "collect"
-    )
+    ...await notifyRunEnd(reporters, runEndArgs, options.reporterErrorPolicy ?? "collect")
   );
   return result;
 }
@@ -1444,13 +1473,14 @@ function resolveRun(options, startedAtMs) {
       throw new TypeError(`Evaluation run ${label} must contain 1 to 256 characters`);
     }
   }
-  return {
+  const run = {
     id,
-    startedAt: new Date(startedAtMs).toISOString(),
-    ...options.run?.datasetName === void 0 ? {} : { datasetName: options.run.datasetName },
-    ...options.run?.datasetVersion === void 0 ? {} : { datasetVersion: options.run.datasetVersion },
-    ...options.run?.metadata === void 0 ? {} : { metadata: options.run.metadata }
+    startedAt: new Date(startedAtMs).toISOString()
   };
+  if (options.run?.datasetName !== void 0) run.datasetName = options.run.datasetName;
+  if (options.run?.datasetVersion !== void 0) run.datasetVersion = options.run.datasetVersion;
+  if (options.run?.metadata !== void 0) run.metadata = options.run.metadata;
+  return run;
 }
 async function notifyRunStart(reporters, args, errorPolicy) {
   const errors = [];
@@ -1737,18 +1767,21 @@ function jsonResult(result) {
 }
 function expectationMismatches(result, expectations) {
   if (expectations === void 0) return [];
-  return [
-    ...expectations.totals === void 0 ? [] : totalMismatches(result, expectations.totals),
-    ...expectations.outcomes === void 0 ? [] : outcomeMismatches(result, expectations.outcomes)
-  ];
+  const mismatches = [];
+  if (expectations.totals !== void 0) {
+    mismatches.push(...totalMismatches(result, expectations.totals));
+  }
+  if (expectations.outcomes !== void 0) {
+    mismatches.push(...outcomeMismatches(result, expectations.outcomes));
+  }
+  return mismatches;
 }
 function totalMismatches(result, expected) {
-  const directMetrics = {
-    ...expected.total === void 0 ? {} : { total: expected.total },
-    ...expected.passed === void 0 ? {} : { passed: expected.passed },
-    ...expected.failed === void 0 ? {} : { failed: expected.failed },
-    ...expected.invalid === void 0 ? {} : { invalid: expected.invalid }
-  };
+  const directMetrics = {};
+  if (expected.total !== void 0) directMetrics.total = expected.total;
+  if (expected.passed !== void 0) directMetrics.passed = expected.passed;
+  if (expected.failed !== void 0) directMetrics.failed = expected.failed;
+  if (expected.invalid !== void 0) directMetrics.invalid = expected.invalid;
   return [
     ...totalsGroupMismatches("metrics", result.metrics, {
       ...directMetrics,
@@ -2115,7 +2148,7 @@ function defineEvalSuite(options) {
   };
 }
 export {
-  AgentEvalApprovalError,
+  AgentEvalSuspensionError,
   EvalAssertionError,
   EvalOutcome,
   EvalReporterDispatchError,