@adcp/sdk 13.0.0-rc.10 → 13.0.0-rc.13

package/dist/lib/testing/storyboard/runner.mjs

@@ -17,7 +17,10 @@ import { detectContextRejectionHints } from "./rejection-hints.mjs";
 import { detectShapeDriftHints } from "./shape-drift-hints.mjs";
 import { detectStrictValidationHints } from "./strict-validation-hints.mjs";
 import {
-  runValidations
+  runValidations,
+  decorateValidationResult,
+  validationFailsStep,
+  RUNNER_CAPABILITY_VERSION
 } from "./validations.mjs";
 import { PARALLEL_DISPATCH_CONTRACT, runParallelDispatches, validateParallelDispatchSpec } from "./parallel-dispatch.mjs";
 import { resolvePath, resolvePortableIdentifierPathAll, toJsonPointer, validatePortableIdentifierPath } from "./path.mjs";
@@ -703,6 +706,7 @@ function buildCapabilityUnsupportedResult(agentUrls, storyboard, detail) {
     passed_count: 0,
     failed_count: 0,
     skipped_count: 1,
+    runner_capability_version: RUNNER_CAPABILITY_VERSION,
     tested_at: (/* @__PURE__ */ new Date()).toISOString(),
     strict_validation_summary: {
       observable: false,
@@ -785,6 +789,7 @@ function buildRequirementUnmetResult(agentUrls, storyboard, requirement, detail)
     passed_count: 0,
     failed_count: 0,
     skipped_count: 1,
+    runner_capability_version: RUNNER_CAPABILITY_VERSION,
     tested_at: (/* @__PURE__ */ new Date()).toISOString(),
     strict_validation_summary: {
       observable: false,
@@ -832,6 +837,7 @@ function buildRequiredAnyOfToolsMissingResult(agentUrls, storyboard, detail) {
     passed_count: 0,
     failed_count: 0,
     skipped_count: 1,
+    runner_capability_version: RUNNER_CAPABILITY_VERSION,
     tested_at: (/* @__PURE__ */ new Date()).toISOString(),
     strict_validation_summary: {
       observable: false,
@@ -1040,6 +1046,7 @@ function buildDiscoveryFailedResult(agentUrls, storyboard, discoveryStep) {
     passed_count: 0,
     failed_count: 1,
     skipped_count: 0,
+    runner_capability_version: RUNNER_CAPABILITY_VERSION,
     tested_at: (/* @__PURE__ */ new Date()).toISOString(),
     strict_validation_summary: {
       observable: false,
@@ -1088,6 +1095,7 @@ function buildRequiredToolsMissingResult(agentUrls, storyboard, detail) {
     passed_count: 0,
     failed_count: 0,
     skipped_count: 1,
+    runner_capability_version: RUNNER_CAPABILITY_VERSION,
     tested_at: (/* @__PURE__ */ new Date()).toISOString(),
     strict_validation_summary: {
       observable: false,
@@ -1809,7 +1817,9 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
       }
       stepResults.push(result2);
       priorStepResults.set(step.id, result2);
-      const schemaInvalidResponse = result2.validations.some((v) => v.check === "response_schema" && !v.passed);
+      const schemaInvalidResponse = result2.validations.some(
+        (v) => v.check === "response_schema" && validationFailsStep(v)
+      );
       for (const spec of assertions) {
         if (!spec.onStep) continue;
         if (stepDisablesAssertion(step.invariants, spec.id)) continue;
@@ -2039,6 +2049,7 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
   const schemasUsed = collectSchemasUsed(phaseResults);
   const strictSummary = summarizeStrictValidation(phaseResults);
   const validationsNotApplicable = countValidationsNotApplicable(phaseResults);
+  const validationsAdvisoryFailed = countValidationsAdvisoryFailed(phaseResults);
   const notices = mergeRunnerNotices([
     ...collectCapabilityNotices(storyboard, profile ?? options._profile),
     ...collectStepNotices(phaseResults)
@@ -2064,7 +2075,9 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
     passed_count: passedCount,
     failed_count: failedCount,
     skipped_count: skippedCount,
+    ...validationsAdvisoryFailed > 0 ? { validations_advisory_failed: validationsAdvisoryFailed } : {},
     ...validationsNotApplicable > 0 ? { validations_not_applicable: validationsNotApplicable } : {},
+    runner_capability_version: RUNNER_CAPABILITY_VERSION,
     tested_at: (/* @__PURE__ */ new Date()).toISOString(),
     ...schemasUsed.length > 0 ? { schemas_used: schemasUsed } : {},
     ...assertionResults.length > 0 ? { assertions: assertionResults } : {},
@@ -2146,6 +2159,7 @@ async function runMultiPass(agentUrls, storyboard, options) {
       passed_count: result.passed_count,
       failed_count: result.failed_count,
       skipped_count: result.skipped_count,
+      ...result.validations_advisory_failed ? { validations_advisory_failed: result.validations_advisory_failed } : {},
       ...result.validations_not_applicable ? { validations_not_applicable: result.validations_not_applicable } : {},
       duration_ms: result.total_duration_ms
     });
@@ -2155,6 +2169,7 @@ async function runMultiPass(agentUrls, storyboard, options) {
   const passed = passes.reduce((sum, p) => sum + p.passed_count, 0);
   const failed = passes.reduce((sum, p) => sum + p.failed_count, 0);
   const skipped = passes.reduce((sum, p) => sum + p.skipped_count, 0);
+  const advisoryFailed = passes.reduce((sum, p) => sum + (p.validations_advisory_failed ?? 0), 0);
   const notApplicable = passes.reduce((sum, p) => sum + (p.validations_not_applicable ?? 0), 0);
   const schemasUsed = passResults.flatMap((r) => r.schemas_used ?? []);
   const schemasDedup = [...new Map(schemasUsed.map((s) => [s.schema_id, s])).values()];
@@ -2174,7 +2189,9 @@ async function runMultiPass(agentUrls, storyboard, options) {
     passed_count: passed,
     failed_count: failed,
     skipped_count: skipped,
+    ...advisoryFailed > 0 ? { validations_advisory_failed: advisoryFailed } : {},
     ...notApplicable > 0 ? { validations_not_applicable: notApplicable } : {},
+    runner_capability_version: RUNNER_CAPABILITY_VERSION,
     tested_at: (/* @__PURE__ */ new Date()).toISOString(),
     ...schemasDedup.length > 0 ? { schemas_used: schemasDedup } : {},
     ...assertionsAgg.length > 0 ? { assertions: assertionsAgg } : {},
@@ -2194,6 +2211,17 @@ function countValidationsNotApplicable(phases) {
   }
   return n;
 }
+function countValidationsAdvisoryFailed(phases) {
+  let n = 0;
+  for (const phase of phases) {
+    for (const step of phase.steps) {
+      for (const validation of step.validations) {
+        if (!validation.passed && validation.severity === "advisory") n++;
+      }
+    }
+  }
+  return n;
+}
 function collectSchemasUsed(phases) {
   const seen = /* @__PURE__ */ new Set();
   const out = [];
@@ -2898,8 +2926,16 @@ async function executeStep(client, step, storyboardId, phaseId, context, allStep
   } else {
     passed = stepResult.passed && (taskResult?.success ?? false);
   }
+  const schemaValidationError = caughtError instanceof ResponseSchemaValidationError ? caughtError : void 0;
+  const validationTaskResult = taskResult ?? (schemaValidationError ? {
+    // ResponseSchemaValidationError is thrown while validating a
+    // successful response arm. Preserve that terminal state so
+    // status_code/error_code checks retain their authored semantics.
+    success: true,
+    data: schemaValidationError.data
+  } : void 0);
   let validations = [];
-  if (step.validations?.length && (taskResult || httpResult)) {
+  if (step.validations?.length && (validationTaskResult || httpResult)) {
     const resolvedValidations = step.validations.map((v) => {
       const resolved = { ...v };
       if (resolved.value !== void 0) {
@@ -2926,7 +2962,7 @@ async function executeStep(client, step, storyboardId, phaseId, context, allStep
       taskName: effectiveStep.task,
       ...options.adcpVersion && { adcpVersion: options.adcpVersion },
       ...options._serverAdcpVersion && { responseAdcpVersion: options._serverAdcpVersion },
-      ...taskResult && { taskResult },
+      ...validationTaskResult && { taskResult: validationTaskResult },
       ...httpResult && { httpResult },
       agentUrl: runState.agentUrl,
       contributions: runState.contributions,
@@ -2983,20 +3019,38 @@ async function executeStep(client, step, storyboardId, phaseId, context, allStep
       }
     }
   }
-  if (caughtError instanceof ResponseSchemaValidationError) {
-    const issues = caughtError.issues.slice(0, 5).map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`).join("; ");
-    const firstIssue = caughtError.issues[0];
+  let schemaRejectionIsAdvisory = false;
+  if (schemaValidationError) {
+    const issues = schemaValidationError.issues.slice(0, 5).map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`).join("; ");
+    const firstIssue = schemaValidationError.issues[0];
     const jsonPointer = firstIssue ? "/" + firstIssue.path.map((s) => String(s)).join("/") : null;
-    const synthSchemaResult = {
+    const baseSchemaResult = {
       check: "response_schema",
       passed: false,
-      description: `Response schema validation for ${caughtError.toolName}`,
+      description: `Response schema validation for ${schemaValidationError.toolName}`,
       error: issues,
       json_pointer: jsonPointer,
-      expected: `response schema for ${caughtError.toolName}`,
-      actual: caughtError.issues
+      expected: `response schema for ${schemaValidationError.toolName}`,
+      actual: schemaValidationError.issues
     };
-    validations = [synthSchemaResult, ...validations];
+    const authoredSchemaValidations = (step.validations ?? []).filter((v) => v.check === "response_schema");
+    const schemaValidationContext = {
+      taskName: effectiveStep.task,
+      ...options.adcpVersion && { adcpVersion: options.adcpVersion },
+      ...options._serverAdcpVersion && { responseAdcpVersion: options._serverAdcpVersion },
+      ...validationTaskResult && { taskResult: validationTaskResult },
+      agentUrl: runState.agentUrl,
+      contributions: runState.contributions,
+      request: requestRecord,
+      ...responseRecord && { response: responseRecord }
+    };
+    const decoratedCandidates = authoredSchemaValidations.map(
+      (validation) => decorateValidationResult(baseSchemaResult, schemaValidationContext, validation)
+    );
+    const schemaResults = decoratedCandidates.length ? decoratedCandidates : [{ ...baseSchemaResult, severity: "required" }];
+    schemaRejectionIsAdvisory = schemaResults.every((result) => !validationFailsStep(result));
+    if (schemaRejectionIsAdvisory && !step.expect_error) passed = true;
+    validations = [...schemaResults, ...validations.filter((result) => result.check !== "response_schema")];
   }
   if (a2aEnvelope && runState.priorA2aEnvelopes) {
     runState.priorA2aEnvelopes.set(step.id, a2aEnvelope);
@@ -3081,7 +3135,7 @@ async function executeStep(client, step, storyboardId, phaseId, context, allStep
       }
     }
   }
-  const allValidationsPassedFinal = validations.every((v) => v.passed);
+  const allValidationsPassedFinal = validations.every((v) => !validationFailsStep(v));
   const stepFailed = !(passed && allValidationsPassedFinal);
   const contextRejectionHints = stepFailed && runState.contextProvenance ? detectContextRejectionHints(taskResult, request, context, runState.contextProvenance, effectiveStep.task) : [];
   const driftPayload = (() => {
@@ -3116,7 +3170,7 @@ async function executeStep(client, step, storyboardId, phaseId, context, allStep
       context_provenance: Object.fromEntries(runState.contextProvenance)
     },
     ...responseDerivedContextResult(runState),
-    error: step.expect_error ? void 0 : truncateError(stepResult.error || taskResult?.error),
+    error: step.expect_error || schemaRejectionIsAdvisory ? void 0 : truncateError(stepResult.error || taskResult?.error),
     ...!step.expect_error && taskResult?.adcp_error && { adcp_error: taskResult.adcp_error },
     next,
     request: requestRecord,
@@ -3422,7 +3476,7 @@ async function executeProbeStep(client, step, phaseId, context, allSteps, option
     ...oauthMetadataGraph && { oauthMetadataGraph }
   };
   const validations = step.validations?.length ? runValidations(step.validations, vctx) : [];
-  const allValidationsPassed = validations.every((v) => v.passed);
+  const allValidationsPassed = validations.every((v) => !validationFailsStep(v));
   const fetchOk = httpResult ? !httpResult.error : true;
   const passed = fetchOk && allValidationsPassed;
   const extraction = httpResult ? httpResult.error ? { path: "error" } : { path: "structured_content", note: "http-probe body parsed as JSON" } : { path: "none" };
@@ -3630,7 +3684,7 @@ async function executeReplayWebhookVectorStep(step, phaseId, context, allSteps,
     body: redactSecrets(httpResult.body)
   };
   const validations = buildWebhookReplayValidations(step, vector, allSteps, options, requestRecord, responseRecord);
-  const allValidationsPassed = validations.every((v) => v.passed);
+  const allValidationsPassed = validations.every((v) => !validationFailsStep(v));
   const fetchOk = !httpResult.error;
   const passed = fetchOk && allValidationsPassed;
   return {