@adcp/sdk 13.0.0-rc.7 → 13.0.0-rc.8

package/dist/lib/testing/storyboard/runner.mjs

@@ -28,6 +28,11 @@ import { injectLegacyEnvelopeStatus, normalizeLegacyMediaBuyStatusForReturn } fr
 import { queryUpstreamTraffic } from "../test-controller.mjs";
 import { IDENTIFIER_DIGEST_LIMIT } from "../../upstream-recorder/constants.mjs";
 import { enrichRequest, hasRequestEnricher } from "./request-builder.mjs";
+import {
+  BUILD_ASSETS_FROM_FORMAT_DIRECTIVE,
+  expandCreativeAssetDirectives,
+  findUnresolvedCreativeAssetDirectives
+} from "./creative-assets.mjs";
 import { resolveAccount, resolveBrand } from "../client.mjs";
 import { isMutatingTask, generateIdempotencyKey } from "../../utils/idempotency.mjs";
 import {
@@ -37,7 +42,7 @@ import {
   schemaAllowsTopLevelField,
   withExternalSchemaRoot
 } from "../../validation/schema-loader.mjs";
-import { parseAdcpMajorVersion } from "../../version.mjs";
+import { ADCP_VERSION, parseAdcpMajorVersion } from "../../version.mjs";
 import {
   PROBE_TASKS,
   probeProtectedResourceMetadata,
@@ -54,6 +59,7 @@ import { probeRequestSigningVector } from "./request-signing/probe-dispatch.mjs"
 import { createWebhookReceiver } from "./webhook-receiver.mjs";
 import { WEBHOOK_ASSERTION_TASKS, armWebhookAssertions, executeWebhookAssertionStep } from "./webhook-assertions.mjs";
 import { runControllerSeeding } from "./seeding.mjs";
+import { applyFixtureBindingsToRequest } from "./fixture-resolution.mjs";
 import { getComplianceCacheDir } from "./compliance.mjs";
 import { signWebhook } from "../../signing/client.mjs";
 import {
@@ -1361,15 +1367,64 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
   let seedingFailed = false;
   let seedingMissingController = false;
   let seedingUnsupported = false;
+  let fixtureUnsatisfied = false;
+  let fixtureBindings;
+  let fixtureResolutionRecords;
+  let fixtureCoverageGap;
   {
-    const seeding = skipControllerSeedingForPhaseGates ? null : preSeeded !== void 0 ? preSeeded.result : await runControllerSeeding(clients[0], storyboard, options, context);
+    let fixtureSeedClient = clients[0];
+    let fixtureDiscoveryClient = clients[0];
+    if (routingContext && options.agents) {
+      const fixtureStep = (task) => ({
+        id: `__fixture_resolution_${task}__`,
+        title: `Fixture resolution via ${task}`,
+        task
+      });
+      if (options.agentTools?.includes("comply_test_controller")) {
+        fixtureSeedClient = dispatch.nextFor(fixtureStep("comply_test_controller")).client;
+      }
+      if (options.agentTools?.includes("get_products")) {
+        fixtureDiscoveryClient = dispatch.nextFor(fixtureStep("get_products")).client;
+      }
+    }
+    const seeding = skipControllerSeedingForPhaseGates ? null : preSeeded !== void 0 ? preSeeded.result : await runControllerSeeding(fixtureSeedClient, storyboard, options, context, fixtureDiscoveryClient);
     if (seeding) {
       const attach = preSeeded === void 0 || preSeeded.attach;
       if (attach) {
         seedingPhaseResult = seeding.phase;
         passedCount += seeding.passedCount;
         failedCount += seeding.failedCount;
-        if (seeding.missingController || seeding.seedUnsupported) skippedCount += seeding.phase.steps.length;
+        if (seeding.missingController || seeding.seedUnsupported || seeding.fixtureUnsatisfied) {
+          skippedCount += seeding.phase.steps.filter((step) => step.skipped).length;
+        }
+      }
+      fixtureBindings = seeding.bindings;
+      fixtureResolutionRecords = seeding.resolutionRecords;
+      fixtureUnsatisfied = seeding.fixtureUnsatisfied === true && seeding.failedCount === 0;
+      if (fixtureUnsatisfied) {
+        const noPhasesIndex = phaseResults.findIndex((phase) => phase.phase_id === "no_phases");
+        if (noPhasesIndex >= 0) {
+          skippedCount -= phaseResults[noPhasesIndex].steps.filter((step) => step.skipped).length;
+          phaseResults.splice(noPhasesIndex, 1);
+        }
+        if (attach) skippedCount += 1;
+      }
+      const hasUnsatisfiedFixtures = fixtureResolutionRecords?.some((record) => record.status === "unsatisfied") === true;
+      if (hasUnsatisfiedFixtures && fixtureResolutionRecords) {
+        const fixtures = fixtureResolutionRecords.filter((record) => record.status === "unsatisfied").map((record) => ({
+          fixture_type: record.fixture_type,
+          handle: record.handle,
+          ...record.product_handle && { product_handle: record.product_handle },
+          requirements: record.requirements
+        }));
+        const names = fixtures.map(
+          (fixture) => fixture.fixture_type === "pricing_option" ? `pricing_option "${fixture.product_handle}/${fixture.handle}"` : `product "${fixture.handle}"`
+        );
+        fixtureCoverageGap = {
+          reason: "fixture_unsatisfied",
+          detail: `fixture_unsatisfied: no seller fixture satisfied ${names.join(", ")}`,
+          fixtures
+        };
       }
       if (seeding.missingController) {
         seedingMissingController = true;
@@ -1383,6 +1438,16 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
   for (const phase of storyboard.phases) {
     options.signal?.throwIfAborted();
     const phaseStart = Date.now();
+    if (fixtureUnsatisfied) {
+      phaseResults.push({
+        phase_id: phase.id,
+        phase_title: phase.title,
+        passed: true,
+        steps: [],
+        duration_ms: 0
+      });
+      continue;
+    }
     const phaseCapabilitySkipDetail = phaseCapabilitySkipDetails.get(phase.id);
     if (phaseCapabilitySkipDetail !== void 0) {
       const skippedSteps = buildPhaseCapabilitySkippedSteps(storyboard, phase, phaseCapabilitySkipDetail, context);
@@ -1441,6 +1506,9 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
       } : seedingUnsupported ? {
         skip_reason: "fixture_seed_unsupported",
         skip: { reason: "not_applicable", detail: FIXTURE_SEED_UNSUPPORTED_DETAIL }
+      } : seedingFailed ? {
+        skip_reason: "controller_seeding_failed",
+        skip: { reason: "prerequisite_failed", detail: CONTROLLER_SEEDING_FAILED_DETAIL }
       } : {
         skip_reason: "controller_seeding_failed",
         skip: { reason: "prerequisite_failed", detail: CONTROLLER_SEEDING_FAILED_DETAIL }
@@ -1584,7 +1652,8 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
           responseDerivedNotApplicableContextKeys,
           agentProfile: assignment.profile,
           agentLibraryVersion: assignment.profile?.library_version,
-          storyboardRequiresRequestSigner: allRequires.includes("request_signer")
+          storyboardRequiresRequestSigner: allRequires.includes("request_signer"),
+          fixtureBindings
         }
       );
       const result2 = { ...rawResult, storyboard_id: storyboard.id };
@@ -1780,7 +1849,7 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
     return !!phaseResult && phaseResult.passed && phaseResult.steps.some((s) => !s.skipped && s.passed);
   });
   const requiredPhasesPassed = !hasExecutableSteps || requiredPhaseHasExecutedPass || failedCount === 0 && requiredPhasesCoveredByCapabilityGates;
-  const storyboardWideFixtureSeedUnsupported = seedingUnsupported && failedCount === 0 && phaseResults.some((p) => p.steps.some((s) => s.skipped && s.skip?.reason === "not_applicable")) && phaseResults.every((p) => p.steps.every((s) => s.skipped && s.skip?.reason === "not_applicable"));
+  const storyboardWideFixtureUnavailable = (seedingUnsupported || fixtureUnsatisfied) && failedCount === 0;
   if (seedingPhaseResult) phaseResults.unshift(seedingPhaseResult);
   const schemasUsed = collectSchemasUsed(phaseResults);
   const strictSummary = summarizeStrictValidation(phaseResults);
@@ -1799,7 +1868,7 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
     // `multi-pass`.
     ...isMultiInstance && { multi_instance_strategy: "round-robin" },
     ...routingContext && { agent_map: { ...routingContext.agentMap } },
-    overall_passed: failedCount === 0 && (requiredPhasesPassed || storyboardWideFixtureSeedUnsupported) && !assertionsFailed,
+    overall_passed: failedCount === 0 && (requiredPhasesPassed || storyboardWideFixtureUnavailable) && !assertionsFailed,
     phases: phaseResults,
     context,
     total_duration_ms: Date.now() - start,
@@ -1812,6 +1881,8 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
     ...assertionResults.length > 0 ? { assertions: assertionResults } : {},
     strict_validation_summary: strictSummary,
     notices,
+    ...fixtureResolutionRecords && { fixture_resolutions: fixtureResolutionRecords },
+    ...fixtureCoverageGap && { coverage_gaps: [fixtureCoverageGap] },
     ...routingContext && routingContext.discoveryFailures.length > 0 ? {
       discovery_failures: routingContext.discoveryFailures.map((f) => ({
         agent_key: f.agentKey,
@@ -1905,6 +1976,8 @@ async function runMultiPass(agentUrls, storyboard, options) {
     tested_at: (/* @__PURE__ */ new Date()).toISOString(),
     ...schemasDedup.length > 0 ? { schemas_used: schemasDedup } : {},
     ...assertionsAgg.length > 0 ? { assertions: assertionsAgg } : {},
+    ...first.fixture_resolutions ? { fixture_resolutions: first.fixture_resolutions } : {},
+    ...first.coverage_gaps ? { coverage_gaps: first.coverage_gaps } : {},
     notices: noticesDedup
   };
 }
@@ -2185,8 +2258,43 @@ async function executeStep(client, step, storyboardId, phaseId, context, allStep
     request = applyDisableSandboxHint(request, effectiveStep.task);
   }
   request = applyIdempotencyInvariant(request, effectiveStep.task, step);
-  const unresolvedVars = findUnresolvedContextVars(request);
-  if (unresolvedVars.length > 0 && !step.expect_error) {
+  const fixtureBinding = applyFixtureBindingsSafely(request, effectiveStep.task, options, runState);
+  if (!fixtureBinding.ok) {
+    const next2 = getNextStepPreview(step.id, allSteps, context, runState.runnerVars);
+    return {
+      step_id: step.id,
+      phase_id: phaseId,
+      title: step.title,
+      task: effectiveStep.task,
+      passed: false,
+      duration_ms: 0,
+      validations: [
+        {
+          check: "unresolved_substitution",
+          passed: false,
+          description: `Fixture handle substitution failed: ${fixtureBinding.error}`,
+          json_pointer: null,
+          expected: "an unambiguous fixture handle binding",
+          actual: fixtureBinding.error,
+          schema_id: null,
+          schema_url: null
+        }
+      ],
+      context,
+      error: fixtureBinding.error,
+      next: next2,
+      extraction: { path: "none" }
+    };
+  }
+  request = fixtureBinding.request;
+  request = expandCreativeAssetDirectives(request, context, options.test_kit);
+  const unresolvedContextVars = findUnresolvedContextVars(request);
+  const unresolvedAssetDirectives = findUnresolvedCreativeAssetDirectives(request).map((path) => ({
+    key: path,
+    token: BUILD_ASSETS_FROM_FORMAT_DIRECTIVE
+  }));
+  const unresolvedVars = [...unresolvedContextVars, ...unresolvedAssetDirectives];
+  if (unresolvedAssetDirectives.length > 0 || unresolvedContextVars.length > 0 && !step.expect_error) {
     const next2 = getNextStepPreview(step.id, allSteps, context, runState.runnerVars);
     const responseDerivedDetails = unresolvedVars.map((v) => runState.responseDerivedNotApplicableContextKeys?.get(v.key)).filter((d) => typeof d === "string");
     const allResponseDerived = responseDerivedDetails.length === unresolvedVars.length && responseDerivedDetails.length > 0;
@@ -2200,7 +2308,7 @@ async function executeStep(client, step, storyboardId, phaseId, context, allStep
         synthesized.push({
           check: "unresolved_substitution",
           passed: false,
-          description: `request token "${v.token}" did not resolve \u2014 prior step did not populate context.${v.key}`,
+          description: v.token === BUILD_ASSETS_FROM_FORMAT_DIRECTIVE ? `request directive "${v.token}" did not resolve \u2014 the test kit could not populate every seller-required asset at ${v.key}` : `request token "${v.token}" did not resolve \u2014 prior step did not populate context.${v.key}`,
           json_pointer: null,
           expected: v.token,
           actual: null,
@@ -2860,9 +2968,42 @@ async function executeProbeStep(client, step, phaseId, context, allSteps, option
         sample_request: rateLimitTrip.trip_target_sample_request,
         omit_idempotency_key: true
       };
-      const resolvedTargetRequest = buildEffectiveStepRequest(targetStep, context, options, runState);
-      const unresolvedVars = findUnresolvedContextVars(resolvedTargetRequest);
-      if (unresolvedVars.length > 0 && !targetStep.expect_error) {
+      const targetRequestResult = buildEffectiveStepRequest(targetStep, context, options, runState);
+      if (!targetRequestResult.ok) {
+        const next = getNextStepPreview(step.id, allSteps, context, runState.runnerVars);
+        return {
+          step_id: step.id,
+          phase_id: phaseId,
+          title: step.title,
+          task: step.task,
+          passed: false,
+          duration_ms: Date.now() - start,
+          validations: [
+            {
+              check: "unresolved_substitution",
+              passed: false,
+              description: `Fixture handle substitution failed: ${targetRequestResult.error}`,
+              json_pointer: null,
+              expected: "an unambiguous fixture handle binding",
+              actual: targetRequestResult.error,
+              schema_id: null,
+              schema_url: null
+            }
+          ],
+          context,
+          next,
+          extraction: { path: "none" },
+          error: targetRequestResult.error
+        };
+      }
+      const resolvedTargetRequest = targetRequestResult.request;
+      const unresolvedContextVars = findUnresolvedContextVars(resolvedTargetRequest);
+      const unresolvedAssetDirectives = findUnresolvedCreativeAssetDirectives(resolvedTargetRequest).map((path) => ({
+        key: path,
+        token: BUILD_ASSETS_FROM_FORMAT_DIRECTIVE
+      }));
+      const unresolvedVars = [...unresolvedContextVars, ...unresolvedAssetDirectives];
+      if (unresolvedAssetDirectives.length > 0 || unresolvedContextVars.length > 0 && !targetStep.expect_error) {
         const next = getNextStepPreview(step.id, allSteps, context, runState.runnerVars);
         const detail = `Skipped: unresolved context variables from rate_limit_trip.trip_target_sample_request: ${unresolvedVars.map((v) => v.key).join(", ")}.`;
         return {
@@ -3026,6 +3167,21 @@ async function executeProbeStep(client, step, phaseId, context, allSteps, option
     extraction
   };
 }
+function applyFixtureBindingsSafely(request, task, options, runState) {
+  try {
+    return {
+      ok: true,
+      request: applyFixtureBindingsToRequest(
+        request,
+        task,
+        runState.fixtureBindings,
+        options.adcpVersion ?? ADCP_VERSION
+      )
+    };
+  } catch (error) {
+    return { ok: false, error: error instanceof Error ? error.message : String(error) };
+  }
+}
 function buildEffectiveStepRequest(step, context, options, runState) {
   let request;
   if (options.request) {
@@ -3046,7 +3202,13 @@ function buildEffectiveStepRequest(step, context, options, runState) {
   if (options.disable_sandbox === true) {
     request = applyDisableSandboxHint(request, step.task);
   }
-  return applyIdempotencyInvariant(request, step.task, step);
+  request = applyIdempotencyInvariant(request, step.task, step);
+  const fixtureBinding = applyFixtureBindingsSafely(request, step.task, options, runState);
+  if (!fixtureBinding.ok) return fixtureBinding;
+  return {
+    ok: true,
+    request: expandCreativeAssetDirectives(fixtureBinding.request, context, options.test_kit)
+  };
 }
 async function executeReplayWebhookVectorStep(step, phaseId, context, allSteps, options, runState) {
   const start = Date.now();