@adcp/sdk 5.23.0

package/dist/lib/testing/local-agent-runner.js

@@ -0,0 +1,290 @@
+"use strict";
+/**
+ * `runAgainstLocalAgent` — one-call compliance harness for server-side
+ * AdCP agents.
+ *
+ * The 300-line "spin up my Express-equivalent, seed fixtures, create a
+ * webhook receiver, loop through storyboards, aggregate results" loop
+ * sellers keep hand-rolling collapses into:
+ *
+ * ```ts
+ * const result = await runAgainstLocalAgent({
+ *   createAgent: () => createAdcpServer({ ...myHandlers, stateStore }),
+ *   capabilities: { supported_protocols: ['media-buy'], specialisms: ['sales-non-guaranteed'] },
+ * });
+ * ```
+ *
+ * What it composes:
+ *   - {@link serve} on an ephemeral port (real HTTP transport — MCP
+ *     wire-format bugs surface, unlike in-memory dispatch)
+ *   - {@link seedComplianceFixtures} against the `AdcpServer` (skipped
+ *     when the caller returns a raw `McpServer`)
+ *   - Optional `createTestAuthorizationServer` for closed-loop OAuth
+ *   - {@link runStoryboard} per applicable storyboard, with
+ *     `webhook_receiver: { mode: 'loopback_mock' }` wired by default so
+ *     the webhook-emission universal grades instead of skipping
+ *
+ * The caller's `createAgent` factory MUST close over a stable
+ * `stateStore` so fixtures seeded once persist across requests. The
+ * `serve()` contract calls the factory once per request — every call
+ * needs to see the same store.
+ *
+ * What this helper does NOT do:
+ *   - Start tunnels (use the CLI's `--webhook-receiver-auto-tunnel` when
+ *     fronting a remote agent instead)
+ *   - Mint tokens per-storyboard automatically — call
+ *     `authorizationServer.issueToken()` in `onListening` and stash the
+ *     result on `runStoryboardOptions.auth` when a flow needs one
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runAgainstLocalAgent = runAgainstLocalAgent;
+const node_net_1 = require("node:net");
+const in_memory_js_1 = require("@modelcontextprotocol/sdk/experimental/tasks/stores/in-memory.js");
+const adcp_server_1 = require("../server/adcp-server");
+const serve_1 = require("../server/serve");
+const compliance_fixtures_1 = require("../compliance-fixtures");
+const test_authorization_server_1 = require("../compliance-fixtures/test-authorization-server");
+const compliance_1 = require("./storyboard/compliance");
+const runner_1 = require("./storyboard/runner");
+/**
+ * True when `url` is an http:// (not https://) URL pointing at a loopback
+ * host — used to decide whether the SSRF guard's HTTPS-only default can be
+ * relaxed. Must be tight: a `http://` prefix alone is NOT sufficient (a
+ * caller override could redirect to `http://attacker.example.com/mcp` and
+ * inherit the allowance). Only `127.0.0.1`, `::1`, and `localhost` count.
+ *
+ * @internal
+ */
+function isLoopbackHttpUrl(url) {
+    try {
+        const parsed = new URL(url);
+        if (parsed.protocol !== 'http:')
+            return false;
+        // `URL.hostname` wraps IPv6 literals in brackets (`[::1]`); strip them
+        // to match the bare forms typical in SSRF loopback checks.
+        const host = parsed.hostname.replace(/^\[|\]$/g, '').toLowerCase();
+        return host === '127.0.0.1' || host === '::1' || host === 'localhost';
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Run storyboards against an in-process AdCP agent. Handles lifecycle:
+ * bind, seed, (optional) token issuance, loop, tear down.
+ */
+async function runAgainstLocalAgent(options) {
+    const startedAt = Date.now();
+    const mountPath = options.serveOptions?.path ?? '/mcp';
+    const taskStore = new in_memory_js_1.InMemoryTaskStore();
+    // Bootstrap the factory once with a synthetic host so seeding and
+    // fixture preparation run before the server ever binds. `serve()` will
+    // pass the real per-request host when it calls the factory again.
+    const ctx = { taskStore, host: '' };
+    let auth;
+    let httpServer;
+    try {
+        if (options.authorizationServer) {
+            const authOptions = typeof options.authorizationServer === 'object' ? options.authorizationServer : {};
+            auth = await (0, test_authorization_server_1.createTestAuthorizationServer)(authOptions);
+        }
+        // Bootstrap: call the factory once ahead of time to get an AdcpServer
+        // we can seed against. serve() will call the factory again for every
+        // real request — the user's stateStore closure guarantees seeds persist.
+        const bootstrapAgent = options.createAgent(ctx);
+        const bootstrapIsAdcp = isAdcpServer(bootstrapAgent);
+        if (bootstrapIsAdcp && options.fixtures !== false) {
+            const seedOptions = typeof options.fixtures === 'object' ? options.fixtures : {};
+            await (0, compliance_fixtures_1.seedComplianceFixtures)(bootstrapAgent, seedOptions);
+        }
+        // Release the bootstrap instance so serve()'s per-request factory calls
+        // produce fresh servers. Closing is idempotent.
+        await bootstrapAgent.close();
+        // serve() validates publicUrl path against mountPath synchronously, so
+        // we need the port known before the call. Grab a free port first.
+        const port = await allocatePort();
+        const publicUrl = `http://127.0.0.1:${port}${mountPath}`;
+        const agentUrl = publicUrl;
+        const protectedResource = auth
+            ? { authorization_servers: [auth.issuer] }
+            : undefined;
+        httpServer = (0, serve_1.serve)(options.createAgent, {
+            ...options.serveOptions,
+            taskStore,
+            port,
+            ...(protectedResource ? { protectedResource, publicUrl } : {}),
+            onListening: () => {
+                /* swallow the default console.log — callers can observe via onListening */
+            },
+        });
+        await waitForListening(httpServer);
+        if (options.onListening) {
+            await options.onListening({ agentUrl, auth });
+        }
+        const toRun = await resolveStoryboardSet(options);
+        const results = [];
+        let passed = 0;
+        let failed = 0;
+        let skipped = 0;
+        const webhookConfig = resolveWebhookReceiver(options.webhookReceiver);
+        const explicitAllowHttp = options.runStoryboardOptions?.allow_http;
+        const baseRunOptions = {
+            ...(options.runStoryboardOptions ?? {}),
+        };
+        // `allow_http` is removed from baseRunOptions and recomputed per
+        // storyboard below — inheriting `true` across storyboards would let a
+        // `resolvePerStoryboard` override that redirected to a remote
+        // `http://...` URL bypass the SSRF guard and silently probe an
+        // attacker-controlled target with private-IP allowance.
+        delete baseRunOptions.allow_http;
+        for (let i = 0; i < toRun.storyboards.length; i++) {
+            const sb = toRun.storyboards[i];
+            const override = (await options.resolvePerStoryboard?.(sb, agentUrl)) ?? undefined;
+            const { agentUrl: overrideUrl, ...overrideRunOptions } = override ?? {};
+            const storyboardUrl = overrideUrl ?? agentUrl;
+            // Tighten the SSRF-guard bypass: only auto-opt-in `allow_http` when
+            // THIS storyboard's URL is loopback-bound (what the helper owns).
+            // An override to `http://attacker.example.com/...` must not
+            // inherit the default — that would re-enable the private-IP
+            // allowance against attacker-controlled infrastructure.
+            const storyboardIsLoopbackHttp = isLoopbackHttpUrl(storyboardUrl);
+            const effectiveAllowHttp = overrideRunOptions.allow_http ?? explicitAllowHttp ?? storyboardIsLoopbackHttp;
+            // webhook_receiver is helper-owned — re-applied after the merge so
+            // overrides can't replace the loopback mock the helper promised.
+            const storyboardOptions = {
+                ...baseRunOptions,
+                ...overrideRunOptions,
+                allow_http: effectiveAllowHttp,
+                ...(webhookConfig ? { webhook_receiver: webhookConfig } : {}),
+            };
+            const result = await (0, runner_1.runStoryboard)(storyboardUrl, sb, storyboardOptions);
+            results.push(result);
+            passed += result.passed_count;
+            failed += result.failed_count;
+            skipped += result.skipped_count;
+            if (options.onStoryboardComplete) {
+                options.onStoryboardComplete(result, i, toRun.storyboards.length);
+            }
+            if (options.bail && !result.overall_passed)
+                break;
+        }
+        return {
+            overall_passed: results.every(r => r.overall_passed),
+            agent_url: agentUrl,
+            results,
+            not_applicable: toRun.not_applicable,
+            passed_count: passed,
+            failed_count: failed,
+            skipped_count: skipped,
+            total_duration_ms: Date.now() - startedAt,
+        };
+    }
+    finally {
+        await closeHttpServer(httpServer);
+        if (auth)
+            await auth.close();
+    }
+}
+// ────────────────────────────────────────────────────────────
+// Helpers
+// ────────────────────────────────────────────────────────────
+function isAdcpServer(candidate) {
+    if (candidate == null || typeof candidate !== 'object')
+        return false;
+    return typeof candidate[adcp_server_1.ADCP_STATE_STORE] === 'object';
+}
+function resolveWebhookReceiver(option) {
+    if (option === false)
+        return undefined;
+    if (option === true || option === undefined)
+        return { mode: 'loopback_mock' };
+    return option;
+}
+async function resolveStoryboardSet(options) {
+    const spec = options.storyboards ?? 'all';
+    const compliance = options.compliance ?? {};
+    if (spec === 'all') {
+        return { storyboards: (0, compliance_1.listAllComplianceStoryboards)(compliance), not_applicable: [] };
+    }
+    if (Array.isArray(spec)) {
+        if (spec.length === 0)
+            return { storyboards: [], not_applicable: [] };
+        if (typeof spec[0] === 'object' && 'phases' in spec[0]) {
+            return { storyboards: spec, not_applicable: [] };
+        }
+        const resolved = [];
+        const unknown = [];
+        for (const id of spec) {
+            const matches = (0, compliance_1.resolveBundleOrStoryboard)(id, compliance);
+            if (matches.length === 0) {
+                unknown.push(id);
+            }
+            else {
+                resolved.push(...matches);
+            }
+        }
+        if (unknown.length > 0) {
+            throw new Error(`runAgainstLocalAgent: unknown storyboard or bundle id(s): ${unknown.join(', ')}. ` +
+                `Run \`adcp storyboard list\` to see available ids.`);
+        }
+        const seen = new Set();
+        return {
+            storyboards: resolved.filter(sb => (seen.has(sb.id) ? false : (seen.add(sb.id), true))),
+            not_applicable: [],
+        };
+    }
+    const resolved = (0, compliance_1.resolveStoryboardsForCapabilities)(spec, compliance);
+    return {
+        storyboards: resolved.storyboards,
+        not_applicable: resolved.not_applicable.map(n => ({
+            storyboard_id: n.storyboard_id,
+            storyboard_title: n.storyboard_title,
+            reason: n.reason,
+        })),
+    };
+}
+function allocatePort() {
+    return new Promise((resolve, reject) => {
+        const probe = (0, node_net_1.createServer)();
+        probe.unref();
+        probe.on('error', reject);
+        probe.listen(0, '127.0.0.1', () => {
+            const { port } = probe.address();
+            probe.close(err => (err ? reject(err) : resolve(port)));
+        });
+    });
+}
+function waitForListening(server) {
+    if (server.listening)
+        return Promise.resolve();
+    return new Promise((resolve, reject) => {
+        const onListening = () => {
+            cleanup();
+            resolve();
+        };
+        const onError = (err) => {
+            cleanup();
+            reject(err);
+        };
+        const cleanup = () => {
+            server.off('listening', onListening);
+            server.off('error', onError);
+        };
+        server.on('listening', onListening);
+        server.on('error', onError);
+    });
+}
+function closeHttpServer(server) {
+    if (!server)
+        return Promise.resolve();
+    return new Promise((resolve, reject) => {
+        server.closeAllConnections?.();
+        server.close(err => {
+            if (err && err.code !== 'ERR_SERVER_NOT_RUNNING')
+                reject(err);
+            else
+                resolve();
+        });
+    });
+}
+//# sourceMappingURL=local-agent-runner.js.map

package/dist/lib/testing/local-agent-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-agent-runner.js","sourceRoot":"","sources":["../../../src/lib/testing/local-agent-runner.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;;AAuMH,oDAyHC;AA9TD,uCAA4E;AAC5E,mGAAqG;AAKrG,uDAAyD;AACzD,2CAA8G;AAC9G,gEAAoG;AACpG,gGAI0D;AAC1D,wDAMiC;AACjC,gDAAoD;AAsJpD;;;;;;;;GAQG;AACH,SAAS,iBAAiB,CAAC,GAAW;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO;YAAE,OAAO,KAAK,CAAC;QAC9C,uEAAuE;QACvE,2DAA2D;QAC3D,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACnE,OAAO,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,WAAW,CAAC;IACxE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,oBAAoB,CAAC,OAAoC;IAC7E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,EAAE,IAAI,IAAI,MAAM,CAAC;IACvD,MAAM,SAAS,GAAc,IAAI,gCAAiB,EAAE,CAAC;IACrD,kEAAkE;IAClE,uEAAuE;IACvE,kEAAkE;IAClE,MAAM,GAAG,GAAiB,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IAElD,IAAI,IAAyC,CAAC;IAC9C,IAAI,UAAkC,CAAC;IAEvC,IAAI,CAAC;QACH,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;YAChC,MAAM,WAAW,GAAG,OAAO,OAAO,CAAC,mBAAmB,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;YACvG,IAAI,GAAG,MAAM,IAAA,yDAA6B,EAAC,WAAW,CAAC,CAAC;QAC1D,CAAC;QAED,sEAAsE;QACtE,qEAAqE;QACrE,yEAAyE;QACzE,MAAM,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,eAAe,GAAG,YAAY,CAAC,cAAc,CAAC,CAAC;QACrD,IAAI,eAAe,IAAI,OAAO,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;YAClD,MAAM,WAAW,GAAG,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YACjF,MAAM,IAAA,4CAAsB,EAAC,cAAc,EAAE,WAAW,CAAC,CAAC;QAC5D,CAAC;QACD,wEAAwE;QACxE,gDAAgD;QAChD,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,uEAAuE;QACvE,kEAAkE;QAClE,MAAM,IAAI,GAAG,MAAM,YAAY,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,oBAAoB,IAAI,GAAG,SAAS,EAAE,CAAC;QACzD,MAAM,QAAQ,GAAG,SAAS,CAAC;QAE3B,MAAM,iBAAiB,GAA0C,IAAI;YACnE,CAAC,CAAC,EAAE,qBAAqB,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YAC1C,CAAC,CAAC,SAAS,CAAC;QAEd,UAAU,GAAG,IAAA,aAAK,EAAC,OAAO,CAAC,WAAW,EAAE;YACtC,GAAG,OAAO,CAAC,YAAY;YACvB,SAAS;YACT,IAAI;YACJ,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,WAAW,EAAE,GAAG,EAAE;gBAChB,2EAA2E;YAC7E,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,gBAAgB,CAAC,UAAU,CAAC,CAAC;QAEnC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;YACxB,MAAM,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,OAAO,GAAuB,EAAE,CAAC;QACvC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,MAAM,aAAa,GAAG,sBAAsB,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QACtE,MAAM,iBAAiB,GAAG,OAAO,CAAC,oBAAoB,EAAE,UAAU,CAAC;QACnE,MAAM,cAAc,GAAkE;YACpF,GAAG,CAAC,OAAO,CAAC,oBAAoB,IAAI,EAAE,CAAC;SACxC,CAAC;QACF,iEAAiE;QACjE,sEAAsE;QACtE,8DAA8D;QAC9D,+DAA+D;QAC/D,wDAAwD;QACxD,OAAQ,cAA2C,CAAC,UAAU,CAAC;QAE/D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClD,MAAM,EAAE,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,CAAC,MAAM,OAAO,CAAC,oBAAoB,EAAE,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,IAAI,SAAS,CAAC;YACnF,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,kBAAkB,EAAE,GAAG,QAAQ,IAAI,EAAE,CAAC;YACxE,MAAM,aAAa,GAAG,WAAW,IAAI,QAAQ,CAAC;YAC9C,oEAAoE;YACpE,kEAAkE;YAClE,4DAA4D;YAC5D,4DAA4D;YAC5D,wDAAwD;YACxD,MAAM,wBAAwB,GAAG,iBAAiB,CAAC,aAAa,CAAC,CAAC;YAClE,MAAM,kBAAkB,GACrB,kBAA+C,CAAC,UAAU,IAAI,iBAAiB,IAAI,wBAAwB,CAAC;YAC/G,mEAAmE;YACnE,iEAAiE;YACjE,MAAM,iBAAiB,GAAyB;gBAC9C,GAAG,cAAc;gBACjB,GAAG,kBAAkB;gBACrB,UAAU,EAAE,kBAAkB;gBAC9B,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9D,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAa,EAAC,aAAa,EAAE,EAAE,EAAE,iBAAiB,CAAC,CAAC;YACzE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;YAC9B,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;YAC9B,OAAO,IAAI,MAAM,CAAC,aAAa,CAAC;YAChC,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;gBACjC,OAAO,CAAC,oBAAoB,CAAC,MAAM,EAAE,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACpE,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc;gBAAE,MAAM;QACpD,CAAC;QAED,OAAO;YACL,cAAc,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC;YACpD,SAAS,EAAE,QAAQ;YACnB,OAAO;YACP,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,YAAY,EAAE,MAAM;YACpB,YAAY,EAAE,MAAM;YACpB,aAAa,EAAE,OAAO;YACtB,iBAAiB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SAC1C,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC;QAClC,IAAI,IAAI;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;AACH,CAAC;AAED,+DAA+D;AAC/D,UAAU;AACV,+DAA+D;AAE/D,SAAS,YAAY,CAAC,SAAkB;IACtC,IAAI,SAAS,IAAI,IAAI,IAAI,OAAO,SAAS,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACrE,OAAO,OAAQ,SAAqC,CAAC,8BAAgB,CAAC,KAAK,QAAQ,CAAC;AACtF,CAAC;AAED,SAAS,sBAAsB,CAC7B,MAAsD;IAEtD,IAAI,MAAM,KAAK,KAAK;QAAE,OAAO,SAAS,CAAC;IACvC,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;IAC9E,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,OAAoC;IAItE,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC;IAC1C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;IAC5C,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnB,OAAO,EAAE,WAAW,EAAE,IAAA,yCAA4B,EAAC,UAAU,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC;IACvF,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC;QACtE,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,QAAQ,IAAK,IAAI,CAAC,CAAC,CAAgB,EAAE,CAAC;YACvE,OAAO,EAAE,WAAW,EAAE,IAAoB,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC;QACnE,CAAC;QACD,MAAM,QAAQ,GAAiB,EAAE,CAAC;QAClC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,EAAE,IAAI,IAAgB,EAAE,CAAC;YAClC,MAAM,OAAO,GAAG,IAAA,sCAAyB,EAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CACb,6DAA6D,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;gBACjF,oDAAoD,CACvD,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,OAAO;YACL,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;YACvF,cAAc,EAAE,EAAE;SACnB,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,IAAA,8CAAiC,EAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACrE,OAAO;QACL,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,cAAc,EAAE,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChD,aAAa,EAAE,CAAC,CAAC,aAAa;YAC9B,gBAAgB,EAAE,CAAC,CAAC,gBAAgB;YACpC,MAAM,EAAE,CAAC,CAAC,MAAM;SACjB,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,SAAS,YAAY;IACnB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,IAAA,uBAAc,GAAE,CAAC;QAC/B,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE;YAChC,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,OAAO,EAAiB,CAAC;YAChD,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAkB;IAC1C,IAAI,MAAM,CAAC,SAAS;QAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/C,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,MAAM,WAAW,GAAG,GAAG,EAAE;YACvB,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,CAAC,GAAU,EAAE,EAAE;YAC7B,OAAO,EAAE,CAAC;YACV,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YACrC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/B,CAAC,CAAC;QACF,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACpC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,eAAe,CAAC,MAA8B;IACrD,IAAI,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IACtC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,MAAM,CAAC,mBAAmB,EAAE,EAAE,CAAC;QAC/B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;YACjB,IAAI,GAAG,IAAK,GAA6B,CAAC,IAAI,KAAK,wBAAwB;gBAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;gBACpF,OAAO,EAAE,CAAC;QACjB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/lib/testing/orchestrator.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Test Suite Orchestrator for AdCP Agents
+ *
+ * Discovers agent capabilities and runs all applicable test scenarios,
+ * aggregating results into a single suite report.
+ */
+import type { TestScenario, TestOptions, SuiteResult } from './types';
+/**
+ * Minimum tools required for each scenario to be applicable.
+ * ALL listed tools must be present in the agent's tool list.
+ * Scenarios with an empty array are always applicable.
+ *
+ * @deprecated Use storyboard-driven testing via `runStoryboard()` instead.
+ * Storyboards define required tools per step via `requires_tool` and per
+ * storyboard via `required_tools`. See `src/lib/testing/storyboard/`.
+ */
+export declare const SCENARIO_REQUIREMENTS: Partial<Record<TestScenario, string[]>>;
+/**
+ * Default set of scenarios the orchestrator will attempt.
+ * Excludes unimplemented scenarios and deduplicates coverage
+ * (pricing_models is omitted since pricing_edge_cases covers the same ground).
+ *
+ * Deterministic scenarios (deterministic_*, controller_validation) are excluded here —
+ * they are managed by the comply engine via TRACK_DEFINITIONS and only run when the
+ * seller exposes comply_test_controller.
+ *
+ * @deprecated Use storyboard-driven testing via `runStoryboard()` instead.
+ */
+export declare const DEFAULT_SCENARIOS: readonly TestScenario[];
+/**
+ * Options for testAllScenarios(), extending single-scenario TestOptions
+ * with an optional scenario filter.
+ */
+export interface OrchestratorOptions extends TestOptions {
+    /** Limit which scenarios to attempt (defaults to DEFAULT_SCENARIOS) */
+    scenarios?: TestScenario[];
+}
+/**
+ * Return the subset of candidate scenarios that are applicable for an agent
+ * with the given tool list.
+ *
+ * @param tools - Tool names from getAgentInfo()
+ * @param filter - Scenarios to consider; defaults to DEFAULT_SCENARIOS
+ */
+export declare function getApplicableScenarios(tools: string[], filter?: readonly TestScenario[]): TestScenario[];
+/**
+ * Run all applicable test scenarios against an agent and return aggregated results.
+ *
+ * Scenarios are run sequentially. Each scenario re-uses the same TestOptions
+ * (including auth, brand, etc.). The total duration includes the
+ * initial capability discovery call.
+ *
+ * Note: testAllScenarios creates a client for capability discovery, then each
+ * testAgent call creates its own client internally. This is intentional — testAgent
+ * is designed as a standalone function and sharing state between scenarios could
+ * cause cross-scenario interference.
+ *
+ * @deprecated Use `runStoryboard()` for storyboard-driven testing instead.
+ * The comply() engine now uses storyboards as its primary testing surface.
+ */
+export declare function testAllScenarios(agentUrl: string, options?: OrchestratorOptions): Promise<SuiteResult>;
+/**
+ * Format suite results as markdown for display in Slack/chat
+ */
+export declare function formatSuiteResults(suite: SuiteResult): string;
+/**
+ * Format suite results as JSON
+ */
+export declare function formatSuiteResultsJSON(suite: SuiteResult): string;
+//# sourceMappingURL=orchestrator.d.ts.map

package/dist/lib/testing/orchestrator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../../src/lib/testing/orchestrator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAc,WAAW,EAAE,MAAM,SAAS,CAAC;AAElF;;;;;;;;GAQG;AACH,eAAO,MAAM,qBAAqB,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC,CA8EzE,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,iBAAiB,EAAE,SAAS,YAAY,EAwC3C,CAAC;AAEX;;;GAGG;AACH,MAAM,WAAW,mBAAoB,SAAQ,WAAW;IACtD,uEAAuE;IACvE,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;CAC5B;AAQD;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,SAAS,YAAY,EAAE,GAAG,YAAY,EAAE,CAGxG;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,WAAW,CAAC,CA2DhH;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,CA0C7D;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,CAEjE"}

package/dist/lib/testing/orchestrator.js

@@ -0,0 +1,278 @@
+"use strict";
+/**
+ * Test Suite Orchestrator for AdCP Agents
+ *
+ * Discovers agent capabilities and runs all applicable test scenarios,
+ * aggregating results into a single suite report.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_SCENARIOS = exports.SCENARIO_REQUIREMENTS = void 0;
+exports.getApplicableScenarios = getApplicableScenarios;
+exports.testAllScenarios = testAllScenarios;
+exports.formatSuiteResults = formatSuiteResults;
+exports.formatSuiteResultsJSON = formatSuiteResultsJSON;
+const agent_tester_1 = require("./agent-tester");
+const client_1 = require("./client");
+/**
+ * Minimum tools required for each scenario to be applicable.
+ * ALL listed tools must be present in the agent's tool list.
+ * Scenarios with an empty array are always applicable.
+ *
+ * @deprecated Use storyboard-driven testing via `runStoryboard()` instead.
+ * Storyboards define required tools per step via `requires_tool` and per
+ * storyboard via `required_tools`. See `src/lib/testing/storyboard/`.
+ */
+exports.SCENARIO_REQUIREMENTS = {
+    // Always applicable
+    health_check: [],
+    discovery: [],
+    capability_discovery: [],
+    // Requires media buy core tools
+    create_media_buy: ['get_products', 'create_media_buy'],
+    full_sales_flow: ['get_products', 'create_media_buy'],
+    reporting_flow: ['get_products', 'create_media_buy', 'get_media_buy_delivery'],
+    creative_inline: ['get_products', 'create_media_buy'],
+    creative_reference: ['get_products', 'create_media_buy', 'build_creative', 'sync_creatives'],
+    temporal_validation: ['get_products', 'create_media_buy'],
+    creative_sync: ['sync_creatives'],
+    // Requires get_products
+    pricing_edge_cases: ['get_products'],
+    pricing_models: ['get_products'],
+    error_handling: ['get_products'],
+    validation: ['get_products'],
+    behavior_analysis: ['get_products'],
+    response_consistency: ['get_products'],
+    // Requires creative agent tools
+    creative_flow: ['build_creative'],
+    creative_lifecycle: ['list_creative_formats'],
+    // Requires signals tools
+    signals_flow: ['get_signals'],
+    // Requires governance tools
+    governance_property_lists: ['create_property_list'],
+    governance_content_standards: ['list_content_standards'],
+    property_list_filters: ['create_property_list', 'get_property_list'],
+    // Requires campaign governance tools
+    campaign_governance: ['sync_plans', 'check_governance'],
+    campaign_governance_denied: ['sync_plans', 'check_governance'],
+    campaign_governance_conditions: ['sync_plans', 'check_governance'],
+    campaign_governance_delivery: ['check_governance'],
+    // Requires seller tools (governance context persistence)
+    seller_governance_context: ['get_products', 'create_media_buy', 'get_media_buys'],
+    // Requires SI tools
+    si_session_lifecycle: ['si_initiate_session'],
+    si_availability: ['si_get_offering'],
+    si_handoff: ['si_initiate_session', 'si_terminate_session'],
+    // Requires audience management tools
+    sync_audiences: ['sync_audiences'],
+    // State machine compliance
+    media_buy_lifecycle: ['get_products', 'create_media_buy', 'update_media_buy'],
+    terminal_state_enforcement: ['get_products', 'create_media_buy', 'update_media_buy'],
+    package_lifecycle: ['get_products', 'create_media_buy', 'update_media_buy'],
+    // Schema compliance (requires product discovery)
+    schema_compliance: ['get_products'],
+    // Error compliance (requires create_media_buy to provoke errors)
+    error_codes: ['create_media_buy'],
+    error_structure: ['create_media_buy'],
+    error_transport: ['create_media_buy'],
+    // Requires brand rights tools
+    brand_identity: ['get_brand_identity'],
+    brand_rights_flow: ['get_rights', 'acquire_rights'],
+    creative_approval: ['creative_approval'],
+    // Deterministic state machine scenarios (require comply_test_controller)
+    deterministic_creative: ['comply_test_controller', 'sync_creatives'],
+    deterministic_media_buy: ['comply_test_controller', 'get_products', 'create_media_buy'],
+    deterministic_account: ['comply_test_controller', 'list_accounts'],
+    deterministic_session: ['comply_test_controller', 'si_initiate_session'],
+    deterministic_delivery: ['comply_test_controller', 'get_products', 'create_media_buy', 'get_media_buy_delivery'],
+    deterministic_budget: ['comply_test_controller', 'get_products', 'create_media_buy'],
+    controller_validation: ['comply_test_controller'],
+};
+/**
+ * Default set of scenarios the orchestrator will attempt.
+ * Excludes unimplemented scenarios and deduplicates coverage
+ * (pricing_models is omitted since pricing_edge_cases covers the same ground).
+ *
+ * Deterministic scenarios (deterministic_*, controller_validation) are excluded here —
+ * they are managed by the comply engine via TRACK_DEFINITIONS and only run when the
+ * seller exposes comply_test_controller.
+ *
+ * @deprecated Use storyboard-driven testing via `runStoryboard()` instead.
+ */
+exports.DEFAULT_SCENARIOS = [
+    'health_check',
+    'discovery',
+    'capability_discovery',
+    'create_media_buy',
+    'full_sales_flow',
+    'creative_sync',
+    'creative_inline',
+    'creative_reference',
+    'pricing_edge_cases',
+    'error_handling',
+    'validation',
+    'temporal_validation',
+    'behavior_analysis',
+    'response_consistency',
+    'creative_flow',
+    'creative_lifecycle',
+    'signals_flow',
+    'governance_property_lists',
+    'governance_content_standards',
+    'property_list_filters',
+    'campaign_governance',
+    'campaign_governance_denied',
+    'campaign_governance_conditions',
+    'campaign_governance_delivery',
+    'seller_governance_context',
+    'si_session_lifecycle',
+    'si_availability',
+    'si_handoff',
+    'schema_compliance',
+    'sync_audiences',
+    'media_buy_lifecycle',
+    'terminal_state_enforcement',
+    'package_lifecycle',
+    'error_codes',
+    'error_structure',
+    'error_transport',
+    'brand_identity',
+    'brand_rights_flow',
+    'creative_approval',
+];
+function isApplicable(scenario, tools) {
+    if (!(scenario in exports.SCENARIO_REQUIREMENTS))
+        return false;
+    const requirements = exports.SCENARIO_REQUIREMENTS[scenario];
+    return requirements.every(tool => tools.includes(tool));
+}
+/**
+ * Return the subset of candidate scenarios that are applicable for an agent
+ * with the given tool list.
+ *
+ * @param tools - Tool names from getAgentInfo()
+ * @param filter - Scenarios to consider; defaults to DEFAULT_SCENARIOS
+ */
+function getApplicableScenarios(tools, filter) {
+    const candidates = filter ?? exports.DEFAULT_SCENARIOS;
+    return candidates.filter(s => isApplicable(s, tools));
+}
+/**
+ * Run all applicable test scenarios against an agent and return aggregated results.
+ *
+ * Scenarios are run sequentially. Each scenario re-uses the same TestOptions
+ * (including auth, brand, etc.). The total duration includes the
+ * initial capability discovery call.
+ *
+ * Note: testAllScenarios creates a client for capability discovery, then each
+ * testAgent call creates its own client internally. This is intentional — testAgent
+ * is designed as a standalone function and sharing state between scenarios could
+ * cause cross-scenario interference.
+ *
+ * @deprecated Use `runStoryboard()` for storyboard-driven testing instead.
+ * The comply() engine now uses storyboards as its primary testing surface.
+ */
+async function testAllScenarios(agentUrl, options = {}) {
+    const start = Date.now();
+    const { scenarios: scenarioFilter, ...testOptions } = options;
+    const effectiveOptions = {
+        ...testOptions,
+        sandbox: testOptions.sandbox !== false,
+        test_session_id: testOptions.test_session_id || `addie-suite-${Date.now()}`,
+    };
+    // Discover the agent's tools to determine which scenarios apply.
+    // If discovery fails (agent unreachable or rejects the request), return early
+    // rather than attempting scenarios with no tool information.
+    const client = (0, client_1.getOrCreateClient)(agentUrl, effectiveOptions);
+    const { profile, step: profileStep } = await (0, client_1.getOrDiscoverProfile)(client, effectiveOptions);
+    effectiveOptions._client = client;
+    effectiveOptions._profile = profile;
+    if (!profileStep.passed) {
+        return {
+            agent_url: agentUrl,
+            agent_profile: profile,
+            scenarios_run: [],
+            scenarios_skipped: [],
+            results: [],
+            overall_passed: false,
+            passed_count: 0,
+            failed_count: 0,
+            total_duration_ms: Date.now() - start,
+            tested_at: new Date().toISOString(),
+        };
+    }
+    const applicable = getApplicableScenarios(profile.tools, scenarioFilter);
+    const candidates = scenarioFilter ?? exports.DEFAULT_SCENARIOS;
+    const skipped = candidates.filter(s => !applicable.includes(s));
+    // Run each applicable scenario sequentially
+    const results = [];
+    for (const scenario of applicable) {
+        const result = await (0, agent_tester_1.testAgent)(agentUrl, scenario, effectiveOptions);
+        results.push(result);
+    }
+    const passedCount = results.filter(r => r.overall_passed).length;
+    const failedCount = results.filter(r => !r.overall_passed).length;
+    return {
+        agent_url: agentUrl,
+        agent_profile: profile,
+        scenarios_run: applicable,
+        scenarios_skipped: skipped,
+        results,
+        overall_passed: failedCount === 0 && passedCount > 0,
+        passed_count: passedCount,
+        failed_count: failedCount,
+        total_duration_ms: Date.now() - start,
+        tested_at: new Date().toISOString(),
+    };
+}
+/**
+ * Format suite results as markdown for display in Slack/chat
+ */
+function formatSuiteResults(suite) {
+    const statusEmoji = suite.overall_passed ? '✅' : '❌';
+    let output = `## ${statusEmoji} Agent Test Suite Results\n\n`;
+    output += `**Agent:** ${suite.agent_url}\n`;
+    output += `**Agent Name:** ${suite.agent_profile.name}\n`;
+    output += `**Duration:** ${suite.total_duration_ms}ms\n`;
+    if (suite.scenarios_run.length === 0) {
+        output += `**Result:** No applicable scenarios found for this agent.\n\n`;
+    }
+    else {
+        output += `**Result:** ${suite.passed_count} passed, ${suite.failed_count} failed out of ${suite.scenarios_run.length} scenario(s)\n\n`;
+    }
+    if (suite.scenarios_skipped.length > 0) {
+        output += `**Skipped (agent does not advertise required tools):** ${suite.scenarios_skipped.join(', ')}\n\n`;
+    }
+    output += `### Scenario Results\n\n`;
+    for (const result of suite.results) {
+        const emoji = result.overall_passed ? '✅' : '❌';
+        output += `${emoji} **${result.scenario}**: ${result.summary} (${result.total_duration_ms}ms)\n`;
+        if (!result.overall_passed) {
+            const failedSteps = (result.steps ?? []).filter(s => !s.passed);
+            for (const step of failedSteps) {
+                output += `   ❌ **${step.step}**`;
+                if (step.task) {
+                    output += ` (\`${step.task}\`)`;
+                }
+                output += ` - ${step.duration_ms}ms\n`;
+                if (step.details)
+                    output += `      ${step.details}\n`;
+                if (step.error)
+                    output += `      ⚠️ Error: ${step.error}\n`;
+                if (step.response_preview) {
+                    output +=
+                        '\n<details><summary>Diagnostic details</summary>\n\n```json\n' +
+                            step.response_preview +
+                            '\n```\n\n</details>\n\n';
+                }
+            }
+        }
+    }
+    return output;
+}
+/**
+ * Format suite results as JSON
+ */
+function formatSuiteResultsJSON(suite) {
+    return JSON.stringify(suite, null, 2);
+}
+//# sourceMappingURL=orchestrator.js.map