@powerduck/openapi-cli 0.2.7 → 0.2.12

package/dist/index.d.cts

@@ -1,6 +1,64 @@
+import { RequestValues, createClient } from '@powerduck/openapi-request';
+
+/**
+ * Assertion engine. Supports two sources:
+ * 1. Declarative `x-tests` array on the operation (simple, JSON-serializable).
+ * 2. Postman `x-postman-scripts.test` string (full pm.* API, executed by
+ *    @powerduck/openapi-request's script sandbox and surfaced via onAssertion).
+ */
+
+interface DeclarativeAssertion {
+    name: string;
+    /** What to assert. */
+    assert: "status" | "header" | "bodyContains" | "jsonPath" | "responseTime" | "bodyEquals";
+    /** Expected status code (for assert=status). */
+    value?: number;
+    /** Header name (for assert=header). */
+    key?: string;
+    /** Substring to check (for assert=header / bodyContains). */
+    contains?: string;
+    /** JSONPath expression (for assert=jsonPath). Simple dot-notation supported. */
+    path?: string;
+    /** Expected value at path (for assert=jsonPath). */
+    equals?: unknown;
+    /** Whether the path must exist (for assert=jsonPath). */
+    exists?: boolean;
+    /** Max response time in ms (for assert=responseTime). */
+    max?: number;
+    /** Expected body string (for assert=bodyEquals). */
+    body?: string;
+}
+interface AssertionContext {
+    status?: number;
+    headers?: Record<string, string>;
+    body?: unknown;
+    bodyText?: string;
+    durationMs: number;
+}
+/**
+ * Run declarative assertions against a response context.
+ */
+declare function runDeclarativeAssertions(assertions: DeclarativeAssertion[], ctx: AssertionContext): AssertionResult[];
+/**
+ * Minimal JSONPath resolver supporting dot notation and array indices.
+ * e.g. "$.data.items[0].id" or "data.user.name"
+ */
+declare function resolveJsonPath(obj: unknown, path: string): unknown;
+/**
+ * Extract declarative assertions from an OpenAPI operation.
+ * Looks for `x-tests` on the operation or path item.
+ */
+declare function extractAssertions(operation: any, pathItem?: any): DeclarativeAssertion[];
+/**
+ * Extract Postman test scripts from an operation.
+ * Looks for `x-postman-scripts.test` on the operation.
+ */
+declare function extractPostmanScripts(operation: any): string | undefined;
+
 /**
  * Public types for @powerduck/openapi-cli.
  */
+
 type ProtocolName = "http" | "sse" | "websocket" | "graphql" | "grpc" | "mcp";
 type ReportFormat = "json" | "cli" | "html";
 type TestStatus = "passed" | "failed" | "skipped" | "error";
@@ -80,6 +138,21 @@ interface AssertionResult {
     passed: boolean;
     error?: string;
 }
+/**
+ * Best-effort snapshot of the request actually sent, after variable
+ * substitution and header merging. Secrets are included as sent; callers
+ * decide what to redact before persisting or displaying.
+ */
+interface RequestSnapshot {
+    method?: string;
+    /** Fully resolved URL for HTTP-style protocols. */
+    url?: string;
+    headers?: Record<string, string>;
+    /** Request body, gRPC message, WS payload or MCP arguments. */
+    body?: unknown;
+    /** Protocol-specific target, e.g. "package.Service/Method" or an MCP tool. */
+    target?: string;
+}
 interface TestResult {
     operationId: string;
     path: string;
@@ -87,6 +160,8 @@ interface TestResult {
     protocol: ProtocolName | string;
     status: TestStatus;
     durationMs: number;
+    /** Snapshot of the resolved request, when the protocol could capture it. */
+    request?: RequestSnapshot;
     response?: {
         status?: number;
         statusText?: string;
@@ -118,11 +193,236 @@ interface TestReport {
     generatedAt: string;
     version: string;
 }
+/** Pull a value out of one step's response into the shared scenario scope. */
+interface ScenarioExtract {
+    /** Variable name written into the scenario scope. */
+    name: string;
+    /** Response part to read from. */
+    from: "body" | "header" | "status";
+    /** Dot/JSONPath into the JSON body when from="body" (e.g. "$.data.token"). */
+    path?: string;
+    /** Response header name when from="header". */
+    key?: string;
+}
+/**
+ * One row of data-driven values. Keys are variable names referenced as
+ * {{name}} in step values; values use the same string contract as scenario
+ * variables.
+ */
+type ScenarioDataRow = Record<string, string>;
+/** Per-step request overrides and verification rules. */
+interface ScenarioStepRequest {
+    /** Request values merged over the operation's sampled defaults. */
+    values?: RequestValues;
+    /** Override the resolved server URL for this step only. */
+    serverUrl?: string;
+    /** Declarative variables extracted after the response arrives. */
+    extract?: ScenarioExtract[];
+    /** Declarative assertions added on top of the operation's spec assertions. */
+    assertions?: DeclarativeAssertion[];
+    /**
+     * Step-level data rows. The step executes once per row; each row's values
+     * layer over the scenario scope only for that execution. Extracts from the
+     * last execution are written back into the scenario scope.
+     */
+    data?: ScenarioDataRow[];
+}
+/** A single ordered operation invocation inside a scenario. */
+interface ScenarioStep {
+    /** Reference as "METHOD /path" (e.g. "POST /login") or an operationId. */
+    ref?: string;
+    /** Explicit method, paired with `path`; an alternative to `ref`. */
+    method?: string;
+    /** Explicit path, paired with `method`; an alternative to `ref`. */
+    path?: string;
+    /** Human-readable step label shown in reports and live events. */
+    name?: string;
+    request?: ScenarioStepRequest;
+    /** Compile-time skip; the step is reported as skipped and never sent. */
+    skip?: boolean;
+}
+/** A named, ordered sequence of operations with shared variable scope. */
+interface ScenarioDefinition {
+    name: string;
+    description?: string;
+    steps: ScenarioStep[];
+    /** Initial scenario variables, merged over CliConfig.variables. */
+    variables?: Record<string, string>;
+    /** Stop at the first failed/error step. Default: true. */
+    stopOnFailure?: boolean;
+    /**
+     * Scenario-level data rows. The whole step sequence runs once per row, each
+     * time with a fresh variable scope seeded from the row; extracts never leak
+     * across iterations. An empty/absent array means a single iteration.
+     */
+    data?: ScenarioDataRow[];
+    /**
+     * Declared number of full-sequence repetitions. The effective iteration
+     * count is max(iterations, data.length); iterations beyond the data rows
+     * run with an empty scope. Useful for "run this flow N times" requests that
+     * carry no row-specific values. Defaults to 1 when omitted.
+     */
+    iterations?: number;
+}
+type ScenarioStatus = "passed" | "failed" | "error" | "cancelled";
+/** Result of executing one scenario step. */
+interface ScenarioStepResult extends TestResult {
+    /** Zero-based position in the scenario. */
+    index: number;
+    /** Resolved operation reference ("METHOD /path"). */
+    ref: string;
+    name?: string;
+    /** Zero-based scenario-level iteration (definition.data row index). */
+    scenarioIteration?: number;
+    /** Total scenario-level iterations for this run. */
+    scenarioIterations?: number;
+    /** Zero-based step-level iteration (step.request.data row index). */
+    stepIteration?: number;
+    /** Total step-level iterations for this step (>=1). */
+    stepIterations?: number;
+    /** Variables written by this step (declarative extraction + scripts). */
+    extracted?: Record<string, string>;
+    /** Full scenario variable scope after this step completed. */
+    variablesAfter?: Record<string, string>;
+    /** True when the step never executed (explicit skip or halted by a prior failure). */
+    notRun?: boolean;
+    notRunReason?: string;
+}
+type ScenarioEventType = "scenario:start" | "step:start" | "step:finish" | "step:skip" | "scenario:finish" | "scenario:cancel";
+/** Live progress event emitted while a scenario runs. */
+interface ScenarioEvent {
+    type: ScenarioEventType;
+    scenario: string;
+    stepIndex?: number;
+    step?: ScenarioStepResult;
+    status?: ScenarioStatus;
+    message?: string;
+    /** Zero-based scenario-level iteration (definition.data row index). */
+    scenarioIteration?: number;
+    /** Total scenario-level iterations. */
+    scenarioIterations?: number;
+    /** Zero-based step-level iteration (step.request.data row index). */
+    stepIteration?: number;
+    /** Total step-level iterations for the current step. */
+    stepIterations?: number;
+}
+/** Iteration shape of a run: how many times the sequence and each step run. */
+interface ScenarioIterations {
+    /** Number of full sequence runs (definition.data rows, defaults to 1). */
+    scenarioCount: number;
+    /** Number of executions per step index (step.request.data rows, >=1). */
+    stepCounts: number[];
+}
+/** Aggregate result of a full scenario execution. */
+interface ScenarioReport {
+    name: string;
+    description?: string;
+    status: ScenarioStatus;
+    summary: TestSummary;
+    steps: ScenarioStepResult[];
+    /** Final shared variable scope. */
+    variables: Record<string, string>;
+    /** Declared execution grid (iteration counts) for report grouping. */
+    iterations?: ScenarioIterations;
+    config: CliConfig;
+    startedAt: string;
+    durationMs: number;
+    generatedAt: string;
+    version: string;
+}
+/** Options for {@link runScenario}. */
+interface RunScenarioOptions {
+    /** Runner configuration (server, auth, proxy, TLS, timeouts, ...). */
+    config: CliConfig;
+    /** Pre-parsed spec document; when omitted it is loaded from config.specPath. */
+    spec?: unknown;
+    /** Live progress callback, invoked defensively (a throw never aborts the run). */
+    onEvent?: (event: ScenarioEvent) => void;
+    /** Aborting cancels the scenario; remaining steps are reported as skipped. */
+    signal?: AbortSignal;
+}
 
+/**
+ * Batch test runner. Iterates over all operations in an OpenAPI spec,
+ * executes each via @powerduck/openapi-request, runs assertions, and collects
+ * results. Supports HTTP, SSE, GraphQL, gRPC (unary/server-streaming),
+ * MCP (one-shot), and WebSocket (connect-send-close).
+ */
+
+interface OperationInfo {
+    path: string;
+    method: string;
+    operation: any;
+    pathItem: any;
+    operationId: string;
+    protocol: string;
+}
+/** Extract a readable message from any assertion error shape. */
+declare function formatAssertionError(err: unknown): string | undefined;
+/**
+ * Build implicit assertions when the user defined none.
+ *
+ * Conventional success criteria per protocol:
+ *   HTTP / SSE : HTTP status 2xx (200-299)
+ *   GraphQL    : HTTP status 2xx AND no top-level "errors" array in the body
+ *   gRPC       : status code 0 (OK)
+ *   MCP        : JSON-RPC response has a "result" and no "error"
+ *   WebSocket  : connection reached the "open" state
+ */
+declare function buildImplicitAssertions(protocol: string, response: TestResult["response"]): Array<{
+    name: string;
+    passed: boolean;
+    error?: string;
+}>;
 /**
  * Run all tests defined in the spec.
  */
 declare function runTests(config: CliConfig): Promise<TestReport>;
+declare function collectOperations(spec: any, config: CliConfig): OperationInfo[];
+interface StepExecutionOptions {
+    config: CliConfig;
+    spec: any;
+    /** Active variable scope injected as {{name}} values. */
+    variables?: Record<string, string>;
+    /** Per-step request values (path/query/header/body). */
+    values?: any;
+    /** Override the resolved server URL for this invocation. */
+    serverUrl?: string;
+    /** Declarative assertions added beyond those declared on the operation. */
+    extraAssertions?: DeclarativeAssertion[];
+}
+interface StepExecution {
+    result: TestResult;
+    response: TestResult["response"];
+    /** Resolved request snapshot for the result card. */
+    request?: TestResult["request"];
+    /** Variable scope produced by Postman scripts during this step. */
+    env: Record<string, string>;
+}
+/**
+ * Execute a single operation with per-step overrides and return the result,
+ * the normalized response, and any variables produced by its scripts. Shared
+ * by the batch runner (runTests) and the ordered scenario engine (runScenario).
+ */
+declare function executeStep(client: ReturnType<typeof createClient>, op: OperationInfo, options: StepExecutionOptions): Promise<StepExecution>;
+declare function buildSummary(results: TestResult[], durationMs: number): TestSummary;
+
+interface ResolvedStep {
+    step: ScenarioStep;
+    op: OperationInfo;
+    ref: string;
+}
+/** Normalize a step reference to "METHOD /path". */
+declare function stepRef(step: ScenarioStep): string;
+/** Resolve every step reference up front so a bad plan fails before any call. */
+declare function resolveSteps(definition: ScenarioDefinition, operations: OperationInfo[]): ResolvedStep[];
+/** Read declared extracts out of a normalized response. Missing values are omitted. */
+declare function applyExtracts(extracts: ScenarioExtract[] | undefined, response: TestResult["response"]): Record<string, string>;
+/**
+ * Execute a scenario in order with a shared variable scope per scenario
+ * iteration. See the file header for data-driven iteration semantics.
+ */
+declare function runScenario(definition: ScenarioDefinition, options: RunScenarioOptions): Promise<ScenarioReport>;
 
 declare const DEFAULT_CONFIG: Required<Pick<CliConfig, "outputDir" | "formats" | "concurrency" | "timeout" | "failOnError" | "grpcReflection" | "mcpTransport">>;
 interface CliArgs {
@@ -168,61 +468,6 @@ declare function isRemoteSpec(specPath: string): boolean;
  */
 declare function loadSpec(config: CliConfig): Promise<any>;
 
-/**
- * Assertion engine. Supports two sources:
- * 1. Declarative `x-tests` array on the operation (simple, JSON-serializable).
- * 2. Postman `x-postman-scripts.test` string (full pm.* API, executed by
- *    @powerduck/openapi-request's script sandbox and surfaced via onAssertion).
- */
-
-interface DeclarativeAssertion {
-    name: string;
-    /** What to assert. */
-    assert: "status" | "header" | "bodyContains" | "jsonPath" | "responseTime" | "bodyEquals";
-    /** Expected status code (for assert=status). */
-    value?: number;
-    /** Header name (for assert=header). */
-    key?: string;
-    /** Substring to check (for assert=header / bodyContains). */
-    contains?: string;
-    /** JSONPath expression (for assert=jsonPath). Simple dot-notation supported. */
-    path?: string;
-    /** Expected value at path (for assert=jsonPath). */
-    equals?: unknown;
-    /** Whether the path must exist (for assert=jsonPath). */
-    exists?: boolean;
-    /** Max response time in ms (for assert=responseTime). */
-    max?: number;
-    /** Expected body string (for assert=bodyEquals). */
-    body?: string;
-}
-interface AssertionContext {
-    status?: number;
-    headers?: Record<string, string>;
-    body?: unknown;
-    bodyText?: string;
-    durationMs: number;
-}
-/**
- * Run declarative assertions against a response context.
- */
-declare function runDeclarativeAssertions(assertions: DeclarativeAssertion[], ctx: AssertionContext): AssertionResult[];
-/**
- * Minimal JSONPath resolver supporting dot notation and array indices.
- * e.g. "$.data.items[0].id" or "data.user.name"
- */
-declare function resolveJsonPath(obj: unknown, path: string): unknown;
-/**
- * Extract declarative assertions from an OpenAPI operation.
- * Looks for `x-tests` on the operation or path item.
- */
-declare function extractAssertions(operation: any, pathItem?: any): DeclarativeAssertion[];
-/**
- * Extract Postman test scripts from an operation.
- * Looks for `x-postman-scripts.test` on the operation.
- */
-declare function extractPostmanScripts(operation: any): string | undefined;
-
 declare function generateJsonReport(report: TestReport, outputDir: string): string;
 
 /**
@@ -238,4 +483,4 @@ declare function printCliReport(report: TestReport): string;
 
 declare function generateHtmlReport(report: TestReport, outputDir: string): string;
 
-export { type AssertionResult, type AuthConfig, type CliArgs, type CliConfig, DEFAULT_CONFIG, type DeclarativeAssertion, type FilterConfig, type ProtocolName, type ReportFormat, type TestReport, type TestResult, type TestStatus, type TestSummary, type TlsConfig, extractAssertions, extractPostmanScripts, generateCliReport, generateHtmlReport, generateJsonReport, isRemoteSpec, loadSpec, printCliReport, resolveConfig, resolveJsonPath, runDeclarativeAssertions, runTests };
+export { type AssertionResult, type AuthConfig, type CliArgs, type CliConfig, DEFAULT_CONFIG, type DeclarativeAssertion, type FilterConfig, type ProtocolName, type ReportFormat, type RunScenarioOptions, type ScenarioDefinition, type ScenarioEvent, type ScenarioEventType, type ScenarioExtract, type ScenarioReport, type ScenarioStatus, type ScenarioStep, type ScenarioStepRequest, type ScenarioStepResult, type TestReport, type TestResult, type TestStatus, type TestSummary, type TlsConfig, applyExtracts, buildImplicitAssertions, buildSummary, collectOperations, executeStep, extractAssertions, extractPostmanScripts, formatAssertionError, generateCliReport, generateHtmlReport, generateJsonReport, isRemoteSpec, loadSpec, printCliReport, resolveConfig, resolveJsonPath, resolveSteps, runDeclarativeAssertions, runScenario, runTests, stepRef };