@typespec/spector 0.1.0-alpha.2 → 0.1.0-alpha.20-dev.0

package/src/actions/serve.ts

@@ -1,5 +1,4 @@
 import { spawn } from "child_process";
-import fetch from "node-fetch";
 import { resolve } from "path";
 import { MockApiApp } from "../app/app.js";
 import { AdminUrls } from "../constants.js";
@@ -39,14 +38,14 @@ export async function startInBackground(config: ServeConfig) {
     const [nodeExe, entrypoint] = process.argv;
     logger.info(`Starting server in background at port ${config.port}`);
     const scenariosPath = Array.isArray(config.scenariosPath)
-      ? config.scenariosPath.join(" ")
-      : config.scenariosPath;
+      ? config.scenariosPath
+      : [config.scenariosPath];
     const cp = spawn(
       nodeExe,
       [
         entrypoint,
         "serve",
-        scenariosPath,
+        ...scenariosPath,
         "--port",
         config.port.toString(),
         "--coverageFile",

package/src/actions/server-test.ts

@@ -1,157 +1,114 @@
-import { MockApiDefinition } from "@typespec/spec-api";
-import * as fs from "fs";
-import * as path from "path";
+import {
+  expandDyns,
+  MockApiDefinition,
+  MockBody,
+  ResolverConfig,
+  ValidationError,
+} from "@typespec/spec-api";
+import deepEqual from "deep-equal";
+import micromatch from "micromatch";
+import { inspect } from "node:util";
 import pc from "picocolors";
 import { logger } from "../logger.js";
 import { loadScenarioMockApis } from "../scenarios-resolver.js";
-import { makeServiceCall, uint8ArrayToString } from "./helper.js";
+import { makeServiceCall } from "./helper.js";
 
 const DEFAULT_BASE_URL = "http://localhost:3000";
 
 export interface ServerTestDiagnostics {
-  scenario_name: string;
-  status: "success" | "failure";
-  message: any;
+  scenarioName: string;
+  message: string;
 }
 
 class ServerTestsGenerator {
   private name: string = "";
   private mockApiDefinition: MockApiDefinition;
   private serverBasePath: string = "";
+  private resolverConfig: ResolverConfig;
 
   constructor(name: string, mockApiDefinition: MockApiDefinition, serverBasePath: string) {
     this.name = name;
     this.mockApiDefinition = mockApiDefinition;
     this.serverBasePath = serverBasePath;
-  }
-
-  private getConfigObj() {
-    let config = {};
-    if (this.mockApiDefinition.request.status) {
-      const validStatusCode = this.mockApiDefinition.request.status;
-      config = {
-        validateStatus: function (status: number) {
-          return (status >= 200 && status < 300) || validStatusCode === status;
-        },
-      };
-    }
-    if (this.mockApiDefinition.request.params) {
-      config = {
-        ...config,
-        params: this.mockApiDefinition.request.params,
-      };
-    }
-    if (this.mockApiDefinition.request.headers) {
-      config = {
-        ...config,
-        headers: this.mockApiDefinition.request.headers,
-      };
-    }
-    if (
-      ["head", "get", "delete"].includes(this.mockApiDefinition.method) &&
-      this.mockApiDefinition.request.body
-    ) {
-      config = {
-        ...config,
-        data: this.mockApiDefinition.request.body,
-      };
-    }
-    return config;
+    this.resolverConfig = {
+      baseUrl: serverBasePath,
+    };
   }
 
   public async executeScenario() {
-    logger.info(`Executing ${this.name} endpoint - Method: ${this.mockApiDefinition.method}`);
-
-    const response = await makeServiceCall(this.mockApiDefinition.method, {
-      endPoint: `${this.serverBasePath}${this.mockApiDefinition.uri}`,
-      options: {
-        requestBody: this.mockApiDefinition.request.body,
-        files: this.mockApiDefinition.request.files,
-        config: this.getConfigObj(),
+    log(`Executing ${this.name} endpoint - Method: ${this.mockApiDefinition.method}`);
+
+    const response = await makeServiceCall(
+      {
+        method: this.mockApiDefinition.method,
+        url: `${this.serverBasePath}${this.mockApiDefinition.uri}`,
+        body: this.mockApiDefinition.request?.body,
+        headers: this.mockApiDefinition.request?.headers,
+        query: this.mockApiDefinition.request?.query,
+        pathParams: this.mockApiDefinition.request?.pathParams,
       },
-    });
+      this.resolverConfig,
+    );
 
     if (this.mockApiDefinition.response.status !== response.status) {
-      logger.error(`Status code mismatch for ${this.name} endpoint`);
-      logger.error(
-        `Expected: ${this.mockApiDefinition.response.status} - Actual: ${response.status}`,
+      throw new ValidationError(
+        "Status code mismatch",
+        this.mockApiDefinition.response.status,
+        response.status,
       );
-      throw new Error(`Status code mismatch for ${this.name} endpoint`);
     }
+
     if (this.mockApiDefinition.response.body) {
-      if (this.mockApiDefinition.response.body.contentType === "application/xml") {
-        if (
-          JSON.stringify(this.mockApiDefinition.response.body.rawContent) !==
-          JSON.stringify(response.data)
-        ) {
-          logger.error(`Response data mismatch for ${this.name} endpoint`);
-          logger.error(
-            `Expected: ${this.mockApiDefinition.response.body["rawContent"]} - Actual: ${response.data}`,
-          );
-          throw new Error(`Response data mismatch for ${this.name} endpoint`);
-        }
-      } else if (Buffer.isBuffer(this.mockApiDefinition.response.body.rawContent)) {
-        if (
-          this.mockApiDefinition.request.headers &&
-          this.mockApiDefinition.request.headers["accept"] === "application/json"
-        ) {
-          if (
-            response.data.content !==
-            this.mockApiDefinition.response.body.rawContent.toString("base64")
-          ) {
-            throw new Error(`Response data mismatch for ${this.name} endpoint`);
-          }
-        } else {
-          if (
-            uint8ArrayToString(response.data, "utf-8") !==
-            this.mockApiDefinition.response.body.rawContent.toString()
-          ) {
-            throw new Error(`Response data mismatch for ${this.name} endpoint`);
-          }
-        }
-      } else if (this.mockApiDefinition.response.body.contentType === "text/plain") {
-        if (this.mockApiDefinition.response.body.rawContent !== response.data) {
-          logger.error(`Response data mismatch for ${this.name} endpoint`);
-          logger.error(
-            `Expected: ${this.mockApiDefinition.response.body} - Actual: ${response.data}`,
-          );
-          throw new Error(`Response data mismatch for ${this.name} endpoint`);
-        }
-      } else {
-        const responseData = JSON.stringify(response.data);
-        if (
-          this.mockApiDefinition.response.body.rawContent !==
-          responseData.replace(this.serverBasePath, "")
-        ) {
-          logger.error(`Response data mismatch for ${this.name} endpoint`);
-          logger.error(
-            `Expected: ${this.mockApiDefinition.response.body} - Actual: ${response.data}`,
-          );
-          throw new Error(`Response data mismatch for ${this.name} endpoint`);
-        }
-      }
+      await this.#validateBody(response, this.mockApiDefinition.response.body);
     }
+
     if (this.mockApiDefinition.response.headers) {
-      for (const key in this.mockApiDefinition.response.headers) {
-        if (
-          this.mockApiDefinition.response.headers[key] !==
-          response.headers[key].replace(this.serverBasePath, "")
-        ) {
-          logger.error(`Response headers mismatch for ${this.name} endpoint`);
-          logger.error(
-            `Expected: ${this.mockApiDefinition.response.headers[key]} - Actual: ${response.headers[key]}`,
+      const headers = expandDyns(this.mockApiDefinition.response.headers, this.resolverConfig);
+      for (const key in headers) {
+        if (headers[key] !== response.headers.get(key)) {
+          throw new ValidationError(
+            `Response headers mismatch`,
+            headers[key],
+            response.headers.get(key),
           );
-          throw new Error(`Response headers mismatch for ${this.name} endpoint`);
         }
       }
     }
   }
+
+  async #validateBody(response: Response, body: MockBody) {
+    if (Buffer.isBuffer(body.rawContent)) {
+      const responseData = Buffer.from(await response.arrayBuffer());
+      if (!deepEqual(responseData, body.rawContent)) {
+        throw new ValidationError(`Raw body mismatch`, body.rawContent, responseData);
+      }
+    } else {
+      const responseData = await response.text();
+      const raw =
+        typeof body.rawContent === "string"
+          ? body.rawContent
+          : body.rawContent?.serialize(this.resolverConfig);
+      switch (body.contentType) {
+        case "application/xml":
+        case "text/plain":
+          if (body.rawContent !== responseData) {
+            throw new ValidationError("Response data mismatch", raw, responseData);
+          }
+          break;
+        case "application/json":
+          const expected = JSON.parse(raw as any);
+          const actual = JSON.parse(responseData);
+          if (!deepEqual(actual, expected, { strict: true })) {
+            throw new ValidationError("Response data mismatch", expected, actual);
+          }
+      }
+    }
+  }
 }
 
 export interface ServerTestOptions {
   baseUrl?: string;
-  runSingleScenario?: string;
-  runScenariosFromFile?: string;
+  filter?: string;
 }
 
 async function delay(ms: number) {
@@ -159,7 +116,7 @@ async function delay(ms: number) {
 }
 
 async function waitForServer(baseUrl: string) {
-  logger.info(`Executing server tests with base URL: ${baseUrl}`);
+  logger.debug(`Executing server tests with base URL: ${baseUrl}`);
   let retry = 0;
 
   while (retry < 3) {
@@ -178,40 +135,44 @@ async function waitForServer(baseUrl: string) {
 export async function serverTest(scenariosPath: string, options: ServerTestOptions = {}) {
   const baseUrl = options.baseUrl ?? DEFAULT_BASE_URL;
   await waitForServer(baseUrl);
-  // 1. Get Testcases to run
-  const testCasesToRun: string[] = [];
-  if (options.runSingleScenario) {
-    testCasesToRun.push(options.runSingleScenario);
-  } else if (options.runScenariosFromFile) {
-    const data = fs.readFileSync(path.resolve(options.runScenariosFromFile), "utf8");
-    const lines = data.split("\n");
-    lines.forEach((line) => {
-      testCasesToRun.push(line.trim());
-    });
-  }
-  // 2. Load all the scenarios
   const scenarios = await loadScenarioMockApis(scenariosPath);
-  const success_diagnostics: ServerTestDiagnostics[] = [];
-  const failure_diagnostics: ServerTestDiagnostics[] = [];
-
+  const successfullScenarios: { name: string }[] = [];
+  const failureDiagnostics: ServerTestDiagnostics[] = [];
+
+  const allScenarioEntries = Object.entries(scenarios);
+  const scenarioEntries = allScenarioEntries.filter(([name]) => {
+    const pathlikeName = name.replaceAll("_", "/").toLowerCase();
+    const filter = options.filter?.toLowerCase();
+    if (filter && !micromatch.isMatch(pathlikeName, filter)) {
+      logger.debug(`Skipping scenario: ${pathlikeName}, does not match filter: ${filter}`);
+      return false;
+    }
+    return true;
+  });
   // 3. Execute each scenario
-  for (const [name, scenario] of Object.entries(scenarios)) {
+  for (const [name, scenario] of scenarioEntries) {
     if (!Array.isArray(scenario.apis)) continue;
     for (const api of scenario.apis) {
       if (api.kind !== "MockApiDefinition") continue;
-      if (testCasesToRun.length === 0 || testCasesToRun.includes(name)) {
-        const obj: ServerTestsGenerator = new ServerTestsGenerator(name, api, baseUrl);
-        try {
-          await obj.executeScenario();
-          success_diagnostics.push({
-            scenario_name: name,
-            status: "success",
-            message: "executed successfully",
+      const obj: ServerTestsGenerator = new ServerTestsGenerator(name, api, baseUrl);
+      try {
+        await obj.executeScenario();
+        successfullScenarios.push({
+          name,
+        });
+      } catch (e: any) {
+        if (e instanceof ValidationError) {
+          failureDiagnostics.push({
+            scenarioName: name,
+            message: [
+              `Validation failed: ${e.message}:`,
+              ` Expected:\n    ${inspect(e.expected)}`,
+              ` Actual:\n    ${inspect(e.actual)}`,
+            ].join("\n"),
           });
-        } catch (e: any) {
-          failure_diagnostics.push({
-            scenario_name: name,
-            status: "failure",
+        } else {
+          failureDiagnostics.push({
+            scenarioName: name,
             message: `code = ${e.code} \n message = ${e.message} \n name = ${e.name} \n stack = ${e.stack} \n status = ${e.status}`,
           });
         }
@@ -220,20 +181,35 @@ export async function serverTest(scenariosPath: string, options: ServerTestOptio
   }
 
   // 4. Print diagnostics
-  logger.info("Server Tests Diagnostics Summary");
+  log("");
+  log("Server Tests Diagnostics Summary");
+
+  if (successfullScenarios.length === 0 && failureDiagnostics.length === 0) {
+    logger.error("No scenarios were executed");
+    process.exit(-1);
+  }
 
-  if (success_diagnostics.length > 0) logger.info("Success Scenarios");
-  success_diagnostics.forEach((diagnostic) => {
-    logger.info(`${pc.green("✓")} Scenario: ${diagnostic.scenario_name} - ${diagnostic.message}`);
+  if (successfullScenarios.length > 0) log("Successfull scenarios");
+  successfullScenarios.forEach((diagnostic) => {
+    log(`${pc.green("✓")} Scenario: ${pc.cyan(diagnostic.name)}`);
   });
 
-  if (failure_diagnostics.length > 0) logger.error("Failure Scenarios");
-  if (failure_diagnostics.length > 0) {
-    logger.error("Failed Scenario details");
-    failure_diagnostics.forEach((diagnostic) => {
-      logger.error(`${pc.red("✘")} Scenario: ${diagnostic.scenario_name}`);
-      logger.error(`${diagnostic.message}`);
+  if (failureDiagnostics.length > 0) {
+    log("Failed scenarios");
+    failureDiagnostics.forEach((diagnostic) => {
+      log(`${pc.red("✘")} Scenario: ${pc.cyan(diagnostic.scenarioName)}`);
+      log(`${diagnostic.message}`);
     });
-    process.exit(-1);
   }
+  log(pc.bold(pc.green(`✓ ${scenarioEntries.length} passed`)));
+  if (failureDiagnostics.length > 0) {
+    log(pc.red(`✘ ${failureDiagnostics.length} failed`));
+  }
+
+  process.exit(failureDiagnostics.length > 0 ? 1 : 0);
+}
+
+function log(message: string) {
+  // eslint-disable-next-line no-console
+  console.log(message);
 }

package/src/actions/upload-coverage-report.ts

@@ -11,6 +11,7 @@ export interface UploadCoverageReportConfig {
   generatorVersion: string;
   generatorCommit?: string;
   generatorMode: string;
+  containerName: string;
 }
 
 export async function uploadCoverageReport({
@@ -20,11 +21,16 @@ export async function uploadCoverageReport({
   generatorVersion,
   generatorCommit: geenratorCommit,
   generatorMode,
+  containerName,
 }: UploadCoverageReportConfig) {
   const content = await readFile(coverageFile);
   const coverage: CoverageReport = JSON.parse(content.toString());
 
-  const client = new SpecCoverageClient(storageAccountName, new AzureCliCredential());
+  const client = new SpecCoverageClient(storageAccountName, {
+    credential: new AzureCliCredential(),
+    containerName,
+  });
+  await client.createIfNotExists();
   const generatorMetadata: GeneratorMetadata = {
     name: generatorName,
     version: generatorVersion,

package/src/actions/upload-scenario-manifest.ts

@@ -9,27 +9,31 @@ import { logger } from "../logger.js";
 export interface UploadScenarioManifestConfig {
   scenariosPaths: string[];
   storageAccountName: string;
-  setName: string;
+  setNames: string[];
+  containerName: string;
 }
 
 export async function uploadScenarioManifest({
   scenariosPaths,
   storageAccountName,
-  setName,
+  setNames,
+  containerName,
 }: UploadScenarioManifestConfig) {
   const manifests = [];
-  for (const scenariosPath of scenariosPaths) {
-    const path = resolve(process.cwd(), scenariosPath);
+  for (let idx = 0; idx < scenariosPaths.length; idx++) {
+    const path = resolve(process.cwd(), scenariosPaths[idx]);
     logger.info(`Computing scenario manifest for ${path}`);
-    const [manifest, diagnostics] = await computeScenarioManifest(path, setName);
+    const [manifest, diagnostics] = await computeScenarioManifest(path, setNames[idx]);
     if (manifest === undefined || diagnostics.length > 0) {
       process.exit(-1);
     }
     manifests.push(manifest);
   }
-
   await writeFile("manifest.json", JSON.stringify(manifests, null, 2));
-  const client = new SpecCoverageClient(storageAccountName, new AzureCliCredential());
+  const client = new SpecCoverageClient(storageAccountName, {
+    credential: new AzureCliCredential(),
+    containerName,
+  });
   await client.createIfNotExists();
   await client.manifest.upload(manifests);
 

package/src/actions/validate-mock-apis.ts

@@ -41,7 +41,7 @@ export async function validateMockApis({
     );
 
     if (programDiagnostics.length > 0) {
-      specCompiler.logDiagnostics(programDiagnostics, { log: logger.error });
+      specCompiler.logDiagnostics(programDiagnostics, specCompiler.NodeHost.logSink);
       diagnostics.reportDiagnostic({
         message: `Scenario ${name} is invalid.`,
       });

package/src/app/app.ts

@@ -1,4 +1,13 @@
-import { MockApiDefinition, MockRequest, RequestExt, ScenarioMockApi } from "@typespec/spec-api";
+import {
+  expandDyns,
+  MockApiDefinition,
+  MockBody,
+  MockMultipartBody,
+  MockRequest,
+  RequestExt,
+  ResolverConfig,
+  ScenarioMockApi,
+} from "@typespec/spec-api";
 import { ScenariosMetadata } from "@typespec/spec-coverage-sdk";
 import { Response, Router } from "express";
 import { getScenarioMetadata } from "../coverage/common.js";
@@ -19,6 +28,7 @@ export class MockApiApp {
   private router = Router();
   private server: MockApiServer;
   private coverageTracker: CoverageTracker;
+  private resolverConfig!: ResolverConfig;
 
   constructor(private config: ApiMockAppConfig) {
     this.server = new MockApiServer({ port: config.port });
@@ -54,79 +64,77 @@ export class MockApiApp {
     });
 
     this.server.use("/", this.router);
-    this.server.start();
+    // Getting the resolved port as setting 0 in the config will have express resolve on of the available ports
+    const port = await this.server.start();
+    this.resolverConfig = {
+      baseUrl: `http://localhost:${port}`,
+    };
   }
 
   private registerScenario(name: string, scenario: ScenarioMockApi) {
     for (const endpoint of scenario.apis) {
-      if (endpoint.kind !== "MockApiDefinition") {
-        this.router.route(endpoint.uri)[endpoint.method]((req: RequestExt, res: Response) => {
-          processRequest(
-            this.coverageTracker,
-            name,
-            endpoint.uri,
-            req,
-            res,
-            endpoint.handler,
-          ).catch((e) => {
-            logger.error("Unexpected request error", e);
-            res.status(500).end();
-          });
-        });
-      } else {
-        if (!endpoint.handler) {
-          endpoint.handler = createHandler(endpoint);
-        }
-        this.router.route(endpoint.uri)[endpoint.method]((req: RequestExt, res: Response) => {
-          processRequest(
-            this.coverageTracker,
-            name,
-            endpoint.uri,
-            req,
-            res,
-            endpoint.handler!,
-          ).catch((e) => {
-            logger.error("Unexpected request error", e);
-            res.status(500).end();
-          });
-        });
+      if (!endpoint.handler) {
+        endpoint.handler = createHandler(endpoint, this.resolverConfig);
       }
+      this.router.route(endpoint.uri)[endpoint.method]((req: RequestExt, res: Response) => {
+        processRequest(
+          this.coverageTracker,
+          name,
+          endpoint.uri,
+          req,
+          res,
+          endpoint.handler!,
+          this.resolverConfig,
+        ).catch((e) => {
+          logger.error("Unexpected request error", e);
+          res.status(500).end();
+        });
+      });
     }
   }
 }
 
-function isObject(value: any): boolean {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
+function validateBody(
+  req: MockRequest,
+  body: MockBody | MockMultipartBody,
+  config: ResolverConfig,
+) {
+  if ("kind" in body) {
+    // custom handler for now.
+  } else {
+    if (Buffer.isBuffer(body.rawContent)) {
+      req.expect.rawBodyEquals(body.rawContent);
+    } else {
+      const raw =
+        typeof body.rawContent === "string" ? body.rawContent : body.rawContent?.serialize(config);
+      switch (body.contentType) {
+        case "application/json":
+          req.expect.coercedBodyEquals(JSON.parse(raw as any));
+          break;
+        case "application/xml":
+          req.expect.xmlBodyEquals(
+            (raw as any).replace(`<?xml version='1.0' encoding='UTF-8'?>`, ""),
+          );
+          break;
+        default:
+          req.expect.rawBodyEquals(raw);
+      }
+    }
+  }
 }
 
-function createHandler(apiDefinition: MockApiDefinition) {
+function createHandler(apiDefinition: MockApiDefinition, config: ResolverConfig) {
   return (req: MockRequest) => {
+    const body = apiDefinition.request?.body;
     // Validate body if present in the request
-    if (apiDefinition.request.body) {
-      if (
-        apiDefinition.request.headers &&
-        apiDefinition.request.headers["Content-Type"] === "application/xml"
-      ) {
-        req.expect.xmlBodyEquals(
-          apiDefinition.request.body.rawContent.replace(
-            `<?xml version='1.0' encoding='UTF-8'?>`,
-            "",
-          ),
-        );
-      } else {
-        if (isObject(apiDefinition.request.body)) {
-          Object.entries(apiDefinition.request.body).forEach(([key, value]) => {
-            req.expect.deepEqual(req.body[key], value);
-          });
-        } else {
-          req.expect.coercedBodyEquals(apiDefinition.request.body);
-        }
-      }
+    if (body) {
+      validateBody(req, body, config);
     }
 
     // Validate headers if present in the request
-    if (apiDefinition.request.headers) {
-      Object.entries(apiDefinition.request.headers).forEach(([key, value]) => {
+    if (apiDefinition.request?.headers) {
+      const headers = expandDyns(apiDefinition.request.headers, config);
+      Object.entries(headers).forEach(([key, value]) => {
         if (key.toLowerCase() !== "content-type") {
           if (Array.isArray(value)) {
             req.expect.deepEqual(req.headers[key], value);
@@ -137,21 +145,12 @@ function createHandler(apiDefinition: MockApiDefinition) {
       });
     }
 
-    // Validate query params if present in the request
-    if (apiDefinition.request.params) {
-      Object.entries(apiDefinition.request.params).forEach(([key, value]) => {
-        if (!req.query[key]) {
-          if (Array.isArray(value)) {
-            req.expect.deepEqual(req.params[key], value);
-          } else {
-            req.expect.deepEqual(req.params[key], String(value));
-          }
+    if (apiDefinition.request?.query) {
+      Object.entries(apiDefinition.request.query).forEach(([key, value]) => {
+        if (Array.isArray(value)) {
+          req.expect.deepEqual(req.query[key], value);
         } else {
-          if (Array.isArray(value)) {
-            req.expect.deepEqual(req.query[key], value);
-          } else {
-            req.expect.containsQueryParam(key, String(value));
-          }
+          req.expect.containsQueryParam(key, String(value));
         }
       });
     }