@typespec/spector 0.1.0-alpha.9 → 0.1.0-dev.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typespec/spector",
-  "version": "0.1.0-alpha.9",
+  "version": "0.1.0-dev.1",
   "description": "Typespec Core Tool to validate, run mock api, collect coverage.",
   "exports": {
     ".": {
@@ -26,50 +26,44 @@
   },
   "homepage": "https://github.com/microsoft/typespec#readme",
   "dependencies": {
-    "@azure/identity": "~4.7.0",
-    "@types/js-yaml": "^4.0.5",
-    "ajv": "~8.17.1",
-    "axios": "^1.8.1",
-    "body-parser": "^1.20.3",
+    "@azure/identity": "~4.13.0",
+    "@typespec/compiler": "^1.10.0 || >= 1.11.0-dev.3",
+    "@typespec/http": "^1.10.0 || >= 1.11.0-dev.1",
+    "@typespec/rest": "^0.80.0 || >= 0.81.0-dev.0",
+    "@typespec/spec-api": "^0.1.0-alpha.13 || >= 0.1.0-alpha.14-dev.1",
+    "@typespec/spec-coverage-sdk": "^0.1.0-alpha.16 || >= 0.1.0-dev.0",
+    "@typespec/versioning": "^0.80.0 || >= 0.81.0-dev.0",
+    "ajv": "~8.18.0",
+    "body-parser": "^2.2.0",
     "deep-equal": "^2.2.0",
-    "express": "^4.21.2",
-    "express-promise-router": "^4.1.1",
-    "form-data": "^4.0.2",
-    "globby": "~14.1.0",
-    "jackspeak": "4.1.0",
-    "js-yaml": "^4.1.0",
+    "express": "^5.2.1",
+    "globby": "~16.1.0",
+    "micromatch": "^4.0.8",
     "morgan": "^1.10.0",
-    "multer": "^1.4.5-lts.1",
-    "node-fetch": "^3.3.1",
+    "multer": "^2.0.1",
     "picocolors": "~1.1.1",
     "source-map-support": "~0.5.21",
-    "winston": "^3.17.0",
     "xml2js": "^0.6.2",
-    "yargs": "~17.7.2",
-    "@typespec/compiler": "^0.67.0",
-    "@typespec/http": "^0.67.0",
-    "@typespec/spec-api": "^0.1.0-alpha.2",
-    "@typespec/rest": "^0.67.0",
-    "@typespec/spec-coverage-sdk": "^0.1.0-alpha.4",
-    "@typespec/versioning": "^0.67.0"
+    "yaml": "~2.8.2",
+    "yargs": "~18.0.0"
   },
   "devDependencies": {
     "@types/body-parser": "^1.19.2",
     "@types/deep-equal": "^1.0.1",
-    "@types/express": "^5.0.0",
-    "@types/morgan": "^1.9.4",
-    "@types/multer": "^1.4.10",
-    "@types/node": "~22.13.9",
-    "@types/node-fetch": "^2.6.12",
+    "@types/express": "^5.0.6",
+    "@types/micromatch": "^4.0.9",
+    "@types/morgan": "^1.9.9",
+    "@types/multer": "^2.0.0",
+    "@types/node": "~25.3.0",
     "@types/xml2js": "^0.4.11",
     "@types/yargs": "~17.0.33",
-    "rimraf": "~6.0.1",
-    "typescript": "~5.8.2",
-    "@typespec/tspd": "^0.46.0"
+    "@typespec/tspd": "^0.74.1 || >= 0.74.2-dev.1",
+    "rimraf": "~6.1.3",
+    "typescript": "~5.9.3"
   },
   "scripts": {
     "watch": "tsc -p ./tsconfig.build.json --watch",
-    "build": "npm run gen-extern-signature && tsc -p tsconfig.build.json",
+    "build": "pnpm gen-extern-signature && tsc -p tsconfig.build.json",
     "clean": "rimraf dist/ temp/",
     "gen-extern-signature": "tspd --enable-experimental gen-extern-signature .",
     "test": "vitest run",

package/src/actions/helper.ts

@@ -1,113 +1,97 @@
-import { HttpMethod, ServiceRequestFile } from "@typespec/spec-api";
-import axios, { AxiosRequestConfig, AxiosResponse } from "axios";
-import FormData from "form-data";
+import {
+  expandDyns,
+  HttpMethod,
+  MockBody,
+  MockMultipartBody,
+  ResolverConfig,
+} from "@typespec/spec-api";
 
 export interface ServiceRequest {
-  endPoint: string;
-  options?: {
-    requestBody?: any;
-    files?: ServiceRequestFile[];
-    config?: AxiosRequestConfig<any> | undefined;
-  };
+  method: HttpMethod;
+  url: string;
+  body?: MockBody | MockMultipartBody;
+  headers?: Record<string, unknown>;
+  query?: Record<string, unknown>;
+  pathParams?: Record<string, unknown>;
 }
 
-function checkAndAddFormDataIfRequired(request: ServiceRequest) {
-  if (request.options?.config?.headers?.["Content-Type"] === "multipart/form-data") {
-    const formData = new FormData();
-    if (request.options?.requestBody) {
-      for (const key in request.options.requestBody) {
-        formData.append(key, JSON.stringify(request.options.requestBody[key]));
-      }
-    }
-    if (request.options.files) {
-      request.options.files.forEach((file) => {
-        formData.append(`${file.fieldname}`, file.buffer, {
-          filename: file.originalname,
-          contentType: file.mimetype,
-        });
-      });
+function renderMultipartRequest(body: MockMultipartBody) {
+  const formData = new FormData();
+  if (body.parts) {
+    for (const key in body.parts) {
+      formData.append(key, JSON.stringify(body.parts[key]));
     }
-    request.options.requestBody = formData;
-    request.options.config = {
-      ...request.options.config,
-      headers: formData.getHeaders(),
-    };
-  }
-}
-
-function checkAndUpdateEndpoint(request: ServiceRequest) {
-  if (request.options?.config?.params) {
-    for (const key in request.options.config.params) {
-      request.endPoint = request.endPoint.replace(`:${key}`, request.options.config.params[key]);
-    }
-  }
-  request.endPoint = request.endPoint.replace(/\[:\]/g, ":");
-}
-
-export async function makeServiceCall(
-  serviceCallType: HttpMethod,
-  request: ServiceRequest,
-): Promise<AxiosResponse<any, any>> {
-  checkAndUpdateEndpoint(request);
-  checkAndAddFormDataIfRequired(request);
-  if (serviceCallType === "put") {
-    return await makePutCall(request);
-  }
-  if (serviceCallType === "post") {
-    return await makePostCall(request);
-  }
-  if (serviceCallType === "get") {
-    return await makeGetCall(request);
-  }
-  if (serviceCallType === "delete") {
-    return await makeDeleteCall(request);
   }
-  if (serviceCallType === "head") {
-    return await makeHeadCall(request);
+  if (body.files) {
+    body.files.forEach((file) => {
+      formData.append(
+        `${file.fieldname}`,
+        new Blob([file.buffer as any], { type: file.mimetype }),
+        file.originalname,
+      );
+    });
   }
-  return await makePatchCall(request);
-}
 
-export async function makePutCall(request: ServiceRequest): Promise<AxiosResponse<any, any>> {
-  const response = await axios.put(
-    request.endPoint,
-    request.options?.requestBody,
-    request.options?.config,
-  );
-  return response;
+  return formData;
 }
 
-export async function makePostCall(request: ServiceRequest): Promise<AxiosResponse<any, any>> {
-  const response = await axios.post(
-    request.endPoint,
-    request.options?.requestBody,
-    request.options?.config,
-  );
-  return response;
-}
+function resolveUrl(request: ServiceRequest) {
+  let endpoint = request.url;
 
-export async function makeGetCall(request: ServiceRequest): Promise<AxiosResponse<any, any>> {
-  const response = await axios.get(request.endPoint, request.options?.config);
-  return response;
-}
+  if (request.pathParams) {
+    for (const [key, value] of Object.entries(request.pathParams)) {
+      endpoint = endpoint.replaceAll(`:${key}`, String(value));
+    }
+  }
 
-export async function makePatchCall(request: ServiceRequest): Promise<AxiosResponse<any, any>> {
-  const response = await axios.patch(
-    request.endPoint,
-    request.options?.requestBody,
-    request.options?.config,
-  );
-  return response;
-}
+  endpoint = endpoint.replaceAll("\\:", ":");
 
-export async function makeDeleteCall(request: ServiceRequest): Promise<AxiosResponse<any, any>> {
-  const response = await axios.delete(request.endPoint, request.options?.config);
-  return response;
+  if (request.query) {
+    const query = new URLSearchParams();
+    for (const [key, value] of Object.entries(request.query)) {
+      if (Array.isArray(value)) {
+        for (const v of value) {
+          query.append(key, v);
+        }
+      } else {
+        query.append(key, value as any);
+      }
+    }
+    endpoint = `${endpoint}?${query.toString()}`;
+  }
+  return endpoint;
 }
 
-export async function makeHeadCall(request: ServiceRequest): Promise<AxiosResponse<any, any>> {
-  const response = await axios.head(request.endPoint, request.options?.config);
-  return response;
+export async function makeServiceCall(
+  request: ServiceRequest,
+  config: ResolverConfig,
+): Promise<Response> {
+  const url = resolveUrl(request);
+  let body;
+  let headers = expandDyns(request.headers, config) as Record<string, string>;
+  if (request.body) {
+    if ("kind" in request.body) {
+      const formData = renderMultipartRequest(request.body);
+      body = formData;
+    } else {
+      if (typeof request.body.rawContent === "string" || Buffer.isBuffer(request.body.rawContent)) {
+        body = request.body.rawContent as any;
+      } else {
+        body = request.body.rawContent?.serialize(config);
+      }
+      headers = {
+        ...headers,
+        ...(request.body?.contentType && {
+          "Content-Type": request.body.contentType,
+        }),
+      };
+    }
+  }
+  return await fetch(url, {
+    method: request.method.toUpperCase(),
+    body,
+    headers,
+  });
 }
 
 type EncodingType = "utf-8" | "base64" | "base64url" | "hex";

package/src/actions/serve.ts

@@ -1,5 +1,4 @@
 import { spawn } from "child_process";
-import fetch from "node-fetch";
 import { resolve } from "path";
 import { MockApiApp } from "../app/app.js";
 import { AdminUrls } from "../constants.js";

package/src/actions/server-test.ts

@@ -1,157 +1,114 @@
-import { MockApiDefinition } from "@typespec/spec-api";
-import * as fs from "fs";
-import * as path from "path";
+import {
+  expandDyns,
+  MockApiDefinition,
+  MockBody,
+  ResolverConfig,
+  ValidationError,
+} from "@typespec/spec-api";
+import deepEqual from "deep-equal";
+import micromatch from "micromatch";
+import { inspect } from "node:util";
 import pc from "picocolors";
 import { logger } from "../logger.js";
 import { loadScenarioMockApis } from "../scenarios-resolver.js";
-import { makeServiceCall, uint8ArrayToString } from "./helper.js";
+import { makeServiceCall } from "./helper.js";
 
 const DEFAULT_BASE_URL = "http://localhost:3000";
 
 export interface ServerTestDiagnostics {
-  scenario_name: string;
-  status: "success" | "failure";
-  message: any;
+  scenarioName: string;
+  message: string;
 }
 
 class ServerTestsGenerator {
   private name: string = "";
   private mockApiDefinition: MockApiDefinition;
   private serverBasePath: string = "";
+  private resolverConfig: ResolverConfig;
 
   constructor(name: string, mockApiDefinition: MockApiDefinition, serverBasePath: string) {
     this.name = name;
     this.mockApiDefinition = mockApiDefinition;
     this.serverBasePath = serverBasePath;
-  }
-
-  private getConfigObj() {
-    let config = {};
-    if (this.mockApiDefinition.request.status) {
-      const validStatusCode = this.mockApiDefinition.request.status;
-      config = {
-        validateStatus: function (status: number) {
-          return (status >= 200 && status < 300) || validStatusCode === status;
-        },
-      };
-    }
-    if (this.mockApiDefinition.request.params) {
-      config = {
-        ...config,
-        params: this.mockApiDefinition.request.params,
-      };
-    }
-    if (this.mockApiDefinition.request.headers) {
-      config = {
-        ...config,
-        headers: this.mockApiDefinition.request.headers,
-      };
-    }
-    if (
-      ["head", "get", "delete"].includes(this.mockApiDefinition.method) &&
-      this.mockApiDefinition.request.body
-    ) {
-      config = {
-        ...config,
-        data: this.mockApiDefinition.request.body,
-      };
-    }
-    return config;
+    this.resolverConfig = {
+      baseUrl: serverBasePath,
+    };
   }
 
   public async executeScenario() {
-    logger.info(`Executing ${this.name} endpoint - Method: ${this.mockApiDefinition.method}`);
-
-    const response = await makeServiceCall(this.mockApiDefinition.method, {
-      endPoint: `${this.serverBasePath}${this.mockApiDefinition.uri}`,
-      options: {
-        requestBody: this.mockApiDefinition.request.body,
-        files: this.mockApiDefinition.request.files,
-        config: this.getConfigObj(),
+    log(`Executing ${this.name} endpoint - Method: ${this.mockApiDefinition.method}`);
+
+    const response = await makeServiceCall(
+      {
+        method: this.mockApiDefinition.method,
+        url: `${this.serverBasePath}${this.mockApiDefinition.uri}`,
+        body: this.mockApiDefinition.request?.body,
+        headers: this.mockApiDefinition.request?.headers,
+        query: this.mockApiDefinition.request?.query,
+        pathParams: this.mockApiDefinition.request?.pathParams,
       },
-    });
+      this.resolverConfig,
+    );
 
     if (this.mockApiDefinition.response.status !== response.status) {
-      logger.error(`Status code mismatch for ${this.name} endpoint`);
-      logger.error(
-        `Expected: ${this.mockApiDefinition.response.status} - Actual: ${response.status}`,
+      throw new ValidationError(
+        "Status code mismatch",
+        this.mockApiDefinition.response.status,
+        response.status,
       );
-      throw new Error(`Status code mismatch for ${this.name} endpoint`);
     }
+
     if (this.mockApiDefinition.response.body) {
-      if (this.mockApiDefinition.response.body.contentType === "application/xml") {
-        if (
-          JSON.stringify(this.mockApiDefinition.response.body.rawContent) !==
-          JSON.stringify(response.data)
-        ) {
-          logger.error(`Response data mismatch for ${this.name} endpoint`);
-          logger.error(
-            `Expected: ${this.mockApiDefinition.response.body["rawContent"]} - Actual: ${response.data}`,
-          );
-          throw new Error(`Response data mismatch for ${this.name} endpoint`);
-        }
-      } else if (Buffer.isBuffer(this.mockApiDefinition.response.body.rawContent)) {
-        if (
-          this.mockApiDefinition.request.headers &&
-          this.mockApiDefinition.request.headers["accept"] === "application/json"
-        ) {
-          if (
-            response.data.content !==
-            this.mockApiDefinition.response.body.rawContent.toString("base64")
-          ) {
-            throw new Error(`Response data mismatch for ${this.name} endpoint`);
-          }
-        } else {
-          if (
-            uint8ArrayToString(response.data, "utf-8") !==
-            this.mockApiDefinition.response.body.rawContent.toString()
-          ) {
-            throw new Error(`Response data mismatch for ${this.name} endpoint`);
-          }
-        }
-      } else if (this.mockApiDefinition.response.body.contentType === "text/plain") {
-        if (this.mockApiDefinition.response.body.rawContent !== response.data) {
-          logger.error(`Response data mismatch for ${this.name} endpoint`);
-          logger.error(
-            `Expected: ${this.mockApiDefinition.response.body} - Actual: ${response.data}`,
-          );
-          throw new Error(`Response data mismatch for ${this.name} endpoint`);
-        }
-      } else {
-        const responseData = JSON.stringify(response.data);
-        if (
-          this.mockApiDefinition.response.body.rawContent !==
-          responseData.replace(this.serverBasePath, "")
-        ) {
-          logger.error(`Response data mismatch for ${this.name} endpoint`);
-          logger.error(
-            `Expected: ${this.mockApiDefinition.response.body} - Actual: ${response.data}`,
-          );
-          throw new Error(`Response data mismatch for ${this.name} endpoint`);
-        }
-      }
+      await this.#validateBody(response, this.mockApiDefinition.response.body);
     }
+
     if (this.mockApiDefinition.response.headers) {
-      for (const key in this.mockApiDefinition.response.headers) {
-        if (
-          this.mockApiDefinition.response.headers[key] !==
-          response.headers[key].replace(this.serverBasePath, "")
-        ) {
-          logger.error(`Response headers mismatch for ${this.name} endpoint`);
-          logger.error(
-            `Expected: ${this.mockApiDefinition.response.headers[key]} - Actual: ${response.headers[key]}`,
+      const headers = expandDyns(this.mockApiDefinition.response.headers, this.resolverConfig);
+      for (const key in headers) {
+        if (headers[key] !== response.headers.get(key)) {
+          throw new ValidationError(
+            `Response headers mismatch`,
+            headers[key],
+            response.headers.get(key),
           );
-          throw new Error(`Response headers mismatch for ${this.name} endpoint`);
         }
       }
     }
   }
+
+  async #validateBody(response: Response, body: MockBody) {
+    if (Buffer.isBuffer(body.rawContent)) {
+      const responseData = Buffer.from(await response.arrayBuffer());
+      if (!deepEqual(responseData, body.rawContent)) {
+        throw new ValidationError(`Raw body mismatch`, body.rawContent, responseData);
+      }
+    } else {
+      const responseData = await response.text();
+      const raw =
+        typeof body.rawContent === "string"
+          ? body.rawContent
+          : body.rawContent?.serialize(this.resolverConfig);
+      switch (body.contentType) {
+        case "application/xml":
+        case "text/plain":
+          if (raw !== responseData) {
+            throw new ValidationError("Response data mismatch", raw, responseData);
+          }
+          break;
+        case "application/json":
+          const expected = JSON.parse(raw as any);
+          const actual = JSON.parse(responseData);
+          if (!deepEqual(actual, expected, { strict: true })) {
+            throw new ValidationError("Response data mismatch", expected, actual);
+          }
+      }
+    }
+  }
 }
 
 export interface ServerTestOptions {
   baseUrl?: string;
-  runSingleScenario?: string;
-  runScenariosFromFile?: string;
+  filter?: string;
 }
 
 async function delay(ms: number) {
@@ -159,7 +116,7 @@ async function delay(ms: number) {
 }
 
 async function waitForServer(baseUrl: string) {
-  logger.info(`Executing server tests with base URL: ${baseUrl}`);
+  logger.debug(`Executing server tests with base URL: ${baseUrl}`);
   let retry = 0;
 
   while (retry < 3) {
@@ -178,40 +135,44 @@ async function waitForServer(baseUrl: string) {
 export async function serverTest(scenariosPath: string, options: ServerTestOptions = {}) {
   const baseUrl = options.baseUrl ?? DEFAULT_BASE_URL;
   await waitForServer(baseUrl);
-  // 1. Get Testcases to run
-  const testCasesToRun: string[] = [];
-  if (options.runSingleScenario) {
-    testCasesToRun.push(options.runSingleScenario);
-  } else if (options.runScenariosFromFile) {
-    const data = fs.readFileSync(path.resolve(options.runScenariosFromFile), "utf8");
-    const lines = data.split("\n");
-    lines.forEach((line) => {
-      testCasesToRun.push(line.trim());
-    });
-  }
-  // 2. Load all the scenarios
   const scenarios = await loadScenarioMockApis(scenariosPath);
-  const success_diagnostics: ServerTestDiagnostics[] = [];
-  const failure_diagnostics: ServerTestDiagnostics[] = [];
-
+  const successfullScenarios: { name: string }[] = [];
+  const failureDiagnostics: ServerTestDiagnostics[] = [];
+
+  const allScenarioEntries = Object.entries(scenarios);
+  const scenarioEntries = allScenarioEntries.filter(([name]) => {
+    const pathlikeName = name.replaceAll("_", "/").toLowerCase();
+    const filter = options.filter?.toLowerCase();
+    if (filter && !micromatch.isMatch(pathlikeName, filter)) {
+      logger.debug(`Skipping scenario: ${pathlikeName}, does not match filter: ${filter}`);
+      return false;
+    }
+    return true;
+  });
   // 3. Execute each scenario
-  for (const [name, scenario] of Object.entries(scenarios)) {
+  for (const [name, scenario] of scenarioEntries) {
     if (!Array.isArray(scenario.apis)) continue;
     for (const api of scenario.apis) {
       if (api.kind !== "MockApiDefinition") continue;
-      if (testCasesToRun.length === 0 || testCasesToRun.includes(name)) {
-        const obj: ServerTestsGenerator = new ServerTestsGenerator(name, api, baseUrl);
-        try {
-          await obj.executeScenario();
-          success_diagnostics.push({
-            scenario_name: name,
-            status: "success",
-            message: "executed successfully",
+      const obj: ServerTestsGenerator = new ServerTestsGenerator(name, api, baseUrl);
+      try {
+        await obj.executeScenario();
+        successfullScenarios.push({
+          name,
+        });
+      } catch (e: any) {
+        if (e instanceof ValidationError) {
+          failureDiagnostics.push({
+            scenarioName: name,
+            message: [
+              `Validation failed: ${e.message}:`,
+              ` Expected:\n    ${inspect(e.expected)}`,
+              ` Actual:\n    ${inspect(e.actual)}`,
+            ].join("\n"),
           });
-        } catch (e: any) {
-          failure_diagnostics.push({
-            scenario_name: name,
-            status: "failure",
+        } else {
+          failureDiagnostics.push({
+            scenarioName: name,
             message: `code = ${e.code} \n message = ${e.message} \n name = ${e.name} \n stack = ${e.stack} \n status = ${e.status}`,
           });
         }
@@ -220,20 +181,35 @@ export async function serverTest(scenariosPath: string, options: ServerTestOptio
   }
 
   // 4. Print diagnostics
-  logger.info("Server Tests Diagnostics Summary");
+  log("");
+  log("Server Tests Diagnostics Summary");
+
+  if (successfullScenarios.length === 0 && failureDiagnostics.length === 0) {
+    logger.error("No scenarios were executed");
+    process.exit(-1);
+  }
 
-  if (success_diagnostics.length > 0) logger.info("Success Scenarios");
-  success_diagnostics.forEach((diagnostic) => {
-    logger.info(`${pc.green("✓")} Scenario: ${diagnostic.scenario_name} - ${diagnostic.message}`);
+  if (successfullScenarios.length > 0) log("Successfull scenarios");
+  successfullScenarios.forEach((diagnostic) => {
+    log(`${pc.green("✓")} Scenario: ${pc.cyan(diagnostic.name)}`);
   });
 
-  if (failure_diagnostics.length > 0) logger.error("Failure Scenarios");
-  if (failure_diagnostics.length > 0) {
-    logger.error("Failed Scenario details");
-    failure_diagnostics.forEach((diagnostic) => {
-      logger.error(`${pc.red("✘")} Scenario: ${diagnostic.scenario_name}`);
-      logger.error(`${diagnostic.message}`);
+  if (failureDiagnostics.length > 0) {
+    log("Failed scenarios");
+    failureDiagnostics.forEach((diagnostic) => {
+      log(`${pc.red("✘")} Scenario: ${pc.cyan(diagnostic.scenarioName)}`);
+      log(`${diagnostic.message}`);
     });
-    process.exit(-1);
   }
+  log(pc.bold(pc.green(`✓ ${scenarioEntries.length} passed`)));
+  if (failureDiagnostics.length > 0) {
+    log(pc.red(`✘ ${failureDiagnostics.length} failed`));
+  }
+
+  process.exit(failureDiagnostics.length > 0 ? 1 : 0);
+}
+
+function log(message: string) {
+  // eslint-disable-next-line no-console
+  console.log(message);
 }