@typespec/spector 0.1.0-alpha.0

package/src/actions/server-test.ts

@@ -0,0 +1,198 @@
+import { MockApiDefinition } from "@typespec/spec-api";
+import * as fs from "fs";
+import * as path from "path";
+import { logger } from "../logger.js";
+import { loadScenarioMockApis } from "../scenarios-resolver.js";
+import { makeServiceCall, uint8ArrayToString } from "./helper.js";
+
+const DEFAULT_BASE_URL = "http://localhost:3000";
+
+class ServerTestsGenerator {
+  private name: string = "";
+  private mockApiDefinition: MockApiDefinition;
+  private serverBasePath: string = "";
+
+  constructor(name: string, mockApiDefinition: MockApiDefinition, serverBasePath: string) {
+    this.name = name;
+    this.mockApiDefinition = mockApiDefinition;
+    this.serverBasePath = serverBasePath;
+  }
+
+  private getConfigObj() {
+    let config = {};
+    if (this.mockApiDefinition.request.status) {
+      const validStatusCode = this.mockApiDefinition.request.status;
+      config = {
+        validateStatus: function (status: number) {
+          return (status >= 200 && status < 300) || validStatusCode === status;
+        },
+      };
+    }
+    if (this.mockApiDefinition.request.params) {
+      config = {
+        ...config,
+        params: this.mockApiDefinition.request.params,
+      };
+    }
+    if (this.mockApiDefinition.request.headers) {
+      config = {
+        ...config,
+        headers: this.mockApiDefinition.request.headers,
+      };
+    }
+    if (
+      ["head", "get", "delete"].includes(this.mockApiDefinition.method) &&
+      this.mockApiDefinition.request.body
+    ) {
+      config = {
+        ...config,
+        data: this.mockApiDefinition.request.body,
+      };
+    }
+    return config;
+  }
+
+  public async executeScenario() {
+    logger.info(`Executing ${this.name} endpoint - Method: ${this.mockApiDefinition.method}`);
+
+    const response = await makeServiceCall(this.mockApiDefinition.method, {
+      endPoint: `${this.serverBasePath}${this.mockApiDefinition.uri}`,
+      options: {
+        requestBody: this.mockApiDefinition.request.body,
+        files: this.mockApiDefinition.request.files,
+        config: this.getConfigObj(),
+      },
+    });
+
+    if (this.mockApiDefinition.response.status !== response.status) {
+      logger.error(`Status code mismatch for ${this.name} endpoint`);
+      logger.error(
+        `Expected: ${this.mockApiDefinition.response.status} - Actual: ${response.status}`,
+      );
+      throw new Error(`Status code mismatch for ${this.name} endpoint`);
+    }
+    if (this.mockApiDefinition.response.body) {
+      if (this.mockApiDefinition.response.body.contentType === "application/xml") {
+        if (
+          JSON.stringify(this.mockApiDefinition.response.body.rawContent) !==
+          JSON.stringify(response.data)
+        ) {
+          logger.error(`Response data mismatch for ${this.name} endpoint`);
+          logger.error(
+            `Expected: ${this.mockApiDefinition.response.body["rawContent"]} - Actual: ${response.data}`,
+          );
+          throw new Error(`Response data mismatch for ${this.name} endpoint`);
+        }
+      } else if (Buffer.isBuffer(this.mockApiDefinition.response.body.rawContent)) {
+        if (
+          this.mockApiDefinition.request.headers &&
+          this.mockApiDefinition.request.headers["accept"] === "application/json"
+        ) {
+          if (
+            response.data.content !==
+            this.mockApiDefinition.response.body.rawContent.toString("base64")
+          ) {
+            throw new Error(`Response data mismatch for ${this.name} endpoint`);
+          }
+        } else {
+          if (
+            uint8ArrayToString(response.data, "utf-8") !==
+            this.mockApiDefinition.response.body.rawContent.toString()
+          ) {
+            throw new Error(`Response data mismatch for ${this.name} endpoint`);
+          }
+        }
+      } else if (this.mockApiDefinition.response.body.contentType === "text/plain") {
+        if (this.mockApiDefinition.response.body.rawContent !== response.data) {
+          logger.error(`Response data mismatch for ${this.name} endpoint`);
+          logger.error(
+            `Expected: ${this.mockApiDefinition.response.body} - Actual: ${response.data}`,
+          );
+          throw new Error(`Response data mismatch for ${this.name} endpoint`);
+        }
+      } else {
+        const responseData = JSON.stringify(response.data);
+        if (
+          this.mockApiDefinition.response.body.rawContent !==
+          responseData.replace(this.serverBasePath, "")
+        ) {
+          logger.error(`Response data mismatch for ${this.name} endpoint`);
+          logger.error(
+            `Expected: ${this.mockApiDefinition.response.body} - Actual: ${response.data}`,
+          );
+          throw new Error(`Response data mismatch for ${this.name} endpoint`);
+        }
+      }
+    }
+    if (this.mockApiDefinition.response.headers) {
+      for (const key in this.mockApiDefinition.response.headers) {
+        if (
+          this.mockApiDefinition.response.headers[key] !==
+          response.headers[key].replace(this.serverBasePath, "")
+        ) {
+          logger.error(`Response headers mismatch for ${this.name} endpoint`);
+          logger.error(
+            `Expected: ${this.mockApiDefinition.response.headers[key]} - Actual: ${response.headers[key]}`,
+          );
+          throw new Error(`Response headers mismatch for ${this.name} endpoint`);
+        }
+      }
+    }
+  }
+}
+
+export interface ServerTestOptions {
+  baseUrl?: string;
+  runSingleScenario?: string;
+  runScenariosFromFile?: string;
+}
+
+async function delay(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function waitForServer(baseUrl: string) {
+  logger.info(`Executing server tests with base URL: ${baseUrl}`);
+  let retry = 0;
+
+  while (retry < 3) {
+    try {
+      await fetch(baseUrl);
+      break;
+    } catch (e) {
+      retry++;
+      logger.info("Retrying...");
+      await delay(retry * 1000);
+    }
+  }
+  logger.info(`  ${baseUrl} is ready!`);
+}
+
+export async function serverTest(scenariosPath: string, options: ServerTestOptions = {}) {
+  const baseUrl = options.baseUrl ?? DEFAULT_BASE_URL;
+  await waitForServer(baseUrl);
+  // 1. Get Testcases to run
+  const testCasesToRun: string[] = [];
+  if (options.runSingleScenario) {
+    testCasesToRun.push(options.runSingleScenario);
+  } else if (options.runScenariosFromFile) {
+    const data = fs.readFileSync(path.resolve(options.runScenariosFromFile), "utf8");
+    const lines = data.split("\n");
+    lines.forEach((line) => {
+      testCasesToRun.push(line.trim());
+    });
+  }
+  // 2. Load all the scenarios
+  const scenarios = await loadScenarioMockApis(scenariosPath);
+  // 3. Execute each scenario
+  for (const [name, scenario] of Object.entries(scenarios)) {
+    if (!Array.isArray(scenario.apis)) continue;
+    for (const api of scenario.apis) {
+      if (api.kind !== "MockApiDefinition") continue;
+      if (testCasesToRun.length === 0 || testCasesToRun.includes(name)) {
+        const obj: ServerTestsGenerator = new ServerTestsGenerator(name, api, baseUrl);
+        await obj.executeScenario();
+      }
+    }
+  }
+}

package/src/actions/upload-coverage-report.ts

@@ -0,0 +1,41 @@
+import { AzureCliCredential } from "@azure/identity";
+import { CoverageReport, GeneratorMetadata, SpecCoverageClient } from "@typespec/spec-coverage-sdk";
+import { readFile } from "fs/promises";
+import pc from "picocolors";
+import { logger } from "../logger.js";
+
+export interface UploadCoverageReportConfig {
+  coverageFile: string;
+  storageAccountName: string;
+  generatorName: string;
+  generatorVersion: string;
+  generatorCommit?: string;
+  generatorMode: string;
+}
+
+export async function uploadCoverageReport({
+  coverageFile,
+  storageAccountName,
+  generatorName,
+  generatorVersion,
+  generatorCommit: geenratorCommit,
+  generatorMode,
+}: UploadCoverageReportConfig) {
+  const content = await readFile(coverageFile);
+  const coverage: CoverageReport = JSON.parse(content.toString());
+
+  const client = new SpecCoverageClient(storageAccountName, new AzureCliCredential());
+  const generatorMetadata: GeneratorMetadata = {
+    name: generatorName,
+    version: generatorVersion,
+    mode: generatorMode,
+    commit: geenratorCommit,
+  };
+  await client.coverage.upload(generatorMetadata, coverage);
+
+  logger.info(
+    `${pc.green(
+      "✓",
+    )} Scenario coverage file "${coverageFile}" uploaded to ${storageAccountName} storage account for ${generatorName}@${generatorVersion}.`,
+  );
+}

package/src/actions/upload-scenario-manifest.ts

@@ -0,0 +1,30 @@
+import { AzureCliCredential } from "@azure/identity";
+import { SpecCoverageClient } from "@typespec/spec-coverage-sdk";
+import { writeFile } from "fs/promises";
+import pc from "picocolors";
+import { computeScenarioManifest } from "../coverage/scenario-manifest.js";
+import { logger } from "../logger.js";
+
+export interface UploadScenarioManifestConfig {
+  scenariosPath: string;
+  storageAccountName: string;
+}
+
+export async function uploadScenarioManifest({
+  scenariosPath,
+  storageAccountName,
+}: UploadScenarioManifestConfig) {
+  const [manifest, diagnostics] = await computeScenarioManifest(scenariosPath);
+  if (manifest === undefined || diagnostics.length > 0) {
+    process.exit(-1);
+  }
+
+  await writeFile("manifest.json", JSON.stringify(manifest, null, 2));
+  const client = new SpecCoverageClient(storageAccountName, new AzureCliCredential());
+  await client.createIfNotExists();
+  await client.manifest.upload(manifest);
+
+  logger.info(
+    `${pc.green("✓")} Scenario manifest uploaded to ${storageAccountName} storage account.`,
+  );
+}

package/src/actions/validate-mock-apis.ts

@@ -0,0 +1,91 @@
+import pc from "picocolors";
+import { logger } from "../logger.js";
+import { findScenarioSpecFiles, loadScenarioMockApiFiles } from "../scenarios-resolver.js";
+import { importSpecExpect, importTypeSpec } from "../spec-utils/import-spec.js";
+import { createDiagnosticReporter } from "../utils/diagnostic-reporter.js";
+
+export interface ValidateMockApisConfig {
+  scenariosPath: string;
+  exitDueToPreviousError?: boolean;
+  hasMoreScenarios?: boolean;
+}
+
+export async function validateMockApis({
+  scenariosPath,
+  exitDueToPreviousError,
+  hasMoreScenarios,
+}: ValidateMockApisConfig) {
+  const mockApis = await loadScenarioMockApiFiles(scenariosPath);
+  const scenarioFiles = await findScenarioSpecFiles(scenariosPath);
+
+  const specCompiler = await importTypeSpec(scenariosPath);
+  const specExpect = await importSpecExpect(scenariosPath);
+  const diagnostics = createDiagnosticReporter();
+  for (const { name, specFilePath } of scenarioFiles) {
+    logger.debug(`Found scenario "${specFilePath}"`);
+    const program = await specCompiler.compile(specCompiler.NodeHost, specFilePath, {
+      noEmit: true,
+      warningAsError: true,
+    });
+
+    // Workaround https://github.com/Azure/cadl-azure/issues/2458
+    const programDiagnostics = program.diagnostics.filter(
+      (d) =>
+        !(
+          d.code === "@azure-tools/typespec-azure-core/casing-style" &&
+          typeof d.target === "object" &&
+          "kind" in d.target &&
+          d.target.kind === "Namespace" &&
+          d.target.name === "DPG"
+        ),
+    );
+
+    if (programDiagnostics.length > 0) {
+      specCompiler.logDiagnostics(programDiagnostics, { log: logger.error });
+      diagnostics.reportDiagnostic({
+        message: `Scenario ${name} is invalid.`,
+      });
+      continue;
+    }
+
+    const mockApiFile = mockApis.find((x) => x.path.endsWith(`/${name}/mockapi.js`));
+    if (mockApiFile === undefined) {
+      diagnostics.reportDiagnostic({
+        message: `Scenario ${name} is missing a mockapi file. Make sure to have a mockapi.ts that is built.`,
+      });
+      logger.debug(`Expected mock api file at "${name}/mockapi.js"`);
+
+      continue;
+    }
+
+    const scenarios = specExpect.listScenarios(program);
+
+    let foundFailure = false;
+    for (const scenario of scenarios) {
+      if (mockApiFile.scenarios[scenario.name] === undefined) {
+        foundFailure = true;
+        diagnostics.reportDiagnostic({
+          message: `Scenario ${scenario.name} is missing implementation in for ${name} scenario file.`,
+        });
+      }
+    }
+
+    if (!foundFailure) {
+      logger.info(`${pc.green("✓")} Scenario ${name} has all implemented mock apis.`);
+    }
+  }
+
+  if (diagnostics.diagnostics.length === 0) {
+    if (exitDueToPreviousError && !hasMoreScenarios) {
+      process.exit(1);
+    }
+    if (exitDueToPreviousError) return exitDueToPreviousError;
+    else return false;
+  } else {
+    if (hasMoreScenarios) {
+      return true;
+    } else {
+      process.exit(1);
+    }
+  }
+}

package/src/actions/validate-scenarios.ts

@@ -0,0 +1,32 @@
+import pc from "picocolors";
+import { logger } from "../logger.js";
+import { loadScenarios } from "../scenarios-resolver.js";
+
+export interface ValidateScenarioConfig {
+  scenariosPath: string;
+  exitDueToPreviousError?: boolean;
+  hasMoreScenarios?: boolean;
+}
+
+export async function validateScenarios({
+  scenariosPath,
+  exitDueToPreviousError,
+  hasMoreScenarios,
+}: ValidateScenarioConfig) {
+  const [_, diagnostics] = await loadScenarios(scenariosPath);
+
+  if (diagnostics.length === 0) {
+    logger.info(`${pc.green("✓")} All scenarios are valid.`);
+    if (exitDueToPreviousError && !hasMoreScenarios) {
+      process.exit(-1);
+    }
+    if (exitDueToPreviousError) return exitDueToPreviousError;
+    else return false;
+  } else {
+    if (hasMoreScenarios) {
+      return true;
+    } else {
+      process.exit(-1);
+    }
+  }
+}

package/src/app/app.ts

@@ -0,0 +1,166 @@
+import { MockApiDefinition, MockRequest, RequestExt, ScenarioMockApi } from "@typespec/spec-api";
+import { ScenariosMetadata } from "@typespec/spec-coverage-sdk";
+import { Response, Router } from "express";
+import { getScenarioMetadata } from "../coverage/common.js";
+import { CoverageTracker } from "../coverage/coverage-tracker.js";
+import { logger } from "../logger.js";
+import { internalRouter } from "../routes/index.js";
+import { loadScenarioMockApis } from "../scenarios-resolver.js";
+import { MockApiServer } from "../server/index.js";
+import { ApiMockAppConfig } from "./config.js";
+import { processRequest } from "./request-processor.js";
+
+export interface ScenariosAndScenariosMetadata {
+  scenarios: Record<string, ScenarioMockApi>;
+  scenariosMetadata: ScenariosMetadata;
+}
+
+export class MockApiApp {
+  private router = Router();
+  private server: MockApiServer;
+  private coverageTracker: CoverageTracker;
+
+  constructor(private config: ApiMockAppConfig) {
+    this.server = new MockApiServer({ port: config.port });
+    this.coverageTracker = new CoverageTracker(config.coverageFile);
+  }
+
+  public async start(): Promise<void> {
+    this.server.use("/", internalRouter);
+
+    const ScenariosAndScenariosMetadata: ScenariosAndScenariosMetadata[] = [];
+    if (Array.isArray(this.config.scenarioPath)) {
+      for (let idx = 0; idx < this.config.scenarioPath.length; idx++) {
+        const scenarios = await loadScenarioMockApis(this.config.scenarioPath[idx]);
+        const scenariosMetadata = await getScenarioMetadata(this.config.scenarioPath[idx]);
+        ScenariosAndScenariosMetadata.push({ scenarios, scenariosMetadata });
+      }
+    } else {
+      const scenarios = await loadScenarioMockApis(this.config.scenarioPath);
+      const scenariosMetadata = await getScenarioMetadata(this.config.scenarioPath);
+      ScenariosAndScenariosMetadata.push({ scenarios, scenariosMetadata });
+    }
+
+    this.coverageTracker.setScenarios(ScenariosAndScenariosMetadata);
+
+    for (const { scenarios } of ScenariosAndScenariosMetadata) {
+      for (const [name, scenario] of Object.entries(scenarios)) {
+        this.registerScenario(name, scenario);
+      }
+    }
+
+    this.router.get("/.coverage", (req, res) => {
+      res.json(this.coverageTracker.computeCoverage());
+    });
+
+    this.server.use("/", this.router);
+    this.server.start();
+  }
+
+  private registerScenario(name: string, scenario: ScenarioMockApi) {
+    for (const endpoint of scenario.apis) {
+      if (endpoint.kind !== "MockApiDefinition") {
+        this.router.route(endpoint.uri)[endpoint.method]((req: RequestExt, res: Response) => {
+          processRequest(
+            this.coverageTracker,
+            name,
+            endpoint.uri,
+            req,
+            res,
+            endpoint.handler,
+          ).catch((e) => {
+            logger.error("Unexpected request error", e);
+            res.status(500).end();
+          });
+        });
+      } else {
+        if (!endpoint.handler) {
+          endpoint.handler = createHandler(endpoint);
+        }
+        this.router.route(endpoint.uri)[endpoint.method]((req: RequestExt, res: Response) => {
+          processRequest(
+            this.coverageTracker,
+            name,
+            endpoint.uri,
+            req,
+            res,
+            endpoint.handler!,
+          ).catch((e) => {
+            logger.error("Unexpected request error", e);
+            res.status(500).end();
+          });
+        });
+      }
+    }
+  }
+}
+
+function isObject(value: any): boolean {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function createHandler(apiDefinition: MockApiDefinition) {
+  return (req: MockRequest) => {
+    // Validate body if present in the request
+    if (apiDefinition.request.body) {
+      if (
+        apiDefinition.request.headers &&
+        apiDefinition.request.headers["Content-Type"] === "application/xml"
+      ) {
+        req.expect.xmlBodyEquals(
+          apiDefinition.request.body.rawContent.replace(
+            `<?xml version='1.0' encoding='UTF-8'?>`,
+            "",
+          ),
+        );
+      } else {
+        if (isObject(apiDefinition.request.body)) {
+          Object.entries(apiDefinition.request.body).forEach(([key, value]) => {
+            req.expect.deepEqual(req.body[key], value);
+          });
+        } else {
+          req.expect.coercedBodyEquals(apiDefinition.request.body);
+        }
+      }
+    }
+
+    // Validate headers if present in the request
+    if (apiDefinition.request.headers) {
+      Object.entries(apiDefinition.request.headers).forEach(([key, value]) => {
+        if (key.toLowerCase() !== "content-type") {
+          if (Array.isArray(value)) {
+            req.expect.deepEqual(req.headers[key], value);
+          } else {
+            req.expect.containsHeader(key.toLowerCase(), String(value));
+          }
+        }
+      });
+    }
+
+    // Validate query params if present in the request
+    if (apiDefinition.request.params) {
+      Object.entries(apiDefinition.request.params).forEach(([key, value]) => {
+        if (!req.query[key]) {
+          if (Array.isArray(value)) {
+            req.expect.deepEqual(req.params[key], value);
+          } else {
+            req.expect.deepEqual(req.params[key], String(value));
+          }
+        } else {
+          if (Array.isArray(value)) {
+            req.expect.deepEqual(req.query[key], value);
+          } else {
+            req.expect.containsQueryParam(key, String(value));
+          }
+        }
+      });
+    }
+
+    // Validations are done. Now return the response
+    return {
+      status: apiDefinition.response.status,
+      body: apiDefinition.response.body,
+      headers: apiDefinition.response.headers,
+    };
+  };
+}

package/src/app/config.ts

@@ -0,0 +1,16 @@
+export interface ApiMockAppConfig {
+  /**
+   * Mock API server port.
+   */
+  port: number;
+
+  /**
+   * Path where are the scenarios.
+   */
+  scenarioPath: string | string[];
+
+  /**
+   * Coverage file Path.
+   */
+  coverageFile: string;
+}

package/src/app/index.ts

@@ -0,0 +1,2 @@
+export * from "./app.js";
+export * from "./config.js";

package/src/app/request-processor.ts

@@ -0,0 +1,71 @@
+import {
+  MockRequest,
+  MockRequestHandler,
+  MockResponse,
+  RequestExt,
+  ValidationError,
+} from "@typespec/spec-api";
+import { Response } from "express";
+import { inspect } from "util";
+import { CoverageTracker } from "../coverage/coverage-tracker.js";
+import { logger } from "../logger.js";
+
+export async function processRequest(
+  coverageTracker: CoverageTracker,
+  scenarioName: string,
+  scenarioUri: string,
+  request: RequestExt,
+  response: Response,
+  func: MockRequestHandler,
+): Promise<void> {
+  const mockRequest = new MockRequest(request);
+  const mockResponse = await callHandler(mockRequest, response, func);
+  if (mockResponse === undefined) {
+    return;
+  }
+
+  await coverageTracker.trackEndpointResponse(scenarioName, scenarioUri, mockResponse);
+  processResponse(response, mockResponse);
+}
+
+const processResponse = (response: Response, mockResponse: MockResponse) => {
+  response.status(mockResponse.status);
+
+  if (mockResponse.headers) {
+    response.set(mockResponse.headers);
+  }
+
+  if (mockResponse.body) {
+    response.contentType(mockResponse.body.contentType).send(mockResponse.body.rawContent);
+  }
+
+  response.end();
+};
+
+const callHandler = async (
+  mockRequest: MockRequest,
+  response: Response,
+  func: MockRequestHandler,
+): Promise<MockResponse | undefined> => {
+  try {
+    return func(mockRequest);
+  } catch (e) {
+    if (!(e instanceof ValidationError)) {
+      throw e;
+    }
+
+    logger.warn(
+      [
+        `Request validation failed: ${e.message}:`,
+        ` Expected:\n${inspect(e.expected)}`,
+        ` Actual: \n${inspect(e.actual)}`,
+      ].join("\n"),
+    );
+    response
+      .status(400)
+      .contentType("application/json")
+      .send(e.toJSON ? e.toJSON() : JSON.stringify(e.message))
+      .end();
+    return undefined;
+  }
+};