@typespec/spector 0.1.0-alpha.0

package/src/cli/cli.ts

@@ -0,0 +1,368 @@
+import { join, resolve } from "path";
+import "source-map-support/register.js";
+import yargs from "yargs";
+import { checkCoverage } from "../actions/check-coverage.js";
+import { generateScenarioSummary } from "../actions/generate-scenario-summary.js";
+import { validateScenarios } from "../actions/index.js";
+import { serve, startInBackground, stop } from "../actions/serve.js";
+import { serverTest } from "../actions/server-test.js";
+import { uploadCoverageReport } from "../actions/upload-coverage-report.js";
+import { uploadScenarioManifest } from "../actions/upload-scenario-manifest.js";
+import { validateMockApis } from "../actions/validate-mock-apis.js";
+import { logger } from "../logger.js";
+import { getCommit } from "../utils/misc-utils.js";
+
+export const DEFAULT_PORT = 3000;
+
+async function main() {
+  await yargs(process.argv.slice(2))
+    .scriptName("tsp-spector")
+    .strict()
+    .help()
+    .strict()
+    .parserConfiguration({
+      "greedy-arrays": false,
+      "boolean-negation": false,
+    })
+    .option("debug", {
+      type: "boolean",
+      description: "Output debug log messages.",
+      default: false,
+    })
+    .middleware((args) => {
+      if (args.debug) {
+        logger.level = "debug";
+      }
+    })
+    .command(
+      "validate-scenarios <scenariosPaths..>",
+      "Compile and validate all the Spec scenarios.",
+      (cmd) => {
+        return cmd.positional("scenariosPaths", {
+          description: "Path(s) to the scenarios",
+          type: "string",
+          array: true,
+          demandOption: true,
+        });
+      },
+      async (args) => {
+        let exitDueToPreviousError = false;
+        let hasMoreScenarios = true;
+        for (let idx = 0; idx < args.scenariosPaths.length; idx++) {
+          logger.info(`Validating scenarios at ${args.scenariosPaths[idx]}`);
+          if (idx === args.scenariosPaths.length - 1) hasMoreScenarios = false;
+          exitDueToPreviousError = await validateScenarios({
+            scenariosPath: resolve(process.cwd(), args.scenariosPaths[idx]),
+            exitDueToPreviousError,
+            hasMoreScenarios,
+          });
+        }
+      },
+    )
+    .command(
+      "generate-scenarios-summary <scenariosPaths..>",
+      "Compile and validate all the Spec scenarios.",
+      (cmd) => {
+        return cmd
+          .positional("scenariosPaths", {
+            description: "Path(s) to the scenarios",
+            type: "string",
+            array: true,
+            demandOption: true,
+          })
+          .option("outputFile", {
+            type: "string",
+            description: "Path to the generated summary file(Markdown).",
+            default: join(process.cwd(), "spec-summary.md"),
+          });
+      },
+      async (args) => {
+        let overrideOutputFile = false;
+        for (const scenariosPath of args.scenariosPaths) {
+          logger.info(`Generating scenarios summary for ${scenariosPath}`);
+          await generateScenarioSummary({
+            scenariosPath: resolve(process.cwd(), scenariosPath),
+            outputFile: resolve(process.cwd(), args.outputFile),
+            overrideOutputFile,
+          });
+          overrideOutputFile = true;
+        }
+      },
+    )
+    .command("server", "Server management", (cmd) => {
+      cmd
+        .command(
+          "start <scenariosPath>",
+          "Start the server in the background.",
+          (cmd) => {
+            return cmd
+              .positional("scenariosPath", {
+                description: "Path to the scenarios and mock apis",
+                type: "string",
+                demandOption: true,
+              })
+              .option("port", {
+                alias: "p",
+                type: "number",
+                description: "Port where to host the server",
+                default: DEFAULT_PORT,
+              })
+              .option("coverageFile", {
+                type: "string",
+                description: "Path to the coverage file.",
+                default: join(process.cwd(), "spec-coverage.json"),
+              });
+          },
+          async (args) =>
+            startInBackground({
+              scenariosPath: resolve(process.cwd(), args.scenariosPath),
+              port: args.port,
+              coverageFile: args.coverageFile,
+            }),
+        )
+        .command(
+          "stop",
+          "Stop the server running.",
+          (cmd) => {
+            return cmd.option("port", {
+              alias: "p",
+              type: "number",
+              description: "Port where to host the server",
+              default: DEFAULT_PORT,
+            });
+          },
+          async (args) => stop({ port: args.port }),
+        );
+    })
+    .command(
+      "serve <scenariosPaths..>",
+      "Serve the mock api at the given paths.",
+      (cmd) => {
+        return cmd
+          .positional("scenariosPaths", {
+            description: "Path(s) to the scenarios and mock apis",
+            type: "string",
+            array: true,
+            demandOption: true,
+          })
+          .option("port", {
+            alias: "p",
+            type: "number",
+            description: "Port where to host the server",
+            default: DEFAULT_PORT,
+          })
+          .option("coverageFile", {
+            type: "string",
+            description: "Path to the coverage file.",
+            default: join(process.cwd(), "spec-coverage.json"),
+          });
+      },
+      async (args) => {
+        await serve({
+          scenariosPath: args.scenariosPaths,
+          port: args.port,
+          coverageFile: args.coverageFile,
+        });
+      },
+    )
+    .command(
+      "server-test <scenariosPaths..>",
+      "Executes the test cases against the service",
+      (cmd) => {
+        return cmd
+          .positional("scenariosPaths", {
+            description: "Path(s) to the scenarios and mock apis",
+            type: "string",
+            array: true,
+            demandOption: true,
+          })
+          .option("baseUrl", {
+            description: "Path to the server",
+            type: "string",
+          })
+          .option("runSingleScenario", {
+            description: "Single Scenario Case to run",
+            type: "string",
+          })
+          .option("runScenariosFromFile", {
+            description: "File that has the Scenarios to run",
+            type: "string",
+          })
+          .demandOption("scenariosPaths", "serverBasePath");
+      },
+      async (args) => {
+        for (const scenariosPath of args.scenariosPaths) {
+          logger.info(`Executing server tests for scenarios at ${scenariosPath}`);
+          await serverTest(scenariosPath, {
+            baseUrl: args.baseUrl,
+            runSingleScenario: args.runSingleScenario,
+            runScenariosFromFile: args.runScenariosFromFile,
+          });
+        }
+      },
+    )
+    .command(
+      "check-coverage <scenariosPaths..>",
+      "Serve the mock api at the given paths.",
+      (cmd) => {
+        return cmd
+          .positional("scenariosPaths", {
+            description: "Path(s) to the scenarios and mock apis",
+            type: "string",
+            demandOption: true,
+          })
+          .option("configFile", {
+            type: "string",
+            description: "Path to config file for generator.",
+          })
+          .option("coverageFiles", {
+            type: "string",
+            array: true,
+            description: "Path to the created coverage files.",
+            default: [join(process.cwd(), "spec-coverage.json")],
+          })
+          .demandOption("coverageFiles")
+          .option("mergedCoverageFile", {
+            type: "string",
+            description: "Output Path to the merged coverage file.",
+            default: join(process.cwd(), "spec-coverage.json"),
+          })
+          .option("ignoreNotImplemented", {
+            type: "boolean",
+            description: "Do not fail if there is some non implemented scenarios.",
+            default: false,
+          });
+      },
+      async (args) => {
+        let exitDueToPreviousError = false;
+        let hasMoreScenarios = true;
+        for (let idx = 0; idx < args.scenariosPaths.length; idx++) {
+          logger.info(`Checking coverage for scenarios at ${args.scenariosPaths[idx]}`);
+          if (idx === args.scenariosPaths.length - 1) hasMoreScenarios = false;
+          exitDueToPreviousError = await checkCoverage({
+            scenariosPath: resolve(process.cwd(), args.scenariosPaths[idx]),
+            configFile: args.configFile,
+            mergedCoverageFile: resolve(process.cwd(), args.mergedCoverageFile),
+            coverageFiles: args.coverageFiles.map((x) => resolve(process.cwd(), x)),
+            ignoreNotImplemented: args.ignoreNotImplemented,
+            exitDueToPreviousError,
+            hasMoreScenarios,
+          });
+        }
+      },
+    )
+    .command(
+      "validate-mock-apis <scenariosPaths..>",
+      "Validate mock apis have all the scenarios specified",
+      (cmd) => {
+        return cmd.positional("scenariosPaths", {
+          description: "Path to the scenarios and mock apis",
+          type: "string",
+          array: true,
+          demandOption: true,
+        });
+      },
+      async (args) => {
+        let exitDueToPreviousError = false;
+        let hasMoreScenarios = true;
+        for (let idx = 0; idx < args.scenariosPaths.length; idx++) {
+          logger.info(`Validating mock apis for scenarios at ${args.scenariosPaths[idx]}`);
+          if (idx === args.scenariosPaths.length - 1) hasMoreScenarios = false;
+          exitDueToPreviousError = await validateMockApis({
+            scenariosPath: resolve(process.cwd(), args.scenariosPaths[idx]),
+            exitDueToPreviousError,
+            hasMoreScenarios,
+          });
+        }
+      },
+    )
+    .command(
+      "upload-manifest <scenariosPaths..>",
+      "Upload the scenario manifest. DO NOT CALL in generator.",
+      (cmd) => {
+        return cmd
+          .positional("scenariosPaths", {
+            description: "Path to the scenarios and mock apis",
+            type: "string",
+            array: true,
+            demandOption: true,
+          })
+          .option("storageAccountName", {
+            type: "string",
+            description: "Name of the storage account",
+          })
+          .demandOption("storageAccountName");
+      },
+      async (args) => {
+        for (const scenariosPath of args.scenariosPaths) {
+          logger.info(`Uploading scenario manifest for scenarios at ${scenariosPath}`);
+          await uploadScenarioManifest({
+            scenariosPath: resolve(process.cwd(), scenariosPath),
+            storageAccountName: args.storageAccountName,
+          });
+        }
+      },
+    )
+    .command(
+      "upload-coverage",
+      "Upload the coverage report.",
+      (cmd) => {
+        return cmd
+          .option("coverageFile", {
+            type: "string",
+            description: "Path to the coverage file to upload.",
+            default: join(process.cwd(), "spec-coverage.json"),
+          })
+          .demandOption("coverageFile")
+          .option("storageAccountName", {
+            type: "string",
+            description: "Name of the storage account",
+            default: join(process.cwd(), "spec-coverage.json"),
+          })
+          .demandOption("storageAccountName")
+          .option("generatorName", {
+            type: "string",
+            description: "Name of generator",
+          })
+          .demandOption("generatorName")
+          .option("generatorVersion", {
+            type: "string",
+            description: "Version of generator",
+          })
+          .demandOption("generatorVersion")
+          .option("generatorCommit", {
+            type: "string",
+            description:
+              "Git sha of the generator. Resolved automatically if command is run inside of repository.",
+          })
+          .option("generatorMode", {
+            type: "string",
+            description: "Mode of generator to upload.",
+          })
+          .demandOption("generatorMode");
+      },
+      async (args) => {
+        await uploadCoverageReport({
+          coverageFile: resolve(process.cwd(), args.coverageFile),
+          storageAccountName: args.storageAccountName,
+          generatorName: args.generatorName,
+          generatorVersion: args.generatorVersion,
+          generatorCommit: args.generatorCommit ?? getCommit(process.cwd()),
+          generatorMode: args.generatorMode,
+        });
+      },
+    )
+    .demandCommand(1, "You must use one of the supported commands.")
+    .parse();
+}
+
+main().catch((error) => {
+  // eslint-disable-next-line no-console
+  console.log("Error", error);
+  process.exit(1);
+});
+
+process.once("SIGTERM", () => process.exit(143));
+process.once("SIGINT", () => process.exit(2));
+process.once("SIGUSR1", () => process.exit(2));
+process.once("SIGUSR2", () => process.exit(2));

package/src/config/config-schema.ts

@@ -0,0 +1,16 @@
+import { JSONSchemaType } from "ajv";
+import { SpecConfig } from "./types.js";
+
+export const SpecConfigJsonSchema: JSONSchemaType<SpecConfig> = {
+  type: "object",
+  additionalProperties: false,
+  properties: {
+    unsupportedScenarios: {
+      type: "array",
+      items: {
+        type: "string",
+      },
+    },
+  },
+  required: [],
+} as const;

package/src/config/config.ts

@@ -0,0 +1,15 @@
+import { readFile } from "fs/promises";
+import yaml from "js-yaml";
+import { Diagnostic } from "../utils/diagnostic-reporter.js";
+import { SpecConfigJsonSchema } from "./config-schema.js";
+import { SchemaValidator } from "./schema-validator.js";
+import { SpecConfig } from "./types.js";
+
+const validator = new SchemaValidator(SpecConfigJsonSchema);
+
+export async function loadSpecConfig(path: string): Promise<[SpecConfig, Diagnostic[]]> {
+  const content = await readFile(path);
+  const config: any = yaml.load(content.toString(), { filename: path });
+  const diagnostics = validator.validate(config, path);
+  return [config, diagnostics];
+}

package/src/config/schema-validator.ts

@@ -0,0 +1,57 @@
+import Ajv, { ErrorObject, JSONSchemaType } from "ajv";
+import { Diagnostic } from "../utils/diagnostic-reporter.js";
+
+export interface SchemaValidatorOptions {
+  coerceTypes?: boolean;
+}
+export class SchemaValidator<T> {
+  private ajv: any;
+
+  public constructor(
+    private schema: JSONSchemaType<T>,
+    options: SchemaValidatorOptions = {},
+  ) {
+    // https://github.com/ajv-validator/ajv/issues/2047
+    this.ajv = new (Ajv as any)({
+      strict: true,
+      coerceTypes: options.coerceTypes,
+    });
+  }
+
+  /**
+   * Validate the config is valid
+   * @param config Configuration
+   * @param target @optional file for errors tracing.
+   * @returns Validation
+   */
+  public validate(config: unknown, target: string): Diagnostic[] {
+    const validate = this.ajv.compile(this.schema);
+    validate(config);
+
+    const diagnostics = [];
+    for (const error of validate.errors ?? []) {
+      const diagnostic = ajvErrorToDiagnostic(error, target);
+      diagnostics.push(diagnostic);
+    }
+
+    return diagnostics;
+  }
+}
+
+const IGNORED_AJV_PARAMS = new Set(["type", "errors"]);
+
+function ajvErrorToDiagnostic(error: ErrorObject, target: string): Diagnostic {
+  const messageLines = [`Schema violation: ${error.message} (${error.instancePath || "/"})`];
+  for (const [name, value] of Object.entries(error.params).filter(
+    ([name]) => !IGNORED_AJV_PARAMS.has(name),
+  )) {
+    const formattedValue = Array.isArray(value) ? [...new Set(value)].join(", ") : value;
+    messageLines.push(`  ${name}: ${formattedValue}`);
+  }
+
+  const message = messageLines.join("\n");
+  return {
+    message,
+    target,
+  };
+}

package/src/config/types.ts

@@ -0,0 +1,3 @@
+export interface SpecConfig {
+  unsupportedScenarios: string[];
+}

package/src/constants.ts

@@ -0,0 +1,3 @@
+export const AdminUrls = {
+  stop: "/.admin/stop",
+};

package/src/coverage/common.ts

@@ -0,0 +1,10 @@
+import { getPackageJson, getCommit } from "../utils/misc-utils.js";
+import { ScenariosMetadata } from "@typespec/spec-coverage-sdk";
+
+export async function getScenarioMetadata(scenariosPath: string): Promise<ScenariosMetadata> {
+  const pkg = await getPackageJson(scenariosPath);
+  return {
+    commit: getCommit(scenariosPath),
+    version: pkg?.version ?? "?",
+  };
+}

package/src/coverage/coverage-report.ts

@@ -0,0 +1,13 @@
+import { CoverageReport, ScenarioStatus } from "@typespec/spec-coverage-sdk";
+import { getScenarioMetadata } from "./common.js";
+
+export async function createCoverageReport(
+  scenariosPath: string,
+  results: Record<string, ScenarioStatus>,
+): Promise<CoverageReport> {
+  return {
+    scenariosMetadata: await getScenarioMetadata(scenariosPath),
+    results,
+    createdAt: new Date().toISOString(),
+  };
+}

package/src/coverage/coverage-tracker.ts

@@ -0,0 +1,141 @@
+import { Fail, KeyedMockResponse, MockResponse, PassByKeyScenario, PassByServiceKeyScenario, ScenarioMockApi } from "@typespec/spec-api";
+import { logger } from "../logger.js";
+import { CoverageReport, ScenariosMetadata, ScenarioStatus } from "@typespec/spec-coverage-sdk";
+import { writeFileSync } from "fs";
+import { ScenariosAndScenariosMetadata } from "../app/app.js";
+
+export class CoverageTracker {
+  private scenarios: Record<string, ScenarioMockApi> = {};
+  private scenariosAndScenariosMetaData: ScenariosAndScenariosMetadata[] = [];
+  private hits = new Map<string, Map<string, MockResponse[]>>();
+
+  public constructor(private coverageFile: string) {
+    process.on("exit", () => {
+      logger.info("Saving coverage");
+      this.saveCoverageSync();
+      logger.info("Coverage saved!");
+    });
+  }
+
+  public setScenarios(scenariosMetadata: ScenariosMetadata, scenarios: Record<string, ScenarioMockApi>): void;
+  public setScenarios(scenariosAndScenariosMetaData: ScenariosAndScenariosMetadata[]): void;
+
+  public setScenarios(arg1: ScenariosMetadata | ScenariosAndScenariosMetadata[], arg2?: Record<string, ScenarioMockApi>) {
+    if (arg1 instanceof Array === false && arg2 !== undefined) {
+      this.scenariosAndScenariosMetaData = [
+        {
+          scenariosMetadata: arg1,
+          scenarios: arg2,
+        }
+      ];
+    } else {
+      this.scenariosAndScenariosMetaData = arg1 as ScenariosAndScenariosMetadata[];
+      for (const {scenarios} of this.scenariosAndScenariosMetaData) {
+        this.scenarios = {...this.scenarios, ...scenarios};
+      }
+    }
+  }
+
+  public async trackEndpointResponse(scenarioName: string, endpoint: string, response: MockResponse) {
+    let scenarioHits = this.hits.get(scenarioName);
+    if (scenarioHits === undefined) {
+      scenarioHits = new Map();
+      this.hits.set(scenarioName, scenarioHits);
+    }
+
+    let responses = scenarioHits.get(endpoint);
+    if (responses === undefined) {
+      responses = [];
+      scenarioHits.set(endpoint, responses);
+    }
+    responses.push(response);
+  }
+
+  public computeCoverage(): CoverageReport[] {
+    const coverageReports:CoverageReport[] = [];
+    for(const {scenarios, scenariosMetadata} of this.scenariosAndScenariosMetaData) {
+      const results: Record<string, ScenarioStatus> = {};
+      for (const [name, mockApi] of Object.entries(scenarios)) {
+        results[name] = this.computeScenarioStatus(name, mockApi);        
+      }
+      coverageReports.push({
+        scenariosMetadata: scenariosMetadata,
+        results,
+        createdAt: new Date().toISOString(),
+      })
+    }
+    return coverageReports;
+  }
+
+  private saveCoverageSync() {
+    const coverage = this.computeCoverage();
+
+    try {
+      writeFileSync(this.coverageFile, JSON.stringify(coverage, null, 2));
+    } catch (e) {
+      logger.warn("Error while saving coverage", e);
+    }
+  }
+
+  private computeScenarioStatus(name: string, mockApi: ScenarioMockApi): ScenarioStatus {
+    const scenarioHits = this.hits.get(name);
+    switch (mockApi.passCondition) {
+      case "response-success":
+        return checkAll((x) => x.status >= 200 && x.status < 300);
+      case "status-code":
+        return checkAll((x) => x.status === mockApi.code);
+      case "by-key":
+        return checkByKeys(mockApi);
+      default:
+        const _assertNever: never = mockApi;
+        throw new Error("Unreachable");
+    }
+
+    function checkAll(condition: (res: MockResponse) => boolean): ScenarioStatus {
+      for (const endpoint of mockApi.apis) {
+        const hits = scenarioHits?.get(endpoint.uri);
+        if (hits === undefined) {
+          return "not-implemented";
+        }
+
+        if (!condition(hits[hits.length - 1])) {
+          return "fail";
+        }
+      }
+
+      return "pass";
+    }
+
+    function checkByKeys(scenario: PassByKeyScenario | PassByServiceKeyScenario) {
+      for (const endpoint of scenario.apis) {
+        const hits = scenarioHits?.get(endpoint.uri);
+        if (hits === undefined) {
+          return "not-implemented";
+        }
+        const keys = new Set(scenario.keys);
+
+        for (const hit of hits) {
+          if (!isKeyedMockResponse(hit)) {
+            continue;
+          }
+
+          if (hit.pass === Fail) {
+            return "fail";
+          }
+          keys.delete(hit.pass);
+        }
+
+        if (keys.size === 0) {
+          return "pass";
+        } else {
+          return "fail";
+        }
+      }
+      return "fail";
+    }
+  }
+}
+
+function isKeyedMockResponse(response: MockResponse): response is KeyedMockResponse {
+  return "pass" in response;
+}

package/src/coverage/index.ts

@@ -0,0 +1 @@
+export * from "./coverage-report.js";

package/src/coverage/scenario-manifest.ts

@@ -0,0 +1,43 @@
+import { loadScenarios } from "../scenarios-resolver.js";
+import { Diagnostic } from "../utils/diagnostic-reporter.js";
+import { getCommit, getPackageJson } from "../utils/misc-utils.js";
+import { ScenarioLocation, ScenarioManifest, GeneratorMode } from "@typespec/spec-coverage-sdk";
+import { getSourceLocation, normalizePath } from "@typespec/compiler";
+import { relative } from "path";
+import type { Scenario } from "../lib/decorators.js";
+
+export async function computeScenarioManifest(
+  scenariosPath: string,
+): Promise<[ScenarioManifest | undefined, readonly Diagnostic[]]> {
+  const [scenarios, diagnostics] = await loadScenarios(scenariosPath);
+  if (diagnostics.length > 0) {
+    return [undefined, diagnostics];
+  }
+
+  const commit = getCommit(scenariosPath);
+  const pkg = await getPackageJson(scenariosPath);
+  return [createScenarioManifest(scenariosPath, pkg?.version ?? "?", commit, scenarios), []];
+}
+
+export function createScenarioManifest(
+  scenariosPath: string,
+  version: string,
+  commit: string,
+  scenarios: Scenario[],
+): ScenarioManifest {
+  const sortedScenarios = [...scenarios].sort((a, b) => a.name.localeCompare(b.name));
+  return {
+    version,
+    commit,
+    scenarios: sortedScenarios.map(({ name, scenarioDoc, target }) => {
+      const tspLocation = getSourceLocation(target);
+      const location: ScenarioLocation = {
+        path: normalizePath(relative(scenariosPath, tspLocation.file.path)),
+        start: tspLocation.file.getLineAndCharacterOfPosition(tspLocation.pos),
+        end: tspLocation.file.getLineAndCharacterOfPosition(tspLocation.end),
+      };
+      return { name, scenarioDoc, location };
+    }),
+    modes: GeneratorMode,
+  };
+}

package/src/index.ts

@@ -0,0 +1 @@
+export * from "./lib/index.js";