@typespec/spector 0.1.0-alpha.1-dev.1 → 0.1.0-alpha.10-dev.0

package/dist/src/utils/request-utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"request-utils.d.ts","sourceRoot":"","sources":["../../../src/utils/request-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,eAAO,MAAM,iBAAiB,YAAa,OAAO,KAAG,MACL,CAAC"}
+{"version":3,"file":"request-utils.d.ts","sourceRoot":"","sources":["../../../src/utils/request-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,eAAO,MAAM,iBAAiB,GAAI,SAAS,OAAO,KAAG,MACL,CAAC"}

package/docs/decorators.md

@@ -0,0 +1,24 @@
+## Spector decorators
+
+### `@scenarioService`
+
+Decorator setting up the boilerplate for specs service namespace. Will automatically set:
+
+- `@service{title: '<namespace>', value: '1.0.0'}` using the namespace as a value
+- `@server` to `localhost:3000`
+- `@route` using the parameter passed.
+
+Usage:
+
+```tsp
+@scenarioSpec("/my-spec")
+namespace MySpec;
+```
+
+### `@scenario`
+
+Mark an operation, interface or namespace as a scenario. Optionally can provide the name of the scenario.
+
+### `@scenarioDoc`
+
+Specify how to implement this scenario. Value is markdown. Differ from @doc which describe the scenario to the end user.

package/docs/using-spector.md

@@ -0,0 +1,99 @@
+# Using spector
+
+Install `spector` CLI globally or as a local dependency:
+
+- **Globally**: run `npm install -g @typespec/spector`. You can then use `tsp-spector` as a CLI tool.
+- **Locally**: run `npm install @typespec/spector` which should add the dependency to package.json. You can then use `tsp-spector` with `npx tsp-spector`.
+
+**NOTE** With local install you'll need to prefix `tsp-spector` with `npx`(unless commands are written directly as npm scripts)
+
+## Test against scenarios
+
+### Spector Config
+
+Spector supports a config file where you can configure scenarios that are not supported by your generator.
+
+Create a `spector-config.yaml` file.
+
+```yaml
+# List of unsupported scenarios
+unsupportedScenarios:
+  - Foo_Bar
+```
+
+### Run mock api server
+
+```bash
+# Minimal
+tsp-spector serve ./path/to/scenarios
+
+# Change the port
+tsp-spector serve ./path/to/scenarios --port 1234
+
+# Specify where the coverage file should go
+tsp-spector serve ./path/to/scenarios --coverageFile ./path/to/spector-coverage.json
+```
+
+Alternative to start in background
+
+```bash
+tsp-spector server start ./path/to/scenarios # Takes the same arguments as serve
+```
+
+### Stop running server
+
+```bash
+tsp-spector server stop             # Stop at the default port
+tsp-spector server stop --port 1234 # If started the server at another port
+```
+
+### Validate and merge coverage
+
+```bash
+# Minimal
+tsp-spector check-coverage ./path/to/scenarios
+
+# Path to tsp-spector config file for generator
+tsp-spector check-coverage ./path/to/scenarios --configFile ./spector-config.yaml
+
+# In case where there was multiple serve instance each creating their own coverage file
+tsp-spector check-coverage ./path/to/scenarios --coverageFiles ./path/to/*-coverage.json --coverageFiles ./other/to/*-coverage.json
+
+# Specify where the merged coverage file should go
+tsp-spector check-coverage ./path/to/scenarios --mergedCoverageFile ./path/to/spector-final-coverage.json
+```
+
+### Upload coverage
+
+Upload the coverage. Upload from the `main` branch. DO NOT upload on PR this WILL override the latest index.
+
+```bash
+# Minimal
+tsp-spector upload-coverage --generatorName typescript --version=0.1.0
+
+# Specify Coverage file
+tsp-spector upload-coverage --generatorName typescript --version=0.1.0 --coverageFile ./path/to/spector-final-coverage.json
+```
+
+Options:
+
+- `--storageAccountName`: Name of the storage account to publish coverage. Use `typespec` for Spector/Azure Spector dashboard.
+- `--generatorName`: Name of the generator. Must be one of `"@typespec/http-client-python", "@typespec/http-client-csharp", "@azure-tools/typespec-ts-rlc", "@azure-tools/typespec-ts-modular", "@typespec/http-client-js", "@typespec/http-client-java"`.
+- `--generatorVersion`: Version of the generator
+- `--coverageFile`: Path to the coverage file
+
+#### Upload in azure devops
+
+**This is applicable in the azure-sdk/internal ado project only.**
+
+Add the following step
+
+```yaml
+- task: AzureCLI@2
+  displayName: Upload scenario coverage
+  inputs:
+    azureSubscription: "Typespec Storage"
+    scriptType: "bash"
+    scriptLocation: "inlineScript"
+    inlineScript: `tsp-spector upload-coverage --storageAccountName typespec --containerName coverages --generatorMode standard  ... FILL options fitting your generator here as described above...`
+```

package/docs/writing-mock-apis.md

@@ -0,0 +1,116 @@
+# Writing mock apis
+
+1. Create a `mockapi.ts` file next to the `main.tsp` of the scenario
+2. Create and export a variable called `Scenarios` with a `Record<string, ScenarioMockApi>` type
+3. For each of the scenario assign a new property to the `Scenarios` variable. The value use one of the following:
+   - `passOnSuccess`: This will take one or multiple routes and will only pass teh scenario if all the routes gets called with a 2xx success code.
+
+## Example
+
+```ts
+import { passOnSuccess, mockapi } from "@typespec/spec-api";
+import { ScenarioMockApi } from "@typespec/spec-api";
+
+export const Scenarios: Record<string, ScenarioMockApi> = {};
+
+Scenarios.Hello_world = passOnSuccess(
+  mockapi.get("/hello/world", () => {
+    return {
+      status: 200,
+      body: {
+        contentType: "application/json",
+        rawContent: `"Hello World!"`,
+      },
+    };
+  }),
+);
+```
+
+## How to build response
+
+Return the response object. [See type](../../spec-api/src/types.ts)
+
+```ts
+// Minimum requirement is the status code.
+return {
+  status: 200,
+};
+```
+
+### Return a body
+
+```ts
+// Return json
+return {
+  status: 200,
+  body: json({ foo: 123 }),
+};
+
+// Return raw content
+return {
+  status: 200,
+  body: {
+    contentType: "application/text",
+    rawContent: "foobar",
+  },
+};
+```
+
+### Return headers
+
+```ts
+// Return json
+return {
+  status: 200,
+  headers: {
+      MyHeader: "value-1"
+      MyHeaderOther: req.headers.MyRequestHeader
+  }
+};
+
+```
+
+## How to validate the request:
+
+All built-in validation tools can be accessed using `req.expect.`
+
+### Validate the body
+
+- With `req.expect.bodyEquals`
+
+This will do a deep equals of the body to make sure it match.
+
+```ts
+app.post("/example", "Example", (req) => {
+  req.bodyEquals({ foo: "123", bar: "456" });
+});
+```
+
+- With `req.expect.rawBodyEquals`
+
+This will compare the raw body sent.
+
+```ts
+app.post("/example", "Example", (req) => {
+  req.rawBodyEquals('"foo"');
+});
+```
+
+### Custom validation
+
+You can do any kind of validation accessing the `req: MockRequest` object and deciding to return a different response in some cases.
+You can also always `throw` a `ValidationError`
+
+Example:
+
+```ts
+app.post("/example", "Example", (req) => {
+  if (req.headers.MyCustomHeader.startsWith("x-foo")) {
+    throw new ValidationError(
+      "MyCustomHeader shouldn't start with x-foo",
+      null,
+      req.headers.MyCustomHeader,
+    );
+  }
+});
+```

package/docs/writing-scenario-spec.md

@@ -0,0 +1,49 @@
+## Defining scenario
+
+The goal of the testserver is to define scenarios that needs to be supported in client generators. This means to have a meaningful coverage we need scenarios to:
+
+- have a specific behavior that is meant to be tested
+  - ❌ DO NOT use the same scenario for testing multiple things.
+- have a meaningful name that can be used in a compatibility table to see what a given generator support(e.g. `get_string`)
+- a description of what the scenario is validating(e.g. `"Support passing a simple string as JSON"`)
+- have a good description on what the client is expecting to generate/receive/send(e.g `Validate that this operation returns a JSON string that match "abc"`)
+  - ✅ DO describe how to validate that this scenario is working from the client point of view
+
+When naming scenario always think about what would it look like in the [compatibility table](#compatibility-table) and would that name be meaningful to someone looking to see what is supported.
+
+```cadl
+import "@typespec/spector";
+
+@scenarioService("/strings")
+namespace String;
+
+@scenario("get_string")
+@doc("Support passing a simple string as JSON")
+@scenarioDoc("In this scenario the Client should expect a string matching 'abc' to be returned.")
+@get
+@route("/simple")
+op returnString(): string;
+```
+
+Decorators that should be provided in this test library `@typespec/spector`:
+
+- `@scenario`: Specify that this operation, interface or namespace is a scenario. Optionally take a scenario name otherwise default to the namespace name + operation/interface name
+- `@scenarioDoc`: Specify how to implement this scenario. Differ from `@doc` which describe the scenario to the end user.
+- `@supportedBy`: Specify if something is supported only by some kind of SDK. Option: `arm`, `dpg`. By default everything.
+
+## Compatibility table
+
+With all this information, a detailed compatibility table should be able to be produced by compiling each one of the scenarios and extracting the cases. Providing something like
+
+| Scenario                     | CSharp | Python | Go  | Java | TS/JS |
+| ---------------------------- | ------ | ------ | --- | ---- | ----- |
+| Vanilla                      |
+| `get_string`                 | ✅     | ✅     | ✅  | ✅   | ✅    |
+| `put_string`                 | ✅     | ✅     | ✅  | ✅   | ✅    |
+| Azure                        |
+| `pageable_nextLink`          | ✅     | ✅     | ✅  | ✅   | ✅    |
+| `pageable_continuationToken` | ❌     | ✅     | ✅  | ✅   | ✅    |
+
+## Writing the mock api
+
+Each scenario should be accompanied by a mock api. See [writing a mock api docs](./writing-mock-apis.md)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typespec/spector",
-  "version": "0.1.0-alpha.1-dev.1",
+  "version": "0.1.0-alpha.10-dev.0",
   "description": "Typespec Core Tool to validate, run mock api, collect coverage.",
   "exports": {
     ".": {
@@ -26,30 +26,30 @@
   },
   "homepage": "https://github.com/microsoft/typespec#readme",
   "dependencies": {
-    "@azure/identity": "~4.4.1",
+    "@azure/identity": "~4.7.0",
     "@types/js-yaml": "^4.0.5",
-    "@typespec/compiler": "~0.61.2 || >=0.62.0-dev <0.62.0",
-    "@typespec/versioning": "~0.61.0 || >=0.62.0-dev <0.62.0",
-    "@typespec/http": "~0.61.0 || >=0.62.0-dev <0.62.0",
-    "@typespec/rest": "~0.61.0 || >=0.62.0-dev <0.62.0",
-    "@typespec/spec-api": "~0.1.0-alpha.0 || >=0.1.0-alpha.1-dev <0.1.0-alpha.1",
-    "@typespec/spec-coverage-sdk": "~0.1.0-alpha.0 || >=0.1.0-alpha.1-dev <0.1.0-alpha.1",
+    "@typespec/compiler": "^0.67.0 || >=0.68.0-dev <0.68.0",
+    "@typespec/http": "^0.67.0 || >=0.68.0-dev <0.68.0",
+    "@typespec/rest": "^0.67.0 || >=0.68.0-dev <0.68.0",
+    "@typespec/spec-api": "^0.1.0-alpha.2 || >=0.1.0-alpha.3-dev <0.1.0-alpha.3",
+    "@typespec/spec-coverage-sdk": "^0.1.0-alpha.4 || >=0.1.0-alpha.5-dev <0.1.0-alpha.5",
+    "@typespec/versioning": "^0.67.0 || >=0.68.0-dev <0.68.0",
     "ajv": "~8.17.1",
-    "axios": "^1.7.5",
+    "axios": "^1.8.1",
     "body-parser": "^1.20.3",
     "deep-equal": "^2.2.0",
-    "express": "^4.21.1",
+    "express": "^4.21.2",
     "express-promise-router": "^4.1.1",
-    "form-data": "^4.0.1",
-    "globby": "~14.0.2",
-    "jackspeak": "4.0.2",
+    "form-data": "^4.0.2",
+    "globby": "~14.1.0",
+    "jackspeak": "4.1.0",
     "js-yaml": "^4.1.0",
     "morgan": "^1.10.0",
     "multer": "^1.4.5-lts.1",
     "node-fetch": "^3.3.1",
-    "picocolors": "~1.1.0",
+    "picocolors": "~1.1.1",
     "source-map-support": "~0.5.21",
-    "winston": "^3.15.0",
+    "winston": "^3.17.0",
     "xml2js": "^0.6.2",
     "yargs": "~17.7.2"
   },
@@ -59,13 +59,13 @@
     "@types/express": "^5.0.0",
     "@types/morgan": "^1.9.4",
     "@types/multer": "^1.4.10",
-    "@types/node": "~22.7.5",
-    "@types/node-fetch": "^2.6.3",
+    "@types/node": "~22.13.9",
+    "@types/node-fetch": "^2.6.12",
     "@types/xml2js": "^0.4.11",
     "@types/yargs": "~17.0.33",
     "rimraf": "~6.0.1",
-    "typescript": "~5.6.3",
-    "@typespec/tspd": "~0.46.0"
+    "typescript": "~5.8.2",
+    "@typespec/tspd": "^0.46.0"
   },
   "peerDependencies": {},
   "scripts": {

package/src/actions/serve.ts

@@ -39,14 +39,14 @@ export async function startInBackground(config: ServeConfig) {
     const [nodeExe, entrypoint] = process.argv;
     logger.info(`Starting server in background at port ${config.port}`);
     const scenariosPath = Array.isArray(config.scenariosPath)
-      ? config.scenariosPath.join(" ")
-      : config.scenariosPath;
+      ? config.scenariosPath
+      : [config.scenariosPath];
     const cp = spawn(
       nodeExe,
       [
         entrypoint,
         "serve",
-        scenariosPath,
+        ...scenariosPath,
         "--port",
         config.port.toString(),
         "--coverageFile",

package/src/actions/server-test.ts

@@ -1,12 +1,19 @@
 import { MockApiDefinition } from "@typespec/spec-api";
 import * as fs from "fs";
 import * as path from "path";
+import pc from "picocolors";
 import { logger } from "../logger.js";
 import { loadScenarioMockApis } from "../scenarios-resolver.js";
 import { makeServiceCall, uint8ArrayToString } from "./helper.js";
 
 const DEFAULT_BASE_URL = "http://localhost:3000";
 
+export interface ServerTestDiagnostics {
+  scenario_name: string;
+  status: "success" | "failure";
+  message: any;
+}
+
 class ServerTestsGenerator {
   private name: string = "";
   private mockApiDefinition: MockApiDefinition;
@@ -184,6 +191,9 @@ export async function serverTest(scenariosPath: string, options: ServerTestOptio
   }
   // 2. Load all the scenarios
   const scenarios = await loadScenarioMockApis(scenariosPath);
+  const success_diagnostics: ServerTestDiagnostics[] = [];
+  const failure_diagnostics: ServerTestDiagnostics[] = [];
+
   // 3. Execute each scenario
   for (const [name, scenario] of Object.entries(scenarios)) {
     if (!Array.isArray(scenario.apis)) continue;
@@ -191,8 +201,39 @@ export async function serverTest(scenariosPath: string, options: ServerTestOptio
       if (api.kind !== "MockApiDefinition") continue;
       if (testCasesToRun.length === 0 || testCasesToRun.includes(name)) {
         const obj: ServerTestsGenerator = new ServerTestsGenerator(name, api, baseUrl);
-        await obj.executeScenario();
+        try {
+          await obj.executeScenario();
+          success_diagnostics.push({
+            scenario_name: name,
+            status: "success",
+            message: "executed successfully",
+          });
+        } catch (e: any) {
+          failure_diagnostics.push({
+            scenario_name: name,
+            status: "failure",
+            message: `code = ${e.code} \n message = ${e.message} \n name = ${e.name} \n stack = ${e.stack} \n status = ${e.status}`,
+          });
+        }
       }
     }
   }
+
+  // 4. Print diagnostics
+  logger.info("Server Tests Diagnostics Summary");
+
+  if (success_diagnostics.length > 0) logger.info("Success Scenarios");
+  success_diagnostics.forEach((diagnostic) => {
+    logger.info(`${pc.green("✓")} Scenario: ${diagnostic.scenario_name} - ${diagnostic.message}`);
+  });
+
+  if (failure_diagnostics.length > 0) logger.error("Failure Scenarios");
+  if (failure_diagnostics.length > 0) {
+    logger.error("Failed Scenario details");
+    failure_diagnostics.forEach((diagnostic) => {
+      logger.error(`${pc.red("✘")} Scenario: ${diagnostic.scenario_name}`);
+      logger.error(`${diagnostic.message}`);
+    });
+    process.exit(-1);
+  }
 }

package/src/actions/upload-coverage-report.ts

@@ -11,6 +11,7 @@ export interface UploadCoverageReportConfig {
   generatorVersion: string;
   generatorCommit?: string;
   generatorMode: string;
+  containerName: string;
 }
 
 export async function uploadCoverageReport({
@@ -20,11 +21,16 @@ export async function uploadCoverageReport({
   generatorVersion,
   generatorCommit: geenratorCommit,
   generatorMode,
+  containerName,
 }: UploadCoverageReportConfig) {
   const content = await readFile(coverageFile);
   const coverage: CoverageReport = JSON.parse(content.toString());
 
-  const client = new SpecCoverageClient(storageAccountName, new AzureCliCredential());
+  const client = new SpecCoverageClient(storageAccountName, {
+    credential: new AzureCliCredential(),
+    containerName,
+  });
+  await client.createIfNotExists();
   const generatorMetadata: GeneratorMetadata = {
     name: generatorName,
     version: generatorVersion,

package/src/actions/upload-scenario-manifest.ts

@@ -1,28 +1,41 @@
 import { AzureCliCredential } from "@azure/identity";
 import { SpecCoverageClient } from "@typespec/spec-coverage-sdk";
 import { writeFile } from "fs/promises";
+import { resolve } from "path";
 import pc from "picocolors";
 import { computeScenarioManifest } from "../coverage/scenario-manifest.js";
 import { logger } from "../logger.js";
 
 export interface UploadScenarioManifestConfig {
-  scenariosPath: string;
+  scenariosPaths: string[];
   storageAccountName: string;
+  setNames: string[];
+  containerName: string;
 }
 
 export async function uploadScenarioManifest({
-  scenariosPath,
+  scenariosPaths,
   storageAccountName,
+  setNames,
+  containerName,
 }: UploadScenarioManifestConfig) {
-  const [manifest, diagnostics] = await computeScenarioManifest(scenariosPath);
-  if (manifest === undefined || diagnostics.length > 0) {
-    process.exit(-1);
+  const manifests = [];
+  for (let idx = 0; idx < scenariosPaths.length; idx++) {
+    const path = resolve(process.cwd(), scenariosPaths[idx]);
+    logger.info(`Computing scenario manifest for ${path}`);
+    const [manifest, diagnostics] = await computeScenarioManifest(path, setNames[idx]);
+    if (manifest === undefined || diagnostics.length > 0) {
+      process.exit(-1);
+    }
+    manifests.push(manifest);
   }
-
-  await writeFile("manifest.json", JSON.stringify(manifest, null, 2));
-  const client = new SpecCoverageClient(storageAccountName, new AzureCliCredential());
+  await writeFile("manifest.json", JSON.stringify(manifests, null, 2));
+  const client = new SpecCoverageClient(storageAccountName, {
+    credential: new AzureCliCredential(),
+    containerName,
+  });
   await client.createIfNotExists();
-  await client.manifest.upload(manifest);
+  await client.manifest.upload(manifests);
 
   logger.info(
     `${pc.green("✓")} Scenario manifest uploaded to ${storageAccountName} storage account.`,

package/src/cli/cli.ts

@@ -92,13 +92,14 @@ async function main() {
     .command("server", "Server management", (cmd) => {
       cmd
         .command(
-          "start <scenariosPath>",
+          "start <scenariosPaths..>",
           "Start the server in the background.",
           (cmd) => {
             return cmd
-              .positional("scenariosPath", {
-                description: "Path to the scenarios and mock apis",
+              .positional("scenariosPaths", {
+                description: "Path(s) to the scenarios and mock apis",
                 type: "string",
+                array: true,
                 demandOption: true,
               })
               .option("port", {
@@ -115,7 +116,7 @@ async function main() {
           },
           async (args) =>
             startInBackground({
-              scenariosPath: resolve(process.cwd(), args.scenariosPath),
+              scenariosPath: args.scenariosPaths,
               port: args.port,
               coverageFile: args.coverageFile,
             }),
@@ -287,20 +288,30 @@ async function main() {
             array: true,
             demandOption: true,
           })
+          .option("setName", {
+            type: "string",
+            description: "Set used to generate the manifest.",
+            array: true,
+            demandOption: true,
+          })
           .option("storageAccountName", {
             type: "string",
             description: "Name of the storage account",
           })
+          .option("containerName", {
+            type: "string",
+            description: "Name of the Container",
+            demandOption: true,
+          })
           .demandOption("storageAccountName");
       },
       async (args) => {
-        for (const scenariosPath of args.scenariosPaths) {
-          logger.info(`Uploading scenario manifest for scenarios at ${scenariosPath}`);
-          await uploadScenarioManifest({
-            scenariosPath: resolve(process.cwd(), scenariosPath),
-            storageAccountName: args.storageAccountName,
-          });
-        }
+        await uploadScenarioManifest({
+          scenariosPaths: args.scenariosPaths,
+          storageAccountName: args.storageAccountName,
+          setNames: args.setName,
+          containerName: args.containerName,
+        });
       },
     )
     .command(
@@ -339,6 +350,11 @@ async function main() {
             type: "string",
             description: "Mode of generator to upload.",
           })
+          .option("containerName", {
+            type: "string",
+            description: "Name of the Container",
+            demandOption: true,
+          })
           .demandOption("generatorMode");
       },
       async (args) => {
@@ -349,6 +365,7 @@ async function main() {
           generatorVersion: args.generatorVersion,
           generatorCommit: args.generatorCommit ?? getCommit(process.cwd()),
           generatorMode: args.generatorMode,
+          containerName: args.containerName,
         });
       },
     )

package/src/coverage/common.ts

@@ -6,5 +6,6 @@ export async function getScenarioMetadata(scenariosPath: string): Promise<Scenar
   return {
     commit: getCommit(scenariosPath),
     version: pkg?.version ?? "?",
+    packageName: pkg?.name ?? "@typespec/http-specs",
   };
 }

package/src/coverage/scenario-manifest.ts

@@ -1,13 +1,14 @@
 import { loadScenarios } from "../scenarios-resolver.js";
 import { Diagnostic } from "../utils/diagnostic-reporter.js";
 import { getCommit, getPackageJson } from "../utils/misc-utils.js";
-import { ScenarioLocation, ScenarioManifest, GeneratorMode } from "@typespec/spec-coverage-sdk";
+import { ScenarioLocation, ScenarioManifest } from "@typespec/spec-coverage-sdk";
 import { getSourceLocation, normalizePath } from "@typespec/compiler";
 import { relative } from "path";
 import type { Scenario } from "../lib/decorators.js";
 
 export async function computeScenarioManifest(
   scenariosPath: string,
+  setName: string
 ): Promise<[ScenarioManifest | undefined, readonly Diagnostic[]]> {
   const [scenarios, diagnostics] = await loadScenarios(scenariosPath);
   if (diagnostics.length > 0) {
@@ -16,7 +17,7 @@ export async function computeScenarioManifest(
 
   const commit = getCommit(scenariosPath);
   const pkg = await getPackageJson(scenariosPath);
-  return [createScenarioManifest(scenariosPath, pkg?.version ?? "?", commit, scenarios), []];
+  return [createScenarioManifest(scenariosPath, pkg?.version ?? "?", commit, scenarios, setName), []];
 }
 
 export function createScenarioManifest(
@@ -24,6 +25,7 @@ export function createScenarioManifest(
   version: string,
   commit: string,
   scenarios: Scenario[],
+  setName: string
 ): ScenarioManifest {
   const sortedScenarios = [...scenarios].sort((a, b) => a.name.localeCompare(b.name));
   return {
@@ -38,6 +40,6 @@ export function createScenarioManifest(
       };
       return { name, scenarioDoc, location };
     }),
-    modes: GeneratorMode,
+    setName
   };
 }