@typespec/emitter-framework 0.9.0-dev.8 → 0.9.0-dev.9

package/src/testing/scenario-test/code-block-expectation.test.ts

@@ -0,0 +1,95 @@
+import { d } from "@alloy-js/core/testing";
+import { beforeAll, describe, expect, it } from "vitest";
+import { getExcerptForQuery, parseCodeBlockHeading } from "./code-block-expectation.js";
+import {
+  createSnippetExtractor,
+  createTypeScriptExtractorConfig,
+  type SnippetExtractor,
+} from "./snippet-extractor.js";
+
+describe("parseCodeBlockHeading", () => {
+  it("parse whole file expectation", () => {
+    expect(parseCodeBlockHeading("ts path/to/file.ts")).toEqual({
+      lang: "ts",
+      file: "path/to/file.ts",
+    });
+  });
+
+  it("throws error when no file is provided", () => {
+    expect(() => parseCodeBlockHeading("ts")).toThrow(
+      'Invalid code block heading: "ts". Missing file path. Expected format: "<lang> <path>"',
+    );
+  });
+
+  it("parse parse with type and name", () => {
+    expect(parseCodeBlockHeading("ts path/to/file.ts interface foo")).toEqual({
+      lang: "ts",
+      file: "path/to/file.ts",
+      query: {
+        type: "interface",
+        name: "foo",
+      },
+    });
+  });
+
+  it("throws error when using type but no name is provided", () => {
+    expect(() => parseCodeBlockHeading("ts path/to/file.ts interface")).toThrow(
+      'Invalid code block heading: "ts path/to/file.ts interface". Missing name when using type. Expected format: "<lang> <path> [type] [name]"',
+    );
+  });
+});
+
+describe("getExcerptForQuery", () => {
+  let snippetExtractor: SnippetExtractor;
+  beforeAll(async () => {
+    const tsExtractorConfig = await createTypeScriptExtractorConfig();
+    snippetExtractor = createSnippetExtractor(tsExtractorConfig);
+  });
+
+  it("gets a whole file", async () => {
+    const expectation = {
+      lang: "ts",
+      file: "file.ts",
+    };
+    const outputs = {
+      "file.ts": d`
+        interface bar {
+        
+        }
+        interface foo {
+          bar: string;
+        }      
+      `,
+    };
+    const excerpt = getExcerptForQuery(snippetExtractor, expectation, outputs);
+    expect(excerpt).toBe(outputs["file.ts"]);
+  });
+
+  it("gets an interface for typescript", async () => {
+    const expectation = {
+      lang: "ts",
+      file: "file.ts",
+      query: {
+        type: "interface",
+        name: "foo",
+      },
+    };
+    const outputs = {
+      "file.ts": d`
+        interface bar {
+        
+        }
+
+        interface foo {
+          bar: string;
+        }
+      `,
+    };
+    const excerpt = getExcerptForQuery(snippetExtractor, expectation, outputs);
+    expect(excerpt).toBe(d`
+      interface foo {
+        bar: string;
+      }
+    `);
+  });
+});

package/src/testing/scenario-test/code-block-expectation.ts

@@ -0,0 +1,115 @@
+import type { SnippetExtractor } from "./snippet-extractor.js";
+
+export interface ElementQuery {
+  /** Type to query */
+  type: string;
+
+  /** Name of the type to query */
+  name: string;
+}
+
+export interface CodeBlockQuery {
+  /** Language */
+  lang: string;
+
+  /** File path */
+  file: string;
+
+  /** Query for content in the file */
+  query?: ElementQuery;
+}
+
+export interface CodeBlockExpectation extends CodeBlockQuery {
+  /** Expected content of the code block */
+  expected: string;
+}
+
+/**
+ * Parse a markdown code block following the expectation syntax
+ */
+export function parseCodeblockExpectation(heading: string, content: string): CodeBlockExpectation {
+  const query = parseCodeBlockHeading(heading);
+  return {
+    ...query,
+    expected: content,
+  };
+}
+
+/**
+ * Parse the codeblock heading for what it should validate.
+ * Expected format
+ * ```
+ * ts path/to/file.ts {type} {name}
+ * ```
+ */
+export function parseCodeBlockHeading(heading: string): CodeBlockQuery {
+  const [lang, file, type, name] = heading.split(" ");
+  if (!file) {
+    throw new Error(
+      `Invalid code block heading: "${heading}". Missing file path. Expected format: "<lang> <path>"`,
+    );
+  }
+
+  if (type && !name) {
+    throw new Error(
+      `Invalid code block heading: "${heading}". Missing name when using type. Expected format: "<lang> <path> [type] [name]"`,
+    );
+  }
+
+  return { lang, file, query: type ? { type, name } : undefined };
+}
+
+export function getExcerptForQuery(
+  snippetExtractor: SnippetExtractor,
+  expectation: CodeBlockQuery,
+  outputs: Record<string, string>,
+): string {
+  const content = outputs[expectation.file];
+
+  if (!content) {
+    throw new Error(
+      `File ${expectation.file} not found in emitted files:\n ${Object.keys(outputs).join("\n")}`,
+    );
+  }
+
+  return getExcerptInFile(snippetExtractor, expectation, content);
+}
+
+function getExcerptInFile(
+  snippetExtractor: SnippetExtractor,
+  expectation: CodeBlockQuery,
+  sourceFile: string,
+): string {
+  if (expectation.query) {
+    const excerpt = tryGetExcerptInFile(snippetExtractor, expectation.query, sourceFile);
+    if (!excerpt) {
+      throw new Error(
+        `Could not find ${expectation.query.type} "${expectation.query.name}" in file "${expectation.file}".`,
+      );
+    }
+    return excerpt;
+  } else {
+    return sourceFile;
+  }
+}
+
+function tryGetExcerptInFile(
+  snippetExtractor: SnippetExtractor,
+  query: ElementQuery,
+  sourceFile: string,
+): string | null {
+  switch (query.type) {
+    case "interface":
+      return snippetExtractor.getInterface(sourceFile, query.name);
+    case "type":
+      return snippetExtractor.getTypeAlias(sourceFile, query.name);
+    case "enum":
+      return snippetExtractor.getEnum(sourceFile, query.name);
+    case "function":
+      return snippetExtractor.getFunction(sourceFile, query.name);
+    case "class":
+      return snippetExtractor.getClass(sourceFile, query.name);
+    default:
+      throw new Error("Unsupported type in code block expectation: " + query.type);
+  }
+}

package/src/testing/scenario-test/harness.ts

@@ -1,176 +1,59 @@
-import { normalizePath } from "@typespec/compiler";
-import type { TypeSpecTestLibrary } from "@typespec/compiler/testing";
+import { logDiagnostics, NodeHost } from "@typespec/compiler";
+import { expectDiagnosticEmpty, type EmitterTester } from "@typespec/compiler/testing";
 import { readdirSync, readFileSync, statSync, writeFileSync } from "fs";
-import minimist from "minimist";
-import path from "path";
+import { join } from "pathe";
 import { format } from "prettier";
-import { afterAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import {
+  getExcerptForQuery,
+  parseCodeblockExpectation,
+  type CodeBlockExpectation,
+} from "./code-block-expectation.js";
 import type { LanguageConfiguration, SnippetExtractor } from "./snippet-extractor.js";
-import { emitWithDiagnostics } from "./test-host.js";
-
-const rawArgs = process.env.TEST_ARGS ? process.env.TEST_ARGS.split(" ") : [];
-
-// Parse command-line arguments with minimist
-const args = minimist(rawArgs, {
-  alias: {
-    filter: "f", // Short alias for `--filter`
-  },
-  default: {
-    filter: undefined, // Default to undefined if no filter is provided
-  },
-});
-
-// Extract the filter paths from the parsed arguments
-const filterPaths = args.filter
-  ? Array.isArray(args.filter) // Handle single or multiple file paths
-    ? args.filter
-    : [args.filter]
-  : undefined;
 
 const SCENARIOS_UPDATE =
   process.env["RECORD"] === "true" || process.env["SCENARIOS_UPDATE"] === "true";
 
-type EmitterFunction = (tsp: string, namedArgs: Record<string, string>) => Promise<string>;
-
-async function assertGetEmittedFile(
-  testLibrary: TypeSpecTestLibrary,
-  emitterOutputDir: string,
-  file: string,
-  code: string,
-) {
-  const [emittedFiles, diagnostics] = await emitWithDiagnostics(
-    testLibrary,
-    emitterOutputDir,
-    code,
-  );
-
-  const errors = diagnostics.filter((d) => d.severity === "error");
-  const warnings = diagnostics.filter((d) => d.severity === "warning");
-  if (warnings.length > 0) {
-    // eslint-disable-next-line no-console
-    console.warn(`Warning compiling code:\n ${warnings.map((x) => x.message).join("\n")}`);
-  }
-  if (errors.length > 0) {
-    throw new Error(`Error compiling code:\n ${errors.map((x) => x.message).join("\n")}`);
-  }
-
-  const normalizedTarget = normalizePath(file);
-  const sourceFile = emittedFiles.find((x) => normalizePath(x.path) === normalizedTarget);
-
-  if (!sourceFile) {
-    throw new Error(
-      `File ${file} not found in emitted files:\n ${emittedFiles.map((f) => f.path).join("\n")}`,
-    );
-  }
-  return sourceFile;
-}
-
-/**
- * Mapping of different snapshot types to how to get them.
- * Snapshot types can take single-word string arguments templated in curly braces {} and are otherwise regex
- */
-function getCodeBlockTypes(
-  testLibrary: TypeSpecTestLibrary,
-  languageConfiguration: LanguageConfiguration,
-  emitterOutputDir: string,
-  snippetExtractor: SnippetExtractor,
-): Record<string, EmitterFunction> {
-  const languageTags = languageConfiguration.codeBlockTypes.join("|");
-  return {
-    // Snapshot of a particular interface named {name} in the models file
-    [`(${languageTags}) {file} interface {name}`]: async (code, { file, name }) => {
-      const sourceFile = await assertGetEmittedFile(testLibrary, emitterOutputDir, file, code);
-      const snippet = snippetExtractor.getInterface(sourceFile.content, name);
-
-      if (!snippet) {
-        throw new Error(`Interface ${name} not found in ${file}`);
-      }
-
-      return snippet;
-    },
-
-    [`(${languageTags}) {file} type {name}`]: async (code, { file, name }) => {
-      const sourceFile = await assertGetEmittedFile(testLibrary, emitterOutputDir, file, code);
-      const snippet = snippetExtractor.getTypeAlias(sourceFile.content, name);
-
-      if (!snippet) {
-        throw new Error(`Type alias ${name} not found in ${file}`);
-      }
-
-      return snippet;
-    },
-
-    // Snapshot of a particular function named {name} in the models file
-    [`(${languageTags}) {file} function {name}`]: async (code, { file, name }) => {
-      const sourceFile = await assertGetEmittedFile(testLibrary, emitterOutputDir, file, code);
-      const snippet = snippetExtractor.getFunction(sourceFile.content, name);
-
-      if (!snippet) {
-        throw new Error(`Function ${name} not found in ${file}`);
-      }
-
-      return snippet;
-    },
-
-    // Snapshot of a particular class named {name} in the models file
-    [`(${languageTags}) {file} class {name}`]: async (code, { file, name }) => {
-      const sourceFile = await assertGetEmittedFile(testLibrary, emitterOutputDir, file, code);
-      const snippet = snippetExtractor.getClass(sourceFile.content, name);
-
-      if (!snippet) {
-        throw new Error(`Class ${name} not found in ${file}`);
-      }
-
-      return snippet;
-    },
-
-    // Snapshot of the entire file
-    [`(${languageTags}) {file}`]: async (code, { file }) => {
-      const sourceFile = await assertGetEmittedFile(testLibrary, emitterOutputDir, file, code);
-      return sourceFile.content;
-    },
-  };
-}
-
 export async function executeScenarios(
-  testLibrary: TypeSpecTestLibrary,
+  tester: EmitterTester,
   languageConfiguration: LanguageConfiguration,
   scenariosLocation: string,
-  emitterOutputDir: string,
   snippetExtractor: SnippetExtractor,
 ) {
-  const scenarioList = filterPaths ?? [];
-  // eslint-disable-next-line no-console
-  scenarioList.length && console.log("Filtering scenarios: ", scenarioList);
+  const scenarioList = discoverAllScenarios(scenariosLocation);
 
-  if (!scenarioList.length) {
-    // Add all scenarios.
-    discoverAllScenarios(scenariosLocation, scenarioList);
-  }
-
-  describeScenarios(
-    scenarioList,
-    testLibrary,
-    languageConfiguration,
-    emitterOutputDir,
-    snippetExtractor,
-  );
+  describeScenarios(scenarioList, tester, languageConfiguration, snippetExtractor);
 }
 
-function discoverAllScenarios(location: string, scenarios: string[]) {
-  const children = readdirSync(location);
-  for (const child of children) {
-    const fullPath = path.join(location, child);
-    const stat = statSync(fullPath);
-    if (stat.isDirectory()) {
-      discoverAllScenarios(fullPath, scenarios);
-    } else {
-      scenarios.push(fullPath);
+function discoverAllScenarios(dir: string): ScenarioFileId[] {
+  const scenarios: ScenarioFileId[] = [];
+
+  function recurse(current: string) {
+    const children = readdirSync(join(dir, current));
+    for (const child of children) {
+      const fullPath = join(dir, current, child);
+      const stat = statSync(fullPath);
+      if (stat.isDirectory()) {
+        recurse(join(current, child));
+      } else {
+        scenarios.push({ path: fullPath, relativePath: join(current, child) });
+      }
     }
   }
 
+  recurse("");
   return scenarios;
 }
+
+interface ScenarioFileId {
+  path: string;
+  relativePath: string;
+}
+
+interface ScenarioFile extends ScenarioFileId {
+  scenarios: Scenario[];
+}
+
 interface Scenario {
   // The title of the scenario delimited by H1
   title: string;
@@ -186,54 +69,32 @@ interface ScenarioContents {
 
 interface SpecCodeBlock {
   kind: "spec" | "test";
-  content: string[];
+  content: string;
 }
 
 interface TestCodeBlock {
   kind: "test";
   heading: string;
-  content: string[];
-  matchedTemplate: {
-    template: string;
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
-    fn: Function;
-    namedArgs: Record<string, string> | null;
-  };
+  content: string;
+  expectation: CodeBlockExpectation;
 }
 
 type ScenarioCodeBlock = SpecCodeBlock | TestCodeBlock;
 
-interface ScenarioFile {
-  path: string;
-  scenarios: Scenario[];
-}
-
-function parseFile(
-  path: string,
-  testLibrary: TypeSpecTestLibrary,
-  languageConfiguration: LanguageConfiguration,
-  emitterOutputDir: string,
-  snippetExtractor: SnippetExtractor,
-): ScenarioFile {
+function parseFile(file: ScenarioFileId): ScenarioFile {
   // Read the whole file
-  const rawContent = readFileSync(path, { encoding: "utf-8" });
+  const rawContent = readFileSync(file.path, { encoding: "utf-8" });
 
   // Split the content by H1
   const sections = splitByH1(rawContent);
 
   const scenarioFile: ScenarioFile = {
-    path,
+    ...file,
     scenarios: [],
   };
 
   for (const section of sections) {
-    const scenarioContent = parseScenario(
-      section.content,
-      testLibrary,
-      languageConfiguration,
-      emitterOutputDir,
-      snippetExtractor,
-    );
+    const scenarioContent = parseScenario(section.content);
     const scenario: Scenario = {
       title: section.title,
       content: scenarioContent,
@@ -249,59 +110,43 @@ function isTestCodeBlock(codeBlock: ScenarioCodeBlock): codeBlock is TestCodeBlo
   return codeBlock.kind === "test";
 }
 
-function parseScenario(
-  content: string,
-  testLibrary: TypeSpecTestLibrary,
-  languageConfiguration: LanguageConfiguration,
-  emitterOutputDir: string,
-  snippetExtractor: SnippetExtractor,
-): ScenarioContents {
+function parseScenario(content: string): ScenarioContents {
   const rawLines = content.split("\n");
   const scenario: ScenarioContents = {
     lines: [],
-    specBlock: { kind: "spec", content: [] },
+    specBlock: { kind: "spec", content: "" },
     testBlocks: [],
   };
 
-  let currentCodeBlock: ScenarioCodeBlock | null = null;
-
-  // Precompute output code block types once
-  const outputCodeBlockTypes = getCodeBlockTypes(
-    testLibrary,
-    languageConfiguration,
-    emitterOutputDir,
-    snippetExtractor,
-  );
+  let currentCodeBlock: { heading: string; content: string[] } | null = null;
 
   for (const line of rawLines) {
     if (line.startsWith("```") && currentCodeBlock) {
-      // Close the code block
-      scenario.lines.push(currentCodeBlock);
-      if (!isTestCodeBlock(currentCodeBlock)) {
-        scenario.specBlock.content = currentCodeBlock.content;
+      const heading = currentCodeBlock.heading;
+      const codeBlockKind =
+        heading.includes("tsp") || heading.includes("typespec") ? "spec" : "test";
+      const content = currentCodeBlock.content.join("\n");
+      if (codeBlockKind === "spec") {
+        const codeblock: SpecCodeBlock = {
+          kind: "spec",
+          content,
+        };
+        scenario.lines.push(codeblock);
+        scenario.specBlock.content = content;
       } else {
-        for (const [template, fn] of Object.entries(outputCodeBlockTypes)) {
-          const templateRegex = new RegExp(
-            "^" + template.replace(/\{(\w+)\}/g, "(?<$1>[^\\s]+)") + "$",
-          );
-
-          const match = currentCodeBlock.heading.match(templateRegex);
-          if (match) {
-            currentCodeBlock.matchedTemplate = {
-              template,
-              fn,
-              namedArgs: match.groups ?? null,
-            };
-            break;
-          }
-        }
-        scenario.testBlocks.push(currentCodeBlock);
+        const codeblock: TestCodeBlock = {
+          kind: "test",
+          heading: currentCodeBlock.heading,
+          content,
+          expectation: parseCodeblockExpectation(currentCodeBlock.heading, content),
+        };
+        scenario.lines.push(codeblock);
+        scenario.testBlocks.push(codeblock);
       }
       currentCodeBlock = null;
     } else if (line.startsWith("```")) {
-      const codeBlockKind = line.includes("tsp") || line.includes("typespec") ? "spec" : "test";
       // Start a new code block
-      currentCodeBlock = { kind: codeBlockKind, heading: line.substring(3), content: [] };
+      currentCodeBlock = { heading: line.substring(3), content: [] };
     } else if (currentCodeBlock) {
       // Append to code block content
       currentCodeBlock.content.push(line);
@@ -315,37 +160,47 @@ function parseScenario(
 }
 
 function describeScenarios(
-  scenarioFiles: string[],
-  testLibrary: TypeSpecTestLibrary,
+  scenarioFiles: ScenarioFileId[],
+  tester: EmitterTester,
   languageConfiguration: LanguageConfiguration,
-  emitterOutputDir: string,
   snippetExtractor: SnippetExtractor,
 ) {
-  const scenarios = scenarioFiles.map((f) =>
-    parseFile(f, testLibrary, languageConfiguration, emitterOutputDir, snippetExtractor),
-  );
+  const scenarios = scenarioFiles.map((f) => parseFile(f));
 
   for (const scenarioFile of scenarios) {
-    describe(`Scenario File: ${scenarioFile.path}`, () => {
+    describe(`${scenarioFile.relativePath}`, () => {
       for (const scenario of scenarioFile.scenarios) {
         const isOnly = scenario.title.includes("only:");
         const isSkip = scenario.title.includes("skip:");
-
         const describeFn = isSkip ? describe.skip : isOnly ? describe.only : describe;
 
+        let outputFiles: Record<string, string>;
+        beforeAll(async () => {
+          const code = scenario.content.specBlock.content;
+          const [{ outputs }, diagnostics] = await tester.compileAndDiagnose(code);
+          const errors = diagnostics.filter((d) => d.severity === "error");
+          const warnings = diagnostics.filter((d) => d.severity === "warning");
+          if (warnings.length > 0) {
+            // TODO: this should ideally fail the test or be part of the expectation.
+            logDiagnostics(warnings, NodeHost.logSink);
+          }
+          expectDiagnosticEmpty(errors);
+          outputFiles = outputs;
+        });
+
         describeFn(`Scenario: ${scenario.title}`, () => {
           for (const testBlock of scenario.content.testBlocks) {
             it(`Test: ${testBlock.heading}`, async () => {
-              const { fn, namedArgs } = testBlock.matchedTemplate;
-              const result = await fn(
-                scenario.content.specBlock.content.join("\n"),
-                namedArgs ?? {},
+              const result = getExcerptForQuery(
+                snippetExtractor,
+                testBlock.expectation,
+                outputFiles,
               );
 
               if (SCENARIOS_UPDATE) {
-                testBlock.content = (await languageConfiguration.format(result)).split("\n");
+                testBlock.content = await languageConfiguration.format(result);
               } else {
-                const expected = await languageConfiguration.format(testBlock.content.join("\n"));
+                const expected = await languageConfiguration.format(testBlock.content);
                 const actual = await languageConfiguration.format(result);
                 expect(actual).toBe(expected);
               }
@@ -375,7 +230,7 @@ async function updateFile(scenarioFile: ScenarioFile) {
       } else {
         const heading = isTestCodeBlock(line) ? line.heading : "tsp";
         newContent.push("```" + heading);
-        newContent.push(...line.content);
+        newContent.push(line.content);
         newContent.push("```");
       }
     }

package/src/testing/scenario-test/index.ts

@@ -1,3 +1,2 @@
 export * from "./harness.js";
 export * from "./snippet-extractor.js";
-export * from "./test-host.js";

package/src/testing/scenario-test/snippet-extractor.ts

@@ -92,7 +92,7 @@ export interface LanguageConfiguration {
 
 await Parser.init();
 
-export function createSnipperExtractor(
+export function createSnippetExtractor(
   languageConfiguration: LanguageConfiguration,
 ): SnippetExtractor {
   return new SnippetExtractorImpl(languageConfiguration);

package/test/testing/snippet-extractor-csharp.test.ts

@@ -2,14 +2,14 @@ import { d } from "@alloy-js/core/testing";
 import { beforeEach, describe, expect, it } from "vitest";
 import {
   createCSharpExtractorConfig,
-  createSnipperExtractor,
+  createSnippetExtractor,
   type SnippetExtractor,
 } from "../../src/testing/index.js";
 describe("C# Snippet Extractor", () => {
   let extractor: SnippetExtractor;
 
   beforeEach(async () => {
-    extractor = createSnipperExtractor(await createCSharpExtractorConfig());
+    extractor = createSnippetExtractor(await createCSharpExtractorConfig());
   });
 
   it("should extract a class", () => {
@@ -66,7 +66,7 @@ describe("C# Snippet Extractor - Enums", () => {
   let extractor: SnippetExtractor;
 
   beforeEach(async () => {
-    extractor = createSnipperExtractor(await createCSharpExtractorConfig());
+    extractor = createSnippetExtractor(await createCSharpExtractorConfig());
   });
 
   it("should extract a basic enum", async () => {