@typespec/emitter-framework 0.9.0-dev.8 → 0.10.0-dev.0

package/src/testing/scenario-test/harness.ts

@@ -1,176 +1,59 @@
-import { normalizePath } from "@typespec/compiler";
-import type { TypeSpecTestLibrary } from "@typespec/compiler/testing";
+import { logDiagnostics, NodeHost } from "@typespec/compiler";
+import { expectDiagnosticEmpty, type EmitterTester } from "@typespec/compiler/testing";
 import { readdirSync, readFileSync, statSync, writeFileSync } from "fs";
-import minimist from "minimist";
-import path from "path";
+import { join } from "pathe";
 import { format } from "prettier";
-import { afterAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import {
+  getExcerptForQuery,
+  parseCodeblockExpectation,
+  type CodeBlockExpectation,
+} from "./code-block-expectation.js";
 import type { LanguageConfiguration, SnippetExtractor } from "./snippet-extractor.js";
-import { emitWithDiagnostics } from "./test-host.js";
-
-const rawArgs = process.env.TEST_ARGS ? process.env.TEST_ARGS.split(" ") : [];
-
-// Parse command-line arguments with minimist
-const args = minimist(rawArgs, {
-  alias: {
-    filter: "f", // Short alias for `--filter`
-  },
-  default: {
-    filter: undefined, // Default to undefined if no filter is provided
-  },
-});
-
-// Extract the filter paths from the parsed arguments
-const filterPaths = args.filter
-  ? Array.isArray(args.filter) // Handle single or multiple file paths
-    ? args.filter
-    : [args.filter]
-  : undefined;
 
 const SCENARIOS_UPDATE =
   process.env["RECORD"] === "true" || process.env["SCENARIOS_UPDATE"] === "true";
 
-type EmitterFunction = (tsp: string, namedArgs: Record<string, string>) => Promise<string>;
-
-async function assertGetEmittedFile(
-  testLibrary: TypeSpecTestLibrary,
-  emitterOutputDir: string,
-  file: string,
-  code: string,
-) {
-  const [emittedFiles, diagnostics] = await emitWithDiagnostics(
-    testLibrary,
-    emitterOutputDir,
-    code,
-  );
-
-  const errors = diagnostics.filter((d) => d.severity === "error");
-  const warnings = diagnostics.filter((d) => d.severity === "warning");
-  if (warnings.length > 0) {
-    // eslint-disable-next-line no-console
-    console.warn(`Warning compiling code:\n ${warnings.map((x) => x.message).join("\n")}`);
-  }
-  if (errors.length > 0) {
-    throw new Error(`Error compiling code:\n ${errors.map((x) => x.message).join("\n")}`);
-  }
-
-  const normalizedTarget = normalizePath(file);
-  const sourceFile = emittedFiles.find((x) => normalizePath(x.path) === normalizedTarget);
-
-  if (!sourceFile) {
-    throw new Error(
-      `File ${file} not found in emitted files:\n ${emittedFiles.map((f) => f.path).join("\n")}`,
-    );
-  }
-  return sourceFile;
-}
-
-/**
- * Mapping of different snapshot types to how to get them.
- * Snapshot types can take single-word string arguments templated in curly braces {} and are otherwise regex
- */
-function getCodeBlockTypes(
-  testLibrary: TypeSpecTestLibrary,
-  languageConfiguration: LanguageConfiguration,
-  emitterOutputDir: string,
-  snippetExtractor: SnippetExtractor,
-): Record<string, EmitterFunction> {
-  const languageTags = languageConfiguration.codeBlockTypes.join("|");
-  return {
-    // Snapshot of a particular interface named {name} in the models file
-    [`(${languageTags}) {file} interface {name}`]: async (code, { file, name }) => {
-      const sourceFile = await assertGetEmittedFile(testLibrary, emitterOutputDir, file, code);
-      const snippet = snippetExtractor.getInterface(sourceFile.content, name);
-
-      if (!snippet) {
-        throw new Error(`Interface ${name} not found in ${file}`);
-      }
-
-      return snippet;
-    },
-
-    [`(${languageTags}) {file} type {name}`]: async (code, { file, name }) => {
-      const sourceFile = await assertGetEmittedFile(testLibrary, emitterOutputDir, file, code);
-      const snippet = snippetExtractor.getTypeAlias(sourceFile.content, name);
-
-      if (!snippet) {
-        throw new Error(`Type alias ${name} not found in ${file}`);
-      }
-
-      return snippet;
-    },
-
-    // Snapshot of a particular function named {name} in the models file
-    [`(${languageTags}) {file} function {name}`]: async (code, { file, name }) => {
-      const sourceFile = await assertGetEmittedFile(testLibrary, emitterOutputDir, file, code);
-      const snippet = snippetExtractor.getFunction(sourceFile.content, name);
-
-      if (!snippet) {
-        throw new Error(`Function ${name} not found in ${file}`);
-      }
-
-      return snippet;
-    },
-
-    // Snapshot of a particular class named {name} in the models file
-    [`(${languageTags}) {file} class {name}`]: async (code, { file, name }) => {
-      const sourceFile = await assertGetEmittedFile(testLibrary, emitterOutputDir, file, code);
-      const snippet = snippetExtractor.getClass(sourceFile.content, name);
-
-      if (!snippet) {
-        throw new Error(`Class ${name} not found in ${file}`);
-      }
-
-      return snippet;
-    },
-
-    // Snapshot of the entire file
-    [`(${languageTags}) {file}`]: async (code, { file }) => {
-      const sourceFile = await assertGetEmittedFile(testLibrary, emitterOutputDir, file, code);
-      return sourceFile.content;
-    },
-  };
-}
-
 export async function executeScenarios(
-  testLibrary: TypeSpecTestLibrary,
+  tester: EmitterTester,
   languageConfiguration: LanguageConfiguration,
   scenariosLocation: string,
-  emitterOutputDir: string,
   snippetExtractor: SnippetExtractor,
 ) {
-  const scenarioList = filterPaths ?? [];
-  // eslint-disable-next-line no-console
-  scenarioList.length && console.log("Filtering scenarios: ", scenarioList);
+  const scenarioList = discoverAllScenarios(scenariosLocation);
 
-  if (!scenarioList.length) {
-    // Add all scenarios.
-    discoverAllScenarios(scenariosLocation, scenarioList);
-  }
-
-  describeScenarios(
-    scenarioList,
-    testLibrary,
-    languageConfiguration,
-    emitterOutputDir,
-    snippetExtractor,
-  );
+  describeScenarios(scenarioList, tester, languageConfiguration, snippetExtractor);
 }
 
-function discoverAllScenarios(location: string, scenarios: string[]) {
-  const children = readdirSync(location);
-  for (const child of children) {
-    const fullPath = path.join(location, child);
-    const stat = statSync(fullPath);
-    if (stat.isDirectory()) {
-      discoverAllScenarios(fullPath, scenarios);
-    } else {
-      scenarios.push(fullPath);
+function discoverAllScenarios(dir: string): ScenarioFileId[] {
+  const scenarios: ScenarioFileId[] = [];
+
+  function recurse(current: string) {
+    const children = readdirSync(join(dir, current));
+    for (const child of children) {
+      const fullPath = join(dir, current, child);
+      const stat = statSync(fullPath);
+      if (stat.isDirectory()) {
+        recurse(join(current, child));
+      } else {
+        scenarios.push({ path: fullPath, relativePath: join(current, child) });
+      }
     }
   }
 
+  recurse("");
   return scenarios;
 }
+
+interface ScenarioFileId {
+  path: string;
+  relativePath: string;
+}
+
+interface ScenarioFile extends ScenarioFileId {
+  scenarios: Scenario[];
+}
+
 interface Scenario {
   // The title of the scenario delimited by H1
   title: string;
@@ -186,54 +69,32 @@ interface ScenarioContents {
 
 interface SpecCodeBlock {
   kind: "spec" | "test";
-  content: string[];
+  content: string;
 }
 
 interface TestCodeBlock {
   kind: "test";
   heading: string;
-  content: string[];
-  matchedTemplate: {
-    template: string;
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
-    fn: Function;
-    namedArgs: Record<string, string> | null;
-  };
+  content: string;
+  expectation: CodeBlockExpectation;
 }
 
 type ScenarioCodeBlock = SpecCodeBlock | TestCodeBlock;
 
-interface ScenarioFile {
-  path: string;
-  scenarios: Scenario[];
-}
-
-function parseFile(
-  path: string,
-  testLibrary: TypeSpecTestLibrary,
-  languageConfiguration: LanguageConfiguration,
-  emitterOutputDir: string,
-  snippetExtractor: SnippetExtractor,
-): ScenarioFile {
+function parseFile(file: ScenarioFileId): ScenarioFile {
   // Read the whole file
-  const rawContent = readFileSync(path, { encoding: "utf-8" });
+  const rawContent = readFileSync(file.path, { encoding: "utf-8" });
 
   // Split the content by H1
   const sections = splitByH1(rawContent);
 
   const scenarioFile: ScenarioFile = {
-    path,
+    ...file,
     scenarios: [],
   };
 
   for (const section of sections) {
-    const scenarioContent = parseScenario(
-      section.content,
-      testLibrary,
-      languageConfiguration,
-      emitterOutputDir,
-      snippetExtractor,
-    );
+    const scenarioContent = parseScenario(section.content);
     const scenario: Scenario = {
       title: section.title,
       content: scenarioContent,
@@ -249,59 +110,43 @@ function isTestCodeBlock(codeBlock: ScenarioCodeBlock): codeBlock is TestCodeBlo
   return codeBlock.kind === "test";
 }
 
-function parseScenario(
-  content: string,
-  testLibrary: TypeSpecTestLibrary,
-  languageConfiguration: LanguageConfiguration,
-  emitterOutputDir: string,
-  snippetExtractor: SnippetExtractor,
-): ScenarioContents {
+function parseScenario(content: string): ScenarioContents {
   const rawLines = content.split("\n");
   const scenario: ScenarioContents = {
     lines: [],
-    specBlock: { kind: "spec", content: [] },
+    specBlock: { kind: "spec", content: "" },
     testBlocks: [],
   };
 
-  let currentCodeBlock: ScenarioCodeBlock | null = null;
-
-  // Precompute output code block types once
-  const outputCodeBlockTypes = getCodeBlockTypes(
-    testLibrary,
-    languageConfiguration,
-    emitterOutputDir,
-    snippetExtractor,
-  );
+  let currentCodeBlock: { heading: string; content: string[] } | null = null;
 
   for (const line of rawLines) {
     if (line.startsWith("```") && currentCodeBlock) {
-      // Close the code block
-      scenario.lines.push(currentCodeBlock);
-      if (!isTestCodeBlock(currentCodeBlock)) {
-        scenario.specBlock.content = currentCodeBlock.content;
+      const heading = currentCodeBlock.heading;
+      const codeBlockKind =
+        heading.includes("tsp") || heading.includes("typespec") ? "spec" : "test";
+      const content = currentCodeBlock.content.join("\n");
+      if (codeBlockKind === "spec") {
+        const codeblock: SpecCodeBlock = {
+          kind: "spec",
+          content,
+        };
+        scenario.lines.push(codeblock);
+        scenario.specBlock.content = content;
       } else {
-        for (const [template, fn] of Object.entries(outputCodeBlockTypes)) {
-          const templateRegex = new RegExp(
-            "^" + template.replace(/\{(\w+)\}/g, "(?<$1>[^\\s]+)") + "$",
-          );
-
-          const match = currentCodeBlock.heading.match(templateRegex);
-          if (match) {
-            currentCodeBlock.matchedTemplate = {
-              template,
-              fn,
-              namedArgs: match.groups ?? null,
-            };
-            break;
-          }
-        }
-        scenario.testBlocks.push(currentCodeBlock);
+        const codeblock: TestCodeBlock = {
+          kind: "test",
+          heading: currentCodeBlock.heading,
+          content,
+          expectation: parseCodeblockExpectation(currentCodeBlock.heading, content),
+        };
+        scenario.lines.push(codeblock);
+        scenario.testBlocks.push(codeblock);
       }
       currentCodeBlock = null;
     } else if (line.startsWith("```")) {
-      const codeBlockKind = line.includes("tsp") || line.includes("typespec") ? "spec" : "test";
       // Start a new code block
-      currentCodeBlock = { kind: codeBlockKind, heading: line.substring(3), content: [] };
+      currentCodeBlock = { heading: line.substring(3), content: [] };
     } else if (currentCodeBlock) {
       // Append to code block content
       currentCodeBlock.content.push(line);
@@ -315,37 +160,47 @@ function parseScenario(
 }
 
 function describeScenarios(
-  scenarioFiles: string[],
-  testLibrary: TypeSpecTestLibrary,
+  scenarioFiles: ScenarioFileId[],
+  tester: EmitterTester,
   languageConfiguration: LanguageConfiguration,
-  emitterOutputDir: string,
   snippetExtractor: SnippetExtractor,
 ) {
-  const scenarios = scenarioFiles.map((f) =>
-    parseFile(f, testLibrary, languageConfiguration, emitterOutputDir, snippetExtractor),
-  );
+  const scenarios = scenarioFiles.map((f) => parseFile(f));
 
   for (const scenarioFile of scenarios) {
-    describe(`Scenario File: ${scenarioFile.path}`, () => {
+    describe(`${scenarioFile.relativePath}`, () => {
       for (const scenario of scenarioFile.scenarios) {
         const isOnly = scenario.title.includes("only:");
         const isSkip = scenario.title.includes("skip:");
-
         const describeFn = isSkip ? describe.skip : isOnly ? describe.only : describe;
 
+        let outputFiles: Record<string, string>;
+        beforeAll(async () => {
+          const code = scenario.content.specBlock.content;
+          const [{ outputs }, diagnostics] = await tester.compileAndDiagnose(code);
+          const errors = diagnostics.filter((d) => d.severity === "error");
+          const warnings = diagnostics.filter((d) => d.severity === "warning");
+          if (warnings.length > 0) {
+            // TODO: this should ideally fail the test or be part of the expectation.
+            logDiagnostics(warnings, NodeHost.logSink);
+          }
+          expectDiagnosticEmpty(errors);
+          outputFiles = outputs;
+        });
+
         describeFn(`Scenario: ${scenario.title}`, () => {
           for (const testBlock of scenario.content.testBlocks) {
             it(`Test: ${testBlock.heading}`, async () => {
-              const { fn, namedArgs } = testBlock.matchedTemplate;
-              const result = await fn(
-                scenario.content.specBlock.content.join("\n"),
-                namedArgs ?? {},
+              const result = getExcerptForQuery(
+                snippetExtractor,
+                testBlock.expectation,
+                outputFiles,
               );
 
               if (SCENARIOS_UPDATE) {
-                testBlock.content = (await languageConfiguration.format(result)).split("\n");
+                testBlock.content = await languageConfiguration.format(result);
               } else {
-                const expected = await languageConfiguration.format(testBlock.content.join("\n"));
+                const expected = await languageConfiguration.format(testBlock.content);
                 const actual = await languageConfiguration.format(result);
                 expect(actual).toBe(expected);
               }
@@ -375,7 +230,7 @@ async function updateFile(scenarioFile: ScenarioFile) {
       } else {
         const heading = isTestCodeBlock(line) ? line.heading : "tsp";
         newContent.push("```" + heading);
-        newContent.push(...line.content);
+        newContent.push(line.content);
         newContent.push("```");
       }
     }

package/src/testing/scenario-test/index.ts

@@ -1,3 +1,2 @@
 export * from "./harness.js";
 export * from "./snippet-extractor.js";
-export * from "./test-host.js";

package/src/testing/scenario-test/snippet-extractor.ts

@@ -92,7 +92,7 @@ export interface LanguageConfiguration {
 
 await Parser.init();
 
-export function createSnipperExtractor(
+export function createSnippetExtractor(
   languageConfiguration: LanguageConfiguration,
 ): SnippetExtractor {
   return new SnippetExtractorImpl(languageConfiguration);

package/test/testing/snippet-extractor-csharp.test.ts

@@ -2,14 +2,14 @@ import { d } from "@alloy-js/core/testing";
 import { beforeEach, describe, expect, it } from "vitest";
 import {
   createCSharpExtractorConfig,
-  createSnipperExtractor,
+  createSnippetExtractor,
   type SnippetExtractor,
 } from "../../src/testing/index.js";
 describe("C# Snippet Extractor", () => {
   let extractor: SnippetExtractor;
 
   beforeEach(async () => {
-    extractor = createSnipperExtractor(await createCSharpExtractorConfig());
+    extractor = createSnippetExtractor(await createCSharpExtractorConfig());
   });
 
   it("should extract a class", () => {
@@ -66,7 +66,7 @@ describe("C# Snippet Extractor - Enums", () => {
   let extractor: SnippetExtractor;
 
   beforeEach(async () => {
-    extractor = createSnipperExtractor(await createCSharpExtractorConfig());
+    extractor = createSnippetExtractor(await createCSharpExtractorConfig());
   });
 
   it("should extract a basic enum", async () => {

package/test/testing/snippet-extractor-java.test.ts

@@ -2,7 +2,7 @@ import { d } from "@alloy-js/core/testing";
 import { beforeEach, describe, expect, it } from "vitest";
 import {
   createJavaExtractorConfig,
-  createSnipperExtractor,
+  createSnippetExtractor,
   type SnippetExtractor,
 } from "../../src/testing/index.js";
 
@@ -10,7 +10,7 @@ describe("Java Snippet Extractor", () => {
   let extractor: SnippetExtractor;
 
   beforeEach(async () => {
-    extractor = createSnipperExtractor(await createJavaExtractorConfig());
+    extractor = createSnippetExtractor(await createJavaExtractorConfig());
   });
 
   it("should extract a class", () => {
@@ -67,7 +67,7 @@ describe("Java Snippet Extractor - Enums", () => {
   let extractor: SnippetExtractor;
 
   beforeEach(async () => {
-    extractor = createSnipperExtractor(await createJavaExtractorConfig());
+    extractor = createSnippetExtractor(await createJavaExtractorConfig());
   });
 
   it("should extract a basic enum", async () => {

package/test/testing/snippet-extractor-python.test.ts

@@ -2,7 +2,7 @@ import { d } from "@alloy-js/core/testing";
 import { beforeEach, describe, expect, it } from "vitest";
 import {
   createPythonExtractorConfig,
-  createSnipperExtractor,
+  createSnippetExtractor,
   type SnippetExtractor,
 } from "../../src/testing/index.js";
 
@@ -10,7 +10,7 @@ describe("Python Snippet Extractor", () => {
   let extractor: SnippetExtractor;
 
   beforeEach(async () => {
-    extractor = createSnipperExtractor(await createPythonExtractorConfig());
+    extractor = createSnippetExtractor(await createPythonExtractorConfig());
   });
 
   it("should extract a class", () => {

package/test/testing/snippet-extractor-typescript.test.ts

@@ -1,7 +1,7 @@
 import { d } from "@alloy-js/core/testing";
 import { beforeEach, describe, expect, it } from "vitest";
 import {
-  createSnipperExtractor,
+  createSnippetExtractor,
   createTypeScriptExtractorConfig,
   type SnippetExtractor,
 } from "../../src/testing/index.js";
@@ -9,7 +9,7 @@ import {
 describe("TypeScript Snippet Extractor", () => {
   let extractor: SnippetExtractor;
   beforeEach(async () => {
-    extractor = createSnipperExtractor(await createTypeScriptExtractorConfig());
+    extractor = createSnippetExtractor(await createTypeScriptExtractorConfig());
   });
 
   it("should extract a class", async () => {
@@ -138,7 +138,7 @@ describe("TypeScript Snippet Extractor", () => {
 describe("TypeScript Snippet Extractor - Enums", () => {
   let extractor: SnippetExtractor;
   beforeEach(async () => {
-    extractor = createSnipperExtractor(await createTypeScriptExtractorConfig());
+    extractor = createSnippetExtractor(await createTypeScriptExtractorConfig());
   });
 
   it("should extract a basic enum", async () => {

package/dist/src/testing/scenario-test/test-host.d.ts

@@ -1,8 +0,0 @@
-import type { Diagnostic } from "@typespec/compiler";
-import { type TypeSpecTestLibrary } from "@typespec/compiler/testing";
-export interface EmittedFile {
-    path: string;
-    content: string;
-}
-export declare function emitWithDiagnostics(testLibrary: TypeSpecTestLibrary, emitterOutputDir: string, code: string): Promise<[EmittedFile[], readonly Diagnostic[]]>;
-//# sourceMappingURL=test-host.d.ts.map

package/dist/src/testing/scenario-test/test-host.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"test-host.d.ts","sourceRoot":"","sources":["../../../../src/testing/scenario-test/test-host.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAmB,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAKL,KAAK,mBAAmB,EACzB,MAAM,4BAA4B,CAAC;AAKpC,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AA4BD,wBAAsB,mBAAmB,CACvC,WAAW,EAAE,mBAAmB,EAChC,gBAAgB,EAAE,MAAM,EACxB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE,SAAS,UAAU,EAAE,CAAC,CAAC,CAOjD"}

package/dist/src/testing/scenario-test/test-host.js

@@ -1,49 +0,0 @@
-import { createTestHost, createTestWrapper } from "@typespec/compiler/testing";
-import { HttpTestLibrary } from "@typespec/http/testing";
-import { RestTestLibrary } from "@typespec/rest/testing";
-import { join, relative } from "path";
-async function createEmitterTestRunner(testLibrary, options = {}) {
-  const libraries = options.testHostConfig?.libraries ?? [testLibrary, HttpTestLibrary, RestTestLibrary];
-  const host = await createTestHost({
-    libraries
-  });
-  return createTestWrapper(host, {
-    autoImports: options.autoImports ?? ["@typespec/http", "@typespec/rest"],
-    autoUsings: options.autoUsings ?? ["TypeSpec.Http", "TypeSpec.Rest"],
-    compilerOptions: options.compilerOptions ?? {
-      noEmit: false,
-      emit: [testLibrary.name]
-    }
-  });
-}
-export async function emitWithDiagnostics(testLibrary, emitterOutputDir, code) {
-  const runner = await createEmitterTestRunner(testLibrary);
-  await runner.compileAndDiagnose(code, {
-    outputDir: "tsp-output"
-  });
-  const result = await readFilesRecursively(emitterOutputDir, emitterOutputDir, runner);
-  return [result, runner.program.diagnostics];
-}
-async function readFilesRecursively(currentDir, emitterOutputDir, runner) {
-  const entries = await runner.program.host.readDir(currentDir);
-  const result = [];
-  for (const entry of entries) {
-    const fullPath = join(currentDir, entry);
-    const stat = await runner.program.host.stat(fullPath);
-    if (stat.isDirectory()) {
-      // Recursively read files in the directory
-      const nestedFiles = await readFilesRecursively(fullPath, emitterOutputDir, runner);
-      result.push(...nestedFiles);
-    } else if (stat.isFile()) {
-      // Read the file
-      // Read the file and store it with a relative path
-      const relativePath = relative(emitterOutputDir, fullPath);
-      const fileContent = await runner.program.host.readFile(fullPath);
-      result.push({
-        path: relativePath,
-        content: fileContent.text
-      });
-    }
-  }
-  return result;
-}