@typespec/http-client-python 0.6.11 → 0.7.1

package/emitter/src/emitter.ts

@@ -8,13 +8,15 @@ import { EmitContext, NoTarget } from "@typespec/compiler";
 import { execSync } from "child_process";
 import fs from "fs";
 import path, { dirname } from "path";
+import process from "process";
 import { loadPyodide } from "pyodide";
 import { fileURLToPath } from "url";
 import { emitCodeModel } from "./code-model.js";
 import { saveCodeModelAsYaml } from "./external-process.js";
 import { PythonEmitterOptions, PythonSdkContext, reportDiagnostic } from "./lib.js";
 import { runPython3 } from "./run-python3.js";
-import { removeUnderscoresFromNamespace } from "./utils.js";
+import { disableGenerationMap, simpleTypesMap, typesMap } from "./types.js";
+import { md2Rst, removeUnderscoresFromNamespace } from "./utils.js";
 
 export function getModelsMode(context: SdkContext): "dpg" | "none" {
   const specifiedModelsMode = context.emitContext.options["models-mode"];
@@ -23,9 +25,11 @@ export function getModelsMode(context: SdkContext): "dpg" | "none" {
     if (modelModes.includes(specifiedModelsMode)) {
       return specifiedModelsMode;
     }
-    throw new Error(
-      `Need to specify models mode with the following values: ${modelModes.join(", ")}`,
-    );
+    reportDiagnostic(context.program, {
+      code: "invalid-models-mode",
+      target: NoTarget,
+      format: { inValidValue: specifiedModelsMode },
+    });
   }
   return "dpg";
 }
@@ -76,7 +80,54 @@ async function createPythonSdkContext<TServiceOperation extends SdkServiceOperat
   };
 }
 
+function walkThroughNodes(yamlMap: Record<string, any>): Record<string, any> {
+  const stack = [yamlMap];
+  const seen = new WeakSet();
+
+  while (stack.length > 0) {
+    const current = stack.pop();
+
+    if (seen.has(current!)) {
+      continue;
+    }
+    if (current !== undefined && current !== null) {
+      seen.add(current);
+    }
+
+    if (Array.isArray(current)) {
+      for (let i = 0; i < current.length; i++) {
+        if (current[i] !== undefined && typeof current[i] === "object") {
+          stack.push(current[i]);
+        }
+      }
+    } else {
+      for (const key in current) {
+        if (key === "description" || key === "summary") {
+          if (current[key] !== undefined) {
+            current[key] = md2Rst(current[key]);
+          }
+        } else if (Array.isArray(current[key])) {
+          stack.push(current[key]);
+        } else if (current[key] !== undefined && typeof current[key] === "object") {
+          stack.push(current[key]);
+        }
+      }
+    }
+  }
+
+  return yamlMap;
+}
+
+function cleanAllCache() {
+  typesMap.clear();
+  simpleTypesMap.clear();
+  disableGenerationMap.clear();
+}
+
 export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
+  // clean all cache to make sure emitter could work in watch mode
+  cleanAllCache();
+
   const program = context.program;
   const sdkContext = await createPythonSdkContext<SdkHttpOperation>(context);
   const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", "..");
@@ -90,7 +141,10 @@ export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
     });
     return;
   }
-  const yamlPath = await saveCodeModelAsYaml("python-yaml-path", yamlMap);
+
+  const parsedYamlMap = walkThroughNodes(yamlMap);
+
+  const yamlPath = await saveCodeModelAsYaml("python-yaml-path", parsedYamlMap);
   const resolvedOptions = sdkContext.emitContext.options;
   const commandArgs: Record<string, string> = {};
   if (resolvedOptions["packaging-files-config"]) {
@@ -134,54 +188,68 @@ export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
       }
     }
 
-    if (resolvedOptions["use-pyodide"]) {
-      // here we run with pyodide
-      const pyodide = await setupPyodideCall(root);
-      // create the output folder if not exists
-      if (!fs.existsSync(outputDir)) {
-        fs.mkdirSync(outputDir, { recursive: true });
-      }
-      // mount output folder to pyodide
-      pyodide.FS.mkdirTree("/output");
-      pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: outputDir }, "/output");
-      // mount yaml file to pyodide
-      pyodide.FS.mkdirTree("/yaml");
-      pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: path.dirname(yamlPath) }, "/yaml");
-      const globals = pyodide.toPy({
-        outputFolder: "/output",
-        yamlFile: `/yaml/${path.basename(yamlPath)}`,
-        commandArgs,
-      });
-      const pythonCode = `
-        async def main():
-          import warnings
-          with warnings.catch_warnings():
-            warnings.simplefilter("ignore", SyntaxWarning) # bc of m2r2 dep issues
-            from pygen import m2r, preprocess, codegen, black
-          m2r.M2R(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
-          preprocess.PreProcessPlugin(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
-          codegen.CodeGenerator(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
-          black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process()
-    
-        await main()`;
-      await pyodide.runPythonAsync(pythonCode, { globals });
-    } else {
-      // here we run with native python
-      let venvPath = path.join(root, "venv");
-      if (fs.existsSync(path.join(venvPath, "bin"))) {
-        venvPath = path.join(venvPath, "bin", "python");
-      } else if (fs.existsSync(path.join(venvPath, "Scripts"))) {
-        venvPath = path.join(venvPath, "Scripts", "python.exe");
+    try {
+      if (resolvedOptions["use-pyodide"]) {
+        // here we run with pyodide
+        const pyodide = await setupPyodideCall(root);
+        // create the output folder if not exists
+        if (!fs.existsSync(outputDir)) {
+          fs.mkdirSync(outputDir, { recursive: true });
+        }
+        // mount output folder to pyodide
+        pyodide.FS.mkdirTree("/output");
+        pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: outputDir }, "/output");
+        // mount yaml file to pyodide
+        pyodide.FS.mkdirTree("/yaml");
+        pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: path.dirname(yamlPath) }, "/yaml");
+        const globals = pyodide.toPy({
+          outputFolder: "/output",
+          yamlFile: `/yaml/${path.basename(yamlPath)}`,
+          commandArgs,
+        });
+        const pythonCode = `
+          async def main():
+            import warnings
+            with warnings.catch_warnings():
+              from pygen import preprocess, codegen, black
+            preprocess.PreProcessPlugin(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
+            codegen.CodeGenerator(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
+            black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process()
+      
+          await main()`;
+        await pyodide.runPythonAsync(pythonCode, { globals });
       } else {
-        throw new Error("Virtual environment doesn't exist.");
+        // here we run with native python
+        let venvPath = path.join(root, "venv");
+        if (fs.existsSync(path.join(venvPath, "bin"))) {
+          venvPath = path.join(venvPath, "bin", "python");
+        } else if (fs.existsSync(path.join(venvPath, "Scripts"))) {
+          venvPath = path.join(venvPath, "Scripts", "python.exe");
+        } else {
+          reportDiagnostic(program, {
+            code: "pyodide-flag-conflict",
+            target: NoTarget,
+          });
+        }
+        commandArgs["output-folder"] = outputDir;
+        commandArgs["cadl-file"] = yamlPath;
+        const commandFlags = Object.entries(commandArgs)
+          .map(([key, value]) => `--${key}=${value}`)
+          .join(" ");
+        const command = `${venvPath} ${root}/eng/scripts/setup/run_tsp.py ${commandFlags}`;
+        execSync(command, { stdio: [process.stdin, process.stdout] });
       }
-      commandArgs["output-folder"] = outputDir;
-      commandArgs["cadl-file"] = yamlPath;
-      const commandFlags = Object.entries(commandArgs)
-        .map(([key, value]) => `--${key}=${value}`)
-        .join(" ");
-      const command = `${venvPath} ${root}/eng/scripts/setup/run_tsp.py ${commandFlags}`;
-      execSync(command);
+    } catch (error: any) {
+      const errStackStart =
+        "========================================= error stack start ================================================";
+      const errStackEnd =
+        "========================================= error stack end ================================================";
+      const errStack = error.stack ? `\n${errStackStart}\n${error.stack}\n${errStackEnd}` : "";
+      reportDiagnostic(program, {
+        code: "unknown-error",
+        target: NoTarget,
+        format: { stack: errStack },
+      });
     }
   }
 }

package/emitter/src/lib.ts

@@ -1,5 +1,5 @@
 import { SdkContext, SdkServiceOperation } from "@azure-tools/typespec-client-generator-core";
-import { createTypeSpecLibrary, JSONSchemaType } from "@typespec/compiler";
+import { createTypeSpecLibrary, JSONSchemaType, paramMessage } from "@typespec/compiler";
 
 export interface PythonEmitterOptions {
   "package-version"?: string;
@@ -56,6 +56,27 @@ const EmitterOptionsSchema: JSONSchemaType<PythonEmitterOptions> = {
 const libDef = {
   name: "@typespec/http-client-python",
   diagnostics: {
+    // error
+    "unknown-error": {
+      severity: "error",
+      messages: {
+        default: paramMessage`Can't generate Python client code from this TypeSpec. Please open an issue on https://github.com/microsoft/typespec'.${"stack"}`,
+      },
+    },
+    "invalid-models-mode": {
+      severity: "error",
+      messages: {
+        default: paramMessage`Invalid value '${"inValidValue"}' for 'models-mode' of tspconfig.yaml and expected values are 'dpg'/'none'.`,
+      },
+    },
+    "pyodide-flag-conflict": {
+      severity: "error",
+      messages: {
+        default:
+          "Python is not installed. Please follow https://www.python.org/ to install Python or set 'use-pyodide' to true.",
+      },
+    },
+    // warning
     "no-valid-client": {
       severity: "warning",
       messages: {

package/emitter/src/types.ts

@@ -341,13 +341,16 @@ function emitEnum<TServiceOperation extends SdkServiceOperation>(
     if (!type.isFixed) {
       types.push(emitBuiltInType(type.valueType));
     }
-    return {
+
+    const newValue = {
       description: "",
       internal: true,
       type: "combined",
       types,
       xmlMetadata: {},
     };
+    typesMap.set(type, newValue);
+    return newValue;
   }
   const values: Record<string, any>[] = [];
   const name = type.name;

package/emitter/src/utils.ts

@@ -10,6 +10,7 @@ import {
   SdkType,
 } from "@azure-tools/typespec-client-generator-core";
 import { getNamespaceFullName } from "@typespec/compiler";
+import { marked, Token } from "marked";
 import { PythonSdkContext } from "./lib.js";
 import { getSimpleTypeResult, getType } from "./types.js";
 
@@ -232,3 +233,84 @@ export function getClientNamespace<TServiceOperation extends SdkServiceOperation
     ? rootNamespace
     : removeUnderscoresFromNamespace(clientNamespace).toLowerCase();
 }
+
+function parseToken(token: Token): string {
+  let parsed = "";
+  switch (token.type) {
+    case "heading":
+      parsed += `${"=".repeat(token.text.length)}\n${token.text}\n${"=".repeat(
+        token.text.length,
+      )}\n\n`;
+      break;
+    case "paragraph":
+      parsed += `${token.text}\n\n`;
+      break;
+    case "strong":
+      parsed += `**${token.text}**`;
+      break;
+    case "em":
+      parsed += `*${token.text}*`;
+      break;
+    case "codespan":
+      parsed += `\`\`${token.text}\`\``;
+      break;
+    case "code":
+      let codeBlockStyle = token.codeBlockStyle;
+      if (codeBlockStyle === undefined) {
+        codeBlockStyle = token.raw.split("\n")[0].replace("```", "").trim();
+      }
+      parsed += `\n\n.. code-block:: ${codeBlockStyle ?? ""}\n\n   ${token.text.split("\n").join("\n   ")}`;
+      break;
+    case "link":
+      if (token.href !== undefined) {
+        parsed += `\`${token.text} <${token.href}>\`_`;
+        break;
+      }
+      parsed += `${token.text}`;
+      break;
+    case "list":
+      if (!token.ordered) {
+        parsed += `\n\n${token.items.map((item: any) => `* ${item.text}`).join("\n")}`;
+        break;
+      }
+      parsed += `\n\n${token.items.map((item: any, index: number) => `${index + 1}. ${item.text}`).join("\n")}`;
+      break;
+    default:
+      parsed += token.raw;
+  }
+  return parsed;
+}
+
+export function md2Rst(text?: string): string | undefined {
+  try {
+    if (!text || text === "") return text;
+    const tokens = marked.lexer(text);
+    let rst = "";
+
+    tokens.forEach((token: Token) => {
+      if (token.type === "heading") {
+        // Heading tokens are block level, so we should check if there are additional tokens inside
+        const parsedHeadingText = md2Rst(token.text);
+        rst += `${"=".repeat(
+          parsedHeadingText!.length,
+        )}\n${parsedHeadingText}\n${"=".repeat(parsedHeadingText!.length)}\n\n`;
+      } else if ("tokens" in token && token.tokens !== undefined && token.tokens.length > 0) {
+        token.tokens.forEach((element: any) => {
+          rst += parseToken(element);
+        });
+      } else {
+        rst += parseToken(token);
+      }
+    });
+
+    // Trim trailing whitespace or tabs
+    return rst.replace(/[ \t]+$/, "");
+  } catch (e) {
+    if (e instanceof RangeError) {
+      // The error is thrown by the tokenizer when the markdown is too long
+      // We can ignore it and return the original text
+      return text;
+    }
+  }
+  return text;
+}