@typespec/http-client-python 0.4.4 → 0.5.1

package/emitter/src/emitter.ts

@@ -8,10 +8,12 @@ import { EmitContext, NoTarget } from "@typespec/compiler";
 import { execSync } from "child_process";
 import fs from "fs";
 import path, { dirname } from "path";
+import { loadPyodide } from "pyodide";
 import { fileURLToPath } from "url";
 import { emitCodeModel } from "./code-model.js";
 import { saveCodeModelAsYaml } from "./external-process.js";
 import { PythonEmitterOptions, PythonSdkContext, reportDiagnostic } from "./lib.js";
+import { runPython3 } from "./run-python3.js";
 import { removeUnderscoresFromNamespace } from "./utils.js";
 
 export function getModelsMode(context: SdkContext): "dpg" | "none" {
@@ -85,51 +87,137 @@ export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
     });
     return;
   }
-
-  addDefaultOptions(sdkContext);
   const yamlPath = await saveCodeModelAsYaml("python-yaml-path", yamlMap);
-  let venvPath = path.join(root, "venv");
-  if (fs.existsSync(path.join(venvPath, "bin"))) {
-    venvPath = path.join(venvPath, "bin", "python");
-  } else if (fs.existsSync(path.join(venvPath, "Scripts"))) {
-    venvPath = path.join(venvPath, "Scripts", "python.exe");
-  } else {
-    throw new Error("Virtual environment doesn't exist.");
-  }
-  const commandArgs = [
-    venvPath,
-    `${root}/eng/scripts/setup/run_tsp.py`,
-    `--output-folder=${outputDir}`,
-    `--cadl-file=${yamlPath}`,
-  ];
+  addDefaultOptions(sdkContext);
   const resolvedOptions = sdkContext.emitContext.options;
+  const commandArgs: Record<string, string> = {};
   if (resolvedOptions["packaging-files-config"]) {
     const keyValuePairs = Object.entries(resolvedOptions["packaging-files-config"]).map(
       ([key, value]) => {
         return `${key}:${value}`;
       },
     );
-    commandArgs.push(`--packaging-files-config='${keyValuePairs.join("|")}'`);
+    commandArgs["packaging-files-config"] = keyValuePairs.join("|");
     resolvedOptions["packaging-files-config"] = undefined;
   }
   if (
     resolvedOptions["package-pprint-name"] !== undefined &&
     !resolvedOptions["package-pprint-name"].startsWith('"')
   ) {
-    resolvedOptions["package-pprint-name"] = `"${resolvedOptions["package-pprint-name"]}"`;
+    resolvedOptions["package-pprint-name"] = `${resolvedOptions["package-pprint-name"]}`;
   }
 
   for (const [key, value] of Object.entries(resolvedOptions)) {
-    commandArgs.push(`--${key}=${value}`);
+    commandArgs[key] = value;
   }
   if (sdkContext.arm === true) {
-    commandArgs.push("--azure-arm=true");
+    commandArgs["azure-arm"] = "true";
   }
   if (resolvedOptions.flavor === "azure") {
-    commandArgs.push("--emit-cross-language-definition-file=true");
+    commandArgs["emit-cross-language-definition-file"] = "true";
   }
-  commandArgs.push("--from-typespec=true");
+  commandArgs["from-typespec"] = "true";
+
   if (!program.compilerOptions.noEmit && !program.hasError()) {
-    execSync(commandArgs.join(" "));
+    // if not using pyodide and there's no venv, we try to create venv
+    if (!resolvedOptions["use-pyodide"] && !fs.existsSync(path.join(root, "venv"))) {
+      try {
+        await runPython3(path.join(root, "/eng/scripts/setup/install.py"));
+        await runPython3(path.join(root, "/eng/scripts/setup/prepare.py"));
+      } catch (error) {
+        // if the python env is not ready, we use pyodide instead
+        resolvedOptions["use-pyodide"] = true;
+      }
+    }
+
+    if (resolvedOptions["use-pyodide"]) {
+      // here we run with pyodide
+      const pyodide = await setupPyodideCall(root);
+      // create the output folder if not exists
+      if (!fs.existsSync(outputDir)) {
+        fs.mkdirSync(outputDir, { recursive: true });
+      }
+      // mount output folder to pyodide
+      pyodide.FS.mkdirTree("/output");
+      pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: outputDir }, "/output");
+      // mount yaml file to pyodide
+      pyodide.FS.mkdirTree("/yaml");
+      pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: path.dirname(yamlPath) }, "/yaml");
+      const globals = pyodide.toPy({
+        outputFolder: "/output",
+        yamlFile: `/yaml/${path.basename(yamlPath)}`,
+        commandArgs,
+      });
+      const pythonCode = `
+        async def main():
+          import warnings
+          with warnings.catch_warnings():
+            warnings.simplefilter("ignore", SyntaxWarning) # bc of m2r2 dep issues
+            from pygen import m2r, preprocess, codegen, black
+          m2r.M2R(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
+          preprocess.PreProcessPlugin(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
+          codegen.CodeGenerator(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
+          black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process()
+    
+        await main()`;
+      await pyodide.runPythonAsync(pythonCode, { globals });
+    } else {
+      // here we run with native python
+      let venvPath = path.join(root, "venv");
+      if (fs.existsSync(path.join(venvPath, "bin"))) {
+        venvPath = path.join(venvPath, "bin", "python");
+      } else if (fs.existsSync(path.join(venvPath, "Scripts"))) {
+        venvPath = path.join(venvPath, "Scripts", "python.exe");
+      } else {
+        throw new Error("Virtual environment doesn't exist.");
+      }
+      commandArgs["output-folder"] = outputDir;
+      commandArgs["cadl-file"] = yamlPath;
+      const commandFlags = Object.entries(commandArgs)
+        .map(([key, value]) => `--${key}=${value}`)
+        .join(" ");
+      const command = `${venvPath} ${root}/eng/scripts/setup/run_tsp.py ${commandFlags}`;
+      execSync(command);
+    }
+  }
+}
+
+async function setupPyodideCall(root: string) {
+  const pyodide = await loadPyodide({
+    indexURL: path.dirname(fileURLToPath(import.meta.resolve("pyodide"))),
+  });
+  const micropipLockPath = path.join(root, "micropip.lock");
+  while (true) {
+    if (fs.existsSync(micropipLockPath)) {
+      try {
+        const stats = fs.statSync(micropipLockPath);
+        const now = new Date().getTime();
+        const lockAge = (now - stats.mtime.getTime()) / 1000;
+        if (lockAge > 300) {
+          fs.unlinkSync(micropipLockPath);
+        }
+      } catch (err) {
+        // ignore
+      }
+    }
+    try {
+      const fd = fs.openSync(micropipLockPath, "wx");
+      // mount generator to pyodide
+      pyodide.FS.mkdirTree("/generator");
+      pyodide.FS.mount(
+        pyodide.FS.filesystems.NODEFS,
+        { root: path.join(root, "generator") },
+        "/generator",
+      );
+      await pyodide.loadPackage("micropip");
+      const micropip = pyodide.pyimport("micropip");
+      await micropip.install("emfs:/generator/dist/pygen-0.1.0-py3-none-any.whl");
+      fs.closeSync(fd);
+      fs.unlinkSync(micropipLockPath);
+      break;
+    } catch (err) {
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+    }
   }
+  return pyodide;
 }

package/emitter/src/http.ts

@@ -204,7 +204,7 @@ function emitHttpOperation(
     apiVersions: [],
     wantTracing: true,
     exposeStreamKeyword: true,
-    crossLanguageDefinitionId: method?.crossLanguageDefintionId,
+    crossLanguageDefinitionId: method?.crossLanguageDefinitionId,
     samples: arrayToRecord(method?.operation.examples),
     internal: method.access === "internal",
   };

package/emitter/src/lib.ts

@@ -17,6 +17,7 @@ export interface PythonEmitterOptions {
   debug?: boolean;
   flavor?: "azure";
   "examples-dir"?: string;
+  "use-pyodide"?: boolean;
 }
 
 export interface PythonSdkContext<TServiceOperation extends SdkServiceOperation>
@@ -43,6 +44,7 @@ const EmitterOptionsSchema: JSONSchemaType<PythonEmitterOptions> = {
     debug: { type: "boolean", nullable: true },
     flavor: { type: "string", nullable: true },
     "examples-dir": { type: "string", nullable: true, format: "absolute-path" },
+    "use-pyodide": { type: "boolean", nullable: true },
   },
   required: [],
 };

package/emitter/src/run-python3.ts

@@ -0,0 +1,20 @@
+// This script wraps logic in @azure-tools/extension to resolve
+// the path to Python 3 so that a Python script file can be run
+// from an npm script in package.json.  It uses the same Python 3
+// path resolution algorithm as AutoRest so that the behavior
+// is fully consistent (and also supports AUTOREST_PYTHON_EXE).
+//
+// Invoke it like so: "tsx run-python3.ts script.py"
+
+import cp from "child_process";
+import { patchPythonPath } from "./system-requirements.js";
+
+export async function runPython3(...args: string[]) {
+  const command = await patchPythonPath(["python", ...args], {
+    version: ">=3.8",
+    environmentVariable: "AUTOREST_PYTHON_EXE",
+  });
+  cp.execSync(command.join(" "), {
+    stdio: [0, 1, 2],
+  });
+}

package/emitter/src/system-requirements.ts

@@ -0,0 +1,261 @@
+import { ChildProcess, spawn, SpawnOptions } from "child_process";
+import { coerce, satisfies } from "semver";
+
+/*
+ * Copied from @autorest/system-requirements
+ */
+
+const execute = (
+  command: string,
+  cmdlineargs: Array<string>,
+  options: MoreOptions = {},
+): Promise<ExecResult> => {
+  return new Promise((resolve, reject) => {
+    const cp = spawn(command, cmdlineargs, { ...options, stdio: "pipe", shell: true });
+    if (options.onCreate) {
+      options.onCreate(cp);
+    }
+
+    options.onStdOutData && cp.stdout.on("data", options.onStdOutData);
+    options.onStdErrData && cp.stderr.on("data", options.onStdErrData);
+
+    let err = "";
+    let out = "";
+    let all = "";
+    cp.stderr.on("data", (chunk) => {
+      err += chunk;
+      all += chunk;
+    });
+    cp.stdout.on("data", (chunk) => {
+      out += chunk;
+      all += chunk;
+    });
+
+    cp.on("error", (err) => {
+      reject(err);
+    });
+    cp.on("close", (code, signal) =>
+      resolve({
+        stdout: out,
+        stderr: err,
+        log: all,
+        error: code ? new Error("Process Failed.") : null,
+        code,
+      }),
+    );
+  });
+};
+
+const versionIsSatisfied = (version: string, requirement: string): boolean => {
+  const cleanedVersion = coerce(version);
+  if (!cleanedVersion) {
+    throw new Error(`Invalid version ${version}.`);
+  }
+  return satisfies(cleanedVersion, requirement, true);
+};
+
+/**
+ * Validate the provided system requirement resolution is satisfying the version requirement if applicable.
+ * @param resolution Command resolution.
+ * @param actualVersion Version for that resolution.
+ * @param requirement Requirement.
+ * @returns the resolution if it is valid or an @see SystemRequirementError if not.
+ */
+const validateVersionRequirement = (
+  resolution: SystemRequirementResolution,
+  actualVersion: string,
+  requirement: SystemRequirement,
+): SystemRequirementResolution | SystemRequirementError => {
+  if (!requirement.version) {
+    return resolution; // No version requirement.
+  }
+
+  try {
+    if (versionIsSatisfied(actualVersion, requirement.version)) {
+      return resolution;
+    }
+    return {
+      ...resolution,
+      error: true,
+      message: `'${resolution.command}' version is '${actualVersion}' but doesn't satisfy requirement '${requirement.version}'. Please update.`,
+      actualVersion: actualVersion,
+      neededVersion: requirement.version,
+    };
+  } catch {
+    return {
+      ...resolution,
+      error: true,
+      message: `Couldn't parse the version ${actualVersion}. This is not a valid semver version.`,
+      actualVersion: actualVersion,
+      neededVersion: requirement.version,
+    };
+  }
+};
+
+const tryPython = async (
+  requirement: SystemRequirement,
+  command: string,
+  additionalArgs: string[] = [],
+): Promise<SystemRequirementResolution | SystemRequirementError> => {
+  const resolution: SystemRequirementResolution = {
+    name: PythonRequirement,
+    command,
+    additionalArgs: additionalArgs.length > 0 ? additionalArgs : undefined,
+  };
+
+  try {
+    const result = await execute(command, [
+      ...additionalArgs,
+      "-c",
+      `"${PRINT_PYTHON_VERSION_SCRIPT}"`,
+    ]);
+    return validateVersionRequirement(resolution, result.stdout.trim(), requirement);
+  } catch (e) {
+    return {
+      error: true,
+      ...resolution,
+      message: `'${command}' command line is not found in the path. Make sure to have it installed.`,
+    };
+  }
+};
+
+/**
+ * Returns the path to the executable as asked in the requirement.
+ * @param requirement System requirement definition.
+ * @returns If the requirement provide an environment variable for the path returns the value of that environment variable. undefined otherwise.
+ */
+const getExecutablePath = (requirement: SystemRequirement): string | undefined =>
+  requirement.environmentVariable && process.env[requirement.environmentVariable];
+
+const createPythonErrorMessage = (
+  requirement: SystemRequirement,
+  errors: SystemRequirementError[],
+): SystemRequirementError => {
+  const versionReq = requirement.version ?? "*";
+  const lines = [
+    `Couldn't find a valid python interpreter satisfying the requirement (version: ${versionReq}). Tried:`,
+    ...errors.map((x) => ` - ${x.command} (${x.message})`),
+  ];
+
+  return {
+    error: true,
+    name: "python",
+    command: "python",
+    message: lines.join("\n"),
+  };
+};
+
+const resolvePythonRequirement = async (
+  requirement: SystemRequirement,
+): Promise<SystemRequirementResolution | SystemRequirementError> => {
+  // Hardcoding AUTOREST_PYTHON_EXE is for backward compatibility
+  const path = getExecutablePath(requirement) ?? process.env["AUTOREST_PYTHON_EXE"];
+  if (path) {
+    return await tryPython(requirement, path);
+  }
+
+  const errors: SystemRequirementError[] = [];
+  // On windows try `py` executable with `-3` flag.
+  if (process.platform === "win32") {
+    const pyResult = await tryPython(requirement, "py", ["-3"]);
+    if ("error" in pyResult) {
+      errors.push(pyResult);
+    } else {
+      return pyResult;
+    }
+  }
+
+  const python3Result = await tryPython(requirement, "python3");
+  if ("error" in python3Result) {
+    errors.push(python3Result);
+  } else {
+    return python3Result;
+  }
+
+  const pythonResult = await tryPython(requirement, "python");
+  if ("error" in pythonResult) {
+    errors.push(pythonResult);
+  } else {
+    return pythonResult;
+  }
+
+  return createPythonErrorMessage(requirement, errors);
+};
+
+/**
+ * @param command list of the command and arguments. First item in array must be a python exe @see KnownPythonExe. (e.g. ["python", "my_python_file.py"]
+ * @param requirement
+ */
+export const patchPythonPath = async (
+  command: PythonCommandLine,
+  requirement: SystemRequirement,
+): Promise<string[]> => {
+  const [_, ...args] = command;
+  const resolution = await resolvePythonRequirement(requirement);
+  if ("error" in resolution) {
+    throw new Error(`Failed to find compatible python version. ${resolution.message}`);
+  }
+  return [resolution.command, ...(resolution.additionalArgs ?? []), ...args];
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// TYPES
+const PythonRequirement = "python";
+const PRINT_PYTHON_VERSION_SCRIPT = "import sys; print('.'.join(map(str, sys.version_info[:3])))";
+
+type KnownPythonExe = "python.exe" | "python3.exe" | "python" | "python3";
+type PythonCommandLine = [KnownPythonExe, ...string[]];
+
+interface MoreOptions extends SpawnOptions {
+  onCreate?(cp: ChildProcess): void;
+  onStdOutData?(chunk: any): void;
+  onStdErrData?(chunk: any): void;
+}
+
+interface SystemRequirement {
+  version?: string;
+  /**
+   * Name of an environment variable where the user could provide the path to the exe.
+   * @example "AUTOREST_PYTHON_PATH"
+   */
+  environmentVariable?: string;
+}
+
+interface SystemRequirementResolution {
+  /**
+   * Name of the requirement.
+   * @example python, java, etc.
+   */
+  name: string;
+
+  /**
+   * Name of the command
+   * @example python3, /home/my_user/python39/python, java, etc.
+   */
+  command: string;
+
+  /**
+   * List of additional arguments to pass to this command.
+   * @example '-3' for 'py' to specify to use python 3
+   */
+  additionalArgs?: string[];
+}
+
+interface ExecResult {
+  stdout: string;
+  stderr: string;
+
+  /**
+   * Union of stdout and stderr.
+   */
+  log: string;
+  error: Error | null;
+  code: number | null;
+}
+
+interface SystemRequirementError extends SystemRequirementResolution {
+  error: true;
+  message: string;
+  neededVersion?: string;
+  actualVersion?: string;
+}