@typespec/http-client-python 0.1.0

package/eng/scripts/setup/system-requirements.ts

@@ -0,0 +1,261 @@
+import { ChildProcess, spawn, SpawnOptions } from "child_process";
+import { coerce, satisfies } from "semver";
+
+/*
+ * Copied from @autorest/system-requirements
+ */
+
+const execute = (
+  command: string,
+  cmdlineargs: Array<string>,
+  options: MoreOptions = {},
+): Promise<ExecResult> => {
+  return new Promise((resolve, reject) => {
+    const cp = spawn(command, cmdlineargs, { ...options, stdio: "pipe", shell: true });
+    if (options.onCreate) {
+      options.onCreate(cp);
+    }
+
+    options.onStdOutData && cp.stdout.on("data", options.onStdOutData);
+    options.onStdErrData && cp.stderr.on("data", options.onStdErrData);
+
+    let err = "";
+    let out = "";
+    let all = "";
+    cp.stderr.on("data", (chunk) => {
+      err += chunk;
+      all += chunk;
+    });
+    cp.stdout.on("data", (chunk) => {
+      out += chunk;
+      all += chunk;
+    });
+
+    cp.on("error", (err) => {
+      reject(err);
+    });
+    cp.on("close", (code, signal) =>
+      resolve({
+        stdout: out,
+        stderr: err,
+        log: all,
+        error: code ? new Error("Process Failed.") : null,
+        code,
+      }),
+    );
+  });
+};
+
+const versionIsSatisfied = (version: string, requirement: string): boolean => {
+  const cleanedVersion = coerce(version);
+  if (!cleanedVersion) {
+    throw new Error(`Invalid version ${version}.`);
+  }
+  return satisfies(cleanedVersion, requirement, true);
+};
+
+/**
+ * Validate the provided system requirement resolution is satisfying the version requirement if applicable.
+ * @param resolution Command resolution.
+ * @param actualVersion Version for that resolution.
+ * @param requirement Requirement.
+ * @returns the resolution if it is valid or an @see SystemRequirementError if not.
+ */
+const validateVersionRequirement = (
+  resolution: SystemRequirementResolution,
+  actualVersion: string,
+  requirement: SystemRequirement,
+): SystemRequirementResolution | SystemRequirementError => {
+  if (!requirement.version) {
+    return resolution; // No version requirement.
+  }
+
+  try {
+    if (versionIsSatisfied(actualVersion, requirement.version)) {
+      return resolution;
+    }
+    return {
+      ...resolution,
+      error: true,
+      message: `'${resolution.command}' version is '${actualVersion}' but doesn't satisfy requirement '${requirement.version}'. Please update.`,
+      actualVersion: actualVersion,
+      neededVersion: requirement.version,
+    };
+  } catch {
+    return {
+      ...resolution,
+      error: true,
+      message: `Couldn't parse the version ${actualVersion}. This is not a valid semver version.`,
+      actualVersion: actualVersion,
+      neededVersion: requirement.version,
+    };
+  }
+};
+
+const tryPython = async (
+  requirement: SystemRequirement,
+  command: string,
+  additionalArgs: string[] = [],
+): Promise<SystemRequirementResolution | SystemRequirementError> => {
+  const resolution: SystemRequirementResolution = {
+    name: PythonRequirement,
+    command,
+    additionalArgs: additionalArgs.length > 0 ? additionalArgs : undefined,
+  };
+
+  try {
+    const result = await execute(command, [
+      ...additionalArgs,
+      "-c",
+      `"${PRINT_PYTHON_VERSION_SCRIPT}"`,
+    ]);
+    return validateVersionRequirement(resolution, result.stdout.trim(), requirement);
+  } catch (e) {
+    return {
+      error: true,
+      ...resolution,
+      message: `'${command}' command line is not found in the path. Make sure to have it installed.`,
+    };
+  }
+};
+
+/**
+ * Returns the path to the executable as asked in the requirement.
+ * @param requirement System requirement definition.
+ * @returns If the requirement provide an environment variable for the path returns the value of that environment variable. undefined otherwise.
+ */
+const getExecutablePath = (requirement: SystemRequirement): string | undefined =>
+  requirement.environmentVariable && process.env[requirement.environmentVariable];
+
+const createPythonErrorMessage = (
+  requirement: SystemRequirement,
+  errors: SystemRequirementError[],
+): SystemRequirementError => {
+  const versionReq = requirement.version ?? "*";
+  const lines = [
+    `Couldn't find a valid python interpreter satisfying the requirement (version: ${versionReq}). Tried:`,
+    ...errors.map((x) => ` - ${x.command} (${x.message})`),
+  ];
+
+  return {
+    error: true,
+    name: "python",
+    command: "python",
+    message: lines.join("\n"),
+  };
+};
+
+const resolvePythonRequirement = async (
+  requirement: SystemRequirement,
+): Promise<SystemRequirementResolution | SystemRequirementError> => {
+  // Hardcoding AUTOREST_PYTHON_EXE is for backward compatibility
+  const path = getExecutablePath(requirement) ?? process.env["AUTOREST_PYTHON_EXE"];
+  if (path) {
+    return await tryPython(requirement, path);
+  }
+
+  const errors: SystemRequirementError[] = [];
+  // On windows try `py` executable with `-3` flag.
+  if (process.platform === "win32") {
+    const pyResult = await tryPython(requirement, "py", ["-3"]);
+    if ("error" in pyResult) {
+      errors.push(pyResult);
+    } else {
+      return pyResult;
+    }
+  }
+
+  const python3Result = await tryPython(requirement, "python3");
+  if ("error" in python3Result) {
+    errors.push(python3Result);
+  } else {
+    return python3Result;
+  }
+
+  const pythonResult = await tryPython(requirement, "python");
+  if ("error" in pythonResult) {
+    errors.push(pythonResult);
+  } else {
+    return pythonResult;
+  }
+
+  return createPythonErrorMessage(requirement, errors);
+};
+
+/**
+ * @param command list of the command and arguments. First item in array must be a python exe @see KnownPythonExe. (e.g. ["python", "my_python_file.py"]
+ * @param requirement
+ */
+export const patchPythonPath = async (
+  command: PythonCommandLine,
+  requirement: SystemRequirement,
+): Promise<string[]> => {
+  const [_, ...args] = command;
+  const resolution = await resolvePythonRequirement(requirement);
+  if ("error" in resolution) {
+    throw new Error(`Failed to find compatible python version. ${resolution.message}`);
+  }
+  return [resolution.command, ...(resolution.additionalArgs ?? []), ...args];
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// TYPES
+const PythonRequirement = "python";
+const PRINT_PYTHON_VERSION_SCRIPT = "import sys; print('.'.join(map(str, sys.version_info[:3])))";
+
+type KnownPythonExe = "python.exe" | "python3.exe" | "python" | "python3";
+type PythonCommandLine = [KnownPythonExe, ...string[]];
+
+interface MoreOptions extends SpawnOptions {
+  onCreate?(cp: ChildProcess): void;
+  onStdOutData?(chunk: any): void;
+  onStdErrData?(chunk: any): void;
+}
+
+interface SystemRequirement {
+  version?: string;
+  /**
+   * Name of an environment variable where the user could provide the path to the exe.
+   * @example "AUTOREST_PYTHON_PATH"
+   */
+  environmentVariable?: string;
+}
+
+interface SystemRequirementResolution {
+  /**
+   * Name of the requirement.
+   * @example python, java, etc.
+   */
+  name: string;
+
+  /**
+   * Name of the command
+   * @example python3, /home/my_user/python39/python, java, etc.
+   */
+  command: string;
+
+  /**
+   * List of additional arguments to pass to this command.
+   * @example '-3' for 'py' to specify to use python 3
+   */
+  additionalArgs?: string[];
+}
+
+interface ExecResult {
+  stdout: string;
+  stderr: string;
+
+  /**
+   * Union of stdout and stderr.
+   */
+  log: string;
+  error: Error | null;
+  code: number | null;
+}
+
+interface SystemRequirementError extends SystemRequirementResolution {
+  error: true;
+  message: string;
+  neededVersion?: string;
+  actualVersion?: string;
+}

package/eng/scripts/setup/venvtools.py

@@ -0,0 +1,87 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from contextlib import contextmanager
+import tempfile
+import subprocess
+import venv
+import sys
+from pathlib import Path
+
+
+_ROOT_DIR = Path(__file__).parent.parent.parent.parent
+
+
+class ExtendedEnvBuilder(venv.EnvBuilder):
+    """An extended env builder which saves the context, to have access
+    easily to bin path and such.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.context = None
+        if sys.version_info < (3, 9, 0):
+            # Not supported on Python 3.8, and we don't need it
+            kwargs.pop("upgrade_deps", None)
+        super().__init__(*args, **kwargs)
+
+    def ensure_directories(self, env_dir):
+        self.context = super(ExtendedEnvBuilder, self).ensure_directories(env_dir)
+        return self.context
+
+
+def create(
+    env_dir,
+    system_site_packages=False,
+    clear=False,
+    symlinks=False,
+    with_pip=False,
+    prompt=None,
+    upgrade_deps=False,
+):
+    """Create a virtual environment in a directory."""
+    builder = ExtendedEnvBuilder(
+        system_site_packages=system_site_packages,
+        clear=clear,
+        symlinks=symlinks,
+        with_pip=with_pip,
+        prompt=prompt,
+        upgrade_deps=upgrade_deps,
+    )
+    builder.create(env_dir)
+    return builder.context
+
+
+@contextmanager
+def create_venv_with_package(packages):
+    """Create a venv with these packages in a temp dir and yield the env.
+
+    packages should be an iterable of pip version instruction (e.g. package~=1.2.3)
+    """
+    with tempfile.TemporaryDirectory() as tempdir:
+        my_env = create(tempdir, with_pip=True, upgrade_deps=True)
+        pip_call = [
+            my_env.env_exe,
+            "-m",
+            "pip",
+            "install",
+        ]
+        subprocess.check_call(pip_call + ["-U", "pip"])
+        if packages:
+            subprocess.check_call(pip_call + packages)
+        yield my_env
+
+
+def python_run(venv_context, module, command=None, *, additional_dir="."):
+    try:
+        cmd_line = [venv_context.env_exe, "-m", module] + (command if command else [])
+        print("Executing: {}".format(" ".join(cmd_line)))
+        subprocess.run(
+            cmd_line,
+            cwd=_ROOT_DIR / additional_dir,
+            check=True,
+        )
+    except subprocess.CalledProcessError as err:
+        print(err)
+        sys.exit(1)

package/generator/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

package/generator/README.md

@@ -0,0 +1 @@
+# Core Library for Python Generation

package/generator/dev_requirements.txt

@@ -0,0 +1,13 @@
+-e ./generator
+pyright==1.1.375
+pylint==3.2.6
+tox==4.16.0
+mypy==1.10.1
+azure-pylint-guidelines-checker==0.0.8
+colorama==0.4.6
+debugpy==1.8.2
+pytest==8.3.2
+coverage==7.6.1
+black==24.8.0
+ptvsd==4.3.2
+types-PyYAML==6.0.12.8

package/generator/pygen/__init__.py

@@ -0,0 +1,107 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import logging
+from pathlib import Path
+import json
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Union, List
+
+import yaml
+
+from ._version import VERSION
+
+
+__version__ = VERSION
+_LOGGER = logging.getLogger(__name__)
+
+
+class ReaderAndWriter:
+    def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None:
+        self.output_folder = Path(output_folder)
+        self._list_file: List[str] = []
+        try:
+            with open(
+                Path(self.output_folder) / Path("..") / Path("python.json"),
+                "r",
+                encoding="utf-8-sig",
+            ) as fd:
+                python_json = json.load(fd)
+        except Exception:  # pylint: disable=broad-except
+            python_json = {}
+        self.options = kwargs
+        if python_json:
+            _LOGGER.warning("Loading python.json file. This behavior will be depreacted")
+        self.options.update(python_json)
+
+    def read_file(self, path: Union[str, Path]) -> str:
+        """Directly reading from disk"""
+        # make path relative to output folder
+        try:
+            with open(self.output_folder / Path(path), "r", encoding="utf-8-sig") as fd:
+                return fd.read()
+        except FileNotFoundError:
+            return ""
+
+    def write_file(self, filename: Union[str, Path], file_content: str) -> None:
+        """Directly writing to disk"""
+        file_folder = Path(filename).parent
+        if not Path.is_dir(self.output_folder / file_folder):
+            Path.mkdir(self.output_folder / file_folder, parents=True)
+        with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd:
+            fd.write(file_content)
+
+    def list_file(self) -> List[str]:
+        return [str(f.relative_to(self.output_folder)) for f in self.output_folder.glob("**/*") if f.is_file()]
+
+
+class Plugin(ReaderAndWriter, ABC):
+    """A base class for autorest plugin.
+
+    :param autorestapi: An autorest API instance
+    """
+
+    @abstractmethod
+    def process(self) -> bool:
+        """The plugin process.
+
+        :rtype: bool
+        :returns: True if everything's ok, False optherwise
+        :raises Exception: Could raise any exception, stacktrace will be sent to autorest API
+        """
+        raise NotImplementedError()
+
+
+class YamlUpdatePlugin(Plugin):
+    """A plugin that update the YAML as input."""
+
+    def get_yaml(self) -> Dict[str, Any]:
+        # cadl file doesn't have to be relative to output folder
+        with open(self.options["cadl_file"], "r", encoding="utf-8-sig") as fd:
+            return yaml.safe_load(fd.read())
+
+    def write_yaml(self, yaml_string: str) -> None:
+        with open(self.options["cadl_file"], "w", encoding="utf-8-sig") as fd:
+            fd.write(yaml_string)
+
+    def process(self) -> bool:
+        # List the input file, should be only one
+        yaml_data = self.get_yaml()
+
+        self.update_yaml(yaml_data)
+
+        yaml_string = yaml.safe_dump(yaml_data)
+
+        self.write_yaml(yaml_string)
+        return True
+
+    @abstractmethod
+    def update_yaml(self, yaml_data: Dict[str, Any]) -> None:
+        """The code-model-v4-no-tags yaml model tree.
+
+        :rtype: updated yaml
+        :raises Exception: Could raise any exception, stacktrace will be sent to autorest API
+        """
+        raise NotImplementedError()

package/generator/pygen/_version.py

@@ -0,0 +1,7 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+VERSION = "0.1.0"

package/generator/pygen/black.py

@@ -0,0 +1,71 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import logging
+from pathlib import Path
+import os
+import black
+from black.report import NothingChanged
+
+from . import Plugin
+from .utils import parse_args
+
+_LOGGER = logging.getLogger("blib2to3")
+
+_BLACK_MODE = black.Mode()  # pyright: ignore [reportPrivateImportUsage]
+_BLACK_MODE.line_length = 120
+
+
+class BlackScriptPlugin(Plugin):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        output_folder = self.options.get("output_folder", str(self.output_folder))
+        if output_folder.startswith("file:"):
+            output_folder = output_folder[5:]
+        if os.name == "nt" and output_folder.startswith("///"):
+            output_folder = output_folder[3:]
+        self.output_folder = Path(output_folder)
+
+    def process(self) -> bool:
+        # apply format_file on every .py file in the output folder
+        list(
+            map(
+                self.format_file,
+                [
+                    Path(f)
+                    for f in self.list_file()
+                    if Path(f).parts[0]
+                    not in (
+                        "__pycache__",
+                        "node_modules",
+                        "venv",
+                        "env",
+                    )
+                    and not Path(f).parts[0].startswith(".")
+                    and Path(f).suffix == ".py"
+                ],
+            )
+        )
+        return True
+
+    def format_file(self, file: Path) -> None:
+        try:
+            file_content = self.read_file(file)
+            file_content = black.format_file_contents(file_content, fast=True, mode=_BLACK_MODE)
+        except NothingChanged:
+            pass
+        except:
+            _LOGGER.error("Error: failed to format %s", file)
+            raise
+        else:
+            if len(file_content.splitlines()) > 1000:
+                file_content = "# pylint: disable=too-many-lines\n" + file_content
+            self.write_file(file, file_content)
+
+
+if __name__ == "__main__":
+    # CADL pipeline will call this
+    args, unknown_args = parse_args(need_cadl_file=False)
+    BlackScriptPlugin(output_folder=args.output_folder, **unknown_args).process()