@typespec/http-client-python 0.4.4 → 0.5.0

package/eng/scripts/setup/install.py

@@ -8,17 +8,23 @@
 import sys
 
 if not sys.version_info >= (3, 8, 0):
-    raise Exception("Autorest for Python extension requires Python 3.8 at least")
+    raise Warning(
+        "Autorest for Python extension requires Python 3.8 at least. We will run your code with Pyodide since your Python version isn't adequate."
+    )
 
 try:
     import pip
 except ImportError:
-    raise Exception("Your Python installation doesn't have pip available")
+    raise Warning(
+        "Your Python installation doesn't have pip available. We will run your code with Pyodide since your Python version isn't adequate."
+    )
 
 try:
     import venv
 except ImportError:
-    raise Exception("Your Python installation doesn't have venv available")
+    raise Warning(
+        "Your Python installation doesn't have venv available. We will run your code with Pyodide since your Python version isn't adequate."
+    )
 
 
 # Now we have pip and Py >= 3.8, go to work

package/eng/scripts/setup/install.ts

@@ -0,0 +1,32 @@
+import fs from "fs";
+import path, { dirname } from "path";
+import { loadPyodide } from "pyodide";
+import { fileURLToPath } from "url";
+import { runPython3 } from "./run-python3.js";
+
+async function main() {
+  try {
+    await runPython3("./eng/scripts/setup/install.py");
+    console.log("Found Python on your local environment and created a venv with all requirements."); // eslint-disable-line no-console
+  } catch (error) {
+    console.log("No Python found on your local environment. We will use Pyodide instead."); // eslint-disable-line no-console
+  } finally {
+    await installPyodideDeps();
+    console.log("Successfully installed all required Python packages in Pyodide"); // eslint-disable-line no-console
+  }
+}
+
+async function installPyodideDeps() {
+  const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
+  const pyodide = await loadPyodide({
+    indexURL: path.dirname(fileURLToPath(import.meta.resolve("pyodide"))),
+  });
+  await pyodide.loadPackage("micropip");
+  const micropip = pyodide.pyimport("micropip");
+  const requirementsPath = path.join(root, "generator", "requirements.txt");
+  const requirementsText = fs.readFileSync(requirementsPath, "utf-8");
+  const requirementsArray = requirementsText.split("\n").filter((line) => line.trim() !== "");
+  await micropip.install(requirementsArray);
+}
+
+main();

package/eng/scripts/setup/prepare.py

@@ -10,7 +10,9 @@ import os
 import argparse
 
 if not sys.version_info >= (3, 8, 0):
-    raise Exception("Autorest for Python extension requires Python 3.8 at least")
+    raise Warning(
+        "Autorest for Python extension requires Python 3.8 at least. We will run your code with Pyodide since your Python version isn't adequate."
+    )
 
 from pathlib import Path
 import venv

package/eng/scripts/setup/prepare.ts

@@ -0,0 +1,11 @@
+import { runPython3 } from "./run-python3.js";
+
+async function main() {
+  try {
+    await runPython3("./eng/scripts/setup/prepare.py");
+  } catch (error) {
+    console.log("No Python found on your local environment. We will use Pyodide instead."); // eslint-disable-line no-console
+  }
+}
+
+main();

package/eng/scripts/setup/run-python3.ts

@@ -9,7 +9,7 @@
 import cp from "child_process";
 import { patchPythonPath } from "./system-requirements.js";
 
-async function runPython3(...args: string[]) {
+export async function runPython3(...args: string[]) {
   const command = await patchPythonPath(["python", ...args], {
     version: ">=3.8",
     environmentVariable: "AUTOREST_PYTHON_EXE",
@@ -18,8 +18,3 @@ async function runPython3(...args: string[]) {
     stdio: [0, 1, 2],
   });
 }
-
-runPython3(...process.argv.slice(2)).catch((err) => {
-  console.error(err.toString()); // eslint-disable-line no-console
-  process.exit(1);
-});

package/generator/build/lib/pygen/__init__.py

@@ -0,0 +1,107 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import logging
+from pathlib import Path
+import json
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Union, List
+
+import yaml
+
+from ._version import VERSION
+
+
+__version__ = VERSION
+_LOGGER = logging.getLogger(__name__)
+
+
+class ReaderAndWriter:
+    def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None:
+        self.output_folder = Path(output_folder)
+        self._list_file: List[str] = []
+        try:
+            with open(
+                Path(self.output_folder) / Path("..") / Path("python.json"),
+                "r",
+                encoding="utf-8-sig",
+            ) as fd:
+                python_json = json.load(fd)
+        except Exception:  # pylint: disable=broad-except
+            python_json = {}
+        self.options = kwargs
+        if python_json:
+            _LOGGER.warning("Loading python.json file. This behavior will be depreacted")
+        self.options.update(python_json)
+
+    def read_file(self, path: Union[str, Path]) -> str:
+        """Directly reading from disk"""
+        # make path relative to output folder
+        try:
+            with open(self.output_folder / Path(path), "r", encoding="utf-8-sig") as fd:
+                return fd.read()
+        except FileNotFoundError:
+            return ""
+
+    def write_file(self, filename: Union[str, Path], file_content: str) -> None:
+        """Directly writing to disk"""
+        file_folder = Path(filename).parent
+        if not Path.is_dir(self.output_folder / file_folder):
+            Path.mkdir(self.output_folder / file_folder, parents=True)
+        with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd:
+            fd.write(file_content)
+
+    def list_file(self) -> List[str]:
+        return [str(f.relative_to(self.output_folder)) for f in self.output_folder.glob("**/*") if f.is_file()]
+
+
+class Plugin(ReaderAndWriter, ABC):
+    """A base class for autorest plugin.
+
+    :param autorestapi: An autorest API instance
+    """
+
+    @abstractmethod
+    def process(self) -> bool:
+        """The plugin process.
+
+        :rtype: bool
+        :returns: True if everything's ok, False optherwise
+        :raises Exception: Could raise any exception, stacktrace will be sent to autorest API
+        """
+        raise NotImplementedError()
+
+
+class YamlUpdatePlugin(Plugin):
+    """A plugin that update the YAML as input."""
+
+    def get_yaml(self) -> Dict[str, Any]:
+        # cadl file doesn't have to be relative to output folder
+        with open(self.options["cadl_file"], "r", encoding="utf-8-sig") as fd:
+            return yaml.safe_load(fd.read())
+
+    def write_yaml(self, yaml_string: str) -> None:
+        with open(self.options["cadl_file"], "w", encoding="utf-8-sig") as fd:
+            fd.write(yaml_string)
+
+    def process(self) -> bool:
+        # List the input file, should be only one
+        yaml_data = self.get_yaml()
+
+        self.update_yaml(yaml_data)
+
+        yaml_string = yaml.safe_dump(yaml_data)
+
+        self.write_yaml(yaml_string)
+        return True
+
+    @abstractmethod
+    def update_yaml(self, yaml_data: Dict[str, Any]) -> None:
+        """The code-model-v4-no-tags yaml model tree.
+
+        :rtype: updated yaml
+        :raises Exception: Could raise any exception, stacktrace will be sent to autorest API
+        """
+        raise NotImplementedError()

package/generator/build/lib/pygen/_version.py

@@ -0,0 +1,7 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+VERSION = "0.1.0"

package/generator/build/lib/pygen/black.py

@@ -0,0 +1,71 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import logging
+from pathlib import Path
+import os
+import black
+from black.report import NothingChanged
+
+from . import Plugin
+from .utils import parse_args
+
+_LOGGER = logging.getLogger("blib2to3")
+
+_BLACK_MODE = black.Mode()  # pyright: ignore [reportPrivateImportUsage]
+_BLACK_MODE.line_length = 120
+
+
+class BlackScriptPlugin(Plugin):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        output_folder = self.options.get("output_folder", str(self.output_folder))
+        if output_folder.startswith("file:"):
+            output_folder = output_folder[5:]
+        if os.name == "nt" and output_folder.startswith("///"):
+            output_folder = output_folder[3:]
+        self.output_folder = Path(output_folder)
+
+    def process(self) -> bool:
+        # apply format_file on every .py file in the output folder
+        list(
+            map(
+                self.format_file,
+                [
+                    Path(f)
+                    for f in self.list_file()
+                    if Path(f).parts[0]
+                    not in (
+                        "__pycache__",
+                        "node_modules",
+                        "venv",
+                        "env",
+                    )
+                    and not Path(f).parts[0].startswith(".")
+                    and Path(f).suffix == ".py"
+                ],
+            )
+        )
+        return True
+
+    def format_file(self, file: Path) -> None:
+        file_content = ""
+        try:
+            file_content = self.read_file(file)
+            file_content = black.format_file_contents(file_content, fast=True, mode=_BLACK_MODE)
+        except NothingChanged:
+            pass
+        except:
+            _LOGGER.error("Error: failed to format %s", file)
+            raise
+        if len(file_content.splitlines()) > 1000:
+            file_content = "# pylint: disable=too-many-lines\n" + file_content
+        self.write_file(file, file_content)
+
+
+if __name__ == "__main__":
+    # CADL pipeline will call this
+    args, unknown_args = parse_args(need_cadl_file=False)
+    BlackScriptPlugin(output_folder=args.output_folder, **unknown_args).process()

package/generator/build/lib/pygen/codegen/__init__.py

@@ -0,0 +1,357 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import logging
+from typing import Dict, Any, Union, Optional
+from pathlib import Path
+import yaml
+
+
+from .. import Plugin
+from ..utils import parse_args
+from .models.code_model import CodeModel
+from .serializers import JinjaSerializer
+from ._utils import DEFAULT_HEADER_TEXT, VALID_PACKAGE_MODE, TYPESPEC_PACKAGE_MODE
+
+
+def _default_pprint(package_name: str) -> str:
+    return " ".join([i.capitalize() for i in package_name.split("-")])
+
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class OptionsRetriever:
+    OPTIONS_TO_DEFAULT = {
+        "azure-arm": False,
+        "flavor": "azure",  # need to default to azure in shared code so we don't break swagger generation
+        "no-async": False,
+        "low-level-client": False,
+        "version-tolerant": True,
+        "keep-version-file": False,
+        "no-namespace-folders": False,
+        "basic-setup-py": False,
+        "client-side-validation": False,
+        "multiapi": False,
+        "polymorphic-examples": 5,
+        "generate-sample": False,
+        "generate-test": False,
+        "from-typespec": False,
+        "emit-cross-language-definition-file": False,
+    }
+
+    @property
+    def is_azure_flavor(self) -> bool:
+        return self.flavor == "azure"
+
+    def __init__(self, options: Dict[str, Any]) -> None:
+        self.options = options
+
+    def __getattr__(self, prop: str) -> Any:
+        key = prop.replace("_", "-")
+        return self.options.get(key, self.OPTIONS_TO_DEFAULT.get(key))
+
+    @property
+    def company_name(self) -> str:
+        return self.options.get("company-name", "Microsoft" if self.is_azure_flavor else "")
+
+    @property
+    def license_header(self) -> str:
+        license_header = self.options.get(
+            "header-text",
+            (DEFAULT_HEADER_TEXT.format(company_name=self.company_name) if self.company_name else ""),
+        )
+        if license_header:
+            license_header = license_header.replace("\n", "\n# ")
+            license_header = (
+                "# --------------------------------------------------------------------------\n# " + license_header
+            )
+            license_header += "\n# --------------------------------------------------------------------------"
+        return license_header
+
+    @property
+    def show_operations(self) -> bool:
+        return self.options.get("show-operations", not self.low_level_client)
+
+    @property
+    def _models_mode_default(self) -> str:
+        models_mode_default = "none" if self.low_level_client or self.version_tolerant else "msrest"
+        if self.options.get("cadl_file") is not None:
+            models_mode_default = "dpg"
+        return models_mode_default
+
+    @property
+    def original_models_mode(self) -> str:
+        return self.options.get("models-mode", self._models_mode_default)
+
+    @property
+    def models_mode(self) -> Union[str, bool]:
+        # switch to falsy value for easier code writing
+        return False if self.original_models_mode == "none" else self.original_models_mode
+
+    @property
+    def tracing(self) -> bool:
+        return self.options.get(
+            "tracing",
+            self.show_operations and self.is_azure_flavor,
+        )
+
+    @property
+    def show_send_request(self) -> bool:
+        return self.options.get(
+            "show-send-request",
+            self._low_level_or_version_tolerant,
+        )
+
+    @property
+    def _low_level_or_version_tolerant(self) -> bool:
+        return self.low_level_client or self.version_tolerant
+
+    @property
+    def only_path_and_body_params_positional(self) -> bool:
+        return self.options.get(
+            "only-path-and-body-params-positional",
+            self._low_level_or_version_tolerant,
+        )
+
+    @property
+    def combine_operation_files(self) -> bool:
+        return self.options.get(
+            "combine-operation-files",
+            self.version_tolerant,
+        )
+
+    @property
+    def package_pprint_name(self) -> str:
+        return self.options.get("package-pprint-name") or _default_pprint(str(self.package_name))
+
+    @property
+    def default_optional_constants_to_none(self) -> bool:
+        return self.options.get(
+            "default-optional-constants-to-none",
+            self._low_level_or_version_tolerant,
+        )
+
+    @property
+    def builders_visibility(self) -> str:
+        builders_visibility = self.options.get("builders-visibility")
+        if builders_visibility is None:
+            return "public" if self.low_level_client else "embedded"
+        return builders_visibility.lower()
+
+    @property
+    def head_as_boolean(self) -> bool:
+        head_as_boolean = self.options.get("head-as-boolean", True)
+        # Force some options in ARM MODE
+        return True if self.azure_arm else head_as_boolean
+
+    @property
+    def package_mode(self) -> str:
+        return self.options.get("packaging-files-dir") or self.options.get("package-mode", "")
+
+    @property
+    def packaging_files_config(self) -> Optional[Dict[str, Any]]:
+        packaging_files_config = self.options.get("packaging-files-config")
+        if packaging_files_config is None:
+            return None
+        # packaging-files-config is either a string or a dict
+        # if it's a string, we can split on the comma to get the dict
+        # otherwise we just return
+        try:
+            return {k.strip(): v.strip() for k, v in [i.split(":") for i in packaging_files_config.split("|")]}
+        except AttributeError:
+            return packaging_files_config
+
+    @property
+    def package_version(self) -> Optional[str]:
+        return str(self.options.get("package-version", ""))
+
+
+class CodeGenerator(Plugin):
+    def __init__(self, *args, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.options_retriever = OptionsRetriever(self.options)
+
+    def _validate_code_model_options(self) -> None:
+        if self.options_retriever.builders_visibility not in [
+            "public",
+            "hidden",
+            "embedded",
+        ]:
+            raise ValueError("The value of --builders-visibility must be either 'public', 'hidden', or 'embedded'")
+
+        if self.options_retriever.original_models_mode not in ["msrest", "dpg", "none"]:
+            raise ValueError(
+                "--models-mode can only be 'msrest', 'dpg' or 'none'. "
+                "Pass in 'msrest' if you want msrest models, or "
+                "'none' if you don't want any."
+            )
+
+        if not self.options_retriever.show_operations and self.options_retriever.builders_visibility == "embedded":
+            raise ValueError(
+                "Can not embed builders without operations. "
+                "Either set --show-operations to True, or change the value of --builders-visibility "
+                "to 'public' or 'hidden'."
+            )
+
+        if self.options_retriever.basic_setup_py and not self.options_retriever.package_version:
+            raise ValueError("--basic-setup-py must be used with --package-version")
+
+        if self.options_retriever.package_mode and not self.options_retriever.package_version:
+            raise ValueError("--package-mode must be used with --package-version")
+
+        if not self.options_retriever.show_operations and self.options_retriever.combine_operation_files:
+            raise ValueError(
+                "Can not combine operation files if you are not showing operations. "
+                "If you want operation files, pass in flag --show-operations"
+            )
+
+        if self.options_retriever.package_mode:
+            if (
+                (
+                    self.options_retriever.package_mode not in TYPESPEC_PACKAGE_MODE
+                    and self.options_retriever.from_typespec
+                )
+                or (
+                    self.options_retriever.package_mode not in VALID_PACKAGE_MODE
+                    and not self.options_retriever.from_typespec
+                )
+            ) and not Path(self.options_retriever.package_mode).exists():
+                raise ValueError(
+                    f"--package-mode can only be {' or '.join(TYPESPEC_PACKAGE_MODE)} or directory which contains template files"  # pylint: disable=line-too-long
+                )
+
+        if self.options_retriever.multiapi and self.options_retriever.version_tolerant:
+            raise ValueError(
+                "Can not currently generate version tolerant multiapi SDKs. "
+                "We are working on creating a new multiapi SDK for version tolerant and it is not available yet."
+            )
+
+        if self.options_retriever.client_side_validation and self.options_retriever.version_tolerant:
+            raise ValueError("Can not generate version tolerant with --client-side-validation. ")
+
+        if not (self.options_retriever.azure_arm or self.options_retriever.version_tolerant):
+            _LOGGER.warning(
+                "You are generating with options that would not allow the SDK to be shipped as an official Azure SDK. "
+                "Please read https://aka.ms/azsdk/dpcodegen for more details."
+            )
+
+        if not self.options_retriever.is_azure_flavor and self.options_retriever.tracing:
+            raise ValueError("Can only have tracing turned on for Azure SDKs.")
+
+    @staticmethod
+    def sort_exceptions(yaml_data: Dict[str, Any]) -> None:
+        for client in yaml_data["clients"]:
+            for group in client["operationGroups"]:
+                for operation in group["operations"]:
+                    if not operation.get("exceptions"):
+                        continue
+                    # sort exceptions by status code, first single status code, then range, then default
+                    operation["exceptions"] = sorted(
+                        operation["exceptions"],
+                        key=lambda x: (
+                            3
+                            if x["statusCodes"][0] == "default"
+                            else (1 if isinstance(x["statusCodes"][0], int) else 2)
+                        ),
+                    )
+
+    @staticmethod
+    def remove_cloud_errors(yaml_data: Dict[str, Any]) -> None:
+        for client in yaml_data["clients"]:
+            for group in client["operationGroups"]:
+                for operation in group["operations"]:
+                    if not operation.get("exceptions"):
+                        continue
+                    i = 0
+                    while i < len(operation["exceptions"]):
+                        exception = operation["exceptions"][i]
+                        if (
+                            exception.get("schema")
+                            and exception["schema"]["language"]["default"]["name"] == "CloudError"
+                        ):
+                            del operation["exceptions"][i]
+                            i -= 1
+                        i += 1
+        if yaml_data.get("schemas") and yaml_data["schemas"].get("objects"):
+            for i in range(len(yaml_data["schemas"]["objects"])):
+                obj_schema = yaml_data["schemas"]["objects"][i]
+                if obj_schema["language"]["default"]["name"] == "CloudError":
+                    del yaml_data["schemas"]["objects"][i]
+                    break
+
+    def _build_code_model_options(self) -> Dict[str, Any]:
+        flags = [
+            "azure_arm",
+            "head_as_boolean",
+            "license_header",
+            "keep_version_file",
+            "no_async",
+            "no_namespace_folders",
+            "basic_setup_py",
+            "package_name",
+            "package_version",
+            "client_side_validation",
+            "tracing",
+            "multiapi",
+            "polymorphic_examples",
+            "models_mode",
+            "builders_visibility",
+            "show_operations",
+            "show_send_request",
+            "only_path_and_body_params_positional",
+            "version_tolerant",
+            "low_level_client",
+            "combine_operation_files",
+            "package_mode",
+            "package_pprint_name",
+            "packaging_files_config",
+            "default_optional_constants_to_none",
+            "generate_sample",
+            "generate_test",
+            "default_api_version",
+            "from_typespec",
+            "flavor",
+            "company_name",
+            "emit_cross_language_definition_file",
+        ]
+        return {f: getattr(self.options_retriever, f) for f in flags}
+
+    def get_yaml(self) -> Dict[str, Any]:
+        # cadl file doesn't have to be relative to output folder
+        with open(self.options["cadl_file"], "r", encoding="utf-8-sig") as fd:
+            return yaml.safe_load(fd.read())
+
+    def get_serializer(self, code_model: CodeModel):
+        return JinjaSerializer(code_model, output_folder=self.output_folder)
+
+    def process(self) -> bool:
+        # List the input file, should be only one
+        self._validate_code_model_options()
+        options = self._build_code_model_options()
+        yaml_data = self.get_yaml()
+
+        self.sort_exceptions(yaml_data)
+
+        if self.options_retriever.azure_arm:
+            self.remove_cloud_errors(yaml_data)
+
+        code_model = CodeModel(yaml_data=yaml_data, options=options)
+        if not self.options_retriever.is_azure_flavor and any(client.lro_operations for client in code_model.clients):
+            raise ValueError("Only support LROs for Azure SDKs")
+        serializer = self.get_serializer(code_model)
+        serializer.serialize()
+
+        return True
+
+
+if __name__ == "__main__":
+    # CADL pipeline will call this
+    parsed_args, unknown_args = parse_args()
+    CodeGenerator(
+        output_folder=parsed_args.output_folder,
+        cadl_file=parsed_args.cadl_file,
+        **unknown_args,
+    ).process()

package/generator/build/lib/pygen/codegen/_utils.py

@@ -0,0 +1,17 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+DEFAULT_HEADER_TEXT = (
+    "Copyright (c) {company_name} Corporation. All rights reserved.\n"
+    "Licensed under the MIT License. See License.txt in the project root for license information.\n"
+    "Code generated by {company_name} (R) Python Code Generator.\n"
+    "Changes may cause incorrect behavior and will be lost if the code is regenerated."
+)
+
+SWAGGER_PACKAGE_MODE = ["mgmtplane", "dataplane"]  # for backward compatibility
+TYPESPEC_PACKAGE_MODE = ["azure-mgmt", "azure-dataplane", "generic"]
+VALID_PACKAGE_MODE = SWAGGER_PACKAGE_MODE + TYPESPEC_PACKAGE_MODE
+NAME_LENGTH_LIMIT = 40