@autorest/python 6.0.0 → 6.1.1

package/autorest/multiapi/__init__.py

@@ -11,53 +11,70 @@ import shutil
 from collections import defaultdict
 from pathlib import Path
 from typing import Dict, List, Optional, cast, Any
-from .serializers import MultiAPISerializer
+
+from .serializers import MultiAPISerializer, MultiAPISerializerAutorest
 from .models import CodeModel
 from .utils import _get_default_api_version_from_list
-from ..jsonrpc import AutorestAPI
 
-from .. import Plugin
+from .. import Plugin, PluginAutorest, ReaderAndWriter, ReaderAndWriterAutorest
 
 _LOGGER = logging.getLogger(__name__)
 
 
-class MultiApiScriptPlugin(Plugin):
+class MultiApiScriptPlugin(Plugin):  # pylint: disable=abstract-method
     def process(self) -> bool:
-        input_package_name: str = self._autorestapi.get_value("package-name")
-        output_folder: str = self._autorestapi.get_value("output-folder")
-        user_specified_default_api: str = self._autorestapi.get_value("default-api")
-        no_async = self._autorestapi.get_boolean_value("no-async")
-        generator = MultiAPI(
-            input_package_name,
-            output_folder,
-            self._autorestapi,
-            no_async,
-            user_specified_default_api,
+        return self.generator.process()
+
+    @property
+    def generator(self) -> "MultiAPI":
+        return MultiAPI(
+            input_package_name=self.options.get("package-name"),
+            output_folder=self.options["output-folder"],
+            user_specified_default_api=self.options.get("default-api"),
+            no_async=self.options.get("no-async", False),
+        )
+
+
+class MultiApiScriptPluginAutorest(MultiApiScriptPlugin, PluginAutorest):
+    @property
+    def generator(self) -> "MultiAPI":
+        return MultiAPIAutorest(
+            autorestapi=self._autorestapi,
+            input_package_name=self.options.get("package-name"),
+            output_folder=self.output_folder,
+            user_specified_default_api=self.options.get("default-api"),
+            no_async=self.options.get("no-async", False),
         )
-        return generator.process()
 
+    def get_options(self) -> Dict[str, Any]:
+        options = {
+            "package-name": self._autorestapi.get_value("package-name"),
+            "default-api": self._autorestapi.get_value("default-api"),
+            "no-async": self._autorestapi.get_value("no-async"),
+        }
+        return {k: v for k, v in options.items() if v is not None}
 
-class MultiAPI:
+
+class MultiAPI(ReaderAndWriter):  # pylint: disable=abstract-method
     def __init__(
         self,
-        input_package_name: str,
+        *,
+        input_package_name: Optional[str] = None,
         output_folder: str,
-        autorestapi: AutorestAPI,
         no_async: Optional[bool] = False,
         user_specified_default_api: Optional[str] = None,
+        **kwargs: Any,
     ) -> None:
+        super().__init__(output_folder=Path(output_folder).resolve(), **kwargs)
         if input_package_name is None:
             raise ValueError(
                 "package-name is required, either provide it as args or check your readme configuration"
             )
         self.input_package_name = input_package_name
         _LOGGER.debug("Received package name %s", input_package_name)
-
-        self.output_folder = Path(output_folder).resolve()
         _LOGGER.debug("Received output-folder %s", output_folder)
         self.output_package_name: str = ""
         self.no_async = no_async
-        self._autorestapi = autorestapi
         self.user_specified_default_api = user_specified_default_api
 
     @property
@@ -77,9 +94,7 @@ class MultiAPI:
             if self.default_api_version.replace("-", "_") == path_to_version.stem:
                 path_to_default_version = path_to_version
                 break
-        return json.loads(
-            self._autorestapi.read_file(path_to_default_version / "_metadata.json")
-        )
+        return json.loads(self.read_file(path_to_default_version / "_metadata.json"))
 
     @property
     def module_name(self) -> str:
@@ -120,9 +135,7 @@ class MultiAPI:
     @property
     def version_path_to_metadata(self) -> Dict[Path, Dict[str, Any]]:
         return {
-            version_path: json.loads(
-                self._autorestapi.read_file(version_path / "_metadata.json")
-            )
+            version_path: json.loads(self.read_file(version_path / "_metadata.json"))
             for version_path in self.paths_to_versions
         }
 
@@ -130,9 +143,7 @@ class MultiAPI:
     def mod_to_api_version(self) -> Dict[str, str]:
         mod_to_api_version: Dict[str, str] = defaultdict(str)
         for version_path in self.paths_to_versions:
-            metadata_json = json.loads(
-                self._autorestapi.read_file(version_path / "_metadata.json")
-            )
+            metadata_json = json.loads(self.read_file(version_path / "_metadata.json"))
             version = metadata_json["chosen_version"]
             total_api_version_list = metadata_json["total_api_version_list"]
             if not version:
@@ -145,6 +156,10 @@ class MultiAPI:
             mod_to_api_version[version_path.name] = version
         return mod_to_api_version
 
+    @property
+    def serializer(self) -> MultiAPISerializer:
+        return MultiAPISerializer()
+
     def process(self) -> bool:
         _LOGGER.info("Generating multiapi client")
 
@@ -163,8 +178,19 @@ class MultiAPI:
         shutil.rmtree(str(self.output_folder / "operations"), ignore_errors=True)
         shutil.rmtree(str(self.output_folder / "models"), ignore_errors=True)
 
-        multiapi_serializer = MultiAPISerializer(self._autorestapi)
+        multiapi_serializer = self.serializer
         multiapi_serializer.serialize(code_model, self.no_async)
 
         _LOGGER.info("Done!")
         return True
+
+
+class MultiAPIAutorest(MultiAPI, ReaderAndWriterAutorest):
+    def __init__(self, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+
+    @property
+    def serializer(self) -> MultiAPISerializer:
+        return MultiAPISerializerAutorest(
+            self._autorestapi, output_folder=self.output_folder
+        )

package/autorest/multiapi/serializers/__init__.py

@@ -4,13 +4,14 @@
 # license information.
 # --------------------------------------------------------------------------
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Optional, Union
 from jinja2 import PackageLoader, Environment
 
 from .import_serializer import FileImportSerializer
 
 from ...jsonrpc import AutorestAPI
 from ..models import CodeModel
+from ... import ReaderAndWriter, ReaderAndWriterAutorest
 
 __all__ = [
     "MultiAPISerializer",
@@ -32,9 +33,9 @@ def _get_file_path(filename: str, async_mode: bool) -> Path:
     return Path(filename)
 
 
-class MultiAPISerializer(object):
-    def __init__(self, autorestapi: AutorestAPI) -> None:
-        self._autorestapi = autorestapi
+class MultiAPISerializer(ReaderAndWriter):  # pylint: disable=abstract-method
+    def __init__(self, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
         self.env = Environment(
             loader=PackageLoader("autorest.multiapi", "templates"),
             keep_trailing_newline=True,
@@ -52,20 +53,20 @@ class MultiAPISerializer(object):
             )
 
         # serialize init file
-        self._autorestapi.write_file(
+        self.write_file(
             _get_file_path("__init__", async_mode), _render_template("init")
         )
 
         # serialize service client file
         imports = FileImportSerializer(code_model.client.imports(async_mode))
-        self._autorestapi.write_file(
+        self.write_file(
             _get_file_path(code_model.client.filename, async_mode),
             _render_template("client", imports=imports),
         )
 
         # serialize config file
         imports = FileImportSerializer(code_model.config.imports(async_mode))
-        self._autorestapi.write_file(
+        self.write_file(
             _get_file_path("_configuration", async_mode),
             _render_template("config", imports=imports),
         )
@@ -75,26 +76,22 @@ class MultiAPISerializer(object):
             imports = FileImportSerializer(
                 code_model.operation_mixin_group.imports(async_mode)
             )
-            self._autorestapi.write_file(
+            self.write_file(
                 _get_file_path("_operations_mixin", async_mode),
                 _render_template("operations_mixin", imports=imports),
             )
 
         # serialize models
-        self._autorestapi.write_file(Path("models.py"), _render_template("models"))
+        self.write_file(Path("models.py"), _render_template("models"))
 
     def _serialize_version_file(self) -> None:
-        if self._autorestapi.read_file("_version.py"):
-            self._autorestapi.write_file(
-                "_version.py", self._autorestapi.read_file("_version.py")
-            )
-        elif self._autorestapi.read_file("version.py"):
-            self._autorestapi.write_file(
-                "_version.py", self._autorestapi.read_file("version.py")
-            )
+        if self.read_file("_version.py"):
+            self.write_file("_version.py", self.read_file("_version.py"))
+        elif self.read_file("version.py"):
+            self.write_file("_version.py", self.read_file("version.py"))
         else:
             template = self.env.get_template("multiapi_version.py.jinja2")
-            self._autorestapi.write_file(Path("_version.py"), template.render())
+            self.write_file(Path("_version.py"), template.render())
 
     def serialize(self, code_model: CodeModel, no_async: Optional[bool]) -> None:
         self._serialize_helper(code_model, async_mode=False)
@@ -104,12 +101,10 @@ class MultiAPISerializer(object):
         self._serialize_version_file()
 
         # don't erase patch file
-        if self._autorestapi.read_file("_patch.py"):
-            self._autorestapi.write_file(
-                "_patch.py", self._autorestapi.read_file("_patch.py")
-            )
+        if self.read_file("_patch.py"):
+            self.write_file("_patch.py", self.read_file("_patch.py"))
 
-        self._autorestapi.write_file(Path("py.typed"), "# Marker file for PEP 561.")
+        self.write_file(Path("py.typed"), "# Marker file for PEP 561.")
 
         if not code_model.client.client_side_validation:
             codegen_env = Environment(
@@ -120,7 +115,14 @@ class MultiAPISerializer(object):
                 trim_blocks=True,
                 lstrip_blocks=True,
             )
-            self._autorestapi.write_file(
+            self.write_file(
                 Path("_serialization.py"),
                 codegen_env.get_template("serialization.py.jinja2").render(),
             )
+
+
+class MultiAPISerializerAutorest(MultiAPISerializer, ReaderAndWriterAutorest):
+    def __init__(
+        self, autorestapi: AutorestAPI, *, output_folder: Union[str, Path]
+    ) -> None:
+        super().__init__(autorestapi=autorestapi, output_folder=output_folder)

package/autorest/multiclient/__init__.py

@@ -0,0 +1,50 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import logging
+from typing import Any, Dict
+from pathlib import Path
+from jinja2 import Environment, PackageLoader
+from .. import Plugin, PluginAutorest
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class MultiClientPlugin(Plugin):  # pylint: disable=abstract-method
+    def process(self) -> bool:
+        _LOGGER.info("Generating files for multi client")
+
+        env = Environment(
+            loader=PackageLoader("autorest.multiclient", "templates"),
+            keep_trailing_newline=True,
+            line_statement_prefix="##",
+            line_comment_prefix="###",
+            trim_blocks=True,
+            lstrip_blocks=True,
+        )
+
+        # __init__.py
+        template = env.get_template("init.py.jinja2")
+        self.write_file(Path("__init__.py"), template.render())
+
+        # _version.py
+        template = env.get_template("version.py.jinja2")
+        self.write_file(
+            Path("_version.py"),
+            template.render(
+                package_version=self.options.get("package-version") or "1.0.0b1"
+            ),
+        )
+
+        # py.typed
+        self.write_file(Path("py.typed"), "# Marker file for PEP 561.")
+
+        _LOGGER.info("Generating Done for multi client!")
+        return True
+
+
+class MultiClientPluginAutorest(MultiClientPlugin, PluginAutorest):
+    def get_options(self) -> Dict[str, Any]:
+        return {"package-version": self._autorestapi.get_value("package-version")}

package/autorest/multiclient/templates/init.py.jinja2

@@ -0,0 +1,8 @@
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# Code generated by Microsoft (R) AutoRest Code Generator.
+# Changes may cause incorrect behavior and will be lost if the code is regenerated.
+# --------------------------------------------------------------------------
+from ._version import VERSION
+__version__ = VERSION

package/autorest/multiclient/templates/version.py.jinja2

@@ -0,0 +1,8 @@
+# coding=utf-8
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+VERSION = "{{ package_version }}"

package/autorest/postprocess/__init__.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-from typing import Tuple
+from typing import Tuple, Any, Dict
 from pathlib import Path
 import os
 import shutil
@@ -11,7 +11,7 @@ from venv import EnvBuilder
 import black
 from .venvtools import ExtendedEnvBuilder, python_run
 
-from .. import Plugin
+from .. import Plugin, PluginAutorest
 
 _BLACK_MODE = black.Mode()
 _BLACK_MODE.line_length = 120
@@ -29,10 +29,10 @@ def format_file(file: Path, file_content: str) -> str:
     return file_content
 
 
-class PostProcessPlugin(Plugin):
-    def __init__(self, autorestapi):
-        super().__init__(autorestapi)
-        output_folder_uri = self._autorestapi.get_value("outputFolderUri")
+class PostProcessPlugin(Plugin):  # pylint: disable=abstract-method
+    def __init__(self, **kwargs: Any):
+        super().__init__(**kwargs)
+        output_folder_uri = self.options["outputFolderUri"]
         if output_folder_uri.startswith("file:"):
             output_folder_uri = output_folder_uri[5:]
         if os.name == "nt" and output_folder_uri.startswith("///"):
@@ -65,10 +65,8 @@ class PostProcessPlugin(Plugin):
         try:
             init_file = next(d for d in dir.iterdir() if d.name == "__init__.py")
             # we don't care about pkgutil inits, we skip over them
-            file_content = self._autorestapi.read_file(
-                init_file.relative_to(self.output_folder)
-            )
-            if not "pkgutil" in file_content:
+            file_content = self.read_file(init_file.relative_to(self.output_folder))
+            if "pkgutil" not in file_content:
                 return dir, namespace
         except StopIteration:
             pass
@@ -157,7 +155,7 @@ class PostProcessPlugin(Plugin):
             k: None for k in customized_objects_str.split(",")
         }.keys()  # filter out duplicates
         file = (folder_path / "__init__.py").relative_to(self.output_folder)
-        file_content = self._autorestapi.read_file(file)
+        file_content = self.read_file(file)
         added_objs = []
         for obj in customized_objects:
             if f" import {obj}\n" in file_content:
@@ -199,4 +197,9 @@ class PostProcessPlugin(Plugin):
                 "__all__ = [", f"__all__ = [\n{added_objs_all}", 1
             )
         formatted_file = format_file(file, file_content)
-        self._autorestapi.write_file(file, formatted_file)
+        self.write_file(file, formatted_file)
+
+
+class PostProcessPluginAutorest(PostProcessPlugin, PluginAutorest):
+    def get_options(self) -> Dict[str, Any]:
+        return {"outputFolderUri": self._autorestapi.get_value("outputFolderUri")}

package/autorest/preprocess/__init__.py

@@ -6,10 +6,12 @@
 """The preprocessing autorest plugin.
 """
 from typing import Callable, Dict, Any, List, Optional
-from .helpers import to_snake_case, pad_reserved_words, add_redefined_builtin_info
+from .._utils import to_snake_case
+from .helpers import pad_reserved_words, add_redefined_builtin_info
 from .python_mappings import PadType
 
-from .. import YamlUpdatePlugin
+from .. import YamlUpdatePlugin, YamlUpdatePluginAutorest
+from .._utils import parse_args
 
 
 def _remove_paging_maxpagesize(yaml_data: Dict[str, Any]) -> None:
@@ -92,12 +94,12 @@ def update_paging_response(yaml_data: Dict[str, Any]) -> None:
     )
 
 
-class PreProcessPlugin(YamlUpdatePlugin):
+class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
     """Add Python naming information."""
 
     @property
     def version_tolerant(self) -> bool:
-        return bool(self._autorestapi.get_boolean_value("version-tolerant", True))
+        return self.options.get("version-tolerant", True)
 
     def get_operation_updater(
         self, yaml_data: Dict[str, Any]
@@ -133,7 +135,7 @@ class PreProcessPlugin(YamlUpdatePlugin):
             response["discriminator"] = "operation"
 
     def _update_lro_operation_helper(self, yaml_data: Dict[str, Any]) -> None:
-        azure_arm = self._autorestapi.get_boolean_value("azure-arm", False)
+        azure_arm = self.options.get("azure-arm", False)
         for response in yaml_data.get("responses", []):
             response["discriminator"] = "lro"
             response["pollerSync"] = (
@@ -228,3 +230,20 @@ class PreProcessPlugin(YamlUpdatePlugin):
         update_client(yaml_data["client"])
         update_types(yaml_data["types"])
         self.update_operation_groups(yaml_data)
+
+
+class PreProcessPluginAutorest(YamlUpdatePluginAutorest, PreProcessPlugin):
+    def get_options(self) -> Dict[str, Any]:
+        options = {
+            "version-tolerant": self._autorestapi.get_boolean_value("version-tolerant"),
+            "azure-arm": self._autorestapi.get_boolean_value("azure-arm"),
+        }
+        return {k: v for k, v in options.items() if v is not None}
+
+
+if __name__ == "__main__":
+    # CADL pipeline will call this
+    args = parse_args()
+    PreProcessPlugin(
+        output_folder=args.output_folder, cadl_file=args.cadl_file
+    ).process()

package/autorest/preprocess/helpers.py

@@ -4,39 +4,9 @@
 # license information.
 # --------------------------------------------------------------------------
 from typing import Any, Dict
-import re
 from .python_mappings import PadType, RESERVED_WORDS, REDEFINED_BUILTINS
 
 
-def to_snake_case(name: str) -> str:
-    def replace_upper_characters(m) -> str:
-        match_str = m.group().lower()
-        if m.start() > 0 and name[m.start() - 1] == "_":
-            # we are good if a '_' already exists
-            return match_str
-        # the first letter should not have _
-        prefix = "_" if m.start() > 0 else ""
-
-        # we will add an extra _ if there are multiple upper case chars together
-        next_non_upper_case_char_location = m.start() + len(match_str)
-        if (
-            len(match_str) > 2
-            and len(name) - next_non_upper_case_char_location > 1
-            and name[next_non_upper_case_char_location].isalpha()
-        ):
-
-            return (
-                prefix
-                + match_str[: len(match_str) - 1]
-                + "_"
-                + match_str[len(match_str) - 1]
-            )
-
-        return prefix + match_str
-
-    return re.sub("[A-Z]+", replace_upper_characters, name)
-
-
 def pad_reserved_words(name: str, pad_type: PadType):
     # we want to pad hidden variables as well
     if not name:

package/install.py

@@ -6,8 +6,8 @@
 # license information.
 # --------------------------------------------------------------------------
 import sys
-if not sys.version_info >= (3, 6, 0):
-    raise Exception("Autorest for Python extension requires Python 3.6 at least")
+if not sys.version_info >= (3, 7, 0):
+    raise Exception("Autorest for Python extension requires Python 3.7 at least")
 
 try:
     import pip
@@ -20,7 +20,7 @@ except ImportError:
     raise Exception("Your Python installation doesn't have venv available")
 
 
-# Now we have pip and Py >= 3.6, go to work
+# Now we have pip and Py >= 3.7, go to work
 
 import subprocess
 from pathlib import Path

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@autorest/python",
-    "version": "6.0.0",
+    "version": "6.1.1",
     "description": "The Python extension for generators in AutoRest.",
     "scripts": {
         "prepare": "node run-python3.js prepare.py",
@@ -38,6 +38,7 @@
         "start.py",
         "venvtools.py",
         "run-python3.js",
-        "requirements.txt"
+        "requirements.txt",
+        "run_cadl.py"
     ]
 }

package/prepare.py

@@ -6,8 +6,8 @@
 # license information.
 # --------------------------------------------------------------------------
 import sys
-if not sys.version_info >= (3, 6, 0):
-    raise Exception("Autorest for Python extension requires Python 3.6 at least")
+if not sys.version_info >= (3, 7, 0):
+    raise Exception("Autorest for Python extension requires Python 3.7 at least")
 
 from pathlib import Path
 import venv

package/requirements.txt

@@ -1,14 +1,13 @@
-black==21.12b0
-click==8.0.3
-docutils==0.18.1
-Jinja2==3.0.3
+black==22.6.0
+click==8.1.3
+docutils==0.18
+Jinja2==3.1.2
 json-rpc==1.13.0
-m2r==0.2.1
-MarkupSafe==2.0.1
+m2r2==0.3.2
+MarkupSafe==2.1.1
 mistune==0.8.4
 mypy-extensions==0.4.3
 pathspec==0.9.0
-platformdirs==2.4.0
+platformdirs==2.5.2
 PyYAML==6.0
-tomli==1.2.2
-typing-extensions==4.0.1
+tomli==2.0.1

package/run-python3.js

@@ -9,8 +9,8 @@
 const cp = require("child_process");
 const extension = require("@autorest/system-requirements");
 
-async function runPython3(scriptName, debug = "") {
-  const command = await extension.patchPythonPath(["python", scriptName, debug], { version: ">=3.6", environmentVariable: "AUTOREST_PYTHON_EXE" });
+async function runPython3(scriptName, ...args) {
+  const command = await extension.patchPythonPath(["python", scriptName, ...args], { version: ">=3.7", environmentVariable: "AUTOREST_PYTHON_EXE" });
   cp.execSync(command.join(" "), {
     stdio: [0, 1, 2]
   });

package/run_cadl.py

@@ -0,0 +1,26 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import sys
+import venv
+from pathlib import Path
+from venvtools import python_run
+
+_ROOT_DIR = Path(__file__).parent
+
+if __name__ == "__main__":
+    venv_path = _ROOT_DIR / "venv"
+    venv_prexists = venv_path.exists()
+
+    assert venv_prexists  # Otherwise install was not done
+
+    env_builder = venv.EnvBuilder(with_pip=True)
+    venv_context = env_builder.ensure_directories(venv_path)
+
+    # run m2r
+    python_run(venv_context, "autorest.m2r.__init__", command=sys.argv[1:])
+    python_run(venv_context, "autorest.preprocess.__init__", command=sys.argv[1:])
+    python_run(venv_context, "autorest.codegen.__init__", command=sys.argv[1:])
+    python_run(venv_context, "autorest.black.__init__", command=sys.argv[1:])

package/setup.py

@@ -36,7 +36,6 @@ setup(
         'Development Status :: 4 - Beta',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'License :: OSI Approved :: MIT License',
@@ -48,8 +47,8 @@ setup(
         "json-rpc",
         "Jinja2 >= 2.11", # I need "include" and auto-context + blank line are not indented by default
         "pyyaml",
-        "mistune < 2.0.0",  # Need to pin mistune's max version so m2r doesn't break
-        "m2r",
+        "m2r2",
         "black",
+        "docutils<0.19", # m2r2 fails with docutils 0.19
     ],
 )

package/start.py

@@ -6,8 +6,8 @@
 # license information.
 # --------------------------------------------------------------------------
 import sys
-if not sys.version_info >= (3, 6, 0):
-    raise Exception("Autorest for Python extension requires Python 3.6 at least")
+if not sys.version_info >= (3, 7, 0):
+    raise Exception("Autorest for Python extension requires Python 3.7 at least")
 
 from pathlib import Path
 import venv