@autorest/python 5.16.0 → 5.19.0

package/autorest/multiapi/models/client.py

@@ -5,9 +5,10 @@
 # --------------------------------------------------------------------------
 import sys
 import json
+import re
 from typing import Any, Dict, List
 from pathlib import Path
-from .imports import FileImport
+from .imports import FileImport, TypingSection, ImportType
 
 
 def _extract_version(metadata_json: Dict[str, Any], version_path: Path) -> str:
@@ -43,9 +44,18 @@ class Client:
 
     def imports(self, async_mode: bool) -> FileImport:
         imports_to_load = "async_imports" if async_mode else "sync_imports"
-        return FileImport(
+        file_import = FileImport(
             json.loads(self.default_version_metadata["client"][imports_to_load])
         )
+        local_imports = file_import.imports.get(TypingSection.REGULAR, {}).get(
+            ImportType.LOCAL, {}
+        )
+        for key in local_imports:
+            if re.search("^\\.*_serialization$", key):
+                relative_path = ".." if async_mode else "."
+                local_imports[f"{relative_path}_serialization"] = local_imports.pop(key)
+                break
+        return file_import
 
     @property
     def parameterized_host_template_to_api_version(self) -> Dict[str, List[str]]:

package/autorest/multiapi/models/code_model.py

@@ -34,7 +34,7 @@ class CodeModel:  # pylint: disable=too-many-instance-attributes
         self.azure_arm = default_version_metadata["client"]["azure_arm"]
         self.default_version_metadata = default_version_metadata
         self.version_path_to_metadata = version_path_to_metadata
-        self.service_client = Client(
+        self.client = Client(
             self.azure_arm, default_version_metadata, version_path_to_metadata
         )
         self.config = Config(default_version_metadata)

package/autorest/multiapi/serializers/__init__.py

@@ -18,7 +18,7 @@ __all__ = [
 
 _FILE_TO_TEMPLATE = {
     "init": "multiapi_init.py.jinja2",
-    "service_client": "multiapi_service_client.py.jinja2",
+    "client": "multiapi_service_client.py.jinja2",
     "config": "multiapi_config.py.jinja2",
     "models": "multiapi_models.py.jinja2",
     "operations_mixin": "multiapi_operations_mixin.py.jinja2",
@@ -58,11 +58,11 @@ class MultiAPISerializer(object):
 
         # serialize service client file
         imports = FileImportSerializer(
-            code_model.service_client.imports(async_mode), is_python3_file=async_mode
+            code_model.client.imports(async_mode), is_python3_file=async_mode
         )
         self._autorestapi.write_file(
-            _get_file_path(code_model.service_client.filename, async_mode),
-            _render_template("service_client", imports=imports),
+            _get_file_path(code_model.client.filename, async_mode),
+            _render_template("client", imports=imports),
         )
 
         # serialize config file
@@ -115,3 +115,17 @@ class MultiAPISerializer(object):
             )
 
         self._autorestapi.write_file(Path("py.typed"), "# Marker file for PEP 561.")
+
+        if not code_model.client.client_side_validation:
+            codegen_env = Environment(
+                loader=PackageLoader("autorest.codegen", "templates"),
+                keep_trailing_newline=True,
+                line_statement_prefix="##",
+                line_comment_prefix="###",
+                trim_blocks=True,
+                lstrip_blocks=True,
+            )
+            self._autorestapi.write_file(
+                Path("_serialization.py"),
+                codegen_env.get_template("serialization.py.jinja2").render(),
+            )

package/autorest/multiapi/templates/multiapi_config.py.jinja2

@@ -28,8 +28,8 @@ def __init__(
 # --------------------------------------------------------------------------
 {{ imports }}
 
-class {{ code_model.service_client.name }}Configuration(Configuration):
-    """Configuration for {{ code_model.service_client.name }}.
+class {{ code_model.client.name }}Configuration(Configuration):
+    """Configuration for {{ code_model.client.name }}.
 
     Note that all parameters used to create this instance are saved as instance
     attributes.
@@ -52,7 +52,7 @@ class {{ code_model.service_client.name }}Configuration(Configuration):
             raise ValueError("Parameter '{{ parameter.name }}' must not be None.")
     {% endif %}
 {% endfor %}
-        super({{ code_model.service_client.name }}Configuration, self).__init__(**kwargs)
+        super({{ code_model.client.name }}Configuration, self).__init__(**kwargs)
 
 {% for parameter in code_model.global_parameters.parameters %}
         self.{{ parameter.name }} = {{ parameter.name }}

package/autorest/multiapi/templates/multiapi_init.py.jinja2

@@ -6,8 +6,8 @@
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 
-from .{{ code_model.service_client.filename }} import {{ code_model.service_client.name }}
-__all__ = ['{{ code_model.service_client.name }}']
+from .{{ code_model.client.filename }} import {{ code_model.client.name }}
+__all__ = ['{{ code_model.client.name }}']
 {% if not async_mode %}
 
 try:

package/autorest/multiapi/templates/multiapi_operations_mixin.py.jinja2

@@ -8,13 +8,13 @@
 # Changes may cause incorrect behavior and will be lost if the code is
 # regenerated.
 # --------------------------------------------------------------------------
-from msrest import Serializer, Deserializer
+from {{ ".." if async_mode else "." }}_serialization import Serializer, Deserializer
 {% if imports %}
 {{ imports }}
 {% endif %}
 
 
-class {{ code_model.service_client.name }}OperationsMixin(object):
+class {{ code_model.client.name }}OperationsMixin(object):
 {% for mixin_operation in code_model.operation_mixin_group.mixin_operations %}
 
     {{ mixin_operation.signature(async_mode) | indent }}        {{ mixin_operation.description(async_mode) | indent(8) }}
@@ -22,7 +22,7 @@ class {{ code_model.service_client.name }}OperationsMixin(object):
         {% for api in mixin_operation.available_apis|sort %}
         {% set if_statement = "if" if loop.first else "elif" %}
         {{ if_statement }} api_version == '{{ code_model.mod_to_api_version[api] }}':
-            from {{ ".." if async_mode else "." }}{{ api }}{{ ".aio" if async_mode else "" }}.operations import {{ code_model.service_client.name }}OperationsMixin as OperationClass
+            from {{ ".." if async_mode else "." }}{{ api }}{{ ".aio" if async_mode else "" }}.operations import {{ code_model.client.name }}OperationsMixin as OperationClass
         {% endfor %}
         else:
             raise ValueError("API version {} does not have operation '{{ mixin_operation.name }}'".format(api_version))
@@ -30,7 +30,7 @@ class {{ code_model.service_client.name }}OperationsMixin(object):
         mixin_instance._client = self._client
         mixin_instance._config = self._config
         mixin_instance._serialize = Serializer(self._models_dict(api_version))
-        {% if not code_model.service_client.client_side_validation %}
+        {% if not code_model.client.client_side_validation %}
         mixin_instance._serialize.client_side_validation = False
         {% endif %}
         mixin_instance._deserialize = Deserializer(self._models_dict(api_version))

package/autorest/multiapi/templates/multiapi_service_client.py.jinja2

@@ -43,8 +43,8 @@ class _SDKClient(object):
         """
         pass
 
-class {{ code_model.service_client.name }}({% if code_model.operation_mixin_group.mixin_operations %}{{ code_model.service_client.name }}OperationsMixin, {% endif %}MultiApiClientMixin, _SDKClient):
-    """{{ code_model.service_client.description }}
+class {{ code_model.client.name }}({% if code_model.operation_mixin_group.mixin_operations %}{{ code_model.client.name }}OperationsMixin, {% endif %}MultiApiClientMixin, _SDKClient):
+    """{{ code_model.client.description }}
 
     This ready contains multiple API versions, to help you deal with all of the Azure clouds
     (Azure Stack, Azure Government, Azure China, etc.).
@@ -64,13 +64,13 @@ class {{ code_model.service_client.name }}({% if code_model.operation_mixin_grou
     :param {{ parameter.name }}: {{ parameter.description(async_mode) }}
     :type {{ parameter.name }}: {{ parameter.docstring_type(async_mode) }}
 {% endfor %}
-    {% if code_model.service_client.has_lro_operations %}
+    {% if code_model.client.has_lro_operations %}
     :keyword int polling_interval: Default waiting time between two polls for LRO operations if no Retry-After header is present.
     {% endif %}
     """
 
     DEFAULT_API_VERSION = '{{ code_model.mod_to_api_version[code_model.default_api_version] }}'
-    _PROFILE_TAG = "{{ code_model.module_name }}.{{ code_model.service_client.name }}"
+    _PROFILE_TAG = "{{ code_model.module_name }}.{{ code_model.client.name }}"
     LATEST_PROFILE = ProfileDefinition({
         _PROFILE_TAG: {
             None: DEFAULT_API_VERSION,
@@ -82,8 +82,8 @@ class {{ code_model.service_client.name }}({% if code_model.operation_mixin_grou
     )
 
     {{ method_signature()|indent }}
-        {% if not code_model.service_client.host_value %}
-        {% for parameterized_host_template, api_versions in code_model.service_client.parameterized_host_template_to_api_version|dictsort %}
+        {% if not code_model.client.host_value %}
+        {% for parameterized_host_template, api_versions in code_model.client.parameterized_host_template_to_api_version|dictsort %}
         {% set if_statement = "if" if loop.first else "elif" %}
         {{ if_statement ~ " api_version == '" ~ api_versions|join("' or api_version == '") ~ "'" }}:
             base_url = {{ parameterized_host_template }}
@@ -91,9 +91,9 @@ class {{ code_model.service_client.name }}({% if code_model.operation_mixin_grou
         else:
             raise ValueError("API version {} is not available".format(api_version))
         {% endif %}
-        self._config = {{ code_model.service_client.name }}Configuration({{ code_model.global_parameters.call }}{{ ", " if code_model.global_parameters.call }}**kwargs)
-        self._client = {{ async_prefix }}{{ code_model.service_client.pipeline_client }}(base_url=base_url, config=self._config, **kwargs)
-        super({{ code_model.service_client.name }}, self).__init__(
+        self._config = {{ code_model.client.name }}Configuration({{ code_model.global_parameters.call }}{{ ", " if code_model.global_parameters.call }}**kwargs)
+        self._client = {{ async_prefix }}{{ code_model.client.pipeline_client }}(base_url=base_url, config=self._config, **kwargs)
+        super({{ code_model.client.name }}, self).__init__(
             api_version=api_version,
             profile=profile
         )

package/autorest/postprocess/__init__.py

@@ -0,0 +1,202 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from typing import Tuple
+from pathlib import Path
+import os
+import shutil
+from venv import EnvBuilder
+import black
+from .venvtools import ExtendedEnvBuilder, python_run
+
+from .. import Plugin
+
+_BLACK_MODE = black.Mode()
+_BLACK_MODE.line_length = 120
+
+
+def format_file(file: Path, file_content: str) -> str:
+    if not file.suffix == ".py":
+        return file_content
+    try:
+        file_content = black.format_file_contents(
+            file_content, fast=True, mode=_BLACK_MODE
+        )
+    except black.NothingChanged:
+        pass
+    return file_content
+
+
+class PostProcessPlugin(Plugin):
+    def __init__(self, autorestapi):
+        super().__init__(autorestapi)
+        output_folder_uri = self._autorestapi.get_value("outputFolderUri")
+        if output_folder_uri.startswith("file:"):
+            output_folder_uri = output_folder_uri[5:]
+        if os.name == "nt" and output_folder_uri.startswith("///"):
+            output_folder_uri = output_folder_uri[3:]
+        self.output_folder = Path(output_folder_uri)  # path to where the setup.py is
+        self.setup_venv()
+
+        # set up the venv
+        # base folder is where the code starts, i.e. where we
+        self.base_folder, self.namespace = self.get_namespace(self.output_folder, "")
+
+    def setup_venv(self):
+        venv_path = self.output_folder / Path(".temp_folder") / Path("temp_venv")
+
+        if venv_path.exists():
+            env_builder = EnvBuilder(with_pip=True)
+            self.venv_context = env_builder.ensure_directories(venv_path)
+        else:
+            env_builder = ExtendedEnvBuilder(with_pip=True)
+            env_builder.create(venv_path)
+            self.venv_context = env_builder.context
+            python_run(
+                self.venv_context,
+                "pip",
+                ["install", "-e", str(self.output_folder)],
+                directory=self.output_folder,
+            )
+
+    def get_namespace(self, dir: Path, namespace: str) -> Tuple[Path, str]:
+        try:
+            init_file = next(d for d in dir.iterdir() if d.name == "__init__.py")
+            # we don't care about pkgutil inits, we skip over them
+            file_content = self._autorestapi.read_file(
+                init_file.relative_to(self.output_folder)
+            )
+            if not "pkgutil" in file_content:
+                return dir, namespace
+        except StopIteration:
+            pass
+
+        try:
+            # first, see if we can get a folder that has the same name as the current output folder
+            start = self.output_folder.stem.split("-")[0]
+            next_dir = next(d for d in dir.iterdir() if d.is_dir() and d.name == start)
+        except StopIteration:
+            invalid_start_chars = [".", "_"]
+            invalid_dirs = [
+                "swagger",
+                "out",
+                "tests",
+                "samples",
+            ]
+
+            next_dir = next(
+                d
+                for d in dir.iterdir()
+                if d.is_dir()
+                and not str(d).endswith("egg-info")
+                and d.name[0] not in invalid_start_chars
+                and d.name not in invalid_dirs
+            )
+
+        namespace = f"{namespace}.{next_dir.name}" if namespace else next_dir.name
+        return self.get_namespace(next_dir, namespace)
+
+    def process(self) -> bool:
+        folders = [
+            f
+            for f in self.base_folder.glob("**/*")
+            if f.is_dir() and not f.stem.startswith("__")
+        ]
+        # will always have the root
+        self.fix_imports_in_init(
+            generated_file_name="_client",
+            folder_path=self.base_folder,
+            namespace=self.namespace,
+        )
+        try:
+            aio_folder = next(f for f in folders if f.stem == "aio")
+            self.fix_imports_in_init(
+                generated_file_name="_client",
+                folder_path=aio_folder,
+                namespace=f"{self.namespace}.aio",
+            )
+        except StopIteration:
+            pass
+
+        try:
+            models_folder = next(f for f in folders if f.stem == "models")
+            self.fix_imports_in_init(
+                generated_file_name="_models",
+                folder_path=models_folder,
+                namespace=f"{self.namespace}.models",
+            )
+        except StopIteration:
+            pass
+        operations_folders = [
+            f for f in folders if f.stem in ["operations", "_operations"]
+        ]
+        for operations_folder in operations_folders:
+            aio = ".aio" if operations_folder.parent.stem == "aio" else ""
+            self.fix_imports_in_init(
+                generated_file_name="_operations",
+                folder_path=operations_folder,
+                namespace=f"{self.namespace}{aio}.{operations_folder.stem}",
+            )
+        shutil.rmtree(f"{str(self.output_folder)}/.temp_folder")
+        return True
+
+    def fix_imports_in_init(
+        self, generated_file_name: str, folder_path: Path, namespace: str
+    ) -> None:
+        customized_objects_str = python_run(
+            self.venv_context,
+            command=[namespace, str(self.output_folder)],
+            module="get_all",
+        )
+
+        if not customized_objects_str:
+            return
+        customized_objects = {
+            k: None for k in customized_objects_str.split(",")
+        }.keys()  # filter out duplicates
+        file = (folder_path / "__init__.py").relative_to(self.output_folder)
+        file_content = self._autorestapi.read_file(file)
+        added_objs = []
+        for obj in customized_objects:
+            if f" import {obj}\n" in file_content:
+                # means we're overriding a generated model
+                file_content = file_content.replace(
+                    f"from .{generated_file_name} import {obj}\n",
+                    f"from ._patch import {obj}\n",
+                )
+            else:
+                added_objs.append(obj)
+        file_content = file_content.replace(
+            "try:\n    from ._patch import __all__ as _patch_all\n    "
+            "from ._patch import *  # type: ignore # pylint: disable=unused-wildcard-import"
+            "\nexcept ImportError:\n    _patch_all = []",
+            "",
+        )
+        file_content = file_content.replace(
+            "from ._patch import __all__ as _patch_all", ""
+        )
+        file_content = file_content.replace(
+            "from ._patch import *  # type: ignore # pylint: disable=unused-wildcard-import\n",
+            "",
+        )
+        file_content = file_content.replace(
+            "__all__.extend([p for p in _patch_all if p not in __all__])", ""
+        )
+        if added_objs:
+            # add import
+            patch_sdk_import = "from ._patch import patch_sdk as _patch_sdk"
+            imports = "\n".join([f"from ._patch import {obj}" for obj in added_objs])
+            if imports:
+                replacement = f"{imports}\n{patch_sdk_import}"
+            else:
+                replacement = patch_sdk_import
+            file_content = file_content.replace(patch_sdk_import, replacement)
+            # add to __all__
+            added_objs_all = "\n".join([f'    "{obj}",' for obj in added_objs]) + "\n"
+            file_content = file_content.replace(
+                "__all__ = [", f"__all__ = [\n{added_objs_all}", 1
+            )
+        formatted_file = format_file(file, file_content)
+        self._autorestapi.write_file(file, formatted_file)

package/autorest/postprocess/get_all.py

@@ -0,0 +1,19 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import sys
+import importlib
+
+
+def main(namespace):
+    sdk = importlib.import_module(namespace)
+    return sdk._patch.__all__  # pylint: disable=protected-access
+
+
+if __name__ == "__main__":
+    patched = ",".join(main(sys.argv[1]))
+    output_folder = sys.argv[2]
+    with open(f"{output_folder}/.temp_folder/patched.txt", "w") as f:
+        f.write(patched)

package/autorest/postprocess/venvtools.py

@@ -0,0 +1,73 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from typing import Optional
+import subprocess
+import venv
+import sys
+from pathlib import Path
+
+
+_ROOT_DIR = Path(__file__).parent
+
+
+class ExtendedEnvBuilder(venv.EnvBuilder):
+    """An extended env builder which saves the context, to have access
+    easily to bin path and such.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.context = None
+        super(ExtendedEnvBuilder, self).__init__(*args, **kwargs)
+
+    def ensure_directories(self, env_dir):
+        self.context = super(ExtendedEnvBuilder, self).ensure_directories(env_dir)
+        return self.context
+
+
+def create(
+    env_dir,
+    system_site_packages=False,
+    clear=False,
+    symlinks=False,
+    with_pip=False,
+    prompt=None,
+):
+    """Create a virtual environment in a directory."""
+    builder = ExtendedEnvBuilder(
+        system_site_packages=system_site_packages,
+        clear=clear,
+        symlinks=symlinks,
+        with_pip=with_pip,
+        prompt=prompt,
+    )
+    builder.create(env_dir)
+    return builder.context
+
+
+def python_run(  # pylint: disable=inconsistent-return-statements
+    venv_context, module, command, directory=_ROOT_DIR, *, error_ok=False
+) -> Optional[str]:
+    try:
+        cmd_line = [
+            venv_context.env_exe,
+            "-m",
+            module,
+        ] + command
+        print("Executing: {}".format(" ".join(cmd_line)))
+        subprocess.run(
+            cmd_line,
+            cwd=directory,
+            check=True,
+            stdout=False,
+        )
+        if module == "get_all":
+            with open(f"{command[1]}/.temp_folder/patched.txt", "r") as f:
+                return f.read()
+    except subprocess.CalledProcessError as err:
+        print(err)
+        if not error_ok:
+            sys.exit(1)
+    return None