@autorest/python 6.0.1 → 6.1.2

package/autorest/preprocess/__init__.py

@@ -5,11 +5,107 @@
 # --------------------------------------------------------------------------
 """The preprocessing autorest plugin.
 """
+import copy
 from typing import Callable, Dict, Any, List, Optional
-from .helpers import to_snake_case, pad_reserved_words, add_redefined_builtin_info
+
+from .._utils import to_snake_case
+from .helpers import pad_reserved_words, add_redefined_builtin_info
 from .python_mappings import PadType
 
-from .. import YamlUpdatePlugin, PluginAutorest
+from .. import YamlUpdatePlugin, YamlUpdatePluginAutorest
+from .._utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES
+
+
+def add_body_param_type(code_model: Dict[str, Any], body_parameter: Dict[str, Any]):
+    if (
+        body_parameter
+        and body_parameter["type"]["type"] in ("model", "dict", "list")
+        and any(
+            ct for ct in body_parameter.get("contentTypes", []) if JSON_REGEXP.match(ct)
+        )
+        and not body_parameter["type"].get("xmlMetadata")
+        and not any(t for t in ["flattened", "groupedBy"] if body_parameter.get(t))
+    ):
+        body_parameter["type"] = {
+            "type": "combined",
+            "types": [body_parameter["type"], KNOWN_TYPES["binary"]],
+        }
+        code_model["types"].append(body_parameter["type"])
+
+
+def update_overload_section(
+    overload: Dict[str, Any],
+    yaml_data: Dict[str, Any],
+    section: str,
+):
+    for overload_s, original_s in zip(overload[section], yaml_data[section]):
+        if overload_s.get("type"):
+            overload_s["type"] = original_s["type"]
+        if overload_s.get("headers"):
+            for overload_h, original_h in zip(
+                overload_s["headers"], original_s["headers"]
+            ):
+                if overload_h.get("type"):
+                    overload_h["type"] = original_h["type"]
+
+
+def add_overload(yaml_data: Dict[str, Any], body_type: Dict[str, Any]):
+    overload = copy.deepcopy(yaml_data)
+    overload["isOverload"] = True
+    overload["bodyParameter"]["type"] = body_type
+
+    overload["overloads"] = []
+
+    # for yaml sync, we need to make sure all of the responses, parameters, and exceptions' types have the same yaml id
+    for overload_p, original_p in zip(overload["parameters"], yaml_data["parameters"]):
+        overload_p["type"] = original_p["type"]
+    update_overload_section(overload, yaml_data, "responses")
+    update_overload_section(overload, yaml_data, "exceptions")
+
+    # update content type to be an overloads content type
+    content_type_param = next(
+        p for p in overload["parameters"] if p["restApiName"].lower() == "content-type"
+    )
+    content_type_param["inOverload"] = True
+    content_type_param["inDocstring"] = True
+    body_type_description = get_body_type_for_description(overload["bodyParameter"])
+    content_type_param[
+        "description"
+    ] = f"Body Parameter content-type. Content type parameter for {body_type_description} body."
+    content_types = yaml_data["bodyParameter"]["contentTypes"]
+    if body_type["type"] == "binary" and len(content_types) > 1:
+        content_types = "'" + "', '".join(content_types) + "'"
+        content_type_param["description"] += f" Known values are: {content_types}."
+    return overload
+
+
+def add_overloads_for_body_param(yaml_data: Dict[str, Any]) -> None:
+    """If we added a body parameter type, add overloads for that type"""
+    body_parameter = yaml_data["bodyParameter"]
+    if not (
+        body_parameter["type"]["type"] == "combined"
+        and len(yaml_data["bodyParameter"]["type"]["types"])
+        > len(yaml_data["overloads"])
+    ):
+        return
+    for body_type in body_parameter["type"]["types"]:
+        if any(
+            o
+            for o in yaml_data["overloads"]
+            if id(o["bodyParameter"]["type"]) == id(body_type)
+        ):
+            continue
+        yaml_data["overloads"].append(add_overload(yaml_data, body_type))
+    content_type_param = next(
+        p for p in yaml_data["parameters"] if p["restApiName"].lower() == "content-type"
+    )
+    content_type_param["inOverload"] = False
+    content_type_param["inOverriden"] = True
+    content_type_param["inDocstring"] = True
+    content_type_param[
+        "clientDefaultValue"
+    ] = None  # make it none bc it will be overriden, we depend on default of overloads
+    content_type_param["optional"] = True
 
 
 def _remove_paging_maxpagesize(yaml_data: Dict[str, Any]) -> None:
@@ -101,7 +197,7 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
 
     def get_operation_updater(
         self, yaml_data: Dict[str, Any]
-    ) -> Callable[[Dict[str, Any]], None]:
+    ) -> Callable[[Dict[str, Any], Dict[str, Any]], None]:
         if yaml_data["discriminator"] == "lropaging":
             return self.update_lro_paging_operation
         if yaml_data["discriminator"] == "lro":
@@ -110,7 +206,13 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
             return self.update_paging_operation
         return self.update_operation
 
-    def update_operation(self, yaml_data: Dict[str, Any]) -> None:
+    def update_operation(
+        self,
+        code_model: Dict[str, Any],
+        yaml_data: Dict[str, Any],
+        *,
+        is_overload: bool = False,
+    ) -> None:
         yaml_data["groupName"] = pad_reserved_words(
             yaml_data["groupName"], PadType.OPERATION_GROUP
         )
@@ -121,6 +223,7 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
             yaml_data["description"], yaml_data["name"]
         )
         yaml_data["summary"] = update_description(yaml_data.get("summary", ""))
+        body_parameter = yaml_data.get("bodyParameter")
         for parameter in yaml_data["parameters"]:
             update_parameter(parameter)
         if yaml_data.get("bodyParameter"):
@@ -128,9 +231,13 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
             for entry in yaml_data["bodyParameter"].get("entries", []):
                 update_parameter(entry)
         for overload in yaml_data.get("overloads", []):
-            self.update_operation(overload)
+            self.update_operation(code_model, overload, is_overload=True)
         for response in yaml_data.get("responses", []):
             response["discriminator"] = "operation"
+        if body_parameter and not is_overload:
+            # if we have a JSON body, we add a binary overload
+            add_body_param_type(code_model, body_parameter)
+            add_overloads_for_body_param(yaml_data)
 
     def _update_lro_operation_helper(self, yaml_data: Dict[str, Any]) -> None:
         azure_arm = self.options.get("azure-arm", False)
@@ -155,23 +262,29 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
                     else "azure.core.polling.async_base_polling.AsyncLROBasePolling"
                 )
 
-    def update_lro_paging_operation(self, yaml_data: Dict[str, Any]) -> None:
-        self.update_lro_operation(yaml_data)
-        self.update_paging_operation(yaml_data)
+    def update_lro_paging_operation(
+        self, code_model: Dict[str, Any], yaml_data: Dict[str, Any]
+    ) -> None:
+        self.update_lro_operation(code_model, yaml_data)
+        self.update_paging_operation(code_model, yaml_data)
         yaml_data["discriminator"] = "lropaging"
         for response in yaml_data.get("responses", []):
             response["discriminator"] = "lropaging"
         for overload in yaml_data.get("overloads", []):
-            self.update_lro_paging_operation(overload)
+            self.update_lro_paging_operation(code_model, overload)
 
-    def update_lro_operation(self, yaml_data: Dict[str, Any]) -> None:
-        self.update_operation(yaml_data)
+    def update_lro_operation(
+        self, code_model: Dict[str, Any], yaml_data: Dict[str, Any]
+    ) -> None:
+        self.update_operation(code_model, yaml_data)
         self._update_lro_operation_helper(yaml_data)
         for overload in yaml_data.get("overloads", []):
             self._update_lro_operation_helper(overload)
 
-    def update_paging_operation(self, yaml_data: Dict[str, Any]) -> None:
-        self.update_operation(yaml_data)
+    def update_paging_operation(
+        self, code_model: Dict[str, Any], yaml_data: Dict[str, Any]
+    ) -> None:
+        self.update_operation(code_model, yaml_data)
         if not yaml_data.get("pagerSync"):
             yaml_data["pagerSync"] = "azure.core.paging.ItemPaged"
         if not yaml_data.get("pagerAsync"):
@@ -206,7 +319,7 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
             update_paging_response(response)
             response["itemType"] = item_type
         for overload in yaml_data.get("overloads", []):
-            self.update_paging_operation(overload)
+            self.update_paging_operation(code_model, overload)
 
     def update_operation_groups(self, yaml_data: Dict[str, Any]) -> None:
         operation_groups_yaml_data = yaml_data["operationGroups"]
@@ -221,19 +334,27 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
                 yaml_data, operation_group["className"]
             )
             for operation in operation_group["operations"]:
-                self.get_operation_updater(operation)(operation)
+                self.get_operation_updater(operation)(yaml_data, operation)
 
     def update_yaml(self, yaml_data: Dict[str, Any]) -> None:
         """Convert in place the YAML str."""
         update_client(yaml_data["client"])
-        update_types(yaml_data["types"])
         self.update_operation_groups(yaml_data)
+        update_types(yaml_data["types"])
 
 
-class PreProcessPluginAutorest(PluginAutorest, PreProcessPlugin):
+class PreProcessPluginAutorest(YamlUpdatePluginAutorest, PreProcessPlugin):
     def get_options(self) -> Dict[str, Any]:
         options = {
             "version-tolerant": self._autorestapi.get_boolean_value("version-tolerant"),
             "azure-arm": self._autorestapi.get_boolean_value("azure-arm"),
         }
         return {k: v for k, v in options.items() if v is not None}
+
+
+if __name__ == "__main__":
+    # CADL pipeline will call this
+    args = parse_args()
+    PreProcessPlugin(
+        output_folder=args.output_folder, cadl_file=args.cadl_file
+    ).process()

package/autorest/preprocess/helpers.py

@@ -4,39 +4,9 @@
 # license information.
 # --------------------------------------------------------------------------
 from typing import Any, Dict
-import re
 from .python_mappings import PadType, RESERVED_WORDS, REDEFINED_BUILTINS
 
 
-def to_snake_case(name: str) -> str:
-    def replace_upper_characters(m) -> str:
-        match_str = m.group().lower()
-        if m.start() > 0 and name[m.start() - 1] == "_":
-            # we are good if a '_' already exists
-            return match_str
-        # the first letter should not have _
-        prefix = "_" if m.start() > 0 else ""
-
-        # we will add an extra _ if there are multiple upper case chars together
-        next_non_upper_case_char_location = m.start() + len(match_str)
-        if (
-            len(match_str) > 2
-            and len(name) - next_non_upper_case_char_location > 1
-            and name[next_non_upper_case_char_location].isalpha()
-        ):
-
-            return (
-                prefix
-                + match_str[: len(match_str) - 1]
-                + "_"
-                + match_str[len(match_str) - 1]
-            )
-
-        return prefix + match_str
-
-    return re.sub("[A-Z]+", replace_upper_characters, name)
-
-
 def pad_reserved_words(name: str, pad_type: PadType):
     # we want to pad hidden variables as well
     if not name:

package/install.py

@@ -6,8 +6,8 @@
 # license information.
 # --------------------------------------------------------------------------
 import sys
-if not sys.version_info >= (3, 6, 0):
-    raise Exception("Autorest for Python extension requires Python 3.6 at least")
+if not sys.version_info >= (3, 7, 0):
+    raise Exception("Autorest for Python extension requires Python 3.7 at least")
 
 try:
     import pip
@@ -20,7 +20,7 @@ except ImportError:
     raise Exception("Your Python installation doesn't have venv available")
 
 
-# Now we have pip and Py >= 3.6, go to work
+# Now we have pip and Py >= 3.7, go to work
 
 import subprocess
 from pathlib import Path

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@autorest/python",
-    "version": "6.0.1",
+    "version": "6.1.2",
     "description": "The Python extension for generators in AutoRest.",
     "scripts": {
         "prepare": "node run-python3.js prepare.py",
@@ -38,6 +38,7 @@
         "start.py",
         "venvtools.py",
         "run-python3.js",
-        "requirements.txt"
+        "requirements.txt",
+        "run_cadl.py"
     ]
 }

package/prepare.py

@@ -6,8 +6,8 @@
 # license information.
 # --------------------------------------------------------------------------
 import sys
-if not sys.version_info >= (3, 6, 0):
-    raise Exception("Autorest for Python extension requires Python 3.6 at least")
+if not sys.version_info >= (3, 7, 0):
+    raise Exception("Autorest for Python extension requires Python 3.7 at least")
 
 from pathlib import Path
 import venv

package/requirements.txt

@@ -1,14 +1,13 @@
-black==21.12b0
-click==8.0.3
-docutils==0.18.1
-Jinja2==3.0.3
+black==22.6.0
+click==8.1.3
+docutils==0.18
+Jinja2==3.1.2
 json-rpc==1.13.0
-m2r==0.2.1
-MarkupSafe==2.0.1
+m2r2==0.3.2
+MarkupSafe==2.1.1
 mistune==0.8.4
 mypy-extensions==0.4.3
 pathspec==0.9.0
-platformdirs==2.4.0
+platformdirs==2.5.2
 PyYAML==6.0
-tomli==1.2.2
-typing-extensions==4.0.1
+tomli==2.0.1

package/run-python3.js

@@ -9,8 +9,8 @@
 const cp = require("child_process");
 const extension = require("@autorest/system-requirements");
 
-async function runPython3(scriptName, debug = "") {
-  const command = await extension.patchPythonPath(["python", scriptName, debug], { version: ">=3.6", environmentVariable: "AUTOREST_PYTHON_EXE" });
+async function runPython3(scriptName, ...args) {
+  const command = await extension.patchPythonPath(["python", scriptName, ...args], { version: ">=3.7", environmentVariable: "AUTOREST_PYTHON_EXE" });
   cp.execSync(command.join(" "), {
     stdio: [0, 1, 2]
   });

package/run_cadl.py

@@ -0,0 +1,27 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import sys
+import venv
+from pathlib import Path
+from venvtools import python_run
+
+_ROOT_DIR = Path(__file__).parent
+
+if __name__ == "__main__":
+    venv_path = _ROOT_DIR / "venv"
+    venv_prexists = venv_path.exists()
+
+    assert venv_prexists  # Otherwise install was not done
+
+    env_builder = venv.EnvBuilder(with_pip=True)
+    venv_context = env_builder.ensure_directories(venv_path)
+
+    # run m2r
+    python_run(venv_context, "autorest.m2r.__init__", command=sys.argv[1:])
+    python_run(venv_context, "autorest.preprocess.__init__", command=sys.argv[1:])
+    python_run(venv_context, "autorest.cadlflags.__init__", command=sys.argv[1:])
+    python_run(venv_context, "autorest.codegen.__init__", command=sys.argv[1:])
+    python_run(venv_context, "autorest.black.__init__", command=sys.argv[1:])

package/setup.py

@@ -36,7 +36,6 @@ setup(
         'Development Status :: 4 - Beta',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'License :: OSI Approved :: MIT License',
@@ -48,8 +47,8 @@ setup(
         "json-rpc",
         "Jinja2 >= 2.11", # I need "include" and auto-context + blank line are not indented by default
         "pyyaml",
-        "mistune < 2.0.0",  # Need to pin mistune's max version so m2r doesn't break
-        "m2r",
+        "m2r2",
         "black",
+        "docutils<0.19", # m2r2 fails with docutils 0.19
     ],
 )

package/start.py

@@ -6,8 +6,8 @@
 # license information.
 # --------------------------------------------------------------------------
 import sys
-if not sys.version_info >= (3, 6, 0):
-    raise Exception("Autorest for Python extension requires Python 3.6 at least")
+if not sys.version_info >= (3, 7, 0):
+    raise Exception("Autorest for Python extension requires Python 3.7 at least")
 
 from pathlib import Path
 import venv

package/autorest/multiapi/serializers/multiapi_serializer.py

@@ -1,121 +0,0 @@
-# -------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for
-# license information.
-# --------------------------------------------------------------------------
-from typing import Any, Dict
-from pathlib import Path
-from jinja2 import Environment, PackageLoader
-
-from ...jsonrpc import AutorestAPI
-from ... import ReaderAndWriter, ReaderAndWriterAutorest
-
-
-class MultiAPISerializer(ReaderAndWriter):  # pylint: disable=abstract-method
-    def __init__(
-        self,
-        conf: Dict[str, Any],
-        async_mode: bool,
-        service_client_filename: str,
-        **kwargs: Any
-    ):
-        super().__init__(**kwargs)
-        self.conf = conf
-        self.async_mode = async_mode
-        self.service_client_filename = service_client_filename
-        self.env = Environment(
-            loader=PackageLoader("autorest.multiapi", "templates"),
-            keep_trailing_newline=True,
-            line_statement_prefix="##",
-            line_comment_prefix="###",
-            trim_blocks=True,
-            lstrip_blocks=True,
-        )
-
-    def _get_file_path(self, filename: str) -> Path:
-        if self.async_mode:
-            return Path("aio") / filename
-        return Path(filename)
-
-    def serialize(self):
-        self.write_file(
-            self._get_file_path("__init__.py"), self.serialize_multiapi_init()
-        )
-
-        service_client_filename_with_py_extension = self.service_client_filename + ".py"
-        self.write_file(
-            self._get_file_path(service_client_filename_with_py_extension),
-            self.serialize_multiapi_client(),
-        )
-
-        configuration_filename = "_configuration.py"
-        self.write_file(
-            self._get_file_path(configuration_filename),
-            self.serialize_multiapi_config(),
-        )
-
-        operation_mixins_filename = "_operations_mixin.py"
-        if self.conf["mixin_operations"]:
-            self.write_file(
-                self._get_file_path(operation_mixins_filename),
-                self.serialize_multiapi_operation_mixins(),
-            )
-
-        if self.read_file("_version.py"):
-            self.write_file("_version.py", self.read_file("_version.py"))
-        elif self.read_file("version.py"):
-            self.write_file("_version.py", self.read_file("version.py"))
-        else:
-            self.write_file(Path("_version.py"), self.serialize_multiapi_version())
-
-        # don't erase patch file
-        if self.read_file("_patch.py"):
-            self.write_file("_patch.py", self.read_file("_patch.py"))
-
-        self.write_file(Path("models.py"), self.serialize_multiapi_models())
-
-        self.write_file(Path("py.typed"), "# Marker file for PEP 561.")
-
-    def serialize_multiapi_init(self) -> str:
-        template = self.env.get_template("multiapi_init.py.jinja2")
-        return template.render(
-            service_client_filename=self.service_client_filename,
-            client_name=self.conf["client_name"],
-            async_mode=self.async_mode,
-        )
-
-    def serialize_multiapi_client(self) -> str:
-        template = self.env.get_template("multiapi_service_client.py.jinja2")
-        return template.render(**self.conf, async_mode=self.async_mode)
-
-    def serialize_multiapi_config(self) -> str:
-        template = self.env.get_template("multiapi_config.py.jinja2")
-        return template.render(**self.conf, async_mode=self.async_mode)
-
-    def serialize_multiapi_models(self) -> str:
-        template = self.env.get_template("multiapi_models.py.jinja2")
-        return template.render(**self.conf)
-
-    def serialize_multiapi_version(self) -> str:
-        template = self.env.get_template("multiapi_version.py.jinja2")
-        return template.render()
-
-    def serialize_multiapi_operation_mixins(self) -> str:
-        template = self.env.get_template("multiapi_operations_mixin.py.jinja2")
-        return template.render(**self.conf, async_mode=self.async_mode)
-
-
-class MultiAPISerializerAutorest(MultiAPISerializer, ReaderAndWriterAutorest):
-    def __init__(
-        self,
-        autorestapi: AutorestAPI,
-        conf: Dict[str, Any],
-        async_mode: bool,
-        service_client_filename: str,
-    ):
-        super().__init__(
-            autorestapi=autorestapi,
-            conf=conf,
-            async_mode=async_mode,
-            service_client_filename=service_client_filename,
-        )