@autorest/python 5.17.0 → 6.0.0-rc.1

package/autorest/codegen/templates/setup.py.jinja2

@@ -82,7 +82,11 @@ setup(
     include_package_data=True,
     {% endif %}
     install_requires=[
+        {% if code_model.is_legacy %}
         "{{ dependency_msrest }}",
+        {% else %}
+        "isodate>=0.6.1",
+        {% endif %}
         {% if azure_arm %}
         "{{ dependency_azure_mgmt_core }}",
         {% else %}

package/autorest/codegen/templates/vendor.py.jinja2

@@ -34,3 +34,13 @@ class MixinABC(ABC):
     _serialize: "Serializer"
     _deserialize: "Deserializer"
 {% endif %}
+{% if code_model.has_abstract_operations %}
+
+def raise_if_not_implemented(cls, abstract_methods):
+    not_implemented = [f for f in abstract_methods if not callable(getattr(cls, f, None))]
+    if not_implemented:
+        raise NotImplementedError("The following methods on operation group '{}' are not implemented: '{}'."
+        " Please refer to https://aka.ms/azsdk/python/dpcodegen/python/customize to learn how to customize.".format(
+        cls.__name__, '\', \''.join(not_implemented))
+        )
+{% endif %}

package/autorest/m4reformatter/__init__.py

@@ -134,7 +134,7 @@ def update_property(
 
 def update_discriminated_subtypes(yaml_data: Dict[str, Any]) -> Dict[str, Any]:
     return {
-        obj["discriminatorValue"]: obj["language"]["default"]["name"]
+        obj["discriminatorValue"]: update_type(obj)
         for obj in yaml_data.get("discriminator", {}).get("immediate", {}).values()
     }
 
@@ -431,39 +431,6 @@ def update_client_url(yaml_data: Dict[str, Any]) -> str:
     ]["uri"]
 
 
-def update_content_type_parameter(
-    yaml_data: Dict[str, Any],
-    body_parameter: Optional[Dict[str, Any]],
-    request_media_types: List[str],
-    *,
-    in_overload: bool = False,
-    in_overriden: bool = False,
-) -> Dict[str, Any]:
-    # override content type type to string
-    if not body_parameter:
-        return yaml_data
-    param = copy.deepcopy(yaml_data)
-    param["schema"] = KNOWN_TYPES["string"]  # override to string type
-    param["required"] = False
-    description = param["language"]["default"]["description"]
-    if description and description[-1] != ".":
-        description += "."
-    if not (in_overriden or in_overload):
-        param["inDocstring"] = False
-    elif in_overload:
-        description += (
-            " Content type parameter for "
-            f"{get_body_type_for_description(body_parameter)} body."
-        )
-    elif not in_overload:
-        content_types = "'" + "', '".join(request_media_types) + "'"
-        description += f" Known values are: {content_types}."
-    if not in_overload and not in_overriden:
-        param["clientDefaultValue"] = body_parameter["defaultContentType"]
-    param["language"]["default"]["description"] = description
-    return param
-
-
 class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-methods
     """Add Python naming information."""
 
@@ -471,11 +438,23 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
     def azure_arm(self) -> bool:
         return bool(self._autorestapi.get_boolean_value("azure-arm"))
 
+    @property
+    def version_tolerant(self) -> bool:
+        return bool(self._autorestapi.get_boolean_value("version-tolerant"))
+
+    @property
+    def low_level_client(self) -> bool:
+        return bool(self._autorestapi.get_boolean_value("low-level-client"))
+
+    @property
+    def legacy(self) -> bool:
+        return not (self.version_tolerant or self.low_level_client)
+
     @property
     def default_optional_constants_to_none(self) -> bool:
         return bool(
             self._autorestapi.get_boolean_value("default-optional-constants-to-none")
-            or self._autorestapi.get_boolean_value("version-tolerant")
+            or self.version_tolerant
         )
 
     def update_overloads(
@@ -515,6 +494,13 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
         in_overriden = (
             body_parameter["type"]["type"] == "combined" if body_parameter else False
         )
+        abstract = False
+        if body_parameter and (
+            body_parameter.get("entries")
+            or len(body_parameter["type"].get("types", [])) > 2
+        ):
+            # this means it's formdata or urlencoded, or there are more than 2 types of body
+            abstract = True
         return {
             "name": yaml_data["language"]["default"]["name"],
             "description": yaml_data["language"]["default"]["description"],
@@ -541,6 +527,7 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
             "discriminator": "operation",
             "isOverload": is_overload,
             "apiVersions": _get_api_versions(yaml_data.get("apiVersions", [])),
+            "abstract": abstract,
         }
 
     def get_operation_creator(
@@ -689,6 +676,9 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
         body_param["defaultContentType"] = _get_default_content_type(
             body_param["contentTypes"]
         )
+        # python supports IO input with all kinds of content_types
+        if body_type["type"] == "binary":
+            body_param["contentTypes"] = content_types or list(yaml_data.keys())
         if body_param["type"]["type"] == "constant":
             if not body_param["optional"] or (
                 body_param["optional"] and not self.default_optional_constants_to_none
@@ -771,11 +761,53 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
         param["inFlattenedBody"] = True
         return param
 
+    def _update_content_type_parameter(
+        self,
+        yaml_data: Dict[str, Any],
+        body_parameter: Optional[Dict[str, Any]],
+        request_media_types: List[str],
+        *,
+        in_overload: bool = False,
+        in_overriden: bool = False,
+    ) -> Dict[str, Any]:
+        # override content type type to string
+        if not body_parameter:
+            return yaml_data
+        param = copy.deepcopy(yaml_data)
+        param["schema"] = KNOWN_TYPES["string"]  # override to string type
+        if (
+            body_parameter["type"]["type"] == "binary"
+            and not body_parameter["defaultContentType"]
+            and not self.legacy
+        ):
+            param["required"] = True
+        else:
+            param["required"] = False
+        description = param["language"]["default"]["description"]
+        if description and description[-1] != ".":
+            description += "."
+        if not (in_overriden or in_overload):
+            param["inDocstring"] = False
+        elif in_overload:
+            description += (
+                " Content type parameter for "
+                f"{get_body_type_for_description(body_parameter)} body."
+            )
+        if not in_overload or (
+            body_parameter["type"]["type"] == "binary" and len(request_media_types) > 1
+        ):
+            content_types = "'" + "', '".join(request_media_types) + "'"
+            description += f" Known values are: {content_types}."
+        if not in_overload and not in_overriden:
+            param["clientDefaultValue"] = body_parameter["defaultContentType"]
+        param["language"]["default"]["description"] = description
+        return param
+
     def _update_parameters_helper(
         self,
         parameters: List[Dict[str, Any]],
         body_parameter: Optional[Dict[str, Any]],
-        seen_rest_api_names: Set[str],
+        seen_client_names: Set[str],
         groupers: Dict[str, Dict[str, Any]],
         request_media_types: List[str],
         *,
@@ -785,15 +817,17 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
         retval: List[Dict[str, Any]] = []
         has_flattened_body = body_parameter and body_parameter.get("flattened")
         for param in parameters:
-            serialized_name = param["language"]["default"].get("serializedName")
+            client_name = param["language"]["default"]["name"]
             if param["language"]["default"]["name"] == "$host" or (
-                serialized_name and serialized_name in seen_rest_api_names
+                client_name in seen_client_names
             ):
                 continue
+            seen_client_names.add(client_name)
             if param.get("origin") == "modelerfour:synthesized/api-version":
                 param["inDocstring"] = False
-                param["implementation"] = "Method"
-                param["checkClientInput"] = True
+                if self.legacy:
+                    param["implementation"] = "Method"
+                    param["checkClientInput"] = True
             if has_flattened_body and param.get("targetProperty"):
                 retval.append(self.update_flattened_parameter(param, body_parameter))
                 continue
@@ -806,8 +840,8 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
                 continue
             if is_body(param):
                 continue
-            if serialized_name == "Content-Type":
-                param = update_content_type_parameter(
+            if param["language"]["default"].get("serializedName") == "Content-Type":
+                param = self._update_content_type_parameter(
                     param,
                     body_parameter,
                     request_media_types,
@@ -818,7 +852,6 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
                 param, in_overload=in_overload, in_overriden=in_overriden
             )
             retval.append(updated_param)
-            seen_rest_api_names.add(updated_param["restApiName"])
         return retval
 
     def update_parameters(
@@ -830,7 +863,7 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
         in_overriden: bool = False,
     ) -> List[Dict[str, Any]]:
         retval: List[Dict[str, Any]] = []
-        seen_rest_api_names: Set[str] = set()
+        seen_client_names: Set[str] = set()
         groupers: Dict[str, Dict[str, Any]] = {}
         # first update top level parameters
         request_media_types = yaml_data.get("requestMediaTypes", [])
@@ -838,7 +871,7 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
             self._update_parameters_helper(
                 yaml_data["parameters"],
                 body_parameter,
-                seen_rest_api_names,
+                seen_client_names,
                 groupers,
                 request_media_types,
                 in_overload=in_overload,
@@ -857,7 +890,7 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
                 self._update_parameters_helper(
                     request.get("parameters", []),
                     body_parameter,
-                    seen_rest_api_names,
+                    seen_client_names,
                     groupers,
                     request_media_types,
                     in_overload=in_overload,
@@ -921,15 +954,8 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
             name = global_parameter["language"]["default"]["name"]
             if name == "$host":
                 # I am the non-parameterized endpoint. Modify name based off of flag
-                version_tolerant = self._autorestapi.get_boolean_value(
-                    "version-tolerant", False
-                )
-                low_level_client = self._autorestapi.get_boolean_value(
-                    "low-level-client", False
-                )
-                client_name = (
-                    "endpoint" if (version_tolerant or low_level_client) else "base_url"
-                )
+
+                client_name = "base_url" if self.legacy else "endpoint"
                 global_parameter["language"]["default"]["description"] = "Service URL."
             global_params.append(
                 self.update_parameter(
@@ -1054,9 +1080,7 @@ class M4Reformatter(YamlUpdatePlugin):  # pylint: disable=too-many-public-method
             "skipUrlEncoding": True,
             "inOverload": False,
         }
-        if self._autorestapi.get_boolean_value(
-            "version-tolerant"
-        ) or self._autorestapi.get_boolean_value("low-level-client"):
+        if self.version_tolerant or self.low_level_client:
             parameters.append(credential)
         else:
             parameters.insert(0, credential)

package/autorest/multiapi/models/client.py

@@ -5,9 +5,10 @@
 # --------------------------------------------------------------------------
 import sys
 import json
+import re
 from typing import Any, Dict, List
 from pathlib import Path
-from .imports import FileImport
+from .imports import FileImport, TypingSection, ImportType
 
 
 def _extract_version(metadata_json: Dict[str, Any], version_path: Path) -> str:
@@ -43,9 +44,18 @@ class Client:
 
     def imports(self, async_mode: bool) -> FileImport:
         imports_to_load = "async_imports" if async_mode else "sync_imports"
-        return FileImport(
+        file_import = FileImport(
             json.loads(self.default_version_metadata["client"][imports_to_load])
         )
+        local_imports = file_import.imports.get(TypingSection.REGULAR, {}).get(
+            ImportType.LOCAL, {}
+        )
+        for key in local_imports:
+            if re.search("^\\.*_serialization$", key):
+                relative_path = ".." if async_mode else "."
+                local_imports[f"{relative_path}_serialization"] = local_imports.pop(key)
+                break
+        return file_import
 
     @property
     def parameterized_host_template_to_api_version(self) -> Dict[str, List[str]]:

package/autorest/multiapi/serializers/__init__.py

@@ -57,18 +57,14 @@ class MultiAPISerializer(object):
         )
 
         # serialize service client file
-        imports = FileImportSerializer(
-            code_model.client.imports(async_mode), is_python3_file=async_mode
-        )
+        imports = FileImportSerializer(code_model.client.imports(async_mode))
         self._autorestapi.write_file(
             _get_file_path(code_model.client.filename, async_mode),
             _render_template("client", imports=imports),
         )
 
         # serialize config file
-        imports = FileImportSerializer(
-            code_model.config.imports(async_mode), is_python3_file=async_mode
-        )
+        imports = FileImportSerializer(code_model.config.imports(async_mode))
         self._autorestapi.write_file(
             _get_file_path("_configuration", async_mode),
             _render_template("config", imports=imports),
@@ -77,8 +73,7 @@ class MultiAPISerializer(object):
         # serialize mixins
         if code_model.operation_mixin_group.mixin_operations:
             imports = FileImportSerializer(
-                code_model.operation_mixin_group.imports(async_mode),
-                is_python3_file=async_mode,
+                code_model.operation_mixin_group.imports(async_mode)
             )
             self._autorestapi.write_file(
                 _get_file_path("_operations_mixin", async_mode),
@@ -115,3 +110,17 @@ class MultiAPISerializer(object):
             )
 
         self._autorestapi.write_file(Path("py.typed"), "# Marker file for PEP 561.")
+
+        if not code_model.client.client_side_validation:
+            codegen_env = Environment(
+                loader=PackageLoader("autorest.codegen", "templates"),
+                keep_trailing_newline=True,
+                line_statement_prefix="##",
+                line_comment_prefix="###",
+                trim_blocks=True,
+                lstrip_blocks=True,
+            )
+            self._autorestapi.write_file(
+                Path("_serialization.py"),
+                codegen_env.get_template("serialization.py.jinja2").render(),
+            )

package/autorest/multiapi/serializers/import_serializer.py

@@ -54,9 +54,8 @@ def _get_import_clauses(
 
 
 class FileImportSerializer:
-    def __init__(self, file_import: FileImport, is_python3_file: bool) -> None:
+    def __init__(self, file_import: FileImport) -> None:
         self._file_import = file_import
-        self.is_python3_file = is_python3_file
 
     def _switch_typing_section_key(self, new_key: TypingSection):
         switched_dictionary = {}
@@ -84,10 +83,7 @@ class FileImportSerializer:
         return file_import_copy.imports.get(baseline_typing_section, {})
 
     def _add_type_checking_import(self):
-        if self._file_import.imports.get(TypingSection.TYPING) or (
-            not self.is_python3_file
-            and self._file_import.imports.get(TypingSection.CONDITIONAL)
-        ):
+        if self._file_import.imports.get(TypingSection.TYPING):
             self._file_import.add_submodule_import(
                 "typing", "TYPE_CHECKING", ImportType.STDLIB
             )
@@ -97,7 +93,7 @@ class FileImportSerializer:
         regular_imports = ""
         regular_imports_dict = self._get_imports_dict(
             baseline_typing_section=TypingSection.REGULAR,
-            add_conditional_typing=self.is_python3_file,
+            add_conditional_typing=True,
         )
 
         if regular_imports_dict:
@@ -108,7 +104,7 @@ class FileImportSerializer:
         typing_imports = ""
         typing_imports_dict = self._get_imports_dict(
             baseline_typing_section=TypingSection.TYPING,
-            add_conditional_typing=not self.is_python3_file,
+            add_conditional_typing=False,
         )
         if typing_imports_dict:
             typing_imports += "\n\nif TYPE_CHECKING:\n    # pylint: disable=unused-import,ungrouped-imports\n    "

package/autorest/multiapi/templates/multiapi_operations_mixin.py.jinja2

@@ -8,7 +8,7 @@
 # Changes may cause incorrect behavior and will be lost if the code is
 # regenerated.
 # --------------------------------------------------------------------------
-from msrest import Serializer, Deserializer
+from {{ ".." if async_mode else "." }}_serialization import Serializer, Deserializer
 {% if imports %}
 {{ imports }}
 {% endif %}

package/autorest/preprocess/__init__.py

@@ -62,6 +62,7 @@ def update_types(yaml_data: List[Dict[str, Any]]) -> None:
             add_redefined_builtin_info(property["clientName"], property)
         if type.get("name"):
             type["description"] = update_description(type["description"], type["name"])
+            type["snakeCaseName"] = to_snake_case(type["name"])
 
 
 def update_client(yaml_data: Dict[str, Any]) -> None:

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@autorest/python",
-    "version": "5.17.0",
+    "version": "6.0.0-rc.1",
     "description": "The Python extension for generators in AutoRest.",
     "scripts": {
         "prepare": "node run-python3.js prepare.py",
@@ -27,7 +27,7 @@
         "@autorest/system-requirements": "~1.0.0"
     },
     "devDependencies": {
-        "@microsoft.azure/autorest.testserver": "^3.3.26"
+        "@microsoft.azure/autorest.testserver": "^3.3.29"
     },
     "files": [
         "autorest/**/*.py",

package/autorest/codegen/serializers/model_generic_serializer.py

@@ -1,32 +0,0 @@
-# -------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for
-# license information.
-# --------------------------------------------------------------------------
-from typing import List
-from jinja2 import Environment
-from .model_base_serializer import ModelBaseSerializer
-from ..models import ModelType, CodeModel, Property
-
-
-class ModelGenericSerializer(ModelBaseSerializer):
-    def __init__(self, code_model: CodeModel, env: Environment) -> None:
-        super().__init__(code_model=code_model, env=env, is_python3_file=False)
-
-    def init_line(self, model: ModelType) -> List[str]:
-        return []
-
-    def properties_to_pass_to_super(self, model: ModelType) -> str:
-        return "**kwargs"
-
-    def required_property_no_default_init(self, prop: Property) -> str:
-        return f"self.{prop.client_name} = kwargs['{prop.client_name}']"
-
-    def optional_property_init(self, prop: Property) -> str:
-        return f"self.{prop.client_name} = kwargs.get('{prop.client_name}', {prop.client_default_value_declaration})"
-
-    def initialize_standard_arg(self, prop: Property) -> str:
-        return self.initialize_standard_property(prop)
-
-    def super_call_template(self, model: ModelType) -> str:
-        return "super(" + model.name + ", self).__init__({})"

package/autorest/codegen/serializers/model_python3_serializer.py

@@ -1,72 +0,0 @@
-# -------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for
-# license information.
-# --------------------------------------------------------------------------
-from typing import List
-from jinja2 import Environment
-from .model_base_serializer import ModelBaseSerializer
-from ..models import ModelType, CodeModel, Property
-from ..models.imports import FileImport
-
-
-class ModelPython3Serializer(ModelBaseSerializer):
-    def __init__(self, code_model: CodeModel, env: Environment) -> None:
-        super().__init__(code_model=code_model, env=env, is_python3_file=True)
-
-    def init_line(self, model: ModelType) -> List[str]:
-        init_properties_declaration = []
-        init_line_parameters = [
-            p
-            for p in model.properties
-            if not p.readonly and not p.is_discriminator and not p.constant
-        ]
-        init_line_parameters.sort(key=lambda x: x.optional)
-        if init_line_parameters:
-            init_properties_declaration.append("*,")
-        for param in init_line_parameters:
-            init_properties_declaration.append(self.initialize_standard_property(param))
-
-        return init_properties_declaration
-
-    def properties_to_pass_to_super(self, model: ModelType) -> str:
-        properties_to_pass_to_super = []
-        for parent in model.parents:
-            for prop in model.properties:
-                if (
-                    prop in parent.properties
-                    and not prop.is_discriminator
-                    and not prop.constant
-                    and not prop.readonly
-                ):
-                    properties_to_pass_to_super.append(
-                        f"{prop.client_name}={prop.client_name}"
-                    )
-        properties_to_pass_to_super.append("**kwargs")
-        return ", ".join(properties_to_pass_to_super)
-
-    def required_property_no_default_init(self, prop: Property) -> str:
-        return f"{prop.client_name}: {prop.type_annotation()},{prop.pylint_disable}"
-
-    def optional_property_init(self, prop: Property) -> str:
-        return (
-            f"{prop.client_name}: {prop.type_annotation()} = "
-            f"{prop.client_default_value_declaration},{prop.pylint_disable}"
-        )
-
-    def initialize_standard_arg(self, prop: Property) -> str:
-        return f"self.{prop.client_name} = {prop.client_name}"
-
-    def super_call_template(self, model: ModelType) -> str:
-        return "super().__init__({})"
-
-    def imports(self) -> FileImport:
-        file_import = super(ModelPython3Serializer, self).imports()
-        for model in self.code_model.model_types:
-            init_line_parameters = [
-                p for p in model.properties if not p.readonly and not p.is_discriminator
-            ]
-            for param in init_line_parameters:
-                file_import.merge(param.imports())
-
-        return file_import