@autorest/python 5.17.0 → 6.0.0-rc.1

package/ChangeLog.md

@@ -1,4 +1,66 @@
-# Change Log
+# Release History
+
+### 2022-06-17 - 6.0.0-rc.1
+
+| Library                                                                 | Min Version |
+| ----------------------------------------------------------------------- | ----------- |
+| `@autorest/core`                                                        | `3.8.1`     |
+| `@autorest/modelerfour`                                                 | `4.23.5`    |
+| `azure-core` dep of generated code                                      | `1.24.0`    |
+| `msrest` dep of generated code (If generating legacy code               | `0.7.1`     |
+| `azure-mgmt-core` dep of generated code (If generating mgmt plane code) | `1.3.0`     |
+
+**Breaking Changes**
+
+- Default to generating DPG SDKs with `--version-tolerant` now defaulting to `true`. For a list of flag default changes, please
+see [here](https://github.com/Azure/autorest.python/issues/1186)  #1304
+- Only generate Python3 SDKs  #1297
+- Don't reformat initial query parameters into the next link  #1297
+- Don't generate operations with more than two body types. SDK authors need to implement this operation themselves  #1300
+
+**New Features**
+
+- Automatically format generated code with `black`. To not format, pass in `--black=false`  #1304
+
+**Other**
+
+- Remove testing support for `--low-level-client` SDKs  #1303
+
+### 2022-06-13 - 5.19.0
+
+| Library                                                                 | Min Version |
+| ----------------------------------------------------------------------- | ----------- |
+| `@autorest/core`                                                        | `3.8.1`     |
+| `@autorest/modelerfour`                                                 | `4.23.5`    |
+| `azure-core` dep of generated code                                      | `1.24.0`    |
+| `msrest` dep of generated code                                          | `0.7.0`    |
+| `azure-mgmt-core` dep of generated code (If generating mgmt plane code) | `1.3.0`     |
+
+**New Features**
+
+- Add _serialization.py for `--client-side-validation=false` generation, and migrate serilization from msrest to _serialization.py #1236
+
+### 2022-06-09 - 5.18.0
+
+| Library                                                                 | Min Version |
+| ----------------------------------------------------------------------- | ----------- |
+| `@autorest/core`                                                        | `3.8.1`     |
+| `@autorest/modelerfour`                                                 | `4.23.5`    |
+| `azure-core` dep of generated code                                      | `1.24.0`    |
+| `msrest` dep of generated code                                          | `0.7.0`    |
+| `azure-mgmt-core` dep of generated code (If generating mgmt plane code) | `1.3.0`     |
+
+**Breaking Changes in Version Tolerant**
+
+- No longer allow users to specify `api_version` on the method level  #1281
+- Make `content_type` param required with no default if streaming with no `application/octet-stream`  #1288
+
+**Bug Fixes**
+
+- Fix duplicate params in signature with `--payload-flattening-threshold`  #1289
+- Fix overloaded request builder signatures  #1289
+- Improve docstring templates, specifically for polymorphic bodies  #1279
+
 
 ### 2022-06-02 - 5.17.0
 

package/README.md

@@ -23,6 +23,15 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio
 
 #### Python code gen
 
+```yaml !$(multiapiscript)
+# default values for version tolerant and black
+black: true
+```
+
+```yaml !$(low-level-client)
+version-tolerant: true
+```
+
 ```yaml !$(low-level-client) && !$(version-tolerant)
 modelerfour:
   group-parameters: true

package/autorest/codegen/__init__.py

@@ -57,13 +57,6 @@ def _validate_code_model_options(options: Dict[str, Any]) -> None:
             "to 'public' or 'hidden'."
         )
 
-    if not options["show_operations"] and options["add_python3_operation_files"]:
-        raise ValueError(
-            "Can not add typed sync operation files if you are not showing operations. "
-            "If you want typed synced operation files, you have to add flag "
-            "--show-operations"
-        )
-
     if options["basic_setup_py"] and not options["package_version"]:
         raise ValueError("--basic-setup-py must be used with --package-version")
 
@@ -85,18 +78,17 @@ def _validate_code_model_options(options: Dict[str, Any]) -> None:
                 "--package-mode can only be 'mgmtplane' or 'dataplane' or directory which contains template files"
             )
 
-    if options["reformat_next_link"] and options["version_tolerant"]:
-        raise ValueError(
-            "--reformat-next-link can not be true for version tolerant generations. "
-            "Please remove --reformat-next-link from your call for version tolerant generations."
-        )
-
     if options["multiapi"] and options["version_tolerant"]:
         raise ValueError(
             "Can not currently generate version tolerant multiapi SDKs. "
             "We are working on creating a new multiapi SDK for version tolerant and it is not available yet."
         )
 
+    if options["client_side_validation"] and options["version_tolerant"]:
+        raise ValueError(
+            "Can not generate version tolerant with --client-side-validation. "
+        )
+
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -130,8 +122,8 @@ class CodeGenerator(Plugin):
     def _build_package_dependency() -> Dict[str, str]:
         return {
             "dependency_azure_mgmt_core": "azure-mgmt-core<2.0.0,>=1.3.0",
-            "dependency_azure_core": "azure-core<2.0.0,>=1.23.0",
-            "dependency_msrest": "msrest>=0.6.21",
+            "dependency_azure_core": "azure-core<2.0.0,>=1.24.0",
+            "dependency_msrest": "msrest>=0.7.1",
         }
 
     def _create_code_model(
@@ -184,7 +176,7 @@ class CodeGenerator(Plugin):
             bool, self._autorestapi.get_boolean_value("low-level-client", False)
         )
         version_tolerant = cast(
-            bool, self._autorestapi.get_boolean_value("version-tolerant", False)
+            bool, self._autorestapi.get_boolean_value("version-tolerant")
         )
         show_operations = self._autorestapi.get_boolean_value(
             "show-operations", not low_level_client
@@ -192,12 +184,22 @@ class CodeGenerator(Plugin):
         models_mode_default = (
             "none" if low_level_client or version_tolerant else "msrest"
         )
-        python3_only = cast(
-            bool,
-            self._autorestapi.get_boolean_value(
-                "python3-only", low_level_client or version_tolerant
-            ),
-        )
+        if self._autorestapi.get_boolean_value("python3-only") is False:
+            _LOGGER.warning(
+                "You have passed in --python3-only=False. We have force overriden "
+                "this to True."
+            )
+        if self._autorestapi.get_boolean_value("add-python3-operation-files"):
+            _LOGGER.warning(
+                "You have passed in --add-python3-operation-files. "
+                "This flag no longer has an effect bc all SDKs are now Python3 only."
+            )
+        if self._autorestapi.get_boolean_value("reformat-next-link"):
+            _LOGGER.warning(
+                "You have passed in --reformat-next-link. We have force overriden "
+                "this to False because we no longer reformat initial query parameters into next "
+                "calls unless explicitly defined in the service definition."
+            )
 
         options: Dict[str, Any] = {
             "azure_arm": azure_arm,
@@ -236,16 +238,11 @@ class CodeGenerator(Plugin):
                 "only-path-and-body-params-positional",
                 low_level_client or version_tolerant,
             ),
-            "add_python3_operation_files": self._autorestapi.get_boolean_value(
-                "add-python3-operation-files",
-                python3_only and not (low_level_client or version_tolerant),
-            ),
             "version_tolerant": version_tolerant,
             "low_level_client": low_level_client,
             "combine_operation_files": self._autorestapi.get_boolean_value(
                 "combine-operation-files", version_tolerant
             ),
-            "python3_only": python3_only,
             "package_mode": self._autorestapi.get_value("package-mode"),
             "package_pprint_name": self._autorestapi.get_value("package-pprint-name"),
             "package_configuration": self._autorestapi.get_value(
@@ -255,9 +252,6 @@ class CodeGenerator(Plugin):
                 "default-optional-constants-to-none",
                 low_level_client or version_tolerant,
             ),
-            "reformat_next_link": self._autorestapi.get_boolean_value(
-                "reformat-next-link", not version_tolerant
-            ),
         }
 
         if options["builders_visibility"] is None:

package/autorest/codegen/models/base_builder.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
+import logging
 from abc import abstractmethod
 from typing import Any, Dict, List, Optional, TypeVar, Union, TYPE_CHECKING, Generic
 from .base_model import BaseModel
@@ -27,6 +28,8 @@ if TYPE_CHECKING:
     from .operation import Operation
     from .request_builder import RequestBuilder
 
+_LOGGER = logging.getLogger(__name__)
+
 
 class BaseBuilder(Generic[ParameterListType], BaseModel):
     """Base class for Operations and Request Builders"""
@@ -39,7 +42,6 @@ class BaseBuilder(Generic[ParameterListType], BaseModel):
         parameters: ParameterListType,
         *,
         overloads=None,
-        abstract: bool = False,
         want_tracing: bool = True,
     ) -> None:
         super().__init__(yaml_data=yaml_data, code_model=code_model)
@@ -50,14 +52,23 @@ class BaseBuilder(Generic[ParameterListType], BaseModel):
             overloads or []
         )
         self._summary: str = yaml_data.get("summary", "")
-        # for operations where we don't know what to do, we mark them as abstract so users implement
-        # in patch.py
-        self.abstract = abstract
         self.want_tracing = want_tracing
         self.group_name: str = yaml_data["groupName"]
         self.is_overload: bool = yaml_data["isOverload"]
         self.api_versions: List[str] = yaml_data["apiVersions"]
 
+        if code_model.options["version_tolerant"] and yaml_data.get("abstract"):
+            _LOGGER.warning(
+                'Not going to generate operation "%s" because we are unable to generate this '
+                "type of operation right now. "
+                'Please write your own custom operation in the "_patch.py" file '
+                "following https://aka.ms/azsdk/python/dpcodegen/python/customize",
+                name,
+            )
+            self.abstract = True
+        else:
+            self.abstract = False
+
     @property
     def summary(self) -> Optional[str]:
         if self.abstract:
@@ -89,7 +100,7 @@ class BaseBuilder(Generic[ParameterListType], BaseModel):
             )
         return self._description or self.name
 
-    def method_signature(self, is_python3_file: bool, async_mode: bool) -> List[str]:
+    def method_signature(self, async_mode: bool) -> List[str]:
         if self.abstract:
             return ["*args,", "**kwargs"]
-        return self.parameters.method_signature(is_python3_file, async_mode)
+        return self.parameters.method_signature(async_mode)

package/autorest/codegen/models/base_type.py

@@ -11,6 +11,7 @@ from .imports import FileImport
 
 if TYPE_CHECKING:
     from .code_model import CodeModel
+    from .model_type import ModelType
 
 
 class BaseType(BaseModel, ABC):
@@ -143,6 +144,11 @@ class BaseType(BaseModel, ABC):
         """Template of what this schema would look like as JSON input"""
         ...
 
+    def get_polymorphic_subtypes(  # pylint: disable=no-self-use
+        self, polymorphic_subtypes: List["ModelType"]  # pylint: disable=unused-argument
+    ) -> None:
+        return None
+
     @property
     @abstractmethod
     def instance_check_template(self) -> str:

package/autorest/codegen/models/client.py

@@ -7,7 +7,7 @@ from typing import Any, Dict, TYPE_CHECKING, TypeVar, Generic, Union
 
 from .base_model import BaseModel
 from .parameter_list import ClientGlobalParameterList, ConfigGlobalParameterList
-from .imports import FileImport, ImportType, TypingSection
+from .imports import FileImport, ImportType, TypingSection, MsrestImportType
 from .utils import add_to_pylint_disable
 
 ParameterListType = TypeVar(
@@ -89,10 +89,6 @@ class Client(_ClientConfigBase[ClientGlobalParameterList]):
     def _imports_shared(self, async_mode: bool) -> FileImport:
         file_import = FileImport()
 
-        file_import.add_submodule_import("msrest", "Serializer", ImportType.THIRDPARTY)
-        file_import.add_submodule_import(
-            "msrest", "Deserializer", ImportType.THIRDPARTY
-        )
         file_import.add_submodule_import(
             "typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL
         )
@@ -113,6 +109,13 @@ class Client(_ClientConfigBase[ClientGlobalParameterList]):
             ImportType.LOCAL,
         )
 
+        file_import.add_msrest_import(
+            self.code_model,
+            ".." if async_mode else ".",
+            MsrestImportType.SerializerDeserializer,
+            TypingSection.REGULAR,
+        )
+
         return file_import
 
     def imports(self, async_mode: bool) -> FileImport:
@@ -156,7 +159,7 @@ class Client(_ClientConfigBase[ClientGlobalParameterList]):
                 # Also in this case, we're in version tolerant, so python3 only is true
                 file_import.add_submodule_import(
                     f"{path_to_models}models",
-                    f"{self.code_model.get_models_filename(is_python3_file=True)}",
+                    self.code_model.models_filename,
                     ImportType.LOCAL,
                     alias="models",
                 )

package/autorest/codegen/models/code_model.py

@@ -200,8 +200,15 @@ class CodeModel:  # pylint: disable=too-many-instance-attributes, too-many-publi
             name = f"_{name}"
         return name
 
+    @property
+    def has_abstract_operations(self) -> bool:
+        """Whether there is abstract operation in any operation group."""
+        return any(og.has_abstract_operations for og in self.operation_groups)
+
     def need_vendored_code(self, async_mode: bool) -> bool:
         """Whether we need to vendor code in the _vendor.py file for this SDK"""
+        if self.has_abstract_operations:
+            return True
         if async_mode:
             return self.need_mixin_abc
         return (
@@ -228,11 +235,7 @@ class CodeModel:  # pylint: disable=too-many-instance-attributes, too-many-publi
     @property
     def need_mixin_abc(self) -> bool:
         """Do we want a mixin ABC class for typing purposes?"""
-        return any(
-            o
-            for o in self.operation_groups
-            if o.is_mixin and self.options["python3_only"]
-        )
+        return any(o for o in self.operation_groups if o.is_mixin)
 
     @property
     def has_lro_operations(self) -> bool:
@@ -245,22 +248,25 @@ class CodeModel:  # pylint: disable=too-many-instance-attributes, too-many-publi
             ]
         )
 
-    def get_models_filename(self, is_python3_file: bool) -> str:
+    @property
+    def models_filename(self) -> str:
         """Get the names of the model file(s)"""
-        if (
-            self.options["version_tolerant"] or self.options["low_level_client"]
-        ) and self.options["python3_only"]:
-            return "_models"
-        if is_python3_file:
+        if self.is_legacy:
             return "_models_py3"
         return "_models"
 
     @property
     def enums_filename(self) -> str:
         """The name of the enums file"""
-        if self.options["version_tolerant"] or self.options["low_level_client"]:
-            return "_enums"
-        return f"_{self.module_name}_enums"
+        if self.is_legacy:
+            return f"_{self.module_name}_enums"
+        return "_enums"
+
+    @property
+    def is_legacy(self) -> bool:
+        return not (
+            self.options["version_tolerant"] or self.options["low_level_client"]
+        )
 
     @property
     def rest_layer_name(self) -> str:

package/autorest/codegen/models/dictionary_type.py

@@ -3,12 +3,13 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-from typing import Any, Dict, Optional, TYPE_CHECKING
+from typing import Any, Dict, Optional, TYPE_CHECKING, List
 from .base_type import BaseType
 from .imports import FileImport, ImportType, TypingSection
 
 if TYPE_CHECKING:
     from .code_model import CodeModel
+    from .model_type import ModelType
 
 
 class DictionaryType(BaseType):
@@ -79,6 +80,20 @@ class DictionaryType(BaseType):
             )
         }
 
+    def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None:
+        from .model_type import ModelType
+
+        if isinstance(self.element_type, ModelType):
+            is_polymorphic_subtype = (
+                self.element_type.discriminator_value
+                and not self.element_type.discriminated_subtypes
+            )
+            if (
+                self.element_type.name not in (m.name for m in polymorphic_subtypes)
+                and is_polymorphic_subtype
+            ):
+                polymorphic_subtypes.append(self.element_type)
+
     @classmethod
     def from_yaml(
         cls, yaml_data: Dict[str, Any], code_model: "CodeModel"

package/autorest/codegen/models/imports.py

@@ -3,8 +3,11 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-from enum import Enum
-from typing import Dict, List, Optional, Tuple, Union, Set, Mapping
+from enum import Enum, auto
+from typing import Dict, List, Optional, Tuple, Union, Set, Mapping, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .code_model import CodeModel
 
 
 class ImportType(str, Enum):
@@ -20,6 +23,16 @@ class TypingSection(str, Enum):
     TYPING = "typing"  # never a typing import
 
 
+class MsrestImportType(Enum):
+    Module = auto()  # import _serialization.py or msrest.serialization as Module
+    Serializer = (
+        auto()
+    )  # from _serialization.py or msrest.serialization import Serializer
+    SerializerDeserializer = (
+        auto()
+    )  # from _serialization.py or msrest.serialization import Serializer and Deserializer
+
+
 class ImportModel:
     def __init__(
         self,
@@ -190,3 +203,45 @@ class FileImport:
                 i.import_type, dict()
             ).setdefault(i.module_name, set()).add(name_import)
         return retval
+
+    def add_msrest_import(
+        self,
+        code_model: "CodeModel",
+        relative_path: str,
+        msrest_import_type: MsrestImportType,
+        typing_section: TypingSection,
+    ):
+        if code_model.options["client_side_validation"]:
+            if msrest_import_type == MsrestImportType.Module:
+                self.add_import(
+                    "msrest.serialization", ImportType.AZURECORE, typing_section
+                )
+            else:
+                self.add_submodule_import(
+                    "msrest", "Serializer", ImportType.THIRDPARTY, typing_section
+                )
+                if msrest_import_type == MsrestImportType.SerializerDeserializer:
+                    self.add_submodule_import(
+                        "msrest", "Deserializer", ImportType.THIRDPARTY, typing_section
+                    )
+        else:
+            if code_model.options["multiapi"]:
+                relative_path += "."
+            if msrest_import_type == MsrestImportType.Module:
+                self.add_submodule_import(
+                    relative_path, "_serialization", ImportType.LOCAL, typing_section
+                )
+            else:
+                self.add_submodule_import(
+                    f"{relative_path}_serialization",
+                    "Serializer",
+                    ImportType.LOCAL,
+                    typing_section,
+                )
+                if msrest_import_type == MsrestImportType.SerializerDeserializer:
+                    self.add_submodule_import(
+                        f"{relative_path}_serialization",
+                        "Deserializer",
+                        ImportType.LOCAL,
+                        typing_section,
+                    )

package/autorest/codegen/models/list_type.py

@@ -3,12 +3,13 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-from typing import Any, Dict, Optional, Union, TYPE_CHECKING
+from typing import Any, Dict, Optional, Union, TYPE_CHECKING, List
 from .base_type import BaseType
 from .imports import FileImport, ImportType, TypingSection
 
 if TYPE_CHECKING:
     from .code_model import CodeModel
+    from .model_type import ModelType
 
 
 class ListType(BaseType):
@@ -104,6 +105,20 @@ class ListType(BaseType):
             )
         ]
 
+    def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None:
+        from .model_type import ModelType
+
+        if isinstance(self.element_type, ModelType):
+            is_polymorphic_subtype = (
+                self.element_type.discriminator_value
+                and not self.element_type.discriminated_subtypes
+            )
+            if (
+                self.element_type.name not in (m.name for m in polymorphic_subtypes)
+                and is_polymorphic_subtype
+            ):
+                polymorphic_subtypes.append(self.element_type)
+
     @property
     def instance_check_template(self) -> str:
         return "isinstance({}, list)"

package/autorest/codegen/models/lro_operation.py

@@ -33,7 +33,6 @@ class LROOperationBase(OperationBase[LROResponseType]):
         overloads: Optional[List[Operation]] = None,
         public: bool = True,
         want_tracing: bool = True,
-        abstract: bool = False,
     ) -> None:
         super().__init__(
             code_model=code_model,
@@ -46,7 +45,6 @@ class LROOperationBase(OperationBase[LROResponseType]):
             overloads=overloads,
             public=public,
             want_tracing=want_tracing,
-            abstract=abstract,
         )
         self.name = "begin_" + self.name
         self.lro_options: Dict[str, Any] = self.yaml_data.get("lroOptions", {})
@@ -120,10 +118,10 @@ class LROOperationBase(OperationBase[LROResponseType]):
     def get_no_polling_method(self, async_mode: bool) -> str:
         return self.responses[0].get_no_polling_method(async_mode)
 
-    def imports(
-        self, async_mode: bool, is_python3_file: bool, **kwargs: Any
-    ) -> FileImport:
-        file_import = super().imports(async_mode, is_python3_file, **kwargs)
+    def imports(self, async_mode: bool, **kwargs: Any) -> FileImport:
+        file_import = super().imports(async_mode, **kwargs)
+        if self.abstract:
+            return file_import
         if async_mode:
             file_import.add_submodule_import(
                 f"azure.core.tracing.decorator_async",

package/autorest/codegen/models/lro_paging_operation.py

@@ -25,15 +25,9 @@ class LROPagingOperation(
     def cls_type_annotation(self, *, async_mode: bool) -> str:
         return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode)}]"  # pylint: disable=no-member
 
-    def imports(
-        self, async_mode: bool, is_python3_file: bool, **kwargs: Any
-    ) -> FileImport:
-        lro_imports = LROOperationBase.imports(
-            self, async_mode, is_python3_file, **kwargs
-        )
-        paging_imports = PagingOperationBase.imports(
-            self, async_mode, is_python3_file, **kwargs
-        )
+    def imports(self, async_mode: bool, **kwargs: Any) -> FileImport:
+        lro_imports = LROOperationBase.imports(self, async_mode, **kwargs)
+        paging_imports = PagingOperationBase.imports(self, async_mode, **kwargs)
 
         file_import = lro_imports
         file_import.merge(paging_imports)