@autorest/python 5.14.0 → 5.15.0

package/ChangeLog.md

@@ -1,5 +1,34 @@
 # Change Log
 
+### 2022-04-07 - 5.15.0
+
+| Library                                                                 | Min Version |
+| ----------------------------------------------------------------------- | ----------- |
+| `@autorest/core`                                                        | `3.6.2`     |
+| `@autorest/modelerfour`                                                 | `4.19.1`    |
+| `azure-core` dep of generated code                                      | `1.23.0`    |
+| `msrest` dep of generated code                                          | `0.6.21`    |
+| `azure-mgmt-core` dep of generated code (If generating mgmt plane code) | `1.3.0`     |
+
+**New Features**
+
+- Add support for security configurations in the swagger. For more information, see https://github.com/Azure/autorest/blob/main/docs/generate/authentication.md #1161
+- Add support for handwritten customizations of generated code. For more information, see https://aka.ms/azsdk/python/dpcodegen/python/customize #1153
+- Allow `header` and `params` as kwargs in operation and request-build function to hand over REST Header and Query parameters case insensitively #1183
+
+**Bug Fixes**
+
+- Make `--version-tolerant` generated code mypy compatible in the `azure-sdk-for-python` repo. Tested only with the `--black` flag #1185
+- Remove unnecessary vendored code in the `_vendor` file if the SDK has no operations #1196
+- Fix the generation of the root `__init__` files for packages with only models #1195
+- Add pylint and mypy support for `--version-tolerant` generations with `--models-mode=msrest` #1202
+
+**Breaking Changes in Version Tolerant Generation**
+
+- Change client filenames to `_client.py`  #1206
+- Change the models filename from `_models_py3.py` to `_models.py` #1204
+- Change the enums filename to `_enums.py` #1204
+
 ### 2022-03-08 - 5.14.0
 
 | Library                                                                 | Min Version |

package/autorest/codegen/__init__.py

@@ -18,7 +18,11 @@ from .models.parameter_list import GlobalParameterList
 from .models.rest import Rest
 from .serializers import JinjaSerializer
 from .models.credential_schema_policy import CredentialSchemaPolicy, get_credential_schema_policy_type
-from .models.credential_schema import AzureKeyCredentialSchema, TokenCredentialSchema
+from .models.credential_schema_policy import BearerTokenCredentialPolicy, AzureKeyCredentialPolicy
+from .models.credential_model import CredentialModel
+
+_AAD_TYPE = "AADToken"
+_KEY_TYPE = "AzureKey"
 
 def _build_convenience_layer(yaml_data: Dict[str, Any], code_model: CodeModel) -> None:
     # Create operations
@@ -32,7 +36,6 @@ def _build_convenience_layer(yaml_data: Dict[str, Any], code_model: CodeModel) -
             code_model.sort_schemas()
 
     if code_model.options["show_operations"]:
-        code_model.add_schema_link_to_operation()
         code_model.generate_single_parameter_from_multiple_content_types_operation()
         code_model.link_operation_to_request_builder()
         # LRO operation
@@ -92,6 +95,12 @@ def _validate_code_model_options(options: Dict[str, Any]) -> None:
             "Please remove --reformat-next-link from your call for version tolerant generations."
         )
 
+    if options["multiapi"] and options["version_tolerant"]:
+        raise ValueError(
+            "Can not currently generate version tolerant multiapi SDKs. "
+            "We are working on creating a new multiapi SDK for version tolerant and it is not available yet."
+        )
+
 _LOGGER = logging.getLogger(__name__)
 class CodeGenerator(Plugin):
     @staticmethod
@@ -131,32 +140,53 @@ class CodeGenerator(Plugin):
     def _build_package_dependency() -> Dict[str, str]:
         return {
             "dependency_azure_mgmt_core": "azure-mgmt-core<2.0.0,>=1.3.0",
-            "dependency_azure_core": "azure-core<2.0.0,>=1.20.1",
+            "dependency_azure_core": "azure-core<2.0.0,>=1.23.0",
             "dependency_msrest": "msrest>=0.6.21",
         }
 
+    @staticmethod
+    def _build_with_security_definition(yaml_data: Dict[str, Any], credential_model: CredentialModel):
+        security_yaml = yaml_data.get("security", {})
+        if security_yaml.get("authenticationRequired"):
+            for scheme in security_yaml.get("schemes"):
+                if _AAD_TYPE == scheme["type"]:
+                    credential_model.credential_scopes.update(scheme["scopes"])
+                elif _KEY_TYPE == scheme["type"]:
+                    # only accept the last one
+                    credential_model.key_header_name = scheme["headerName"]
+
+        if credential_model.credential_scopes:
+            credential_model.policy_type = BearerTokenCredentialPolicy
+        elif credential_model.key_header_name:
+            credential_model.policy_type = AzureKeyCredentialPolicy
+
+    @staticmethod
+    def _build_credential_model(code_model: CodeModel, credential_model: CredentialModel):
+        if credential_model.policy_type:
+            code_model.options["credential"] = True
+            credential_model.build_authentication_policy()
+            code_model.credential_model = credential_model
+
+    def _handle_credential_model(self, yaml_data: Dict[str, Any], code_model: CodeModel):
+        credential_model = CredentialModel(code_model.options["azure_arm"])
+
+        # credential info with security definition will be overridded by credential flags
+        self._build_with_security_definition(yaml_data, credential_model)
+        self._build_with_credential_flags(code_model, credential_model)
+
+        self._build_credential_model(code_model, credential_model)
+
     def _create_code_model(self, yaml_data: Dict[str, Any], options: Dict[str, Union[str, bool]]) -> CodeModel:
         # Create a code model
 
         code_model = CodeModel(options=options)
-        if code_model.options['credential']:
-            self._handle_default_authentication_policy(code_model)
+        self._handle_credential_model(yaml_data, code_model)
         code_model.module_name = yaml_data["info"]["python_title"]
         code_model.class_name = yaml_data["info"]["pascal_case_title"]
         code_model.description = (
             yaml_data["info"]["description"] if yaml_data["info"].get("description") else ""
         )
 
-        # Global parameters
-        code_model.global_parameters = GlobalParameterList(
-            code_model,
-            [Parameter.from_yaml(param, code_model=code_model) for param in yaml_data.get("globalParameters", [])],
-        )
-        code_model.global_parameters.code_model = code_model
-
-        # Custom URL
-        code_model.setup_client_input_parameters(yaml_data)
-
         # Get my namespace
         namespace = self._autorestapi.get_value("namespace")
         _LOGGER.debug("Namespace parameter was %s", namespace)
@@ -164,16 +194,24 @@ class CodeGenerator(Plugin):
             namespace = yaml_data["info"]["python_title"]
         code_model.namespace = namespace
 
-        code_model.rest = Rest.from_yaml(yaml_data, code_model=code_model)
         if yaml_data.get("schemas"):
             exceptions_set = CodeGenerator._build_exceptions_set(yaml_data=yaml_data["operationGroups"])
 
             for type_list in yaml_data["schemas"].values():
                 for schema in type_list:
                     build_schema(yaml_data=schema, exceptions_set=exceptions_set, code_model=code_model)
-            code_model.add_schema_link_to_request_builder()
-            code_model.add_schema_link_to_global_parameters()
 
+        # Global parameters
+        code_model.global_parameters = GlobalParameterList(
+            code_model,
+            [Parameter.from_yaml(param, code_model=code_model) for param in yaml_data.get("globalParameters", [])],
+        )
+        code_model.global_parameters.code_model = code_model
+
+        # Custom URL
+        code_model.setup_client_input_parameters(yaml_data)
+
+        code_model.rest = Rest.from_yaml(yaml_data, code_model=code_model)
         _build_convenience_layer(yaml_data=yaml_data, code_model=code_model)
 
         if options["credential"]:
@@ -196,9 +234,13 @@ class CodeGenerator(Plugin):
             )
         return credential_scopes
 
-    def _initialize_credential_schema_policy(
-        self, code_model: CodeModel, credential_schema_policy: Type[CredentialSchemaPolicy]
-    ) -> CredentialSchemaPolicy:
+    def _update_with_credential_flags(
+        self,
+        code_model: CodeModel,
+        credential_schema_policy: Type[CredentialSchemaPolicy],
+        credential_model: CredentialModel
+    ):
+        credential_model.policy_type = credential_schema_policy
         credential_scopes = self._get_credential_scopes(code_model.options['credential'])
         credential_key_header_name = self._autorestapi.get_value('credential-key-header-name')
         azure_arm = code_model.options['azure_arm']
@@ -227,38 +269,36 @@ class CodeGenerator(Plugin):
                     "name is tied with AzureKeyCredentialPolicy. Instead, with this policy it is recommend you "
                     "pass in --credential-scopes."
                 )
-            return credential_schema_policy(
-                credential=TokenCredentialSchema(async_mode=False),
-                credential_scopes=credential_scopes,
-            )
-        # currently the only other credential policy is AzureKeyCredentialPolicy
-        if credential_scopes:
-            raise ValueError(
-                "You have passed in credential scopes with default credential policy type "
-                "AzureKeyCredentialPolicy. This is not allowed, since credential scopes is tied with "
-                f"{code_model.default_authentication_policy.name()}. Instead, with this policy you must pass in "
-                "--credential-key-header-name."
-            )
-        if not credential_key_header_name:
-            credential_key_header_name = "api-key"
-            _LOGGER.info(
-                "Defaulting the AzureKeyCredentialPolicy header's name to 'api-key'"
-            )
-        return credential_schema_policy(
-            credential=AzureKeyCredentialSchema(),
-            credential_key_header_name=credential_key_header_name,
-        )
+            credential_model.credential_scopes = set(credential_scopes)
+        else:
+            # currently the only other credential policy is AzureKeyCredentialPolicy
+            if credential_scopes:
+                raise ValueError(
+                    "You have passed in credential scopes with default credential policy type "
+                    "AzureKeyCredentialPolicy. This is not allowed, since credential scopes is tied with "
+                    f"{credential_model.default_authentication_policy.name()}. Instead, with this policy "
+                    "you must pass in --credential-key-header-name."
+                )
+            if not credential_key_header_name:
+                credential_key_header_name = "api-key"
+                _LOGGER.info(
+                    "Defaulting the AzureKeyCredentialPolicy header's name to 'api-key'"
+                )
+
+            credential_model.key_header_name = credential_key_header_name
+
+    def _build_with_credential_flags(self, code_model: CodeModel, credential_model: CredentialModel):
+        if not code_model.options["credential"]:
+            return
 
-    def _handle_default_authentication_policy(self, code_model: CodeModel):
         credential_schema_policy_name = (
             self._autorestapi.get_value("credential-default-policy-type") or
-            code_model.default_authentication_policy.name()
+            credential_model.default_authentication_policy.name()
         )
         credential_schema_policy_type = get_credential_schema_policy_type(credential_schema_policy_name)
-        credential_schema_policy = self._initialize_credential_schema_policy(
-            code_model, credential_schema_policy_type
+        self._update_with_credential_flags(
+            code_model, credential_schema_policy_type, credential_model
         )
-        code_model.credential_schema_policy = credential_schema_policy
 
     def _build_code_model_options(self) -> Dict[str, Any]:
         """Build en options dict from the user input while running autorest.

package/autorest/codegen/models/base_schema.py

@@ -90,18 +90,14 @@ class BaseSchema(BaseModel, ABC):
         """
         ...
 
-    @property
-    def type_annotation(self) -> str:
+    @abstractmethod
+    def type_annotation(self, *, is_operation_file: bool = False) -> str:
         """The python type used for type annotation
 
         Special case for enum, for instance: Union[str, "EnumName"]
         """
         ...
 
-    @property
-    def operation_type_annotation(self) -> str:
-        return self.type_annotation
-
     def get_declaration(self, value: Any) -> str:  # pylint: disable=no-self-use
         """Return the current value from YAML as a Python string that represents the constant.
 

package/autorest/codegen/models/client.py

@@ -102,3 +102,9 @@ class Client:
     def send_request_signature(self, is_python3_file: bool) -> List[str]:
         request_signature = ["request: HttpRequest," if is_python3_file else "request,  # type: HttpRequest"]
         return request_signature + self.parameters.method_signature_kwargs(is_python3_file)
+
+    @property
+    def filename(self) -> str:
+        if self.code_model.options["version_tolerant"] or self.code_model.options["low_level_client"]:
+            return "_client"
+        return f"_{self.code_model.module_name}"

package/autorest/codegen/models/code_model.py

@@ -3,14 +3,10 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-from itertools import chain
 import logging
-from typing import cast, List, Dict, Optional, Any, Set, Type
+from typing import cast, List, Dict, Optional, Any, Set
 
 from .base_schema import BaseSchema
-from .credential_schema_policy import (
-    ARMChallengeAuthenticationPolicy, BearerTokenCredentialPolicy, CredentialSchemaPolicy
-)
 from .enum_schema import EnumSchema
 from .object_schema import ObjectSchema
 from .operation_group import OperationGroup
@@ -20,12 +16,10 @@ from .paging_operation import PagingOperation
 from .parameter import Parameter
 from .client import Client
 from .parameter_list import GlobalParameterList
-from .schema_response import SchemaResponse
 from .property import Property
-from .primitive_schemas import IOSchema
 from .request_builder import RequestBuilder
 from .rest import Rest
-
+from .credential_model import CredentialModel
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -53,6 +47,8 @@ class CodeModel:  # pylint: disable=too-many-instance-attributes, too-many-publi
     :type operation_groups: list[~autorest.models.OperationGroup]
     :param package_dependency: All the dependencies needed in setup.py
     :type package_dependency: Dict[str, str]
+    :param credential_model: The class contains all the credential info
+    :type credential_model: CredentialMode
     """
 
     def __init__(
@@ -77,8 +73,8 @@ class CodeModel:  # pylint: disable=too-many-instance-attributes, too-many-publi
         self.service_client: Client = Client(self, params)
         self._rest: Optional[Rest] = None
         self.request_builder_ids: Dict[int, RequestBuilder] = {}
-        self._credential_schema_policy: Optional[CredentialSchemaPolicy] = None
         self.package_dependency: Dict[str, str] = {}
+        self._credential_model: Optional[CredentialModel] = None
 
     @property
     def global_parameters(self) -> GlobalParameterList:
@@ -211,18 +207,14 @@ class CodeModel:  # pylint: disable=too-many-instance-attributes, too-many-publi
         return self.schemas or self.enums
 
     @property
-    def default_authentication_policy(self) -> Type[CredentialSchemaPolicy]:
-        return ARMChallengeAuthenticationPolicy if self.options['azure_arm'] else BearerTokenCredentialPolicy
-
-    @property
-    def credential_schema_policy(self) -> CredentialSchemaPolicy:
-        if not self._credential_schema_policy:
-            raise ValueError("You want to find the Credential Schema Policy, but have not given a value")
-        return self._credential_schema_policy
+    def credential_model(self) -> CredentialModel:
+        if not self._credential_model:
+            raise ValueError("You want to find the Credential Model, but have not given a value")
+        return self._credential_model
 
-    @credential_schema_policy.setter
-    def credential_schema_policy(self, val: CredentialSchemaPolicy) -> None:
-        self._credential_schema_policy = val
+    @credential_model.setter
+    def credential_model(self, val: CredentialModel) -> None:
+        self._credential_model = val
 
     @staticmethod
     def _add_properties_from_inheritance_helper(schema, properties) -> List[Property]:
@@ -303,74 +295,33 @@ class CodeModel:  # pylint: disable=too-many-instance-attributes, too-many-publi
                     return
         raise KeyError("Didn't find the target property")
 
-    def _populate_schema(self, obj: Any) -> None:
-        schema_obj = obj.schema
-        if schema_obj and not isinstance(schema_obj, dict):
-            return
-
-        if schema_obj:
-            schema_obj_id = id(obj.schema)
-            _LOGGER.debug("Looking for id %s for member %s", schema_obj_id, obj)
-            try:
-                obj.schema = self.lookup_schema(schema_obj_id)
-            except KeyError:
-                _LOGGER.critical("Unable to ref the object")
-                raise
-        if isinstance(obj, Parameter) and obj.target_property_name:
-            self._populate_target_property(obj)
-        if isinstance(obj, SchemaResponse) and obj.is_stream_response:
-            obj.schema = IOSchema(namespace=None, yaml_data={})
-
-    def add_schema_link_to_operation(self) -> None:
-        """Puts created schemas into operation classes `schema` property
-
-        :return: None
-        :rtype: None
-        """
-        # Index schemas
-        for operation_group in self.operation_groups:
-            for operation in operation_group.operations:
-                for obj in chain(
-                    operation.parameters,
-                    operation.multiple_content_type_parameters or [],
-                    operation.responses,
-                    operation.exceptions,
-                    chain.from_iterable(response.headers for response in operation.responses),
-                ):
-                    self._populate_schema(obj)
-
-    def add_schema_link_to_request_builder(self) -> None:
-        for request_builder in self.rest.request_builders:
-            for obj in chain(
-                request_builder.parameters,
-                chain.from_iterable(request.parameters for request in request_builder.schema_requests),
-                request_builder.responses,
-            ):
-                self._populate_schema(obj)
-
-
-    def add_schema_link_to_global_parameters(self) -> None:
-        for parameter in self.global_parameters:
-            self._populate_schema(parameter)
-
     def generate_single_parameter_from_multiple_content_types_operation(self) -> None:
         for operation_group in self.operation_groups:
             for operation in operation_group.operations:
                 if operation.multiple_content_type_parameters:
                     operation.convert_multiple_content_type_parameters()
 
-    @property
-    def need_vendored_code(self) -> bool:
-        return self.need_request_converter or self.need_format_url
+    def need_vendored_code(self, async_mode: bool) -> bool:
+        if async_mode:
+            return self.need_mixin_abc
+        return self.need_request_converter or self.need_format_url or self.need_mixin_abc
 
     @property
     def need_request_converter(self) -> bool:
-        return self.options["show_operations"] and not self.options["version_tolerant"]
+        return (
+            self.options["show_operations"] and
+            bool(self.rest.request_builders) and
+            not self.options["version_tolerant"]
+        )
 
     @property
     def need_format_url(self) -> bool:
         return any(rq for rq in self.rest.request_builders if rq.parameters.path)
 
+    @property
+    def need_mixin_abc(self) -> bool:
+        return any(o for o in self.operation_groups if o.is_empty_operation_group and self.options["python3_only"])
+
     @property
     def has_lro_operations(self) -> bool:
         return any([
@@ -400,3 +351,18 @@ class CodeModel:  # pylint: disable=too-many-instance-attributes, too-many-publi
                     request_builder.name = request_builder.name + "_initial"
                 operation.request_builder = request_builder
                 operation.link_body_kwargs_to_body_params()
+
+    def get_models_filename(self, is_python3_file: bool) -> str:
+        if (
+            self.options["version_tolerant"] or self.options["low_level_client"]
+        ) and self.options["python3_only"]:
+            return "_models"
+        if is_python3_file:
+            return "_models_py3"
+        return "_models"
+
+    @property
+    def enums_filename(self) -> str:
+        if self.options["version_tolerant"] or self.options["low_level_client"]:
+            return "_enums"
+        return f"_{self.module_name}_enums"

package/autorest/codegen/models/constant_schema.py

@@ -60,9 +60,8 @@ class ConstantSchema(BaseSchema):
         """
         return self.schema.docstring_type
 
-    @property
-    def type_annotation(self) -> str:
-        return self.schema.type_annotation
+    def type_annotation(self, *, is_operation_file: bool = False) -> str:
+        return self.schema.type_annotation(is_operation_file=is_operation_file)
 
     @classmethod
     def from_yaml(cls, namespace: str, yaml_data: Dict[str, Any], **kwargs) -> "ConstantSchema":
@@ -95,3 +94,8 @@ class ConstantSchema(BaseSchema):
         file_import = FileImport()
         file_import.merge(self.schema.imports())
         return file_import
+
+    def model_file_imports(self) -> FileImport:
+        file_import = self.imports()
+        file_import.merge(self.schema.model_file_imports())
+        return file_import

package/autorest/codegen/models/credential_model.py

@@ -0,0 +1,47 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from typing import Set, Optional, Type
+from .credential_schema_policy import CredentialSchemaPolicy, BearerTokenCredentialPolicy
+from .credential_schema_policy import ARMChallengeAuthenticationPolicy
+from .credential_schema import TokenCredentialSchema, AzureKeyCredentialSchema
+
+
+class CredentialModel:
+    """Store info about credential.
+    """
+
+    def __init__(
+        self,
+        azure_arm: bool
+    ) -> None:
+        self.azure_arm: bool = azure_arm
+        self.credential_scopes: Set[str] = set()
+        self.key_header_name: str = ""
+        self.policy_type: Optional[Type[CredentialSchemaPolicy]] = None
+        self._credential_schema_policy: Optional[CredentialSchemaPolicy] = None
+
+    @property
+    def default_authentication_policy(self) -> Type[CredentialSchemaPolicy]:
+        return ARMChallengeAuthenticationPolicy if self.azure_arm else BearerTokenCredentialPolicy
+
+    @property
+    def credential_schema_policy(self) -> CredentialSchemaPolicy:
+        if not self._credential_schema_policy:
+            raise ValueError(
+                "You want to find the Credential Schema Policy, but have not given a value")
+        return self._credential_schema_policy
+
+    def build_authentication_policy(self):
+        if hasattr(self.policy_type, "credential_scopes"):
+            self._credential_schema_policy = self.policy_type( # pylint: disable=not-callable
+                credential=TokenCredentialSchema(async_mode=False),
+                credential_scopes=list(self.credential_scopes),
+            )
+        elif hasattr(self.policy_type, "credential_key_header_name"):
+            self._credential_schema_policy = self.policy_type( # pylint: disable=not-callable
+                credential=AzureKeyCredentialSchema(),
+                credential_key_header_name=self.key_header_name
+            )

package/autorest/codegen/models/credential_schema.py

@@ -11,6 +11,9 @@ class CredentialSchema(BaseSchema):
     def __init__(self) -> None:  # pylint: disable=super-init-not-called
         self.default_value = None
 
+    def type_annotation(self, *, is_operation_file: bool = False) -> str:
+        raise ValueError("Children classes should set their own type annotation")
+
     @property
     def docstring_type(self) -> str:
         return self.serialization_type
@@ -38,8 +41,7 @@ class AzureKeyCredentialSchema(CredentialSchema):
     def serialization_type(self) -> str:
         return "~azure.core.credentials.AzureKeyCredential"
 
-    @property
-    def type_annotation(self) -> str:
+    def type_annotation(self, *, is_operation_file: bool = False) -> str:  # pylint: disable=unused-argument
         return "AzureKeyCredential"
 
     def imports(self) -> FileImport:
@@ -66,8 +68,7 @@ class TokenCredentialSchema(CredentialSchema):
             return self.async_type
         return self.sync_type
 
-    @property
-    def type_annotation(self) -> str:
+    def type_annotation(self, *, is_operation_file: bool = False) -> str:  # pylint: disable=unused-argument
         if self.async_mode:
             return '"AsyncTokenCredential"'
         return '"TokenCredential"'

package/autorest/codegen/models/dictionary_schema.py

@@ -34,18 +34,13 @@ class DictionarySchema(BaseSchema):
         """
         return f"{{{self.element_type.serialization_type}}}"
 
-    @property
-    def type_annotation(self) -> str:
+    def type_annotation(self, *, is_operation_file: bool = False) -> str:
         """The python type used for type annotation
 
         :return: The type annotation for this schema
         :rtype: str
         """
-        return f"Dict[str, {self.element_type.type_annotation}]"
-
-    @property
-    def operation_type_annotation(self) -> str:
-        return f"Dict[str, {self.element_type.operation_type_annotation}]"
+        return f"Dict[str, {self.element_type.type_annotation(is_operation_file=is_operation_file)}]"
 
     @property
     def docstring_text(self) -> str:
@@ -101,3 +96,8 @@ class DictionarySchema(BaseSchema):
         file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB, TypingSection.CONDITIONAL)
         file_import.merge(self.element_type.imports())
         return file_import
+
+    def model_file_imports(self) -> FileImport:
+        file_import = self.imports()
+        file_import.merge(self.element_type.model_file_imports())
+        return file_import