@autorest/python 6.1.4 → 6.1.6

package/ChangeLog.md

@@ -1,5 +1,40 @@
 # Release History
 
+### 2022-09-15 - 6.1.6
+
+| Library                                                                 | Min Version |
+| ----------------------------------------------------------------------- | ----------- |
+| `@autorest/core`                                                        | `3.8.4`     |
+| `@autorest/modelerfour`                                                 | `4.23.5`    |
+| `azure-core` dep of generated code                                      | `1.24.0`    |
+| `isodate` dep of generated code                                         | `0.6.1`     |
+| `msrest` dep of generated code (If generating legacy code)              | `0.7.1`     |
+| `azure-mgmt-core` dep of generated code (If generating mgmt plane code) | `1.3.2`     |
+
+**Bug Fixes**
+
+- Fix `--clear-output-folder` when `--black=true`  #1410
+
+### 2022-09-06 - 6.1.5
+
+| Library                                                                 | Min Version |
+| ----------------------------------------------------------------------- | ----------- |
+| `@autorest/core`                                                        | `3.8.4`     |
+| `@autorest/modelerfour`                                                 | `4.23.5`    |
+| `azure-core` dep of generated code                                      | `1.24.0`    |
+| `isodate` dep of generated code                                         | `0.6.1`     |
+| `msrest` dep of generated code (If generating legacy code)              | `0.7.1`     |
+| `azure-mgmt-core` dep of generated code (If generating mgmt plane code) | `1.3.2`     |
+
+**Bug Fixes**
+
+- Fix `api_version` error when there are multi different `api-version`(not multiapi)   #1429
+- Fix generator raising `KeyError` in corner case when generating an LRO-paging operation  #1425
+
+**Other Changes**
+
+- Default `304` errors to throw `azure.core.exception.ResourceNotFoundError`s  #1415
+
 ### 2022-08-31 - 6.1.4
 
 | Library                                                                 | Min Version |

package/autorest/__init__.py

@@ -7,7 +7,7 @@ import logging
 from pathlib import Path
 import json
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Union
+from typing import Any, Dict, Union, List
 
 import yaml
 
@@ -22,6 +22,7 @@ _LOGGER = logging.getLogger(__name__)
 class ReaderAndWriter:
     def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None:
         self.output_folder = Path(output_folder)
+        self._list_file: List[str] = []
         try:
             with open(
                 Path(self.output_folder) / Path("..") / Path("python.json"), "r"
@@ -37,7 +38,7 @@ class ReaderAndWriter:
         self.options.update(python_json)
 
     def read_file(self, path: Union[str, Path]) -> str:
-        """How does one read a file in cadl?"""
+        """Directly reading from disk"""
         # make path relative to output folder
         try:
             with open(self.output_folder / Path(path), "r") as fd:
@@ -46,13 +47,16 @@ class ReaderAndWriter:
             return ""
 
     def write_file(self, filename: Union[str, Path], file_content: str) -> None:
-        """How does writing work in cadl?"""
+        """Directly writing to disk"""
         file_folder = Path(filename).parent
         if not Path.is_dir(self.output_folder / file_folder):
             Path.mkdir(self.output_folder / file_folder, parents=True)
         with open(self.output_folder / Path(filename), "w") as fd:
             fd.write(file_content)
 
+    def list_file(self) -> List[str]:
+        return [str(f) for f in self.output_folder.glob("**/*") if f.is_file()]
+
 
 class ReaderAndWriterAutorest(ReaderAndWriter):
     def __init__(
@@ -67,6 +71,9 @@ class ReaderAndWriterAutorest(ReaderAndWriter):
     def write_file(self, filename: Union[str, Path], file_content: str) -> None:
         return self._autorestapi.write_file(filename, file_content)
 
+    def list_file(self) -> List[str]:
+        return self._autorestapi.list_inputs()
+
 
 class Plugin(ReaderAndWriter, ABC):
     """A base class for autorest plugin.

package/autorest/black/__init__.py

@@ -30,16 +30,10 @@ class BlackScriptPlugin(Plugin):  # pylint: disable=abstract-method
 
     def process(self) -> bool:
         # apply format_file on every file in the output folder
-        list(
-            map(
-                self.format_file,
-                [f for f in self.output_folder.glob("**/*") if f.is_file()],
-            )
-        )
+        list(map(self.format_file, [Path(f) for f in self.list_file()]))
         return True
 
-    def format_file(self, full_path) -> None:
-        file = full_path.relative_to(self.output_folder)
+    def format_file(self, file: Path) -> None:
         file_content = self.read_file(file)
         if not file.suffix == ".py":
             self.write_file(file, file_content)

package/autorest/codegen/models/base_builder.py

@@ -31,7 +31,9 @@ if TYPE_CHECKING:
 _LOGGER = logging.getLogger(__name__)
 
 
-class BaseBuilder(Generic[ParameterListType], BaseModel):
+class BaseBuilder(
+    Generic[ParameterListType], BaseModel
+):  # pylint: disable=too-many-instance-attributes
     """Base class for Operations and Request Builders"""
 
     def __init__(
@@ -56,6 +58,7 @@ class BaseBuilder(Generic[ParameterListType], BaseModel):
         self.group_name: str = yaml_data["groupName"]
         self.is_overload: bool = yaml_data["isOverload"]
         self.api_versions: List[str] = yaml_data["apiVersions"]
+        self.added_on: Optional[str] = yaml_data.get("addedOn")
 
         if code_model.options["version_tolerant"] and yaml_data.get("abstract"):
             _LOGGER.warning(

package/autorest/codegen/models/credential_types.py

@@ -159,18 +159,18 @@ class TokenCredentialType(
     """Type of a token credential. Used by BearerAuth and ARMChallenge policies"""
 
     def type_annotation(self, **kwargs: Any) -> str:  # pylint: disable=no-self-use
-        if kwargs.pop("async_mode"):
+        if kwargs.get("async_mode"):
             return '"AsyncTokenCredential"'
         return '"TokenCredential"'
 
     def docstring_type(self, **kwargs: Any) -> str:  # pylint: disable=no-self-use
-        if kwargs.pop("async_mode"):
+        if kwargs.get("async_mode"):
             return "~azure.core.credentials_async.AsyncTokenCredential"
         return "~azure.core.credentials.TokenCredential"
 
     def imports(self, **kwargs: Any) -> FileImport:  # pylint: disable=no-self-use
         file_import = FileImport()
-        if kwargs.pop("async_mode"):
+        if kwargs.get("async_mode"):
             file_import.add_submodule_import(
                 "azure.core.credentials_async",
                 "AsyncTokenCredential",

package/autorest/codegen/models/operation.py

@@ -209,6 +209,12 @@ class OperationBase(  # pylint: disable=too-many-public-methods
             file_import.add_submodule_import(
                 "typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL
             )
+        if self.added_on:
+            file_import.add_submodule_import(
+                f"{'.' if async_mode else ''}.._validation",
+                "api_version_validation",
+                ImportType.LOCAL,
+            )
         return file_import
 
     def imports_for_multiapi(self, async_mode: bool, **kwargs: Any) -> FileImport:
@@ -244,6 +250,11 @@ class OperationBase(  # pylint: disable=too-many-public-methods
             for kwarg in kwargs_to_pop
         )
 
+    @property
+    def need_validation(self) -> bool:
+        """Whether we need parameter / operation validation. For API version."""
+        return bool(self.added_on) or any(p for p in self.parameters if p.added_on)
+
     def get_request_builder_import(
         self,
         request_builder: Union[RequestBuilder, OverloadedRequestBuilder],
@@ -291,27 +302,22 @@ class OperationBase(  # pylint: disable=too-many-public-methods
             file_import.merge(self.parameters.body_parameter.type.imports(**kwargs))
 
         # Exceptions
-        file_import.add_submodule_import(
-            "azure.core.exceptions", "map_error", ImportType.AZURECORE
-        )
+        errors = [
+            "map_error",
+            "HttpResponseError",
+            "ClientAuthenticationError",
+            "ResourceNotFoundError",
+            "ResourceExistsError",
+            "ResourceNotModifiedError",
+        ]
+        for error in errors:
+            file_import.add_submodule_import(
+                "azure.core.exceptions", error, ImportType.AZURECORE
+            )
         if self.code_model.options["azure_arm"]:
             file_import.add_submodule_import(
                 "azure.mgmt.core.exceptions", "ARMErrorFormat", ImportType.AZURECORE
             )
-        file_import.add_submodule_import(
-            "azure.core.exceptions", "HttpResponseError", ImportType.AZURECORE
-        )
-        file_import.add_submodule_import(
-            "azure.core.exceptions",
-            "ClientAuthenticationError",
-            ImportType.AZURECORE,
-        )
-        file_import.add_submodule_import(
-            "azure.core.exceptions", "ResourceNotFoundError", ImportType.AZURECORE
-        )
-        file_import.add_submodule_import(
-            "azure.core.exceptions", "ResourceExistsError", ImportType.AZURECORE
-        )
 
         if self.has_kwargs_to_pop_with_default(
             self.parameters.kwargs_to_pop, ParameterLocation.HEADER

package/autorest/codegen/models/operation_group.py

@@ -66,6 +66,11 @@ class OperationGroup(BaseModel):
             return "  # type: ignore"
         return ""
 
+    @property
+    def need_validation(self) -> bool:
+        """Whether any of its operations need validation"""
+        return any(o for o in self.operations if o.need_validation)
+
     def imports(self, async_mode: bool) -> FileImport:
         file_import = FileImport()
 

package/autorest/codegen/models/parameter.py

@@ -82,6 +82,7 @@ class _ParameterBase(
         self.in_flattened_body: bool = self.yaml_data.get("inFlattenedBody", False)
         self.grouper: bool = self.yaml_data.get("grouper", False)
         self.check_client_input: bool = self.yaml_data.get("checkClientInput", False)
+        self.added_on: Optional[str] = self.yaml_data.get("addedOn")
 
     @property
     def constant(self) -> bool:
@@ -152,6 +153,12 @@ class _ParameterBase(
         )
         if self.optional and self.client_default_value is None:
             file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB)
+        if self.added_on:
+            file_import.add_submodule_import(
+                f"{'.' if async_mode else ''}.._validation",
+                "api_version_validation",
+                ImportType.LOCAL,
+            )
         return file_import
 
     @property

package/autorest/codegen/models/primitive_types.py

@@ -122,13 +122,13 @@ class BinaryIteratorType(PrimitiveType):
         return "IO"
 
     def docstring_type(self, **kwargs: Any) -> str:
-        return "AsyncIterator[bytes]" if kwargs.pop("async_mode") else "Iterator[bytes]"
+        return "AsyncIterator[bytes]" if kwargs.get("async_mode") else "Iterator[bytes]"
 
     def type_annotation(self, **kwargs: Any) -> str:
         return self.docstring_type(**kwargs)
 
     def docstring_text(self, **kwargs: Any) -> str:
-        iterator = "Async iterator" if kwargs.pop("async_mode") else "Iterator"
+        iterator = "Async iterator" if kwargs.get("async_mode") else "Iterator"
         return f"{iterator} of the response bytes"
 
     @property
@@ -137,7 +137,7 @@ class BinaryIteratorType(PrimitiveType):
 
     def imports(self, **kwargs: Any) -> FileImport:
         file_import = FileImport()
-        iterator = "AsyncIterator" if kwargs.pop("async_mode") else "Iterator"
+        iterator = "AsyncIterator" if kwargs.get("async_mode") else "Iterator"
         file_import.add_submodule_import("typing", iterator, ImportType.STDLIB)
         return file_import
 

package/autorest/codegen/models/response.py

@@ -170,7 +170,7 @@ class PagingResponse(Response):
 
     def _imports_shared(self, **kwargs: Any) -> FileImport:
         file_import = super()._imports_shared(**kwargs)
-        async_mode = kwargs.pop("async_mode")
+        async_mode = kwargs.get("async_mode", False)
         pager_import_path = ".".join(self.get_pager_path(async_mode).split(".")[:-1])
         pager = self.get_pager(async_mode)
 
@@ -179,7 +179,7 @@ class PagingResponse(Response):
 
     def imports(self, **kwargs: Any) -> FileImport:
         file_import = self._imports_shared(**kwargs)
-        async_mode = kwargs.pop("async_mode")
+        async_mode = kwargs.get("async_mode")
         if async_mode:
             file_import.add_submodule_import(
                 "azure.core.async_paging", "AsyncList", ImportType.AZURECORE
@@ -234,16 +234,14 @@ class LROResponse(Response):
         return self.get_base_polling_method_path(async_mode).split(".")[-1]
 
     def type_annotation(self, **kwargs: Any) -> str:
-        return f"{self.get_poller(kwargs.pop('async_mode'))}[{super().type_annotation(**kwargs)}]"
+        return f"{self.get_poller(kwargs.get('async_mode', False))}[{super().type_annotation(**kwargs)}]"
 
     def docstring_type(self, **kwargs: Any) -> str:
-        return f"~{self.get_poller_path(kwargs.pop('async_mode'))}[{super().docstring_type(**kwargs)}]"
+        return f"~{self.get_poller_path(kwargs.get('async_mode', False))}[{super().docstring_type(**kwargs)}]"
 
     def docstring_text(self, **kwargs) -> str:
         super_text = super().docstring_text(**kwargs)
-        base_description = (
-            f"An instance of {self.get_poller(kwargs.pop('async_mode'))} that returns "
-        )
+        base_description = f"An instance of {self.get_poller(kwargs.get('async_mode', False))} that returns "
         if not self.code_model.options["version_tolerant"]:
             base_description += "either "
         return base_description + super_text
@@ -297,11 +295,11 @@ class LROResponse(Response):
 class LROPagingResponse(LROResponse, PagingResponse):
     def type_annotation(self, **kwargs: Any) -> str:
         paging_type_annotation = PagingResponse.type_annotation(self, **kwargs)
-        return f"{self.get_poller(kwargs.pop('async_mode'))}[{paging_type_annotation}]"
+        return f"{self.get_poller(kwargs.get('async_mode', False))}[{paging_type_annotation}]"
 
     def docstring_type(self, **kwargs: Any) -> str:
         paging_docstring_type = PagingResponse.docstring_type(self, **kwargs)
-        return f"~{self.get_poller_path(kwargs.pop('async_mode'))}[{paging_docstring_type}]"
+        return f"~{self.get_poller_path(kwargs.get('async_mode', False))}[{paging_docstring_type}]"
 
     def docstring_text(self, **kwargs) -> str:
         base_description = (

package/autorest/codegen/serializers/__init__.py

@@ -428,6 +428,12 @@ class JinjaSerializer(ReaderAndWriter):  # pylint: disable=abstract-method
                 general_serializer.serialize_serialization_file(),
             )
 
+        if any(og for og in self.code_model.operation_groups if og.need_validation):
+            self.write_file(
+                namespace_path / Path("_validation.py"),
+                general_serializer.serialize_validation_file(),
+            )
+
         # Write the config file
         if self.code_model.request_builders:
             self.write_file(

package/autorest/codegen/serializers/builder_serializer.py

@@ -173,6 +173,22 @@ def _get_json_response_template_to_status_codes(
     return retval
 
 
+def _api_version_validation(builder: OperationType) -> str:
+    retval: List[str] = []
+    if builder.added_on:
+        retval.append(f'    method_added_on="{builder.added_on}",')
+    params_added_on = defaultdict(list)
+    for parameter in builder.parameters:
+        if parameter.added_on:
+            params_added_on[parameter.added_on].append(parameter.client_name)
+    if params_added_on:
+        retval.append(f"    params_added_on={dict(params_added_on)},")
+    if retval:
+        retval_str = "\n".join(retval)
+        return f"@api_version_validation(\n{retval_str}\n)"
+    return ""
+
+
 class _BuilderBaseSerializer(Generic[BuilderType]):  # pylint: disable=abstract-method
     def __init__(self, code_model: CodeModel, async_mode: bool) -> None:
         self.code_model = code_model
@@ -590,8 +606,10 @@ class _OperationSerializer(
 
     def decorators(self, builder: OperationType) -> List[str]:
         """Decorators for the method"""
-        super_decorators = super().decorators(builder)
-        return super_decorators
+        retval = super().decorators(builder)
+        if _api_version_validation(builder):
+            retval.append(_api_version_validation(builder))
+        return retval
 
     def param_description(
         self, builder: OperationType
@@ -1043,6 +1061,8 @@ class _OperationSerializer(
                 retval.append("    404: ResourceNotFoundError,")
             if not 409 in builder.non_default_error_status_codes:
                 retval.append("    409: ResourceExistsError,")
+            if not 304 in builder.non_default_error_status_codes:
+                retval.append("    304: ResourceNotModifiedError,")
             for excep in builder.non_default_errors:
                 error_model_str = ""
                 if (
@@ -1071,6 +1091,11 @@ class _OperationSerializer(
                             "    409: lambda response: ResourceExistsError(response=response"
                             f"{error_model_str}{error_format_str}),"
                         )
+                    elif status_code == 304:
+                        retval.append(
+                            "    304: lambda response: ResourceNotModifiedError(response=response"
+                            f"{error_model_str}{error_format_str}),"
+                        )
                     elif not error_model_str and not error_format_str:
                         retval.append(f"    {status_code}: HttpResponseError,")
                     else:
@@ -1080,7 +1105,8 @@ class _OperationSerializer(
                         )
         else:
             retval.append(
-                "    401: ClientAuthenticationError, 404: ResourceNotFoundError, 409: ResourceExistsError"
+                "    401: ClientAuthenticationError, 404: ResourceNotFoundError, 409: ResourceExistsError, "
+                "304: ResourceNotModifiedError"
             )
         retval.append("}")
         retval.append("error_map.update(kwargs.pop('error_map', {}) or {})")
@@ -1130,6 +1156,8 @@ class _PagingOperationSerializer(
             return ["@overload"]
         if self.code_model.options["tracing"] and builder.want_tracing:
             retval.append("@distributed_trace")
+        if _api_version_validation(builder):
+            retval.append(_api_version_validation(builder))
         return retval
 
     def call_next_link_request_builder(self, builder: PagingOperationType) -> List[str]:

package/autorest/codegen/serializers/general_serializer.py

@@ -120,3 +120,7 @@ class GeneralSerializer:
     def serialize_serialization_file(self) -> str:
         template = self.env.get_template("serialization.py.jinja2")
         return template.render(code_model=self.code_model)
+
+    def serialize_validation_file(self) -> str:
+        template = self.env.get_template("validation.py.jinja2")
+        return template.render(code_model=self.code_model)

package/autorest/codegen/templates/operation_groups_container.py.jinja2

@@ -17,4 +17,4 @@
 {% endif %}
 {% for operation_group in operation_groups %}
     {% include "operation_group.py.jinja2" %}
-{% endfor %}
+{% endfor %}

package/autorest/codegen/templates/validation.py.jinja2

@@ -0,0 +1,38 @@
+{{ code_model.options['license_header'] }}
+import functools
+
+def api_version_validation(**kwargs):
+    params_added_on = kwargs.pop("params_added_on", {})
+    method_added_on = kwargs.pop("method_added_on", "")
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                # this assumes the client has an _api_version attribute
+                client = args[0]
+                client_api_version = client._config.api_version  # pylint: disable=protected-access
+            except AttributeError:
+                return func(*args, **kwargs)
+
+            if method_added_on > client_api_version:
+                raise ValueError(
+                    f"'{func.__name__}' is not available in API version "
+                    f"{client_api_version}. Pass service API version {method_added_on} or newer to your client."
+                )
+
+            unsupported = {
+                parameter: api_version
+                for api_version, parameters in params_added_on.items()
+                for parameter in parameters
+                if parameter in kwargs and api_version > client_api_version
+            }
+            if unsupported:
+                raise ValueError("".join([
+                    f"'{param}' is not available in API version {client_api_version}. "
+                    f"Use service API version {version} or newer.\n"
+                    for param, version in unsupported.items()
+                ]))
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator

package/autorest/m4reformatter/__init__.py

@@ -426,6 +426,10 @@ class M4Reformatter(
 ):  # pylint: disable=too-many-public-methods
     """Add Python naming information."""
 
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.check_client_input: bool = False
+
     @property
     def azure_arm(self) -> bool:
         return bool(self._autorestapi.get_boolean_value("azure-arm"))
@@ -802,7 +806,7 @@ class M4Reformatter(
                 param["inDocstring"] = False
                 if self.legacy:
                     param["implementation"] = "Method"
-                    param["checkClientInput"] = True
+                    param["checkClientInput"] = self.check_client_input
             if has_flattened_body and param.get("targetProperty"):
                 retval.append(self.update_flattened_parameter(param, body_parameter))
                 continue
@@ -935,6 +939,8 @@ class M4Reformatter(
 
                 client_name = "base_url" if self.legacy else "endpoint"
                 global_parameter["language"]["default"]["description"] = "Service URL."
+            elif name == "api_version":
+                self.check_client_input = True
             global_params.append(
                 self.update_parameter(
                     global_parameter, override_client_name=client_name

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorest/python",
-  "version": "6.1.4",
+  "version": "6.1.6",
   "description": "The Python extension for generators in AutoRest.",
   "scripts": {
     "prepare": "node run-python3.js prepare.py",