@autorest/python 6.7.0 → 6.7.1

package/autorest/codegen/models/__init__.py

@@ -28,6 +28,7 @@ from .primitive_types import (
     BooleanType,
     AnyObjectType,
     UnixTimeType,
+    AzureCoreType,
 )
 from .enum_type import EnumType
 from .base import BaseType
@@ -144,6 +145,7 @@ TYPE_TO_OBJECT = {
     "any-object": AnyObjectType,
     "unixtime": UnixTimeType,
     "credential": StringType,
+    "azurecore": AzureCoreType,
 }
 _LOGGER = logging.getLogger(__name__)
 

package/autorest/codegen/models/client.py

@@ -76,6 +76,8 @@ class Client(_ClientConfigBase[ClientGlobalParameterList]):
                 for op_group in self.yaml_data.get("operationGroups", [])
             ]
             self.link_lro_initial_operations()
+        self.request_id_header_name = self.yaml_data.get("requestIdHeaderName", None)
+        self.has_etag: bool = yaml_data.get("hasEtag", False)
 
     def _build_request_builders(
         self,

package/autorest/codegen/models/code_model.py

@@ -15,10 +15,10 @@ from .constant_type import ConstantType
 
 
 def _is_legacy(options) -> bool:
-    return not (options["version_tolerant"] or options["low_level_client"])
+    return not (options.get("version_tolerant") or options.get("low_level_client"))
 
 
-class CodeModel:  # pylint: disable=too-many-public-methods
+class CodeModel:  # pylint: disable=too-many-public-methods, disable=too-many-instance-attributes
     """Top level code model
 
     :param options: Options of the code model. I.e., whether this is for management generation
@@ -77,6 +77,10 @@ class CodeModel:  # pylint: disable=too-many-public-methods
             t for t in self.types_map.values() if isinstance(t, CombinedType) and t.name
         ]
 
+    @property
+    def has_etag(self) -> bool:
+        return any(client.has_etag for client in self.clients)
+
     @property
     def has_operations(self) -> bool:
         if any(c for c in self.clients if c.has_operations):

package/autorest/codegen/models/dictionary_type.py

@@ -30,6 +30,10 @@ class DictionaryType(BaseType):
         super().__init__(yaml_data=yaml_data, code_model=code_model)
         self.element_type = element_type
 
+    @property
+    def format(self) -> Optional[str]:
+        return self.element_type.format if hasattr(self.element_type, "format") else None  # type: ignore
+
     @property
     def serialization_type(self) -> str:
         """Returns the serialization value for msrest.

package/autorest/codegen/models/list_type.py

@@ -25,6 +25,10 @@ class ListType(BaseType):
         self.min_items: Optional[int] = yaml_data.get("minItems")
         self.unique_items: bool = yaml_data.get("uniqueItems", False)
 
+    @property
+    def format(self) -> Optional[str]:
+        return self.element_type.format if hasattr(self.element_type, "format") else None  # type: ignore
+
     @property
     def serialization_type(self) -> str:
         return f"[{self.element_type.serialization_type}]"

package/autorest/codegen/models/operation.py

@@ -90,6 +90,7 @@ class OperationBase(  # pylint: disable=too-many-public-methods
         self.internal: bool = self.yaml_data.get("internal", False)
         if self.internal:
             self.name = "_" + self.name
+        self.has_etag: bool = self.yaml_data.get("hasEtag", False)
 
     @property
     def expose_stream_keyword(self) -> bool:
@@ -331,7 +332,7 @@ class OperationBase(  # pylint: disable=too-many-public-methods
             )
         return file_import
 
-    def imports(  # pylint: disable=too-many-branches
+    def imports(  # pylint: disable=too-many-branches, disable=too-many-statements
         self, async_mode: bool, **kwargs: Any
     ) -> FileImport:
         if self.abstract:
@@ -394,11 +395,22 @@ class OperationBase(  # pylint: disable=too-many-public-methods
         if self.deprecated:
             file_import.add_import("warnings", ImportType.STDLIB)
 
+        relative_path = "..." if async_mode else ".."
         if self.code_model.need_request_converter:
-            relative_path = "..." if async_mode else ".."
             file_import.add_submodule_import(
                 f"{relative_path}_vendor", "_convert_request", ImportType.LOCAL
             )
+        if self.has_etag:
+            file_import.add_submodule_import(
+                "azure.core.exceptions", "ResourceModifiedError", ImportType.AZURECORE
+            )
+            if not async_mode:
+                file_import.add_submodule_import(
+                    f"{relative_path}_vendor", "prep_if_match", ImportType.LOCAL
+                )
+                file_import.add_submodule_import(
+                    f"{relative_path}_vendor", "prep_if_none_match", ImportType.LOCAL
+                )
         if self.code_model.need_request_converter:
             if async_mode:
                 file_import.add_submodule_import(

package/autorest/codegen/models/parameter.py

@@ -52,9 +52,6 @@ class ParameterDelimeter(str, Enum):
     COMMA = "comma"
 
 
-SPECIAL_HANDLE_HEADERS = ["repeatability-request-id", "repeatability-first-sent"]
-
-
 class _ParameterBase(
     BaseModel, abc.ABC
 ):  # pylint: disable=too-many-instance-attributes
@@ -93,10 +90,7 @@ class _ParameterBase(
         self.default_to_unset_sentinel: bool = self.yaml_data.get(
             "defaultToUnsetSentinel", False
         )
-        self.is_special_handle_header: bool = (
-            self.location == ParameterLocation.HEADER
-            and self.wire_name.lower() in SPECIAL_HANDLE_HEADERS
-        )
+        self.hide_in_method: bool = self.yaml_data.get("hideInMethod", False)
 
     @property
     def constant(self) -> bool:

package/autorest/codegen/models/parameter_list.py

@@ -215,7 +215,7 @@ class _ParameterListBase(
             for p in self.parameters
             if p.in_method_signature
             and p.implementation == self.implementation
-            and not p.is_special_handle_header
+            and (self.code_model.is_legacy or not p.hide_in_method)
         ]
         if self._body_parameter:
             if self._body_parameter.in_method_signature:

package/autorest/codegen/models/primitive_types.py

@@ -4,7 +4,6 @@
 # license information.
 # --------------------------------------------------------------------------
 import datetime
-from enum import Enum
 from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING
 
 from .base import BaseType
@@ -356,17 +355,17 @@ class StringType(PrimitiveType):
 class DatetimeType(PrimitiveType):
     def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None:
         super().__init__(yaml_data=yaml_data, code_model=code_model)
-        self.format = self.Formats(yaml_data.get("format", "date-time"))
-
-    class Formats(str, Enum):
-        datetime = "date-time"
-        rfc1123 = "date-time-rfc1123"
+        self.format = (
+            "rfc3339"
+            if yaml_data.get("format", "date-time") == "date-time"
+            else "rfc7231"
+        )
 
     @property
     def serialization_type(self) -> str:
         formats_to_attribute_type = {
-            self.Formats.datetime: "iso-8601",
-            self.Formats.rfc1123: "rfc-1123",
+            "rfc3339": "iso-8601",
+            "rfc7231": "rfc-1123",
         }
         return formats_to_attribute_type[self.format]
 
@@ -454,6 +453,10 @@ class TimeType(PrimitiveType):
 
 
 class UnixTimeType(PrimitiveType):
+    @property
+    def format(self) -> str:
+        return "unix-timestamp"
+
     @property
     def serialization_type(self) -> str:
         return "unix-time"
@@ -588,7 +591,7 @@ class DurationType(PrimitiveType):
 class ByteArraySchema(PrimitiveType):
     def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None:
         super().__init__(yaml_data=yaml_data, code_model=code_model)
-        self.format = yaml_data.get("format", "bytes")
+        self.format = yaml_data.get("format", "base64")
 
     @property
     def serialization_type(self) -> str:
@@ -605,3 +608,28 @@ class ByteArraySchema(PrimitiveType):
     @property
     def instance_check_template(self) -> str:
         return "isinstance({}, bytes)"
+
+
+class AzureCoreType(PrimitiveType):
+    def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None:
+        super().__init__(yaml_data=yaml_data, code_model=code_model)
+        self.name = yaml_data.get("name", "")
+
+    def docstring_type(self, **kwargs: Any) -> str:
+        return "~azure.core." + self.type_annotation(**kwargs)
+
+    def type_annotation(self, **kwargs: Any) -> str:
+        return self.name
+
+    def imports(self, **kwargs: Any) -> FileImport:
+        file_import = FileImport()
+        file_import.add_submodule_import("azure.core", self.name, ImportType.AZURECORE)
+        return file_import
+
+    @property
+    def instance_check_template(self) -> str:
+        return f"isinstance({{}}, {self.name})"
+
+    @property
+    def serialization_type(self) -> str:
+        return self.name

package/autorest/codegen/serializers/__init__.py

@@ -52,6 +52,7 @@ _PACKAGE_FILES = [
 
 _REGENERATE_FILES = {"setup.py", "MANIFEST.in"}
 
+
 # extract sub folders. For example, source_file_path is like:
 # "xxx/resource-manager/Microsoft.XX/stable/2023-04-01/examples/Compute/createOrUpdate/AKSCompute.json",
 # and we want to extract the sub folders after "examples/", which is "compute/create_or_update"

package/autorest/codegen/serializers/builder_serializer.py

@@ -18,7 +18,6 @@ from ..models import (
     ModelType,
     DictionaryType,
     ListType,
-    Parameter,
     RequestBuilder,
     ParameterLocation,
     Response,
@@ -422,36 +421,6 @@ class _BuilderBaseSerializer(Generic[BuilderType]):  # pylint: disable=abstract-
         )
         return template
 
-    def _serialize_parameter(self, param: Parameter, kwarg_name: str) -> List[str]:
-        set_parameter = "_{}['{}'] = {}".format(
-            kwarg_name,
-            param.wire_name,
-            self.parameter_serializer.serialize_parameter(param, self.serializer_name),
-        )
-        if not param.optional:
-            retval = [set_parameter]
-        else:
-            retval = [
-                f"if {param.full_client_name} is not None:",
-                f"    {set_parameter}",
-            ]
-        return retval
-
-    @staticmethod
-    def _serialize_special_handle_header(param: Parameter) -> List[str]:
-        if param.wire_name.lower() == "repeatability-request-id":
-            return [
-                """if "Repeatability-Request-ID" not in _headers:""",
-                """    _headers["Repeatability-Request-ID"] = str(uuid.uuid4())""",
-            ]
-        if param.wire_name.lower() == "repeatability-first-sent":
-            return [
-                """if "Repeatability-First-Sent" not in _headers:""",
-                """    _headers["Repeatability-First-Sent"] = _SERIALIZER.serialize_data(datetime.datetime.now(),
-                "rfc-1123")""",
-            ]
-        raise ValueError(f"Unsupported special header: {param}")
-
     def serialize_path(self, builder: BuilderType) -> List[str]:
         return self.parameter_serializer.serialize_path(
             builder.parameters.path, self.serializer_name
@@ -563,24 +532,25 @@ class RequestBuilderSerializer(
     def serialize_headers(self, builder: RequestBuilderType) -> List[str]:
         retval = ["# Construct headers"]
         for parameter in builder.parameters.headers:
-            if parameter.is_special_handle_header:
-                retval.extend(self._serialize_special_handle_header(parameter))
-            else:
-                retval.extend(
-                    self._serialize_parameter(
-                        parameter,
-                        kwarg_name="headers",
-                    )
+            retval.extend(
+                self.parameter_serializer.serialize_query_header(
+                    parameter,
+                    "headers",
+                    self.serializer_name,
+                    self.code_model.is_legacy,
                 )
+            )
         return retval
 
     def serialize_query(self, builder: RequestBuilderType) -> List[str]:
         retval = ["# Construct parameters"]
         for parameter in builder.parameters.query:
             retval.extend(
-                self._serialize_parameter(
+                self.parameter_serializer.serialize_query_header(
                     parameter,
-                    kwarg_name="params",
+                    "params",
+                    self.serializer_name,
+                    self.code_model.is_legacy,
                 )
             )
         return retval
@@ -1119,9 +1089,17 @@ class _OperationSerializer(
         return retval
 
     def handle_error_response(self, builder: OperationType) -> List[str]:
+        async_await = "await " if self.async_mode else ""
         retval = [
             f"if response.status_code not in {str(builder.success_status_codes)}:"
         ]
+        if not self.code_model.need_request_converter:
+            retval.extend(
+                [
+                    "    if _stream:",
+                    f"        {async_await} response.read()  # Load the body in memory and close the socket",
+                ]
+            )
         retval.append(
             "    map_error(status_code=response.status_code, response=response, error_map=error_map)"
         )
@@ -1283,6 +1261,17 @@ class _OperationSerializer(
                 "304: ResourceNotModifiedError"
             )
         retval.append("}")
+        if builder.has_etag:
+            retval.extend(
+                [
+                    "if match_condition == MatchConditions.IfNotModified:",
+                    "    error_map[412] = ResourceModifiedError",
+                    "elif match_condition == MatchConditions.IfPresent:",
+                    "    error_map[412] = ResourceNotFoundError",
+                    "elif match_condition == MatchConditions.IfMissing:",
+                    "    error_map[412] = ResourceExistsError",
+                ]
+            )
         retval.append("error_map.update(kwargs.pop('error_map', {}) or {})")
         return retval
 

package/autorest/codegen/serializers/client_serializer.py

@@ -109,9 +109,15 @@ class ClientSerializer:
 
     def initialize_pipeline_client(self, async_mode: bool) -> str:
         pipeline_client_name = self.client.pipeline_class(async_mode)
+        params = {
+            "base_url": self.host_variable_name,
+            "config": "self._config",
+        }
+        if not self.client.code_model.is_legacy and self.client.request_id_header_name:
+            params["request_id_header_name"] = f'"{self.client.request_id_header_name}"'
         return (
-            f"self._client: {pipeline_client_name} = {pipeline_client_name}(base_url={self.host_variable_name}, "
-            "config=self._config, **kwargs)"
+            f"self._client: {pipeline_client_name} = {pipeline_client_name}("
+            f"{', '.join(f'{k}={v}' for k, v in params.items())}, **kwargs)"
         )
 
     def serializers_and_operation_groups_properties(self) -> List[str]:

package/autorest/codegen/serializers/general_serializer.py

@@ -133,6 +133,13 @@ class GeneralSerializer:
                     f"{client.name}Configuration",
                     ImportType.LOCAL,
                 )
+        if self.code_model.has_etag:
+            file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB)
+            file_import.add_submodule_import(
+                "azure.core",
+                "MatchConditions",
+                ImportType.AZURECORE,
+            )
 
         return template.render(
             code_model=self.code_model,

package/autorest/codegen/serializers/model_serializer.py

@@ -255,13 +255,13 @@ class DpgModelSerializer(_ModelSerializer):
         args = []
         if prop.client_name != prop.wire_name or prop.is_discriminator:
             args.append(f'name="{prop.wire_name}"')
-        if prop.readonly:
-            args.append("readonly=True")
         if prop.visibility:
             v_list = ", ".join(f'"{x}"' for x in prop.visibility)
             args.append(f"visibility=[{v_list}]")
         if prop.client_default_value is not None:
             args.append(f"default={prop.client_default_value_declaration}")
+        if hasattr(prop.type, "format") and prop.type.format:  # type: ignore
+            args.append(f'format="{prop.type.format}"')  # type: ignore
 
         field = "rest_discriminator" if prop.is_discriminator else "rest_field"
         type_ignore = (

package/autorest/codegen/serializers/parameter_serializer.py

@@ -3,10 +3,9 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-from typing import List, Sequence, Union, Optional
+from typing import List, Sequence, Union, Optional, Dict
 from enum import Enum, auto
 
-
 from ..models import (
     Parameter,
     ParameterLocation,
@@ -26,6 +25,32 @@ class PopKwargType(Enum):
     CASE_INSENSITIVE = auto()
 
 
+SPECIAL_HEADER_SERIALIZATION: Dict[str, List[str]] = {
+    "repeatability-request-id": [
+        """if "Repeatability-Request-ID" not in _headers:""",
+        """    _headers["Repeatability-Request-ID"] = str(uuid.uuid4())""",
+    ],
+    "repeatability-first-sent": [
+        """if "Repeatability-First-Sent" not in _headers:""",
+        """    _headers["Repeatability-First-Sent"] = _SERIALIZER.serialize_data(datetime.datetime.now(),
+        "rfc-1123")""",
+    ],
+    "client-request-id": [],
+    "x-ms-client-request-id": [],
+    "return-client-request-id": [],
+    "etag": [
+        """if_match = prep_if_match(etag, match_condition)""",
+        """if if_match is not None:""",
+        """    _headers["If-Match"] = _SERIALIZER.header("if_match", if_match, "str")""",
+    ],
+    "match-condition": [
+        """if_none_match = prep_if_none_match(etag, match_condition)""",
+        """if if_none_match is not None:""",
+        """    _headers["If-None-Match"] = _SERIALIZER.header("if_none_match", if_none_match, "str")""",
+    ],
+}
+
+
 class ParameterSerializer:
     @staticmethod
     def serialize_parameter(parameter: ParameterType, serializer_name: str) -> str:
@@ -81,8 +106,8 @@ class ParameterSerializer:
             return f"[{serialize_line} if q is not None else '' for q in {origin_name}]"
         return serialize_line
 
+    @staticmethod
     def serialize_path(
-        self,
         parameters: Union[
             List[Parameter],
             List[RequestBuilderParameter],
@@ -96,7 +121,9 @@ class ParameterSerializer:
             [
                 '    "{}": {},'.format(
                     path_parameter.wire_name,
-                    self.serialize_parameter(path_parameter, serializer_name),
+                    ParameterSerializer.serialize_parameter(
+                        path_parameter, serializer_name
+                    ),
                 )
                 for path_parameter in parameters
             ]
@@ -104,6 +131,34 @@ class ParameterSerializer:
         retval.append("}")
         return retval
 
+    @staticmethod
+    def serialize_query_header(
+        param: Parameter,
+        kwarg_name: str,
+        serializer_name: str,
+        is_legacy: bool,
+    ) -> List[str]:
+        if (
+            not is_legacy
+            and param.location == ParameterLocation.HEADER
+            and param.wire_name.lower() in SPECIAL_HEADER_SERIALIZATION
+        ):
+            return SPECIAL_HEADER_SERIALIZATION[param.wire_name.lower()]
+
+        set_parameter = "_{}['{}'] = {}".format(
+            kwarg_name,
+            param.wire_name,
+            ParameterSerializer.serialize_parameter(param, serializer_name),
+        )
+        if not param.optional:
+            retval = [set_parameter]
+        else:
+            retval = [
+                f"if {param.full_client_name} is not None:",
+                f"    {set_parameter}",
+            ]
+        return retval
+
     @staticmethod
     def pop_kwargs_from_signature(
         parameters: Sequence[_ParameterBase],

package/autorest/codegen/templates/model_base.py.jinja2

@@ -7,6 +7,7 @@
 # pylint: disable=protected-access, arguments-differ, signature-differs, broad-except
 # pyright: reportGeneralTypeIssues=false
 
+import calendar
 import functools
 import sys
 import logging
@@ -14,13 +15,14 @@ import base64
 import re
 import copy
 import typing
+import email
 from datetime import datetime, date, time, timedelta, timezone
 from json import JSONEncoder
 import isodate
 from azure.core.exceptions import DeserializationError
 from azure.core import CaseInsensitiveEnumMeta
 from azure.core.pipeline import PipelineResponse
-from azure.core.serialization import _Null  # pylint: disable=protected-access
+from azure.core.serialization import _Null
 
 if sys.version_info >= (3, 9):
     from collections.abc import MutableMapping
@@ -31,9 +33,9 @@ _LOGGER = logging.getLogger(__name__)
 
 __all__ = ["AzureJSONEncoder", "Model", "rest_field", "rest_discriminator"]
 
-
 TZ_UTC = timezone.utc
 
+
 def _timedelta_as_isostr(td: timedelta) -> str:
     """Converts a datetime.timedelta object into an ISO 8601 formatted string, e.g. 'P4DT12H30M05S'
 
@@ -91,38 +93,20 @@ def _timedelta_as_isostr(td: timedelta) -> str:
     return "P" + date_str + time_str
 
 
-def _datetime_as_isostr(dt: typing.Union[datetime, date, time, timedelta]) -> str:
-    """Converts a datetime.(datetime|date|time|timedelta) object into an ISO 8601 formatted string
-
-    :param timedelta dt: The date object to convert
-    :rtype: str
-    :return: ISO8601 version of this datetime
-    """
-    # First try datetime.datetime
-    if hasattr(dt, "year") and hasattr(dt, "hour"):
-        dt = typing.cast(datetime, dt)
-        # astimezone() fails for naive times in Python 2.7, so make make sure dt is aware (tzinfo is set)
-        if not dt.tzinfo:
-            iso_formatted = dt.replace(tzinfo=TZ_UTC).isoformat()
-        else:
-            iso_formatted = dt.astimezone(TZ_UTC).isoformat()
-        # Replace the trailing "+00:00" UTC offset with "Z" (RFC 3339: https://www.ietf.org/rfc/rfc3339.txt)
-        return iso_formatted.replace("+00:00", "Z")
-    # Next try datetime.date or datetime.time
-    try:
-        dt = typing.cast(typing.Union[date, time], dt)
-        return dt.isoformat()
-    # Last, try datetime.timedelta
-    except AttributeError:
-        dt = typing.cast(timedelta, dt)
-        return _timedelta_as_isostr(dt)
-
-def _serialize_bytes(o) -> str:
-    return base64.b64encode(o).decode()
+def _serialize_bytes(o, format: typing.Optional[str] = None) -> str:
+    encoded = base64.b64encode(o).decode()
+    if format == "base64url":
+        return encoded.strip("=").replace("+", "-").replace("/", "_")
+    return encoded
 
 
-def _serialize_datetime(o):
+def _serialize_datetime(o, format: typing.Optional[str] = None):
     if hasattr(o, "year") and hasattr(o, "hour"):
+        if format == "rfc7231":
+            return email.utils.format_datetime(o, usegmt=True)
+        if format == "unix-timestamp":
+            return int(calendar.timegm(o.utctimetuple()))
+
         # astimezone() fails for naive times in Python 2.7, so make make sure o is aware (tzinfo is set)
         if not o.tzinfo:
             iso_formatted = o.replace(tzinfo=TZ_UTC).isoformat()
@@ -146,7 +130,7 @@ class AzureJSONEncoder(JSONEncoder):
 
     def default(self, o):  # pylint: disable=too-many-return-statements
         if _is_model(o):
-            readonly_props = [p._rest_name for p in o._attr_to_rest_field.values() if _is_readonly(p)]  # pylint: disable=protected-access
+            readonly_props = [p._rest_name for p in o._attr_to_rest_field.values() if _is_readonly(p)]
             return {k: v for k, v in o.items() if k not in readonly_props}
         if isinstance(o, (bytes, bytearray)):
             return base64.b64encode(o).decode()
@@ -172,6 +156,10 @@ class AzureJSONEncoder(JSONEncoder):
 
 
 _VALID_DATE = re.compile(r"\d{4}[-]\d{2}[-]\d{2}T\d{2}:\d{2}:\d{2}" + r"\.?\d*Z?[-+]?[\d{2}]?:?[\d{2}]?")
+_VALID_RFC7231 = re.compile(
+    r"(Mon|Tue|Wed|Thu|Fri|Sat|Sun),\s\d{2}\s"
+    r"(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{4}\s\d{2}:\d{2}:\d{2}\sGMT"
+)
 
 
 def _deserialize_datetime(attr: typing.Union[str, datetime]) -> datetime:
@@ -207,6 +195,36 @@ def _deserialize_datetime(attr: typing.Union[str, datetime]) -> datetime:
     return date_obj
 
 
+def _deserialize_datetime_rfc7231(attr: typing.Union[str, datetime]) -> datetime:
+    """Deserialize RFC7231 formatted string into Datetime object.
+
+    :param str attr: response string to be deserialized.
+    :rtype: ~datetime.datetime
+    :returns: The datetime object from that input
+    """
+    if isinstance(attr, datetime):
+        # i'm already deserialized
+        return attr
+    match = _VALID_RFC7231.match(attr)
+    if not match:
+        raise ValueError("Invalid datetime string: " + attr)
+
+    return email.utils.parsedate_to_datetime(attr)
+
+
+def _deserialize_datetime_unix_timestamp(attr: typing.Union[float, datetime]) -> datetime:
+    """Deserialize unix timestamp into Datetime object.
+
+    :param str attr: response string to be deserialized.
+    :rtype: ~datetime.datetime
+    :returns: The datetime object from that input
+    """
+    if isinstance(attr, datetime):
+        # i'm already deserialized
+        return attr
+    return datetime.fromtimestamp(attr, TZ_UTC)
+
+
 def _deserialize_date(attr: typing.Union[str, date]) -> date:
     """Deserialize ISO-8601 formatted string into Date object.
     :param str attr: response string to be deserialized.
@@ -231,13 +249,22 @@ def _deserialize_time(attr: typing.Union[str, time]) -> time:
     return isodate.parse_time(attr)
 
 
-def deserialize_bytes(attr):
+def _deserialize_bytes(attr):
     if isinstance(attr, (bytes, bytearray)):
         return attr
     return bytes(base64.b64decode(attr))
 
 
-def deserialize_duration(attr):
+def _deserialize_bytes_base64(attr):
+    if isinstance(attr, (bytes, bytearray)):
+        return attr
+    padding = "=" * (3 - (len(attr) + 3) % 4)  # type: ignore
+    attr = attr + padding  # type: ignore
+    encoded = attr.replace("-", "+").replace("_", "/")
+    return bytes(base64.b64decode(encoded))
+
+
+def _deserialize_duration(attr):
     if isinstance(attr, timedelta):
         return attr
     return isodate.parse_duration(attr)
@@ -247,11 +274,26 @@ _DESERIALIZE_MAPPING = {
     datetime: _deserialize_datetime,
     date: _deserialize_date,
     time: _deserialize_time,
-    bytes: deserialize_bytes,
-    timedelta: deserialize_duration,
+    bytes: _deserialize_bytes,
+    bytearray: _deserialize_bytes,
+    timedelta: _deserialize_duration,
     typing.Any: lambda x: x,
 }
 
+_DESERIALIZE_MAPPING_WITHFORMAT = {
+    "rfc3339": _deserialize_datetime,
+    "rfc7231": _deserialize_datetime_rfc7231,
+    "unix-timestamp": _deserialize_datetime_unix_timestamp,
+    "base64": _deserialize_bytes,
+    "base64url": _deserialize_bytes_base64,
+}
+
+
+def get_deserializer(annotation: typing.Any, rf: typing.Optional["_RestField"] = None):
+    if rf and rf._format:
+        return _DESERIALIZE_MAPPING_WITHFORMAT.get(rf._format)
+    return _DESERIALIZE_MAPPING.get(annotation)
+
 
 def _get_model(module_name: str, model_name: str):
     models = {k: v for k, v in sys.modules[module_name].__dict__.items() if isinstance(v, type)}
@@ -358,12 +400,20 @@ def _is_model(obj: typing.Any) -> bool:
     return getattr(obj, "_is_model", False)
 
 
-def _serialize(o):
+def _serialize(o, format: typing.Optional[str] = None):  # pylint: disable=too-many-return-statements
+    if isinstance(o, list):
+        return [_serialize(x, format) for x in o]
+    if isinstance(o, dict):
+        return {k: _serialize(v, format) for k, v in o.items()}
+    if isinstance(o, set):
+        return {_serialize(x, format) for x in o}
+    if isinstance(o, tuple):
+        return tuple(_serialize(x, format) for x in o)
     if isinstance(o, (bytes, bytearray)):
-        return _serialize_bytes(o)
+        return _serialize_bytes(o, format)
     try:
         # First try datetime.datetime
-        return _serialize_datetime(o)
+        return _serialize_datetime(o, format)
     except AttributeError:
         pass
     # Last, try datetime.timedelta
@@ -385,7 +435,7 @@ def _get_rest_field(
 
 
 def _create_value(rf: typing.Optional["_RestField"], value: typing.Any) -> typing.Any:
-    return _deserialize(rf._type, value) if (rf and rf._is_model) else _serialize(value)
+    return _deserialize(rf._type, value) if (rf and rf._is_model) else _serialize(value, rf._format if rf else None)
 
 
 class Model(_MyMutableMapping):
@@ -409,10 +459,13 @@ class Model(_MyMutableMapping):
             if non_attr_kwargs:
                 # actual type errors only throw the first wrong keyword arg they see, so following that.
                 raise TypeError(f"{class_name}.__init__() got an unexpected keyword argument '{non_attr_kwargs[0]}'")
-            dict_to_pass.update({
-                self._attr_to_rest_field[k]._rest_name: _serialize(v)
-                for k, v in kwargs.items() if v is not None
-            })
+            dict_to_pass.update(
+                {
+                    self._attr_to_rest_field[k]._rest_name: _serialize(v, self._attr_to_rest_field[k]._format)
+                    for k, v in kwargs.items()
+                    if v is not None
+                }
+            )
         super().__init__(dict_to_pass)
 
     def copy(self) -> "Model":
@@ -464,145 +517,157 @@ class Model(_MyMutableMapping):
 
 
 def _get_deserialize_callable_from_annotation(  # pylint: disable=too-many-return-statements, too-many-statements
-        annotation: typing.Any, module: typing.Optional[str], rf: typing.Optional["_RestField"] = None
-    ) -> typing.Optional[typing.Callable[[typing.Any], typing.Any]]:
-        if not annotation or annotation in [int, float]:
-            return None
+    annotation: typing.Any,
+    module: typing.Optional[str],
+    rf: typing.Optional["_RestField"] = None,
+) -> typing.Optional[typing.Callable[[typing.Any], typing.Any]]:
+    if not annotation or annotation in [int, float]:
+        return None
 
-        try:
-            if module and _is_model(_get_model(module, annotation)):
-                if rf:
-                    rf._is_model = True
-                def _deserialize_model(model_deserializer: typing.Optional[typing.Callable], obj):
-                    if _is_model(obj):
-                        return obj
-                    return _deserialize(model_deserializer, obj)
+    try:
+        if module and _is_model(_get_model(module, annotation)):
+            if rf:
+                rf._is_model = True
 
-                return functools.partial(_deserialize_model, _get_model(module, annotation))
-        except Exception:
-            pass
+            def _deserialize_model(model_deserializer: typing.Optional[typing.Callable], obj):
+                if _is_model(obj):
+                    return obj
+                return _deserialize(model_deserializer, obj)
 
-        # is it a literal?
-        try:
-            if sys.version_info >= (3, 8):
-                from typing import Literal  # pylint: disable=no-name-in-module, ungrouped-imports
-            else:
-                from typing_extensions import Literal  # type: ignore  # pylint: disable=ungrouped-imports
+            return functools.partial(_deserialize_model, _get_model(module, annotation))
+    except Exception:
+        pass
 
-            if annotation.__origin__ == Literal:
-                return None
-        except AttributeError:
-            pass
+    # is it a literal?
+    try:
+        if sys.version_info >= (3, 8):
+            from typing import (
+                Literal,
+            )  # pylint: disable=no-name-in-module, ungrouped-imports
+        else:
+            from typing_extensions import Literal  # type: ignore  # pylint: disable=ungrouped-imports
 
-        if getattr(annotation, "__origin__", None) is typing.Union:
-            def _deserialize_with_union(union_annotation, obj):
-                for t in union_annotation.__args__:
-                    try:
-                        return _deserialize(t, obj, module)
-                    except DeserializationError:
-                        pass
-                raise DeserializationError()
-            return functools.partial(_deserialize_with_union, annotation)
-
-        # is it optional?
-        try:
-            # right now, assuming we don't have unions, since we're getting rid of the only
-            # union we used to have in msrest models, which was union of str and enum
-            if any(a for a in annotation.__args__ if a == type(None)):
+        if annotation.__origin__ == Literal:
+            return None
+    except AttributeError:
+        pass
 
-                if_obj_deserializer = _get_deserialize_callable_from_annotation(
-                    next(a for a in annotation.__args__ if a != type(None)), module, rf
-                )
+    if getattr(annotation, "__origin__", None) is typing.Union:
 
-                def _deserialize_with_optional(if_obj_deserializer: typing.Optional[typing.Callable], obj):
-                    if obj is None:
-                        return obj
-                    return _deserialize_with_callable(if_obj_deserializer, obj)
+        def _deserialize_with_union(union_annotation, obj):
+            for t in union_annotation.__args__:
+                try:
+                    return _deserialize(t, obj, module, rf)
+                except DeserializationError:
+                    pass
+            raise DeserializationError()
 
-                return functools.partial(_deserialize_with_optional, if_obj_deserializer)
-        except AttributeError:
-            pass
+        return functools.partial(_deserialize_with_union, annotation)
 
-        # is it a forward ref / in quotes?
-        if isinstance(annotation, (str, typing.ForwardRef)):
-            try:
-                model_name = annotation.__forward_arg__  # type: ignore
-            except AttributeError:
-                model_name = annotation
-            if module is not None:
-                annotation = _get_model(module, model_name)
+    # is it optional?
+    try:
+        # right now, assuming we don't have unions, since we're getting rid of the only
+        # union we used to have in msrest models, which was union of str and enum
+        if any(a for a in annotation.__args__ if a == type(None)):
+            if_obj_deserializer = _get_deserialize_callable_from_annotation(
+                next(a for a in annotation.__args__ if a != type(None)), module, rf
+            )
 
+            def _deserialize_with_optional(if_obj_deserializer: typing.Optional[typing.Callable], obj):
+                if obj is None:
+                    return obj
+                return _deserialize_with_callable(if_obj_deserializer, obj)
+
+            return functools.partial(_deserialize_with_optional, if_obj_deserializer)
+    except AttributeError:
+        pass
+
+    # is it a forward ref / in quotes?
+    if isinstance(annotation, (str, typing.ForwardRef)):
         try:
-            if annotation._name == "Dict":
-                key_deserializer = _get_deserialize_callable_from_annotation(annotation.__args__[0], module, rf)
-                value_deserializer = _get_deserialize_callable_from_annotation(annotation.__args__[1], module, rf)
-
-                def _deserialize_dict(
-                    key_deserializer: typing.Optional[typing.Callable],
-                    value_deserializer: typing.Optional[typing.Callable],
-                    obj: typing.Dict[typing.Any, typing.Any],
-                ):
-                    if obj is None:
-                        return obj
-                    return {
-                        _deserialize(key_deserializer, k, module): _deserialize(value_deserializer, v, module) for k, v in obj.items()
-                    }
-
-                return functools.partial(
-                    _deserialize_dict,
-                    key_deserializer,
-                    value_deserializer,
-                )
-        except (AttributeError, IndexError):
-            pass
-        try:
-            if annotation._name in ["List", "Set", "Tuple", "Sequence"]:
-                if len(annotation.__args__) > 1:
-
-                    def _deserialize_multiple_sequence(
-                        entry_deserializers: typing.List[typing.Optional[typing.Callable]], obj
-                    ):
-                        if obj is None:
-                            return obj
-                        return type(obj)(
-                            _deserialize(deserializer, entry, module) for entry, deserializer in zip(obj, entry_deserializers)
-                        )
-
-                    entry_deserializers = [
-                        _get_deserialize_callable_from_annotation(dt, module, rf) for dt in annotation.__args__
-                    ]
-                    return functools.partial(_deserialize_multiple_sequence, entry_deserializers)
-                deserializer = _get_deserialize_callable_from_annotation(annotation.__args__[0], module, rf)
-
-                def _deserialize_sequence(
-                    deserializer: typing.Optional[typing.Callable],
+            model_name = annotation.__forward_arg__  # type: ignore
+        except AttributeError:
+            model_name = annotation
+        if module is not None:
+            annotation = _get_model(module, model_name)
+
+    try:
+        if annotation._name == "Dict":
+            key_deserializer = _get_deserialize_callable_from_annotation(annotation.__args__[0], module, rf)
+            value_deserializer = _get_deserialize_callable_from_annotation(annotation.__args__[1], module, rf)
+
+            def _deserialize_dict(
+                key_deserializer: typing.Optional[typing.Callable],
+                value_deserializer: typing.Optional[typing.Callable],
+                obj: typing.Dict[typing.Any, typing.Any],
+            ):
+                if obj is None:
+                    return obj
+                return {
+                    _deserialize(key_deserializer, k, module): _deserialize(value_deserializer, v, module)
+                    for k, v in obj.items()
+                }
+
+            return functools.partial(
+                _deserialize_dict,
+                key_deserializer,
+                value_deserializer,
+            )
+    except (AttributeError, IndexError):
+        pass
+    try:
+        if annotation._name in ["List", "Set", "Tuple", "Sequence"]:
+            if len(annotation.__args__) > 1:
+
+                def _deserialize_multiple_sequence(
+                    entry_deserializers: typing.List[typing.Optional[typing.Callable]],
                     obj,
                 ):
                     if obj is None:
                         return obj
-                    return type(obj)(_deserialize(deserializer, entry, module) for entry in obj)
+                    return type(obj)(
+                        _deserialize(deserializer, entry, module)
+                        for entry, deserializer in zip(obj, entry_deserializers)
+                    )
+
+                entry_deserializers = [
+                    _get_deserialize_callable_from_annotation(dt, module, rf) for dt in annotation.__args__
+                ]
+                return functools.partial(_deserialize_multiple_sequence, entry_deserializers)
+            deserializer = _get_deserialize_callable_from_annotation(annotation.__args__[0], module, rf)
+
+            def _deserialize_sequence(
+                deserializer: typing.Optional[typing.Callable],
+                obj,
+            ):
+                if obj is None:
+                    return obj
+                return type(obj)(_deserialize(deserializer, entry, module) for entry in obj)
+
+            return functools.partial(_deserialize_sequence, deserializer)
+    except (TypeError, IndexError, AttributeError, SyntaxError):
+        pass
 
-                return functools.partial(_deserialize_sequence, deserializer)
-        except (TypeError, IndexError, AttributeError, SyntaxError):
+    def _deserialize_default(
+        annotation,
+        deserializer_from_mapping,
+        obj,
+    ):
+        if obj is None:
+            return obj
+        try:
+            return _deserialize_with_callable(annotation, obj)
+        except Exception:
             pass
+        return _deserialize_with_callable(deserializer_from_mapping, obj)
 
-        def _deserialize_default(
-            annotation,
-            deserializer_from_mapping,
-            obj,
-        ):
-            if obj is None:
-                return obj
-            try:
-                return _deserialize_with_callable(annotation, obj)
-            except Exception:
-                pass
-            return _deserialize_with_callable(deserializer_from_mapping, obj)
-
-        return functools.partial(_deserialize_default, annotation, _DESERIALIZE_MAPPING.get(annotation))
+    return functools.partial(_deserialize_default, annotation, get_deserializer(annotation, rf))
 
 
-def _deserialize_with_callable(deserializer: typing.Optional[typing.Callable[[typing.Any], typing.Any]], value: typing.Any):
+def _deserialize_with_callable(
+    deserializer: typing.Optional[typing.Callable[[typing.Any], typing.Any]],
+    value: typing.Any,
+):
     try:
         if value is None:
             return None
@@ -621,12 +686,18 @@ def _deserialize_with_callable(deserializer: typing.Optional[typing.Callable[[ty
         raise DeserializationError() from e
 
 
-def _deserialize(deserializer: typing.Any, value: typing.Any, module: typing.Optional[str] = None) -> typing.Any:
+def _deserialize(
+    deserializer: typing.Any,
+    value: typing.Any,
+    module: typing.Optional[str] = None,
+    rf: typing.Optional["_RestField"] = None,
+) -> typing.Any:
     if isinstance(value, PipelineResponse):
         value = value.http_response.json()
-    deserializer = _get_deserialize_callable_from_annotation(deserializer, module)
+    deserializer = _get_deserialize_callable_from_annotation(deserializer, module, rf)
     return _deserialize_with_callable(deserializer, value)
 
+
 class _RestField:
     def __init__(
         self,
@@ -636,6 +707,7 @@ class _RestField:
         is_discriminator: bool = False,
         visibility: typing.Optional[typing.List[str]] = None,
         default: typing.Any = _UNSET,
+        format: typing.Optional[str] = None,
     ):
         self._type = type
         self._rest_name_input = name
@@ -644,6 +716,7 @@ class _RestField:
         self._visibility = visibility
         self._is_model = False
         self._default = default
+        self._format = format
 
     @property
     def _rest_name(self) -> str:
@@ -657,7 +730,7 @@ class _RestField:
         item = obj.get(self._rest_name)
         if item is None:
             return item
-        return _deserialize(self._type, _serialize(item))
+        return _deserialize(self._type, _serialize(item, self._format), rf=self)
 
     def __set__(self, obj: Model, value) -> None:
         if value is None:
@@ -669,7 +742,7 @@ class _RestField:
             return
         if self._is_model and not _is_model(value):
             obj.__setitem__(self._rest_name, _deserialize(self._type, value))
-        obj.__setitem__(self._rest_name, _serialize(value))
+        obj.__setitem__(self._rest_name, _serialize(value, self._format))
 
     def _get_deserialize_callable_from_annotation(
         self, annotation: typing.Any
@@ -683,8 +756,9 @@ def rest_field(
     type: typing.Optional[typing.Callable] = None,  # pylint: disable=redefined-builtin
     visibility: typing.Optional[typing.List[str]] = None,
     default: typing.Any = _UNSET,
+    format: typing.Optional[str] = None,
 ) -> typing.Any:
-    return _RestField(name=name, type=type, visibility=visibility, default=default)
+    return _RestField(name=name, type=type, visibility=visibility, default=default, format=format)
 
 
 def rest_discriminator(

package/autorest/codegen/templates/packaging_templates/setup.py.jinja2

@@ -90,7 +90,7 @@ setup(
         {% if azure_arm %}
         "azure-mgmt-core<2.0.0,>=1.3.2",
         {% else %}
-        "azure-core<2.0.0,>=1.27.0",
+        "azure-core<2.0.0,>=1.28.0",
         {% endif %}
         {% if code_model.need_typing_extensions %}
         "typing-extensions>=4.3.0; python_version<'3.8.0'",

package/autorest/codegen/templates/serialization.py.jinja2

@@ -744,6 +744,8 @@ class Serializer(object):
 
         :param data: The data to be serialized.
         :param str data_type: The type to be serialized from.
+        :keyword bool skip_quote: Whether to skip quote the serialized result.
+        Defaults to False.
         :rtype: str
         :raises: TypeError if serialization fails.
         :raises: ValueError if data is None
@@ -752,10 +754,8 @@ class Serializer(object):
             # Treat the list aside, since we don't want to encode the div separator
             if data_type.startswith("["):
                 internal_data_type = data_type[1:-1]
-                data = [self.serialize_data(d, internal_data_type, **kwargs) if d is not None else "" for d in data]
-                if not kwargs.get("skip_quote", False):
-                    data = [quote(str(d), safe="") for d in data]
-                return str(self.serialize_iter(data, internal_data_type, **kwargs))
+                do_quote = not kwargs.get('skip_quote', False)
+                return str(self.serialize_iter(data, internal_data_type, do_quote=do_quote, **kwargs))
 
             # Not a list, regular serialization
             output = self.serialize_data(data, data_type, **kwargs)
@@ -894,6 +894,8 @@ class Serializer(object):
          not be None or empty.
         :param str div: If set, this str will be used to combine the elements
          in the iterable into a combined string. Default is 'None'.
+        :keyword bool do_quote: Whether to quote the serialized result of each iterable element.
+        Defaults to False.
         :rtype: list, str
         """
         if isinstance(data, str):
@@ -911,6 +913,13 @@ class Serializer(object):
                     raise
                 serialized.append(None)
 
+        if kwargs.get('do_quote', False):
+            serialized = [
+                '' if s is None else quote(str(s), safe='')
+                for s
+                in serialized
+            ]
+
         if div:
             serialized = ["" if s is None else str(s) for s in serialized]
             serialized = div.join(serialized)

package/autorest/codegen/templates/vendor.py.jinja2

@@ -32,3 +32,32 @@ def raise_if_not_implemented(cls, abstract_methods):
         cls.__name__, '\', \''.join(not_implemented))
         )
 {% endif %}
+
+{% if code_model.has_etag %}
+def quote_etag(etag: Optional[str]) -> Optional[str]:
+    if not etag or etag == "*":
+        return etag
+    if etag.startswith('"') and etag.endswith('"'):
+        return etag
+    if etag.startswith("'") and etag.endswith("'"):
+        return etag
+    return '"' + etag + '"'
+
+
+def prep_if_match(etag: Optional[str], match_condition: Optional[MatchConditions]) -> Optional[str]:
+    if match_condition == MatchConditions.IfNotModified:
+        if_match = quote_etag(etag) if etag else None
+        return if_match
+    if match_condition == MatchConditions.IfPresent:
+        return "*"
+    return None
+
+
+def prep_if_none_match(etag: Optional[str], match_condition: Optional[MatchConditions]) -> Optional[str]:
+    if match_condition == MatchConditions.IfModified:
+        if_none_match = quote_etag(etag) if etag else None
+        return if_none_match
+    if match_condition == MatchConditions.IfMissing:
+        return "*"
+    return None
+{% endif %}

package/autorest/preprocess/__init__.py

@@ -176,6 +176,37 @@ def update_paging_response(yaml_data: Dict[str, Any]) -> None:
     )
 
 
+HEADERS_HIDE_IN_METHOD = (
+    "repeatability-request-id",
+    "repeatability-first-sent",
+    "x-ms-client-request-id",
+    "client-request-id",
+    "return-client-request-id",
+)
+HEADERS_CONVERT_IN_METHOD = {
+    "if-match": {
+        "clientName": "etag",
+        "wireName": "etag",
+        "description": "check if resource is changed. Set None to skip checking etag.",
+    },
+    "if-none-match": {
+        "clientName": "match_condition",
+        "wireName": "match-condition",
+        "description": "The match condition to use upon the etag.",
+        "type": {
+            "type": "azurecore",
+            "name": "MatchConditions",
+        },
+    },
+}
+
+
+def headers_convert(yaml_data: Dict[str, Any], replace_data: Any) -> None:
+    if isinstance(replace_data, dict):
+        for k, v in replace_data.items():
+            yaml_data[k] = v
+
+
 class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
     """Add Python naming information."""
 
@@ -242,6 +273,11 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
                         )
                 type["values"].extend(values_to_add)
 
+        # add type for reference
+        for v in HEADERS_CONVERT_IN_METHOD.values():
+            if isinstance(v, dict) and "type" in v:
+                yaml_data.append(v["type"])
+
     def update_client(self, yaml_data: Dict[str, Any]) -> None:
         yaml_data["description"] = update_description(
             yaml_data["description"], default_description=yaml_data["name"]
@@ -253,6 +289,21 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
         if prop_name.endswith("Client"):
             prop_name = prop_name[: len(prop_name) - len("Client")]
         yaml_data["builderPadName"] = to_snake_case(prop_name)
+        for og in yaml_data["operationGroups"]:
+            for o in og["operations"]:
+                for p in o["parameters"]:
+                    if (
+                        p["location"] == "header"
+                        and p["wireName"] == "client-request-id"
+                    ):
+                        yaml_data["requestIdHeaderName"] = p["wireName"]
+                    if (
+                        self.version_tolerant
+                        and p["location"] == "header"
+                        and p["clientName"] in ("if_match", "if_none_match")
+                    ):
+                        o["hasEtag"] = True
+                        yaml_data["hasEtag"] = True
 
     def get_operation_updater(
         self, yaml_data: Dict[str, Any]
@@ -282,6 +333,18 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
                 .lower()
                 for prop, param_name in yaml_data["propertyToParameterName"].items()
             }
+        wire_name_lower = (yaml_data.get("wireName") or "").lower()
+        if (
+            yaml_data["location"] == "header"
+            and wire_name_lower in HEADERS_HIDE_IN_METHOD
+        ):
+            yaml_data["hideInMethod"] = True
+        if (
+            self.version_tolerant
+            and yaml_data["location"] == "header"
+            and wire_name_lower in HEADERS_CONVERT_IN_METHOD
+        ):
+            headers_convert(yaml_data, HEADERS_CONVERT_IN_METHOD[wire_name_lower])
 
     def update_operation(
         self,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorest/python",
-  "version": "6.7.0",
+  "version": "6.7.1",
   "description": "The Python extension for generators in AutoRest.",
   "main": "index.js",
   "repository": {