@autorest/python 6.11.0 → 6.12.0

package/autorest/codegen/models/code_model.py

@@ -77,6 +77,15 @@ class CodeModel:  # pylint: disable=too-many-public-methods, disable=too-many-in
             t for t in self.types_map.values() if isinstance(t, CombinedType) and t.name
         ]
 
+    @property
+    def has_form_data(self) -> bool:
+        for client in self.clients:
+            for operation_group in client.operation_groups:
+                for operation in operation_group.operations:
+                    if operation.has_form_data_body:
+                        return True
+        return False
+
     @property
     def has_etag(self) -> bool:
         return any(client.has_etag for client in self.clients)
@@ -133,7 +142,12 @@ class CodeModel:  # pylint: disable=too-many-public-methods, disable=too-many-in
             return True
         if async_mode:
             return self.need_mixin_abc
-        return self.need_request_converter or self.need_mixin_abc or self.has_etag
+        return (
+            self.need_request_converter
+            or self.need_mixin_abc
+            or self.has_etag
+            or self.has_form_data
+        )
 
     @property
     def need_request_converter(self) -> bool:

package/autorest/codegen/models/imports.py

@@ -184,8 +184,7 @@ class FileImport:
             self._append_import(i)
         self.type_definitions.update(file_import.type_definitions)
 
-    def define_mutable_mapping_type(self) -> None:
-        """Helper function for defining the mutable mapping type"""
+    def add_mutable_mapping_import(self) -> None:
         self.add_import("sys", ImportType.STDLIB)
         self.add_submodule_import(
             "typing",
@@ -195,6 +194,10 @@ class FileImport:
             None,
             (((3, 9), "collections.abc", None),),
         )
+
+    def define_mutable_mapping_type(self) -> None:
+        """Helper function for defining the mutable mapping type"""
+        self.add_mutable_mapping_import()
         self.define_mypy_type(
             "JSON",
             "MutableMapping[str, Any] # pylint: disable=unsubscriptable-object",

package/autorest/codegen/models/model_type.py

@@ -6,7 +6,7 @@
 from collections import OrderedDict
 from typing import Any, Dict, List, Optional, TYPE_CHECKING, cast
 import sys
-from autorest.codegen.models.utils import add_to_pylint_disable
+from autorest.codegen.models.utils import add_to_pylint_disable, NAME_LENGTH_LIMIT
 from .base import BaseType
 from .constant_type import ConstantType
 from .property import Property
@@ -237,6 +237,8 @@ class ModelType(  # pylint: disable=abstract-method
         retval: str = ""
         if len(self.properties) > 10:
             retval = add_to_pylint_disable(retval, "too-many-instance-attributes")
+        if len(self.name) > NAME_LENGTH_LIMIT:
+            retval = add_to_pylint_disable(retval, "name-too-long")
         return retval
 
     @property

package/autorest/codegen/models/operation.py

@@ -92,6 +92,10 @@ class OperationBase(  # pylint: disable=too-many-public-methods
             self.name = "_" + self.name
         self.has_etag: bool = self.yaml_data.get("hasEtag", False)
 
+    @property
+    def has_form_data_body(self):
+        return self.parameters.has_form_data_body
+
     @property
     def expose_stream_keyword(self) -> bool:
         return self.yaml_data.get("exposeStreamKeyword", False)
@@ -568,13 +572,36 @@ class Operation(OperationBase[Response]):
             and not self.code_model.options["models_mode"]
         ):
             file_import.add_submodule_import("typing", "cast", ImportType.STDLIB)
+        relative_path = "..." if async_mode else ".."
         if self.code_model.options["models_mode"] == "dpg":
-            relative_path = "..." if async_mode else ".."
             if self.parameters.has_body:
-                file_import.add_submodule_import(
-                    f"{relative_path}_model_base", "SdkJSONEncoder", ImportType.LOCAL
-                )
-                file_import.add_import("json", ImportType.STDLIB)
+                if not self.parameters.body_parameter.is_form_data:
+                    file_import.add_submodule_import(
+                        f"{relative_path}_model_base",
+                        "SdkJSONEncoder",
+                        ImportType.LOCAL,
+                    )
+                    file_import.add_import("json", ImportType.STDLIB)
+                else:
+                    file_import.add_submodule_import(
+                        relative_path, "_model_base", ImportType.LOCAL
+                    )
+                    file_import.add_submodule_import("io", "IOBase", ImportType.STDLIB)
+                    file_import.add_submodule_import(
+                        f"{relative_path}_vendor",
+                        "multipart_file",
+                        ImportType.LOCAL,
+                    )
+                    file_import.add_submodule_import(
+                        f"{relative_path}_vendor",
+                        "multipart_data",
+                        ImportType.LOCAL,
+                    )
+                    file_import.add_submodule_import(
+                        f"{relative_path}_vendor",
+                        "handle_multipart_form_data_model",
+                        ImportType.LOCAL,
+                    )
             if self.default_error_deserialization or any(
                 r.type for r in self.responses
             ):

package/autorest/codegen/models/parameter.py

@@ -243,6 +243,10 @@ class _ParameterBase(
 class BodyParameter(_ParameterBase):
     """Body parameter."""
 
+    @property
+    def is_form_data(self) -> bool:
+        return self.default_content_type == "multipart/form-data"
+
     @property
     def is_partial_body(self) -> bool:
         """Whether it's part of a bigger body parameter, i.e. a MultipartBodyParameter"""

package/autorest/codegen/models/parameter_list.py

@@ -133,6 +133,10 @@ class _ParameterListBase(
             params.append(self.body_parameter)
         return params
 
+    @property
+    def has_form_data_body(self):
+        return self.has_body and self.body_parameter.is_form_data
+
     @property
     def has_body(self) -> bool:
         """Whether there is a body parameter in the parameter list"""

package/autorest/codegen/models/request_builder.py

@@ -57,6 +57,10 @@ class RequestBuilderBase(BaseBuilder[ParameterListType]):
         self.method: str = yaml_data["method"]
         self.want_tracing = False
 
+    @property
+    def has_form_data_body(self):
+        return self.parameters.has_form_data_body
+
     @property
     def is_lro(self) -> bool:
         return self.yaml_data.get("discriminator") in ("lro", "lropaging")

package/autorest/codegen/serializers/builder_serializer.py

@@ -38,6 +38,7 @@ from ..models import (
 from .parameter_serializer import ParameterSerializer, PopKwargType
 from ..models.parameter_list import ParameterType
 from . import utils
+from ..._utils import JSON_REGEXP
 
 T = TypeVar("T")
 OrderedSet = Dict[T, None]
@@ -59,6 +60,14 @@ OperationType = TypeVar(
 )
 
 
+def _need_type_ignore(builder: OperationType) -> bool:
+    for excep in builder.non_default_errors:
+        for status_code in excep.status_codes:
+            if status_code in (401, 404, 409, 304):
+                return True
+    return False
+
+
 def _xml_config(send_xml: bool, content_types: List[str]) -> str:
     if not (send_xml and "xml" in str(content_types)):
         return ""
@@ -547,11 +556,16 @@ class RequestBuilderSerializer(
         return retval
 
     def serialize_headers(self, builder: RequestBuilderType) -> List[str]:
-        retval = ["# Construct headers"]
-        for parameter in builder.parameters.headers:
+        headers = [
+            h
+            for h in builder.parameters.headers
+            if not builder.has_form_data_body or h.wire_name.lower() != "content-type"
+        ]
+        retval = ["# Construct headers"] if headers else []
+        for header in headers:
             retval.extend(
                 self.parameter_serializer.serialize_query_header(
-                    parameter,
+                    header,
                     "headers",
                     self.serializer_name,
                     self.code_model.is_legacy,
@@ -747,8 +761,17 @@ class _OperationSerializer(
 
         This function serializes the body params that need to be serialized.
         """
-        retval: List[str] = []
         body_param = cast(BodyParameter, builder.parameters.body_parameter)
+        if body_param.is_form_data:
+            return [
+                f"if isinstance({body_param.client_name}, _model_base.Model):",
+                f"    _body = handle_multipart_form_data_model({body_param.client_name})",
+                "else:",
+                f"    _body = {body_param.client_name}",
+                "_files = {k: multipart_file(v) for k, v in _body.items() if isinstance(v, (IOBase, bytes))}",
+                "_data = {k: multipart_data(v) for k, v in _body.items() if not isinstance(v, (IOBase, bytes))}",
+            ]
+        retval: List[str] = []
         body_kwarg_name = builder.request_builder.parameters.body_parameter.client_name
         send_xml = builder.parameters.body_parameter.type.is_xml
         xml_serialization_ctxt = (
@@ -769,17 +792,20 @@ class _OperationSerializer(
                 f"'{body_param.type.serialization_type}'{is_xml_cmd}{serialization_ctxt_cmd})"
             )
         elif self.code_model.options["models_mode"] == "dpg":
-            if hasattr(body_param.type, "encode") and body_param.type.encode:  # type: ignore
-                create_body_call = (
-                    f"_{body_kwarg_name} = json.dumps({body_param.client_name}, "
-                    "cls=SdkJSONEncoder, exclude_readonly=True, "
-                    f"format='{body_param.type.encode}')  # type: ignore"  # type: ignore
-                )
+            if JSON_REGEXP.match(body_param.default_content_type):
+                if hasattr(body_param.type, "encode") and body_param.type.encode:  # type: ignore
+                    create_body_call = (
+                        f"_{body_kwarg_name} = json.dumps({body_param.client_name}, "
+                        "cls=SdkJSONEncoder, exclude_readonly=True, "
+                        f"format='{body_param.type.encode}')  # type: ignore"  # type: ignore
+                    )
+                else:
+                    create_body_call = (
+                        f"_{body_kwarg_name} = json.dumps({body_param.client_name}, "
+                        "cls=SdkJSONEncoder, exclude_readonly=True)  # type: ignore"
+                    )
             else:
-                create_body_call = (
-                    f"_{body_kwarg_name} = json.dumps({body_param.client_name}, "
-                    "cls=SdkJSONEncoder, exclude_readonly=True)  # type: ignore"
-                )
+                create_body_call = f"_{body_kwarg_name} = {body_param.client_name}"
         else:
             create_body_call = f"_{body_kwarg_name} = {body_param.client_name}"
         if body_param.optional:
@@ -959,7 +985,10 @@ class _OperationSerializer(
                 f"    {parameter.client_name}={parameter.name_in_high_level_operation},"
                 f"{'  # type: ignore' if type_ignore else ''}"
             )
-        if request_builder.overloads:
+        if request_builder.has_form_data_body:
+            retval.append("    data=_data,")
+            retval.append("    files=_files,")
+        elif request_builder.overloads:
             seen_body_params = set()
             for overload in request_builder.overloads:
                 body_param = cast(
@@ -1030,7 +1059,9 @@ class _OperationSerializer(
                     builder.parameters.body_parameter, builder.parameters.parameters
                 )
             )
-        if builder.overloads:
+        if builder.has_form_data_body:
+            retval.extend(self._create_body_parameter(builder))
+        elif builder.overloads:
             # we are only dealing with two overloads. If there are three, we generate an abstract operation
             retval.extend(self._initialize_overloads(builder, is_paging=is_paging))
         elif builder.parameters.has_body:
@@ -1097,12 +1128,17 @@ class _OperationSerializer(
                         and response.default_content_type == "application/json"
                         else ""
                     )
+                    response_attr = (
+                        "json"
+                        if JSON_REGEXP.match(str(response.default_content_type))
+                        else "text"
+                    )
                     deserialize_code.append("deserialized = _deserialize(")
                     deserialize_code.append(
                         f"    {response.type.type_annotation(is_operation_file=True)},{pylint_disable}"
                     )
                     deserialize_code.append(
-                        f"    response.json(){response.result_property}{format_filed}"
+                        f"    response.{response_attr}(){response.result_property}{format_filed}"
                     )
                     deserialize_code.append(")")
 
@@ -1138,8 +1174,9 @@ class _OperationSerializer(
                     f"        {async_await} response.read()  # Load the body in memory and close the socket",
                 ]
             )
+        type_ignore = "  # type: ignore" if _need_type_ignore(builder) else ""
         retval.append(
-            "    map_error(status_code=response.status_code, response=response, error_map=error_map)"
+            f"    map_error(status_code=response.status_code, response=response, error_map=error_map){type_ignore}"
         )
         error_model = ""
         if (

package/autorest/codegen/serializers/general_serializer.py

@@ -130,6 +130,18 @@ class GeneralSerializer(BaseSerializer):
                 "MatchConditions",
                 ImportType.SDKCORE,
             )
+        if self.code_model.has_form_data:
+            file_import.add_submodule_import("typing", "Union", ImportType.STDLIB)
+            file_import.add_submodule_import("typing", "Any", ImportType.STDLIB)
+            file_import.add_submodule_import("io", "IOBase", ImportType.STDLIB)
+            file_import.add_submodule_import("io", "BytesIO", ImportType.STDLIB)
+            file_import.add_import("uuid", ImportType.STDLIB)
+            file_import.add_import("json", ImportType.STDLIB)
+            file_import.add_mutable_mapping_import()
+            file_import.add_submodule_import("._model_base", "Model", ImportType.LOCAL)
+            file_import.add_submodule_import(
+                "._model_base", "SdkJSONEncoder", ImportType.LOCAL
+            )
 
         return template.render(
             code_model=self.code_model,

package/autorest/codegen/templates/packaging_templates/README.md.jinja2

@@ -3,7 +3,7 @@
 # Microsoft Azure SDK for Python
 
 This is the Microsoft {{package_pprint_name}} Client Library.
-This package has been tested with Python 3.7+.
+This package has been tested with Python 3.8+.
 For a more complete view of Azure libraries, see the [azure sdk python release](https://aka.ms/azsdk/python/all).
 
 # Usage
@@ -36,7 +36,7 @@ python -m pip install {{ package_name }}
 
 #### Prequisites
 
-- Python 3.7 or later is required to use this package.
+- Python 3.8 or later is required to use this package.
 - You need an [Azure subscription][azure_sub] to use this package.
 - An existing {{ package_pprint_name }} instance.
 

package/autorest/codegen/templates/packaging_templates/setup.py.jinja2

@@ -56,11 +56,11 @@ setup(
         "Programming Language :: Python",
         "Programming Language :: Python :: 3 :: Only",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "License :: OSI Approved :: MIT License",
     ],
     zip_safe=False,
@@ -94,14 +94,14 @@ setup(
         {% elif azure_arm %}
         "azure-mgmt-core<2.0.0,>=1.3.2",
         {% else %}
-        "azure-core<2.0.0,>=1.28.0",
+        "azure-core<2.0.0,>=1.29.5",
         {% endif %}
         {% if code_model.need_typing_extensions %}
         "typing-extensions>=4.3.0; python_version<'3.8.0'",
         {% endif %}
     ],
     {% if package_mode %}
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     {% else %}
     long_description="""\
     {{ code_model.description }}

package/autorest/codegen/templates/vendor.py.jinja2

@@ -65,3 +65,38 @@ def prep_if_none_match(etag: Optional[str], match_condition: Optional[MatchCondi
         return "*"
     return None
 {% endif %}
+{% if code_model.has_form_data %}
+class NamedBytesIO(BytesIO):
+    def __init__(self, name: str, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.name = name
+
+def multipart_file(file: Union[IOBase, bytes]) -> IOBase:
+    if isinstance(file, IOBase):
+        return file
+    return NamedBytesIO("auto-name-" + str(uuid.uuid4()), file)
+
+def multipart_data(data: Any) -> Any:
+    if isinstance(data, (list, tuple, dict, Model)):
+        return json.dumps(data, cls=SdkJSONEncoder, exclude_readonly=True)
+    return data
+
+def handle_multipart_form_data_model(body: Model) -> MutableMapping[str, Any]:  # pylint: disable=unsubscriptable-object
+    """handle first layer of model. 
+    If its value is bytes or IO, replace it with raw value instead of serialized value.
+
+    :param body: The model to handle.
+    :type body: ~payload.multipart._model_base.Model
+    :return: The handled model.
+    :rtype: MutableMapping[str, Any]
+    """
+    result = body.as_dict()
+    rest_name_attr = {v._rest_name: k for k, v in body._attr_to_rest_field.items()}  # pylint: disable=protected-access
+    for rest_name in result.keys():
+        attr = rest_name_attr.get(rest_name)
+        if attr is not None:
+            raw_value = getattr(body, attr, None)
+            if isinstance(raw_value, (bytes, IOBase)):
+                result[rest_name] = raw_value
+    return result
+{% endif %}

package/autorest/preprocess/__init__.py

@@ -171,6 +171,16 @@ def headers_convert(yaml_data: Dict[str, Any], replace_data: Any) -> None:
             yaml_data[k] = v
 
 
+def has_json_content_type(yaml_data: Dict[str, Any]) -> bool:
+    return any(ct for ct in yaml_data.get("contentTypes", []) if JSON_REGEXP.match(ct))
+
+
+def has_multi_part_content_type(yaml_data: Dict[str, Any]) -> bool:
+    return any(
+        ct for ct in yaml_data.get("contentTypes", []) if ct == "multipart/form-data"
+    )
+
+
 class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
     """Add Python naming information."""
 
@@ -195,13 +205,13 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
         code_model: Dict[str, Any],
         body_parameter: Dict[str, Any],
     ):
-        if (
+        # only add overload for special content type
+        if (  # pylint: disable=too-many-boolean-expressions
             body_parameter
             and body_parameter["type"]["type"] in ("model", "dict", "list")
-            and any(
-                ct
-                for ct in body_parameter.get("contentTypes", [])
-                if JSON_REGEXP.match(ct)
+            and (
+                has_json_content_type(body_parameter)
+                or (self.is_cadl and has_multi_part_content_type(body_parameter))
             )
             and not body_parameter["type"].get("xmlMetadata")
             and not any(t for t in ["flattened", "groupedBy"] if body_parameter.get(t))
@@ -210,8 +220,12 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
             is_dpg_model = body_parameter["type"].get("base") == "dpg"
             body_parameter["type"] = {
                 "type": "combined",
-                "types": [body_parameter["type"], KNOWN_TYPES["binary"]],
+                "types": [body_parameter["type"]],
             }
+            # don't add binary overload for multipart content type
+            if not (self.is_cadl and has_multi_part_content_type(body_parameter)):
+                body_parameter["type"]["types"].append(KNOWN_TYPES["binary"])
+
             if origin_type == "model" and is_dpg_model and self.models_mode == "dpg":
                 body_parameter["type"]["types"].insert(1, KNOWN_TYPES["any-object"])
             code_model["types"].append(body_parameter["type"])
@@ -223,8 +237,10 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
             return name
 
         if self.is_cadl:
-            reserved_words = copy.copy(CADL_RESERVED_WORDS)
-            reserved_words.update(RESERVED_WORDS)
+            reserved_words = {
+                k: (v + CADL_RESERVED_WORDS.get(k, []))
+                for k, v in RESERVED_WORDS.items()
+            }
         else:
             reserved_words = RESERVED_WORDS
         name = pad_special_chars(name)
@@ -371,9 +387,9 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
                 for prop, param_name in yaml_data["propertyToParameterName"].items()
             }
         wire_name_lower = (yaml_data.get("wireName") or "").lower()
-        if (
-            yaml_data["location"] == "header"
-            and wire_name_lower in HEADERS_HIDE_IN_METHOD
+        if yaml_data["location"] == "header" and (
+            wire_name_lower in HEADERS_HIDE_IN_METHOD
+            or yaml_data.get("clientDefaultValue") == "multipart/form-data"
         ):
             yaml_data["hideInMethod"] = True
         if (

package/autorest/preprocess/python_mappings.py

@@ -96,6 +96,18 @@ _always_reserved = [
     "int",
 ]
 
+RESERVED_MODEL_PROPERTIES = [
+    "keys",
+    "items",
+    "values",
+    "popitem",
+    "clear",
+    "update",
+    "setdefault",
+    "pop",
+    "get",
+]
+
 RESERVED_WORDS = {
     PadType.METHOD: [*_always_reserved],
     PadType.PARAMETER: [
@@ -169,7 +181,10 @@ RESERVED_WORDS = {
     PadType.OPERATION_GROUP: [*_always_reserved],
 }
 
-CADL_RESERVED_WORDS = {PadType.PARAMETER: ["stream"]}
+CADL_RESERVED_WORDS = {
+    PadType.PARAMETER: ["stream"],
+    PadType.PROPERTY: RESERVED_MODEL_PROPERTIES,
+}
 
 REDEFINED_BUILTINS = [  # we don't pad, but we need to do lint ignores
     "id",

package/install.py

@@ -7,7 +7,7 @@
 # --------------------------------------------------------------------------
 import sys
 if not sys.version_info >= (3, 7, 0):
-    raise Exception("Autorest for Python extension requires Python 3.7 at least")
+    raise Exception("Autorest for Python extension requires Python 3.8 at least")
 
 try:
     import pip
@@ -20,7 +20,7 @@ except ImportError:
     raise Exception("Your Python installation doesn't have venv available")
 
 
-# Now we have pip and Py >= 3.7, go to work
+# Now we have pip and Py >= 3.8, go to work
 
 import subprocess
 from pathlib import Path

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorest/python",
-  "version": "6.11.0",
+  "version": "6.12.0",
   "description": "The Python extension for generators in AutoRest.",
   "main": "index.js",
   "repository": {

package/prepare.py

@@ -7,7 +7,7 @@
 # --------------------------------------------------------------------------
 import sys
 if not sys.version_info >= (3, 7, 0):
-    raise Exception("Autorest for Python extension requires Python 3.7 at least")
+    raise Exception("Autorest for Python extension requires Python 3.8 at least")
 
 from pathlib import Path
 import venv

package/requirements.txt

@@ -10,4 +10,4 @@ pathspec==0.11.1
 platformdirs==3.2.0
 PyYAML==6.0.1
 tomli==2.0.1
-setuptools==58.3.0
+setuptools==65.5.1

package/run-python3.js

@@ -10,7 +10,7 @@ const cp = require("child_process");
 const extension = require("@autorest/system-requirements");
 
 async function runPython3(scriptName, ...args) {
-  const command = await extension.patchPythonPath(["python", scriptName, ...args], { version: ">=3.7", environmentVariable: "AUTOREST_PYTHON_EXE" });
+  const command = await extension.patchPythonPath(["python", scriptName, ...args], { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE" });
   cp.execSync(command.join(" "), {
     stdio: [0, 1, 2]
   });

package/setup.py

@@ -36,11 +36,11 @@ setup(
         'Development Status :: 4 - Beta',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
         'License :: OSI Approved :: MIT License',
     ],
     packages=find_packages(exclude=[

package/start.py

@@ -7,7 +7,7 @@
 # --------------------------------------------------------------------------
 import sys
 if not sys.version_info >= (3, 7, 0):
-    raise Exception("Autorest for Python extension requires Python 3.7 at least")
+    raise Exception("Autorest for Python extension requires Python 3.8 at least")
 
 from pathlib import Path
 import venv