@autorest/python 6.37.2 → 6.38.1

package/generator/build/lib/pygen/codegen/serializers/builder_serializer.py

@@ -1032,7 +1032,7 @@ class _OperationSerializer(_BuilderBaseSerializer[OperationType]):
                             is_operation_file=True, skip_quote=True, serialize_namespace=self.serialize_namespace
                         )
                         if self.code_model.options["models-mode"] == "dpg":
-                            retval.append(f"        error = _failsafe_deserialize({type_annotation},  response.json())")
+                            retval.append(f"        error = _failsafe_deserialize({type_annotation},  response)")
                         else:
                             retval.append(
                                 f"        error = self._deserialize.failsafe_deserialize({type_annotation}, "
@@ -1068,10 +1068,10 @@ class _OperationSerializer(_BuilderBaseSerializer[OperationType]):
                     if self.code_model.options["models-mode"] == "dpg":
                         if xml_serializable(str(e.default_content_type)):
                             retval.append(
-                                f"        error = _failsafe_deserialize_xml({type_annotation},  response.text())"
+                                f"        error = _failsafe_deserialize_xml({type_annotation}, response)"
                             )
                         else:
-                            retval.append(f"        error = _failsafe_deserialize({type_annotation},  response.json())")
+                            retval.append(f"        error = _failsafe_deserialize({type_annotation}, response)")
                     else:
                         retval.append(
                             f"        error = self._deserialize.failsafe_deserialize({type_annotation}, "
@@ -1079,18 +1079,19 @@ class _OperationSerializer(_BuilderBaseSerializer[OperationType]):
                         )
                     condition = "elif"
         # default error handling
-        if builder.default_error_deserialization and self.code_model.options["models-mode"]:
+        default_error_deserialization = builder.default_error_deserialization(self.serialize_namespace)
+        if default_error_deserialization and self.code_model.options["models-mode"]:
             error_model = ", model=error"
             indent = "        " if builder.non_default_errors else "    "
             if builder.non_default_errors:
                 retval.append("    else:")
             if self.code_model.options["models-mode"] == "dpg":
                 retval.append(
-                    f"{indent}error = _failsafe_deserialize({builder.default_error_deserialization},  response.json())"
+                    f"{indent}error = _failsafe_deserialize({default_error_deserialization}, response)"
                 )
             else:
                 retval.append(
-                    f"{indent}error = self._deserialize.failsafe_deserialize({builder.default_error_deserialization}, "
+                    f"{indent}error = self._deserialize.failsafe_deserialize({default_error_deserialization}, "
                     "pipeline_response)"
                 )
         retval.append(

package/generator/build/lib/pygen/codegen/serializers/general_serializer.py

@@ -4,7 +4,10 @@
 # license information.
 # --------------------------------------------------------------------------
 import json
-from typing import Any, List
+from typing import Any, List, TYPE_CHECKING
+import re
+import tomli as tomllib
+from packaging.version import parse as parse_version
 from .import_serializer import FileImportSerializer, TypingSection
 from ..models.imports import MsrestImportType, FileImport
 from ..models import (
@@ -16,6 +19,9 @@ from ..models.utils import NamespaceType
 from .client_serializer import ClientSerializer, ConfigSerializer
 from .base_serializer import BaseSerializer
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
 VERSION_MAP = {
     "msrest": "0.7.1",
     "isodate": "0.6.1",
@@ -42,8 +48,73 @@ class GeneralSerializer(BaseSerializer):
         params.update({"options": self.code_model.options})
         return template.render(code_model=self.code_model, **params)
 
-    def serialize_package_file(self, template_name: str, **kwargs: Any) -> str:
+    def _extract_min_dependency(self, s):
+        # Extract the minimum version from a dependency string.
+        #
+        # Handles formats like:
+        # - >=1.2.3
+        # - >=0.1.0b1 (beta versions)
+        # - >=1.2.3rc2 (release candidates)
+        #
+        # Returns the parsed version if found, otherwise version "0".
+        m = re.search(r"[>=]=?([\d.]+(?:[a-z]+\d+)?)", s)
+        return parse_version(m.group(1)) if m else parse_version("0")
+
+    def _keep_pyproject_fields(self, file_path: "Path") -> dict:
+        # Load the pyproject.toml file if it exists and extract fields to keep.
+        result: dict = {"KEEP_FIELDS": {}}
+        try:
+            with open(file_path, "rb") as f:
+                loaded_pyproject_toml = tomllib.load(f)
+        except Exception:  # pylint: disable=broad-except
+            # If parsing the pyproject.toml fails, we assume the it does not exist or is incorrectly formatted.
+            return result
+
+        # Keep azure-sdk-build configuration
+        if "tool" in loaded_pyproject_toml and "azure-sdk-build" in loaded_pyproject_toml["tool"]:
+            result["KEEP_FIELDS"]["tool.azure-sdk-build"] = loaded_pyproject_toml["tool"]["azure-sdk-build"]
+
+        # Process dependencies
+        if "project" in loaded_pyproject_toml:
+            # Handle main dependencies
+            if "dependencies" in loaded_pyproject_toml["project"]:
+                kept_deps = []
+                for dep in loaded_pyproject_toml["project"]["dependencies"]:
+                    dep_name = re.split(r"[<>=\[]", dep)[0].strip()
+
+                    # Check if dependency is one we track in VERSION_MAP
+                    if dep_name in VERSION_MAP:
+                        # For tracked dependencies, check if the version is higher than our default
+                        default_version = parse_version(VERSION_MAP[dep_name])
+                        dep_version = self._extract_min_dependency(dep)
+                        # If the version is higher than the default, update VERSION_MAP
+                        # with higher min dependency version
+                        if dep_version > default_version:
+                            VERSION_MAP[dep_name] = str(dep_version)
+                    else:
+                        # Keep non-default dependencies
+                        kept_deps.append(dep)
+
+                if kept_deps:
+                    result["KEEP_FIELDS"]["project.dependencies"] = kept_deps
+
+            # Keep optional dependencies
+            if "optional-dependencies" in loaded_pyproject_toml["project"]:
+                result["KEEP_FIELDS"]["project.optional-dependencies"] = loaded_pyproject_toml["project"][
+                    "optional-dependencies"
+                ]
+
+        return result
+
+    def serialize_package_file(self, template_name: str, file_path: "Path", **kwargs: Any) -> str:
         template = self.env.get_template(template_name)
+
+        # Add fields to keep from an existing pyproject.toml
+        if template_name == "pyproject.toml.jinja2":
+            params = self._keep_pyproject_fields(file_path)
+        else:
+            params = {}
+
         package_parts = (
             self.code_model.namespace.split(".")[:-1]
             if self.code_model.is_tsp
@@ -57,17 +128,19 @@ class GeneralSerializer(BaseSerializer):
             dev_status = "4 - Beta"
         else:
             dev_status = "5 - Production/Stable"
-        params = {
-            "code_model": self.code_model,
-            "dev_status": dev_status,
-            "token_credential": token_credential,
-            "pkgutil_names": [".".join(package_parts[: i + 1]) for i in range(len(package_parts))],
-            "init_names": ["/".join(package_parts[: i + 1]) + "/__init__.py" for i in range(len(package_parts))],
-            "client_name": self.code_model.clients[0].name if self.code_model.clients else "",
-            "VERSION_MAP": VERSION_MAP,
-            "MIN_PYTHON_VERSION": MIN_PYTHON_VERSION,
-            "MAX_PYTHON_VERSION": MAX_PYTHON_VERSION,
-        }
+        params.update(
+            {
+                "code_model": self.code_model,
+                "dev_status": dev_status,
+                "token_credential": token_credential,
+                "pkgutil_names": [".".join(package_parts[: i + 1]) for i in range(len(package_parts))],
+                "init_names": ["/".join(package_parts[: i + 1]) + "/__init__.py" for i in range(len(package_parts))],
+                "client_name": self.code_model.clients[0].name if self.code_model.clients else "",
+                "VERSION_MAP": VERSION_MAP,
+                "MIN_PYTHON_VERSION": MIN_PYTHON_VERSION,
+                "MAX_PYTHON_VERSION": MAX_PYTHON_VERSION,
+            }
+        )
         params.update({"options": self.code_model.options})
         params.update(kwargs)
         return template.render(file_import=FileImport(self.code_model), **params)

package/generator/build/lib/pygen/codegen/templates/model_base.py.jinja2

@@ -25,6 +25,7 @@ from {{ code_model.core_library }}.exceptions import DeserializationError
 from {{ code_model.core_library }}{{ "" if code_model.is_azure_flavor else ".utils" }} import CaseInsensitiveEnumMeta
 from {{ code_model.core_library }}.{{ "" if code_model.is_azure_flavor else "runtime." }}pipeline import PipelineResponse
 from {{ code_model.core_library }}.serialization import _Null
+from {{ code_model.core_library }}.rest import HttpResponse
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -935,13 +936,13 @@ def _deserialize(
 
 def _failsafe_deserialize(
   deserializer: typing.Any,
-  value: typing.Any,
+  response: HttpResponse,
   module: typing.Optional[str] = None,
   rf: typing.Optional["_RestField"] = None,
   format: typing.Optional[str] = None,
 ) -> typing.Any:
   try:
-      return _deserialize(deserializer, value, module, rf, format)
+      return _deserialize(deserializer, response.json(), module, rf, format)
   except DeserializationError:
       _LOGGER.warning(
           "Ran into a deserialization error. Ignoring since this is failsafe deserialization",
@@ -952,10 +953,10 @@ def _failsafe_deserialize(
 
 def _failsafe_deserialize_xml(
   deserializer: typing.Any,
-  value: typing.Any,
+  response: HttpResponse, 
 ) -> typing.Any:
   try:
-      return _deserialize_xml(deserializer, value)
+      return _deserialize_xml(deserializer, response.text())
   except DeserializationError:
       _LOGGER.warning(
           "Ran into a deserialization error. Ignoring since this is failsafe deserialization",

package/generator/build/lib/pygen/codegen/templates/packaging_templates/pyproject.toml.jinja2

@@ -0,0 +1,109 @@
+{% set min_version = MIN_PYTHON_VERSION.split('.')[1] | int %}
+{% set max_version = MAX_PYTHON_VERSION.split('.')[1] | int %}
+{% if code_model.license_header %}
+{{ code_model.license_header }}
+{% endif %}
+
+[build-system]
+requires = ["setuptools>=77.0.3", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "{{ options.get('package-name')|lower }}"
+{% if options.get('package-mode') %}
+authors = [
+  { name = "{{ code_model.company_name }}"{% if code_model.is_azure_flavor %}, email = "azpysdkhelp@microsoft.com"{% endif %} },
+]
+description = "{{ code_model.company_name }} {% if code_model.is_azure_flavor and not options.get('package-pprint-name').startswith('Azure ') %}Azure {% endif %}{{ options.get('package-pprint-name') }} Client Library for Python"
+license = "MIT"
+classifiers = [
+    "Development Status :: {{ dev_status }}",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3",
+    {% for version in range(min_version, max_version + 1) %}
+    "Programming Language :: Python :: 3.{{ version }}",
+    {% endfor %}
+]
+requires-python = ">={{ MIN_PYTHON_VERSION }}"
+{% else %}
+description = "{{ options.get('package-name') }}"
+{% endif %}
+{% if code_model.is_azure_flavor %}
+keywords = ["azure", "azure sdk"]
+{% endif %}
+
+dependencies = [
+    {% if code_model.is_legacy %}
+    "msrest>={{ VERSION_MAP['msrest'] }}",
+    {% else %}
+    "isodate>={{ VERSION_MAP['isodate'] }}",
+    {% endif %}
+    {% if options.get('azure_arm') %}
+    "azure-mgmt-core>={{ VERSION_MAP['azure-mgmt-core'] }}",
+    {% elif code_model.is_azure_flavor %}
+    "azure-core>={{ VERSION_MAP['azure-core'] }}",
+    {% else %}
+    "corehttp[requests]>={{ VERSION_MAP['corehttp'] }}",
+    {% endif %}
+    "typing-extensions>={{ VERSION_MAP['typing-extensions'] }}",
+    {% if KEEP_FIELDS and KEEP_FIELDS.get('project.dependencies') %}
+    {% for dep in KEEP_FIELDS.get('project.dependencies') %}
+    "{{ dep }}",
+    {% endfor %}
+    {% endif %}
+]
+dynamic = [
+    {% if options.get('package-mode') %}"version", {% endif %}"readme"
+]
+{% if not options.get('package-mode') %}
+version = "{{ options.get("package-version", "unknown") }}"
+{% endif %}
+{% if KEEP_FIELDS and KEEP_FIELDS.get('project.optional-dependencies') %}
+
+[project.optional-dependencies]
+{% for key, val in KEEP_FIELDS.get('project.optional-dependencies').items() %}
+{{ key }} = [
+  {% for dep in val %}
+  "{{ dep }}",
+  {% endfor %}
+]
+{% endfor %}
+{% endif %}
+{% if code_model.is_azure_flavor %}
+
+[project.urls]
+repository = "https://github.com/Azure/azure-sdk-for-python/tree/main/sdk"
+{% endif %}
+
+[tool.setuptools.dynamic]
+{% if options.get('package-mode') %}
+    {% if code_model.is_tsp %}
+version = {attr = "{{ code_model.namespace|lower }}._version.VERSION"}
+    {% else %}
+version = {attr = "{{ options.get('package-name')|lower|replace('-', '.') }}._version.VERSION"}
+    {% endif %}
+{% endif %}
+readme = {file = ["README.md"], content-type = "text/markdown"}
+{% if options.get('package-mode') %}
+
+[tool.setuptools.packages.find]
+exclude = [
+    "tests*",
+    "samples*",
+    "doc*",
+    {% for pkgutil_name in pkgutil_names %}
+    "{{ pkgutil_name }}",
+    {% endfor %}
+]
+
+[tool.setuptools.package-data]
+pytyped = ["py.typed"]
+{% endif %}
+{% if KEEP_FIELDS and KEEP_FIELDS.get('tool.azure-sdk-build') %}
+
+[tool.azure-sdk-build]
+{% for key, val in KEEP_FIELDS.get('tool.azure-sdk-build').items() %}
+{{ key }} = {{ val|tojson }}
+{% endfor %}
+{% endif %}

package/generator/pygen/__init__.py

@@ -25,10 +25,10 @@ class OptionsDict(MutableMapping):
         "azure-arm": False,
         "basic-setup-py": False,
         "client-side-validation": False,
-        "emit-cross-language-definition-file": False,
         "flavor": "azure",  # need to default to azure in shared code so we don't break swagger generation
         "from-typespec": False,
         "generate-sample": False,
+        "keep-setup-py": False,
         "generate-test": False,
         "head-as-boolean": True,
         "keep-version-file": False,
@@ -39,13 +39,14 @@ class OptionsDict(MutableMapping):
         "polymorphic-examples": 5,
         "validate-versioning": True,
         "version-tolerant": True,
+        "generation-subdir": None,  # subdirectory to generate the code in
     }
 
     def __init__(self, options: Optional[Dict[str, Any]] = None) -> None:
         self._data = options.copy() if options else {}
         self._validate_combinations()
 
-    def __getitem__(self, key: str) -> Any:
+    def __getitem__(self, key: str) -> Any: # pylint: disable=too-many-return-statements
         if key == "head-as-boolean" and self.get("azure-arm"):
             # override to always true if azure-arm is set
             return True
@@ -60,6 +61,18 @@ class OptionsDict(MutableMapping):
         if key == "package-mode" and self._data.get("packaging-files-dir"):
             # if packaging-files-dir is set, use it as package-mode
             return self._data["packaging-files-dir"]
+        if key == "generation-subdir":
+            data = self._data.get("generation-subdir")
+            if data:
+                # Remove leading dot or ./ from generation-subdir
+                if data.startswith("./"):
+                    data = data[2:]
+                elif data.startswith("."):
+                    data = data[1:]
+                # Remove trailing slashes
+                if data.endswith("/") or data.endswith("\\"):
+                    data = data[:-1]
+            return data
         return self._get_default(key)
 
     def __setitem__(self, key: str, value: Any) -> None:
@@ -106,6 +119,8 @@ class OptionsDict(MutableMapping):
                 models_mode_default = "dpg"
             # switch to falsy value for easier code writing
             return models_mode_default
+        if key == "emit-cross-language-definition-file":
+            return self.get("flavor") == "azure"
         return self.DEFAULTS[key]
 
     def _validate_combinations(self) -> None:
@@ -134,6 +149,10 @@ class OptionsDict(MutableMapping):
                 "We are working on creating a new multiapi SDK for version tolerant and it is not available yet."
             )
 
+        # If multiapi, do not generate default pyproject.toml
+        if self.get("multiapi"):
+            self["keep-setup-py"] = True
+
         if self.get("client-side-validation") and self.get("version-tolerant"):
             raise ValueError("Can not generate version tolerant with --client-side-validation. ")
 
@@ -210,6 +229,9 @@ class ReaderAndWriter:
             _LOGGER.warning("Loading python.json file. This behavior will be depreacted")
         self.options.update(python_json)
 
+    def get_output_folder(self) -> Path:
+        return self.output_folder
+
     def read_file(self, path: Union[str, Path]) -> str:
         """Directly reading from disk"""
         # make path relative to output folder
@@ -227,6 +249,14 @@ class ReaderAndWriter:
         with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd:
             fd.write(file_content)
 
+    def remove_file(self, filename: Union[str, Path]) -> None:
+        try:
+            file_path = self.output_folder / Path(filename)
+            if file_path.is_file():
+                file_path.unlink()
+        except FileNotFoundError:
+            pass
+
     def list_file(self) -> List[str]:
         return [str(f.relative_to(self.output_folder)) for f in self.output_folder.glob("**/*") if f.is_file()]
 

package/generator/pygen/codegen/models/code_model.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
+from pathlib import Path
 from typing import List, Dict, Any, Set, Union, Literal, Optional, cast
 
 from .base import BaseType
@@ -452,3 +453,34 @@ class CodeModel:  # pylint: disable=too-many-public-methods, disable=too-many-in
             return self.yaml_data.get("licenseInfo", {}).get("company", "")
         # typespec azure case without custom license and swagger case
         return "Microsoft Corporation"
+
+    def get_root_dir(self) -> Path:
+        if self.options["no-namespace-folders"]:
+            # when output folder contains parts different from the namespace, we fall back to current folder directly.
+            return Path(".")
+        return Path(*self.namespace.split("."))
+
+    def get_generation_dir(self, namespace: str) -> Path:
+        """The directory to generate the code in.
+
+        If 'generation-subdir' is specified, it will be used as a subdirectory.
+        """
+        root_dir = self.get_root_dir()
+        retval = self._get_relative_generation_dir(root_dir, namespace)
+        return retval
+
+    def _get_relative_generation_dir(self, root_dir: Path, namespace: str) -> Path:
+        if self.options["no-namespace-folders"]:
+            return Path(".")
+        if self.options.get("generation-subdir"):
+            # For the main namespace, return root_dir + generation-subdir
+            if namespace in ("", self.namespace):
+                return root_dir / self.options["generation-subdir"]
+
+            # For subnamespaces, extract the subnamespace part and append it to generation-subdir
+            if namespace.startswith(self.namespace + "."):
+                subnamespace_parts = namespace[len(self.namespace) + 1 :].split(".")
+                return root_dir / self.options["generation-subdir"] / Path(*subnamespace_parts)
+
+        # No generation-subdir specified, use the namespace path directly
+        return Path(*namespace.split("."))

package/generator/pygen/codegen/models/enum_type.py

@@ -52,7 +52,10 @@ class EnumValue(BaseType):
         """The python type used for RST syntax input and type annotation."""
 
         type_annotation = self.value_type.type_annotation(**kwargs)
-        enum_type_annotation = f"{self.enum_type.client_namespace}.models.{self.name}"
+        client_namespace = self.enum_type.client_namespace
+        if self.code_model.options.get("generation-subdir"):
+            client_namespace += f".{self.code_model.options['generation-subdir']}"
+        enum_type_annotation = f"{client_namespace}.models.{self.name}"
         return f"{type_annotation} or ~{enum_type_annotation}"
 
     def get_json_template_representation(

package/generator/pygen/codegen/models/model_type.py

@@ -292,7 +292,10 @@ class GeneratedModelType(ModelType):
 
     def docstring_type(self, **kwargs: Any) -> str:
         type_annotation = self.type_annotation(need_model_alias=False, skip_quote=True, **kwargs)
-        return f"~{self.client_namespace}.models.{type_annotation}"
+        client_namespace = self.client_namespace
+        if self.code_model.options.get("generation-subdir"):
+            client_namespace += f".{self.code_model.options['generation-subdir']}"
+        return f"~{client_namespace}.models.{type_annotation}"
 
     def docstring_text(self, **kwargs: Any) -> str:
         return self.name

package/generator/pygen/codegen/models/operation.py

@@ -194,14 +194,13 @@ class OperationBase(  # pylint: disable=too-many-public-methods,too-many-instanc
     def any_response_has_headers(self) -> bool:
         return any(response.headers for response in self.responses)
 
-    @property
-    def default_error_deserialization(self) -> Optional[str]:
+    def default_error_deserialization(self, serialize_namespace: str) -> Optional[str]:
         default_exceptions = [e for e in self.exceptions if "default" in e.status_codes and e.type]
         if not default_exceptions:
             return None
         exception_schema = default_exceptions[0].type
         if isinstance(exception_schema, ModelType):
-            return exception_schema.type_annotation(skip_quote=True)
+            return exception_schema.type_annotation(skip_quote=True, serialize_namespace=serialize_namespace)
         return None if self.code_model.options["models-mode"] == "dpg" else "'object'"
 
     @property
@@ -469,7 +468,7 @@ class OperationBase(  # pylint: disable=too-many-public-methods,too-many-instanc
                 file_import.add_submodule_import(relative_path, "_deserialize_xml", ImportType.LOCAL)
             elif self.need_deserialize:
                 file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL)
-            if self.default_error_deserialization or self.non_default_errors:
+            if self.default_error_deserialization(serialize_namespace) or self.non_default_errors:
                 file_import.add_submodule_import(relative_path, "_failsafe_deserialize", ImportType.LOCAL)
         return file_import
 

package/generator/pygen/codegen/models/paging_operation.py

@@ -173,7 +173,7 @@ class PagingOperationBase(OperationBase[PagingResponseType]):
                 serialize_namespace, module_name="_utils.model_base"
             )
             file_import.merge(self.item_type.imports(**kwargs))
-            if self.default_error_deserialization or self.need_deserialize:
+            if self.default_error_deserialization(serialize_namespace) or self.need_deserialize:
                 file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL)
         return file_import