@autorest/python 6.1.5 → 6.1.7

package/autorest/__init__.py

@@ -7,7 +7,7 @@ import logging
 from pathlib import Path
 import json
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Union
+from typing import Any, Dict, Union, List
 
 import yaml
 
@@ -22,6 +22,7 @@ _LOGGER = logging.getLogger(__name__)
 class ReaderAndWriter:
     def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None:
         self.output_folder = Path(output_folder)
+        self._list_file: List[str] = []
         try:
             with open(
                 Path(self.output_folder) / Path("..") / Path("python.json"), "r"
@@ -37,7 +38,7 @@ class ReaderAndWriter:
         self.options.update(python_json)
 
     def read_file(self, path: Union[str, Path]) -> str:
-        """How does one read a file in cadl?"""
+        """Directly reading from disk"""
         # make path relative to output folder
         try:
             with open(self.output_folder / Path(path), "r") as fd:
@@ -46,13 +47,16 @@ class ReaderAndWriter:
             return ""
 
     def write_file(self, filename: Union[str, Path], file_content: str) -> None:
-        """How does writing work in cadl?"""
+        """Directly writing to disk"""
         file_folder = Path(filename).parent
         if not Path.is_dir(self.output_folder / file_folder):
             Path.mkdir(self.output_folder / file_folder, parents=True)
         with open(self.output_folder / Path(filename), "w") as fd:
             fd.write(file_content)
 
+    def list_file(self) -> List[str]:
+        return [str(f) for f in self.output_folder.glob("**/*") if f.is_file()]
+
 
 class ReaderAndWriterAutorest(ReaderAndWriter):
     def __init__(
@@ -67,6 +71,9 @@ class ReaderAndWriterAutorest(ReaderAndWriter):
     def write_file(self, filename: Union[str, Path], file_content: str) -> None:
         return self._autorestapi.write_file(filename, file_content)
 
+    def list_file(self) -> List[str]:
+        return self._autorestapi.list_inputs()
+
 
 class Plugin(ReaderAndWriter, ABC):
     """A base class for autorest plugin.

package/autorest/black/__init__.py

@@ -30,16 +30,10 @@ class BlackScriptPlugin(Plugin):  # pylint: disable=abstract-method
 
     def process(self) -> bool:
         # apply format_file on every file in the output folder
-        list(
-            map(
-                self.format_file,
-                [f for f in self.output_folder.glob("**/*") if f.is_file()],
-            )
-        )
+        list(map(self.format_file, [Path(f) for f in self.list_file()]))
         return True
 
-    def format_file(self, full_path) -> None:
-        file = full_path.relative_to(self.output_folder)
+    def format_file(self, file: Path) -> None:
         file_content = self.read_file(file)
         if not file.suffix == ".py":
             self.write_file(file, file_content)

package/autorest/codegen/__init__.py

@@ -44,9 +44,9 @@ def _validate_code_model_options(options: Dict[str, Any]) -> None:
             "or 'embedded'"
         )
 
-    if options["models_mode"] not in ["msrest", "none"]:
+    if options["models_mode"] not in ["msrest", "dpg", "none"]:
         raise ValueError(
-            "--models-mode can only be 'msrest' or 'none'. "
+            "--models-mode can only be 'msrest', 'dpg' or 'none'. "
             "Pass in 'msrest' if you want msrest models, or "
             "'none' if you don't want any."
         )
@@ -179,6 +179,8 @@ class CodeGenerator(Plugin):
         models_mode_default = (
             "none" if low_level_client or version_tolerant else "msrest"
         )
+        if self.options.get("cadl_file") is not None:
+            models_mode_default = "dpg"
 
         options: Dict[str, Any] = {
             "azure_arm": azure_arm,

package/autorest/codegen/models/__init__.py

@@ -158,7 +158,7 @@ def build_type(yaml_data: Dict[str, Any], code_model: CodeModel) -> BaseType:
         code_model.types_map[yaml_id] = response
         response.fill_instance_from_yaml(yaml_data, code_model)
     else:
-        object_type = yaml_data.get("type", None)
+        object_type = yaml_data.get("type")
         if object_type is None:
             _LOGGER.warning(
                 'Unrecognized definition type "%s" is found, falling back it as "string"! ',

package/autorest/codegen/models/base_builder.py

@@ -31,7 +31,9 @@ if TYPE_CHECKING:
 _LOGGER = logging.getLogger(__name__)
 
 
-class BaseBuilder(Generic[ParameterListType], BaseModel):
+class BaseBuilder(
+    Generic[ParameterListType], BaseModel
+):  # pylint: disable=too-many-instance-attributes
     """Base class for Operations and Request Builders"""
 
     def __init__(
@@ -56,6 +58,7 @@ class BaseBuilder(Generic[ParameterListType], BaseModel):
         self.group_name: str = yaml_data["groupName"]
         self.is_overload: bool = yaml_data["isOverload"]
         self.api_versions: List[str] = yaml_data["apiVersions"]
+        self.added_on: Optional[str] = yaml_data.get("addedOn")
 
         if code_model.options["version_tolerant"] and yaml_data.get("abstract"):
             _LOGGER.warning(

package/autorest/codegen/models/client.py

@@ -108,7 +108,6 @@ class Client(_ClientConfigBase[ClientGlobalParameterList]):
             f"{self.code_model.client.name}Configuration",
             ImportType.LOCAL,
         )
-
         file_import.add_msrest_import(
             self.code_model,
             ".." if async_mode else ".",
@@ -148,7 +147,10 @@ class Client(_ClientConfigBase[ClientGlobalParameterList]):
                 ImportType.LOCAL,
             )
 
-        if self.code_model.model_types and self.code_model.options["models_mode"]:
+        if (
+            self.code_model.model_types
+            and self.code_model.options["models_mode"] == "msrest"
+        ):
             path_to_models = ".." if async_mode else "."
             if len(self.code_model.model_types) != len(
                 self.code_model.public_model_types
@@ -167,7 +169,7 @@ class Client(_ClientConfigBase[ClientGlobalParameterList]):
                 file_import.add_submodule_import(
                     path_to_models, "models", ImportType.LOCAL
                 )
-        else:
+        elif self.code_model.options["models_mode"] == "msrest":
             # in this case, we have client_models = {} in the service client, which needs a type annotation
             # this import will always be commented, so will always add it to the typing section
             file_import.add_submodule_import(

package/autorest/codegen/models/combined_type.py

@@ -4,7 +4,7 @@
 # license information.
 # --------------------------------------------------------------------------
 from typing import Any, Dict, List, Optional, TYPE_CHECKING
-
+import re
 from autorest.codegen.models.imports import FileImport, ImportType
 from .base_type import BaseType
 
@@ -67,9 +67,11 @@ class CombinedType(BaseType):
         Special case for enum, for instance: Union[str, "EnumName"]
         """
         kwargs["is_operation_file"] = True
-        return (
-            f'Union[{", ".join(type.type_annotation(**kwargs) for type in self.types)}]'
-        )
+        inside_types = [type.type_annotation(**kwargs) for type in self.types]
+
+        # If the inside types has been a Union, peel first and then re-union
+        pattern = re.compile(r"Union\[.*\]")
+        return f'Union[{", ".join(map(lambda x: x[6: -1] if pattern.match(x) else x, inside_types))}]'
 
     def get_json_template_representation(
         self,

package/autorest/codegen/models/enum_type.py

@@ -107,10 +107,12 @@ class EnumType(BaseType):
         :rtype: str
         """
         if self.code_model.options["models_mode"]:
-            return (
-                f"Union[{self.value_type.type_annotation(**kwargs)},"
-                f' "_models.{self.name}"]'
-            )
+            model_name = f"_models.{self.name}"
+            # we don't need quoted annotation in operation files, and need it in model folder files.
+            if not kwargs.get("is_operation_file", False):
+                model_name = f'"{model_name}"'
+
+            return f"Union[{self.value_type.type_annotation(**kwargs)}, {model_name}]"
         return self.value_type.type_annotation(**kwargs)
 
     def get_declaration(self, value: Any) -> str:

package/autorest/codegen/models/model_type.py

@@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional, TYPE_CHECKING, cast
 
 from autorest.codegen.models.utils import add_to_pylint_disable
 from .base_type import BaseType
+from .constant_type import ConstantType
 from .property import Property
 from .imports import FileImport, ImportType, TypingSection
 
@@ -32,7 +33,9 @@ def _get_properties(type: "ModelType", properties: List[Property]) -> List[Prope
     return properties
 
 
-class ModelType(BaseType):  # pylint: disable=too-many-instance-attributes
+class ModelType(
+    BaseType
+):  # pylint: disable=too-many-instance-attributes, too-many-public-methods
     """Represents a class ready to be serialized in Python.
 
     :param str name: The name of the class.
@@ -71,14 +74,16 @@ class ModelType(BaseType):  # pylint: disable=too-many-instance-attributes
 
     @property
     def serialization_type(self) -> str:
-        if self.code_model.options["models_mode"]:
-            return (
-                self.name
-                if self.is_public
-                else f"{self.code_model.models_filename}.{self.name}"
-            )
+        if self.code_model.options["models_mode"] == "msrest":
+            return f"{'' if self.is_public else (self.code_model.models_filename + '.')}{self.name}"
+        if self.code_model.options["models_mode"] == "dpg":
+            return f"{'' if self.is_public else '_models.'}_models.{self.name}"
         return "object"
 
+    @property
+    def is_polymorphic(self) -> bool:
+        return any(p.is_polymorphic for p in self.properties)
+
     def type_annotation(self, **kwargs: Any) -> str:
         if self.code_model.options["models_mode"]:
             is_operation_file = kwargs.pop("is_operation_file", False)
@@ -229,10 +234,26 @@ class ModelType(BaseType):  # pylint: disable=too-many-instance-attributes
         except StopIteration:
             return None
 
+    @property
+    def discriminator_property(self) -> Optional[Property]:
+        try:
+            return next(
+                p
+                for p in self.properties
+                if p.is_discriminator
+                and isinstance(p.type, ConstantType)
+                and p.type.value == self.discriminator_value
+            )
+        except StopIteration:
+            return None
+
     @property
     def instance_check_template(self) -> str:
-        if self.code_model.options["models_mode"]:
+        models_mode = self.code_model.options["models_mode"]
+        if models_mode == "msrest":
             return "isinstance({}, msrest.Model)"
+        if models_mode == "dpg":
+            return "isinstance({}, _model_base.Model)"
         return "isinstance({}, MutableMapping)"
 
     @property
@@ -257,7 +278,7 @@ class ModelType(BaseType):  # pylint: disable=too-many-instance-attributes
             file_import.add_submodule_import(
                 relative_path, "models", ImportType.LOCAL, alias="_models"
             )
-        if self.code_model.options["models_mode"]:
+        if self.code_model.options["models_mode"] == "msrest":
             return file_import
         file_import.add_submodule_import(
             "typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL

package/autorest/codegen/models/operation.py

@@ -154,6 +154,14 @@ class OperationBase(  # pylint: disable=too-many-public-methods
         retval = self._response_docstring_helper("docstring_text", **kwargs)
         if not self.code_model.options["version_tolerant"]:
             retval += " or the result of cls(response)"
+        if self.code_model.options["models_mode"] == "dpg" and any(
+            isinstance(r.type, ModelType) for r in self.responses
+        ):
+            r = next(r for r in self.responses if isinstance(r.type, ModelType))
+            type_name = getattr(r, "item_type", getattr(r, "type")).docstring_text(
+                **kwargs
+            )
+            retval += f". The {type_name} is compatible with MutableMapping"
         return retval
 
     def response_docstring_type(self, **kwargs) -> str:
@@ -209,6 +217,12 @@ class OperationBase(  # pylint: disable=too-many-public-methods
             file_import.add_submodule_import(
                 "typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL
             )
+        if self.added_on:
+            file_import.add_submodule_import(
+                f"{'.' if async_mode else ''}.._validation",
+                "api_version_validation",
+                ImportType.LOCAL,
+            )
         return file_import
 
     def imports_for_multiapi(self, async_mode: bool, **kwargs: Any) -> FileImport:
@@ -244,6 +258,11 @@ class OperationBase(  # pylint: disable=too-many-public-methods
             for kwarg in kwargs_to_pop
         )
 
+    @property
+    def need_validation(self) -> bool:
+        """Whether we need parameter / operation validation. For API version."""
+        return bool(self.added_on) or any(p for p in self.parameters if p.added_on)
+
     def get_request_builder_import(
         self,
         request_builder: Union[RequestBuilder, OverloadedRequestBuilder],
@@ -450,6 +469,19 @@ class Operation(OperationBase[Response]):
             and not self.code_model.options["models_mode"]
         ):
             file_import.add_submodule_import("typing", "cast", ImportType.STDLIB)
+        if self.code_model.options["models_mode"] == "dpg":
+            relative_path = "..." if async_mode else ".."
+            if self.parameters.has_body:
+                file_import.add_submodule_import(
+                    f"{relative_path}_model_base", "AzureJSONEncoder", ImportType.LOCAL
+                )
+                file_import.add_import("json", ImportType.STDLIB)
+            if self.default_error_deserialization or any(
+                [r.type for r in self.responses]
+            ):
+                file_import.add_submodule_import(
+                    f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL
+                )
 
         return file_import
 

package/autorest/codegen/models/operation_group.py

@@ -66,6 +66,11 @@ class OperationGroup(BaseModel):
             return "  # type: ignore"
         return ""
 
+    @property
+    def need_validation(self) -> bool:
+        """Whether any of its operations need validation"""
+        return any(o for o in self.operations if o.need_validation)
+
     def imports(self, async_mode: bool) -> FileImport:
         file_import = FileImport()
 
@@ -75,13 +80,12 @@ class OperationGroup(BaseModel):
                 operation.imports(async_mode, relative_path=relative_path)
             )
         # for multiapi
-        if not self.code_model.options["version_tolerant"]:
-            if (
-                self.code_model.model_types or self.code_model.enums
-            ) and self.code_model.options["models_mode"]:
-                file_import.add_submodule_import(
-                    relative_path, "models", ImportType.LOCAL, alias="_models"
-                )
+        if (
+            self.code_model.model_types or self.code_model.enums
+        ) and self.code_model.options["models_mode"] == "msrest":
+            file_import.add_submodule_import(
+                relative_path, "models", ImportType.LOCAL, alias="_models"
+            )
         if self.code_model.need_mixin_abc:
             file_import.add_submodule_import(".._vendor", "MixinABC", ImportType.LOCAL)
         if self.has_abstract_operations:

package/autorest/codegen/models/parameter.py

@@ -5,6 +5,7 @@
 # --------------------------------------------------------------------------
 import abc
 from enum import Enum, auto
+import re
 
 from typing import (
     Dict,
@@ -21,6 +22,8 @@ from .imports import FileImport, ImportType
 from .base_model import BaseModel
 from .base_type import BaseType
 from .constant_type import ConstantType
+from .model_type import ModelType
+from .combined_type import CombinedType
 from .utils import add_to_description
 
 if TYPE_CHECKING:
@@ -82,6 +85,7 @@ class _ParameterBase(
         self.in_flattened_body: bool = self.yaml_data.get("inFlattenedBody", False)
         self.grouper: bool = self.yaml_data.get("grouper", False)
         self.check_client_input: bool = self.yaml_data.get("checkClientInput", False)
+        self.added_on: Optional[str] = self.yaml_data.get("addedOn")
 
     @property
     def constant(self) -> bool:
@@ -152,6 +156,12 @@ class _ParameterBase(
         )
         if self.optional and self.client_default_value is None:
             file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB)
+        if self.added_on:
+            file_import.add_submodule_import(
+                f"{'.' if async_mode else ''}.._validation",
+                "api_version_validation",
+                ImportType.LOCAL,
+            )
         return file_import
 
     @property
@@ -228,6 +238,40 @@ class BodyParameter(_BodyParameterBase):
             type=code_model.lookup_type(id(yaml_data["type"])),
         )
 
+    def type_annotation(self, **kwargs: Any) -> str:
+        annotation = super().type_annotation(**kwargs)
+        model_seq = BodyParameter.get_model_seq(self.type)
+        if self.code_model.options["models_mode"] == "dpg" and model_seq >= 0:
+            pattern = re.compile(r"Union\[.*\]")
+            union_content = (
+                annotation[6:-1] if pattern.match(annotation) else annotation
+            )
+            items = union_content.split(", ")
+            items.insert(model_seq + 1, "JSON")
+            annotation = f'Union[{", ".join(items)}]'
+        return annotation
+
+    def docstring_type(self, **kwargs: Any) -> str:
+        docstring = super().docstring_type(**kwargs)
+        model_seq = BodyParameter.get_model_seq(self.type)
+        if self.code_model.options["models_mode"] == "dpg" and model_seq >= 0:
+            items = docstring.split(" or ")
+            items.insert(model_seq + 1, "JSON")
+            docstring = " or ".join(items)
+        return docstring
+
+    @staticmethod
+    def get_model_seq(t: BaseType):
+        if isinstance(t, ModelType):
+            return 0
+        if isinstance(t, CombinedType):
+            sub_num = len(t.types)
+            for i in range(sub_num):
+                sub_seq = BodyParameter.get_model_seq(t.types[i])
+                if sub_seq >= 0:
+                    return i + sub_seq
+        return -1
+
 
 EntryBodyParameterType = TypeVar(
     "EntryBodyParameterType", bound=Union[BodyParameter, "RequestBuilderBodyParameter"]

package/autorest/codegen/models/property.py

@@ -29,6 +29,7 @@ class Property(BaseModel):  # pylint: disable=too-many-instance-attributes
         self.type = type
         self.optional: bool = self.yaml_data["optional"]
         self.readonly: bool = self.yaml_data.get("readonly", False)
+        self.is_polymorphic: bool = self.yaml_data.get("isPolymorphic", False)
         self.is_discriminator: bool = yaml_data.get("isDiscriminator", False)
         self.client_default_value = yaml_data.get("clientDefaultValue", None)
         if self.client_default_value is None:
@@ -45,7 +46,7 @@ class Property(BaseModel):  # pylint: disable=too-many-instance-attributes
     def description(self, *, is_operation_file: bool) -> str:
         from .model_type import ModelType
 
-        description = self.yaml_data["description"]
+        description = self.yaml_data.get("description", "")
         if not (self.optional or self.client_default_value):
             description = add_to_description(description, "Required.")
         # don't want model type documentation as part of property doc
@@ -124,20 +125,6 @@ class Property(BaseModel):  # pylint: disable=too-many-instance-attributes
         retval.update(self.type.validation or {})
         return retval or None
 
-    @property
-    def attribute_map(self) -> str:
-        if self.flattened_names:
-            attribute_key = ".".join(
-                n.replace(".", "\\\\.") for n in self.flattened_names
-            )
-        else:
-            attribute_key = self.rest_api_name.replace(".", "\\\\.")
-        if self.type.xml_serialization_ctxt:
-            xml_metadata = f", 'xml': {{{self.type.xml_serialization_ctxt}}}"
-        else:
-            xml_metadata = ""
-        return f'"{self.client_name}": {{"key": "{attribute_key}", "type": "{self.serialization_type}"{xml_metadata}}},'
-
     def imports(self) -> FileImport:
         from .model_type import ModelType
 
@@ -152,6 +139,12 @@ class Property(BaseModel):  # pylint: disable=too-many-instance-attributes
                 TypingSection.TYPING,
                 alias="_models",
             )
+        if self.code_model.options["models_mode"] == "dpg":
+            file_import.add_submodule_import(
+                ".._model_base",
+                "rest_discriminator" if self.is_discriminator else "rest_field",
+                ImportType.LOCAL,
+            )
         return file_import
 
     @classmethod

package/autorest/codegen/models/request_builder_parameter.py

@@ -24,8 +24,10 @@ class RequestBuilderBodyParameter(BodyParameter):
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
-        if isinstance(self.type, (BinaryType, StringType)) or any(
-            "xml" in ct for ct in self.content_types
+        if (
+            isinstance(self.type, (BinaryType, StringType))
+            or any("xml" in ct for ct in self.content_types)
+            or self.code_model.options["models_mode"] == "dpg"
         ):
             self.client_name = "content"
         else:

package/autorest/codegen/serializers/__init__.py

@@ -16,7 +16,7 @@ from ..models import TokenCredentialType
 from .enum_serializer import EnumSerializer
 from .general_serializer import GeneralSerializer
 from .model_init_serializer import ModelInitSerializer
-from .model_serializer import ModelSerializer
+from .model_serializer import DpgModelSerializer, MsrestModelSerializer
 from .operations_init_serializer import OperationsInitSerializer
 from .operation_groups_serializer import OperationGroupsSerializer
 from .metadata_serializer import MetadataSerializer
@@ -215,10 +215,15 @@ class JinjaSerializer(ReaderAndWriter):  # pylint: disable=abstract-method
     ) -> None:
         # Write the models folder
         models_path = namespace_path / Path("models")
+        serializer = (
+            DpgModelSerializer
+            if self.code_model.options["models_mode"] == "dpg"
+            else MsrestModelSerializer
+        )
         if self.code_model.model_types:
             self.write_file(
                 models_path / Path(f"{self.code_model.models_filename}.py"),
-                ModelSerializer(code_model=self.code_model, env=env).serialize(),
+                serializer(code_model=self.code_model, env=env).serialize(),
             )
         if self.code_model.enums:
             self.write_file(
@@ -427,6 +432,17 @@ class JinjaSerializer(ReaderAndWriter):  # pylint: disable=abstract-method
                 namespace_path / Path("_serialization.py"),
                 general_serializer.serialize_serialization_file(),
             )
+        if self.code_model.options["models_mode"] == "dpg":
+            self.write_file(
+                namespace_path / Path("_model_base.py"),
+                general_serializer.serialize_model_base_file(),
+            )
+
+        if any(og for og in self.code_model.operation_groups if og.need_validation):
+            self.write_file(
+                namespace_path / Path("_validation.py"),
+                general_serializer.serialize_validation_file(),
+            )
 
         # Write the config file
         if self.code_model.request_builders: