@autorest/python 6.18.0 → 6.19.0

package/autorest/m4reformatter/__init__.py

@@ -28,6 +28,20 @@ KEY_TYPE = "Key"
 _LOGGER = logging.getLogger(__name__)
 
 
+def get_item_type(yaml_data: Dict[str, Any], item_name: str, enable_exception: bool = True) -> Optional[Dict[str, Any]]:
+    try:
+        return next(p["type"] for p in yaml_data.get("properties", []) if p["wireName"] == item_name)
+    except StopIteration:
+        pass
+    for parent in yaml_data.get("parents", []):
+        result = get_item_type(parent, item_name, False)
+        if result:
+            return result
+    if enable_exception:
+        raise StopIteration(f"Could not find item type {item_name} from type {yaml_data['name']}")
+    return None
+
+
 def is_body(yaml_data: Dict[str, Any]) -> bool:
     """Return true if passed in parameter is a body param"""
     return yaml_data["protocol"]["http"]["in"] == "body"
@@ -544,9 +558,7 @@ class M4Reformatter(YamlUpdatePluginAutorest):  # pylint: disable=too-many-publi
         if self.version_tolerant:
             # if we're in version tolerant, hide the paging model
             returned_response_object["type"]["internal"] = True
-        operation["itemType"] = next(
-            p["type"] for p in returned_response_object["type"]["properties"] if p["wireName"] == operation["itemName"]
-        )
+        operation["itemType"] = get_item_type(returned_response_object["type"], operation["itemName"])
         if yaml_data["language"]["default"]["paging"].get("nextLinkOperation"):
             operation["nextOperation"] = self.update_operation(
                 group_name=group_name,

package/generator/pygen/codegen/models/__init__.py

@@ -30,6 +30,7 @@ from .primitive_types import (
     UnixTimeType,
     SdkCoreType,
     DecimalType,
+    MultiPartFileType,
 )
 from .enum_type import EnumType, EnumValue
 from .base import BaseType
@@ -149,6 +150,7 @@ TYPE_TO_OBJECT = {
     "unixtime": UnixTimeType,
     "credential": StringType,
     "sdkcore": SdkCoreType,
+    "multipartfile": MultiPartFileType,
 }
 _LOGGER = logging.getLogger(__name__)
 

package/generator/pygen/codegen/models/code_model.py

@@ -7,7 +7,7 @@ from typing import List, Dict, Any, Set, Union, Literal
 
 from .base import BaseType
 from .enum_type import EnumType
-from .model_type import ModelType
+from .model_type import ModelType, UsageFlags
 from .combined_type import CombinedType
 from .client import Client
 from .request_builder import RequestBuilder, OverloadedRequestBuilder
@@ -162,9 +162,7 @@ class CodeModel:  # pylint: disable=too-many-public-methods, disable=too-many-in
         """All of the model types in this class"""
         if not self._model_types:
             self._model_types = [
-                t
-                for t in self.types_map.values()
-                if isinstance(t, ModelType) and not (self.options["models_mode"] == "dpg" and t.page_result_model)
+                t for t in self.types_map.values() if isinstance(t, ModelType) and t.usage != UsageFlags.Default.value
             ]
         return self._model_types
 

package/generator/pygen/codegen/models/model_type.py

@@ -84,9 +84,8 @@ class ModelType(  # pylint: disable=abstract-method
         self._got_polymorphic_subtypes = False
         self.internal: bool = self.yaml_data.get("internal", False)
         self.snake_case_name: str = self.yaml_data["snakeCaseName"]
-        self.page_result_model: bool = self.yaml_data.get("pageResultModel", False)
         self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId")
-        self.usage: int = self.yaml_data.get("usage", 0)
+        self.usage: int = self.yaml_data.get("usage", UsageFlags.Input.value | UsageFlags.Output.value)
 
     @property
     def is_usage_output(self) -> bool:

package/generator/pygen/codegen/models/operation.py

@@ -418,8 +418,19 @@ class OperationBase(  # pylint: disable=too-many-public-methods,too-many-instanc
         file_import.merge(self.get_request_builder_import(self.request_builder, async_mode))
         if self.overloads:
             file_import.add_submodule_import("typing", "overload", ImportType.STDLIB)
-        if self.non_default_errors and self.code_model.options["models_mode"] == "dpg":
-            file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL)
+        if self.code_model.options["models_mode"] == "dpg":
+            if self.parameters.has_body:
+                if self.has_form_data_body:
+                    file_import.add_submodule_import(relative_path, "_model_base", ImportType.LOCAL)
+                else:
+                    file_import.add_submodule_import(
+                        f"{relative_path}_model_base",
+                        "SdkJSONEncoder",
+                        ImportType.LOCAL,
+                    )
+                    file_import.add_import("json", ImportType.STDLIB)
+            if (self.default_error_deserialization or any(r.type for r in self.responses)) or self.non_default_errors:
+                file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL)
         return file_import
 
     def get_response_from_status(self, status_code: Optional[Union[str, int]]) -> ResponseType:
@@ -492,20 +503,6 @@ class Operation(OperationBase[Response]):
             )
         if self.has_response_body and not self.has_optional_return_type and not self.code_model.options["models_mode"]:
             file_import.add_submodule_import("typing", "cast", ImportType.STDLIB)
-        relative_path = "..." if async_mode else ".."
-        if self.code_model.options["models_mode"] == "dpg":
-            if self.parameters.has_body:
-                if self.has_form_data_body:
-                    file_import.add_submodule_import(relative_path, "_model_base", ImportType.LOCAL)
-                else:
-                    file_import.add_submodule_import(
-                        f"{relative_path}_model_base",
-                        "SdkJSONEncoder",
-                        ImportType.LOCAL,
-                    )
-                    file_import.add_import("json", ImportType.STDLIB)
-            if self.default_error_deserialization or any(r.type for r in self.responses):
-                file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL)
 
         return file_import
 

package/generator/pygen/codegen/models/primitive_types.py

@@ -624,3 +624,29 @@ class SdkCoreType(PrimitiveType):
     @property
     def serialization_type(self) -> str:
         return self.name
+
+
+class MultiPartFileType(PrimitiveType):
+    def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None:
+        super().__init__(yaml_data=yaml_data, code_model=code_model)
+        self.name = "FileType"
+
+    def type_annotation(self, **kwargs: Any) -> str:
+        return self.name
+
+    def docstring_type(self, **kwargs: Any) -> str:
+        return f"~{self.code_model.namespace}._vendor.{self.name}"
+
+    def imports(self, **kwargs: Any) -> FileImport:
+        file_import = super().imports(**kwargs)
+        relative_path = "..." if kwargs.get("async_mode") else ".."
+        file_import.add_submodule_import(f"{relative_path}_vendor", self.name, ImportType.LOCAL)
+        return file_import
+
+    @property
+    def default_template_representation_declaration(self) -> str:
+        return '"filetype"' if self.code_model.for_test else "filetype"
+
+    @property
+    def instance_check_template(self) -> str:
+        return f"isinstance({{}}, {self.name})"

package/generator/pygen/codegen/models/property.py

@@ -97,12 +97,9 @@ class Property(BaseModel):  # pylint: disable=too-many-instance-attributes
         return self.is_discriminator and self.is_polymorphic and cast(ConstantType, self.type).value is None
 
     def type_annotation(self, *, is_operation_file: bool = False) -> str:
-        types_type_annotation = self.type.type_annotation(is_operation_file=is_operation_file)
-        if self.is_multipart_file_input:
-            # we only support FileType or list of FileType
-            types_type_annotation = types_type_annotation.replace("bytes", "FileType")
         if self.is_base_discriminator:
             return "str"
+        types_type_annotation = self.type.type_annotation(is_operation_file=is_operation_file)
         if self.optional and self.client_default_value is None:
             return f"Optional[{types_type_annotation}]"
         return types_type_annotation
@@ -115,9 +112,6 @@ class Property(BaseModel):  # pylint: disable=too-many-instance-attributes
         *,
         client_default_value_declaration: Optional[str] = None,
     ) -> Any:
-        if self.is_multipart_file_input:
-            file_type_str = '"filetype"' if self.code_model.for_test else "filetype"
-            return f"[{file_type_str}]" if self.type.type == "list" else file_type_str
         if self.client_default_value:
             client_default_value_declaration = self.get_declaration(self.client_default_value)
         # make sure there is no \n otherwise the json template will be invalid
@@ -156,8 +150,6 @@ class Property(BaseModel):  # pylint: disable=too-many-instance-attributes
                 "rest_discriminator" if self.is_discriminator else "rest_field",
                 ImportType.LOCAL,
             )
-        if self.is_multipart_file_input:
-            file_import.add_submodule_import(".._vendor", "FileType", ImportType.LOCAL)
         return file_import
 
     @classmethod

package/generator/pygen/codegen/serializers/builder_serializer.py

@@ -535,9 +535,9 @@ class _OperationSerializer(_BuilderBaseSerializer[OperationType]):  # pylint: di
         return "response"
 
     def example_template(self, builder: OperationType) -> List[str]:
+        if self.code_model.options["models_mode"] in ("msrest", "dpg"):
+            return []
         retval = super().example_template(builder)
-        if self.code_model.options["models_mode"] == "msrest":
-            return retval
         for response in builder.responses:
             polymorphic_subtypes: List[ModelType] = []
             if not response.type:

package/generator/pygen/codegen/serializers/general_serializer.py

@@ -124,7 +124,6 @@ class GeneralSerializer(BaseSerializer):
             file_import.add_submodule_import("typing", "Union", ImportType.STDLIB)
             file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB)
             file_import.add_submodule_import("typing", "Mapping", ImportType.STDLIB)
-            file_import.add_submodule_import("typing", "Sequence", ImportType.STDLIB)
             file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB)
             file_import.add_submodule_import("typing", "Any", ImportType.STDLIB)
             file_import.add_submodule_import("typing", "List", ImportType.STDLIB)

package/generator/pygen/codegen/serializers/model_serializer.py

@@ -194,6 +194,8 @@ class DpgModelSerializer(_ModelSerializer):
         )
 
         for model in self.code_model.model_types:
+            if model.base == "json":
+                continue
             file_import.merge(model.imports(is_operation_file=False))
             for prop in model.properties:
                 file_import.merge(prop.imports())

package/generator/pygen/codegen/serializers/sample_serializer.py

@@ -42,7 +42,7 @@ class SampleSerializer(BaseSerializer):
         self.operation = operation
         self.sample = sample
         self.file_name = file_name
-        self.sample_params = {to_snake_case(k): v for k, v in sample.get("parameters", {}).items()}
+        self.sample_params = sample.get("parameters", {})
 
     def _imports(self) -> FileImportSerializer:
         imports = FileImport(self.code_model)
@@ -66,8 +66,8 @@ class SampleSerializer(BaseSerializer):
                 "AzureKeyCredential",
                 ImportType.SDKCORE,
             )
-        for param in self.operation.parameters.positional:
-            if not param.client_default_value and not param.optional and param.client_name in self.sample_params:
+        for param in self.operation.parameters.positional + self.operation.parameters.keyword_only:
+            if not param.client_default_value and not param.optional and param.wire_name in self.sample_params:
                 imports.merge(param.type.imports_for_sample())
         return FileImportSerializer(imports, True)
 
@@ -80,15 +80,19 @@ class SampleSerializer(BaseSerializer):
         elif isinstance(credential_type, KeyCredentialType):
             special_param.update({"credential": 'AzureKeyCredential(key=os.getenv("AZURE_KEY"))'})
 
-        params_positional = [
-            p for p in self.code_model.clients[0].parameters.positional if not (p.optional or p.client_default_value)
+        params = [
+            p
+            for p in (
+                self.code_model.clients[0].parameters.positional + self.code_model.clients[0].parameters.keyword_only
+            )
+            if not (p.optional or p.client_default_value)
         ]
         client_params = {
             p.client_name: special_param.get(
                 p.client_name,
-                f'"{self.sample_params.get(p.client_name) or p.client_name.upper()}"',
+                f'"{self.sample_params.get(p.wire_name) or p.client_name.upper()}"',
             )
-            for p in params_positional
+            for p in params
         }
 
         return client_params
@@ -103,15 +107,18 @@ class SampleSerializer(BaseSerializer):
 
     # prepare operation parameters
     def _operation_params(self) -> Dict[str, Any]:
-        params_positional = [p for p in self.operation.parameters.positional if not p.client_default_value]
+        params = [
+            p
+            for p in (self.operation.parameters.positional + self.operation.parameters.keyword_only)
+            if not p.client_default_value
+        ]
         failure_info = "fail to find required param named {}"
         operation_params = {}
-        for param in params_positional:
-            name = param.client_name
-            param_value = self.sample_params.get(name)
+        for param in params:
             if not param.optional:
+                param_value = self.sample_params.get(param.wire_name)
                 if not param_value:
-                    raise Exception(failure_info.format(name))  # pylint: disable=broad-exception-raised
+                    raise Exception(failure_info.format(param.client_name))  # pylint: disable=broad-exception-raised
                 operation_params[param.client_name] = self.handle_param(param, param_value)
         return operation_params
 

package/generator/pygen/codegen/templates/model_base.py.jinja2

@@ -489,6 +489,9 @@ def _create_value(rf: typing.Optional["_RestField"], value: typing.Any) -> typin
 
 class Model(_MyMutableMapping):
     _is_model = True
+    # label whether current class's _attr_to_rest_field has been calculated
+    # could not see _attr_to_rest_field directly because subclass inherits it from parent class
+    _calculated: typing.Set[str] = set()
 
     def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None:
         class_name = self.__class__.__name__
@@ -521,24 +524,27 @@ class Model(_MyMutableMapping):
         return Model(self.__dict__)
 
     def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self:  # pylint: disable=unused-argument
-        # we know the last three classes in mro are going to be 'Model', 'dict', and 'object'
-        mros = cls.__mro__[:-3][::-1]  # ignore model, dict, and object parents, and reverse the mro order
-        attr_to_rest_field: typing.Dict[str, _RestField] = {  # map attribute name to rest_field property
-            k: v for mro_class in mros for k, v in mro_class.__dict__.items() if k[0] != "_" and hasattr(v, "_type")
-        }
-        annotations = {
-            k: v
-            for mro_class in mros
-            if hasattr(mro_class, "__annotations__")  # pylint: disable=no-member
-            for k, v in mro_class.__annotations__.items()  # pylint: disable=no-member
-        }
-        for attr, rf in attr_to_rest_field.items():
-            rf._module = cls.__module__
-            if not rf._type:
-                rf._type = rf._get_deserialize_callable_from_annotation(annotations.get(attr, None))
-            if not rf._rest_name_input:
-                rf._rest_name_input = attr
-        cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items())
+        if f"{cls.__module__}.{cls.__qualname__}" not in cls._calculated:
+            # we know the last nine classes in mro are going to be 'Model', '_MyMutableMapping', 'MutableMapping', 
+            # 'Mapping', 'Collection', 'Sized', 'Iterable', 'Container' and 'object'
+            mros = cls.__mro__[:-9][::-1]  # ignore parents, and reverse the mro order
+            attr_to_rest_field: typing.Dict[str, _RestField] = {  # map attribute name to rest_field property
+                k: v for mro_class in mros for k, v in mro_class.__dict__.items() if k[0] != "_" and hasattr(v, "_type")
+            }
+            annotations = {
+                k: v
+                for mro_class in mros
+                if hasattr(mro_class, "__annotations__")  # pylint: disable=no-member
+                for k, v in mro_class.__annotations__.items()  # pylint: disable=no-member
+            }
+            for attr, rf in attr_to_rest_field.items():
+                rf._module = cls.__module__
+                if not rf._type:
+                    rf._type = rf._get_deserialize_callable_from_annotation(annotations.get(attr, None))
+                if not rf._rest_name_input:
+                    rf._rest_name_input = attr
+            cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items())
+            cls._calculated.add(f"{cls.__module__}.{cls.__qualname__}")
 
         return super().__new__(cls)  # pylint: disable=no-value-for-parameter
 

package/generator/pygen/codegen/templates/vendor.py.jinja2

@@ -70,8 +70,6 @@ FileType = Union[
     Tuple[Optional[str], FileContent, Optional[str]],
 ]
 
-FilesType = Union[Mapping[str, FileType], Sequence[Tuple[str, FileType]]]
-
 def serialize_multipart_data_entry(data_entry: Any) -> Any:
     if isinstance(data_entry, (list, tuple, dict, Model)):
         return json.dumps(data_entry, cls=SdkJSONEncoder, exclude_readonly=True)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorest/python",
-  "version": "6.18.0",
+  "version": "6.19.0",
   "description": "The Python extension for generators in AutoRest.",
   "main": "index.js",
   "repository": {
@@ -20,12 +20,13 @@
   "homepage": "https://github.com/Azure/autorest.python/blob/main/README.md",
   "dependencies": {
     "@autorest/system-requirements": "~1.0.2",
-    "fs-extra": "~11.2.0"
+    "fs-extra": "~11.2.0",
+    "tsx": "4.17.0"
   },
   "devDependencies": {
     "@microsoft.azure/autorest.testserver": "^3.3.46",
     "typescript": "~5.1.3",
-    "@azure-tools/typespec-python": "^0.28.0"
+    "@azure-tools/typespec-python": "^0.29.0"
   },
   "files": [
     "autorest/**/*.py",
@@ -39,6 +40,7 @@
     "start": "node ./scripts/run-python3.js ./scripts/start.py",
     "build": "node ./scripts/copy-generator.js --force",
     "install": "node ./scripts/copy-generator.js && node ./scripts/run-python3.js ./scripts/install.py",
-    "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug"
+    "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug",
+    "test": "tsx ./scripts/run-tests.ts"
   }
 }

package/scripts/run-tests.ts

@@ -0,0 +1,81 @@
+/* eslint-disable no-console */
+import { execSync } from "child_process";
+import { readFileSync } from "fs";
+import { join } from "path";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+
+interface Arguments {
+    folder?: string;
+    command?: string;
+}
+
+const validFolders = [
+    "azure/legacy",
+    "azure/version-tolerant",
+    "vanilla/legacy",
+    "vanilla/version-tolerant",
+    "dpg/version-tolerant",
+];
+
+const validCommands = ["ci", "lint", "mypy", "pyright", "apiview"];
+
+// Parse command-line arguments using yargs
+const argv = yargs(hideBin(process.argv))
+    .option("folder", {
+        alias: "f",
+        describe: "Specify the folder to use",
+        choices: validFolders,
+        type: "string",
+    })
+    .option("command", {
+        alias: "c",
+        describe: "Specify the command to run",
+        choices: validCommands,
+        type: "string",
+    }).argv as Arguments;
+
+const foldersToProcess = argv.folder ? [argv.folder] : validFolders;
+
+const commandToRun = argv.command || "all";
+
+function getCommand(command: string, folder: string) {
+    if (!validCommands.includes(command)) throw new Error(`Unknown command '${command}'.`);
+    return `FOLDER=${folder} tox -c ./test/${folder}/tox.ini -e ${command}`;
+}
+
+function sectionExistsInToxIni(command: string, folder: string): boolean {
+    const toxIniPath = join(__dirname, `../test/${folder}/tox.ini`);
+    const toxIniContent = readFileSync(toxIniPath, "utf-8");
+    const sectionHeader = `[testenv:${command}]`;
+    return toxIniContent.includes(sectionHeader);
+}
+
+function myExecSync(command: string, folder: string): void {
+    if (!sectionExistsInToxIni(command, folder)) {
+        console.log(`No section for ${command} in tox.ini for folder ${folder}. Skipping...`);
+        return;
+    }
+    execSync(getCommand(command, folder), { stdio: "inherit" });
+}
+
+foldersToProcess.forEach((folder) => {
+    try {
+        if (commandToRun === "all") {
+            for (const key of validCommands) {
+                console.log(`Running ${key} for folder ${folder}...`);
+                myExecSync(key, folder);
+            }
+        } else if (getCommand(commandToRun, folder)) {
+            console.log(`Running ${commandToRun} for folder ${folder}...`);
+            myExecSync(commandToRun, folder);
+        } else {
+            console.error(`Error: Unknown command '${commandToRun}'.`);
+            process.exit(1);
+        }
+    } catch (error) {
+        console.error((error as Error).message);
+        console.error(`Error executing command for folder ${folder}: ${(error as Error).message}`);
+        process.exit(1);
+    }
+});