@autorest/python 5.14.0 → 5.17.0

package/autorest/postprocess/get_all.py

@@ -0,0 +1,19 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import sys
+import importlib
+
+
+def main(namespace):
+    sdk = importlib.import_module(namespace)
+    return sdk._patch.__all__  # pylint: disable=protected-access
+
+
+if __name__ == "__main__":
+    patched = ",".join(main(sys.argv[1]))
+    output_folder = sys.argv[2]
+    with open(f"{output_folder}/.temp_folder/patched.txt", "w") as f:
+        f.write(patched)

package/autorest/postprocess/venvtools.py

@@ -0,0 +1,73 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from typing import Optional
+import subprocess
+import venv
+import sys
+from pathlib import Path
+
+
+_ROOT_DIR = Path(__file__).parent
+
+
+class ExtendedEnvBuilder(venv.EnvBuilder):
+    """An extended env builder which saves the context, to have access
+    easily to bin path and such.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.context = None
+        super(ExtendedEnvBuilder, self).__init__(*args, **kwargs)
+
+    def ensure_directories(self, env_dir):
+        self.context = super(ExtendedEnvBuilder, self).ensure_directories(env_dir)
+        return self.context
+
+
+def create(
+    env_dir,
+    system_site_packages=False,
+    clear=False,
+    symlinks=False,
+    with_pip=False,
+    prompt=None,
+):
+    """Create a virtual environment in a directory."""
+    builder = ExtendedEnvBuilder(
+        system_site_packages=system_site_packages,
+        clear=clear,
+        symlinks=symlinks,
+        with_pip=with_pip,
+        prompt=prompt,
+    )
+    builder.create(env_dir)
+    return builder.context
+
+
+def python_run(  # pylint: disable=inconsistent-return-statements
+    venv_context, module, command, directory=_ROOT_DIR, *, error_ok=False
+) -> Optional[str]:
+    try:
+        cmd_line = [
+            venv_context.env_exe,
+            "-m",
+            module,
+        ] + command
+        print("Executing: {}".format(" ".join(cmd_line)))
+        subprocess.run(
+            cmd_line,
+            cwd=directory,
+            check=True,
+            stdout=False,
+        )
+        if module == "get_all":
+            with open(f"{command[1]}/.temp_folder/patched.txt", "r") as f:
+                return f.read()
+    except subprocess.CalledProcessError as err:
+        print(err)
+        if not error_ok:
+            sys.exit(1)
+    return None

package/autorest/preprocess/__init__.py

@@ -0,0 +1,209 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+"""The preprocessing autorest plugin.
+"""
+from typing import Callable, Dict, Any, List, Optional
+from .helpers import to_snake_case, pad_reserved_words, add_redefined_builtin_info
+from .python_mappings import PadType
+
+from .. import YamlUpdatePlugin
+
+
+def update_description(
+    description: Optional[str], default_description: str = ""
+) -> str:
+    if not description:
+        description = default_description
+    description.rstrip(" ")
+    if description and description[-1] != ".":
+        description += "."
+    return description
+
+
+def update_operation_group_class_name(
+    yaml_data: Dict[str, Any], class_name: str
+) -> str:
+    if class_name == "":
+        return yaml_data["client"]["name"] + "OperationsMixin"
+    if class_name == "Operations":
+        return "Operations"
+    return class_name + "Operations"
+
+
+def update_parameter(yaml_data: Dict[str, Any]) -> None:
+    yaml_data["description"] = update_description(yaml_data["description"])
+    if not (
+        yaml_data["location"] == "header"
+        and yaml_data["clientName"] in ("content_type", "accept")
+    ):
+        yaml_data["clientName"] = pad_reserved_words(
+            yaml_data["clientName"].lower(), PadType.PARAMETER
+        )
+    if yaml_data.get("propertyToParameterName"):
+        # need to create a new one with padded keys and values
+        yaml_data["propertyToParameterName"] = {
+            pad_reserved_words(prop, PadType.PROPERTY)
+            .lower(): pad_reserved_words(param_name, PadType.PARAMETER)
+            .lower()
+            for prop, param_name in yaml_data["propertyToParameterName"].items()
+        }
+
+
+def update_types(yaml_data: List[Dict[str, Any]]) -> None:
+    for type in yaml_data:
+        for property in type.get("properties", []):
+            property["description"] = update_description(property["description"])
+            property["clientName"] = pad_reserved_words(
+                property["clientName"].lower(), PadType.PROPERTY
+            )
+            add_redefined_builtin_info(property["clientName"], property)
+        if type.get("name"):
+            type["description"] = update_description(type["description"], type["name"])
+
+
+def update_client(yaml_data: Dict[str, Any]) -> None:
+    yaml_data["description"] = update_description(
+        yaml_data["description"], default_description=yaml_data["name"]
+    )
+    yaml_data["moduleName"] = to_snake_case(yaml_data["name"].replace(" ", "_"))
+    for parameter in yaml_data["parameters"]:
+        update_parameter(parameter)
+
+
+def update_paging_response(yaml_data: Dict[str, Any]) -> None:
+    yaml_data["discriminator"] = "paging"
+    yaml_data["pagerSync"] = yaml_data.get("pagerSync") or "azure.core.paging.ItemPaged"
+    yaml_data["pagerAsync"] = (
+        yaml_data.get("pagerAsync") or "azure.core.async_paging.AsyncItemPaged"
+    )
+
+
+class PreProcessPlugin(YamlUpdatePlugin):
+    """Add Python naming information."""
+
+    def get_operation_updater(
+        self, yaml_data: Dict[str, Any]
+    ) -> Callable[[Dict[str, Any]], None]:
+        if yaml_data["discriminator"] == "lropaging":
+            return self.update_lro_paging_operation
+        if yaml_data["discriminator"] == "lro":
+            return self.update_lro_operation
+        if yaml_data["discriminator"] == "paging":
+            return self.update_paging_operation
+        return self.update_operation
+
+    def update_operation(self, yaml_data: Dict[str, Any]) -> None:
+        yaml_data["groupName"] = pad_reserved_words(
+            yaml_data["groupName"], PadType.OPERATION_GROUP
+        )
+        yaml_data["groupName"] = to_snake_case(yaml_data["groupName"])
+        yaml_data["name"] = yaml_data["name"].lower()
+        yaml_data["name"] = pad_reserved_words(yaml_data["name"], PadType.METHOD)
+        yaml_data["description"] = update_description(
+            yaml_data["description"], yaml_data["name"]
+        )
+        yaml_data["summary"] = update_description(yaml_data.get("summary", ""))
+        for parameter in yaml_data["parameters"]:
+            update_parameter(parameter)
+        if yaml_data.get("bodyParameter"):
+            update_parameter(yaml_data["bodyParameter"])
+            for entry in yaml_data["bodyParameter"].get("entries", []):
+                update_parameter(entry)
+        for overload in yaml_data.get("overloads", []):
+            self.update_operation(overload)
+        for response in yaml_data.get("responses", []):
+            response["discriminator"] = "operation"
+
+    def _update_lro_operation_helper(self, yaml_data: Dict[str, Any]) -> None:
+        azure_arm = self._autorestapi.get_boolean_value("azure-arm", False)
+        for response in yaml_data.get("responses", []):
+            response["discriminator"] = "lro"
+            response["pollerSync"] = (
+                response.get("pollerSync") or "azure.core.polling.LROPoller"
+            )
+            response["pollerAsync"] = (
+                response.get("pollerAsync") or "azure.core.polling.AsyncLROPoller"
+            )
+            if not response.get("pollingMethodSync"):
+                response["pollingMethodSync"] = (
+                    "azure.mgmt.core.polling.arm_polling.ARMPolling"
+                    if azure_arm
+                    else "azure.core.polling.base_polling.LROBasePolling"
+                )
+            if not response.get("pollingMethodAsync"):
+                response["pollingMethodAsync"] = (
+                    "azure.mgmt.core.polling.async_arm_polling.AsyncARMPolling"
+                    if azure_arm
+                    else "azure.core.polling.async_base_polling.AsyncLROBasePolling"
+                )
+
+    def update_lro_paging_operation(self, yaml_data: Dict[str, Any]) -> None:
+        self.update_lro_operation(yaml_data)
+        self.update_paging_operation(yaml_data)
+        for response in yaml_data.get("responses", []):
+            response["discriminator"] = "lropaging"
+
+    def update_lro_operation(self, yaml_data: Dict[str, Any]) -> None:
+        self.update_operation(yaml_data)
+        self._update_lro_operation_helper(yaml_data)
+        for overload in yaml_data["overloads"]:
+            self._update_lro_operation_helper(overload)
+
+    def update_paging_operation(self, yaml_data: Dict[str, Any]) -> None:
+        self.update_operation(yaml_data)
+        if not yaml_data.get("pagerSync"):
+            yaml_data["pagerSync"] = "azure.core.paging.ItemPaged"
+        if not yaml_data.get("pagerAsync"):
+            yaml_data["pagerAsync"] = "azure.core.async_paging.AsyncItemPaged"
+        returned_response_object = (
+            yaml_data["nextOperation"]["responses"][0]
+            if yaml_data.get("nextOperation")
+            else yaml_data["responses"][0]
+        )
+        # if we're in version tolerant, hide the paging model
+        if self._autorestapi.get_boolean_value("version-tolerant"):
+            returned_response_object["type"]["isPublic"] = False
+        item_type = next(
+            p["type"]["elementType"]
+            for p in returned_response_object["type"]["properties"]
+            if p["restApiName"] == yaml_data["itemName"]
+        )
+        if yaml_data.get("nextOperation"):
+            yaml_data["nextOperation"]["groupName"] = pad_reserved_words(
+                yaml_data["nextOperation"]["groupName"], PadType.OPERATION_GROUP
+            )
+            yaml_data["nextOperation"]["groupName"] = to_snake_case(
+                yaml_data["nextOperation"]["groupName"]
+            )
+            for response in yaml_data["nextOperation"].get("responses", []):
+                update_paging_response(response)
+                response["itemType"] = item_type
+        for response in yaml_data.get("responses", []):
+            update_paging_response(response)
+            response["itemType"] = item_type
+        for overload in yaml_data.get("overloads", []):
+            self.update_paging_operation(overload)
+
+    def update_operation_groups(self, yaml_data: Dict[str, Any]) -> None:
+        operation_groups_yaml_data = yaml_data["operationGroups"]
+        for operation_group in operation_groups_yaml_data:
+            operation_group["propertyName"] = pad_reserved_words(
+                operation_group["propertyName"], PadType.OPERATION_GROUP
+            )
+            operation_group["propertyName"] = to_snake_case(
+                operation_group["propertyName"]
+            )
+            operation_group["className"] = update_operation_group_class_name(
+                yaml_data, operation_group["className"]
+            )
+            for operation in operation_group["operations"]:
+                self.get_operation_updater(operation)(operation)
+
+    def update_yaml(self, yaml_data: Dict[str, Any]) -> None:
+        """Convert in place the YAML str."""
+        update_client(yaml_data["client"])
+        update_types(yaml_data["types"])
+        self.update_operation_groups(yaml_data)

package/autorest/preprocess/helpers.py

@@ -0,0 +1,54 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from typing import Any, Dict
+import re
+from .python_mappings import PadType, RESERVED_WORDS, REDEFINED_BUILTINS
+
+
+def to_snake_case(name: str) -> str:
+    def replace_upper_characters(m) -> str:
+        match_str = m.group().lower()
+        if m.start() > 0 and name[m.start() - 1] == "_":
+            # we are good if a '_' already exists
+            return match_str
+        # the first letter should not have _
+        prefix = "_" if m.start() > 0 else ""
+
+        # we will add an extra _ if there are multiple upper case chars together
+        next_non_upper_case_char_location = m.start() + len(match_str)
+        if (
+            len(match_str) > 2
+            and len(name) - next_non_upper_case_char_location > 1
+            and name[next_non_upper_case_char_location].isalpha()
+        ):
+
+            return (
+                prefix
+                + match_str[: len(match_str) - 1]
+                + "_"
+                + match_str[len(match_str) - 1]
+            )
+
+        return prefix + match_str
+
+    return re.sub("[A-Z]+", replace_upper_characters, name)
+
+
+def pad_reserved_words(name: str, pad_type: PadType):
+    # we want to pad hidden variables as well
+    if not name:
+        # we'll pass in empty operation groups sometime etc.
+        return name
+    name_prefix = "_" if name[0] == "_" else ""
+    name = name[1:] if name[0] == "_" else name
+    if name.lower() in RESERVED_WORDS[pad_type]:
+        return name_prefix + name + pad_type
+    return name_prefix + name
+
+
+def add_redefined_builtin_info(name: str, yaml_data: Dict[str, Any]) -> None:
+    if name in REDEFINED_BUILTINS:
+        yaml_data["pylintDisable"] = "redefined-builtin"

package/autorest/{namer → preprocess}/python_mappings.py

@@ -50,14 +50,15 @@ basic_latin_chars = {
     "~": "Tilde",
 }
 
-class PadType(Enum):
-    Model = "Model"
-    Method = "Method"
-    Parameter = "Parameter"
-    Enum = "Enum"
-    Property = "Property"
-    OperationGroup = "Operations"
-    BuilderGroup = "Builders"
+
+class PadType(str, Enum):
+    MODEL = "Model"
+    METHOD = "_method"
+    PARAMETER = "_parameter"
+    ENUM = "_enum"
+    PROPERTY = "_property"
+    OPERATION_GROUP = "Operations"
+
 
 _always_reserved = [
     "and",
@@ -91,14 +92,12 @@ _always_reserved = [
     "with",
     "yield",
     "async",
-    "await"
+    "await",
 ]
 
-reserved_words = {
-    PadType.Method: [
-        *_always_reserved
-    ],
-    PadType.Parameter: [
+RESERVED_WORDS = {
+    PadType.METHOD: [*_always_reserved],
+    PadType.PARAMETER: [
         "self",
         # these are kwargs we've reserved for our autorest generated operations
         "content_type",
@@ -161,23 +160,17 @@ reserved_words = {
         "retry_backoff_max",
         "retry_mode",
         "retry_on_status_codes",
-        *_always_reserved
-    ],
-    PadType.Model: [
-        *_always_reserved
-    ],
-    PadType.Property: [
-        "self",
-        *_always_reserved
-    ],
-    PadType.Enum: [
-        "mro",
-        *_always_reserved
-    ],
-    PadType.OperationGroup: [
-        *_always_reserved
+        *_always_reserved,
     ],
-    PadType.BuilderGroup: [
-        *_always_reserved
-    ]
+    PadType.MODEL: [*_always_reserved],
+    PadType.PROPERTY: ["self", *_always_reserved],
+    PadType.ENUM: ["mro", *_always_reserved],
+    PadType.OPERATION_GROUP: [*_always_reserved],
 }
+
+REDEFINED_BUILTINS = [  # we don't pad, but we need to do lint ignores
+    "id",
+    "min",
+    "max",
+    "filter",
+]

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@autorest/python",
-    "version": "5.14.0",
+    "version": "5.17.0",
     "description": "The Python extension for generators in AutoRest.",
     "scripts": {
         "prepare": "node run-python3.js prepare.py",
@@ -24,10 +24,10 @@
     },
     "homepage": "https://github.com/Azure/autorest.python/blob/autorestv3/README.md",
     "dependencies": {
-        "@azure-tools/extension": "~3.2.1"
+        "@autorest/system-requirements": "~1.0.0"
     },
     "devDependencies": {
-        "@microsoft.azure/autorest.testserver": "^3.3.20"
+        "@microsoft.azure/autorest.testserver": "^3.3.26"
     },
     "files": [
         "autorest/**/*.py",

package/run-python3.js

@@ -7,11 +7,10 @@
 // Invoke it like so: "node run-python3.js script.py"
 
 const cp = require("child_process");
-const extension = require("@azure-tools/extension");
+const extension = require("@autorest/system-requirements");
 
 async function runPython3(scriptName, debug = "") {
-  const command = ["python", scriptName, debug];
-  await extension.updatePythonPath(command);
+  const command = await extension.patchPythonPath(["python", scriptName, debug], { version: ">=3.6", environmentVariable: "AUTOREST_PYTHON_EXE" });
   cp.execSync(command.join(" "), {
     stdio: [0, 1, 2]
   });

package/venvtools.py

@@ -21,7 +21,7 @@ class ExtendedEnvBuilder(venv.EnvBuilder):
 
     def __init__(self, *args, **kwargs):
         self.context = None
-        super(ExtendedEnvBuilder, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
     def ensure_directories(self, env_dir):
         self.context = super(ExtendedEnvBuilder, self).ensure_directories(env_dir)

package/autorest/codegen/models/constant_schema.py

@@ -1,97 +0,0 @@
-# -------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for
-# license information.
-# --------------------------------------------------------------------------
-import logging
-from typing import Dict, Any, Optional
-from .base_schema import BaseSchema
-from .primitive_schemas import get_primitive_schema, PrimitiveSchema
-from .imports import FileImport
-
-_LOGGER = logging.getLogger(__name__)
-
-
-class ConstantSchema(BaseSchema):
-    """Schema for constants that will be serialized.
-
-    :param yaml_data: the yaml data for this schema
-    :type yaml_data: dict[str, Any]
-    :param str value: The actual value of this constant.
-    :param schema: The schema for the value of this constant.
-    :type schema: ~autorest.models.PrimitiveSchema
-    """
-
-    def __init__(
-        self, namespace: str, yaml_data: Dict[str, Any], schema: PrimitiveSchema, value: Optional[str],
-    ) -> None:
-        super(ConstantSchema, self).__init__(namespace=namespace, yaml_data=yaml_data)
-        self.value = value
-        self.schema = schema
-
-    def get_declaration(self, value: Any):
-        if value != self.value:
-            _LOGGER.warning(
-                "Passed in value of %s differs from constant value of %s. Choosing constant value",
-                str(value), str(self.value)
-            )
-        if self.value is None:
-            return "None"
-        return self.schema.get_declaration(self.value)
-
-    @property
-    def serialization_type(self) -> str:
-        """Returns the serialization value for msrest.
-
-        :return: The serialization value for msrest
-        :rtype: str
-        """
-        return self.schema.serialization_type
-
-    @property
-    def docstring_text(self) -> str:
-        return "constant"
-
-    @property
-    def docstring_type(self) -> str:
-        """The python type used for RST syntax input and type annotation.
-
-        :param str namespace: Optional. The namespace for the models.
-        """
-        return self.schema.docstring_type
-
-    @property
-    def type_annotation(self) -> str:
-        return self.schema.type_annotation
-
-    @classmethod
-    def from_yaml(cls, namespace: str, yaml_data: Dict[str, Any], **kwargs) -> "ConstantSchema":
-        """Constructs a ConstantSchema from yaml data.
-
-        :param yaml_data: the yaml data from which we will construct this schema
-        :type yaml_data: dict[str, Any]
-
-        :return: A created ConstantSchema
-        :rtype: ~autorest.models.ConstantSchema
-        """
-        name = yaml_data["language"]["python"]["name"] if yaml_data["language"]["python"].get("name") else ""
-        _LOGGER.debug("Parsing %s constant", name)
-        return cls(
-            namespace=namespace,
-            yaml_data=yaml_data,
-            schema=get_primitive_schema(namespace=namespace, yaml_data=yaml_data["valueType"]),
-            value=yaml_data.get("value", {}).get("value", None),
-        )
-
-    def get_json_template_representation(self, **kwargs: Any) -> Any:
-        kwargs['default_value_declaration'] = self.schema.get_declaration(self.value)
-        return self.schema.get_json_template_representation(**kwargs)
-
-    def get_files_and_data_template_representation(self, **kwargs: Any) -> Any:
-        kwargs['default_value_declaration'] = self.schema.get_declaration(self.value)
-        return self.schema.get_files_and_data_template_representation(**kwargs)
-
-    def imports(self) -> FileImport:
-        file_import = FileImport()
-        file_import.merge(self.schema.imports())
-        return file_import

package/autorest/codegen/models/credential_schema.py

@@ -1,90 +0,0 @@
-# -------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for
-# license information.
-# --------------------------------------------------------------------------
-from typing import Any
-from .base_schema import BaseSchema
-from .imports import FileImport, ImportType, TypingSection
-
-class CredentialSchema(BaseSchema):
-    def __init__(self) -> None:  # pylint: disable=super-init-not-called
-        self.default_value = None
-
-    @property
-    def docstring_type(self) -> str:
-        return self.serialization_type
-
-    @property
-    def docstring_text(self) -> str:
-        return "credential"
-
-    @property
-    def serialization_type(self) -> str:
-        # this property is added, because otherwise pylint says that
-        # abstract serialization_type in BaseSchema is not overridden
-        pass
-
-    def get_json_template_representation(self, **kwargs: Any) -> Any:
-        raise TypeError("You should not try to get a JSON template representation of a CredentialSchema")
-
-    def get_files_and_data_template_representation(self, **kwargs: Any) -> Any:
-        raise TypeError("You should not try to get a files template representation of a CredentialSchema")
-
-
-class AzureKeyCredentialSchema(CredentialSchema):
-
-    @property
-    def serialization_type(self) -> str:
-        return "~azure.core.credentials.AzureKeyCredential"
-
-    @property
-    def type_annotation(self) -> str:
-        return "AzureKeyCredential"
-
-    def imports(self) -> FileImport:
-        file_import = FileImport()
-        file_import.add_submodule_import(
-            "azure.core.credentials",
-            "AzureKeyCredential",
-            ImportType.AZURECORE,
-            typing_section=TypingSection.CONDITIONAL
-        )
-        return file_import
-
-
-class TokenCredentialSchema(CredentialSchema):
-    def __init__(self, async_mode) -> None:
-        super(TokenCredentialSchema, self).__init__()
-        self.async_mode = async_mode
-        self.async_type = "~azure.core.credentials_async.AsyncTokenCredential"
-        self.sync_type = "~azure.core.credentials.TokenCredential"
-
-    @property
-    def serialization_type(self) -> str:
-        if self.async_mode:
-            return self.async_type
-        return self.sync_type
-
-    @property
-    def type_annotation(self) -> str:
-        if self.async_mode:
-            return '"AsyncTokenCredential"'
-        return '"TokenCredential"'
-
-
-    def imports(self) -> FileImport:
-        file_import = FileImport()
-        if self.async_mode:
-            file_import.add_submodule_import(
-                "azure.core.credentials_async", "AsyncTokenCredential",
-                ImportType.AZURECORE,
-                typing_section=TypingSection.TYPING
-            )
-        else:
-            file_import.add_submodule_import(
-                "azure.core.credentials", "TokenCredential",
-                ImportType.AZURECORE,
-                typing_section=TypingSection.TYPING
-            )
-        return file_import