@azure-tools/typespec-python 0.24.3 → 0.25.0

package/generator/pygen/preprocess/python_mappings.py

@@ -0,0 +1,222 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from enum import Enum
+
+basic_latin_chars = {
+    " ": "Space",
+    "!": "ExclamationMark",
+    '"': "QuotationMark",
+    "#": "NumberSign",
+    "$": "DollarSign",
+    "%": "PercentSign",
+    "&": "Ampersand",
+    "'": "Apostrophe",
+    "(": "LeftParenthesis",
+    ")": "RightParenthesis",
+    "*": "Asterisk",
+    "+": "PlusSign",
+    ",": "Comma",
+    "-": "HyphenMinus",
+    ".": "FullStop",
+    "/": "Slash",
+    "0": "Zero",
+    "1": "One",
+    "2": "Two",
+    "3": "Three",
+    "4": "Four",
+    "5": "Five",
+    "6": "Six",
+    "7": "Seven",
+    "8": "Eight",
+    "9": "Nine",
+    ":": "Colon",
+    ";": "Semicolon",
+    "<": "LessThanSign",
+    "=": "EqualSign",
+    ">": "GreaterThanSign",
+    "?": "QuestionMark",
+    "@": "AtSign",
+    "[": "LeftSquareBracket",
+    "\\": "Backslash",
+    "]": "RightSquareBracket",
+    "^": "CircumflexAccent",
+    "`": "GraveAccent",
+    "{": "LeftCurlyBracket",
+    "|": "VerticalBar",
+    "}": "RightCurlyBracket",
+    "~": "Tilde",
+}
+
+
+class PadType(str, Enum):
+    MODEL = "Model"
+    METHOD = "_method"
+    PARAMETER = "_parameter"
+    ENUM = "_enum"
+    PROPERTY = "_property"
+    OPERATION_GROUP = "Operations"
+
+
+_always_reserved = [
+    "and",
+    "as",
+    "assert",
+    "break",
+    "class",
+    "continue",
+    "def",
+    "del",
+    "elif",
+    "else",
+    "except",
+    "exec",
+    "finally",
+    "for",
+    "from",
+    "global",
+    "if",
+    "import",
+    "in",
+    "is",
+    "lambda",
+    "not",
+    "or",
+    "pass",
+    "raise",
+    "return",
+    "try",
+    "while",
+    "with",
+    "yield",
+    "async",
+    "await",
+    "int",
+]
+
+RESERVED_MODEL_PROPERTIES = [
+    "keys",
+    "items",
+    "values",
+    "popitem",
+    "clear",
+    "update",
+    "setdefault",
+    "pop",
+    "get",
+]
+
+RESERVED_WORDS = {
+    PadType.METHOD: [*_always_reserved],
+    PadType.PARAMETER: [
+        "self",
+        # these are kwargs we've reserved for our generated operations
+        "content_type",
+        "accept",
+        "cls",
+        "polling",
+        "continuation_token",  # for LRO calls
+        # these are transport kwargs
+        # https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/CLIENT_LIBRARY_DEVELOPER.md#transport
+        "connection_timeout",
+        "connection_verify",
+        "connection_cert",
+        "connection_data_block_size",
+        "use_env_settings",
+        # the following aren't in the readme, but Xiang said these are also transport kwargs
+        "read_timeout",
+        "proxies",
+        "cookies",
+        # these are policy kwargs
+        # https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/CLIENT_LIBRARY_DEVELOPER.md#available-policies
+        "base_headers",
+        "headers",
+        "request_id",
+        "auto_request_id",
+        "base_user_agent",
+        "user_agent",
+        "user_agent_overwrite",
+        "user_agent_use_env",
+        "user_agent",
+        "sdk_moniker",
+        "logging_enable",
+        "logger",
+        "response_encoding",
+        "proxies",
+        "raw_request_hook",
+        "raw_response_hook",
+        "network_span_namer",
+        "tracing_attributes",
+        "permit_redirects",
+        "redirect_max",
+        "redirect_remove_headers",
+        "redirect_on_status_codes",
+        "permit_redirects",
+        "redirect_max",
+        "redirect_remove_headers",
+        "redirect_on_status_codes",
+        "retry_total",
+        "retry_connect",
+        "retry_read",
+        "retry_status",
+        "retry_backoff_factor",
+        "retry_backoff_max",
+        "retry_mode",
+        "retry_on_status_codes",
+        "retry_total",
+        "retry_connect",
+        "retry_read",
+        "retry_status",
+        "retry_backoff_factor",
+        "retry_backoff_max",
+        "retry_mode",
+        "retry_on_status_codes",
+        *_always_reserved,
+    ],
+    PadType.MODEL: [*_always_reserved],
+    PadType.PROPERTY: ["self", *_always_reserved],
+    PadType.ENUM: ["mro", *_always_reserved],
+    PadType.OPERATION_GROUP: [*_always_reserved],
+}
+
+CADL_RESERVED_WORDS = {
+    PadType.PARAMETER: ["stream"],
+    PadType.PROPERTY: RESERVED_MODEL_PROPERTIES,
+}
+
+REDEFINED_BUILTINS = [  # we don't pad, but we need to do lint ignores
+    "id",
+    "min",
+    "max",
+    "filter",
+    "property",
+]
+
+BUILTIN_PACKAGES = [
+    "array",
+    "atexit",
+    "binascii",
+    "builtins",
+    "cmath",
+    "errno",
+    "faulthandler",
+    "fcntl",
+    "gc",
+    "grp",
+    "itertools",
+    "marshal",
+    "math",
+    "posix",
+    "pwd",
+    "pyexpat",
+    "select",
+    "spwd",
+    "sys",
+    "syslog",
+    "time",
+    "unicodedata",
+    "xxsubtype",
+    "zlib",
+]

package/generator/pygen/utils.py

@@ -0,0 +1,149 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from typing import Any, Dict, Tuple, List
+import re
+import argparse
+
+
+def update_enum_value(name: str, value: Any, description: str, enum_type: Dict[str, Any]) -> Dict[str, Any]:
+    return {
+        "name": name,
+        "type": "enumvalue",
+        "value": value,
+        "description": description,
+        "enumType": enum_type,
+        "valueType": enum_type["valueType"],
+    }
+
+
+def to_snake_case(name: str) -> str:
+    def replace_upper_characters(m) -> str:
+        match_str = m.group().lower()
+        if m.start() > 0 and name[m.start() - 1] == "_":
+            # we are good if a '_' already exists
+            return match_str
+        # the first letter should not have _
+        prefix = "_" if m.start() > 0 else ""
+
+        # we will add an extra _ if there are multiple upper case chars together
+        next_non_upper_case_char_location = m.start() + len(match_str)
+        if (
+            len(match_str) > 2
+            and len(name) - next_non_upper_case_char_location > 1
+            and name[next_non_upper_case_char_location].isalpha()
+        ):
+            return prefix + match_str[: len(match_str) - 1] + "_" + match_str[len(match_str) - 1]
+
+        return prefix + match_str
+
+    result = re.sub("[A-Z]+", replace_upper_characters, name)
+    return result.replace(" ", "_").replace("__", "_").replace("-", "")
+
+
+def parse_args(
+    need_cadl_file: bool = True,
+) -> Tuple[argparse.Namespace, Dict[str, Any]]:
+    parser = argparse.ArgumentParser(
+        description="Run mypy against target folder. Add a local custom plugin to the path prior to execution. "
+    )
+    parser.add_argument(
+        "--output-folder",
+        dest="output_folder",
+        help="Output folder for generated SDK",
+        required=True,
+    )
+    parser.add_argument(
+        "--cadl-file",
+        dest="cadl_file",
+        help="Serialized cadl file",
+        required=need_cadl_file,
+    )
+    parser.add_argument(
+        "--debug",
+        dest="debug",
+        help="Debug mode",
+        required=False,
+        action="store",
+    )
+    args, unknown_args = parser.parse_known_args()
+
+    def _get_value(value: Any) -> Any:
+        if value == "true":
+            return True
+        if value == "false":
+            return False
+        try:
+            return int(value)
+        except ValueError:
+            pass
+        return value
+
+    unknown_args_ret = {
+        ua.strip("--").split("=", maxsplit=1)[0]: _get_value(  # pylint: disable=bad-str-strip-call
+            ua.strip("--").split("=", maxsplit=1)[1]  # pylint: disable=bad-str-strip-call
+        )
+        for ua in unknown_args
+    }
+    return args, unknown_args_ret
+
+
+def get_body_type_for_description(body_parameter: Dict[str, Any]) -> str:
+    if body_parameter["type"]["type"] == "binary":
+        return "binary"
+    if body_parameter["type"]["type"] == "string":
+        return "string"
+    return "JSON"
+
+
+# used if we want to get a string / binary type etc
+KNOWN_TYPES: Dict[str, Dict[str, Any]] = {
+    "string": {"type": "string"},
+    "binary": {"type": "binary"},
+    "anydict": {"type": "dict", "elementType": {"type": "any"}},
+    "any-object": {"type": "any-object"},
+}
+
+JSON_REGEXP = re.compile(r"^(application|text)/(.+\+)?json$")
+
+
+def build_policies(
+    is_arm: bool,
+    async_mode: bool,
+    *,
+    is_azure_flavor: bool = False,
+    tracing: bool = True,
+) -> List[str]:
+    if is_azure_flavor:
+        # for Azure
+        async_prefix = "Async" if async_mode else ""
+        policies = [
+            "policies.RequestIdPolicy(**kwargs)",
+            "self._config.headers_policy",
+            "self._config.user_agent_policy",
+            "self._config.proxy_policy",
+            "policies.ContentDecodePolicy(**kwargs)",
+            (f"{async_prefix}ARMAutoResourceProviderRegistrationPolicy()" if is_arm else None),
+            "self._config.redirect_policy",
+            "self._config.retry_policy",
+            "self._config.authentication_policy",
+            "self._config.custom_hook_policy",
+            "self._config.logging_policy",
+            "policies.DistributedTracingPolicy(**kwargs)" if tracing else None,
+            "policies.SensitiveHeaderCleanupPolicy(**kwargs) if self._config.redirect_policy else None",
+            "self._config.http_logging_policy",
+        ]
+    else:
+        # for non-Azure
+        policies = [
+            "self._config.headers_policy",
+            "self._config.user_agent_policy",
+            "self._config.proxy_policy",
+            "policies.ContentDecodePolicy(**kwargs)",
+            "self._config.retry_policy",
+            "self._config.authentication_policy",
+            "self._config.logging_policy",
+        ]
+    return [p for p in policies if p]

package/generator/pygen.egg-info/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.1
+Name: pygen
+Version: 0.1.0
+Summary: Core Library for Python Generation
+Home-page: https://github.com/Azure/autorest.python/packages/core
+Author: Microsoft Corporation
+Author-email: azpysdkhelp@microsoft.com
+License: MIT License
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: Jinja2>=2.11
+Requires-Dist: pyyaml
+Requires-Dist: m2r2
+Requires-Dist: black
+
+# Core Library for Python Generation

package/generator/pygen.egg-info/SOURCES.txt

@@ -0,0 +1,66 @@
+LICENSE
+README.md
+setup.py
+pygen/__init__.py
+pygen/_version.py
+pygen/black.py
+pygen/m2r.py
+pygen/utils.py
+pygen.egg-info/PKG-INFO
+pygen.egg-info/SOURCES.txt
+pygen.egg-info/dependency_links.txt
+pygen.egg-info/requires.txt
+pygen.egg-info/top_level.txt
+pygen/codegen/__init__.py
+pygen/codegen/_utils.py
+pygen/codegen/models/__init__.py
+pygen/codegen/models/base.py
+pygen/codegen/models/base_builder.py
+pygen/codegen/models/client.py
+pygen/codegen/models/code_model.py
+pygen/codegen/models/combined_type.py
+pygen/codegen/models/constant_type.py
+pygen/codegen/models/credential_types.py
+pygen/codegen/models/dictionary_type.py
+pygen/codegen/models/enum_type.py
+pygen/codegen/models/imports.py
+pygen/codegen/models/list_type.py
+pygen/codegen/models/lro_operation.py
+pygen/codegen/models/lro_paging_operation.py
+pygen/codegen/models/model_type.py
+pygen/codegen/models/operation.py
+pygen/codegen/models/operation_group.py
+pygen/codegen/models/paging_operation.py
+pygen/codegen/models/parameter.py
+pygen/codegen/models/parameter_list.py
+pygen/codegen/models/primitive_types.py
+pygen/codegen/models/property.py
+pygen/codegen/models/request_builder.py
+pygen/codegen/models/request_builder_parameter.py
+pygen/codegen/models/response.py
+pygen/codegen/models/utils.py
+pygen/codegen/serializers/__init__.py
+pygen/codegen/serializers/base_serializer.py
+pygen/codegen/serializers/builder_serializer.py
+pygen/codegen/serializers/client_serializer.py
+pygen/codegen/serializers/enum_serializer.py
+pygen/codegen/serializers/general_serializer.py
+pygen/codegen/serializers/import_serializer.py
+pygen/codegen/serializers/metadata_serializer.py
+pygen/codegen/serializers/model_init_serializer.py
+pygen/codegen/serializers/model_serializer.py
+pygen/codegen/serializers/operation_groups_serializer.py
+pygen/codegen/serializers/operations_init_serializer.py
+pygen/codegen/serializers/parameter_serializer.py
+pygen/codegen/serializers/patch_serializer.py
+pygen/codegen/serializers/request_builders_serializer.py
+pygen/codegen/serializers/sample_serializer.py
+pygen/codegen/serializers/test_serializer.py
+pygen/codegen/serializers/types_serializer.py
+pygen/codegen/serializers/utils.py
+pygen/postprocess/__init__.py
+pygen/postprocess/get_all.py
+pygen/postprocess/venvtools.py
+pygen/preprocess/__init__.py
+pygen/preprocess/helpers.py
+pygen/preprocess/python_mappings.py

package/generator/pygen.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

package/generator/pygen.egg-info/requires.txt

@@ -0,0 +1,4 @@
+Jinja2>=2.11
+pyyaml
+m2r2
+black

package/generator/pygen.egg-info/top_level.txt

@@ -0,0 +1 @@
+pygen

package/generator/requirements.txt

@@ -0,0 +1,12 @@
+black==24.4.0
+click==8.1.3
+docutils==0.19
+Jinja2==3.1.4
+m2r2==0.3.3
+MarkupSafe==2.1.2
+mistune==0.8.4
+pathspec==0.11.1
+platformdirs==3.2.0
+PyYAML==6.0.1
+tomli==2.0.1
+setuptools==69.2.0

package/generator/setup.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+
+import os
+import re
+
+from setuptools import setup, find_packages
+
+
+# Version extraction inspired from 'requests'
+with open(os.path.join("pygen", "_version.py"), "r") as fd:
+    version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1)
+
+if not version:
+    raise RuntimeError("Cannot find version information")
+
+setup(
+    name="pygen",
+    version=version,
+    description="Core Library for Python Generation",
+    long_description=open("README.md", "r").read(),
+    long_description_content_type="text/markdown",
+    license="MIT License",
+    author="Microsoft Corporation",
+    author_email="azpysdkhelp@microsoft.com",
+    url="https://github.com/Azure/autorest.python/packages/core",
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "License :: OSI Approved :: MIT License",
+    ],
+    packages=find_packages(
+        exclude=[
+            "test",
+        ]
+    ),
+    install_requires=[
+        "Jinja2 >= 2.11",  # I need "include" and auto-context + blank line are not indented by default
+        "pyyaml",
+        "m2r2",
+        "black",
+    ],
+)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@azure-tools/typespec-python",
-  "version": "0.24.3",
+  "version": "0.25.0",
   "author": "Microsoft Corporation",
   "description": "TypeSpec emitter for Python SDKs",
   "homepage": "https://github.com/Azure/autorest.python",
@@ -27,16 +27,16 @@
     "node": ">=14.0.0"
   },
   "files": [
-    "lib/*.cadl",
     "dist/**",
     "!dist/test/**",
-    "get-autorest-python-path.cjs"
+    "generator/**",
+    "scripts/**"
   ],
   "peerDependencies": {
     "@azure-tools/typespec-azure-core": ">=0.43.0 <1.0.0",
     "@azure-tools/typespec-azure-resource-manager": ">=0.43.0 <1.0.0",
     "@azure-tools/typespec-autorest": ">=0.43.0 <1.0.0",
-    "@azure-tools/typespec-client-generator-core": ">=0.43.0 <1.0.0",
+    "@azure-tools/typespec-client-generator-core": ">=0.43.1 <1.0.0",
     "@typespec/compiler": ">=0.57.0 <1.0.0",
     "@typespec/http": ">=0.57.0 <1.0.0",
     "@typespec/rest": ">=0.57.0 <1.0.0",
@@ -51,7 +51,9 @@
   "dependencies": {
     "js-yaml": "~4.1.0",
     "@typespec/openapi3": "~0.57.0",
-    "@autorest/python": "^6.14.3"
+    "@autorest/system-requirements": "~1.0.2",
+    "fs-extra": "~11.2.0",
+    "semver": "~7.6.2"
   },
   "devDependencies": {
     "@azure-tools/typespec-azure-resource-manager": "~0.43.0",
@@ -69,7 +71,7 @@
     "rimraf": "~5.0.0",
     "typescript": "~5.1.3",
     "@azure-tools/typespec-azure-core": "~0.43.0",
-    "@azure-tools/typespec-client-generator-core": "~0.43.0",
+    "@azure-tools/typespec-client-generator-core": "~0.43.1",
     "@typespec/compiler": "~0.57.0",
     "@typespec/http": "~0.57.0",
     "@typespec/rest": "~0.57.0",
@@ -82,6 +84,7 @@
     "test": "mocha",
     "test-official": "c8 mocha --forbid-only",
     "lint": "eslint . --ext .ts --max-warnings=0",
-    "lint:fix": "eslint . --fix --ext .ts"
+    "lint:fix": "eslint . --fix --ext .ts",
+    "install": "node ./scripts/run-python3.cjs ./scripts/install.py"
   }
 }

package/scripts/install.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import sys
+
+if not sys.version_info >= (3, 8, 0):
+    raise Exception("Autorest for Python extension requires Python 3.8 at least")
+
+try:
+    import pip
+except ImportError:
+    raise Exception("Your Python installation doesn't have pip available")
+
+try:
+    import venv
+except ImportError:
+    raise Exception("Your Python installation doesn't have venv available")
+
+
+# Now we have pip and Py >= 3.8, go to work
+
+from pathlib import Path
+
+from venvtools import ExtendedEnvBuilder, python_run
+
+_ROOT_DIR = Path(__file__).parent.parent
+
+
+def main():
+    venv_path = _ROOT_DIR / "venv"
+    if venv_path.exists():
+        env_builder = venv.EnvBuilder(with_pip=True)
+        venv_context = env_builder.ensure_directories(venv_path)
+    else:
+        env_builder = ExtendedEnvBuilder(with_pip=True, upgrade_deps=True)
+        env_builder.create(venv_path)
+        venv_context = env_builder.context
+
+        python_run(venv_context, "pip", ["install", "-U", "pip"])
+        python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/generator/requirements.txt"])
+        python_run(venv_context, "pip", ["install", "-e", f"{_ROOT_DIR}/generator"])
+
+
+if __name__ == "__main__":
+    main()

package/scripts/prepare.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import sys
+import os
+import argparse
+
+if not sys.version_info >= (3, 8, 0):
+    raise Exception("Autorest for Python extension requires Python 3.8 at least")
+
+from pathlib import Path
+import venv
+
+from venvtools import python_run
+
+_ROOT_DIR = Path(__file__).parent.parent
+
+
+def main():
+    venv_path = _ROOT_DIR / "venv"
+    venv_prexists = venv_path.exists()
+
+    assert venv_prexists  # Otherwise install was not done
+
+    env_builder = venv.EnvBuilder(with_pip=True)
+    venv_context = env_builder.ensure_directories(venv_path)
+    try:
+        python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/generator/dev_requirements.txt"])
+    except FileNotFoundError as e:
+        raise ValueError(e.filename)
+
+
+if __name__ == "__main__":
+    main()