@typespec/http-client-python 0.1.0

package/generator/pygen/codegen/templates/test.py.jinja2

@@ -0,0 +1,50 @@
+{% set prefix_lower = test.prefix|lower %}
+{% set client_var = "self.client" if code_model.options["azure_arm"] else "client" %}
+{% set async = "async " if test.is_async else "" %}
+{% set async_suffix = "_async" if test.is_async else "" %}
+# coding=utf-8
+{{ code_model.options['license_header'] }}
+import pytest
+{{ imports }}
+
+{% if code_model.options["azure_arm"] %}
+AZURE_LOCATION = "eastus"
+{% endif %}
+
+@pytest.mark.skip("you may need to update the auto-generated test case before run it")
+class {{ test.test_class_name }}({{ test.base_test_class_name }}):
+{% if code_model.options["azure_arm"] %}
+    def setup_method(self, method):
+        {% if test.is_async %}
+        self.client = self.create_mgmt_client({{ test.client_name }}, is_async=True)
+        {% else %}
+        self.client = self.create_mgmt_client({{ test.client_name }})
+        {% endif %}
+{% endif %}
+{% for testcase in test.testcases %}
+    {% if code_model.options["azure_arm"] %}
+    @{{ test.preparer_name }}(location=AZURE_LOCATION)
+    {% else %}
+    @{{ test.preparer_name }}()
+    {% endif %}
+    @recorded_by_proxy{{ async_suffix }}
+    {% if code_model.options["azure_arm"] %}
+    {{ async }}def test_{{ testcase.name }}(self, resource_group):
+    {% else %}
+    {{ async }}def test_{{ testcase.name }}(self, {{ prefix_lower }}_endpoint):
+        {{ client_var }} = self.{{ test.create_client_name }}(endpoint={{ prefix_lower }}_endpoint)
+    {% endif %}
+        {{testcase.response }}{{ client_var }}{{ testcase.operation_group_prefix }}.{{ testcase.operation.name }}(
+            {% for key, value in testcase.params.items() %}
+            {% if code_model.options["azure_arm"] and key == "resource_group_name" %}
+            {{ key }}=resource_group.name,
+            {% else %}
+            {{ key }}={{ value|indent(12) }},
+            {% endif %}
+            {% endfor %}
+        ){{ testcase.operation_suffix }}
+        {{ testcase.extra_operation }}
+        # please add some check logic here by yourself
+        # ...
+
+{% endfor %}

package/generator/pygen/codegen/templates/testpreparer.py.jinja2

@@ -0,0 +1,26 @@
+# coding=utf-8
+{{ code_model.options['license_header'] }}
+{{ imports }}
+
+{% for test_name in test_names %}
+{% set extra_async = ", is_async=True" if test_name.is_async else ""%}
+{% set prefix_lower = test_name.prefix|lower %}
+class {{ test_name.base_test_class_name }}(AzureRecordedTestCase):
+
+    def {{ test_name.create_client_name }}(self, endpoint):
+        credential = self.get_credential({{ test_name.client_name }}{{ extra_async }})
+        return self.create_client_from_credential(
+            {{ test_name.client_name }},
+            credential=credential,
+            endpoint=endpoint,
+        )
+
+{% if not test_name.is_async %}
+{{ test_name.preparer_name }} = functools.partial(
+    PowerShellPreparer,
+    "{{ prefix_lower }}",
+    {{ prefix_lower }}_endpoint="https://fake_{{ prefix_lower }}_endpoint.com"
+)
+{% endif %}
+
+{% endfor %}

package/generator/pygen/codegen/templates/types.py.jinja2

@@ -0,0 +1,7 @@
+# coding=utf-8
+{{ code_model.options['license_header'] }}
+
+{{ imports }}
+{% for nu in code_model.named_unions %}
+{{nu.name}} = {{nu.type_definition()}}
+{% endfor %}

package/generator/pygen/codegen/templates/validation.py.jinja2

@@ -0,0 +1,38 @@
+{{ code_model.options['license_header'] }}
+import functools
+
+def api_version_validation(**kwargs):
+    params_added_on = kwargs.pop("params_added_on", {})
+    method_added_on = kwargs.pop("method_added_on", "")
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                # this assumes the client has an _api_version attribute
+                client = args[0]
+                client_api_version = client._config.api_version  # pylint: disable=protected-access
+            except AttributeError:
+                return func(*args, **kwargs)
+
+            if method_added_on > client_api_version:
+                raise ValueError(
+                    f"'{func.__name__}' is not available in API version "
+                    f"{client_api_version}. Pass service API version {method_added_on} or newer to your client."
+                )
+
+            unsupported = {
+                parameter: api_version
+                for api_version, parameters in params_added_on.items()
+                for parameter in parameters
+                if parameter in kwargs and api_version > client_api_version
+            }
+            if unsupported:
+                raise ValueError("".join([
+                    f"'{param}' is not available in API version {client_api_version}. "
+                    f"Use service API version {version} or newer.\n"
+                    for param, version in unsupported.items()
+                ]))
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator

package/generator/pygen/codegen/templates/vendor.py.jinja2

@@ -0,0 +1,95 @@
+{% import 'keywords.jinja2' as keywords with context %}
+{{ code_model.options['license_header'] }}
+
+{{ imports }}
+
+{% if code_model.need_mixin_abc %}
+    {% for client in clients | selectattr("has_mixin") %}
+class {{ client.name }}MixinABC(
+    ABC
+):
+    """DO NOT use this class. It is for internal typing use only."""
+    _client: "{{ keywords.async_class }}PipelineClient"
+    _config: {{ client.name }}Configuration
+    _serialize: "Serializer"
+    _deserialize: "Deserializer"
+    {% endfor %}
+{% endif %}
+{% if code_model.has_abstract_operations %}
+
+def raise_if_not_implemented(cls, abstract_methods):
+    not_implemented = [f for f in abstract_methods if not callable(getattr(cls, f, None))]
+    if not_implemented:
+        raise NotImplementedError("The following methods on operation group '{}' are not implemented: '{}'."
+        " Please refer to https://aka.ms/azsdk/python/dpcodegen/python/customize to learn how to customize.".format(
+        cls.__name__, '\', \''.join(not_implemented))
+        )
+{% endif %}
+
+{% if code_model.has_etag %}
+def quote_etag(etag: Optional[str]) -> Optional[str]:
+    if not etag or etag == "*":
+        return etag
+    if etag.startswith("W/"):
+        return etag
+    if etag.startswith('"') and etag.endswith('"'):
+        return etag
+    if etag.startswith("'") and etag.endswith("'"):
+        return etag
+    return '"' + etag + '"'
+
+
+def prep_if_match(etag: Optional[str], match_condition: Optional[MatchConditions]) -> Optional[str]:
+    if match_condition == MatchConditions.IfNotModified:
+        if_match = quote_etag(etag) if etag else None
+        return if_match
+    if match_condition == MatchConditions.IfPresent:
+        return "*"
+    return None
+
+
+def prep_if_none_match(etag: Optional[str], match_condition: Optional[MatchConditions]) -> Optional[str]:
+    if match_condition == MatchConditions.IfModified:
+        if_none_match = quote_etag(etag) if etag else None
+        return if_none_match
+    if match_condition == MatchConditions.IfMissing:
+        return "*"
+    return None
+{% endif %}
+{% if code_model.has_form_data and code_model.options["models_mode"] == "dpg" and not async_mode  %}
+# file-like tuple could be `(filename, IO (or bytes))` or `(filename, IO (or bytes), content_type)`
+FileContent = Union[str, bytes, IO[str], IO[bytes]]
+
+FileType = Union[
+    # file (or bytes)
+    FileContent,
+    # (filename, file (or bytes))
+    Tuple[Optional[str], FileContent],
+    # (filename, file (or bytes), content_type)
+    Tuple[Optional[str], FileContent, Optional[str]],
+]
+
+def serialize_multipart_data_entry(data_entry: Any) -> Any:
+    if isinstance(data_entry, (list, tuple, dict, Model)):
+        return json.dumps(data_entry, cls=SdkJSONEncoder, exclude_readonly=True)
+    return data_entry
+
+def prepare_multipart_form_data(
+    body: Mapping[str, Any], multipart_fields: List[str], data_fields: List[str]
+) -> Tuple[List[FileType], Dict[str, Any]]:
+    files: List[FileType] = []
+    data: Dict[str, Any] = {}
+    for multipart_field in multipart_fields:
+        multipart_entry = body.get(multipart_field)
+        if isinstance(multipart_entry, list):
+            files.extend([(multipart_field, e) for e in multipart_entry ])
+        elif multipart_entry:
+            files.append((multipart_field, multipart_entry))
+
+    for data_field in data_fields:
+        data_entry = body.get(data_field)
+        if data_entry:
+            data[data_field] = serialize_multipart_data_entry(data_entry)
+
+    return files, data
+{% endif %}

package/generator/pygen/codegen/templates/version.py.jinja2

@@ -0,0 +1,4 @@
+# coding=utf-8
+{{ code_model.options['license_header'] }}
+
+VERSION = "{{ code_model.options['package_version'] }}"

package/generator/pygen/m2r.py

@@ -0,0 +1,65 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+"""An MD to RST plugin.
+"""
+import logging
+from typing import Any, Dict, Set, Union
+
+import m2r2
+
+from . import YamlUpdatePlugin
+from .utils import parse_args
+
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class GeneratorRenderer(m2r2.RestRenderer):
+    """Redefine the concept of inline HTML in the renderer, we don't want to define a new format
+    in the description/summary.
+    """
+
+    def inline_html(self, html: str) -> str:
+        """Do not render inline HTML with a role definition."""
+        return f":code:`{html}`"
+
+
+class M2R(YamlUpdatePlugin):
+    """A plugin to convert any description and summary from MD to RST."""
+
+    def update_yaml(self, yaml_data: Dict[str, Any]) -> None:
+        """Convert in place the YAML str."""
+        self._convert_docstring_no_cycles(yaml_data, set())
+
+    def _convert_docstring_no_cycles(self, yaml_data: Union[Dict[str, Any], str], node_list: Set[int]) -> None:
+        """Walk the YAML tree to convert MD to RST."""
+        if id(yaml_data) in node_list:
+            return
+        node_list.add(id(yaml_data))
+
+        if isinstance(yaml_data, list):
+            for elt in yaml_data:
+                self._convert_docstring_no_cycles(elt, node_list)
+        elif isinstance(yaml_data, dict):
+            for key, value in yaml_data.items():
+                if key in ["description", "summary"]:
+                    yaml_data[key] = self.convert_to_rst(value)
+                    continue
+                self._convert_docstring_no_cycles(value, node_list)
+
+    @staticmethod
+    def convert_to_rst(string_to_convert: str) -> str:
+        """Convert that string from MD to RST."""
+        try:
+            return m2r2.convert(string_to_convert, renderer=GeneratorRenderer()).strip()
+        except Exception:  # pylint: disable=broad-except
+            return string_to_convert
+
+
+if __name__ == "__main__":
+    # CADL pipeline will call this
+    args, unknown_args = parse_args()
+    M2R(output_folder=args.output_folder, cadl_file=args.cadl_file, **unknown_args).process()

package/generator/pygen/postprocess/__init__.py

@@ -0,0 +1,183 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from typing import Tuple, Any
+from pathlib import Path
+import os
+import shutil
+from venv import EnvBuilder
+import black
+from black.report import NothingChanged
+from .venvtools import ExtendedEnvBuilder, python_run
+
+from .. import Plugin
+
+_BLACK_MODE = black.Mode()  # pyright: ignore [reportPrivateImportUsage]
+_BLACK_MODE.line_length = 120
+
+
+def format_file(file: Path, file_content: str) -> str:
+    if not file.suffix == ".py":
+        return file_content
+    try:
+        file_content = black.format_file_contents(file_content, fast=True, mode=_BLACK_MODE)
+    except NothingChanged:
+        pass
+    return file_content
+
+
+class PostProcessPlugin(Plugin):
+    def __init__(self, **kwargs: Any):
+        super().__init__(**kwargs)
+        output_folder_uri = self.options["outputFolderUri"]
+        if output_folder_uri.startswith("file:"):
+            output_folder_uri = output_folder_uri[5:]
+        if os.name == "nt" and output_folder_uri.startswith("///"):
+            output_folder_uri = output_folder_uri[3:]
+        self.output_folder = Path(output_folder_uri)  # path to where the setup.py is
+        self.setup_venv()
+
+        # set up the venv
+        # base folder is where the code starts, i.e. where we
+        self.base_folder, self.namespace = self.get_namespace(self.output_folder, "")
+
+    def setup_venv(self):
+        venv_path = self.output_folder / Path(".temp_folder") / Path("temp_venv")
+
+        if venv_path.exists():
+            env_builder = EnvBuilder(with_pip=True)
+            self.venv_context = env_builder.ensure_directories(venv_path)
+        else:
+            env_builder = ExtendedEnvBuilder(with_pip=True, upgrade_deps=True)
+            env_builder.create(venv_path)
+            self.venv_context = env_builder.context
+            python_run(
+                self.venv_context,
+                "pip",
+                ["install", "-e", str(self.output_folder)],
+                directory=self.output_folder,
+            )
+
+    def get_namespace(self, dir: Path, namespace: str) -> Tuple[Path, str]:
+        try:
+            init_file = next(d for d in dir.iterdir() if d.name == "__init__.py")
+            # we don't care about pkgutil inits, we skip over them
+            file_content = self.read_file(init_file.relative_to(self.output_folder))
+            if "pkgutil" not in file_content:
+                return dir, namespace
+        except StopIteration:
+            pass
+
+        try:
+            # first, see if we can get a folder that has the same name as the current output folder
+            start = self.output_folder.stem.split("-")[0]
+            next_dir = next(d for d in dir.iterdir() if d.is_dir() and d.name == start)
+        except StopIteration:
+            invalid_start_chars = [".", "_"]
+            invalid_dirs = [
+                "swagger",
+                "out",
+                "tests",
+                "samples",
+            ]
+
+            next_dir = next(
+                d
+                for d in dir.iterdir()
+                if d.is_dir()
+                and not str(d).endswith("egg-info")
+                and d.name[0] not in invalid_start_chars
+                and d.name not in invalid_dirs
+            )
+
+        namespace = f"{namespace}.{next_dir.name}" if namespace else next_dir.name
+        return self.get_namespace(next_dir, namespace)
+
+    def process(self) -> bool:
+        folders = [f for f in self.base_folder.glob("**/*") if f.is_dir() and not f.stem.startswith("__")]
+        # will always have the root
+        self.fix_imports_in_init(
+            generated_file_name="_client",
+            folder_path=self.base_folder,
+            namespace=self.namespace,
+        )
+        try:
+            aio_folder = next(f for f in folders if f.stem == "aio")
+            self.fix_imports_in_init(
+                generated_file_name="_client",
+                folder_path=aio_folder,
+                namespace=f"{self.namespace}.aio",
+            )
+        except StopIteration:
+            pass
+
+        try:
+            models_folder = next(f for f in folders if f.stem == "models")
+            self.fix_imports_in_init(
+                generated_file_name="_models",
+                folder_path=models_folder,
+                namespace=f"{self.namespace}.models",
+            )
+        except StopIteration:
+            pass
+        operations_folders = [f for f in folders if f.stem in ["operations", "_operations"]]
+        for operations_folder in operations_folders:
+            sub_namespace = ".".join(str(operations_folder.relative_to(self.base_folder)).split(os.sep))
+            self.fix_imports_in_init(
+                generated_file_name="_operations",
+                folder_path=operations_folder,
+                namespace=f"{self.namespace}.{sub_namespace}",
+            )
+        shutil.rmtree(f"{str(self.output_folder)}/.temp_folder")
+        return True
+
+    def fix_imports_in_init(self, generated_file_name: str, folder_path: Path, namespace: str) -> None:
+        customized_objects_str = python_run(
+            self.venv_context,
+            command=[namespace, str(self.output_folder)],
+            module="get_all",
+        )
+
+        if not customized_objects_str:
+            return
+        customized_objects = {k: None for k in customized_objects_str.split(",")}.keys()  # filter out duplicates
+        file = (folder_path / "__init__.py").relative_to(self.output_folder)
+        file_content = self.read_file(file).replace("\r\n", "\n")
+        added_objs = []
+        for obj in customized_objects:
+            if f" import {obj}\n" in file_content:
+                # means we're overriding a generated model
+                file_content = file_content.replace(
+                    f"from .{generated_file_name} import {obj}\n",
+                    f"from ._patch import {obj}\n",
+                )
+            else:
+                added_objs.append(obj)
+        file_content = file_content.replace(
+            "try:\n    from ._patch import __all__ as _patch_all\n    "
+            "from ._patch import *  # pylint: disable=unused-wildcard-import"
+            "\nexcept ImportError:\n    _patch_all = []",
+            "",
+        )
+        file_content = file_content.replace("from ._patch import __all__ as _patch_all", "")
+        file_content = file_content.replace(
+            "from ._patch import *  # pylint: disable=unused-wildcard-import\n",
+            "",
+        )
+        file_content = file_content.replace("__all__.extend([p for p in _patch_all if p not in __all__])", "")
+        if added_objs:
+            # add import
+            patch_sdk_import = "from ._patch import patch_sdk as _patch_sdk"
+            imports = "\n".join([f"from ._patch import {obj}" for obj in added_objs])
+            if imports:
+                replacement = f"{imports}\n{patch_sdk_import}"
+            else:
+                replacement = patch_sdk_import
+            file_content = file_content.replace(patch_sdk_import, replacement)
+            # add to __all__
+            added_objs_all = "\n".join([f'    "{obj}",' for obj in added_objs]) + "\n"
+            file_content = file_content.replace("__all__ = [", f"__all__ = [\n{added_objs_all}", 1)
+        formatted_file = format_file(file, file_content)
+        self.write_file(file, formatted_file)

package/generator/pygen/postprocess/get_all.py

@@ -0,0 +1,19 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import sys
+import importlib
+
+
+def main(namespace):
+    sdk = importlib.import_module(namespace)
+    return sdk._patch.__all__  # pylint: disable=protected-access
+
+
+if __name__ == "__main__":
+    patched = ",".join(main(sys.argv[1]))
+    output_folder = sys.argv[2]
+    with open(f"{output_folder}/.temp_folder/patched.txt", "w", encoding="utf-8-sig") as f:
+        f.write(patched)

package/generator/pygen/postprocess/venvtools.py

@@ -0,0 +1,75 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from typing import Optional
+import subprocess
+import venv
+import sys
+from pathlib import Path
+
+
+_ROOT_DIR = Path(__file__).parent
+
+
+class ExtendedEnvBuilder(venv.EnvBuilder):
+    """An extended env builder which saves the context, to have access
+    easily to bin path and such.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.context = None
+        if sys.version_info < (3, 9, 0):
+            # Not supported on Python 3.8, and we don't need it
+            kwargs.pop("upgrade_deps", None)
+        super().__init__(*args, **kwargs)
+
+    def ensure_directories(self, env_dir):
+        self.context = super(ExtendedEnvBuilder, self).ensure_directories(env_dir)
+        return self.context
+
+
+def create(
+    env_dir,
+    system_site_packages=False,
+    clear=False,
+    symlinks=False,
+    with_pip=False,
+    prompt=None,
+    upgrade_deps=False,
+):
+    """Create a virtual environment in a directory."""
+    builder = ExtendedEnvBuilder(
+        system_site_packages=system_site_packages,
+        clear=clear,
+        symlinks=symlinks,
+        with_pip=with_pip,
+        prompt=prompt,
+        upgrade_deps=upgrade_deps,
+    )
+    builder.create(env_dir)
+    return builder.context
+
+
+def python_run(venv_context, module, command, directory=_ROOT_DIR) -> Optional[str]:
+    try:
+        cmd_line = [
+            venv_context.env_exe,
+            "-m",
+            module,
+        ] + command
+        print("Executing: {}".format(" ".join(cmd_line)))
+        subprocess.run(
+            cmd_line,
+            cwd=directory,
+            check=True,
+            stdout=False,
+        )
+        if module == "get_all":
+            with open(f"{command[1]}/.temp_folder/patched.txt", "r", encoding="utf-8-sig") as f:
+                return f.read()
+    except subprocess.CalledProcessError as err:
+        print(err)
+        sys.exit(1)
+    return None