@autorest/python 6.0.1 → 6.1.0

package/ChangeLog.md

@@ -1,5 +1,31 @@
 # Release History
 
+### 2022-07-20 - 6.1.0
+
+| Library                                                                 | Min Version |
+| ----------------------------------------------------------------------- | ----------- |
+| `@autorest/core`                                                        | `3.8.1`     |
+| `@autorest/modelerfour`                                                 | `4.23.5`    |
+| `azure-core` dep of generated code                                      | `1.24.0`    |
+| `isodate` dep of generated code                                         | `0.6.1`     |
+| `msrest` dep of generated code (If generating legacy code)              | `0.7.1`     |
+| `azure-mgmt-core` dep of generated code (If generating mgmt plane code) | `1.3.0`     |
+
+**New Features**
+
+- Add new plugin `MultiClient` and new flag `--multiclientscript` to handle package structure of multi client  #1328
+
+**Bug Fixes**
+
+- Fallback unrecognized type as string to avoid a fatal error. #1341
+- Fix regression in default namespace for SDKs generated without `--namespace` flag  #1354
+
+**Other Changes**
+
+- Generated code no longer supports Python 3.6  #1353
+- Order json input and response template entries by whether they are required or not #1335
+- Reduce extreme amount of `black` logs when running in `--debug` mode to just log errors
+
 ### 2022-06-29 - 6.0.1
 
 | Library                                                                 | Min Version |

package/README.md

@@ -23,7 +23,7 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio
 
 #### Python code gen
 
-```yaml !$(multiapiscript)
+```yaml !$(multiapiscript) && !$(multiclientscript)
 # default values for version tolerant and black
 black: true
 ```
@@ -43,7 +43,7 @@ modelerfour:
 allow-no-input: true
 ```
 
-```yaml !$(multiapiscript)
+```yaml !$(multiapiscript) && !$(multiclientscript)
 pass-thru:
   - model-deduplicator
   - subset-reducer
@@ -161,6 +161,25 @@ scope-postprocess/emitter:
 output-artifact: python-files
 ```
 
+# Multi Client pipeline
+
+```yaml $(multiclientscript)
+pipeline:
+  python/multiclientscript:
+    scope: multiclientscript
+    output-artifact: python-files
+
+  python/multiclientscript/emitter:
+    input: multiclientscript
+    scope: scope-multiclientscript/emitter
+
+scope-multiclientscript/emitter:
+  input-artifact: python-files
+  output-uri-expr: $key
+
+output-artifact: python-files
+```
+
 # Help
 
 ```yaml

package/autorest/__init__.py

@@ -19,21 +19,33 @@ _LOGGER = logging.getLogger(__name__)
 
 
 class ReaderAndWriter:
-    def __init__(self, **kwargs: Any) -> None:
-        pass
+    def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None:
+        self.output_folder = Path(output_folder)
+        self.options = kwargs
 
     def read_file(self, path: Union[str, Path]) -> str:
         """How does one read a file in cadl?"""
-        raise NotImplementedError("Haven't plugged in Cadl yet")
+        # make path relative to output folder
+        try:
+            with open(self.output_folder / Path(path), "r") as fd:
+                return fd.read()
+        except FileNotFoundError:
+            return ""
 
     def write_file(self, filename: Union[str, Path], file_content: str) -> None:
         """How does writing work in cadl?"""
-        raise NotImplementedError("Haven't plugged in Cadl yet")
+        file_folder = Path(filename).parent
+        if not Path.is_dir(self.output_folder / file_folder):
+            Path.mkdir(self.output_folder / file_folder, parents=True)
+        with open(self.output_folder / Path(filename), "w") as fd:
+            fd.write(file_content)
 
 
 class ReaderAndWriterAutorest(ReaderAndWriter):
-    def __init__(self, *, autorestapi: AutorestAPI) -> None:
-        super().__init__()
+    def __init__(
+        self, *, output_folder: Union[str, Path], autorestapi: AutorestAPI
+    ) -> None:
+        super().__init__(output_folder=output_folder)
         self._autorestapi = autorestapi
 
     def read_file(self, path: Union[str, Path]) -> str:
@@ -49,10 +61,6 @@ class Plugin(ReaderAndWriter, ABC):
     :param autorestapi: An autorest API instance
     """
 
-    def __init__(self, **kwargs: Any) -> None:
-        super().__init__(**kwargs)
-        self.options: Dict[str, Any] = {}
-
     @abstractmethod
     def process(self) -> bool:
         """The plugin process.
@@ -67,8 +75,10 @@ class Plugin(ReaderAndWriter, ABC):
 class PluginAutorest(Plugin, ReaderAndWriterAutorest):
     """For our Autorest plugins, we want to take autorest api as input as options, then pass it to the Plugin"""
 
-    def __init__(self, autorestapi: AutorestAPI) -> None:
-        super().__init__(autorestapi=autorestapi)
+    def __init__(
+        self, autorestapi: AutorestAPI, *, output_folder: Union[str, Path]
+    ) -> None:
+        super().__init__(autorestapi=autorestapi, output_folder=output_folder)
         self.options = self.get_options()
 
     @abstractmethod
@@ -79,15 +89,24 @@ class PluginAutorest(Plugin, ReaderAndWriterAutorest):
 class YamlUpdatePlugin(Plugin):
     """A plugin that update the YAML as input."""
 
+    def get_yaml(self) -> Dict[str, Any]:
+        # cadl file doesn't have to be relative to output folder
+        with open(self.options["cadl_file"], "r") as fd:
+            return yaml.safe_load(fd.read())
+
+    def write_yaml(self, yaml_string: str) -> None:
+        with open(self.options["cadl_file"], "w") as fd:
+            fd.write(yaml_string)
+
     def process(self) -> bool:
         # List the input file, should be only one
-        yaml_data = yaml.safe_load(self.read_file("code-model-v4-no-tags.yaml"))
+        yaml_data = self.get_yaml()
 
         self.update_yaml(yaml_data)
 
         yaml_string = yaml.safe_dump(yaml_data)
 
-        self.write_file("code-model-v4-no-tags.yaml", yaml_string)
+        self.write_yaml(yaml_string)
         return True
 
     @abstractmethod
@@ -103,6 +122,12 @@ class YamlUpdatePlugin(Plugin):
 class YamlUpdatePluginAutorest(  # pylint: disable=abstract-method
     YamlUpdatePlugin, PluginAutorest
 ):
+    def get_yaml(self) -> Dict[str, Any]:
+        return yaml.safe_load(self.read_file("code-model-v4-no-tags.yaml"))
+
+    def write_yaml(self, yaml_string: str) -> None:
+        self.write_file("code-model-v4-no-tags.yaml", yaml_string)
+
     def get_options(self) -> Dict[str, Any]:
         inputs = self._autorestapi.list_inputs()
         _LOGGER.debug("Possible Inputs: %s", inputs)

package/autorest/_utils.py

@@ -0,0 +1,56 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import re
+import argparse
+
+
+def to_snake_case(name: str) -> str:
+    def replace_upper_characters(m) -> str:
+        match_str = m.group().lower()
+        if m.start() > 0 and name[m.start() - 1] == "_":
+            # we are good if a '_' already exists
+            return match_str
+        # the first letter should not have _
+        prefix = "_" if m.start() > 0 else ""
+
+        # we will add an extra _ if there are multiple upper case chars together
+        next_non_upper_case_char_location = m.start() + len(match_str)
+        if (
+            len(match_str) > 2
+            and len(name) - next_non_upper_case_char_location > 1
+            and name[next_non_upper_case_char_location].isalpha()
+        ):
+
+            return (
+                prefix
+                + match_str[: len(match_str) - 1]
+                + "_"
+                + match_str[len(match_str) - 1]
+            )
+
+        return prefix + match_str
+
+    return re.sub("[A-Z]+", replace_upper_characters, name)
+
+
+def parse_args(need_cadl_file: bool = True):
+    parser = argparse.ArgumentParser(
+        description="Run mypy against target folder. Add a local custom plugin to the path prior to execution. "
+    )
+    parser.add_argument(
+        "--output-folder",
+        dest="output_folder",
+        help="Output folder for generated SDK",
+        required=True,
+    )
+    parser.add_argument(
+        "--cadl-file",
+        dest="cadl_file",
+        help="Serialized cadl file",
+        required=need_cadl_file,
+    )
+
+    return parser.parse_args()

package/autorest/black/__init__.py

@@ -10,8 +10,9 @@ from typing import Any, Dict
 import black
 
 from .. import Plugin, PluginAutorest
+from .._utils import parse_args
 
-_LOGGER = logging.getLogger(__name__)
+logging.getLogger("blib2to3").setLevel(logging.ERROR)
 
 _BLACK_MODE = black.Mode()
 _BLACK_MODE.line_length = 120
@@ -20,12 +21,12 @@ _BLACK_MODE.line_length = 120
 class BlackScriptPlugin(Plugin):  # pylint: disable=abstract-method
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        output_folder_uri = self.options["outputFolderUri"]
-        if output_folder_uri.startswith("file:"):
-            output_folder_uri = output_folder_uri[5:]
-        if os.name == "nt" and output_folder_uri.startswith("///"):
-            output_folder_uri = output_folder_uri[3:]
-        self.output_folder = Path(output_folder_uri)
+        output_folder = self.options.get("output_folder", str(self.output_folder))
+        if output_folder.startswith("file:"):
+            output_folder = output_folder[5:]
+        if os.name == "nt" and output_folder.startswith("///"):
+            output_folder = output_folder[3:]
+        self.output_folder = Path(output_folder)
 
     def process(self) -> bool:
         # apply format_file on every file in the output folder
@@ -54,4 +55,10 @@ class BlackScriptPlugin(Plugin):  # pylint: disable=abstract-method
 
 class BlackScriptPluginAutorest(BlackScriptPlugin, PluginAutorest):
     def get_options(self) -> Dict[str, Any]:
-        return {"outputFolderUri": self._autorestapi.get_value("outputFolderUri")}
+        return {"output_folder": self._autorestapi.get_value("outputFolderUri")}
+
+
+if __name__ == "__main__":
+    # CADL pipeline will call this
+    args = parse_args(need_cadl_file=False)
+    BlackScriptPlugin(output_folder=args.output_folder).process()

package/autorest/codegen/__init__.py

@@ -4,19 +4,20 @@
 # license information.
 # --------------------------------------------------------------------------
 import logging
-import sys
 from typing import Dict, Any, Union, cast
 from pathlib import Path
 import yaml
 
 
 from .. import Plugin, PluginAutorest
+from .._utils import parse_args
 from .models.client import Client, Config
 from .models.code_model import CodeModel
 from .models import build_type
 from .models.request_builder import get_request_builder
 from .models.operation_group import OperationGroup
 from .serializers import JinjaSerializer, JinjaSerializerAutorest
+from ._utils import DEFAULT_HEADER_TEXT
 
 
 def _build_convenience_layer(yaml_data: Dict[str, Any], code_model: CodeModel) -> None:
@@ -163,7 +164,7 @@ class CodeGenerator(Plugin):
     def _build_code_model_options(self) -> Dict[str, Any]:
         """Build en options dict from the user input while running autorest."""
         azure_arm = self.options.get("azure-arm", False)
-        license_header = self.options["header-text"]
+        license_header = self.options.get("header-text", DEFAULT_HEADER_TEXT)
         if license_header:
             license_header = license_header.replace("\n", "\n# ")
             license_header = (
@@ -236,12 +237,12 @@ class CodeGenerator(Plugin):
         return options
 
     def get_yaml(self) -> Dict[str, Any]:
-        # cadl should call this one
-        raise NotImplementedError()
+        # cadl file doesn't have to be relative to output folder
+        with open(self.options["cadl_file"], "r") as fd:
+            return yaml.safe_load(fd.read())
 
-    @staticmethod
-    def get_serializer(code_model: CodeModel):
-        return JinjaSerializer(code_model)
+    def get_serializer(self, code_model: CodeModel):
+        return JinjaSerializer(code_model, output_folder=self.output_folder)
 
     def process(self) -> bool:
         # List the input file, should be only one
@@ -351,20 +352,12 @@ class CodeGeneratorAutorest(CodeGenerator, PluginAutorest):
         return yaml.safe_load(file_content)
 
     def get_serializer(self, code_model: CodeModel):  # type: ignore
-        return JinjaSerializerAutorest(self._autorestapi, code_model)
-
-
-def main(yaml_model_file: str) -> None:
-    from ..jsonrpc.localapi import (  # pylint: disable=import-outside-toplevel
-        LocalAutorestAPI,
-    )
-
-    code_generator = CodeGeneratorAutorest(
-        autorestapi=LocalAutorestAPI(reachable_files=[yaml_model_file])
-    )
-    if not code_generator.process():
-        raise SystemExit("Process didn't finish gracefully")
+        return JinjaSerializerAutorest(
+            self._autorestapi, code_model, output_folder=self.output_folder
+        )
 
 
 if __name__ == "__main__":
-    main(sys.argv[1])
+    # CADL pipeline will call this
+    args = parse_args()
+    CodeGenerator(output_folder=args.output_folder, cadl_file=args.cadl_file).process()

package/autorest/codegen/_utils.py

@@ -0,0 +1,14 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+DEFAULT_HEADER_TEXT = (
+    "# --------------------------------------------------------------------------\n"
+    "# Copyright (c) Microsoft Corporation. All rights reserved.\n"
+    "# Licensed under the MIT License. See License.txt in the project root for license information.\n"
+    "# Code generated by Microsoft (R) Python Code Generator.\n"
+    "# Changes may cause incorrect behavior and will be lost if the code is regenerated.\n"
+    "# --------------------------------------------------------------------------"
+)

package/autorest/codegen/models/__init__.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
+import logging
 from typing import Any, Dict, Union
 from .base_model import BaseModel
 from .code_model import CodeModel
@@ -140,6 +141,7 @@ TYPE_TO_OBJECT = {
     "any-object": AnyObjectType,
     "unixtime": UnixTimeType,
 }
+_LOGGER = logging.getLogger(__name__)
 
 
 def build_type(yaml_data: Dict[str, Any], code_model: CodeModel) -> BaseType:
@@ -155,7 +157,14 @@ def build_type(yaml_data: Dict[str, Any], code_model: CodeModel) -> BaseType:
         code_model.types_map[yaml_id] = response
         response.fill_instance_from_yaml(yaml_data, code_model)
     else:
-        response = TYPE_TO_OBJECT[yaml_data["type"]].from_yaml(yaml_data, code_model)  # type: ignore
+        object_type = yaml_data.get("type", None)
+        if object_type is None:
+            _LOGGER.warning(
+                'Unrecognized definition type "%s" is found, falling back it as "string"! ',
+                yaml_data["type"],
+            )
+            object_type = "string"
+        response = TYPE_TO_OBJECT[object_type].from_yaml(yaml_data, code_model)  # type: ignore
     code_model.types_map[yaml_id] = response
     return response
 

package/autorest/codegen/models/model_type.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
+from collections import OrderedDict
 from typing import Any, Dict, List, Optional, TYPE_CHECKING, cast
 
 from autorest.codegen.models.utils import add_to_pylint_disable
@@ -153,7 +154,17 @@ class ModelType(BaseType):  # pylint: disable=too-many-instance-attributes
         # once we've finished, we want to reset created_json_template_representation to false
         # so we can call it again
         self._created_json_template_representation = False
-        return representation
+        optional_keys = [
+            f'"{p.rest_api_name}"'
+            for p in self.properties
+            if getattr(p, "optional", False)
+        ]
+        return OrderedDict(
+            sorted(
+                representation.items(),
+                key=lambda item: f"{1 if item[0] in optional_keys else 0}{item[0]}",
+            )
+        )
 
     def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None:
         is_polymorphic_subtype = (

package/autorest/codegen/serializers/__init__.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-from typing import Dict, List, Optional, Any, Union
+from typing import Dict, List, Optional, Any, Union, cast
 from pathlib import Path
 from jinja2 import PackageLoader, Environment, FileSystemLoader, StrictUndefined
 from autorest.codegen.models.operation_group import OperationGroup
@@ -40,8 +40,10 @@ _REGENERATE_FILES = {"setup.py", "MANIFEST.in"}
 
 
 class JinjaSerializer(ReaderAndWriter):  # pylint: disable=abstract-method
-    def __init__(self, code_model: CodeModel, **kwargs: Any) -> None:
-        super().__init__(**kwargs)
+    def __init__(
+        self, code_model: CodeModel, *, output_folder: Union[str, Path], **kwargs: Any
+    ) -> None:
+        super().__init__(output_folder=output_folder, **kwargs)
         self.code_model = code_model
 
     @property
@@ -133,7 +135,7 @@ class JinjaSerializer(ReaderAndWriter):  # pylint: disable=abstract-method
             if self.read_file(namespace_path / Path("models.py")):
                 self.write_file(
                     namespace_path / Path("models.py"),
-                    self.read_file(namespace_path / Path("models.py")),
+                    cast(str, self.read_file(namespace_path / Path("models.py"))),
                 )
 
         if self.code_model.options["package_mode"]:
@@ -150,7 +152,9 @@ class JinjaSerializer(ReaderAndWriter):  # pylint: disable=abstract-method
                     self.write_file(output_name, render_result)
 
         def _prepare_params() -> Dict[Any, Any]:
-            package_parts = package_name.split("-")[:-1]
+            package_parts = (self.code_model.options["package_name"] or "").split("-")[
+                :-1
+            ]
             token_cred = isinstance(
                 getattr(self.code_model.credential, "type", None), TokenCredentialType
             )
@@ -177,14 +181,7 @@ class JinjaSerializer(ReaderAndWriter):  # pylint: disable=abstract-method
             params.update(self.code_model.package_dependency)
             return params
 
-        package_name = (
-            self.code_model.options["package_name"]
-            or self.code_model.client.name.lower()
-        )
-        count = package_name.count("-") + 1
-        for _ in range(count):
-            out_path = out_path / Path("..")
-
+        out_path = out_path / Path("../" * (self.code_model.namespace.count(".") + 1))
         if self.code_model.options["package_mode"] in ("dataplane", "mgmtplane"):
             env = Environment(
                 loader=PackageLoader("autorest.codegen", "templates"),
@@ -484,5 +481,13 @@ class JinjaSerializer(ReaderAndWriter):  # pylint: disable=abstract-method
 
 
 class JinjaSerializerAutorest(JinjaSerializer, ReaderAndWriterAutorest):
-    def __init__(self, autorestapi: AutorestAPI, code_model: CodeModel) -> None:
-        super().__init__(autorestapi=autorestapi, code_model=code_model)
+    def __init__(
+        self,
+        autorestapi: AutorestAPI,
+        code_model: CodeModel,
+        *,
+        output_folder: Union[str, Path],
+    ) -> None:
+        super().__init__(
+            autorestapi=autorestapi, code_model=code_model, output_folder=output_folder
+        )

package/autorest/codegen/serializers/builder_serializer.py

@@ -79,9 +79,7 @@ def _improve_json_string(template_representation: str) -> Any:
 
 def _json_dumps_template(template_representation: Any) -> Any:
     # only for template use, since it wraps everything in strings
-    return _improve_json_string(
-        json.dumps(template_representation, sort_keys=True, indent=4)
-    )
+    return _improve_json_string(json.dumps(template_representation, indent=4))
 
 
 def _get_polymorphic_subtype_template(polymorphic_subtype: ModelType) -> List[str]:

package/autorest/codegen/templates/README.md.jinja2

@@ -2,7 +2,7 @@
 # Microsoft Azure SDK for Python
 
 This is the Microsoft {{package_pprint_name}} Client Library.
-This package has been tested with Python 3.6+.
+This package has been tested with Python 3.7+.
 For a more complete view of Azure libraries, see the [azure sdk python release](https://aka.ms/azsdk/python/all).
 
 # Usage
@@ -17,7 +17,7 @@ Additional code samples for different Azure services are available at [Samples R
 
 If you encounter any bugs or have suggestions, please file an issue in the
 [Issues](https://github.com/Azure/azure-sdk-for-python/issues)
-section of the project. 
+section of the project.
 
 
 ![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-python%2F{{package_name}}%2FREADME.png)
@@ -35,7 +35,7 @@ python -m pip install {{ package_name }}
 
 #### Prequisites
 
-- Python 3.6 or later is required to use this package.
+- Python 3.7 or later is required to use this package.
 - You need an [Azure subscription][azure_sub] to use this package.
 - An existing {{ package_pprint_name }} instance.
 

package/autorest/codegen/templates/setup.py.jinja2

@@ -54,7 +54,6 @@ setup(
         "Programming Language :: Python",
         "Programming Language :: Python :: 3 :: Only",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
@@ -94,7 +93,7 @@ setup(
         {% endif %}
     ],
     {% if package_mode %}
-    python_requires=">=3.6",
+    python_requires=">=3.7",
     {% else %}
     long_description="""\
     {{ code_model.client.description }}

package/autorest/jsonrpc/server.py

@@ -26,6 +26,7 @@ def GetPluginNames():
         "black",
         "multiapiscript",
         "postprocess",
+        "multiclientscript",
     ]
 
 
@@ -56,11 +57,16 @@ def Process(plugin_name: str, session_id: str) -> bool:
             from ..black import BlackScriptPluginAutorest as PluginToLoad  # type: ignore
         elif plugin_name == "multiapiscript":
             from ..multiapi import MultiApiScriptPluginAutorest as PluginToLoad  # type: ignore
+        elif plugin_name == "multiclientscript":
+            from ..multiclient import MultiClientPluginAutorest as PluginToLoad  # type: ignore
         else:
             _LOGGER.fatal("Unknown plugin name %s", plugin_name)
             raise RuntimeError(f"Unknown plugin name {plugin_name}")
 
-        plugin = PluginToLoad(autorestapi=stdstream_connection)
+        plugin = PluginToLoad(
+            autorestapi=stdstream_connection,
+            output_folder=stdstream_connection.get_value("output-folder"),
+        )
 
         try:
             _LOGGER.debug("Starting plugin %s", PluginToLoad.__name__)

package/autorest/m2r/__init__.py

@@ -8,20 +8,21 @@
 import logging
 from typing import Any, Dict, Set
 
-import m2r
+import m2r2
 
 from .. import YamlUpdatePluginAutorest, YamlUpdatePlugin
+from .._utils import parse_args
 
 
 _LOGGER = logging.getLogger(__name__)
 
 
-class AutorestRender(m2r.RestRenderer):
+class AutorestRender(m2r2.RestRenderer):
     """Redefine the concept of inline HTML in the renderer, we don't want to define a new format
     in the description/summary.
     """
 
-    def inline_html(self, html: str) -> str:
+    def inline_html(self, html: str) -> str:  # pylint: disable=no-self-use
         """Do not render inline HTML with a role definition."""
         return f":code:`{html}`"
 
@@ -55,7 +56,7 @@ class M2R(YamlUpdatePlugin):  # pylint: disable=abstract-method
     def convert_to_rst(string_to_convert: str) -> str:
         """Convert that string from MD to RST."""
         try:
-            return m2r.convert(string_to_convert, renderer=AutorestRender()).strip()
+            return m2r2.convert(string_to_convert, renderer=AutorestRender()).strip()
         except Exception:  # pylint: disable=broad-except
             return string_to_convert
 
@@ -63,3 +64,9 @@ class M2R(YamlUpdatePlugin):  # pylint: disable=abstract-method
 class M2RAutorest(YamlUpdatePluginAutorest, M2R):
     def get_options(self) -> Dict[str, Any]:
         return {}
+
+
+if __name__ == "__main__":
+    # CADL pipeline will call this
+    args = parse_args()
+    M2R(output_folder=args.output_folder, cadl_file=args.cadl_file).process()

package/autorest/m4reformatter/__init__.py

@@ -11,6 +11,7 @@ import copy
 import logging
 from typing import Callable, Dict, Any, Iterable, List, Optional, Set
 
+from .._utils import to_snake_case
 from .. import YamlUpdatePluginAutorest
 
 JSON_REGEXP = re.compile(r"^(application|text)/(.+\+)?json$")
@@ -1100,7 +1101,7 @@ class M4Reformatter(
             if yaml_data.get("globalParameters")
             else "",
             "namespace": self._autorestapi.get_value("namespace")
-            or yaml_data["language"]["default"]["name"],
+            or to_snake_case(yaml_data["info"]["title"].replace(" ", "")),
         }
 
     def update_yaml(self, yaml_data: Dict[str, Any]) -> None:

package/autorest/multiapi/__init__.py

@@ -41,7 +41,7 @@ class MultiApiScriptPluginAutorest(MultiApiScriptPlugin, PluginAutorest):
         return MultiAPIAutorest(
             autorestapi=self._autorestapi,
             input_package_name=self.options.get("package-name"),
-            output_folder=self.options["output-folder"],
+            output_folder=self.output_folder,
             user_specified_default_api=self.options.get("default-api"),
             no_async=self.options.get("no-async", False),
         )
@@ -49,7 +49,6 @@ class MultiApiScriptPluginAutorest(MultiApiScriptPlugin, PluginAutorest):
     def get_options(self) -> Dict[str, Any]:
         options = {
             "package-name": self._autorestapi.get_value("package-name"),
-            "output-folder": self._autorestapi.get_value("output-folder"),
             "default-api": self._autorestapi.get_value("default-api"),
             "no-async": self._autorestapi.get_value("no-async"),
         }
@@ -66,15 +65,13 @@ class MultiAPI(ReaderAndWriter):  # pylint: disable=abstract-method
         user_specified_default_api: Optional[str] = None,
         **kwargs: Any,
     ) -> None:
-        super().__init__(**kwargs)
+        super().__init__(output_folder=Path(output_folder).resolve(), **kwargs)
         if input_package_name is None:
             raise ValueError(
                 "package-name is required, either provide it as args or check your readme configuration"
             )
         self.input_package_name = input_package_name
         _LOGGER.debug("Received package name %s", input_package_name)
-
-        self.output_folder = Path(output_folder).resolve()
         _LOGGER.debug("Received output-folder %s", output_folder)
         self.output_package_name: str = ""
         self.no_async = no_async
@@ -194,4 +191,6 @@ class MultiAPIAutorest(MultiAPI, ReaderAndWriterAutorest):
 
     @property
     def serializer(self) -> MultiAPISerializer:
-        return MultiAPISerializerAutorest(self._autorestapi)
+        return MultiAPISerializerAutorest(
+            self._autorestapi, output_folder=self.output_folder
+        )

package/autorest/multiapi/serializers/__init__.py

@@ -4,7 +4,7 @@
 # license information.
 # --------------------------------------------------------------------------
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Optional, Union
 from jinja2 import PackageLoader, Environment
 
 from .import_serializer import FileImportSerializer
@@ -122,5 +122,7 @@ class MultiAPISerializer(ReaderAndWriter):  # pylint: disable=abstract-method
 
 
 class MultiAPISerializerAutorest(MultiAPISerializer, ReaderAndWriterAutorest):
-    def __init__(self, autorestapi: AutorestAPI) -> None:
-        super().__init__(autorestapi=autorestapi)
+    def __init__(
+        self, autorestapi: AutorestAPI, *, output_folder: Union[str, Path]
+    ) -> None:
+        super().__init__(autorestapi=autorestapi, output_folder=output_folder)

package/autorest/multiclient/__init__.py

@@ -0,0 +1,50 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import logging
+from typing import Any, Dict
+from pathlib import Path
+from jinja2 import Environment, PackageLoader
+from .. import Plugin, PluginAutorest
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class MultiClientPlugin(Plugin):  # pylint: disable=abstract-method
+    def process(self) -> bool:
+        _LOGGER.info("Generating files for multi client")
+
+        env = Environment(
+            loader=PackageLoader("autorest.multiclient", "templates"),
+            keep_trailing_newline=True,
+            line_statement_prefix="##",
+            line_comment_prefix="###",
+            trim_blocks=True,
+            lstrip_blocks=True,
+        )
+
+        # __init__.py
+        template = env.get_template("init.py.jinja2")
+        self.write_file(Path("__init__.py"), template.render())
+
+        # _version.py
+        template = env.get_template("version.py.jinja2")
+        self.write_file(
+            Path("_version.py"),
+            template.render(
+                package_version=self.options.get("package-version") or "1.0.0b1"
+            ),
+        )
+
+        # py.typed
+        self.write_file(Path("py.typed"), "# Marker file for PEP 561.")
+
+        _LOGGER.info("Generating Done for multi client!")
+        return True
+
+
+class MultiClientPluginAutorest(MultiClientPlugin, PluginAutorest):
+    def get_options(self) -> Dict[str, Any]:
+        return {"package-version": self._autorestapi.get_value("package-version")}

package/autorest/multiclient/templates/init.py.jinja2

@@ -0,0 +1,8 @@
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# Code generated by Microsoft (R) AutoRest Code Generator.
+# Changes may cause incorrect behavior and will be lost if the code is regenerated.
+# --------------------------------------------------------------------------
+from ._version import VERSION
+__version__ = VERSION

package/autorest/multiclient/templates/version.py.jinja2

@@ -0,0 +1,8 @@
+# coding=utf-8
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+VERSION = "{{ package_version }}"

package/autorest/postprocess/__init__.py

@@ -66,7 +66,7 @@ class PostProcessPlugin(Plugin):  # pylint: disable=abstract-method
             init_file = next(d for d in dir.iterdir() if d.name == "__init__.py")
             # we don't care about pkgutil inits, we skip over them
             file_content = self.read_file(init_file.relative_to(self.output_folder))
-            if not "pkgutil" in file_content:
+            if "pkgutil" not in file_content:
                 return dir, namespace
         except StopIteration:
             pass

package/autorest/preprocess/__init__.py

@@ -6,10 +6,12 @@
 """The preprocessing autorest plugin.
 """
 from typing import Callable, Dict, Any, List, Optional
-from .helpers import to_snake_case, pad_reserved_words, add_redefined_builtin_info
+from .._utils import to_snake_case
+from .helpers import pad_reserved_words, add_redefined_builtin_info
 from .python_mappings import PadType
 
-from .. import YamlUpdatePlugin, PluginAutorest
+from .. import YamlUpdatePlugin, YamlUpdatePluginAutorest
+from .._utils import parse_args
 
 
 def _remove_paging_maxpagesize(yaml_data: Dict[str, Any]) -> None:
@@ -230,10 +232,18 @@ class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
         self.update_operation_groups(yaml_data)
 
 
-class PreProcessPluginAutorest(PluginAutorest, PreProcessPlugin):
+class PreProcessPluginAutorest(YamlUpdatePluginAutorest, PreProcessPlugin):
     def get_options(self) -> Dict[str, Any]:
         options = {
             "version-tolerant": self._autorestapi.get_boolean_value("version-tolerant"),
             "azure-arm": self._autorestapi.get_boolean_value("azure-arm"),
         }
         return {k: v for k, v in options.items() if v is not None}
+
+
+if __name__ == "__main__":
+    # CADL pipeline will call this
+    args = parse_args()
+    PreProcessPlugin(
+        output_folder=args.output_folder, cadl_file=args.cadl_file
+    ).process()

package/autorest/preprocess/helpers.py

@@ -4,39 +4,9 @@
 # license information.
 # --------------------------------------------------------------------------
 from typing import Any, Dict
-import re
 from .python_mappings import PadType, RESERVED_WORDS, REDEFINED_BUILTINS
 
 
-def to_snake_case(name: str) -> str:
-    def replace_upper_characters(m) -> str:
-        match_str = m.group().lower()
-        if m.start() > 0 and name[m.start() - 1] == "_":
-            # we are good if a '_' already exists
-            return match_str
-        # the first letter should not have _
-        prefix = "_" if m.start() > 0 else ""
-
-        # we will add an extra _ if there are multiple upper case chars together
-        next_non_upper_case_char_location = m.start() + len(match_str)
-        if (
-            len(match_str) > 2
-            and len(name) - next_non_upper_case_char_location > 1
-            and name[next_non_upper_case_char_location].isalpha()
-        ):
-
-            return (
-                prefix
-                + match_str[: len(match_str) - 1]
-                + "_"
-                + match_str[len(match_str) - 1]
-            )
-
-        return prefix + match_str
-
-    return re.sub("[A-Z]+", replace_upper_characters, name)
-
-
 def pad_reserved_words(name: str, pad_type: PadType):
     # we want to pad hidden variables as well
     if not name:

package/install.py

@@ -6,8 +6,8 @@
 # license information.
 # --------------------------------------------------------------------------
 import sys
-if not sys.version_info >= (3, 6, 0):
-    raise Exception("Autorest for Python extension requires Python 3.6 at least")
+if not sys.version_info >= (3, 7, 0):
+    raise Exception("Autorest for Python extension requires Python 3.7 at least")
 
 try:
     import pip
@@ -20,7 +20,7 @@ except ImportError:
     raise Exception("Your Python installation doesn't have venv available")
 
 
-# Now we have pip and Py >= 3.6, go to work
+# Now we have pip and Py >= 3.7, go to work
 
 import subprocess
 from pathlib import Path

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@autorest/python",
-    "version": "6.0.1",
+    "version": "6.1.0",
     "description": "The Python extension for generators in AutoRest.",
     "scripts": {
         "prepare": "node run-python3.js prepare.py",

package/prepare.py

@@ -6,8 +6,8 @@
 # license information.
 # --------------------------------------------------------------------------
 import sys
-if not sys.version_info >= (3, 6, 0):
-    raise Exception("Autorest for Python extension requires Python 3.6 at least")
+if not sys.version_info >= (3, 7, 0):
+    raise Exception("Autorest for Python extension requires Python 3.7 at least")
 
 from pathlib import Path
 import venv

package/requirements.txt

@@ -1,14 +1,13 @@
-black==21.12b0
-click==8.0.3
-docutils==0.18.1
-Jinja2==3.0.3
+black==22.6.0
+click==8.1.3
+docutils==0.18
+Jinja2==3.1.2
 json-rpc==1.13.0
-m2r==0.2.1
-MarkupSafe==2.0.1
+m2r2==0.3.2
+MarkupSafe==2.1.1
 mistune==0.8.4
 mypy-extensions==0.4.3
 pathspec==0.9.0
-platformdirs==2.4.0
+platformdirs==2.5.2
 PyYAML==6.0
-tomli==1.2.2
-typing-extensions==4.0.1
+tomli==2.0.1

package/run-python3.js

@@ -9,8 +9,8 @@
 const cp = require("child_process");
 const extension = require("@autorest/system-requirements");
 
-async function runPython3(scriptName, debug = "") {
-  const command = await extension.patchPythonPath(["python", scriptName, debug], { version: ">=3.6", environmentVariable: "AUTOREST_PYTHON_EXE" });
+async function runPython3(scriptName, ...args) {
+  const command = await extension.patchPythonPath(["python", scriptName, ...args], { version: ">=3.7", environmentVariable: "AUTOREST_PYTHON_EXE" });
   cp.execSync(command.join(" "), {
     stdio: [0, 1, 2]
   });

package/setup.py

@@ -36,7 +36,6 @@ setup(
         'Development Status :: 4 - Beta',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'License :: OSI Approved :: MIT License',
@@ -48,8 +47,8 @@ setup(
         "json-rpc",
         "Jinja2 >= 2.11", # I need "include" and auto-context + blank line are not indented by default
         "pyyaml",
-        "mistune < 2.0.0",  # Need to pin mistune's max version so m2r doesn't break
-        "m2r",
+        "m2r2",
         "black",
+        "docutils<0.19", # m2r2 fails with docutils 0.19
     ],
 )

package/start.py

@@ -6,8 +6,8 @@
 # license information.
 # --------------------------------------------------------------------------
 import sys
-if not sys.version_info >= (3, 6, 0):
-    raise Exception("Autorest for Python extension requires Python 3.6 at least")
+if not sys.version_info >= (3, 7, 0):
+    raise Exception("Autorest for Python extension requires Python 3.7 at least")
 
 from pathlib import Path
 import venv

package/autorest/multiapi/serializers/multiapi_serializer.py

@@ -1,121 +0,0 @@
-# -------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for
-# license information.
-# --------------------------------------------------------------------------
-from typing import Any, Dict
-from pathlib import Path
-from jinja2 import Environment, PackageLoader
-
-from ...jsonrpc import AutorestAPI
-from ... import ReaderAndWriter, ReaderAndWriterAutorest
-
-
-class MultiAPISerializer(ReaderAndWriter):  # pylint: disable=abstract-method
-    def __init__(
-        self,
-        conf: Dict[str, Any],
-        async_mode: bool,
-        service_client_filename: str,
-        **kwargs: Any
-    ):
-        super().__init__(**kwargs)
-        self.conf = conf
-        self.async_mode = async_mode
-        self.service_client_filename = service_client_filename
-        self.env = Environment(
-            loader=PackageLoader("autorest.multiapi", "templates"),
-            keep_trailing_newline=True,
-            line_statement_prefix="##",
-            line_comment_prefix="###",
-            trim_blocks=True,
-            lstrip_blocks=True,
-        )
-
-    def _get_file_path(self, filename: str) -> Path:
-        if self.async_mode:
-            return Path("aio") / filename
-        return Path(filename)
-
-    def serialize(self):
-        self.write_file(
-            self._get_file_path("__init__.py"), self.serialize_multiapi_init()
-        )
-
-        service_client_filename_with_py_extension = self.service_client_filename + ".py"
-        self.write_file(
-            self._get_file_path(service_client_filename_with_py_extension),
-            self.serialize_multiapi_client(),
-        )
-
-        configuration_filename = "_configuration.py"
-        self.write_file(
-            self._get_file_path(configuration_filename),
-            self.serialize_multiapi_config(),
-        )
-
-        operation_mixins_filename = "_operations_mixin.py"
-        if self.conf["mixin_operations"]:
-            self.write_file(
-                self._get_file_path(operation_mixins_filename),
-                self.serialize_multiapi_operation_mixins(),
-            )
-
-        if self.read_file("_version.py"):
-            self.write_file("_version.py", self.read_file("_version.py"))
-        elif self.read_file("version.py"):
-            self.write_file("_version.py", self.read_file("version.py"))
-        else:
-            self.write_file(Path("_version.py"), self.serialize_multiapi_version())
-
-        # don't erase patch file
-        if self.read_file("_patch.py"):
-            self.write_file("_patch.py", self.read_file("_patch.py"))
-
-        self.write_file(Path("models.py"), self.serialize_multiapi_models())
-
-        self.write_file(Path("py.typed"), "# Marker file for PEP 561.")
-
-    def serialize_multiapi_init(self) -> str:
-        template = self.env.get_template("multiapi_init.py.jinja2")
-        return template.render(
-            service_client_filename=self.service_client_filename,
-            client_name=self.conf["client_name"],
-            async_mode=self.async_mode,
-        )
-
-    def serialize_multiapi_client(self) -> str:
-        template = self.env.get_template("multiapi_service_client.py.jinja2")
-        return template.render(**self.conf, async_mode=self.async_mode)
-
-    def serialize_multiapi_config(self) -> str:
-        template = self.env.get_template("multiapi_config.py.jinja2")
-        return template.render(**self.conf, async_mode=self.async_mode)
-
-    def serialize_multiapi_models(self) -> str:
-        template = self.env.get_template("multiapi_models.py.jinja2")
-        return template.render(**self.conf)
-
-    def serialize_multiapi_version(self) -> str:
-        template = self.env.get_template("multiapi_version.py.jinja2")
-        return template.render()
-
-    def serialize_multiapi_operation_mixins(self) -> str:
-        template = self.env.get_template("multiapi_operations_mixin.py.jinja2")
-        return template.render(**self.conf, async_mode=self.async_mode)
-
-
-class MultiAPISerializerAutorest(MultiAPISerializer, ReaderAndWriterAutorest):
-    def __init__(
-        self,
-        autorestapi: AutorestAPI,
-        conf: Dict[str, Any],
-        async_mode: bool,
-        service_client_filename: str,
-    ):
-        super().__init__(
-            autorestapi=autorestapi,
-            conf=conf,
-            async_mode=async_mode,
-            service_client_filename=service_client_filename,
-        )