@autorest/python 6.19.0 → 6.21.0

package/generator/pygen/codegen/templates/test.py.jinja2

@@ -29,9 +29,9 @@ class {{ test.test_class_name }}({{ test.base_test_class_name }}):
     {% endif %}
     @recorded_by_proxy{{ async_suffix }}
     {% if code_model.options["azure_arm"] %}
-    {{ async }}def test_{{ testcase.operation.name }}(self, resource_group):
+    {{ async }}def test_{{ testcase.name }}(self, resource_group):
     {% else %}
-    {{ async }}def test_{{ testcase.operation.name }}(self, {{ prefix_lower }}_endpoint):
+    {{ async }}def test_{{ testcase.name }}(self, {{ prefix_lower }}_endpoint):
         {{ client_var }} = self.{{ test.create_client_name }}(endpoint={{ prefix_lower }}_endpoint)
     {% endif %}
         {{testcase.response }}{{ client_var }}{{ testcase.operation_group_prefix }}.{{ testcase.operation.name }}(

package/generator/pygen/m2r.py

@@ -27,7 +27,7 @@ class GeneratorRenderer(m2r2.RestRenderer):
         return f":code:`{html}`"
 
 
-class M2R(YamlUpdatePlugin):  # pylint: disable=abstract-method
+class M2R(YamlUpdatePlugin):
     """A plugin to convert any description and summary from MD to RST."""
 
     def update_yaml(self, yaml_data: Dict[str, Any]) -> None:

package/generator/pygen/postprocess/__init__.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-from typing import Tuple, Any, Dict
+from typing import Tuple, Any
 from pathlib import Path
 import os
 import shutil
@@ -28,7 +28,7 @@ def format_file(file: Path, file_content: str) -> str:
     return file_content
 
 
-class PostProcessPlugin(Plugin):  # pylint: disable=abstract-method
+class PostProcessPlugin(Plugin):
     def __init__(self, **kwargs: Any):
         super().__init__(**kwargs)
         output_folder_uri = self.options["outputFolderUri"]

package/generator/pygen/postprocess/venvtools.py

@@ -52,9 +52,7 @@ def create(
     return builder.context
 
 
-def python_run(  # pylint: disable=inconsistent-return-statements
-    venv_context, module, command, directory=_ROOT_DIR
-) -> Optional[str]:
+def python_run(venv_context, module, command, directory=_ROOT_DIR) -> Optional[str]:
     try:
         cmd_line = [
             venv_context.env_exe,

package/generator/pygen/preprocess/__init__.py

@@ -163,7 +163,7 @@ def has_multi_part_content_type(yaml_data: Dict[str, Any]) -> bool:
     return any(ct for ct in yaml_data.get("contentTypes", []) if ct == "multipart/form-data")
 
 
-class PreProcessPlugin(YamlUpdatePlugin):  # pylint: disable=abstract-method
+class PreProcessPlugin(YamlUpdatePlugin):
     """Add Python naming information."""
 
     @property

package/generator/pygen/utils.py

@@ -82,9 +82,7 @@ def parse_args(
         return value
 
     unknown_args_ret = {
-        ua.strip("--").split("=", maxsplit=1)[0]: _get_value(  # pylint: disable=bad-str-strip-call
-            ua.strip("--").split("=", maxsplit=1)[1]  # pylint: disable=bad-str-strip-call
-        )
+        ua.strip("--").split("=", maxsplit=1)[0]: _get_value(ua.strip("--").split("=", maxsplit=1)[1])
         for ua in unknown_args
     }
     return args, unknown_args_ret
@@ -107,6 +105,7 @@ KNOWN_TYPES: Dict[str, Dict[str, Any]] = {
 }
 
 JSON_REGEXP = re.compile(r"^(application|text)/(.+\+)?json$")
+XML_REGEXP = re.compile(r"^(application|text)/(.+\+)?xml$")
 
 
 def build_policies(
@@ -151,3 +150,11 @@ def build_policies(
 
 def extract_original_name(name: str) -> str:
     return name[1 : -len("_initial")]
+
+
+def json_serializable(content_type: str) -> bool:
+    return bool(JSON_REGEXP.match(content_type.split(";")[0].strip().lower()))
+
+
+def xml_serializable(content_type: str) -> bool:
+    return bool(XML_REGEXP.match(content_type.split(";")[0].strip().lower()))

package/generator/setup.py

@@ -15,7 +15,7 @@ from setuptools import setup, find_packages
 
 # Version extraction inspired from 'requests'
 with open(os.path.join("pygen", "_version.py"), "r") as fd:
-    version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1)
+    version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1)  # type: ignore
 
 if not version:
     raise RuntimeError("Cannot find version information")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorest/python",
-  "version": "6.19.0",
+  "version": "6.21.0",
   "description": "The Python extension for generators in AutoRest.",
   "main": "index.js",
   "repository": {
@@ -26,7 +26,8 @@
   "devDependencies": {
     "@microsoft.azure/autorest.testserver": "^3.3.46",
     "typescript": "~5.1.3",
-    "@azure-tools/typespec-python": "^0.29.0"
+    "chalk": "5.3.0",
+    "@azure-tools/typespec-python": "^0.31.0"
   },
   "files": [
     "autorest/**/*.py",
@@ -38,9 +39,11 @@
   ],
   "scripts": {
     "start": "node ./scripts/run-python3.js ./scripts/start.py",
-    "build": "node ./scripts/copy-generator.js --force",
-    "install": "node ./scripts/copy-generator.js && node ./scripts/run-python3.js ./scripts/install.py",
+    "build": "tsx ./scripts/copy-generator.ts --force",
+    "install": "tsx ./scripts/copy-generator.ts && node ./scripts/run-python3.js ./scripts/install.py",
     "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug",
-    "test": "tsx ./scripts/run-tests.ts"
+    "test": "tsx ./scripts/eng/run-tests.ts --validFolders azure/legacy azure/version-tolerant vanilla/legacy vanilla/version-tolerant dpg/version-tolerant",
+    "lint": "tsx ./scripts/eng/lint.ts --folderName autorest --skipEslint true",
+    "format": "tsx ./scripts/eng/format.ts"
   }
 }

package/scripts/copy-generator.ts

@@ -0,0 +1,24 @@
+import { copyFileSync, readdirSync } from "fs";
+import { existsSync, removeSync, copySync } from "fs-extra";
+import { join } from "path";
+
+const force: boolean = process.argv[2] === "--force";
+
+function copyAndCreateDir(sourceDir: string, destDir: string) {
+    // Delete the destination directory if it exists
+    if (existsSync(destDir)) {
+        if (force) removeSync(destDir);
+        else process.exit(0);
+    }
+
+    // Copy the source directory to the destination directory
+    copySync(sourceDir, destDir);
+}
+
+const typespecModulePath: string = join(__dirname, "..", "node_modules", "@azure-tools", "typespec-python");
+
+// Copy the generator directory
+copyAndCreateDir(join(typespecModulePath, "generator"), join(__dirname, "..", "generator"));
+
+// Copy the scripts directory
+copyAndCreateDir(join(typespecModulePath, "scripts", "eng"), join(__dirname, "..", "scripts", "eng"));

package/scripts/eng/format.ts

@@ -0,0 +1,5 @@
+/* eslint-disable no-console */
+
+import { runCommand } from "./utils.js";
+
+runCommand("black .", "black");

package/scripts/eng/lint.ts

@@ -0,0 +1,75 @@
+/* eslint-disable no-console */
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { runCommand, executeCommand } from "./utils.js";
+
+interface Arguments {
+    folderName: string;
+    command?: "pylint" | "mypy" | "pyright" | "eslint";
+    skipWarning?: boolean;
+    skipEslint?: boolean;
+}
+
+const validCommands = ["pylint", "mypy", "pyright", "eslint"];
+
+// PARSE INPUT ARGUMENTS
+const argv = yargs(hideBin(process.argv))
+    .option("folderName", {
+        type: "string",
+        choices: ["generator", "autorest"],
+        description: "Specify the flavor",
+        default: "generator",
+    })
+    .option("command", {
+        alias: "c",
+        type: "string",
+        choices: validCommands,
+        description: "Specify the command to run",
+    })
+    .option("skipWarning", {
+        alias: "s",
+        type: "boolean",
+        description: "Skip to check warnings",
+    })
+    .option("skipEslint", {
+        alias: "e",
+        type: "boolean",
+        description: "Skip to check eslint",
+    }).argv as Arguments;
+
+export function pylint() {
+    runCommand(`pylint ${argv.folderName}/ --rcfile ./scripts/eng/pylintrc`, "pylint");
+}
+
+export function mypy() {
+    runCommand(`mypy ${argv.folderName}/ --config-file ./scripts/eng/mypy.ini`, "mypy");
+}
+
+export function pyright() {
+    runCommand(`pyright ${argv.folderName}/ -p ./scripts/eng/pyrightconfig.json`, "pyright");
+}
+
+export function eslint() {
+    // const checkWarning = argv.skipWarning ? "" : "--max-warnings=0";
+    const checkWarning = "";
+    executeCommand(`npx eslint . --ext .ts ${checkWarning} `, "eslint");
+}
+
+if (argv.command === "pylint") {
+    pylint();
+} else if (argv.command === "mypy") {
+    mypy();
+} else if (argv.command === "pyright") {
+    pyright();
+} else if (argv.command === "eslint") {
+    if (!argv.skipEslint) {
+        eslint();
+    }
+} else {
+    pylint();
+    mypy();
+    pyright();
+    if (!argv.skipEslint) {
+        eslint();
+    }
+}

package/scripts/eng/mypy.ini

@@ -0,0 +1,38 @@
+# global configurations
+[mypy]
+python_version = 3.8
+
+
+# module level configuratiohns
+[mypy-jsonrpc.*]
+ignore_missing_imports = True
+
+[mypy-ptvsd.*]
+ignore_missing_imports = True
+
+[mypy-debugpy.*]
+ignore_missing_imports = True
+
+[mypy-m2r2.*]
+ignore_missing_imports = True
+
+[mypy-autorest.common.utils.*]
+ignore_missing_imports = True
+
+[mypy-autorest.common.python_mappings.*]
+ignore_missing_imports = True
+
+[mypy-pygen.codegen.models.*]
+ignore_missing_imports = True
+
+[mypy-setuptools]
+ignore_missing_imports = True
+
+[mypy-*._patch]
+ignore_missing_imports = True
+
+[mypy-pygen.*]
+ignore_missing_imports = True
+
+[mypy-yaml.*]
+ignore_missing_imports = True

package/scripts/eng/pylintrc

@@ -0,0 +1,58 @@
+[MASTER]
+py-version=3.8
+ignore-patterns=test_*,conftest,setup
+reports=no
+
+# PYLINT DIRECTORY BLACKLIST.
+ignore=_generated,samples,examples,test,tests,doc,.tox,generated_samples
+
+[MESSAGES CONTROL]
+
+# Add enable for useless disables
+enable=useless-suppression
+
+# For all codes, run 'pylint --list-msgs' or go to 'https://pylint.readthedocs.io/en/latest/reference_guide/features.html'
+# locally-disabled: Warning locally suppressed using disable-msg
+# cyclic-import: because of https://github.com/PyCQA/pylint/issues/850
+# too-many-arguments: Due to the nature of the CLI many commands have large arguments set which reflect in large arguments set in corresponding methods.
+# Let's black deal with bad-continuation
+disable=useless-object-inheritance,missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code,too-few-public-methods,consider-using-f-string,super-with-arguments,redefined-builtin,import-outside-toplevel,client-suffix-needed,unnecessary-dunder-call,unnecessary-ellipsis,disallowed-name,consider-using-max-builtin
+
+[FORMAT]
+max-line-length=120
+
+[VARIABLES]
+# Tells whether we should check for unused import in __init__ files.
+init-import=yes
+
+[DESIGN]
+# Maximum number of locals for function / method body
+max-locals=25
+# Maximum number of branch for function / method body
+max-branches=20
+# Maximum number of instance attributes for class
+max-attributes=10
+# Maximum number of ancestors
+max-parents=15
+
+[SIMILARITIES]
+min-similarity-lines=10
+
+[BASIC]
+# Naming hints based on PEP 8 (https://www.python.org/dev/peps/pep-0008/#naming-conventions).
+# Consider these guidelines and not hard rules. Read PEP 8 for more details.
+
+# The invalid-name checker must be **enabled** for these hints to be used.
+include-naming-hint=yes
+
+# keep short; underscores are discouraged
+module-naming-style=snake_case
+const-naming-style=UPPER_CASE
+class-naming-style=PascalCase
+class-attribute-naming-style=snake_case
+attr-naming-style=snake_case
+method-naming-style=snake_case
+function-naming-style=snake_case
+argument-naming-style=snake_case
+variable-naming-style=snake_case
+inlinevar-naming-style=snake_case

package/scripts/eng/pyrightconfig.json

@@ -0,0 +1,6 @@
+{
+    "reportUnnecessaryCast": "warning",
+    "reportTypeCommentUsage": true,
+    "reportMissingImports": false,
+    "pythonVersion": "3.8"
+}

package/scripts/eng/regenerate.ts

@@ -0,0 +1,292 @@
+/* eslint-disable no-console */
+import { exec as execCallback } from "child_process";
+import { promisify } from "util";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { dirname, join, relative, resolve } from "path";
+import { promises, rmSync } from "fs";
+import { fileURLToPath } from "url";
+
+// Promisify the exec function
+const exec = promisify(execCallback);
+
+// Get the directory of the current file
+const PLUGIN_DIR = resolve(fileURLToPath(import.meta.url), "../../../");
+const CADL_RANCH_DIR = resolve(PLUGIN_DIR, "node_modules/@azure-tools/cadl-ranch-specs/http");
+interface TspCommand {
+    outputDir: string;
+    command: string;
+}
+
+const EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, string>[]> = {
+    "resiliency/srv-driven/old.tsp": {
+        "package-name": "resiliency-srv-driven1",
+        "package-mode": "azure-dataplane",
+        "package-pprint-name": "ResiliencySrvDriven1",
+    },
+    "resiliency/srv-driven": {
+        "package-name": "resiliency-srv-driven2",
+        "package-mode": "azure-dataplane",
+        "package-pprint-name": "ResiliencySrvDriven2",
+    },
+    "authentication/http/custom": {
+        "package-name": "authentication-http-custom",
+    },
+    "authentication/union": {
+        "package-name": "authentication-union",
+    },
+    "type/array": {
+        "package-name": "typetest-array",
+    },
+    "type/dictionary": {
+        "package-name": "typetest-dictionary",
+    },
+    "type/enum/extensible": {
+        "package-name": "typetest-enum-extensible",
+    },
+    "type/enum/fixed": {
+        "package-name": "typetest-enum-fixed",
+    },
+    "type/model/empty": {
+        "package-name": "typetest-model-empty",
+    },
+    "type/model/inheritance/enum-discriminator": {
+        "package-name": "typetest-model-enumdiscriminator",
+    },
+    "type/model/inheritance/nested-discriminator": {
+        "package-name": "typetest-model-nesteddiscriminator",
+    },
+    "type/model/inheritance/not-discriminated": {
+        "package-name": "typetest-model-notdiscriminated",
+    },
+    "type/model/inheritance/single-discriminator": {
+        "package-name": "typetest-model-singlediscriminator",
+    },
+    "type/model/inheritance/recursive": {
+        "package-name": "typetest-model-recursive",
+    },
+    "type/model/usage": {
+        "package-name": "typetest-model-usage",
+    },
+    "type/model/visibility": [
+        { "package-name": "typetest-model-visibility" },
+        { "package-name": "headasbooleantrue", "head-as-boolean": "true" },
+        { "package-name": "headasbooleanfalse", "head-as-boolean": "false" },
+    ],
+    "type/property/nullable": {
+        "package-name": "typetest-property-nullable",
+    },
+    "type/property/optionality": {
+        "package-name": "typetest-property-optional",
+    },
+    "type/property/additional-properties": {
+        "package-name": "typetest-property-additionalproperties",
+    },
+    "type/scalar": {
+        "package-name": "typetest-scalar",
+    },
+    "type/property/value-types": {
+        "package-name": "typetest-property-valuetypes",
+    },
+    "type/union": {
+        "package-name": "typetest-union",
+    },
+    "azure/core/lro/rpc": {
+        "package-name": "azurecore-lro-rpc",
+    },
+    "client/structure/multi-client": {
+        "package-name": "client-structure-multiclient",
+    },
+    "client/structure/renamed-operation": {
+        "package-name": "client-structure-renamedoperation",
+    },
+    "client/structure/two-operation-group": {
+        "package-name": "client-structure-twooperationgroup",
+    },
+    "mgmt/sphere": [{ "package-name": "azure-mgmt-spheredpg" }],
+};
+
+function toPosix(dir: string): string {
+    return dir.replace(/\\/g, "/");
+}
+
+function getEmitterOption(spec: string): Record<string, string>[] {
+    const relativeSpec = toPosix(relative(CADL_RANCH_DIR, spec));
+    const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") ? relativeSpec : dirname(relativeSpec);
+    const result = EMITTER_OPTIONS[key] || [{}];
+    return Array.isArray(result) ? result : [result];
+}
+
+// Function to execute CLI commands asynchronously
+async function executeCommand(tspCommand: TspCommand): Promise<void> {
+    try {
+        rmSync(tspCommand.outputDir, { recursive: true, force: true });
+    } catch (error) {
+        console.error(`rm error: ${error}`);
+    }
+    try {
+        console.log(`exec: ${tspCommand.command}`);
+        const { stdout, stderr } = await exec(tspCommand.command);
+        if (stdout) console.log(`stdout: ${stdout}`);
+        if (stderr) console.error(`stderr: ${stderr}`);
+    } catch (error) {
+        console.error(`exec error: ${error}`);
+        throw error;
+    }
+}
+
+interface RegenerateFlagsInput {
+    flavor?: "azure" | "unbranded";
+    debug?: boolean;
+    name?: string;
+}
+
+interface RegenerateFlags {
+    flavor: "azure" | "unbranded";
+    debug: boolean;
+    name?: string;
+}
+
+const SpecialFlags: Record<string, Record<string, any>> = {
+    azure: {
+        "generate-test": true,
+        "generate-sample": true,
+    },
+};
+
+async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promise<string[]> {
+    const subdirectories: string[] = [];
+
+    async function searchDir(currentDir: string) {
+        const items = await promises.readdir(currentDir, { withFileTypes: true });
+
+        const promisesArray = items.map(async (item) => {
+            const subDirPath = join(currentDir, item.name);
+            if (item.isDirectory()) {
+                const mainTspPath = join(subDirPath, "main.tsp");
+                const clientTspPath = join(subDirPath, "client.tsp");
+
+                const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath));
+                if (flags.flavor === "unbranded" && mainTspRelativePath.includes("azure")) return;
+
+                // after fix test generation for nested operation group, remove this check
+                if (mainTspRelativePath.includes("client-operation-group")) return;
+
+                const hasMainTsp = await promises
+                    .access(mainTspPath)
+                    .then(() => true)
+                    .catch(() => false);
+                const hasClientTsp = await promises
+                    .access(clientTspPath)
+                    .then(() => true)
+                    .catch(() => false);
+
+                if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) {
+                    if (mainTspRelativePath.includes("resiliency/srv-driven")) {
+                        subdirectories.push(resolve(subDirPath, "old.tsp"));
+                    }
+                    if (hasClientTsp) {
+                        subdirectories.push(resolve(subDirPath, "client.tsp"));
+                    } else if (hasMainTsp) {
+                        subdirectories.push(resolve(subDirPath, "main.tsp"));
+                    }
+                }
+
+                // Recursively search in the subdirectory
+                await searchDir(subDirPath);
+            }
+        });
+
+        await Promise.all(promisesArray);
+    }
+
+    await searchDir(baseDir);
+    return subdirectories;
+}
+
+function defaultPackageName(spec: string): string {
+    return toPosix(relative(CADL_RANCH_DIR, dirname(spec)))
+        .replace(/\//g, "-")
+        .toLowerCase();
+}
+
+interface EmitterConfig {
+    optionsStr: string;
+    outputDir: string;
+}
+
+function addOptions(spec: string, generatedFolder: string, flags: RegenerateFlags): EmitterConfig[] {
+    const emitterConfigs: EmitterConfig[] = [];
+    for (const config of getEmitterOption(spec)) {
+        const options: Record<string, string> = { ...config };
+        options["flavor"] = flags.flavor;
+        for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) {
+            options[k] = v;
+        }
+        if (options["emitter-output-dir"] === undefined) {
+            const packageName = options["package-name"] || defaultPackageName(spec);
+            options["emitter-output-dir"] = toPosix(`${generatedFolder}/test/${flags.flavor}/generated/${packageName}`);
+        }
+        if (flags.debug) {
+            options["debug"] = "true";
+        }
+        if (flags.flavor === "unbranded") {
+            options["company-name"] = "Unbranded";
+        }
+        options["examples-dir"] = toPosix(join(dirname(spec), "examples"));
+        const configs = Object.entries(options).flatMap(([k, v]) => {
+            return `--option @azure-tools/typespec-python.${k}=${v}`;
+        });
+        emitterConfigs.push({
+            optionsStr: configs.join(" "),
+            outputDir: options["emitter-output-dir"],
+        });
+    }
+    return emitterConfigs;
+}
+function _getCmdList(spec: string, flags: RegenerateFlags): TspCommand[] {
+    return addOptions(spec, PLUGIN_DIR, flags).map((option) => {
+        return {
+            outputDir: option.outputDir,
+            command: `tsp compile ${spec} --emit=${toPosix(PLUGIN_DIR)} ${option.optionsStr}`,
+        };
+    });
+}
+
+async function regenerate(flags: RegenerateFlagsInput): Promise<void> {
+    if (flags.flavor === undefined) {
+        await regenerate({ ...flags, flavor: "azure" });
+        await regenerate({ ...flags, flavor: "unbranded" });
+    } else {
+        const flagsResolved = { debug: false, flavor: flags.flavor, ...flags };
+        const CADL_RANCH_DIR = resolve(PLUGIN_DIR, "node_modules/@azure-tools/cadl-ranch-specs/http");
+        const subdirectories = await getSubdirectories(CADL_RANCH_DIR, flagsResolved);
+        const cmdList: TspCommand[] = subdirectories.flatMap((subdirectory) =>
+            _getCmdList(subdirectory, flagsResolved),
+        );
+        const PromiseCommands = cmdList.map((tspCommand) => executeCommand(tspCommand));
+        await Promise.all(PromiseCommands);
+    }
+}
+
+// PARSE INPUT ARGUMENTS
+const argv = yargs(hideBin(process.argv))
+    .option("flavor", {
+        type: "string",
+        choices: ["azure", "unbranded"],
+        description: "Specify the flavor",
+    })
+    .option("debug", {
+        alias: "d",
+        type: "boolean",
+        description: "Debug mode",
+    })
+    .option("name", {
+        alias: "n",
+        type: "string",
+        description: "Specify filename if you only want to generate a subset",
+    }).argv;
+
+regenerate(argv as RegenerateFlags)
+    .then(() => console.log("Regeneration successful"))
+    .catch((error) => console.error(`Regeneration failed: ${error.message}`));