@azure-tools/typespec-python 0.28.0 → 0.30.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@azure-tools/typespec-python",
-  "version": "0.28.0",
+  "version": "0.30.0",
   "author": "Microsoft Corporation",
   "description": "TypeSpec emitter for Python SDKs",
   "homepage": "https://github.com/Azure/autorest.python",
@@ -36,60 +36,53 @@
     "@azure-tools/typespec-azure-core": ">=0.45.0 <1.0.0",
     "@azure-tools/typespec-azure-resource-manager": ">=0.45.0 <1.0.0",
     "@azure-tools/typespec-autorest": ">=0.45.0 <1.0.0",
-    "@azure-tools/typespec-client-generator-core": ">=0.45.1 <1.0.0",
+    "@azure-tools/typespec-client-generator-core": ">=0.45.4 <1.0.0",
     "@azure-tools/typespec-azure-rulesets": ">=0.45.0 <3.0.0",
-    "@typespec/compiler": ">=0.59.0 <1.0.0",
+    "@typespec/compiler": ">=0.59.1 <1.0.0",
     "@typespec/http": ">=0.59.0 <1.0.0",
     "@typespec/rest": ">=0.59.0 <1.0.0",
     "@typespec/versioning": ">=0.59.0 <1.0.0",
     "@typespec/openapi": ">=0.59.0 <1.0.0"
   },
-  "dependenciesMeta": {
-    "@azure-tools/typespec-client-generator-core": {
-      "injected": true
-    }
-  },
   "dependencies": {
     "js-yaml": "~4.1.0",
     "@typespec/openapi3": "~0.59.0",
-    "@autorest/system-requirements": "~1.0.2",
-    "fs-extra": "~11.2.0",
-    "semver": "~7.6.2"
+    "semver": "~7.6.2",
+    "tsx": "4.17.0"
   },
   "devDependencies": {
     "@azure-tools/typespec-azure-resource-manager": "~0.45.0",
     "@azure-tools/typespec-autorest": "~0.45.0",
-    "@azure-tools/cadl-ranch-expect": "~0.14.1",
-    "@azure-tools/cadl-ranch-specs": "~0.34.5",
+    "@azure-tools/cadl-ranch-expect": "~0.15.1",
+    "@azure-tools/cadl-ranch-specs": "~0.36.1",
     "@types/js-yaml": "~4.0.5",
-    "@types/mocha": "~10.0.1",
     "@types/node": "^18.16.3",
     "@types/yargs": "17.0.32",
+    "@types/semver": "7.5.8",
     "@typespec/eslint-config-typespec": "~0.55.0",
     "@typespec/openapi": "~0.59.0",
     "c8": "~7.13.0",
-    "eslint": "^8.57.0",
-    "mocha": "~10.2.0",
     "rimraf": "~5.0.0",
     "typescript": "~5.1.3",
     "@azure-tools/typespec-azure-core": "~0.45.0",
-    "@azure-tools/typespec-client-generator-core": "0.45.1",
-    "@typespec/compiler": "~0.59.0",
+    "@azure-tools/typespec-client-generator-core": "0.45.4",
+    "@typespec/compiler": "~0.59.1",
     "@typespec/http": "~0.59.0",
     "@typespec/rest": "~0.59.0",
     "@typespec/versioning": "~0.59.0",
     "@azure-tools/typespec-azure-rulesets": "0.45.0",
-    "yargs": "~17.2.1"
+    "yargs": "~17.2.1",
+    "chalk": "5.3.0"
   },
   "scripts": {
-    "clean": "rimraf ./dist ./temp",
+    "clean": "rimraf ./dist ./temp ./venv ./node_modules",
     "build": "tsc -p .",
     "watch": "tsc -p . --watch",
-    "test": "mocha",
-    "test-official": "c8 mocha --forbid-only",
-    "lint": "eslint . --ext .ts --max-warnings=0",
+    "lint": "tsx ./scripts/eng/lint.ts",
     "lint:fix": "eslint . --fix --ext .ts",
-    "install": "node ./scripts/run-python3.cjs ./scripts/install.py",
-    "regenerate": "node ./dist/scripts/regenerate.js"
+    "format": "npx prettier **/*.ts --write && tsx ./scripts/eng/format.ts",
+    "install": "tsx ./scripts/run-python3.ts ./scripts/install.py",
+    "regenerate": "tsx ./scripts/eng/regenerate.ts",
+    "test": "tsx ./scripts/eng/run-tests.ts"
   }
 }

package/scripts/eng/format.ts

@@ -0,0 +1,5 @@
+/* eslint-disable no-console */
+
+import { runCommand } from "./utils.js";
+
+runCommand("black .", "black");

package/scripts/eng/lint.ts

@@ -0,0 +1,75 @@
+/* eslint-disable no-console */
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { runCommand, executeCommand } from "./utils.js";
+
+interface Arguments {
+    folderName: string;
+    command?: "pylint" | "mypy" | "pyright" | "eslint";
+    skipWarning?: boolean;
+    skipEslint?: boolean;
+}
+
+const validCommands = ["pylint", "mypy", "pyright", "eslint"];
+
+// PARSE INPUT ARGUMENTS
+const argv = yargs(hideBin(process.argv))
+    .option("folderName", {
+        type: "string",
+        choices: ["generator", "autorest"],
+        description: "Specify the flavor",
+        default: "generator",
+    })
+    .option("command", {
+        alias: "c",
+        type: "string",
+        choices: validCommands,
+        description: "Specify the command to run",
+    })
+    .option("skipWarning", {
+        alias: "s",
+        type: "boolean",
+        description: "Skip to check warnings",
+    })
+    .option("skipEslint", {
+        alias: "e",
+        type: "boolean",
+        description: "Skip to check eslint",
+    }).argv as Arguments;
+
+export function pylint() {
+    runCommand(`pylint ${argv.folderName}/ --rcfile ./scripts/eng/pylintrc`, "pylint");
+}
+
+export function mypy() {
+    runCommand(`mypy ${argv.folderName}/ --config-file ./scripts/eng/mypy.ini`, "mypy");
+}
+
+export function pyright() {
+    runCommand(`pyright ${argv.folderName}/ -p ./scripts/eng/pyrightconfig.json`, "pyright");
+}
+
+export function eslint() {
+    // const checkWarning = argv.skipWarning ? "" : "--max-warnings=0";
+    const checkWarning = "";
+    executeCommand(`npx eslint . --ext .ts ${checkWarning} `, "eslint");
+}
+
+if (argv.command === "pylint") {
+    pylint();
+} else if (argv.command === "mypy") {
+    mypy();
+} else if (argv.command === "pyright") {
+    pyright();
+} else if (argv.command === "eslint") {
+    if (!argv.skipEslint) {
+        eslint();
+    }
+} else {
+    pylint();
+    mypy();
+    pyright();
+    if (!argv.skipEslint) {
+        eslint();
+    }
+}

package/scripts/eng/mypy.ini

@@ -0,0 +1,38 @@
+# global configurations
+[mypy]
+python_version = 3.8
+
+
+# module level configuratiohns
+[mypy-jsonrpc.*]
+ignore_missing_imports = True
+
+[mypy-ptvsd.*]
+ignore_missing_imports = True
+
+[mypy-debugpy.*]
+ignore_missing_imports = True
+
+[mypy-m2r2.*]
+ignore_missing_imports = True
+
+[mypy-autorest.common.utils.*]
+ignore_missing_imports = True
+
+[mypy-autorest.common.python_mappings.*]
+ignore_missing_imports = True
+
+[mypy-pygen.codegen.models.*]
+ignore_missing_imports = True
+
+[mypy-setuptools]
+ignore_missing_imports = True
+
+[mypy-*._patch]
+ignore_missing_imports = True
+
+[mypy-pygen.*]
+ignore_missing_imports = True
+
+[mypy-yaml.*]
+ignore_missing_imports = True

package/scripts/eng/pylintrc

@@ -0,0 +1,58 @@
+[MASTER]
+py-version=3.8
+ignore-patterns=test_*,conftest,setup
+reports=no
+
+# PYLINT DIRECTORY BLACKLIST.
+ignore=_generated,samples,examples,test,tests,doc,.tox,generated_samples
+
+[MESSAGES CONTROL]
+
+# Add enable for useless disables
+enable=useless-suppression
+
+# For all codes, run 'pylint --list-msgs' or go to 'https://pylint.readthedocs.io/en/latest/reference_guide/features.html'
+# locally-disabled: Warning locally suppressed using disable-msg
+# cyclic-import: because of https://github.com/PyCQA/pylint/issues/850
+# too-many-arguments: Due to the nature of the CLI many commands have large arguments set which reflect in large arguments set in corresponding methods.
+# Let's black deal with bad-continuation
+disable=useless-object-inheritance,missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code,too-few-public-methods,consider-using-f-string,super-with-arguments,redefined-builtin,import-outside-toplevel,client-suffix-needed,unnecessary-dunder-call,unnecessary-ellipsis,disallowed-name,consider-using-max-builtin
+
+[FORMAT]
+max-line-length=120
+
+[VARIABLES]
+# Tells whether we should check for unused import in __init__ files.
+init-import=yes
+
+[DESIGN]
+# Maximum number of locals for function / method body
+max-locals=25
+# Maximum number of branch for function / method body
+max-branches=20
+# Maximum number of instance attributes for class
+max-attributes=10
+# Maximum number of ancestors
+max-parents=15
+
+[SIMILARITIES]
+min-similarity-lines=10
+
+[BASIC]
+# Naming hints based on PEP 8 (https://www.python.org/dev/peps/pep-0008/#naming-conventions).
+# Consider these guidelines and not hard rules. Read PEP 8 for more details.
+
+# The invalid-name checker must be **enabled** for these hints to be used.
+include-naming-hint=yes
+
+# keep short; underscores are discouraged
+module-naming-style=snake_case
+const-naming-style=UPPER_CASE
+class-naming-style=PascalCase
+class-attribute-naming-style=snake_case
+attr-naming-style=snake_case
+method-naming-style=snake_case
+function-naming-style=snake_case
+argument-naming-style=snake_case
+variable-naming-style=snake_case
+inlinevar-naming-style=snake_case

package/scripts/eng/pyrightconfig.json

@@ -0,0 +1,6 @@
+{
+    "reportUnnecessaryCast": "warning",
+    "reportTypeCommentUsage": true,
+    "reportMissingImports": false,
+    "pythonVersion": "3.8"
+}

package/scripts/{regenerate.ts → eng/regenerate.ts}

@@ -4,7 +4,7 @@ import { promisify } from "util";
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 import { dirname, join, relative, resolve } from "path";
-import { promises } from "fs";
+import { promises, rmSync } from "fs";
 import { fileURLToPath } from "url";
 
 // Promisify the exec function
@@ -13,6 +13,10 @@ const exec = promisify(execCallback);
 // Get the directory of the current file
 const PLUGIN_DIR = resolve(fileURLToPath(import.meta.url), "../../../");
 const CADL_RANCH_DIR = resolve(PLUGIN_DIR, "node_modules/@azure-tools/cadl-ranch-specs/http");
+interface TspCommand {
+    outputDir: string;
+    command: string;
+}
 
 const EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, string>[]> = {
     "resiliency/srv-driven/old.tsp": {
@@ -117,9 +121,15 @@ function getEmitterOption(spec: string): Record<string, string>[] {
 }
 
 // Function to execute CLI commands asynchronously
-async function executeCommand(command: string): Promise<void> {
+async function executeCommand(tspCommand: TspCommand): Promise<void> {
+    try {
+        rmSync(tspCommand.outputDir, { recursive: true, force: true });
+    } catch (error) {
+        console.error(`rm error: ${error}`);
+    }
     try {
-        const { stdout, stderr } = await exec(command);
+        console.log(`exec: ${tspCommand.command}`);
+        const { stdout, stderr } = await exec(tspCommand.command);
         if (stdout) console.log(`stdout: ${stdout}`);
         if (stderr) console.error(`stderr: ${stderr}`);
     } catch (error) {
@@ -162,6 +172,12 @@ async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promi
                 const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath));
                 if (flags.flavor === "unbranded" && mainTspRelativePath.includes("azure")) return;
 
+                // after xml support, remove this check
+                if (mainTspRelativePath.includes("xml")) return;
+
+                // after fix test generation for nested operation group, remove this check
+                if (mainTspRelativePath.includes("client-operation-group")) return;
+
                 const hasMainTsp = await promises
                     .access(mainTspPath)
                     .then(() => true)
@@ -200,8 +216,13 @@ function defaultPackageName(spec: string): string {
         .toLowerCase();
 }
 
-function addOptions(spec: string, generatedFolder: string, flags: RegenerateFlags): string[] {
-    const emitterConfigs: string[] = [];
+interface EmitterConfig {
+    optionsStr: string;
+    outputDir: string;
+}
+
+function addOptions(spec: string, generatedFolder: string, flags: RegenerateFlags): EmitterConfig[] {
+    const emitterConfigs: EmitterConfig[] = [];
     for (const config of getEmitterOption(spec)) {
         const options: Record<string, string> = { ...config };
         options["flavor"] = flags.flavor;
@@ -218,51 +239,42 @@ function addOptions(spec: string, generatedFolder: string, flags: RegenerateFlag
         if (flags.flavor === "unbranded") {
             options["company-name"] = "Unbranded";
         }
-        options["examples-directory"] = toPosix(join(dirname(spec), "examples"));
+        options["examples-dir"] = toPosix(join(dirname(spec), "examples"));
         const configs = Object.entries(options).flatMap(([k, v]) => {
             return `--option @azure-tools/typespec-python.${k}=${v}`;
         });
-        emitterConfigs.push(configs.join(" "));
+        emitterConfigs.push({
+            optionsStr: configs.join(" "),
+            outputDir: options["emitter-output-dir"],
+        });
     }
     return emitterConfigs;
 }
-
-async function _regenerateSingle(spec: string, flags: RegenerateFlags): Promise<void> {
-    // Perform some asynchronous operation here
-    const options = addOptions(spec, PLUGIN_DIR, flags);
-    const commandPromises = options.map((option) => {
-        const command = `tsp compile ${spec} --emit=${toPosix(PLUGIN_DIR)} ${option}`;
-        console.log(command);
-        return executeCommand(command);
+function _getCmdList(spec: string, flags: RegenerateFlags): TspCommand[] {
+    return addOptions(spec, PLUGIN_DIR, flags).map((option) => {
+        return {
+            outputDir: option.outputDir,
+            command: `tsp compile ${spec} --emit=${toPosix(PLUGIN_DIR)} ${option.optionsStr}`,
+        };
     });
-    await Promise.all(commandPromises);
 }
 
-async function regenerate(flags: RegenerateFlagsInput): Promise<boolean> {
+async function regenerate(flags: RegenerateFlagsInput): Promise<void> {
     if (flags.flavor === undefined) {
-        const azureGeneration = await regenerate({ ...flags, flavor: "azure" });
-        const unbrandedGeneration = await regenerate({ ...flags, flavor: "unbranded" });
-        return azureGeneration && unbrandedGeneration;
+        await regenerate({ ...flags, flavor: "azure" });
+        await regenerate({ ...flags, flavor: "unbranded" });
     } else {
         const flagsResolved = { debug: false, flavor: flags.flavor, ...flags };
         const CADL_RANCH_DIR = resolve(PLUGIN_DIR, "node_modules/@azure-tools/cadl-ranch-specs/http");
         const subdirectories = await getSubdirectories(CADL_RANCH_DIR, flagsResolved);
-        const promises = subdirectories.map(async (subdirectory) => {
-            // Perform additional asynchronous operations on each subdirectory here
-            await _regenerateSingle(subdirectory, flagsResolved);
-        });
-        await Promise.all(promises);
-        return true;
+        const cmdList: TspCommand[] = subdirectories.flatMap((subdirectory) =>
+            _getCmdList(subdirectory, flagsResolved),
+        );
+        const PromiseCommands = cmdList.map((tspCommand) => executeCommand(tspCommand));
+        await Promise.all(PromiseCommands);
     }
 }
 
-//   try {
-//     const output = await executeCommand('tsp compile');
-//     console.log(`Command output: ${output}`);
-//   } catch (error) {
-//     console.error(`Command failed: ${error}`);
-//   }
-
 // PARSE INPUT ARGUMENTS
 const argv = yargs(hideBin(process.argv))
     .option("flavor", {

package/scripts/eng/run-tests.ts

@@ -0,0 +1,80 @@
+/* eslint-disable no-console */
+import { execSync } from "child_process";
+import { readFileSync } from "fs";
+import { join } from "path";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { fileURLToPath } from "url";
+
+interface Arguments {
+    validFolders: string[];
+    folder?: string;
+    command?: string;
+}
+
+const validCommands = ["ci", "lint", "mypy", "pyright", "apiview"];
+
+// Parse command-line arguments using yargs
+const argv = yargs(hideBin(process.argv))
+    .option("validFolders", {
+        alias: "vf",
+        describe: "Specify the valid folders",
+        type: "array",
+        default: ["azure", "unbranded"],
+    })
+    .option("folder", {
+        alias: "f",
+        describe: "Specify the folder to use",
+        type: "string",
+    })
+    .option("command", {
+        alias: "c",
+        describe: "Specify the command to run",
+        choices: validCommands,
+        type: "string",
+    }).argv as Arguments;
+
+const foldersToProcess = argv.folder ? [argv.folder] : argv.validFolders;
+
+const commandToRun = argv.command || "all";
+
+function getCommand(command: string, folder: string) {
+    if (!validCommands.includes(command)) throw new Error(`Unknown command '${command}'.`);
+    return `FOLDER=${folder} tox -c ./test/${folder}/tox.ini -e ${command}`;
+}
+
+function sectionExistsInToxIni(command: string, folder: string): boolean {
+    const toxIniPath = join(fileURLToPath(import.meta.url), `../../../test/${folder}/tox.ini`);
+    const toxIniContent = readFileSync(toxIniPath, "utf-8");
+    const sectionHeader = `[testenv:${command}]`;
+    return toxIniContent.includes(sectionHeader);
+}
+
+function myExecSync(command: string, folder: string): void {
+    if (!sectionExistsInToxIni(command, folder)) {
+        console.log(`No section for ${command} in tox.ini for folder ${folder}. Skipping...`);
+        return;
+    }
+    execSync(getCommand(command, folder), { stdio: "inherit" });
+}
+
+foldersToProcess.forEach((folder) => {
+    try {
+        if (commandToRun === "all") {
+            for (const key of validCommands) {
+                console.log(`Running ${key} for folder ${folder}...`);
+                myExecSync(key, folder);
+            }
+        } else if (getCommand(commandToRun, folder)) {
+            console.log(`Running ${commandToRun} for folder ${folder}...`);
+            myExecSync(commandToRun, folder);
+        } else {
+            console.error(`Error: Unknown command '${commandToRun}'.`);
+            process.exit(1);
+        }
+    } catch (error) {
+        console.error((error as Error).message);
+        console.error(`Error executing command for folder ${folder}: ${(error as Error).message}`);
+        process.exit(1);
+    }
+});

package/scripts/eng/utils.ts

@@ -0,0 +1,38 @@
+/* eslint-disable no-console */
+import { exec } from "child_process";
+import process from "process";
+import { existsSync } from "fs";
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+import chalk from "chalk";
+
+// execute the command
+export function executeCommand(command: string, prettyName: string) {
+    exec(command, (error, stdout, stderr) => {
+        if (error) {
+            console.error(chalk.red(`Error executing ${command}(stdout): ${stdout}`));
+            console.error(chalk.red(`Error executing ${command}{stderr}: ${stderr}`));
+            process.exit(1);
+        }
+        if (stderr) {
+            // Process stderr output
+            console.log(chalk.yellow(`${command}:\n${stderr}`));
+            return;
+        }
+        console.log(chalk.green(`${prettyName} passed`));
+    });
+}
+
+// Function to run a command and log the output
+export function runCommand(command: string, prettyName: string) {
+    let pythonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "venv/");
+    if (existsSync(join(pythonPath, "bin"))) {
+        pythonPath = join(pythonPath, "bin", "python");
+    } else if (existsSync(join(pythonPath, "Scripts"))) {
+        pythonPath = join(pythonPath, "Scripts", "python");
+    } else {
+        throw new Error(pythonPath);
+    }
+    command = `${pythonPath} -m ${command}`;
+    executeCommand(command, prettyName);
+}

package/scripts/run-python3.ts

@@ -0,0 +1,25 @@
+// This script wraps logic in @azure-tools/extension to resolve
+// the path to Python 3 so that a Python script file can be run
+// from an npm script in package.json.  It uses the same Python 3
+// path resolution algorithm as AutoRest so that the behavior
+// is fully consistent (and also supports AUTOREST_PYTHON_EXE).
+//
+// Invoke it like so: "tsx run-python3.ts script.py"
+
+import cp from "child_process";
+import { patchPythonPath } from "./system-requirements.js";
+
+async function runPython3(...args: string[]) {
+    const command = await patchPythonPath(["python", ...args], {
+        version: ">=3.8",
+        environmentVariable: "AUTOREST_PYTHON_EXE",
+    });
+    cp.execSync(command.join(" "), {
+        stdio: [0, 1, 2],
+    });
+}
+
+runPython3(...process.argv.slice(2)).catch((err) => {
+    console.error(err.toString()); // eslint-disable-line no-console
+    process.exit(1);
+});

package/scripts/run_tsp.py

@@ -7,7 +7,8 @@ import sys
 import venv
 import logging
 from pathlib import Path
-from venvtools import python_run
+from pygen import m2r, preprocess, codegen, black
+from pygen.utils import parse_args
 
 _ROOT_DIR = Path(__file__).parent.parent
 
@@ -34,7 +35,8 @@ if __name__ == "__main__":
         breakpoint()  # pylint: disable=undefined-variable
 
     # run m2r
-    python_run(venv_context, "generator.pygen.m2r", command=sys.argv[1:])
-    python_run(venv_context, "generator.pygen.preprocess.__init__", command=sys.argv[1:])
-    python_run(venv_context, "generator.pygen.codegen.__init__", command=sys.argv[1:])
-    python_run(venv_context, "generator.pygen.black", command=sys.argv[1:])
+    args, unknown_args = parse_args()
+    m2r.M2R(output_folder=args.output_folder, cadl_file=args.cadl_file, **unknown_args).process()
+    preprocess.PreProcessPlugin(output_folder=args.output_folder, cadl_file=args.cadl_file, **unknown_args).process()
+    codegen.CodeGenerator(output_folder=args.output_folder, cadl_file=args.cadl_file, **unknown_args).process()
+    black.BlackScriptPlugin(output_folder=args.output_folder, **unknown_args).process()