@azure-tools/typespec-python 0.27.1 → 0.29.0

package/generator/pygen/codegen/templates/model_base.py.jinja2

@@ -489,6 +489,9 @@ def _create_value(rf: typing.Optional["_RestField"], value: typing.Any) -> typin
 
 class Model(_MyMutableMapping):
     _is_model = True
+    # label whether current class's _attr_to_rest_field has been calculated
+    # could not see _attr_to_rest_field directly because subclass inherits it from parent class
+    _calculated: typing.Set[str] = set()
 
     def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None:
         class_name = self.__class__.__name__
@@ -521,24 +524,27 @@ class Model(_MyMutableMapping):
         return Model(self.__dict__)
 
     def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self:  # pylint: disable=unused-argument
-        # we know the last three classes in mro are going to be 'Model', 'dict', and 'object'
-        mros = cls.__mro__[:-3][::-1]  # ignore model, dict, and object parents, and reverse the mro order
-        attr_to_rest_field: typing.Dict[str, _RestField] = {  # map attribute name to rest_field property
-            k: v for mro_class in mros for k, v in mro_class.__dict__.items() if k[0] != "_" and hasattr(v, "_type")
-        }
-        annotations = {
-            k: v
-            for mro_class in mros
-            if hasattr(mro_class, "__annotations__")  # pylint: disable=no-member
-            for k, v in mro_class.__annotations__.items()  # pylint: disable=no-member
-        }
-        for attr, rf in attr_to_rest_field.items():
-            rf._module = cls.__module__
-            if not rf._type:
-                rf._type = rf._get_deserialize_callable_from_annotation(annotations.get(attr, None))
-            if not rf._rest_name_input:
-                rf._rest_name_input = attr
-        cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items())
+        if f"{cls.__module__}.{cls.__qualname__}" not in cls._calculated:
+            # we know the last nine classes in mro are going to be 'Model', '_MyMutableMapping', 'MutableMapping', 
+            # 'Mapping', 'Collection', 'Sized', 'Iterable', 'Container' and 'object'
+            mros = cls.__mro__[:-9][::-1]  # ignore parents, and reverse the mro order
+            attr_to_rest_field: typing.Dict[str, _RestField] = {  # map attribute name to rest_field property
+                k: v for mro_class in mros for k, v in mro_class.__dict__.items() if k[0] != "_" and hasattr(v, "_type")
+            }
+            annotations = {
+                k: v
+                for mro_class in mros
+                if hasattr(mro_class, "__annotations__")  # pylint: disable=no-member
+                for k, v in mro_class.__annotations__.items()  # pylint: disable=no-member
+            }
+            for attr, rf in attr_to_rest_field.items():
+                rf._module = cls.__module__
+                if not rf._type:
+                    rf._type = rf._get_deserialize_callable_from_annotation(annotations.get(attr, None))
+                if not rf._rest_name_input:
+                    rf._rest_name_input = attr
+            cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items())
+            cls._calculated.add(f"{cls.__module__}.{cls.__qualname__}")
 
         return super().__new__(cls)  # pylint: disable=no-value-for-parameter
 

package/generator/pygen/codegen/templates/vendor.py.jinja2

@@ -70,8 +70,6 @@ FileType = Union[
     Tuple[Optional[str], FileContent, Optional[str]],
 ]
 
-FilesType = Union[Mapping[str, FileType], Sequence[Tuple[str, FileType]]]
-
 def serialize_multipart_data_entry(data_entry: Any) -> Any:
     if isinstance(data_entry, (list, tuple, dict, Model)):
         return json.dumps(data_entry, cls=SdkJSONEncoder, exclude_readonly=True)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@azure-tools/typespec-python",
-  "version": "0.27.1",
+  "version": "0.29.0",
   "author": "Microsoft Corporation",
   "description": "TypeSpec emitter for Python SDKs",
   "homepage": "https://github.com/Azure/autorest.python",
@@ -33,60 +33,54 @@
     "scripts/**"
   ],
   "peerDependencies": {
-    "@azure-tools/typespec-azure-core": ">=0.44.0 <1.0.0",
-    "@azure-tools/typespec-azure-resource-manager": ">=0.44.0 <1.0.0",
-    "@azure-tools/typespec-autorest": ">=0.44.1 <1.0.0",
-    "@azure-tools/typespec-client-generator-core": ">=0.44.3 <1.0.0",
-    "@azure-tools/typespec-azure-rulesets": ">=0.44.0 <3.0.0",
-    "@typespec/compiler": ">=0.58.1 <1.0.0",
-    "@typespec/http": ">=0.58.0 <1.0.0",
-    "@typespec/rest": ">=0.58.0 <1.0.0",
-    "@typespec/versioning": ">=0.58.0 <1.0.0",
-    "@typespec/openapi": ">=0.58.0 <1.0.0"
-  },
-  "dependenciesMeta": {
-    "@azure-tools/typespec-client-generator-core": {
-      "injected": true
-    }
+    "@azure-tools/typespec-azure-core": ">=0.45.0 <1.0.0",
+    "@azure-tools/typespec-azure-resource-manager": ">=0.45.0 <1.0.0",
+    "@azure-tools/typespec-autorest": ">=0.45.0 <1.0.0",
+    "@azure-tools/typespec-client-generator-core": ">=0.45.1 <1.0.0",
+    "@azure-tools/typespec-azure-rulesets": ">=0.45.0 <3.0.0",
+    "@typespec/compiler": ">=0.59.1 <1.0.0",
+    "@typespec/http": ">=0.59.0 <1.0.0",
+    "@typespec/rest": ">=0.59.0 <1.0.0",
+    "@typespec/versioning": ">=0.59.0 <1.0.0",
+    "@typespec/openapi": ">=0.59.0 <1.0.0"
   },
   "dependencies": {
     "js-yaml": "~4.1.0",
-    "@typespec/openapi3": "~0.58.0",
-    "@autorest/system-requirements": "~1.0.2",
-    "fs-extra": "~11.2.0",
-    "semver": "~7.6.2"
+    "@typespec/openapi3": "~0.59.0",
+    "semver": "~7.6.2",
+    "tsx": "4.17.0"
   },
   "devDependencies": {
-    "@azure-tools/typespec-azure-resource-manager": "~0.44.0",
-    "@azure-tools/typespec-autorest": "~0.44.1",
-    "@azure-tools/cadl-ranch-expect": "~0.14.1",
-    "@azure-tools/cadl-ranch-specs": "~0.34.5",
+    "@azure-tools/typespec-azure-resource-manager": "~0.45.0",
+    "@azure-tools/typespec-autorest": "~0.45.0",
+    "@azure-tools/cadl-ranch-expect": "~0.15.1",
+    "@azure-tools/cadl-ranch-specs": "~0.35.4",
     "@types/js-yaml": "~4.0.5",
-    "@types/mocha": "~10.0.1",
     "@types/node": "^18.16.3",
+    "@types/yargs": "17.0.32",
+    "@types/semver": "7.5.8",
     "@typespec/eslint-config-typespec": "~0.55.0",
-    "@typespec/openapi": "~0.58.0",
+    "@typespec/openapi": "~0.59.0",
     "c8": "~7.13.0",
-    "eslint": "^8.57.0",
-    "mocha": "~10.2.0",
     "rimraf": "~5.0.0",
     "typescript": "~5.1.3",
-    "@azure-tools/typespec-azure-core": "~0.44.0",
-    "@azure-tools/typespec-client-generator-core": "~0.44.3",
-    "@typespec/compiler": "~0.58.1",
-    "@typespec/http": "~0.58.0",
-    "@typespec/rest": "~0.58.0",
-    "@typespec/versioning": "~0.58.0",
-    "@azure-tools/typespec-azure-rulesets": "0.44.0"
+    "@azure-tools/typespec-azure-core": "~0.45.0",
+    "@azure-tools/typespec-client-generator-core": "0.45.1",
+    "@typespec/compiler": "~0.59.1",
+    "@typespec/http": "~0.59.0",
+    "@typespec/rest": "~0.59.0",
+    "@typespec/versioning": "~0.59.0",
+    "@azure-tools/typespec-azure-rulesets": "0.45.0",
+    "yargs": "~17.2.1"
   },
   "scripts": {
-    "clean": "rimraf ./dist ./temp",
+    "clean": "rimraf ./dist ./temp ./venv ./node_modules",
     "build": "tsc -p .",
     "watch": "tsc -p . --watch",
-    "test": "mocha",
-    "test-official": "c8 mocha --forbid-only",
     "lint": "eslint . --ext .ts --max-warnings=0",
     "lint:fix": "eslint . --fix --ext .ts",
-    "install": "node ./scripts/run-python3.cjs ./scripts/install.py"
+    "install": "tsx ./scripts/run-python3.ts ./scripts/install.py",
+    "regenerate": "tsx ./scripts/regenerate.ts",
+    "test": "tsx ./scripts/run-tests.ts"
   }
 }

package/scripts/regenerate.ts

@@ -0,0 +1,285 @@
+/* eslint-disable no-console */
+import { exec as execCallback } from "child_process";
+import { promisify } from "util";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { dirname, join, relative, resolve } from "path";
+import { promises } from "fs";
+import { fileURLToPath } from "url";
+
+// Promisify the exec function
+const exec = promisify(execCallback);
+
+// Get the directory of the current file
+const PLUGIN_DIR = resolve(fileURLToPath(import.meta.url), "../../");
+const CADL_RANCH_DIR = resolve(PLUGIN_DIR, "node_modules/@azure-tools/cadl-ranch-specs/http");
+
+const EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, string>[]> = {
+    "resiliency/srv-driven/old.tsp": {
+        "package-name": "resiliency-srv-driven1",
+        "package-mode": "azure-dataplane",
+        "package-pprint-name": "ResiliencySrvDriven1",
+    },
+    "resiliency/srv-driven": {
+        "package-name": "resiliency-srv-driven2",
+        "package-mode": "azure-dataplane",
+        "package-pprint-name": "ResiliencySrvDriven2",
+    },
+    "authentication/http/custom": {
+        "package-name": "authentication-http-custom",
+    },
+    "authentication/union": {
+        "package-name": "authentication-union",
+    },
+    "type/array": {
+        "package-name": "typetest-array",
+    },
+    "type/dictionary": {
+        "package-name": "typetest-dictionary",
+    },
+    "type/enum/extensible": {
+        "package-name": "typetest-enum-extensible",
+    },
+    "type/enum/fixed": {
+        "package-name": "typetest-enum-fixed",
+    },
+    "type/model/empty": {
+        "package-name": "typetest-model-empty",
+    },
+    "type/model/flatten": {
+        "package-name": "typetest-model-flatten",
+    },
+    "type/model/inheritance/enum-discriminator": {
+        "package-name": "typetest-model-enumdiscriminator",
+    },
+    "type/model/inheritance/nested-discriminator": {
+        "package-name": "typetest-model-nesteddiscriminator",
+    },
+    "type/model/inheritance/not-discriminated": {
+        "package-name": "typetest-model-notdiscriminated",
+    },
+    "type/model/inheritance/single-discriminator": {
+        "package-name": "typetest-model-singlediscriminator",
+    },
+    "type/model/inheritance/recursive": {
+        "package-name": "typetest-model-recursive",
+    },
+    "type/model/usage": {
+        "package-name": "typetest-model-usage",
+    },
+    "type/model/visibility": [
+        { "package-name": "typetest-model-visibility" },
+        { "package-name": "headasbooleantrue", "head-as-boolean": "true" },
+        { "package-name": "headasbooleanfalse", "head-as-boolean": "false" },
+    ],
+    "type/property/nullable": {
+        "package-name": "typetest-property-nullable",
+    },
+    "type/property/optionality": {
+        "package-name": "typetest-property-optional",
+    },
+    "type/property/additional-properties": {
+        "package-name": "typetest-property-additionalproperties",
+    },
+    "type/scalar": {
+        "package-name": "typetest-scalar",
+    },
+    "type/property/value-types": {
+        "package-name": "typetest-property-valuetypes",
+    },
+    "type/union": {
+        "package-name": "typetest-union",
+    },
+    "azure/core/lro/rpc": {
+        "package-name": "azurecore-lro-rpc",
+    },
+    "client/structure/multi-client": {
+        "package-name": "client-structure-multiclient",
+    },
+    "client/structure/renamed-operation": {
+        "package-name": "client-structure-renamedoperation",
+    },
+    "client/structure/two-operation-group": {
+        "package-name": "client-structure-twooperationgroup",
+    },
+    "mgmt/sphere": [{ "package-name": "azure-mgmt-spheredpg" }],
+};
+
+function toPosix(dir: string): string {
+    return dir.replace(/\\/g, "/");
+}
+
+function getEmitterOption(spec: string): Record<string, string>[] {
+    const relativeSpec = toPosix(relative(CADL_RANCH_DIR, spec));
+    const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") ? relativeSpec : dirname(relativeSpec);
+    const result = EMITTER_OPTIONS[key] || [{}];
+    return Array.isArray(result) ? result : [result];
+}
+
+// Function to execute CLI commands asynchronously
+async function executeCommand(command: string): Promise<void> {
+    try {
+        const { stdout, stderr } = await exec(command);
+        if (stdout) console.log(`stdout: ${stdout}`);
+        if (stderr) console.error(`stderr: ${stderr}`);
+    } catch (error) {
+        console.error(`exec error: ${error}`);
+        throw error;
+    }
+}
+
+interface RegenerateFlagsInput {
+    flavor?: "azure" | "unbranded";
+    debug?: boolean;
+    name?: string;
+}
+
+interface RegenerateFlags {
+    flavor: "azure" | "unbranded";
+    debug: boolean;
+    name?: string;
+}
+
+const SpecialFlags: Record<string, Record<string, any>> = {
+    azure: {
+        "generate-test": true,
+        "generate-sample": true,
+    },
+};
+
+async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promise<string[]> {
+    const subdirectories: string[] = [];
+
+    async function searchDir(currentDir: string) {
+        const items = await promises.readdir(currentDir, { withFileTypes: true });
+
+        const promisesArray = items.map(async (item) => {
+            const subDirPath = join(currentDir, item.name);
+            if (item.isDirectory()) {
+                const mainTspPath = join(subDirPath, "main.tsp");
+                const clientTspPath = join(subDirPath, "client.tsp");
+
+                const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath));
+                if (flags.flavor === "unbranded" && mainTspRelativePath.includes("azure")) return;
+
+                // after xml support, remove this check
+                if (mainTspRelativePath.includes("xml")) return;
+
+                // after fix test generation for nested operation group, remove this check
+                if (mainTspRelativePath.includes("client-operation-group")) return;
+
+                const hasMainTsp = await promises
+                    .access(mainTspPath)
+                    .then(() => true)
+                    .catch(() => false);
+                const hasClientTsp = await promises
+                    .access(clientTspPath)
+                    .then(() => true)
+                    .catch(() => false);
+
+                if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) {
+                    if (mainTspRelativePath.includes("resiliency/srv-driven")) {
+                        subdirectories.push(resolve(subDirPath, "old.tsp"));
+                    }
+                    if (hasClientTsp) {
+                        subdirectories.push(resolve(subDirPath, "client.tsp"));
+                    } else if (hasMainTsp) {
+                        subdirectories.push(resolve(subDirPath, "main.tsp"));
+                    }
+                }
+
+                // Recursively search in the subdirectory
+                await searchDir(subDirPath);
+            }
+        });
+
+        await Promise.all(promisesArray);
+    }
+
+    await searchDir(baseDir);
+    return subdirectories;
+}
+
+function defaultPackageName(spec: string): string {
+    return toPosix(relative(CADL_RANCH_DIR, dirname(spec)))
+        .replace(/\//g, "-")
+        .toLowerCase();
+}
+
+function addOptions(spec: string, generatedFolder: string, flags: RegenerateFlags): string[] {
+    const emitterConfigs: string[] = [];
+    for (const config of getEmitterOption(spec)) {
+        const options: Record<string, string> = { ...config };
+        options["flavor"] = flags.flavor;
+        for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) {
+            options[k] = v;
+        }
+        if (options["emitter-output-dir"] === undefined) {
+            const packageName = options["package-name"] || defaultPackageName(spec);
+            options["emitter-output-dir"] = toPosix(`${generatedFolder}/test/${flags.flavor}/generated/${packageName}`);
+        }
+        if (flags.debug) {
+            options["debug"] = "true";
+        }
+        if (flags.flavor === "unbranded") {
+            options["company-name"] = "Unbranded";
+        }
+        options["examples-directory"] = toPosix(join(dirname(spec), "examples"));
+        const configs = Object.entries(options).flatMap(([k, v]) => {
+            return `--option @azure-tools/typespec-python.${k}=${v}`;
+        });
+        emitterConfigs.push(configs.join(" "));
+    }
+    return emitterConfigs;
+}
+
+async function _regenerateSingle(spec: string, flags: RegenerateFlags): Promise<void> {
+    // Perform some asynchronous operation here
+    const options = addOptions(spec, PLUGIN_DIR, flags);
+    const commandPromises = options.map((option) => {
+        const command = `tsp compile ${spec} --emit=${toPosix(PLUGIN_DIR)} ${option}`;
+        console.log(command);
+        return executeCommand(command);
+    });
+    await Promise.all(commandPromises);
+}
+
+async function regenerate(flags: RegenerateFlagsInput): Promise<boolean> {
+    if (flags.flavor === undefined) {
+        const azureGeneration = await regenerate({ ...flags, flavor: "azure" });
+        const unbrandedGeneration = await regenerate({ ...flags, flavor: "unbranded" });
+        return azureGeneration && unbrandedGeneration;
+    } else {
+        const flagsResolved = { debug: false, flavor: flags.flavor, ...flags };
+        const CADL_RANCH_DIR = resolve(PLUGIN_DIR, "node_modules/@azure-tools/cadl-ranch-specs/http");
+        const subdirectories = await getSubdirectories(CADL_RANCH_DIR, flagsResolved);
+        const promises = subdirectories.map(async (subdirectory) => {
+            // Perform additional asynchronous operations on each subdirectory here
+            await _regenerateSingle(subdirectory, flagsResolved);
+        });
+        await Promise.all(promises);
+        return true;
+    }
+}
+
+// PARSE INPUT ARGUMENTS
+const argv = yargs(hideBin(process.argv))
+    .option("flavor", {
+        type: "string",
+        choices: ["azure", "unbranded"],
+        description: "Specify the flavor",
+    })
+    .option("debug", {
+        alias: "d",
+        type: "boolean",
+        description: "Debug mode",
+    })
+    .option("name", {
+        alias: "n",
+        type: "string",
+        description: "Specify filename if you only want to generate a subset",
+    }).argv;
+
+regenerate(argv as RegenerateFlags)
+    .then(() => console.log("Regeneration successful"))
+    .catch((error) => console.error(`Regeneration failed: ${error.message}`));

package/scripts/run-python3.ts

@@ -0,0 +1,25 @@
+// This script wraps logic in @azure-tools/extension to resolve
+// the path to Python 3 so that a Python script file can be run
+// from an npm script in package.json.  It uses the same Python 3
+// path resolution algorithm as AutoRest so that the behavior
+// is fully consistent (and also supports AUTOREST_PYTHON_EXE).
+//
+// Invoke it like so: "tsx run-python3.ts script.py"
+
+import cp from "child_process";
+import { patchPythonPath } from "./system-requirements.js";
+
+async function runPython3(...args: string[]) {
+    const command = await patchPythonPath(["python", ...args], {
+        version: ">=3.8",
+        environmentVariable: "AUTOREST_PYTHON_EXE",
+    });
+    cp.execSync(command.join(" "), {
+        stdio: [0, 1, 2],
+    });
+}
+
+runPython3(...process.argv.slice(2)).catch((err) => {
+    console.error(err.toString()); // eslint-disable-line no-console
+    process.exit(1);
+});

package/scripts/run-tests.ts

@@ -0,0 +1,57 @@
+/* eslint-disable no-console */
+import { execSync } from "child_process";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+
+interface Arguments {
+    folder?: string;
+    command?: string;
+}
+
+const validFolders = ["azure", "unbranded"];
+
+const validCommands = ["ci", "lint", "mypy", "pyright", "apiview"];
+
+// Parse command-line arguments using yargs
+const argv = yargs(hideBin(process.argv))
+    .option("folder", {
+        alias: "f",
+        describe: "Specify the folder to use",
+        choices: validFolders,
+        type: "string",
+    })
+    .option("command", {
+        alias: "c",
+        describe: "Specify the command to run",
+        choices: validCommands,
+        type: "string",
+    }).argv as Arguments;
+
+const foldersToProcess = argv.folder ? [argv.folder] : validFolders;
+
+const commandToRun = argv.command || "all";
+
+function getCommand(command: string, folder: string) {
+    if (!validCommands.includes(command)) throw new Error(`Unknown command '${command}'.`);
+    return `FOLDER=${folder} tox -c ./test/${folder}/tox.ini -e ${command}`;
+}
+
+foldersToProcess.forEach((folder) => {
+    try {
+        if (commandToRun === "all") {
+            for (const key of validCommands) {
+                console.log(`Running ${key} for folder ${folder}...`);
+                execSync(getCommand(key, folder), { stdio: "inherit" });
+            }
+        } else if (getCommand(commandToRun, folder)) {
+            console.log(`Running ${commandToRun} for folder ${folder}...`);
+            execSync(getCommand(commandToRun, folder), { stdio: "inherit" });
+        } else {
+            console.error(`Error: Unknown command '${commandToRun}'.`);
+            process.exit(1);
+        }
+    } catch (error) {
+        console.error(`Error executing command for folder ${folder}: ${(error as Error).message}`);
+        process.exit(1);
+    }
+});