@azure-tools/typespec-python 0.28.0 → 0.29.0

package/generator/pygen/codegen/serializers/sample_serializer.py

@@ -42,7 +42,7 @@ class SampleSerializer(BaseSerializer):
         self.operation = operation
         self.sample = sample
         self.file_name = file_name
-        self.sample_params = {to_snake_case(k): v for k, v in sample.get("parameters", {}).items()}
+        self.sample_params = sample.get("parameters", {})
 
     def _imports(self) -> FileImportSerializer:
         imports = FileImport(self.code_model)
@@ -66,8 +66,8 @@ class SampleSerializer(BaseSerializer):
                 "AzureKeyCredential",
                 ImportType.SDKCORE,
             )
-        for param in self.operation.parameters.positional:
-            if not param.client_default_value and not param.optional and param.client_name in self.sample_params:
+        for param in self.operation.parameters.positional + self.operation.parameters.keyword_only:
+            if not param.client_default_value and not param.optional and param.wire_name in self.sample_params:
                 imports.merge(param.type.imports_for_sample())
         return FileImportSerializer(imports, True)
 
@@ -80,15 +80,19 @@ class SampleSerializer(BaseSerializer):
         elif isinstance(credential_type, KeyCredentialType):
             special_param.update({"credential": 'AzureKeyCredential(key=os.getenv("AZURE_KEY"))'})
 
-        params_positional = [
-            p for p in self.code_model.clients[0].parameters.positional if not (p.optional or p.client_default_value)
+        params = [
+            p
+            for p in (
+                self.code_model.clients[0].parameters.positional + self.code_model.clients[0].parameters.keyword_only
+            )
+            if not (p.optional or p.client_default_value)
         ]
         client_params = {
             p.client_name: special_param.get(
                 p.client_name,
-                f'"{self.sample_params.get(p.client_name) or p.client_name.upper()}"',
+                f'"{self.sample_params.get(p.wire_name) or p.client_name.upper()}"',
             )
-            for p in params_positional
+            for p in params
         }
 
         return client_params
@@ -103,15 +107,18 @@ class SampleSerializer(BaseSerializer):
 
     # prepare operation parameters
     def _operation_params(self) -> Dict[str, Any]:
-        params_positional = [p for p in self.operation.parameters.positional if not p.client_default_value]
+        params = [
+            p
+            for p in (self.operation.parameters.positional + self.operation.parameters.keyword_only)
+            if not p.client_default_value
+        ]
         failure_info = "fail to find required param named {}"
         operation_params = {}
-        for param in params_positional:
-            name = param.client_name
-            param_value = self.sample_params.get(name)
+        for param in params:
             if not param.optional:
+                param_value = self.sample_params.get(param.wire_name)
                 if not param_value:
-                    raise Exception(failure_info.format(name))  # pylint: disable=broad-exception-raised
+                    raise Exception(failure_info.format(param.client_name))  # pylint: disable=broad-exception-raised
                 operation_params[param.client_name] = self.handle_param(param, param_value)
         return operation_params
 

package/generator/pygen/codegen/templates/model_base.py.jinja2

@@ -489,6 +489,9 @@ def _create_value(rf: typing.Optional["_RestField"], value: typing.Any) -> typin
 
 class Model(_MyMutableMapping):
     _is_model = True
+    # label whether current class's _attr_to_rest_field has been calculated
+    # could not see _attr_to_rest_field directly because subclass inherits it from parent class
+    _calculated: typing.Set[str] = set()
 
     def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None:
         class_name = self.__class__.__name__
@@ -521,24 +524,27 @@ class Model(_MyMutableMapping):
         return Model(self.__dict__)
 
     def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self:  # pylint: disable=unused-argument
-        # we know the last three classes in mro are going to be 'Model', 'dict', and 'object'
-        mros = cls.__mro__[:-3][::-1]  # ignore model, dict, and object parents, and reverse the mro order
-        attr_to_rest_field: typing.Dict[str, _RestField] = {  # map attribute name to rest_field property
-            k: v for mro_class in mros for k, v in mro_class.__dict__.items() if k[0] != "_" and hasattr(v, "_type")
-        }
-        annotations = {
-            k: v
-            for mro_class in mros
-            if hasattr(mro_class, "__annotations__")  # pylint: disable=no-member
-            for k, v in mro_class.__annotations__.items()  # pylint: disable=no-member
-        }
-        for attr, rf in attr_to_rest_field.items():
-            rf._module = cls.__module__
-            if not rf._type:
-                rf._type = rf._get_deserialize_callable_from_annotation(annotations.get(attr, None))
-            if not rf._rest_name_input:
-                rf._rest_name_input = attr
-        cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items())
+        if f"{cls.__module__}.{cls.__qualname__}" not in cls._calculated:
+            # we know the last nine classes in mro are going to be 'Model', '_MyMutableMapping', 'MutableMapping', 
+            # 'Mapping', 'Collection', 'Sized', 'Iterable', 'Container' and 'object'
+            mros = cls.__mro__[:-9][::-1]  # ignore parents, and reverse the mro order
+            attr_to_rest_field: typing.Dict[str, _RestField] = {  # map attribute name to rest_field property
+                k: v for mro_class in mros for k, v in mro_class.__dict__.items() if k[0] != "_" and hasattr(v, "_type")
+            }
+            annotations = {
+                k: v
+                for mro_class in mros
+                if hasattr(mro_class, "__annotations__")  # pylint: disable=no-member
+                for k, v in mro_class.__annotations__.items()  # pylint: disable=no-member
+            }
+            for attr, rf in attr_to_rest_field.items():
+                rf._module = cls.__module__
+                if not rf._type:
+                    rf._type = rf._get_deserialize_callable_from_annotation(annotations.get(attr, None))
+                if not rf._rest_name_input:
+                    rf._rest_name_input = attr
+            cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items())
+            cls._calculated.add(f"{cls.__module__}.{cls.__qualname__}")
 
         return super().__new__(cls)  # pylint: disable=no-value-for-parameter
 

package/generator/pygen/codegen/templates/vendor.py.jinja2

@@ -70,8 +70,6 @@ FileType = Union[
     Tuple[Optional[str], FileContent, Optional[str]],
 ]
 
-FilesType = Union[Mapping[str, FileType], Sequence[Tuple[str, FileType]]]
-
 def serialize_multipart_data_entry(data_entry: Any) -> Any:
     if isinstance(data_entry, (list, tuple, dict, Model)):
         return json.dumps(data_entry, cls=SdkJSONEncoder, exclude_readonly=True)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@azure-tools/typespec-python",
-  "version": "0.28.0",
+  "version": "0.29.0",
   "author": "Microsoft Corporation",
   "description": "TypeSpec emitter for Python SDKs",
   "homepage": "https://github.com/Azure/autorest.python",
@@ -38,43 +38,35 @@
     "@azure-tools/typespec-autorest": ">=0.45.0 <1.0.0",
     "@azure-tools/typespec-client-generator-core": ">=0.45.1 <1.0.0",
     "@azure-tools/typespec-azure-rulesets": ">=0.45.0 <3.0.0",
-    "@typespec/compiler": ">=0.59.0 <1.0.0",
+    "@typespec/compiler": ">=0.59.1 <1.0.0",
     "@typespec/http": ">=0.59.0 <1.0.0",
     "@typespec/rest": ">=0.59.0 <1.0.0",
     "@typespec/versioning": ">=0.59.0 <1.0.0",
     "@typespec/openapi": ">=0.59.0 <1.0.0"
   },
-  "dependenciesMeta": {
-    "@azure-tools/typespec-client-generator-core": {
-      "injected": true
-    }
-  },
   "dependencies": {
     "js-yaml": "~4.1.0",
     "@typespec/openapi3": "~0.59.0",
-    "@autorest/system-requirements": "~1.0.2",
-    "fs-extra": "~11.2.0",
-    "semver": "~7.6.2"
+    "semver": "~7.6.2",
+    "tsx": "4.17.0"
   },
   "devDependencies": {
     "@azure-tools/typespec-azure-resource-manager": "~0.45.0",
     "@azure-tools/typespec-autorest": "~0.45.0",
-    "@azure-tools/cadl-ranch-expect": "~0.14.1",
-    "@azure-tools/cadl-ranch-specs": "~0.34.5",
+    "@azure-tools/cadl-ranch-expect": "~0.15.1",
+    "@azure-tools/cadl-ranch-specs": "~0.35.4",
     "@types/js-yaml": "~4.0.5",
-    "@types/mocha": "~10.0.1",
     "@types/node": "^18.16.3",
     "@types/yargs": "17.0.32",
+    "@types/semver": "7.5.8",
     "@typespec/eslint-config-typespec": "~0.55.0",
     "@typespec/openapi": "~0.59.0",
     "c8": "~7.13.0",
-    "eslint": "^8.57.0",
-    "mocha": "~10.2.0",
     "rimraf": "~5.0.0",
     "typescript": "~5.1.3",
     "@azure-tools/typespec-azure-core": "~0.45.0",
     "@azure-tools/typespec-client-generator-core": "0.45.1",
-    "@typespec/compiler": "~0.59.0",
+    "@typespec/compiler": "~0.59.1",
     "@typespec/http": "~0.59.0",
     "@typespec/rest": "~0.59.0",
     "@typespec/versioning": "~0.59.0",
@@ -82,14 +74,13 @@
     "yargs": "~17.2.1"
   },
   "scripts": {
-    "clean": "rimraf ./dist ./temp",
+    "clean": "rimraf ./dist ./temp ./venv ./node_modules",
     "build": "tsc -p .",
     "watch": "tsc -p . --watch",
-    "test": "mocha",
-    "test-official": "c8 mocha --forbid-only",
     "lint": "eslint . --ext .ts --max-warnings=0",
     "lint:fix": "eslint . --fix --ext .ts",
-    "install": "node ./scripts/run-python3.cjs ./scripts/install.py",
-    "regenerate": "node ./dist/scripts/regenerate.js"
+    "install": "tsx ./scripts/run-python3.ts ./scripts/install.py",
+    "regenerate": "tsx ./scripts/regenerate.ts",
+    "test": "tsx ./scripts/run-tests.ts"
   }
 }

package/scripts/regenerate.ts

@@ -11,7 +11,7 @@ import { fileURLToPath } from "url";
 const exec = promisify(execCallback);
 
 // Get the directory of the current file
-const PLUGIN_DIR = resolve(fileURLToPath(import.meta.url), "../../../");
+const PLUGIN_DIR = resolve(fileURLToPath(import.meta.url), "../../");
 const CADL_RANCH_DIR = resolve(PLUGIN_DIR, "node_modules/@azure-tools/cadl-ranch-specs/http");
 
 const EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, string>[]> = {
@@ -162,6 +162,12 @@ async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promi
                 const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath));
                 if (flags.flavor === "unbranded" && mainTspRelativePath.includes("azure")) return;
 
+                // after xml support, remove this check
+                if (mainTspRelativePath.includes("xml")) return;
+
+                // after fix test generation for nested operation group, remove this check
+                if (mainTspRelativePath.includes("client-operation-group")) return;
+
                 const hasMainTsp = await promises
                     .access(mainTspPath)
                     .then(() => true)
@@ -256,13 +262,6 @@ async function regenerate(flags: RegenerateFlagsInput): Promise<boolean> {
     }
 }
 
-//   try {
-//     const output = await executeCommand('tsp compile');
-//     console.log(`Command output: ${output}`);
-//   } catch (error) {
-//     console.error(`Command failed: ${error}`);
-//   }
-
 // PARSE INPUT ARGUMENTS
 const argv = yargs(hideBin(process.argv))
     .option("flavor", {

package/scripts/run-python3.ts

@@ -0,0 +1,25 @@
+// This script wraps logic in @azure-tools/extension to resolve
+// the path to Python 3 so that a Python script file can be run
+// from an npm script in package.json.  It uses the same Python 3
+// path resolution algorithm as AutoRest so that the behavior
+// is fully consistent (and also supports AUTOREST_PYTHON_EXE).
+//
+// Invoke it like so: "tsx run-python3.ts script.py"
+
+import cp from "child_process";
+import { patchPythonPath } from "./system-requirements.js";
+
+async function runPython3(...args: string[]) {
+    const command = await patchPythonPath(["python", ...args], {
+        version: ">=3.8",
+        environmentVariable: "AUTOREST_PYTHON_EXE",
+    });
+    cp.execSync(command.join(" "), {
+        stdio: [0, 1, 2],
+    });
+}
+
+runPython3(...process.argv.slice(2)).catch((err) => {
+    console.error(err.toString()); // eslint-disable-line no-console
+    process.exit(1);
+});

package/scripts/run-tests.ts

@@ -0,0 +1,57 @@
+/* eslint-disable no-console */
+import { execSync } from "child_process";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+
+interface Arguments {
+    folder?: string;
+    command?: string;
+}
+
+const validFolders = ["azure", "unbranded"];
+
+const validCommands = ["ci", "lint", "mypy", "pyright", "apiview"];
+
+// Parse command-line arguments using yargs
+const argv = yargs(hideBin(process.argv))
+    .option("folder", {
+        alias: "f",
+        describe: "Specify the folder to use",
+        choices: validFolders,
+        type: "string",
+    })
+    .option("command", {
+        alias: "c",
+        describe: "Specify the command to run",
+        choices: validCommands,
+        type: "string",
+    }).argv as Arguments;
+
+const foldersToProcess = argv.folder ? [argv.folder] : validFolders;
+
+const commandToRun = argv.command || "all";
+
+function getCommand(command: string, folder: string) {
+    if (!validCommands.includes(command)) throw new Error(`Unknown command '${command}'.`);
+    return `FOLDER=${folder} tox -c ./test/${folder}/tox.ini -e ${command}`;
+}
+
+foldersToProcess.forEach((folder) => {
+    try {
+        if (commandToRun === "all") {
+            for (const key of validCommands) {
+                console.log(`Running ${key} for folder ${folder}...`);
+                execSync(getCommand(key, folder), { stdio: "inherit" });
+            }
+        } else if (getCommand(commandToRun, folder)) {
+            console.log(`Running ${commandToRun} for folder ${folder}...`);
+            execSync(getCommand(commandToRun, folder), { stdio: "inherit" });
+        } else {
+            console.error(`Error: Unknown command '${commandToRun}'.`);
+            process.exit(1);
+        }
+    } catch (error) {
+        console.error(`Error executing command for folder ${folder}: ${(error as Error).message}`);
+        process.exit(1);
+    }
+});

package/scripts/system-requirements.ts

@@ -0,0 +1,253 @@
+import { SpawnOptions, ChildProcess, spawn } from "child_process";
+import { coerce, satisfies } from "semver";
+
+/*
+ * Copied from @autorest/system-requirements
+ */
+
+const execute = (command: string, cmdlineargs: Array<string>, options: MoreOptions = {}): Promise<ExecResult> => {
+    return new Promise((resolve, reject) => {
+        const cp = spawn(command, cmdlineargs, { ...options, stdio: "pipe", shell: true });
+        if (options.onCreate) {
+            options.onCreate(cp);
+        }
+
+        options.onStdOutData ? cp.stdout.on("data", options.onStdOutData) : cp;
+        options.onStdErrData ? cp.stderr.on("data", options.onStdErrData) : cp;
+
+        let err = "";
+        let out = "";
+        let all = "";
+        cp.stderr.on("data", (chunk) => {
+            err += chunk;
+            all += chunk;
+        });
+        cp.stdout.on("data", (chunk) => {
+            out += chunk;
+            all += chunk;
+        });
+
+        cp.on("error", (err) => {
+            reject(err);
+        });
+        cp.on("close", (code, signal) =>
+            resolve({
+                stdout: out,
+                stderr: err,
+                log: all,
+                error: code ? new Error("Process Failed.") : null,
+                code,
+            }),
+        );
+    });
+};
+
+const versionIsSatisfied = (version: string, requirement: string): boolean => {
+    const cleanedVersion = coerce(version);
+    if (!cleanedVersion) {
+        throw new Error(`Invalid version ${version}.`);
+    }
+    return satisfies(cleanedVersion, requirement, true);
+};
+
+/**
+ * Validate the provided system requirement resolution is satisfying the version requirement if applicable.
+ * @param resolution Command resolution.
+ * @param actualVersion Version for that resolution.
+ * @param requirement Requirement.
+ * @returns the resolution if it is valid or an @see SystemRequirementError if not.
+ */
+const validateVersionRequirement = (
+    resolution: SystemRequirementResolution,
+    actualVersion: string,
+    requirement: SystemRequirement,
+): SystemRequirementResolution | SystemRequirementError => {
+    if (!requirement.version) {
+        return resolution; // No version requirement.
+    }
+
+    try {
+        if (versionIsSatisfied(actualVersion, requirement.version)) {
+            return resolution;
+        }
+        return {
+            ...resolution,
+            error: true,
+            message: `'${resolution.command}' version is '${actualVersion}' but doesn't satisfy requirement '${requirement.version}'. Please update.`,
+            actualVersion: actualVersion,
+            neededVersion: requirement.version,
+        };
+    } catch {
+        return {
+            ...resolution,
+            error: true,
+            message: `Couldn't parse the version ${actualVersion}. This is not a valid semver version.`,
+            actualVersion: actualVersion,
+            neededVersion: requirement.version,
+        };
+    }
+};
+
+const tryPython = async (
+    requirement: SystemRequirement,
+    command: string,
+    additionalArgs: string[] = [],
+): Promise<SystemRequirementResolution | SystemRequirementError> => {
+    const resolution: SystemRequirementResolution = {
+        name: PythonRequirement,
+        command,
+        additionalArgs: additionalArgs.length > 0 ? additionalArgs : undefined,
+    };
+
+    try {
+        const result = await execute(command, [...additionalArgs, "-c", `"${PRINT_PYTHON_VERSION_SCRIPT}"`]);
+        return validateVersionRequirement(resolution, result.stdout.trim(), requirement);
+    } catch (e) {
+        return {
+            error: true,
+            ...resolution,
+            message: `'${command}' command line is not found in the path. Make sure to have it installed.`,
+        };
+    }
+};
+
+/**
+ * Returns the path to the executable as asked in the requirement.
+ * @param requirement System requirement definition.
+ * @returns If the requirement provide an environment variable for the path returns the value of that environment variable. undefined otherwise.
+ */
+const getExecutablePath = (requirement: SystemRequirement): string | undefined =>
+    requirement.environmentVariable && process.env[requirement.environmentVariable];
+
+const createPythonErrorMessage = (
+    requirement: SystemRequirement,
+    errors: SystemRequirementError[],
+): SystemRequirementError => {
+    const versionReq = requirement.version ?? "*";
+    const lines = [
+        `Couldn't find a valid python interpreter satisfying the requirement (version: ${versionReq}). Tried:`,
+        ...errors.map((x) => ` - ${x.command} (${x.message})`),
+    ];
+
+    return {
+        error: true,
+        name: "python",
+        command: "python",
+        message: lines.join("\n"),
+    };
+};
+
+const resolvePythonRequirement = async (
+    requirement: SystemRequirement,
+): Promise<SystemRequirementResolution | SystemRequirementError> => {
+    // Hardcoding AUTOREST_PYTHON_EXE is for backward compatibility
+    const path = getExecutablePath(requirement) ?? process.env["AUTOREST_PYTHON_EXE"];
+    if (path) {
+        return await tryPython(requirement, path);
+    }
+
+    const errors: SystemRequirementError[] = [];
+    // On windows try `py` executable with `-3` flag.
+    if (process.platform === "win32") {
+        const pyResult = await tryPython(requirement, "py", ["-3"]);
+        if ("error" in pyResult) {
+            errors.push(pyResult);
+        } else {
+            return pyResult;
+        }
+    }
+
+    const python3Result = await tryPython(requirement, "python3");
+    if ("error" in python3Result) {
+        errors.push(python3Result);
+    } else {
+        return python3Result;
+    }
+
+    const pythonResult = await tryPython(requirement, "python");
+    if ("error" in pythonResult) {
+        errors.push(pythonResult);
+    } else {
+        return pythonResult;
+    }
+
+    return createPythonErrorMessage(requirement, errors);
+};
+
+/**
+ * @param command list of the command and arguments. First item in array must be a python exe @see KnownPythonExe. (e.g. ["python", "mypythonfile.py"]
+ * @param requirement
+ */
+export const patchPythonPath = async (
+    command: PythonCommandLine,
+    requirement: SystemRequirement,
+): Promise<string[]> => {
+    const [_, ...args] = command;
+    const resolution = await resolvePythonRequirement(requirement);
+    if ("error" in resolution) {
+        throw new Error(`Failed to find compatible python version. ${resolution.message}`);
+    }
+    return [resolution.command, ...(resolution.additionalArgs ?? []), ...args];
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// TYPES
+const PythonRequirement = "python";
+const PRINT_PYTHON_VERSION_SCRIPT = "import sys; print('.'.join(map(str, sys.version_info[:3])))";
+
+type KnownPythonExe = "python.exe" | "python3.exe" | "python" | "python3";
+type PythonCommandLine = [KnownPythonExe, ...string[]];
+
+interface MoreOptions extends SpawnOptions {
+    onCreate?(cp: ChildProcess): void;
+    onStdOutData?(chunk: any): void;
+    onStdErrData?(chunk: any): void;
+}
+
+interface SystemRequirement {
+    version?: string;
+    /**
+     * Name of an environment variable where the user could provide the path to the exe.
+     * @example "AUTOREST_PYTHON_PATH"
+     */
+    environmentVariable?: string;
+}
+
+interface SystemRequirementResolution {
+    /**
+     * Name of the requirement.
+     * @example python, java, etc.
+     */
+    name: string;
+
+    /**
+     * Name of the command
+     * @example python3, /home/myuser/python39/python, java, etc.
+     */
+    command: string;
+
+    /**
+     * List of additional arguments to pass to this command.
+     * @example '-3' for 'py' to specify to use python 3
+     */
+    additionalArgs?: string[];
+}
+
+interface ExecResult {
+    stdout: string;
+    stderr: string;
+
+    /**
+     * Union of stdout and stderr.
+     */
+    log: string;
+    error: Error | null;
+    code: number | null;
+}
+
+interface SystemRequirementError extends SystemRequirementResolution {
+    error: true;
+    message: string;
+    neededVersion?: string;
+    actualVersion?: string;
+}

package/scripts/run-python3.cjs

@@ -1,22 +0,0 @@
-// This script wraps logic in @azure-tools/extension to resolve
-// the path to Python 3 so that a Python script file can be run
-// from an npm script in package.json.  It uses the same Python 3
-// path resolution algorithm as AutoRest so that the behavior
-// is fully consistent (and also supports AUTOREST_PYTHON_EXE).
-//
-// Invoke it like so: "node run-python3.cjs script.py"
-
-const cp = require("child_process");
-const extension = require("./system-requirements.cjs");
-
-async function runPython3(scriptName, ...args) {
-    const command = await extension.patchPythonPath(["python", scriptName, ...args], { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE" });
-    cp.execSync(command.join(" "), {
-      stdio: [0, 1, 2]
-    });
-  }
-  
-  runPython3(...process.argv.slice(2)).catch(err => {
-    console.error(err.toString());
-    process.exit(1);
-  });