@typespec/http-client-python 0.28.2 → 0.29.0

package/eng/scripts/ci/run_pyright.py

@@ -13,21 +13,25 @@ from subprocess import check_output, CalledProcessError
 import logging
 import sys
 import time
-from util import run_check
+from util import run_check, get_package_namespace_dir
 
 logging.getLogger().setLevel(logging.INFO)
 
 
 def get_pyright_config_file_location():
-    pyright_config = os.path.join(os.getcwd(), "../../eng/scripts/ci/pyrightconfig.json")
+    # When running from tests/ directory via tox
+    pyright_config = os.path.join(os.getcwd(), "../eng/scripts/ci/config/pyrightconfig.json")
     if os.path.exists(pyright_config):
         return pyright_config
-    else:
-        return os.path.join(os.getcwd(), "../../../eng/scripts/ci/pyrightconfig.json")
+    # Fallback for running from different directories
+    return os.path.join(os.path.dirname(__file__), "config/pyrightconfig.json")
 
 
 def _single_dir_pyright(mod):
-    inner_class = next(d for d in mod.iterdir() if d.is_dir() and not str(d).endswith("egg-info"))
+    inner_class = get_package_namespace_dir(mod)
+    if not inner_class:
+        logging.info(f"No package directory found in {mod}, skipping")
+        return True
     retries = 3
     while retries:
         try:

package/eng/scripts/ci/run_sphinx_build.py

@@ -8,18 +8,21 @@
 # This script is used to execute sphinx documentation build within a tox environment.
 # It uses a central sphinx configuration and validates docstrings by running sphinx-build.
 
-from subprocess import check_call, CalledProcessError
+from subprocess import run, TimeoutExpired
 import os
 import logging
 import sys
 from pathlib import Path
-from util import run_check
+from util import run_check, SKIP_PACKAGE_DIRS
 
 logging.getLogger().setLevel(logging.INFO)
 
 # Get the central Sphinx config directory
 SPHINX_CONF_DIR = os.path.abspath(os.path.dirname(__file__))
 
+# Timeout for each sphinx build (seconds)
+SPHINX_TIMEOUT = 120
+
 
 def _create_minimal_index_rst(docs_dir, package_name, module_names):
     """Create a minimal index.rst file for sphinx to process."""
@@ -50,7 +53,12 @@ def _single_dir_sphinx(mod):
 
     # Find the actual Python package directories
     package_dirs = [
-        d for d in mod.iterdir() if d.is_dir() and not d.name.startswith("_") and (d / "__init__.py").exists()
+        d
+        for d in mod.iterdir()
+        if d.is_dir()
+        and not d.name.startswith("_")
+        and d.name not in SKIP_PACKAGE_DIRS
+        and (d / "__init__.py").exists()
     ]
 
     if not package_dirs:
@@ -85,7 +93,7 @@ def _single_dir_sphinx(mod):
     sys.path.insert(0, str(mod.absolute()))
 
     try:
-        result = check_call(
+        result = run(
             [
                 sys.executable,
                 "-m",
@@ -100,12 +108,19 @@ def _single_dir_sphinx(mod):
                 "-q",  # Quiet mode (only show warnings/errors)
                 str(docs_dir.absolute()),  # Source directory
                 str(output_dir.absolute()),  # Output directory
-            ]
+            ],
+            capture_output=True,
+            timeout=SPHINX_TIMEOUT,
         )
-        logging.info(f"Sphinx build completed successfully for {mod.stem}")
-        return True
-    except CalledProcessError as e:
-        logging.error(f"{mod.stem} exited with sphinx build error {e.returncode}")
+        if result.returncode == 0:
+            return True
+        logging.error(f"{mod.stem} sphinx error: {result.stderr.decode()[:500]}")
+        return False
+    except TimeoutExpired:
+        logging.error(f"{mod.stem} timed out after {SPHINX_TIMEOUT}s")
+        return False
+    except Exception as e:
+        logging.error(f"{mod.stem} sphinx error: {e}")
         return False
     finally:
         # Remove from sys.path

package/eng/scripts/ci/typecheck.ts

@@ -0,0 +1,225 @@
+/* eslint-disable no-console */
+import { spawn } from "child_process";
+import fs from "fs";
+import { dirname, join } from "path";
+import pc from "picocolors";
+import { fileURLToPath } from "url";
+import { parseArgs } from "util";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "../../../");
+const testsDir = join(root, "tests");
+
+// Get Python venv path
+function getVenvPython(): string {
+  const venvPath = join(root, "venv");
+  if (fs.existsSync(join(venvPath, "bin"))) {
+    return join(venvPath, "bin", "python");
+  } else if (fs.existsSync(join(venvPath, "Scripts"))) {
+    return join(venvPath, "Scripts", "python.exe");
+  }
+  throw new Error("Virtual environment not found. Run 'npm run install' first.");
+}
+
+const argv = parseArgs({
+  args: process.argv.slice(2),
+  options: {
+    generated: { type: "boolean" },
+    flavor: { type: "string", short: "f" },
+    mypy: { type: "boolean" },
+    pyright: { type: "boolean" },
+    help: { type: "boolean", short: "h" },
+  },
+});
+
+if (argv.values.help) {
+  console.log(`
+${pc.bold("Usage:")} tsx typecheck.ts [options]
+
+${pc.bold("Description:")}
+  Run type checking (mypy + pyright) on Python code.
+
+${pc.bold("Options:")}
+  ${pc.cyan("--generated")}
+      Run type checking on generated SDK packages (via tox).
+      Use with --flavor to specify azure or unbranded.
+
+  ${pc.cyan("-f, --flavor <azure|unbranded>")}
+      SDK flavor to check (only applies to --generated).
+      If not specified, checks both flavors.
+
+  ${pc.cyan("--mypy")}
+      Run only mypy (skip pyright).
+
+  ${pc.cyan("--pyright")}
+      Run only pyright (skip mypy).
+
+  ${pc.cyan("-h, --help")}
+      Show this help message.
+
+${pc.bold("Examples:")}
+  ${pc.dim("# Type check pygen source (default - runs both mypy and pyright)")}
+  tsx typecheck.ts
+
+  ${pc.dim("# Run only mypy on pygen source")}
+  tsx typecheck.ts --mypy
+
+  ${pc.dim("# Run only pyright on pygen source")}
+  tsx typecheck.ts --pyright
+
+  ${pc.dim("# Type check generated SDK packages")}
+  tsx typecheck.ts --generated
+
+  ${pc.dim("# Type check generated SDK packages for azure only")}
+  tsx typecheck.ts --generated --flavor=azure
+`);
+  process.exit(0);
+}
+
+function runCommand(command: string, args: string[], cwd?: string): Promise<boolean> {
+  return new Promise((resolve) => {
+    console.log(`${pc.cyan("[RUN]")} ${command} ${args.join(" ")}`);
+    const proc = spawn(command, args, {
+      cwd: cwd || root,
+      stdio: "inherit",
+      shell: true,
+    });
+
+    proc.on("close", (code) => {
+      if (code === 0) {
+        console.log(`${pc.green("[PASS]")} ${command} completed successfully`);
+        resolve(true);
+      } else {
+        console.log(`${pc.red("[FAIL]")} ${command} failed with code ${code}`);
+        resolve(false);
+      }
+    });
+
+    proc.on("error", (err) => {
+      console.log(`${pc.red("[ERROR]")} ${command}: ${err.message}`);
+      resolve(false);
+    });
+  });
+}
+
+async function runMypyOnPygen(): Promise<boolean> {
+  console.log(`\n${pc.bold("=== Running mypy on pygen ===")}\n`);
+
+  let pythonPath: string;
+  try {
+    pythonPath = getVenvPython();
+  } catch (error) {
+    console.error(pc.red((error as Error).message));
+    return false;
+  }
+
+  return runCommand(pythonPath, [
+    "-m",
+    "mypy",
+    "generator/pygen",
+    "--config-file=eng/scripts/ci/config/mypy.ini",
+    "--ignore-missing",
+  ]);
+}
+
+async function runPyrightOnPygen(): Promise<boolean> {
+  console.log(`\n${pc.bold("=== Running pyright on pygen ===")}\n`);
+
+  let pythonPath: string;
+  try {
+    pythonPath = getVenvPython();
+  } catch (error) {
+    console.error(pc.red((error as Error).message));
+    return false;
+  }
+
+  return runCommand(pythonPath, [
+    "-m",
+    "pyright",
+    "-p",
+    "eng/scripts/ci/config/pyrightconfig.json",
+    "generator/pygen",
+  ]);
+}
+
+async function runTypecheckOnGenerated(
+  flavor?: string,
+  mypyOnly?: boolean,
+  pyrightOnly?: boolean,
+): Promise<boolean> {
+  const flavors = flavor ? [flavor] : ["azure", "unbranded"];
+  console.log(
+    `\n${pc.bold(`=== Type Checking Generated SDK Packages (${flavors.join(", ")}) ===`)}\n`,
+  );
+
+  let pythonPath: string;
+  try {
+    pythonPath = getVenvPython();
+  } catch (error) {
+    console.error(pc.red((error as Error).message));
+    return false;
+  }
+
+  let success = true;
+
+  for (const f of flavors) {
+    if (!pyrightOnly) {
+      const mypyEnv = `mypy-${f}`;
+      const result = await runCommand(
+        pythonPath,
+        ["-m", "tox", "-c", "tox.ini", "-e", mypyEnv],
+        testsDir,
+      );
+      if (!result) success = false;
+    }
+
+    if (!mypyOnly) {
+      const pyrightEnv = `pyright-${f}`;
+      const result = await runCommand(
+        pythonPath,
+        ["-m", "tox", "-c", "tox.ini", "-e", pyrightEnv],
+        testsDir,
+      );
+      if (!result) success = false;
+    }
+  }
+
+  return success;
+}
+
+async function main(): Promise<void> {
+  const runGenerated = argv.values.generated;
+  const mypyOnly = argv.values.mypy && !argv.values.pyright;
+  const pyrightOnly = argv.values.pyright && !argv.values.mypy;
+
+  // If --generated is specified, type check generated packages
+  if (runGenerated) {
+    const result = await runTypecheckOnGenerated(argv.values.flavor, mypyOnly, pyrightOnly);
+    if (!result) process.exit(1);
+    console.log(`\n${pc.green(pc.bold("Generated SDK type checking complete!"))}\n`);
+    return;
+  }
+
+  // Default: type check pygen source
+  let success = true;
+
+  if (!pyrightOnly) {
+    const mypyResult = await runMypyOnPygen();
+    if (!mypyResult) success = false;
+  }
+
+  if (!mypyOnly) {
+    const pyrightResult = await runPyrightOnPygen();
+    if (!pyrightResult) success = false;
+  }
+
+  if (!success) {
+    process.exit(1);
+  }
+
+  console.log(`\n${pc.green(pc.bold("All type checks passed!"))}\n`);
+}
+
+main().catch((error) => {
+  console.error(`${pc.red("Unexpected error:")}`, error);
+  process.exit(1);
+});

package/eng/scripts/ci/util.py

@@ -8,14 +8,32 @@ import os
 import logging
 from pathlib import Path
 import argparse
-from multiprocessing import Pool
+from concurrent.futures import ProcessPoolExecutor, as_completed
 
 logging.getLogger().setLevel(logging.INFO)
 
-ROOT_FOLDER = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..", "..", "generator"))
+# Root is the tests directory (4 levels up from this file: ci -> scripts -> eng -> package_root, then into tests)
+ROOT_FOLDER = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..", "..", "tests"))
 
 IGNORE_FOLDER = []
 
+# Directories inside each generated package that should be skipped by all CI checks.
+# These are auto-generated test/sample scaffolding, not the actual SDK code.
+SKIP_PACKAGE_DIRS = {"generated_tests", "generated_samples", "build", "__pycache__", ".pytest_cache"}
+
+
+def get_package_namespace_dir(mod):
+    """Find the actual namespace directory inside a generated package, skipping non-SDK dirs."""
+    for d in mod.iterdir():
+        if (
+            d.is_dir()
+            and not d.name.startswith("_")
+            and not d.name.endswith("egg-info")
+            and d.name not in SKIP_PACKAGE_DIRS
+        ):
+            return d
+    return None
+
 
 def run_check(name, call_back, log_info):
     parser = argparse.ArgumentParser(
@@ -25,16 +43,9 @@ def run_check(name, call_back, log_info):
         "-t",
         "--test-folder",
         dest="test_folder",
-        help="The test folder we're in. Can be 'azure' or 'vanilla'",
+        help="The test folder we're in. Can be 'azure' or 'unbranded'",
         required=True,
     )
-    parser.add_argument(
-        "-g",
-        "--generator",
-        dest="generator",
-        help="The generator we're using. Can be 'legacy', 'version-tolerant'.",
-        required=False,
-    )
     parser.add_argument(
         "-f",
         "--file-name",
@@ -46,27 +57,52 @@ def run_check(name, call_back, log_info):
         "-s",
         "--subfolder",
         dest="subfolder",
-        help="The specific sub folder to validate, default to Expected/AcceptanceTests. Optional.",
+        help="The subfolder containing generated code, default to 'generated'.",
         required=False,
-        default="Expected/AcceptanceTests",
+        default="generated",
+    )
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        dest="jobs",
+        help="Number of parallel jobs (default: CPU count)",
+        type=int,
+        required=False,
+        default=max(1, os.cpu_count()),
     )
 
     args = parser.parse_args()
 
-    pkg_dir = Path(ROOT_FOLDER) / Path("test") / Path(args.test_folder)
-    if args.generator:
-        pkg_dir /= Path(args.generator)
-    if args.subfolder:
-        pkg_dir /= Path(args.subfolder)
+    # Path structure: tests/generated/{test_folder}/
+    pkg_dir = Path(ROOT_FOLDER) / Path(args.subfolder) / Path(args.test_folder)
     dirs = [d for d in pkg_dir.iterdir() if d.is_dir() and not d.stem.startswith("_") and d.stem not in IGNORE_FOLDER]
     if args.file_name:
         dirs = [d for d in dirs if args.file_name.lower() in d.stem.lower()]
-    if len(dirs) > 1:
-        with Pool() as pool:
-            result = pool.map(call_back, dirs)
-        response = all(result)
-    else:
-        response = call_back(dirs[0])
-    if not response:
-        logging.error("%s fails", log_info)
+
+    if not dirs:
+        logging.info("No directories to process")
+        return
+
+    logging.info(f"Processing {len(dirs)} packages with {args.jobs} parallel jobs...")
+
+    failed = []
+    succeeded = 0
+
+    with ProcessPoolExecutor(max_workers=args.jobs) as executor:
+        futures = {executor.submit(call_back, d): d for d in dirs}
+        for future in as_completed(futures):
+            pkg = futures[future]
+            try:
+                if future.result():
+                    succeeded += 1
+                else:
+                    failed.append(pkg.stem)
+            except Exception as e:
+                logging.error(f"{pkg.stem} raised exception: {e}")
+                failed.append(pkg.stem)
+
+    logging.info(f"{log_info}: {succeeded} succeeded, {len(failed)} failed")
+
+    if failed:
+        logging.error(f"{log_info} failed for: {', '.join(failed)}")
         exit(1)

package/eng/scripts/ci/utils.ts

@@ -1,8 +1,8 @@
 /* eslint-disable no-console */
-import chalk from "chalk";
 import { execFile } from "child_process";
 import { existsSync } from "fs";
 import { dirname, join, resolve } from "path";
+import pc from "picocolors";
 import process from "process";
 import { fileURLToPath } from "url";
 import { parseArgs, promisify } from "util";
@@ -21,11 +21,11 @@ export async function executeCommand(command: string, args: string[]) {
   const execFileAsync = promisify(execFile);
   try {
     await execFileAsync(command, args, { shell: true });
-    console.log(chalk.green(`${command} passed`));
+    console.log(pc.green(`${command} passed`));
   } catch (err: any) {
-    console.error(chalk.red(`Error executing ${command}`));
-    if (err.stdout) console.error(chalk.yellow("STDOUT:"), err.stdout);
-    if (err.stderr) console.error(chalk.yellow("STDERR:"), err.stderr);
+    console.error(pc.red(`Error executing ${command}`));
+    if (err.stdout) console.error(pc.yellow("STDOUT:"), err.stdout);
+    if (err.stderr) console.error(pc.yellow("STDERR:"), err.stderr);
     process.exit(1);
   }
 }

package/eng/scripts/setup/build.ts

@@ -4,7 +4,7 @@ import { runPython3 } from "./run-python3.js";
 async function main() {
   await runPython3("./eng/scripts/setup/build_pygen_wheel.py");
   // remove the venv_build_wheel directory
-  exec("rimraf ./venv_build_wheel", (error, stdout, stderr) => {
+  exec("rimraf ./venv_build_wheel", (error, stdout, _stderr) => {
     if (error) {
       console.error(`Error executing command: ${error.message}`); // eslint-disable-line no-console
       return;

package/eng/scripts/setup/build_pygen_wheel.py

@@ -7,8 +7,8 @@
 # --------------------------------------------------------------------------
 import sys
 
-if not sys.version_info >= (3, 9, 0):
-    raise Exception("Autorest for Python extension requires Python 3.9 at least")
+if not sys.version_info >= (3, 10, 0):
+    raise Exception("Autorest for Python extension requires Python 3.10 at least")
 
 try:
     from package_manager import detect_package_manager, PackageManagerNotFoundError
@@ -18,13 +18,14 @@ except (ImportError, ModuleNotFoundError, PackageManagerNotFoundError):
     raise Exception("Your Python installation doesn't have a suitable package manager (pip or uv) available")
 
 
-# Now we have a package manager (pip or uv) and Py >= 3.9, go to work
+# Now we have a package manager (pip or uv) and Py >= 3.10, go to work
 
 from pathlib import Path
 
 from venvtools import python_run
 from package_manager import install_packages, create_venv_with_package_manager
 
+# eng/scripts/setup/build_pygen_wheel.py -> need to go up 4 levels to get to package root
 _ROOT_DIR = Path(__file__).parent.parent.parent.parent
 
 

package/eng/scripts/setup/install.py

@@ -7,9 +7,9 @@
 # --------------------------------------------------------------------------
 import sys
 
-if not sys.version_info >= (3, 9, 0):
+if not sys.version_info >= (3, 10, 0):
     print(
-        "Warning: Autorest for Python extension requires Python 3.9 at least. We will run your code with Pyodide since your Python version isn't adequate."
+        "Warning: Autorest for Python extension requires Python 3.10 at least. We will run your code with Pyodide since your Python version isn't adequate."
     )
     sys.exit(2)  # Exit code 2 for inadequate environment
 
@@ -32,12 +32,13 @@ except (ImportError, ModuleNotFoundError):
     sys.exit(2)  # Exit code 2 for inadequate environment
 
 
-# Now we have a package manager (uv or pip) and Py >= 3.9, go to work
+# Now we have a package manager (uv or pip) and Py >= 3.10, go to work
 # At this point, both Python and package manager are confirmed to be available
 # Any failures from here should fail the npm install, not fallback to Pyodide
 
 from pathlib import Path
 
+# eng/scripts/setup/install.py -> need to go up 4 levels to get to package root
 _ROOT_DIR = Path(__file__).parent.parent.parent.parent
 
 

package/eng/scripts/setup/install.ts

@@ -7,7 +7,7 @@ async function main() {
   try {
     // First try to resolve Python path
     pythonCommand = await patchPythonPath(["python", "./eng/scripts/setup/install.py"], {
-      version: ">=3.9",
+      version: ">=3.10",
       environmentVariable: "AUTOREST_PYTHON_EXE",
     });
   } catch (error) {

package/eng/scripts/setup/prepare.py

@@ -7,14 +7,15 @@
 # --------------------------------------------------------------------------
 import sys
 
-if not sys.version_info >= (3, 9, 0):
+if not sys.version_info >= (3, 10, 0):
     raise Warning(
-        "Autorest for Python extension requires Python 3.9 at least. We will run your code with Pyodide since your Python version isn't adequate."
+        "Autorest for Python extension requires Python 3.10 at least. We will run your code with Pyodide since your Python version isn't adequate."
     )
 
 from pathlib import Path
 from package_manager import create_venv_with_package_manager, install_packages
 
+# eng/scripts/setup/prepare.py -> need to go up 4 levels to get to package root
 _ROOT_DIR = Path(__file__).parent.parent.parent.parent
 
 

package/eng/scripts/setup/prepare.ts

@@ -3,7 +3,7 @@ import { runPython3 } from "./run-python3.js";
 async function main() {
   try {
     await runPython3("./eng/scripts/setup/prepare.py");
-  } catch (error) {
+  } catch {
     console.log("No Python found on your local environment. We will use Pyodide instead."); // eslint-disable-line no-console
   }
 }

package/eng/scripts/setup/run-python3.ts

@@ -11,7 +11,7 @@ import { patchPythonPath } from "./system-requirements.js";
 
 export async function runPython3(...args: string[]) {
   const command = await patchPythonPath(["python", ...args], {
-    version: ">=3.9",
+    version: ">=3.10",
     environmentVariable: "AUTOREST_PYTHON_EXE",
   });
   cp.execSync(command.join(" "), {