@typespec/http-client-python 0.28.3 → 0.29.0

package/eng/scripts/ci/run_apiview.py

@@ -10,32 +10,38 @@
 
 import os
 import sys
-from subprocess import check_call, CalledProcessError
+from subprocess import run, TimeoutExpired
 import logging
 from util import run_check
 
 logging.getLogger().setLevel(logging.INFO)
 
+# Timeout for each apiview generation (seconds)
+APIVIEW_TIMEOUT = 30
+
 
 def _single_dir_apiview(mod):
-    loop = 0
-    while True:
+    for attempt in range(2):
         try:
-            check_call(
-                [
-                    "apistubgen",
-                    "--pkg-path",
-                    str(mod.absolute()),
-                ]
+            result = run(
+                ["apistubgen", "--pkg-path", str(mod.absolute())],
+                capture_output=True,
+                timeout=APIVIEW_TIMEOUT,
             )
-        except CalledProcessError as e:
-            if loop >= 2:  # retry for maximum 3 times because sometimes the apistubgen has transient failure.
-                logging.error("{} exited with apiview generation error {}".format(mod.stem, e.returncode))
+            if result.returncode == 0:
+                return True
+            if attempt == 1:
+                logging.error(f"{mod.stem} failed: {result.stderr.decode()[:200]}")
+                return False
+        except TimeoutExpired:
+            if attempt == 1:
+                logging.error(f"{mod.stem} timed out after {APIVIEW_TIMEOUT}s")
+                return False
+        except Exception as e:
+            if attempt == 1:
+                logging.error(f"{mod.stem} error: {e}")
                 return False
-            else:
-                loop += 1
-                continue
-        return True
+    return False
 
 
 if __name__ == "__main__":

package/eng/scripts/ci/run_mypy.py

@@ -12,7 +12,7 @@ from subprocess import check_call, CalledProcessError
 import os
 import logging
 import sys
-from util import run_check
+from util import run_check, get_package_namespace_dir
 
 logging.getLogger().setLevel(logging.INFO)
 
@@ -26,24 +26,34 @@ def get_config_file_location():
     return os.path.join(os.path.dirname(__file__), "config/mypy.ini")
 
 
-def _single_dir_mypy(mod):
-    inner_class = next(d for d in mod.iterdir() if d.is_dir() and not str(d).endswith("egg-info"))
-    try:
-        check_call(
-            [
-                sys.executable,
-                "-m",
-                "mypy",
-                "--config-file",
-                get_config_file_location(),
-                "--ignore-missing",
-                str(inner_class.absolute()),
-            ]
-        )
+def _single_dir_mypy(mod, retries=2):
+    inner_class = get_package_namespace_dir(mod)
+    if not inner_class:
+        logging.info(f"No package directory found in {mod}, skipping")
         return True
-    except CalledProcessError as e:
-        logging.error("{} exited with mypy error {}".format(inner_class.stem, e.returncode))
-        return False
+    for attempt in range(1, retries + 2):
+        try:
+            check_call(
+                [
+                    sys.executable,
+                    "-m",
+                    "mypy",
+                    "--config-file",
+                    get_config_file_location(),
+                    "--ignore-missing",
+                    str(inner_class.absolute()),
+                ]
+            )
+            return True
+        except CalledProcessError as e:
+            if attempt <= retries:
+                logging.warning(
+                    "{} mypy attempt {} failed (exit {}), retrying...".format(inner_class.stem, attempt, e.returncode)
+                )
+            else:
+                logging.error("{} exited with mypy error {}".format(inner_class.stem, e.returncode))
+                return False
+    return False
 
 
 if __name__ == "__main__":

package/eng/scripts/ci/run_pylint.py

@@ -12,7 +12,7 @@ from subprocess import check_call, CalledProcessError
 import os
 import logging
 import sys
-from util import run_check
+from util import run_check, get_package_namespace_dir
 
 logging.getLogger().setLevel(logging.INFO)
 
@@ -27,22 +27,29 @@ def get_rfc_file_location():
 
 
 def _single_dir_pylint(mod):
-    inner_class = next(d for d in mod.iterdir() if d.is_dir() and not str(d).endswith("egg-info"))
+    inner_class = get_package_namespace_dir(mod)
+    if not inner_class:
+        logging.info(f"No package directory found in {mod}, skipping")
+        return True
+    # Only load the Azure pylint guidelines checker plugin for azure packages.
+    # The plugin (azure-pylint-guidelines-checker) is only installed in the
+    # lint-azure tox environment and is not available for unbranded packages.
+    is_azure = "azure" in mod.parts
+    pylint_args = [
+        sys.executable,
+        "-m",
+        "pylint",
+        "--rcfile={}".format(get_rfc_file_location()),
+        "--evaluation=(max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention + info)/ statement) * 10)))",
+        "--output-format=parseable",
+        "--recursive=y",
+        "--py-version=3.10",
+    ]
+    if is_azure:
+        pylint_args.append("--load-plugins=pylint_guidelines_checker")
+    pylint_args.append(str(inner_class.absolute()))
     try:
-        check_call(
-            [
-                sys.executable,
-                "-m",
-                "pylint",
-                "--rcfile={}".format(get_rfc_file_location()),
-                "--evaluation=(max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention + info)/ statement) * 10)))",
-                "--load-plugins=pylint_guidelines_checker",
-                "--output-format=parseable",
-                "--recursive=y",
-                "--py-version=3.9",
-                str(inner_class.absolute()),
-            ]
-        )
+        check_call(pylint_args)
         return True
     except CalledProcessError as e:
         logging.error("{} exited with linting error {}".format(str(inner_class.absolute()), e.returncode))

package/eng/scripts/ci/run_pyright.py

@@ -13,7 +13,7 @@ from subprocess import check_output, CalledProcessError
 import logging
 import sys
 import time
-from util import run_check
+from util import run_check, get_package_namespace_dir
 
 logging.getLogger().setLevel(logging.INFO)
 
@@ -28,7 +28,10 @@ def get_pyright_config_file_location():
 
 
 def _single_dir_pyright(mod):
-    inner_class = next(d for d in mod.iterdir() if d.is_dir() and not str(d).endswith("egg-info"))
+    inner_class = get_package_namespace_dir(mod)
+    if not inner_class:
+        logging.info(f"No package directory found in {mod}, skipping")
+        return True
     retries = 3
     while retries:
         try:

package/eng/scripts/ci/run_sphinx_build.py

@@ -8,18 +8,21 @@
 # This script is used to execute sphinx documentation build within a tox environment.
 # It uses a central sphinx configuration and validates docstrings by running sphinx-build.
 
-from subprocess import check_call, CalledProcessError
+from subprocess import run, TimeoutExpired
 import os
 import logging
 import sys
 from pathlib import Path
-from util import run_check
+from util import run_check, SKIP_PACKAGE_DIRS
 
 logging.getLogger().setLevel(logging.INFO)
 
 # Get the central Sphinx config directory
 SPHINX_CONF_DIR = os.path.abspath(os.path.dirname(__file__))
 
+# Timeout for each sphinx build (seconds)
+SPHINX_TIMEOUT = 120
+
 
 def _create_minimal_index_rst(docs_dir, package_name, module_names):
     """Create a minimal index.rst file for sphinx to process."""
@@ -50,7 +53,12 @@ def _single_dir_sphinx(mod):
 
     # Find the actual Python package directories
     package_dirs = [
-        d for d in mod.iterdir() if d.is_dir() and not d.name.startswith("_") and (d / "__init__.py").exists()
+        d
+        for d in mod.iterdir()
+        if d.is_dir()
+        and not d.name.startswith("_")
+        and d.name not in SKIP_PACKAGE_DIRS
+        and (d / "__init__.py").exists()
     ]
 
     if not package_dirs:
@@ -85,7 +93,7 @@ def _single_dir_sphinx(mod):
     sys.path.insert(0, str(mod.absolute()))
 
     try:
-        result = check_call(
+        result = run(
             [
                 sys.executable,
                 "-m",
@@ -100,12 +108,19 @@ def _single_dir_sphinx(mod):
                 "-q",  # Quiet mode (only show warnings/errors)
                 str(docs_dir.absolute()),  # Source directory
                 str(output_dir.absolute()),  # Output directory
-            ]
+            ],
+            capture_output=True,
+            timeout=SPHINX_TIMEOUT,
         )
-        logging.info(f"Sphinx build completed successfully for {mod.stem}")
-        return True
-    except CalledProcessError as e:
-        logging.error(f"{mod.stem} exited with sphinx build error {e.returncode}")
+        if result.returncode == 0:
+            return True
+        logging.error(f"{mod.stem} sphinx error: {result.stderr.decode()[:500]}")
+        return False
+    except TimeoutExpired:
+        logging.error(f"{mod.stem} timed out after {SPHINX_TIMEOUT}s")
+        return False
+    except Exception as e:
+        logging.error(f"{mod.stem} sphinx error: {e}")
         return False
     finally:
         # Remove from sys.path

package/eng/scripts/ci/util.py

@@ -8,14 +8,32 @@ import os
 import logging
 from pathlib import Path
 import argparse
-from multiprocessing import Pool
+from concurrent.futures import ProcessPoolExecutor, as_completed
 
 logging.getLogger().setLevel(logging.INFO)
 
-ROOT_FOLDER = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..", "generator"))
+# Root is the tests directory (4 levels up from this file: ci -> scripts -> eng -> package_root, then into tests)
+ROOT_FOLDER = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..", "..", "tests"))
 
 IGNORE_FOLDER = []
 
+# Directories inside each generated package that should be skipped by all CI checks.
+# These are auto-generated test/sample scaffolding, not the actual SDK code.
+SKIP_PACKAGE_DIRS = {"generated_tests", "generated_samples", "build", "__pycache__", ".pytest_cache"}
+
+
+def get_package_namespace_dir(mod):
+    """Find the actual namespace directory inside a generated package, skipping non-SDK dirs."""
+    for d in mod.iterdir():
+        if (
+            d.is_dir()
+            and not d.name.startswith("_")
+            and not d.name.endswith("egg-info")
+            and d.name not in SKIP_PACKAGE_DIRS
+        ):
+            return d
+    return None
+
 
 def run_check(name, call_back, log_info):
     parser = argparse.ArgumentParser(
@@ -25,16 +43,9 @@ def run_check(name, call_back, log_info):
         "-t",
         "--test-folder",
         dest="test_folder",
-        help="The test folder we're in. Can be 'azure' or 'vanilla'",
+        help="The test folder we're in. Can be 'azure' or 'unbranded'",
         required=True,
     )
-    parser.add_argument(
-        "-g",
-        "--generator",
-        dest="generator",
-        help="The generator we're using. Can be 'legacy', 'version-tolerant'.",
-        required=False,
-    )
     parser.add_argument(
         "-f",
         "--file-name",
@@ -46,27 +57,52 @@ def run_check(name, call_back, log_info):
         "-s",
         "--subfolder",
         dest="subfolder",
-        help="The specific sub folder to validate, default to Expected/AcceptanceTests. Optional.",
+        help="The subfolder containing generated code, default to 'generated'.",
         required=False,
-        default="Expected/AcceptanceTests",
+        default="generated",
+    )
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        dest="jobs",
+        help="Number of parallel jobs (default: CPU count)",
+        type=int,
+        required=False,
+        default=max(1, os.cpu_count()),
     )
 
     args = parser.parse_args()
 
-    pkg_dir = Path(ROOT_FOLDER) / Path("test") / Path(args.test_folder)
-    if args.generator:
-        pkg_dir /= Path(args.generator)
-    if args.subfolder:
-        pkg_dir /= Path(args.subfolder)
+    # Path structure: tests/generated/{test_folder}/
+    pkg_dir = Path(ROOT_FOLDER) / Path(args.subfolder) / Path(args.test_folder)
     dirs = [d for d in pkg_dir.iterdir() if d.is_dir() and not d.stem.startswith("_") and d.stem not in IGNORE_FOLDER]
     if args.file_name:
         dirs = [d for d in dirs if args.file_name.lower() in d.stem.lower()]
-    if len(dirs) > 1:
-        with Pool() as pool:
-            result = pool.map(call_back, dirs)
-        response = all(result)
-    else:
-        response = call_back(dirs[0])
-    if not response:
-        logging.error("%s fails", log_info)
+
+    if not dirs:
+        logging.info("No directories to process")
+        return
+
+    logging.info(f"Processing {len(dirs)} packages with {args.jobs} parallel jobs...")
+
+    failed = []
+    succeeded = 0
+
+    with ProcessPoolExecutor(max_workers=args.jobs) as executor:
+        futures = {executor.submit(call_back, d): d for d in dirs}
+        for future in as_completed(futures):
+            pkg = futures[future]
+            try:
+                if future.result():
+                    succeeded += 1
+                else:
+                    failed.append(pkg.stem)
+            except Exception as e:
+                logging.error(f"{pkg.stem} raised exception: {e}")
+                failed.append(pkg.stem)
+
+    logging.info(f"{log_info}: {succeeded} succeeded, {len(failed)} failed")
+
+    if failed:
+        logging.error(f"{log_info} failed for: {', '.join(failed)}")
         exit(1)

package/eng/scripts/setup/build_pygen_wheel.py

@@ -7,8 +7,8 @@
 # --------------------------------------------------------------------------
 import sys
 
-if not sys.version_info >= (3, 9, 0):
-    raise Exception("Autorest for Python extension requires Python 3.9 at least")
+if not sys.version_info >= (3, 10, 0):
+    raise Exception("Autorest for Python extension requires Python 3.10 at least")
 
 try:
     from package_manager import detect_package_manager, PackageManagerNotFoundError
@@ -18,7 +18,7 @@ except (ImportError, ModuleNotFoundError, PackageManagerNotFoundError):
     raise Exception("Your Python installation doesn't have a suitable package manager (pip or uv) available")
 
 
-# Now we have a package manager (pip or uv) and Py >= 3.9, go to work
+# Now we have a package manager (pip or uv) and Py >= 3.10, go to work
 
 from pathlib import Path
 

package/eng/scripts/setup/install.py

@@ -7,9 +7,9 @@
 # --------------------------------------------------------------------------
 import sys
 
-if not sys.version_info >= (3, 9, 0):
+if not sys.version_info >= (3, 10, 0):
     print(
-        "Warning: Autorest for Python extension requires Python 3.9 at least. We will run your code with Pyodide since your Python version isn't adequate."
+        "Warning: Autorest for Python extension requires Python 3.10 at least. We will run your code with Pyodide since your Python version isn't adequate."
     )
     sys.exit(2)  # Exit code 2 for inadequate environment
 
@@ -32,7 +32,7 @@ except (ImportError, ModuleNotFoundError):
     sys.exit(2)  # Exit code 2 for inadequate environment
 
 
-# Now we have a package manager (uv or pip) and Py >= 3.9, go to work
+# Now we have a package manager (uv or pip) and Py >= 3.10, go to work
 # At this point, both Python and package manager are confirmed to be available
 # Any failures from here should fail the npm install, not fallback to Pyodide
 

package/eng/scripts/setup/install.ts

@@ -7,7 +7,7 @@ async function main() {
   try {
     // First try to resolve Python path
     pythonCommand = await patchPythonPath(["python", "./eng/scripts/setup/install.py"], {
-      version: ">=3.9",
+      version: ">=3.10",
       environmentVariable: "AUTOREST_PYTHON_EXE",
     });
   } catch (error) {

package/eng/scripts/setup/prepare.py

@@ -7,9 +7,9 @@
 # --------------------------------------------------------------------------
 import sys
 
-if not sys.version_info >= (3, 9, 0):
+if not sys.version_info >= (3, 10, 0):
     raise Warning(
-        "Autorest for Python extension requires Python 3.9 at least. We will run your code with Pyodide since your Python version isn't adequate."
+        "Autorest for Python extension requires Python 3.10 at least. We will run your code with Pyodide since your Python version isn't adequate."
     )
 
 from pathlib import Path

package/eng/scripts/setup/run-python3.ts

@@ -11,7 +11,7 @@ import { patchPythonPath } from "./system-requirements.js";
 
 export async function runPython3(...args: string[]) {
   const command = await patchPythonPath(["python", ...args], {
-    version: ">=3.9",
+    version: ">=3.10",
     environmentVariable: "AUTOREST_PYTHON_EXE",
   });
   cp.execSync(command.join(" "), {

package/eng/scripts/setup/run_batch.py

@@ -0,0 +1,197 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+"""
+Batch process multiple TypeSpec YAML files in a single Python process.
+This avoids the overhead of spawning a new Python process for each spec.
+"""
+
+import argparse
+import json
+import sys
+import os
+from pathlib import Path
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from multiprocessing import freeze_support
+
+# Add the generator to the path
+_ROOT_DIR = Path(__file__).parent.parent.parent.parent
+sys.path.insert(0, str(_ROOT_DIR / "generator"))
+
+
+def process_single_spec(config_path_str: str) -> tuple[str, bool, str]:
+    """Process a single spec from its config file.
+
+    Returns: (output_dir, success, error_message)
+    """
+    # Import inside function for multiprocessing compatibility
+    from pygen import preprocess, codegen
+
+    config_path = Path(config_path_str)
+    try:
+        with open(config_path, "r", encoding="utf-8") as f:
+            config = json.load(f)
+
+        yaml_path = config["yamlPath"]
+        command_args = config["commandArgs"]
+        output_dir = config["outputDir"]
+
+        # Pass command args directly to pygen - pygen expects hyphenated keys
+        # Remove keys that shouldn't be passed to pygen.
+        # Also coerce the string values "true"/"false" to real booleans, matching the behavior
+        # of pygen.utils.parse_args (the CLI path). Without this, the emitter passes string
+        # "false" for flags like keep-setup-py, which is truthy in Python and causes pygen to
+        # take the wrong branch (e.g. generating setup.py instead of pyproject.toml).
+        def _coerce(value):
+            if value == "true":
+                return True
+            if value == "false":
+                return False
+            return value
+
+        pygen_args = {k: _coerce(v) for k, v in command_args.items() if k not in ["emit-yaml-only"]}
+
+        # Run preprocess and codegen (black is batched at the end for performance)
+        preprocess.PreProcessPlugin(output_folder=output_dir, tsp_file=yaml_path, **pygen_args).process()
+
+        codegen.CodeGenerator(output_folder=output_dir, tsp_file=yaml_path, **pygen_args).process()
+
+        # Clean up the config file
+        config_path.unlink()
+
+        return (output_dir, True, "")
+    except Exception as e:
+        return (str(config_path), False, str(e))
+
+
+def render_progress_bar(completed: int, failed: int, total: int, width: int = 40) -> str:
+    """Render a progress bar with green for success and red for failures."""
+    success_count = completed - failed
+    success_width = round((success_count / total) * width) if total > 0 else 0
+    fail_width = round((failed / total) * width) if total > 0 else 0
+    empty_width = width - success_width - fail_width
+
+    # ANSI color codes
+    green_bg = "\033[42m"
+    red_bg = "\033[41m"
+    reset = "\033[0m"
+    dim = "\033[2m"
+    cyan = "\033[36m"
+
+    success_bar = f"{green_bg}{' ' * success_width}{reset}"
+    fail_bar = f"{red_bg}{' ' * fail_width}{reset}" if failed > 0 else ""
+    empty_bar = f"{dim}{'░' * max(0, empty_width)}{reset}"
+
+    percent = round((completed / total) * 100) if total > 0 else 0
+    return f"{success_bar}{fail_bar}{empty_bar} {cyan}{percent}%{reset} ({completed}/{total})"
+
+
+def collect_config_files(generated_dir: str, flavor: str) -> list[str]:
+    """Collect all .tsp-codegen-*.json config files from the generated directory."""
+    flavor_dir = Path(generated_dir) / "tests" / "generated" / flavor
+    if not flavor_dir.exists():
+        return []
+
+    config_files = []
+    for pkg_dir in flavor_dir.iterdir():
+        if pkg_dir.is_dir():
+            for f in pkg_dir.iterdir():
+                if f.name.startswith(".tsp-codegen-") and f.name.endswith(".json"):
+                    config_files.append(str(f))
+    return config_files
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Batch process TypeSpec YAML files")
+    parser.add_argument(
+        "--generated-dir",
+        required=True,
+        help="Path to the generator directory (config files are in ../tests/generated/<flavor>/)",
+    )
+    parser.add_argument(
+        "--flavor",
+        required=True,
+        help="Flavor to process (azure or unbranded)",
+    )
+    parser.add_argument(
+        "--jobs",
+        type=int,
+        default=4,
+        help="Number of parallel jobs (default: 4)",
+    )
+    args = parser.parse_args()
+
+    # Discover config files from the generated directory
+    config_files = collect_config_files(args.generated_dir, args.flavor)
+    total = len(config_files)
+
+    if total == 0:
+        print("No config files found, nothing to process")
+        return
+
+    print(f"Processing {total} specs with {args.jobs} parallel jobs...")
+
+    succeeded = 0
+    failed = 0
+    failed_specs = []
+    output_dirs = []
+    is_tty = sys.stdout.isatty()
+
+    def update_progress():
+        if is_tty:
+            sys.stdout.write(f"\r{render_progress_bar(succeeded + failed, failed, total)}")
+            sys.stdout.flush()
+
+    # Initial progress bar
+    update_progress()
+
+    # Use ProcessPoolExecutor for true parallelism (bypasses GIL)
+    with ProcessPoolExecutor(max_workers=args.jobs) as executor:
+        futures = {executor.submit(process_single_spec, cf): cf for cf in config_files}
+
+        for future in as_completed(futures):
+            output_dir, success, error = future.result()
+            if success:
+                succeeded += 1
+                output_dirs.append(output_dir)
+            else:
+                failed += 1
+                failed_specs.append(f"{output_dir}: {error}")
+                # Fail-fast: cancel pending futures on first failure
+                print(f"\n\033[31m[FAIL-FAST] Cancelling remaining tasks after failure...\033[0m")
+                for f in futures:
+                    f.cancel()
+                break
+            update_progress()
+
+    # Clear progress bar line
+    if is_tty:
+        sys.stdout.write("\r" + " " * 60 + "\r")
+        sys.stdout.flush()
+
+    # Print failures at the end
+    if failed_specs:
+        print("\n\033[31mFailed specs:\033[0m")
+        for spec in failed_specs:
+            print(f"  \033[31m•\033[0m {spec}")
+
+    print(f"\nBatch processing complete: {succeeded} succeeded, {failed} failed")
+
+    if failed > 0:
+        sys.exit(1)
+
+    # Run black formatting after all codegen completes. Running black separately
+    # avoids duplicating black's import/startup cost in each worker process.
+    if output_dirs:
+        from pygen.black import BlackScriptPlugin
+
+        print(f"Formatting {len(output_dirs)} packages with black...")
+        for d in output_dirs:
+            BlackScriptPlugin(output_folder=d).process()
+
+
+if __name__ == "__main__":
+    freeze_support()  # Required for Windows multiprocessing
+    main()

package/eng/scripts/setup/venvtools.py

@@ -8,7 +8,6 @@ import venv
 import sys
 from pathlib import Path
 
-
 # eng/scripts/setup/venvtools.py -> need to go up 4 levels to get to package root
 _ROOT_DIR = Path(__file__).parent.parent.parent.parent