@typespec/http-client-python 0.28.2 → 0.28.3

package/eng/scripts/ci/format.ts

@@ -1,3 +1,233 @@
-import { runCommand } from "./utils.js";
+/* eslint-disable no-console */
+import { spawn } from "child_process";
+import fs from "fs";
+import { dirname, join } from "path";
+import pc from "picocolors";
+import { fileURLToPath } from "url";
+import { parseArgs } from "util";
 
-runCommand("black", ["generator/", "eng/", "--config", "./eng/scripts/ci/pyproject.toml"]);
+const root = join(dirname(fileURLToPath(import.meta.url)), "../../../");
+const monorepoRoot = join(root, "../../");
+
+// Get Python venv path
+function getVenvPython(): string {
+  const venvPath = join(root, "venv");
+  if (fs.existsSync(join(venvPath, "bin"))) {
+    return join(venvPath, "bin", "python");
+  } else if (fs.existsSync(join(venvPath, "Scripts"))) {
+    return join(venvPath, "Scripts", "python.exe");
+  }
+  throw new Error("Virtual environment not found. Run 'npm run install' first.");
+}
+
+const argv = parseArgs({
+  args: process.argv.slice(2),
+  options: {
+    emitter: { type: "boolean", short: "e" },
+    generator: { type: "boolean", short: "g" },
+    generated: { type: "boolean" },
+    check: { type: "boolean", short: "c" },
+    help: { type: "boolean", short: "h" },
+  },
+});
+
+if (argv.values.help) {
+  console.log(`
+${pc.bold("Usage:")} tsx format.ts [options]
+
+${pc.bold("Description:")}
+  Format code using Prettier (TypeScript) and Black (Python).
+
+${pc.bold("Options:")}
+  ${pc.cyan("-e, --emitter")}
+      Format TypeScript emitter code with Prettier.
+
+  ${pc.cyan("-g, --generator")}
+      Format Python generator source code (pygen) with Black.
+
+  ${pc.cyan("--generated")}
+      Format generated SDK packages with Black.
+
+  ${pc.cyan("-c, --check")}
+      Check formatting without making changes (exit with error if unformatted).
+
+  ${pc.cyan("-h, --help")}
+      Show this help message.
+
+${pc.bold("Examples:")}
+  ${pc.dim("# Format emitter + pygen source (default)")}
+  tsx format.ts
+
+  ${pc.dim("# Format only TypeScript emitter")}
+  tsx format.ts --emitter
+
+  ${pc.dim("# Format only pygen source code")}
+  tsx format.ts --generator
+
+  ${pc.dim("# Check formatting without making changes")}
+  tsx format.ts --check
+
+  ${pc.dim("# Format generated SDK packages")}
+  tsx format.ts --generated
+`);
+  process.exit(0);
+}
+
+function runCommand(command: string, args: string[]): Promise<boolean> {
+  // Add node_modules/.bin directories to PATH so commands like prettier can be found
+  const pathSep = process.platform === "win32" ? ";" : ":";
+  const binPaths = [
+    join(root, "node_modules", ".bin"),
+    join(monorepoRoot, "node_modules", ".bin"),
+  ].join(pathSep);
+  const env = {
+    ...process.env,
+    PATH: `${binPaths}${pathSep}${process.env.PATH}`,
+  };
+
+  return new Promise((resolve) => {
+    console.log(`${pc.cyan("[RUN]")} ${command} ${args.join(" ")}`);
+    const proc = spawn(command, args, {
+      cwd: root,
+      stdio: "inherit",
+      shell: true,
+      env,
+    });
+
+    proc.on("close", (code) => {
+      if (code === 0) {
+        console.log(`${pc.green("[PASS]")} ${command} completed successfully`);
+        resolve(true);
+      } else {
+        console.log(`${pc.red("[FAIL]")} ${command} failed with code ${code}`);
+        resolve(false);
+      }
+    });
+
+    proc.on("error", (err) => {
+      console.log(`${pc.red("[ERROR]")} ${command}: ${err.message}`);
+      resolve(false);
+    });
+  });
+}
+
+async function formatEmitter(check: boolean): Promise<boolean> {
+  console.log(`\n${pc.bold("=== Formatting TypeScript Emitter ===")}\n`);
+  // Use prettier directly - it's available via node_modules/.bin in PATH
+  // Exclude CHANGELOG.md as it's managed by chronus changelog tool
+  const args = check
+    ? [
+        "--check",
+        "emitter/",
+        "eng/scripts/",
+        "*.json",
+        "README.md",
+        "CONTRIBUTING.md",
+        "ARCHITECTURE.md",
+      ]
+    : [
+        "--write",
+        "emitter/",
+        "eng/scripts/",
+        "*.json",
+        "README.md",
+        "CONTRIBUTING.md",
+        "ARCHITECTURE.md",
+      ];
+  return runCommand("prettier", args);
+}
+
+async function formatPygenSource(check: boolean): Promise<boolean> {
+  console.log(`\n${pc.bold("=== Formatting Python Generator (pygen) ===")}\n`);
+
+  let pythonPath: string;
+  try {
+    pythonPath = getVenvPython();
+  } catch (error) {
+    console.error(pc.red((error as Error).message));
+    return false;
+  }
+
+  const args = [
+    "-m",
+    "black",
+    "generator/pygen",
+    "eng/scripts/",
+    "--config",
+    "./eng/scripts/ci/config/pyproject.toml",
+  ];
+  if (check) {
+    args.push("--check");
+  }
+  return runCommand(pythonPath, args);
+}
+
+async function formatGeneratedPackages(check: boolean): Promise<boolean> {
+  console.log(`\n${pc.bold("=== Formatting Generated SDK Packages ===")}\n`);
+
+  let pythonPath: string;
+  try {
+    pythonPath = getVenvPython();
+  } catch (error) {
+    console.error(pc.red((error as Error).message));
+    return false;
+  }
+
+  const args = [
+    "-m",
+    "black",
+    "generator/test/azure/generated",
+    "generator/test/unbranded/generated",
+    "--config",
+    "./eng/scripts/ci/config/pyproject.toml",
+  ];
+  if (check) {
+    args.push("--check");
+  }
+  return runCommand(pythonPath, args);
+}
+
+async function main(): Promise<void> {
+  const runEmitter = argv.values.emitter;
+  const runGenerator = argv.values.generator;
+  const runGenerated = argv.values.generated;
+  const check = argv.values.check || false;
+
+  // If --generated is specified, only format generated packages
+  if (runGenerated) {
+    const result = await formatGeneratedPackages(check);
+    if (!result) process.exit(1);
+    console.log(
+      `\n${pc.green(pc.bold(`Generated SDK formatting ${check ? "check " : ""}complete!`))}\n`,
+    );
+    return;
+  }
+
+  // Default: format emitter + pygen source
+  const runBoth = !runEmitter && !runGenerator;
+
+  let success = true;
+
+  if (runEmitter || runBoth) {
+    const result = await formatEmitter(check);
+    if (!result) success = false;
+  }
+
+  if (runGenerator || runBoth) {
+    const result = await formatPygenSource(check);
+    if (!result) success = false;
+  }
+
+  if (!success) {
+    process.exit(1);
+  }
+
+  console.log(
+    `\n${pc.green(pc.bold(`All formatting ${check ? "checks passed" : "complete"}!`))}\n`,
+  );
+}
+
+main().catch((error) => {
+  console.error(`${pc.red("Unexpected error:")}`, error);
+  process.exit(1);
+});

package/eng/scripts/ci/lint.ts

@@ -1,36 +1,232 @@
+/* eslint-disable no-console */
+import { spawn } from "child_process";
+import fs from "fs";
+import { dirname, join } from "path";
+import pc from "picocolors";
+import { fileURLToPath } from "url";
 import { parseArgs } from "util";
-import { runCommand } from "./utils.js";
 
-// PARSE INPUT ARGUMENTS
+const root = join(dirname(fileURLToPath(import.meta.url)), "../../../");
+const monorepoRoot = join(root, "../../");
+const testsDir = join(root, "tests");
+
+// Get Python venv path
+function getVenvPython(): string {
+  const venvPath = join(root, "venv");
+  if (fs.existsSync(join(venvPath, "bin"))) {
+    return join(venvPath, "bin", "python");
+  } else if (fs.existsSync(join(venvPath, "Scripts"))) {
+    return join(venvPath, "Scripts", "python.exe");
+  }
+  throw new Error("Virtual environment not found. Run 'npm run install' first.");
+}
 
 const argv = parseArgs({
   args: process.argv.slice(2),
   options: {
-    folderName: { type: "string" },
-    command: { type: "string" },
+    emitter: { type: "boolean", short: "e" },
+    generator: { type: "boolean", short: "g" },
+    generated: { type: "boolean" },
+    flavor: { type: "string", short: "f" },
+    help: { type: "boolean", short: "h" },
   },
 });
 
-export function pylint() {
-  runCommand("pylint", [argv.values.folderName!, "--rcfile", "./eng/scripts/ci/pylintrc"]);
+if (argv.values.help) {
+  console.log(`
+${pc.bold("Usage:")} tsx lint.ts [options]
+
+${pc.bold("Description:")}
+  Run linting checks on the codebase.
+
+${pc.bold("Options:")}
+  ${pc.cyan("-e, --emitter")}
+      Run ESLint on TypeScript emitter code.
+
+  ${pc.cyan("-g, --generator")}
+      Run pylint on Python generator source code (pygen).
+
+  ${pc.cyan("--generated")}
+      Run pylint on generated SDK packages (via tox).
+      Use with --flavor to specify azure or unbranded.
+
+  ${pc.cyan("-f, --flavor <azure|unbranded>")}
+      SDK flavor to lint (only applies to --generated).
+      If not specified, lints both flavors.
+
+  ${pc.cyan("-h, --help")}
+      Show this help message.
+
+${pc.bold("Examples:")}
+  ${pc.dim("# Lint emitter + pygen source (default)")}
+  tsx lint.ts
+
+  ${pc.dim("# Lint only TypeScript emitter")}
+  tsx lint.ts --emitter
+
+  ${pc.dim("# Lint only pygen source code")}
+  tsx lint.ts --generator
+
+  ${pc.dim("# Lint generated SDK packages")}
+  tsx lint.ts --generated
+
+  ${pc.dim("# Lint generated SDK packages for azure only")}
+  tsx lint.ts --generated --flavor=azure
+`);
+  process.exit(0);
+}
+
+function runCommand(
+  command: string,
+  args: string[],
+  cwd?: string,
+  displayName?: string,
+): Promise<boolean> {
+  const workingDir = cwd || root;
+  const name = displayName || command;
+
+  // Add node_modules/.bin directories to PATH so commands like eslint can be found
+  // Also set NODE_PATH so that config files can resolve packages from monorepo's node_modules
+  const pathSep = process.platform === "win32" ? ";" : ":";
+  const binPaths = [
+    join(workingDir, "node_modules", ".bin"),
+    join(root, "node_modules", ".bin"),
+    join(monorepoRoot, "node_modules", ".bin"),
+  ].join(pathSep);
+  const nodePaths = [
+    join(workingDir, "node_modules"),
+    join(root, "node_modules"),
+    join(monorepoRoot, "node_modules"),
+  ].join(pathSep);
+  const env = {
+    ...process.env,
+    PATH: `${binPaths}${pathSep}${process.env.PATH}`,
+    NODE_PATH: nodePaths,
+  };
+
+  return new Promise((resolve) => {
+    // If displayName is provided, show it as-is; otherwise show command + args
+    const logMessage = displayName ? displayName : `${command} ${args.join(" ")}`;
+    console.log(`${pc.cyan("[RUN]")} ${logMessage}`);
+    const proc = spawn(command, args, {
+      cwd: workingDir,
+      stdio: "inherit",
+      shell: true,
+      env,
+    });
+
+    proc.on("close", (code) => {
+      if (code === 0) {
+        console.log(`${pc.green("[PASS]")} ${name} completed successfully`);
+        resolve(true);
+      } else {
+        console.log(`${pc.red("[FAIL]")} ${name} failed with code ${code}`);
+        resolve(false);
+      }
+    });
+
+    proc.on("error", (err) => {
+      console.log(`${pc.red("[ERROR]")} ${name}: ${err.message}`);
+      resolve(false);
+    });
+  });
 }
 
-export function mypy() {
-  runCommand("mypy", [argv.values.folderName!, "--config-file", "./eng/scripts/ci/mypy.ini"]);
+async function lintEmitter(): Promise<boolean> {
+  console.log(`\n${pc.bold("=== Linting TypeScript Emitter ===")}\n`);
+  // Run eslint with local config to avoid dependency on monorepo's eslint.config.js
+  // This ensures the package can be linted in CI without full monorepo dependencies
+  return runCommand(
+    "eslint",
+    ["emitter/", "--config", "eng/scripts/ci/config/eslint-ci.config.mjs", "--max-warnings=0"],
+    root,
+    "eslint emitter/ --max-warnings=0",
+  );
 }
 
-export function pyright() {
-  runCommand("pyright", [argv.values.folderName!, "-p", "./eng/scripts/ci/pyrightconfig.json"]);
+async function lintPygenSource(): Promise<boolean> {
+  console.log(`\n${pc.bold("=== Linting Python Generator (pygen) ===")}\n`);
+
+  let pythonPath: string;
+  try {
+    pythonPath = getVenvPython();
+  } catch (error) {
+    console.error(pc.red((error as Error).message));
+    return false;
+  }
+
+  // Lint pygen source code directly with pylint
+  return runCommand(pythonPath, [
+    "-m",
+    "pylint",
+    "generator/pygen",
+    "--rcfile=eng/scripts/ci/config/pylintrc",
+    "--recursive=y",
+    "--py-version=3.9",
+  ]);
 }
 
-if (argv.values.command === "pylint") {
-  pylint();
-} else if (argv.values.command === "mypy") {
-  mypy();
-} else if (argv.values.command === "pyright") {
-  pyright();
-} else {
-  pylint();
-  mypy();
-  pyright();
+async function lintGeneratedPackages(flavor?: string): Promise<boolean> {
+  const flavors = flavor ? [flavor] : ["azure", "unbranded"];
+  console.log(`\n${pc.bold(`=== Linting Generated SDK Packages (${flavors.join(", ")}) ===`)}\n`);
+
+  let pythonPath: string;
+  try {
+    pythonPath = getVenvPython();
+  } catch (error) {
+    console.error(pc.red((error as Error).message));
+    return false;
+  }
+
+  let success = true;
+  for (const f of flavors) {
+    const toxEnv = `lint-${f}`;
+    const result = await runCommand(
+      pythonPath,
+      ["-m", "tox", "-c", "tox.ini", "-e", toxEnv],
+      testsDir,
+    );
+    if (!result) success = false;
+  }
+  return success;
 }
+
+async function main(): Promise<void> {
+  const runEmitter = argv.values.emitter;
+  const runGenerator = argv.values.generator;
+  const runGenerated = argv.values.generated;
+
+  // If --generated is specified, only lint generated packages
+  if (runGenerated) {
+    const result = await lintGeneratedPackages(argv.values.flavor);
+    if (!result) process.exit(1);
+    console.log(`\n${pc.green(pc.bold("Generated SDK linting complete!"))}\n`);
+    return;
+  }
+
+  // Default: lint emitter + pygen source
+  const runBoth = !runEmitter && !runGenerator;
+
+  let success = true;
+
+  if (runEmitter || runBoth) {
+    const result = await lintEmitter();
+    if (!result) success = false;
+  }
+
+  if (runGenerator || runBoth) {
+    const result = await lintPygenSource();
+    if (!result) success = false;
+  }
+
+  if (!success) {
+    process.exit(1);
+  }
+
+  console.log(`\n${pc.green(pc.bold("All linting checks passed!"))}\n`);
+}
+
+main().catch((error) => {
+  console.error(`${pc.red("Unexpected error:")}`, error);
+  process.exit(1);
+});

package/eng/scripts/ci/regenerate-common.ts

@@ -282,6 +282,7 @@ export interface RegenerateFlagsInput {
   debug?: boolean;
   name?: string;
   pyodide?: boolean;
+  jobs?: number;
 }
 
 export interface RegenerateFlags {
@@ -403,8 +404,9 @@ export function buildOptions(
     }
     if (options["emitter-output-dir"] === undefined) {
       const packageName = options["package-name"] || defaultPackageName(spec, config);
+      // Output to new tests/generated/<flavor>/<package> structure
       options["emitter-output-dir"] = toPosix(
-        `${generatedFolder}/test/${flags.flavor}/generated/${packageName}`,
+        `${generatedFolder}/../tests/generated/${flags.flavor}/${packageName}`,
       );
     }
     if (flags.debug) {
@@ -423,21 +425,21 @@ export async function runTaskPool(
   tasks: Array<() => Promise<void>>,
   poolLimit: number,
 ): Promise<void> {
-  async function worker(start: number, end: number) {
-    while (start < end) {
-      await tasks[start]();
-      start++;
+  const executing: Set<Promise<void>> = new Set();
+
+  for (const task of tasks) {
+    // Start the task and remove from set when done
+    const p: Promise<void> = task().finally(() => executing.delete(p));
+    executing.add(p);
+
+    // If at capacity, wait for one to complete
+    if (executing.size >= poolLimit) {
+      await Promise.race(executing);
     }
   }
 
-  const workers = [];
-  let start = 0;
-  while (start < tasks.length) {
-    const end = Math.min(start + poolLimit, tasks.length);
-    workers.push((async () => await worker(start, end))());
-    start = end;
-  }
-  await Promise.all(workers);
+  // Wait for remaining tasks
+  await Promise.all(executing);
 }
 
 export async function regenerate(
@@ -467,6 +469,8 @@ export async function regenerate(
     });
 
     // Run tasks with a concurrency limit
-    await runTaskPool(tasks, 30);
+    // Default: 30 jobs, or use provided value
+    const poolLimit = flags.jobs ?? 30;
+    await runTaskPool(tasks, poolLimit);
   }
 }