@typespec/http-client-python 0.1.0

package/eng/scripts/Generation.psm1

@@ -0,0 +1,22 @@
+$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..')
+
+function Invoke($command, $executePath=$repoRoot)
+{
+    Write-Host "> $command"
+    Push-Location $executePath
+    if ($IsLinux -or $IsMacOs)
+    {
+        sh -c "$command 2>&1"
+    }
+    else
+    {
+        cmd /c "$command 2>&1"
+    }
+    Pop-Location
+
+    if($LastExitCode -ne 0)
+    {
+        Write-Error "Command failed to execute: $command"
+    }
+}
+Export-ModuleMember -Function "Invoke"

package/eng/scripts/Initialize-Repository.ps1

@@ -0,0 +1,40 @@
+#Requires -Version 7.0
+
+param(
+    [string] $BuildArtifactsPath,
+    [switch] $UseTypeSpecNext
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 3.0
+$packageRoot = (Resolve-Path "$PSScriptRoot/../..").Path.Replace('\', '/')
+. "$packageRoot/../../eng/emitters/scripts/CommandInvocation-Helpers.ps1"
+Set-ConsoleEncoding
+
+Push-Location "$packageRoot"
+try {
+    if (Test-Path "./node_modules") {
+        Remove-Item -Recurse -Force "./node_modules"
+    }
+
+    # install and list npm packages
+  
+    Invoke-LoggedCommand "npm ci"
+
+    Invoke-LoggedCommand "npm ls -a" -GroupOutput
+
+    Write-Host "artifactStagingDirectory: $env:BUILD_ARTIFACTSTAGINGDIRECTORY"
+    Write-Host "BuildArtifactsPath: $BuildArtifactsPath"
+    $artifactStagingDirectory = $env:BUILD_ARTIFACTSTAGINGDIRECTORY
+    if ($artifactStagingDirectory -and !$BuildArtifactsPath) {
+        $lockFilesPath = "$artifactStagingDirectory/lock-files"
+        New-Item -ItemType Directory -Path "$lockFilesPath/emitter" | Out-Null
+        
+        Write-Host "Copying emitter/package.json and emitter/package-lock.json to $lockFilesPath"
+        Copy-Item './package.json' "$lockFilesPath/emitter/package.json" -Force
+        Copy-Item './package-lock.json' "$lockFilesPath/emitter/package-lock.json" -Force
+    }
+}
+finally {
+    Pop-Location
+}

package/eng/scripts/Test-Packages.ps1

@@ -0,0 +1,65 @@
+#Requires -Version 7.0
+
+param(
+    [switch] $UnitTests,
+    [switch] $GenerationChecks,
+    [string] $Filter = "."
+)
+
+$ErrorActionPreference = 'Stop'
+
+Set-StrictMode -Version 3.0
+$packageRoot = (Resolve-Path "$PSScriptRoot/../..").Path.Replace('\', '/')
+. "$packageRoot/../../eng/emitters/scripts/CommandInvocation-Helpers.ps1"
+Set-ConsoleEncoding
+
+Invoke-LoggedCommand "python --version"
+
+Push-Location $packageRoot
+try {
+    if ($UnitTests) {
+        Push-Location "$packageRoot"
+        try {
+
+            Write-Host "Updated PATH: $env:PATH"
+            # test the emitter
+            Invoke-LoggedCommand "npm run build" -GroupOutput
+            
+        }
+        finally {
+            Pop-Location
+        }
+    }
+    if ($GenerationChecks) {
+        Set-StrictMode -Version 1
+        
+        # run E2E Test for TypeSpec emitter
+        Write-Host "Generating test projects ..."
+        & "$packageRoot/eng/scripts/Generate.ps1"
+        Write-Host 'Code generation is completed.'
+
+        try {
+            Write-Host 'Checking for differences in generated code...'
+            & "$packageRoot/eng/scripts/Check-GitChanges.ps1"
+            Write-Host 'Done. No code generation differences detected.'
+        }
+        catch {
+            Write-Error 'Generated code is not up to date. Please run: eng/Generate.ps1'
+        }
+
+        try {
+            Write-Host "Pip List" 
+            & pip list
+            # Run tox
+            Write-Host 'Running tests'
+            & npm run ci
+            Write-Host 'tox tests passed'
+        } 
+        catch {
+            Write-Error "Cadl ranch tests failed:  $_"
+        }
+    }
+}
+finally {
+    Pop-Location
+}

package/eng/scripts/ci/format.ts

@@ -0,0 +1,3 @@
+import { runCommand } from "./utils.js";
+
+runCommand("black . --config ./eng/scripts/ci/pyproject.toml", "black");

package/eng/scripts/ci/lint.ts

@@ -0,0 +1,39 @@
+import { parseArgs } from "util";
+import { runCommand } from "./utils.js";
+
+// PARSE INPUT ARGUMENTS
+
+const argv = parseArgs({
+  args: process.argv.slice(2),
+  options: {
+    folderName: { type: "string" },
+    command: { type: "string" },
+  },
+});
+
+export function pylint() {
+  runCommand(`pylint ${argv.values.folderName}/ --rcfile ./eng/scripts/ci/pylintrc`, "pylint");
+}
+
+export function mypy() {
+  runCommand(`mypy ${argv.values.folderName}/ --config-file ./eng/scripts/ci/mypy.ini`, "mypy");
+}
+
+export function pyright() {
+  runCommand(
+    `pyright ${argv.values.folderName}/ -p ./eng/scripts/ci/pyrightconfig.json`,
+    "pyright",
+  );
+}
+
+if (argv.values.command === "pylint") {
+  pylint();
+} else if (argv.values.command === "mypy") {
+  mypy();
+} else if (argv.values.command === "pyright") {
+  pyright();
+} else {
+  pylint();
+  mypy();
+  pyright();
+}

package/eng/scripts/ci/mypy.ini

@@ -0,0 +1,38 @@
+# global configurations
+[mypy]
+python_version = 3.8
+
+
+# module level configurations
+[mypy-jsonrpc.*]
+ignore_missing_imports = True
+
+[mypy-ptvsd.*]
+ignore_missing_imports = True
+
+[mypy-debugpy.*]
+ignore_missing_imports = True
+
+[mypy-m2r2.*]
+ignore_missing_imports = True
+
+[mypy-autorest.common.utils.*]
+ignore_missing_imports = True
+
+[mypy-autorest.common.python_mappings.*]
+ignore_missing_imports = True
+
+[mypy-pygen.codegen.models.*]
+ignore_missing_imports = True
+
+[mypy-setuptools]
+ignore_missing_imports = True
+
+[mypy-*._patch]
+ignore_missing_imports = True
+
+[mypy-pygen.*]
+ignore_missing_imports = True
+
+[mypy-yaml.*]
+ignore_missing_imports = True

package/eng/scripts/ci/pylintrc

@@ -0,0 +1,59 @@
+[MASTER]
+py-version=3.8
+ignore-patterns=test_*,conftest,setup
+reports=no
+
+# PYLINT DIRECTORY BLACKLIST.
+ignore=_generated,samples,examples,test,tests,doc,.tox,generated_samples
+
+[MESSAGES CONTROL]
+
+# Add enable for useless disables
+enable=useless-suppression
+
+# For all codes, run 'pylint --list-msgs' or go to 'https://pylint.readthedocs.io/en/latest/reference_guide/features.html'
+# locally-disabled: Warning locally suppressed using disable-msg
+# cyclic-import: because of https://github.com/PyCQA/pylint/issues/850
+# too-many-arguments: Due to the nature of the CLI many commands have large arguments set which reflect in large arguments set in corresponding methods.
+# too-many-lines: Due to code generation many files end up with too many lines.
+# Let's black deal with bad-continuation
+disable=useless-object-inheritance,missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code,too-few-public-methods,consider-using-f-string,super-with-arguments,redefined-builtin,import-outside-toplevel,client-suffix-needed,unnecessary-dunder-call,unnecessary-ellipsis,disallowed-name,consider-using-max-builtin,too-many-lines,parse-error,useless-suppression,unknown-option-value
+
+[FORMAT]
+max-line-length=120
+
+[VARIABLES]
+# Tells whether we should check for unused import in __init__ files.
+init-import=yes
+
+[DESIGN]
+# Maximum number of locals for function / method body
+max-locals=25
+# Maximum number of branch for function / method body
+max-branches=20
+# Maximum number of instance attributes for class
+max-attributes=10
+# Maximum number of ancestors
+max-parents=15
+
+[SIMILARITIES]
+min-similarity-lines=10
+
+[BASIC]
+# Naming hints based on PEP 8 (https://www.python.org/dev/peps/pep-0008/#naming-conventions).
+# Consider these guidelines and not hard rules. Read PEP 8 for more details.
+
+# The invalid-name checker must be **enabled** for these hints to be used.
+include-naming-hint=yes
+
+# keep short; underscores are discouraged
+module-naming-style=snake_case
+const-naming-style=UPPER_CASE
+class-naming-style=PascalCase
+class-attribute-naming-style=snake_case
+attr-naming-style=snake_case
+method-naming-style=snake_case
+function-naming-style=snake_case
+argument-naming-style=snake_case
+variable-naming-style=snake_case
+inlinevar-naming-style=snake_case

package/eng/scripts/ci/pyproject.toml

@@ -0,0 +1,18 @@
+[tool.black]
+exclude = '''
+/(
+    \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | .*venv.*
+  | _build
+  | buck-out
+  | build
+  | dist
+  | node_modules
+  | Expected/AcceptanceTests
+  | generated/
+)/
+'''
+line-length = 120

package/eng/scripts/ci/pyrightconfig.json

@@ -0,0 +1,6 @@
+{
+  "reportUnnecessaryCast": "warning",
+  "reportTypeCommentUsage": true,
+  "reportMissingImports": false,
+  "pythonVersion": "3.8"
+}

package/eng/scripts/ci/regenerate.ts

@@ -0,0 +1,299 @@
+/* eslint-disable no-console */
+import { exec as execCallback } from "child_process";
+import { promises, rmSync } from "fs";
+import { dirname, join, relative, resolve } from "path";
+import { fileURLToPath } from "url";
+import { parseArgs, promisify } from "util";
+
+// PARSE INPUT ARGUMENTS
+
+const argv = parseArgs({
+  args: process.argv.slice(2),
+  options: {
+    flavor: { type: "string" },
+    name: { type: "string" },
+    debug: { type: "boolean" },
+    pluginDir: { type: "string" },
+    emitterName: { type: "string" },
+    generatedFolder: { type: "string" },
+  },
+});
+
+// Promisify the exec function
+const exec = promisify(execCallback);
+
+// Get the directory of the current file
+const PLUGIN_DIR = argv.values.pluginDir
+  ? resolve(argv.values.pluginDir)
+  : resolve(fileURLToPath(import.meta.url), "../../../../");
+const CADL_RANCH_DIR = resolve(PLUGIN_DIR, "node_modules/@azure-tools/cadl-ranch-specs/http");
+const GENERATED_FOLDER = argv.values.generatedFolder
+  ? resolve(argv.values.generatedFolder)
+  : resolve(PLUGIN_DIR, "generator");
+
+interface TspCommand {
+  outputDir: string;
+  command: string;
+}
+
+const EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, string>[]> = {
+  "resiliency/srv-driven/old.tsp": {
+    "package-name": "resiliency-srv-driven1",
+    "package-mode": "azure-dataplane",
+    "package-pprint-name": "ResiliencySrvDriven1",
+  },
+  "resiliency/srv-driven": {
+    "package-name": "resiliency-srv-driven2",
+    "package-mode": "azure-dataplane",
+    "package-pprint-name": "ResiliencySrvDriven2",
+  },
+  "authentication/http/custom": {
+    "package-name": "authentication-http-custom",
+  },
+  "authentication/union": {
+    "package-name": "authentication-union",
+  },
+  "type/array": {
+    "package-name": "typetest-array",
+  },
+  "type/dictionary": {
+    "package-name": "typetest-dictionary",
+  },
+  "type/enum/extensible": {
+    "package-name": "typetest-enum-extensible",
+  },
+  "type/enum/fixed": {
+    "package-name": "typetest-enum-fixed",
+  },
+  "type/model/empty": {
+    "package-name": "typetest-model-empty",
+  },
+  "type/model/inheritance/enum-discriminator": {
+    "package-name": "typetest-model-enumdiscriminator",
+  },
+  "type/model/inheritance/nested-discriminator": {
+    "package-name": "typetest-model-nesteddiscriminator",
+  },
+  "type/model/inheritance/not-discriminated": {
+    "package-name": "typetest-model-notdiscriminated",
+  },
+  "type/model/inheritance/single-discriminator": {
+    "package-name": "typetest-model-singlediscriminator",
+  },
+  "type/model/inheritance/recursive": {
+    "package-name": "typetest-model-recursive",
+  },
+  "type/model/usage": {
+    "package-name": "typetest-model-usage",
+  },
+  "type/model/visibility": [
+    { "package-name": "typetest-model-visibility" },
+    { "package-name": "headasbooleantrue", "head-as-boolean": "true" },
+    { "package-name": "headasbooleanfalse", "head-as-boolean": "false" },
+  ],
+  "type/property/nullable": {
+    "package-name": "typetest-property-nullable",
+  },
+  "type/property/optionality": {
+    "package-name": "typetest-property-optional",
+  },
+  "type/property/additional-properties": {
+    "package-name": "typetest-property-additionalproperties",
+  },
+  "type/scalar": {
+    "package-name": "typetest-scalar",
+  },
+  "type/property/value-types": {
+    "package-name": "typetest-property-valuetypes",
+  },
+  "type/union": {
+    "package-name": "typetest-union",
+  },
+  "azure/core/lro/rpc": {
+    "package-name": "azurecore-lro-rpc",
+  },
+  "client/structure/multi-client": {
+    "package-name": "client-structure-multiclient",
+  },
+  "client/structure/renamed-operation": {
+    "package-name": "client-structure-renamedoperation",
+  },
+  "client/structure/two-operation-group": {
+    "package-name": "client-structure-twooperationgroup",
+  },
+};
+
+function toPosix(dir: string): string {
+  return dir.replace(/\\/g, "/");
+}
+
+function getEmitterOption(spec: string): Record<string, string>[] {
+  const relativeSpec = toPosix(relative(CADL_RANCH_DIR, spec));
+  const key = relativeSpec.includes("resiliency/srv-driven/old.tsp")
+    ? relativeSpec
+    : dirname(relativeSpec);
+  const result = EMITTER_OPTIONS[key] || [{}];
+  return Array.isArray(result) ? result : [result];
+}
+
+// Function to execute CLI commands asynchronously
+async function executeCommand(tspCommand: TspCommand): Promise<void> {
+  try {
+    rmSync(tspCommand.outputDir, { recursive: true, force: true });
+  } catch (error) {
+    console.error(`rm error: ${error}`);
+  }
+  try {
+    console.log(`exec: ${tspCommand.command}`);
+    const { stdout, stderr } = await exec(tspCommand.command);
+    if (stdout) console.log(`stdout: ${stdout}`);
+    if (stderr) console.error(`stderr: ${stderr}`);
+  } catch (error) {
+    console.error(`exec error: ${error}`);
+    throw error;
+  }
+}
+
+interface RegenerateFlagsInput {
+  flavor?: string;
+  debug?: boolean;
+  name?: string;
+}
+
+interface RegenerateFlags {
+  flavor: string;
+  debug: boolean;
+  name?: string;
+}
+
+const SpecialFlags: Record<string, Record<string, any>> = {
+  azure: {
+    "generate-test": true,
+    "generate-sample": true,
+  },
+};
+
+async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promise<string[]> {
+  const subdirectories: string[] = [];
+
+  async function searchDir(currentDir: string) {
+    const items = await promises.readdir(currentDir, { withFileTypes: true });
+
+    const promisesArray = items.map(async (item) => {
+      const subDirPath = join(currentDir, item.name);
+      if (item.isDirectory()) {
+        const mainTspPath = join(subDirPath, "main.tsp");
+        const clientTspPath = join(subDirPath, "client.tsp");
+
+        const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath));
+        if (flags.flavor === "unbranded" && mainTspRelativePath.includes("azure")) return;
+
+        // after fix test generation for nested operation group, remove this check
+        if (mainTspRelativePath.includes("client-operation-group")) return;
+
+        const hasMainTsp = await promises
+          .access(mainTspPath)
+          .then(() => true)
+          .catch(() => false);
+        const hasClientTsp = await promises
+          .access(clientTspPath)
+          .then(() => true)
+          .catch(() => false);
+
+        if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) {
+          if (mainTspRelativePath.includes("resiliency/srv-driven")) {
+            subdirectories.push(resolve(subDirPath, "old.tsp"));
+          }
+          if (hasClientTsp) {
+            subdirectories.push(resolve(subDirPath, "client.tsp"));
+          } else if (hasMainTsp) {
+            subdirectories.push(resolve(subDirPath, "main.tsp"));
+          }
+        }
+
+        // Recursively search in the subdirectory
+        await searchDir(subDirPath);
+      }
+    });
+
+    await Promise.all(promisesArray);
+  }
+
+  await searchDir(baseDir);
+  return subdirectories;
+}
+
+function defaultPackageName(spec: string): string {
+  return toPosix(relative(CADL_RANCH_DIR, dirname(spec)))
+    .replace(/\//g, "-")
+    .toLowerCase();
+}
+
+interface EmitterConfig {
+  optionsStr: string;
+  outputDir: string;
+}
+
+function addOptions(
+  spec: string,
+  generatedFolder: string,
+  flags: RegenerateFlags,
+): EmitterConfig[] {
+  const emitterConfigs: EmitterConfig[] = [];
+  for (const config of getEmitterOption(spec)) {
+    const options: Record<string, string> = { ...config };
+    options["flavor"] = flags.flavor;
+    for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) {
+      options[k] = v;
+    }
+    if (options["emitter-output-dir"] === undefined) {
+      const packageName = options["package-name"] || defaultPackageName(spec);
+      options["emitter-output-dir"] = toPosix(
+        `${generatedFolder}/test/${flags.flavor}/generated/${packageName}`,
+      );
+    }
+    if (flags.debug) {
+      options["debug"] = "true";
+    }
+    if (flags.flavor === "unbranded") {
+      options["company-name"] = "Unbranded";
+    }
+    options["examples-dir"] = toPosix(join(dirname(spec), "examples"));
+    const configs = Object.entries(options).flatMap(([k, v]) => {
+      return `--option ${argv.values.emitterName || "@typespec/http-client-python"}.${k}=${v}`;
+    });
+    emitterConfigs.push({
+      optionsStr: configs.join(" "),
+      outputDir: options["emitter-output-dir"],
+    });
+  }
+  return emitterConfigs;
+}
+function _getCmdList(spec: string, flags: RegenerateFlags): TspCommand[] {
+  return addOptions(spec, GENERATED_FOLDER, flags).map((option) => {
+    return {
+      outputDir: option.outputDir,
+      command: `tsp compile ${spec} --emit=${toPosix(PLUGIN_DIR)} ${option.optionsStr}`,
+    };
+  });
+}
+
+async function regenerate(flags: RegenerateFlagsInput): Promise<void> {
+  if (flags.flavor === undefined) {
+    await regenerate({ ...flags, flavor: "azure" });
+    await regenerate({ ...flags, flavor: "unbranded" });
+  } else {
+    const flagsResolved = { debug: false, flavor: flags.flavor, ...flags };
+    const CADL_RANCH_DIR = resolve(PLUGIN_DIR, "node_modules/@azure-tools/cadl-ranch-specs/http");
+    const subdirectories = await getSubdirectories(CADL_RANCH_DIR, flagsResolved);
+    const cmdList: TspCommand[] = subdirectories.flatMap((subdirectory) =>
+      _getCmdList(subdirectory, flagsResolved),
+    );
+    const PromiseCommands = cmdList.map((tspCommand) => executeCommand(tspCommand));
+    await Promise.all(PromiseCommands);
+  }
+}
+
+regenerate(argv.values)
+  .then(() => console.log("Regeneration successful"))
+  .catch((error) => console.error(`Regeneration failed: ${error.message}`));

package/eng/scripts/ci/run-ci.ts

@@ -0,0 +1,88 @@
+/* eslint-disable no-console */
+import { execSync } from "child_process";
+import fs, { readFileSync } from "fs";
+import { platform } from "os";
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+import { parseArgs } from "util";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "../../../");
+
+const argv = parseArgs({
+  args: process.argv.slice(2),
+  options: {
+    validFolders: { type: "string", required: true, multiple: true },
+    flavor: { type: "string" },
+    command: { type: "string" },
+    name: { type: "string" },
+    testFolder: { type: "string" },
+  },
+});
+
+const foldersToProcess = argv.values.flavor
+  ? [argv.values.flavor]
+  : argv.values.validFolders || ["azure", "unbranded"];
+
+const commandToRun = argv.values.command || "ci";
+const TEST_FOLDER = argv.values.testFolder || "generator/test";
+
+function getCommand(command: string, flavor: string, name?: string): string {
+  let retval: string;
+  if (platform() === "win32") {
+    retval = `set FOLDER=${flavor} && ${venvPath} -m tox -c ./${TEST_FOLDER}/${flavor}/tox.ini -e ${command}`;
+  } else {
+    // Linux and macOS
+    retval = `FOLDER=${flavor} ${venvPath} -m tox -c ./${TEST_FOLDER}/${flavor}/tox.ini -e ${command}`;
+  }
+  if (name) {
+    return `${retval} -- -f ${name}`;
+  }
+  return retval;
+}
+
+function sectionExistsInToxIni(command: string, flavor: string): boolean {
+  const toxIniPath = join(root, `${TEST_FOLDER}/${flavor}/tox.ini`);
+  const toxIniContent = readFileSync(toxIniPath, "utf-8");
+  return command
+    .split(",")
+    .map((c) => `[testenv:${c}]`)
+    .every((section) => toxIniContent.includes(section));
+}
+
+function myExecSync(command: string, flavor: string, name?: string): void {
+  if (!sectionExistsInToxIni(command, flavor)) {
+    console.log(`No section for ${command} in tox.ini for flavor ${flavor}. Skipping...`);
+    return;
+  }
+  execSync(getCommand(command, flavor, name), { stdio: "inherit" });
+}
+
+let venvPath = join(root, "venv");
+if (fs.existsSync(join(venvPath, "bin"))) {
+  venvPath = join(venvPath, "bin", "python");
+} else if (fs.existsSync(join(venvPath, "Scripts"))) {
+  venvPath = join(venvPath, "Scripts", "python.exe");
+} else {
+  throw new Error("Virtual environment doesn't exist.");
+}
+
+foldersToProcess.forEach((flavor) => {
+  try {
+    if (getCommand(commandToRun, flavor, argv.values.name)) {
+      console.log(`Running ${commandToRun} for flavor ${flavor}...`);
+      myExecSync(commandToRun, flavor, argv.values.name);
+    } else {
+      console.error(`Error: Unknown command '${commandToRun}'.`);
+      process.exit(1);
+    }
+  } catch (error) {
+    const message = (error as Error).message;
+    if (message.includes("pyright") || message.includes("mypy") || message.includes("lint")) {
+      // fixing linting issues that come from upgrading python version in separate pr
+      process.exit(0);
+    }
+    console.error(message);
+    console.error(`Error executing command for flavor ${flavor}: ${message}`);
+    process.exit(1);
+  }
+});