@typespec/http-client-python 0.27.0 → 0.27.2

package/eng/scripts/ci/regenerate.ts

@@ -2,9 +2,20 @@
 import chalk from "chalk";
 import { execFile } from "child_process";
 import { promises, rmSync } from "fs";
-import { dirname, join, relative, resolve } from "path";
+import { join, resolve } from "path";
 import { fileURLToPath } from "url";
 import { parseArgs, promisify } from "util";
+import {
+  BASE_AZURE_EMITTER_OPTIONS,
+  BASE_EMITTER_OPTIONS,
+  buildOptions,
+  regenerate,
+  toPosix,
+  type RegenerateConfig,
+  type RegenerateFlags,
+  type RegenerateFlagsInput,
+  type TspCommand,
+} from "./regenerate-common.js";
 
 // PARSE INPUT ARGUMENTS
 
@@ -21,9 +32,6 @@ const argv = parseArgs({
   },
 });
 
-// Add this near the top with other constants
-const SKIP_SPECS: string[] = [];
-
 // Get the directory of the current file
 const PLUGIN_DIR = argv.values.pluginDir
   ? resolve(argv.values.pluginDir)
@@ -33,280 +41,38 @@ const HTTP_SPECS = resolve(PLUGIN_DIR, "node_modules/@typespec/http-specs/specs"
 const GENERATED_FOLDER = argv.values.generatedFolder
   ? resolve(argv.values.generatedFolder)
   : resolve(PLUGIN_DIR, "generator");
-
-interface TspCommand {
-  outputDir: string;
-  command: string[];
-}
+const EMITTER_NAME = argv.values.emitterName || "@typespec/http-client-python";
 
 const AZURE_EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, string>[]> = {
-  "azure/client-generator-core/access": {
-    namespace: "specs.azure.clientgenerator.core.access",
-  },
-  "azure/client-generator-core/alternate-type": {
-    namespace: "specs.azure.clientgenerator.core.alternatetype",
-  },
-  "azure/client-generator-core/api-version": {
-    namespace: "specs.azure.clientgenerator.core.apiversion",
-  },
-  "azure/client-generator-core/client-initialization": {
-    namespace: "specs.azure.clientgenerator.core.clientinitialization",
-  },
-  "azure/client-generator-core/client-location": {
-    namespace: "specs.azure.clientgenerator.core.clientlocation",
-  },
-  "azure/client-generator-core/deserialize-empty-string-as-null": {
-    namespace: "specs.azure.clientgenerator.core.emptystring",
-  },
-  "azure/client-generator-core/flatten-property": {
-    namespace: "specs.azure.clientgenerator.core.flattenproperty",
-  },
-  "azure/client-generator-core/usage": {
-    namespace: "specs.azure.clientgenerator.core.usage",
-  },
-  "azure/client-generator-core/override": {
-    namespace: "specs.azure.clientgenerator.core.override",
-  },
-  "azure/client-generator-core/hierarchy-building": {
-    namespace: "specs.azure.clientgenerator.core.hierarchybuilding",
-  },
-  "azure/core/basic": {
-    namespace: "specs.azure.core.basic",
-  },
-  "azure/core/lro/rpc": {
-    namespace: "specs.azure.core.lro.rpc",
-  },
-  "azure/core/lro/standard": {
-    namespace: "specs.azure.core.lro.standard",
-  },
-  "azure/core/model": {
-    namespace: "specs.azure.core.model",
-  },
-  "azure/core/page": {
-    namespace: "specs.azure.core.page",
-  },
-  "azure/core/scalar": {
-    namespace: "specs.azure.core.scalar",
-  },
-  "azure/core/traits": {
-    namespace: "specs.azure.core.traits",
-  },
-  "azure/encode/duration": {
-    namespace: "specs.azure.encode.duration",
-  },
-  "azure/example/basic": {
-    namespace: "specs.azure.example.basic",
-  },
-  "azure/payload/pageable": {
-    namespace: "specs.azure.payload.pageable",
-  },
-  "azure/versioning/previewVersion": {
-    namespace: "specs.azure.versioning.previewversion",
-  },
-  "client/structure/default": {
-    namespace: "client.structure.service",
-  },
+  ...BASE_AZURE_EMITTER_OPTIONS,
   "client/structure/client-operation-group": {
     "package-name": "client-structure-clientoperationgroup",
     namespace: "client.structure.clientoperationgroup",
   },
-  "client/structure/multi-client": {
-    "package-name": "client-structure-multiclient",
-    namespace: "client.structure.multiclient",
-  },
-  "client/structure/renamed-operation": {
-    "package-name": "client-structure-renamedoperation",
-    namespace: "client.structure.renamedoperation",
-  },
-  "client/structure/two-operation-group": {
-    "package-name": "client-structure-twooperationgroup",
-    namespace: "client.structure.twooperationgroup",
-  },
-  "client/naming": {
-    namespace: "client.naming.main",
-  },
-  "client/overload": {
-    namespace: "client.overload",
-  },
-  "encode/duration": {
-    namespace: "encode.duration",
-  },
-  "encode/numeric": {
-    namespace: "encode.numeric",
-  },
-  "parameters/basic": {
-    namespace: "parameters.basic",
-  },
-  "parameters/spread": {
-    namespace: "parameters.spread",
-  },
-  "payload/content-negotiation": {
-    namespace: "payload.contentnegotiation",
-  },
-  "payload/multipart": {
-    namespace: "payload.multipart",
-  },
-  "serialization/encoded-name/json": {
-    namespace: "serialization.encodedname.json",
-  },
-  "special-words": {
-    namespace: "specialwords",
-  },
-  "service/multi-service": {
-    namespace: "service.multiservice",
-  },
 };
 
 const EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, string>[]> = {
-  "resiliency/srv-driven/old.tsp": {
-    "package-name": "resiliency-srv-driven1",
-    namespace: "resiliency.srv.driven1",
-    "package-mode": "azure-dataplane",
-    "package-pprint-name": "ResiliencySrvDriven1",
-  },
-  "resiliency/srv-driven": {
-    "package-name": "resiliency-srv-driven2",
-    namespace: "resiliency.srv.driven2",
-    "package-mode": "azure-dataplane",
-    "package-pprint-name": "ResiliencySrvDriven2",
-  },
-  "authentication/api-key": {
-    "clear-output-folder": "true",
-  },
-  "authentication/http/custom": {
-    "package-name": "authentication-http-custom",
-    namespace: "authentication.http.custom",
-    "package-pprint-name": "Authentication Http Custom",
-  },
-  "authentication/union": [
-    {
-      "package-name": "authentication-union",
-      namespace: "authentication.union",
-    },
-    {
-      "package-name": "setuppy-authentication-union",
-      namespace: "setuppy.authentication.union",
-      "keep-setup-py": "true",
-    },
-  ],
+  ...BASE_EMITTER_OPTIONS,
   "type/array": {
     "package-name": "typetest-array",
     namespace: "typetest.array",
     "use-pyodide": "true",
   },
-  "type/dictionary": {
-    "package-name": "typetest-dictionary",
-    namespace: "typetest.dictionary",
-  },
-  "type/enum/extensible": {
-    "package-name": "typetest-enum-extensible",
-    namespace: "typetest.enum.extensible",
-  },
-  "type/enum/fixed": {
-    "package-name": "typetest-enum-fixed",
-    namespace: "typetest.enum.fixed",
-  },
-  "type/model/empty": {
-    "package-name": "typetest-model-empty",
-    namespace: "typetest.model.empty",
-  },
-  "type/model/inheritance/enum-discriminator": {
-    "package-name": "typetest-model-enumdiscriminator",
-    namespace: "typetest.model.enumdiscriminator",
-  },
-  "type/model/inheritance/nested-discriminator": {
-    "package-name": "typetest-model-nesteddiscriminator",
-    namespace: "typetest.model.nesteddiscriminator",
-  },
-  "type/model/inheritance/not-discriminated": {
-    "package-name": "typetest-model-notdiscriminated",
-    namespace: "typetest.model.notdiscriminated",
-  },
-  "type/model/inheritance/single-discriminator": {
-    "package-name": "typetest-model-singlediscriminator",
-    namespace: "typetest.model.singlediscriminator",
-  },
   "type/model/inheritance/recursive": {
     "package-name": "typetest-model-recursive",
     namespace: "typetest.model.recursive",
     "use-pyodide": "true",
   },
-  "type/model/usage": {
-    "package-name": "typetest-model-usage",
-    namespace: "typetest.model.usage",
-  },
-  "type/model/visibility": [
-    {
-      "package-name": "typetest-model-visibility",
-      namespace: "typetest.model.visibility",
-    },
-    {
-      "package-name": "headasbooleantrue",
-      namespace: "headasbooleantrue",
-      "head-as-boolean": "true",
-    },
-    {
-      "package-name": "headasbooleanfalse",
-      namespace: "headasbooleanfalse",
-      "head-as-boolean": "false",
-    },
-  ],
-  "type/property/nullable": {
-    "package-name": "typetest-property-nullable",
-    namespace: "typetest.property.nullable",
-  },
-  "type/property/optionality": {
-    "package-name": "typetest-property-optional",
-    namespace: "typetest.property.optional",
-  },
-  "type/property/additional-properties": {
-    "package-name": "typetest-property-additionalproperties",
-    namespace: "typetest.property.additionalproperties",
-  },
-  "type/scalar": {
-    "package-name": "typetest-scalar",
-    namespace: "typetest.scalar",
-  },
-  "type/property/value-types": {
-    "package-name": "typetest-property-valuetypes",
-    namespace: "typetest.property.valuetypes",
-  },
-  "type/union": {
-    "package-name": "typetest-union",
-    namespace: "typetest.union",
-  },
-  "type/union/discriminated": {
-    "package-name": "typetest-discriminatedunion",
-    namespace: "typetest.discriminatedunion",
-  },
-  documentation: {
-    "package-name": "specs-documentation",
-    namespace: "specs.documentation",
-  },
 };
 
-function toPosix(dir: string): string {
-  return dir.replace(/\\/g, "/");
-}
-
-function getEmitterOption(spec: string, flavor: string): Record<string, string>[] {
-  const specDir = spec.includes("azure") ? AZURE_HTTP_SPECS : HTTP_SPECS;
-  const relativeSpec = toPosix(relative(specDir, spec));
-  const key = relativeSpec.includes("resiliency/srv-driven/old.tsp")
-    ? relativeSpec
-    : dirname(relativeSpec);
-  const emitter_options = EMITTER_OPTIONS[key] ||
-    (flavor === "azure" ? AZURE_EMITTER_OPTIONS[key] : [{}]) || [{}];
-  return Array.isArray(emitter_options) ? emitter_options : [emitter_options];
-}
-
 // Function to execute CLI commands asynchronously
 async function executeCommand(tspCommand: TspCommand): Promise<void> {
+  const cmd = tspCommand.command as string[];
   const execFileAsync = promisify(execFile);
   try {
-    console.log(chalk.green(`start tsp ${tspCommand.command.join(" ")}`));
-    await execFileAsync("tsp", tspCommand.command, { shell: true });
-    console.log(chalk.green(`tsp ${tspCommand.command.join(" ")} succeeded`));
+    console.log(chalk.green(`start tsp ${cmd.join(" ")}`));
+    await execFileAsync("tsp", cmd, { shell: true });
+    console.log(chalk.green(`tsp ${cmd.join(" ")} succeeded`));
   } catch (err) {
     rmSync(tspCommand.outputDir, { recursive: true, force: true });
     console.error(chalk.red(`exec error: ${err}`));
@@ -314,153 +80,6 @@ async function executeCommand(tspCommand: TspCommand): Promise<void> {
   }
 }
 
-interface RegenerateFlagsInput {
-  flavor?: string;
-  debug?: boolean;
-  name?: string;
-  pyodide?: boolean;
-}
-
-interface RegenerateFlags {
-  flavor: string;
-  debug: boolean;
-  name?: string;
-  pyodide?: boolean;
-}
-
-const SpecialFlags: Record<string, Record<string, any>> = {
-  azure: {
-    "generate-test": true,
-    "generate-sample": true,
-  },
-};
-
-async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promise<string[]> {
-  const subdirectories: string[] = [];
-
-  async function searchDir(currentDir: string) {
-    const items = await promises.readdir(currentDir, { withFileTypes: true });
-
-    const promisesArray = items.map(async (item) => {
-      const subDirPath = join(currentDir, item.name);
-      if (item.isDirectory()) {
-        const mainTspPath = join(subDirPath, "main.tsp");
-        const clientTspPath = join(subDirPath, "client.tsp");
-
-        const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath));
-
-        // Replace the individual skip checks with:
-        if (SKIP_SPECS.some((skipSpec) => mainTspRelativePath.includes(skipSpec))) return;
-
-        const hasMainTsp = await promises
-          .access(mainTspPath)
-          .then(() => true)
-          .catch(() => false);
-        const hasClientTsp = await promises
-          .access(clientTspPath)
-          .then(() => true)
-          .catch(() => false);
-
-        if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) {
-          if (mainTspRelativePath.includes("resiliency/srv-driven")) {
-            subdirectories.push(resolve(subDirPath, "old.tsp"));
-          }
-          if (hasClientTsp) {
-            subdirectories.push(resolve(subDirPath, "client.tsp"));
-          } else if (hasMainTsp) {
-            subdirectories.push(resolve(subDirPath, "main.tsp"));
-          }
-        }
-
-        // Recursively search in the subdirectory
-        await searchDir(subDirPath);
-      }
-    });
-
-    await Promise.all(promisesArray);
-  }
-
-  await searchDir(baseDir);
-  return subdirectories;
-}
-
-function defaultPackageName(spec: string): string {
-  const specDir = spec.includes("azure") ? AZURE_HTTP_SPECS : HTTP_SPECS;
-  return toPosix(relative(specDir, dirname(spec)))
-    .replace(/\//g, "-")
-    .toLowerCase();
-}
-
-interface EmitterConfig {
-  options: string[];
-  outputDir: string;
-}
-
-function addOptions(
-  spec: string,
-  generatedFolder: string,
-  flags: RegenerateFlags,
-): EmitterConfig[] {
-  const emitterConfigs: EmitterConfig[] = [];
-  for (const config of getEmitterOption(spec, flags.flavor)) {
-    const options: Record<string, string> = { ...config };
-    if (flags.pyodide) {
-      options["use-pyodide"] = "true";
-    }
-    options["flavor"] = flags.flavor;
-    for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) {
-      options[k] = v;
-    }
-    if (options["emitter-output-dir"] === undefined) {
-      const packageName = options["package-name"] || defaultPackageName(spec);
-      options["emitter-output-dir"] = toPosix(
-        `${generatedFolder}/test/${flags.flavor}/generated/${packageName}`,
-      );
-    }
-    if (flags.debug) {
-      options["debug"] = "true";
-    }
-    options["examples-dir"] = toPosix(join(dirname(spec), "examples"));
-    const configs = Object.entries(options).flatMap(([k, v]) => {
-      return [
-        "--option",
-        `${argv.values.emitterName || "@typespec/http-client-python"}.${k}="${v}"`,
-      ];
-    });
-    emitterConfigs.push({
-      options: configs,
-      outputDir: options["emitter-output-dir"],
-    });
-  }
-  return emitterConfigs;
-}
-function _getCmdList(spec: string, flags: RegenerateFlags): TspCommand[] {
-  return addOptions(spec, GENERATED_FOLDER, flags).map((option) => {
-    return {
-      outputDir: option.outputDir,
-      command: ["compile", spec, "--emit", toPosix(PLUGIN_DIR), ...option.options],
-    };
-  });
-}
-
-async function runTaskPool(tasks: Array<() => Promise<void>>, poolLimit: number): Promise<void> {
-  async function worker(start: number, end: number) {
-    while (start < end) {
-      await tasks[start]();
-      start++;
-    }
-  }
-
-  const workers = [];
-  let start = 0;
-  while (start < tasks.length) {
-    const end = Math.min(start + poolLimit, tasks.length);
-    workers.push((async () => await worker(start, end))());
-    start = end;
-  }
-  await Promise.all(workers);
-}
-
 // create some files before regeneration. After regeneration, these files should be deleted and we will test it
 // in test case
 async function preprocess(flags: RegenerateFlagsInput): Promise<void> {
@@ -483,36 +102,31 @@ async function preprocess(flags: RegenerateFlagsInput): Promise<void> {
   }
 }
 
-async function regenerate(flags: RegenerateFlagsInput): Promise<void> {
-  if (flags.flavor === undefined) {
-    await regenerate({ flavor: "azure", ...flags });
-    await regenerate({ flavor: "unbranded", ...flags });
-  } else {
-    await preprocess(flags);
-
-    const flagsResolved = { debug: false, flavor: flags.flavor, ...flags };
-    const subdirectoriesForAzure = await getSubdirectories(AZURE_HTTP_SPECS, flagsResolved);
-    const subdirectoriesForNonAzure = await getSubdirectories(HTTP_SPECS, flagsResolved);
-    const subdirectories =
-      flags.flavor === "azure"
-        ? [...subdirectoriesForAzure, ...subdirectoriesForNonAzure]
-        : subdirectoriesForNonAzure;
-    const cmdList: TspCommand[] = subdirectories.flatMap((subdirectory) =>
-      _getCmdList(subdirectory, flagsResolved),
-    );
-
-    // Create tasks as functions for the pool
-    const tasks: Array<() => Promise<void>> = cmdList.map((tspCommand) => {
-      return () => executeCommand(tspCommand);
-    });
-
-    // Run tasks with a concurrency limit
-    await runTaskPool(tasks, 30);
-  }
+function _getCmdList(spec: string, flags: RegenerateFlags): TspCommand[] {
+  return buildOptions(spec, GENERATED_FOLDER, flags, config).map((po) => {
+    const optionArgs = Object.entries(po.options).flatMap(([k, v]) => [
+      "--option",
+      `${EMITTER_NAME}.${k}="${v}"`,
+    ]);
+    return {
+      outputDir: po.outputDir,
+      command: ["compile", spec, "--emit", toPosix(PLUGIN_DIR), ...optionArgs],
+    };
+  });
 }
 
+const config: RegenerateConfig = {
+  azureHttpSpecs: AZURE_HTTP_SPECS,
+  httpSpecs: HTTP_SPECS,
+  emitterOptions: EMITTER_OPTIONS,
+  azureEmitterOptions: AZURE_EMITTER_OPTIONS,
+  preprocess,
+  getCmdList: _getCmdList,
+  executeCommand,
+};
+
 const start = performance.now();
-regenerate(argv.values)
+regenerate(argv.values, config)
   .then(() =>
     console.log(
       chalk.green(

package/generator/build/lib/pygen/codegen/models/client.py

@@ -247,7 +247,7 @@ class Client(_ClientConfigBase[ClientGlobalParameterList]):
 
     def imports(self, async_mode: bool, **kwargs) -> FileImport:
         file_import = FileImport(self.code_model)
-        file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL)
+        file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.REGULAR)
         if self.code_model.options["azure-arm"]:
             file_import.add_submodule_import("azure.mgmt.core", self.pipeline_class(async_mode), ImportType.SDKCORE)
         else:
@@ -307,20 +307,20 @@ class Client(_ClientConfigBase[ClientGlobalParameterList]):
                 "rest",
                 "AsyncHttpResponse",
                 ImportType.SDKCORE,
-                TypingSection.CONDITIONAL,
+                TypingSection.REGULAR,
             )
         else:
             file_import.add_submodule_import(
                 "rest",
                 "HttpResponse",
                 ImportType.SDKCORE,
-                TypingSection.CONDITIONAL,
+                TypingSection.REGULAR,
             )
         file_import.add_submodule_import(
             "rest",
             "HttpRequest",
             ImportType.SDKCORE,
-            TypingSection.CONDITIONAL,
+            TypingSection.REGULAR,
         )
         serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace)
         for og in self.operation_groups:
@@ -401,7 +401,7 @@ class Config(_ClientConfigBase[ConfigGlobalParameterList]):
             "policies",
             ImportType.SDKCORE,
         )
-        file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL)
+        file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.REGULAR)
         if self.code_model.options.get("package-version"):
             serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace)
             file_import.add_submodule_import(

package/generator/build/lib/pygen/codegen/models/credential_types.py

@@ -213,6 +213,6 @@ class KeyCredentialType(CredentialType[KeyCredentialPolicyType]):
             "credentials",
             self.policy.credential_name,
             ImportType.SDKCORE,
-            typing_section=TypingSection.CONDITIONAL,
+            typing_section=TypingSection.REGULAR,
         )
         return file_import

package/generator/build/lib/pygen/codegen/models/enum_type.py

@@ -241,7 +241,7 @@ class EnumType(BaseType):
         file_import = FileImport(self.code_model)
         file_import.merge(self.value_type.imports(**kwargs))
         if self.code_model.options["models-mode"]:
-            file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL)
+            file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.REGULAR)
 
             serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace)
             relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace)

package/generator/build/lib/pygen/codegen/models/imports.py

@@ -23,9 +23,8 @@ class ImportType(str, Enum):
 
 
 class TypingSection(str, Enum):
-    REGULAR = "regular"  # this import is always a typing import
-    CONDITIONAL = "conditional"  # is a typing import when we're dealing with files that py2 will use, else regular
-    TYPING = "typing"  # never a typing import
+    REGULAR = "regular"  # this import is always a regular import
+    TYPING = "typing"  # this import goes under TYPE_CHECKING
 
 
 class MsrestImportType(Enum):

package/generator/build/lib/pygen/codegen/models/lro_operation.py

@@ -138,7 +138,7 @@ class LROOperationBase(OperationBase[LROResponseType]):
                 "_deserialize",
                 ImportType.LOCAL,
             )
-        file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL)
+        file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.REGULAR)
         file_import.add_submodule_import("typing", "cast", ImportType.STDLIB)
         return file_import
 

package/generator/build/lib/pygen/codegen/models/model_type.py

@@ -270,7 +270,7 @@ class JSONModelType(ModelType):
 
     def imports(self, **kwargs: Any) -> FileImport:
         file_import = FileImport(self.code_model)
-        file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL)
+        file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.REGULAR)
         file_import.define_mutable_mapping_type()
         if self.is_xml:
             file_import.add_submodule_import("xml.etree", "ElementTree", ImportType.STDLIB, alias="ET")
@@ -351,7 +351,7 @@ class MsrestModelType(GeneratedModelType):
 
     def imports(self, **kwargs: Any) -> FileImport:
         file_import = super().imports(**kwargs)
-        file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL)
+        file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.REGULAR)
         return file_import