@azure-tools/typespec-python 0.60.0 → 0.60.2

package/scripts/eng/regenerate.ts

@@ -3,9 +3,20 @@ import { exec as execCallback } from "child_process";
 import { promisify } from "util";
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
-import { dirname, join, relative, resolve } from "path";
+import { join, resolve } from "path";
 import { promises, rmSync } from "fs";
 import { fileURLToPath } from "url";
+import {
+    BASE_AZURE_EMITTER_OPTIONS,
+    BASE_EMITTER_OPTIONS,
+    buildOptions,
+    regenerate,
+    toPosix,
+    type RegenerateConfig,
+    type RegenerateFlags,
+    type RegenerateFlagsInput,
+    type TspCommand,
+} from "./regenerate-common.js";
 
 // Promisify the exec function
 const exec = promisify(execCallback);
@@ -14,196 +25,14 @@ const exec = promisify(execCallback);
 const PLUGIN_DIR = resolve(fileURLToPath(import.meta.url), "../../../");
 const AZURE_HTTP_SPECS = resolve(PLUGIN_DIR, "node_modules/@azure-tools/azure-http-specs/specs");
 const HTTP_SPECS = resolve(PLUGIN_DIR, "node_modules/@typespec/http-specs/specs");
-interface TspCommand {
-    outputDir: string;
-    command: string;
-}
-
-// Add this near the top with other constants
-const SKIP_SPECS: string[] = [];
+const EMITTER_NAME = "@azure-tools/typespec-python";
 
 const AZURE_EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, string>[]> = {
-    "azure/client-generator-core/access": {
-        namespace: "specs.azure.clientgenerator.core.access",
-    },
-    "azure/client-generator-core/alternate-type": {
-        namespace: "specs.azure.clientgenerator.core.alternatetype",
-    },
-    "azure/client-generator-core/api-version": {
-        namespace: "specs.azure.clientgenerator.core.apiversion",
-    },
-    "azure/client-generator-core/client-initialization": {
-        namespace: "specs.azure.clientgenerator.core.clientinitialization",
-    },
-    "azure/client-generator-core/client-location": {
-        namespace: "specs.azure.clientgenerator.core.clientlocation",
-    },
-    "azure/client-generator-core/deserialize-empty-string-as-null": {
-        namespace: "specs.azure.clientgenerator.core.emptystring",
-    },
-    "azure/client-generator-core/flatten-property": {
-        namespace: "specs.azure.clientgenerator.core.flattenproperty",
-    },
-    "azure/client-generator-core/usage": {
-        namespace: "specs.azure.clientgenerator.core.usage",
-    },
-    "azure/client-generator-core/override": {
-        namespace: "specs.azure.clientgenerator.core.override",
-    },
-    "azure/client-generator-core/hierarchy-building": {
-        namespace: "specs.azure.clientgenerator.core.hierarchybuilding",
-    },
-    "azure/core/basic": {
-        namespace: "specs.azure.core.basic",
-    },
-    "azure/core/lro/rpc": {
-        namespace: "specs.azure.core.lro.rpc",
-    },
-    "azure/core/lro/standard": {
-        namespace: "specs.azure.core.lro.standard",
-    },
-    "azure/core/model": {
-        namespace: "specs.azure.core.model",
-    },
-    "azure/core/page": {
-        namespace: "specs.azure.core.page",
-    },
-    "azure/core/scalar": {
-        namespace: "specs.azure.core.scalar",
-    },
-    "azure/core/traits": {
-        namespace: "specs.azure.core.traits",
-    },
-    "azure/encode/duration": {
-        namespace: "specs.azure.encode.duration",
-    },
-    "azure/example/basic": {
-        namespace: "specs.azure.example.basic",
-    },
-    "azure/payload/pageable": {
-        namespace: "specs.azure.payload.pageable",
-    },
-    "azure/versioning/previewVersion": {
-        namespace: "specs.azure.versioning.previewversion",
-    },
-    "client/structure/default": {
-        namespace: "client.structure.service",
-    },
-    "client/structure/multi-client": {
-        "package-name": "client-structure-multiclient",
-        "namespace": "client.structure.multiclient",
-    },
-    "client/structure/renamed-operation": {
-        "package-name": "client-structure-renamedoperation",
-        "namespace": "client.structure.renamedoperation",
-    },
-    "client/structure/two-operation-group": {
-        "package-name": "client-structure-twooperationgroup",
-        "namespace": "client.structure.twooperationgroup",
-    },
-    "client/naming": {
-        namespace: "client.naming.main",
-    },
-    "client/overload": {
-        namespace: "client.overload",
-    },
-    "encode/duration": {
-        namespace: "encode.duration",
-    },
-    "encode/numeric": {
-        namespace: "encode.numeric",
-    },
-    "parameters/basic": {
-        namespace: "parameters.basic",
-    },
-    "parameters/spread": {
-        namespace: "parameters.spread",
-    },
-    "payload/content-negotiation": {
-        namespace: "payload.contentnegotiation",
-    },
-    "payload/multipart": {
-        namespace: "payload.multipart",
-    },
-    "serialization/encoded-name/json": {
-        namespace: "serialization.encodedname.json",
-    },
-    "special-words": {
-        namespace: "specialwords",
-    },
-    "service/multi-service": {
-        namespace: "service.multiservice",
-    },
+    ...BASE_AZURE_EMITTER_OPTIONS,
 };
 
 const EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, string>[]> = {
-    "resiliency/srv-driven/old.tsp": {
-        "package-name": "resiliency-srv-driven1",
-        "namespace": "resiliency.srv.driven1",
-        "package-mode": "azure-dataplane",
-        "package-pprint-name": "ResiliencySrvDriven1",
-    },
-    "resiliency/srv-driven": {
-        "package-name": "resiliency-srv-driven2",
-        "namespace": "resiliency.srv.driven2",
-        "package-mode": "azure-dataplane",
-        "package-pprint-name": "ResiliencySrvDriven2",
-    },
-    "authentication/api-key": {
-        "clear-output-folder": "true",
-    },
-    "authentication/http/custom": {
-        "package-name": "authentication-http-custom",
-        "namespace": "authentication.http.custom",
-        "package-pprint-name": "Authentication Http Custom",
-    },
-    "authentication/union": [
-        {
-            "package-name": "authentication-union",
-            "namespace": "authentication.union",
-        },
-        {
-            "package-name": "setuppy-authentication-union",
-            "namespace": "setuppy.authentication.union",
-            "keep-setup-py": "true",
-        },
-    ],
-    "type/array": {
-        "package-name": "typetest-array",
-        "namespace": "typetest.array",
-    },
-    "type/dictionary": {
-        "package-name": "typetest-dictionary",
-        "namespace": "typetest.dictionary",
-    },
-    "type/enum/extensible": {
-        "package-name": "typetest-enum-extensible",
-        "namespace": "typetest.enum.extensible",
-    },
-    "type/enum/fixed": {
-        "package-name": "typetest-enum-fixed",
-        "namespace": "typetest.enum.fixed",
-    },
-    "type/model/empty": {
-        "package-name": "typetest-model-empty",
-        "namespace": "typetest.model.empty",
-    },
-    "type/model/inheritance/enum-discriminator": {
-        "package-name": "typetest-model-enumdiscriminator",
-        "namespace": "typetest.model.enumdiscriminator",
-    },
-    "type/model/inheritance/nested-discriminator": {
-        "package-name": "typetest-model-nesteddiscriminator",
-        "namespace": "typetest.model.nesteddiscriminator",
-    },
-    "type/model/inheritance/not-discriminated": {
-        "package-name": "typetest-model-notdiscriminated",
-        "namespace": "typetest.model.notdiscriminated",
-    },
-    "type/model/inheritance/single-discriminator": {
-        "package-name": "typetest-model-singlediscriminator",
-        "namespace": "typetest.model.singlediscriminator",
-    },
+    ...BASE_EMITTER_OPTIONS,
     "type/model/inheritance/recursive": [
         {
             "package-name": "typetest-model-recursive",
@@ -216,54 +45,6 @@ const EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, st
             "clear-output-folder": "true",
         },
     ],
-    "type/model/usage": {
-        "package-name": "typetest-model-usage",
-        "namespace": "typetest.model.usage",
-    },
-    "type/model/visibility": [
-        {
-            "package-name": "typetest-model-visibility",
-            "namespace": "typetest.model.visibility",
-        },
-        {
-            "package-name": "headasbooleantrue",
-            "namespace": "headasbooleantrue",
-            "head-as-boolean": "true",
-        },
-        {
-            "package-name": "headasbooleanfalse",
-            "namespace": "headasbooleanfalse",
-            "head-as-boolean": "false",
-        },
-    ],
-    "type/property/nullable": {
-        "package-name": "typetest-property-nullable",
-        "namespace": "typetest.property.nullable",
-    },
-    "type/property/optionality": {
-        "package-name": "typetest-property-optional",
-        "namespace": "typetest.property.optional",
-    },
-    "type/property/additional-properties": {
-        "package-name": "typetest-property-additionalproperties",
-        "namespace": "typetest.property.additionalproperties",
-    },
-    "type/scalar": {
-        "package-name": "typetest-scalar",
-        "namespace": "typetest.scalar",
-    },
-    "type/property/value-types": {
-        "package-name": "typetest-property-valuetypes",
-        "namespace": "typetest.property.valuetypes",
-    },
-    "type/union": {
-        "package-name": "typetest-union",
-        "namespace": "typetest.union",
-    },
-    "type/union/discriminated": {
-        "package-name": "typetest-discriminatedunion",
-        "namespace": "typetest.discriminatedunion",
-    },
     "client/structure/client-operation-group": {
         "package-name": "client-structure-clientoperationgroup",
         "namespace": "client.structure.clientoperationgroup",
@@ -280,30 +61,14 @@ const EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, st
         "package-name": "client-structure-twooperationgroup",
         "namespace": "client.structure.twooperationgroup",
     },
-    "documentation": {
-        "package-name": "specs-documentation",
-        "namespace": "specs.documentation",
-    },
 };
 
-function toPosix(dir: string): string {
-    return dir.replace(/\\/g, "/");
-}
-
-function getEmitterOption(spec: string, flavor: string): Record<string, string>[] {
-    const specDir = spec.includes("azure") ? AZURE_HTTP_SPECS : HTTP_SPECS;
-    const relativeSpec = toPosix(relative(specDir, spec));
-    const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") ? relativeSpec : dirname(relativeSpec);
-    const emitter_options = EMITTER_OPTIONS[key] || (flavor === "azure" ? AZURE_EMITTER_OPTIONS[key] : [{}]) || [{}];
-    const result = Array.isArray(emitter_options) ? emitter_options : [emitter_options];
-    return result;
-}
-
 // Function to execute CLI commands asynchronously
 async function executeCommand(tspCommand: TspCommand): Promise<void> {
     try {
-        console.log(`exec: ${tspCommand.command}`);
-        const { stdout, stderr } = await exec(tspCommand.command);
+        const cmd = tspCommand.command as string;
+        console.log(`exec: ${cmd}`);
+        const { stdout, stderr } = await exec(cmd);
         if (stdout) console.log(`stdout: ${stdout}`);
         if (stderr) console.error(`stderr: ${stderr}`);
     } catch (error) {
@@ -313,139 +78,6 @@ async function executeCommand(tspCommand: TspCommand): Promise<void> {
     }
 }
 
-interface RegenerateFlagsInput {
-    flavor?: "azure" | "unbranded";
-    debug?: boolean;
-    name?: string;
-}
-
-interface RegenerateFlags {
-    flavor: "azure" | "unbranded";
-    debug: boolean;
-    name?: string;
-}
-
-const SpecialFlags: Record<string, Record<string, any>> = {
-    azure: {
-        "generate-test": true,
-        "generate-sample": true,
-    },
-};
-
-async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promise<string[]> {
-    const subdirectories: string[] = [];
-
-    async function searchDir(currentDir: string) {
-        const items = await promises.readdir(currentDir, { withFileTypes: true });
-
-        const promisesArray = items.map(async (item) => {
-            const subDirPath = join(currentDir, item.name);
-            if (item.isDirectory()) {
-                const mainTspPath = join(subDirPath, "main.tsp");
-                const clientTspPath = join(subDirPath, "client.tsp");
-
-                const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath));
-
-                // Replace the individual skip checks with:
-                if (SKIP_SPECS.some((skipSpec) => mainTspRelativePath.includes(skipSpec))) return;
-
-                const hasMainTsp = await promises
-                    .access(mainTspPath)
-                    .then(() => true)
-                    .catch(() => false);
-                const hasClientTsp = await promises
-                    .access(clientTspPath)
-                    .then(() => true)
-                    .catch(() => false);
-
-                if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) {
-                    if (mainTspRelativePath.includes("resiliency/srv-driven")) {
-                        subdirectories.push(resolve(subDirPath, "old.tsp"));
-                    }
-                    if (hasClientTsp) {
-                        subdirectories.push(resolve(subDirPath, "client.tsp"));
-                    } else if (hasMainTsp) {
-                        subdirectories.push(resolve(subDirPath, "main.tsp"));
-                    }
-                }
-
-                // Recursively search in the subdirectory
-                await searchDir(subDirPath);
-            }
-        });
-
-        await Promise.all(promisesArray);
-    }
-
-    await searchDir(baseDir);
-    return subdirectories;
-}
-
-function defaultPackageName(spec: string): string {
-    const specDir = spec.includes("azure") ? AZURE_HTTP_SPECS : HTTP_SPECS;
-    return toPosix(relative(specDir, dirname(spec)))
-        .replace(/\//g, "-")
-        .toLowerCase();
-}
-
-interface EmitterConfig {
-    optionsStr: string;
-    outputDir: string;
-}
-
-function addOptions(spec: string, generatedFolder: string, flags: RegenerateFlags): EmitterConfig[] {
-    const emitterConfigs: EmitterConfig[] = [];
-    for (const config of getEmitterOption(spec, flags.flavor)) {
-        const options: Record<string, string> = { ...config };
-        options["flavor"] = flags.flavor;
-        for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) {
-            options[k] = v;
-        }
-        if (options["emitter-output-dir"] === undefined) {
-            const packageName = options["package-name"] || defaultPackageName(spec);
-            options["emitter-output-dir"] = toPosix(`${generatedFolder}/test/${flags.flavor}/generated/${packageName}`);
-        }
-        if (flags.debug) {
-            options["debug"] = "true";
-        }
-        options["examples-dir"] = toPosix(join(dirname(spec), "examples"));
-        const configs = Object.entries(options).flatMap(([k, v]) => {
-            return `--option @azure-tools/typespec-python.${k}=${typeof v === "string" && v.indexOf(" ") > -1 ? `"${v}"` : v}`;
-        });
-        emitterConfigs.push({
-            optionsStr: configs.join(" "),
-            outputDir: options["emitter-output-dir"],
-        });
-    }
-    return emitterConfigs;
-}
-function _getCmdList(spec: string, flags: RegenerateFlags): TspCommand[] {
-    return addOptions(spec, PLUGIN_DIR, flags).map((option) => {
-        return {
-            outputDir: option.outputDir,
-            command: `tsp compile ${spec} --emit=${toPosix(PLUGIN_DIR)} ${option.optionsStr}`,
-        };
-    });
-}
-
-async function runTaskPool(tasks: Array<() => Promise<void>>, poolLimit: number): Promise<void> {
-    async function worker(start: number, end: number) {
-        while (start < end) {
-            await tasks[start]();
-            start++;
-        }
-    }
-
-    const workers = [];
-    let start = 0;
-    while (start < tasks.length) {
-        const end = Math.min(start + poolLimit, tasks.length);
-        workers.push((async () => await worker(start, end))());
-        start = end;
-    }
-    await Promise.all(workers);
-}
-
 // create some files before regeneration. After regeneration, these files should be deleted and we will test it
 // in test case
 async function preprocess(flags: RegenerateFlagsInput): Promise<void> {
@@ -479,33 +111,30 @@ async function preprocess(flags: RegenerateFlagsInput): Promise<void> {
     }
 }
 
-async function regenerate(flags: RegenerateFlagsInput): Promise<void> {
-    if (flags.flavor === undefined) {
-        await regenerate({ ...flags, flavor: "azure" });
-        await regenerate({ ...flags, flavor: "unbranded" });
-    } else {
-        await preprocess(flags);
-        const flagsResolved = { debug: false, flavor: flags.flavor, ...flags };
-        const subdirectoriesForAzure = await getSubdirectories(AZURE_HTTP_SPECS, flagsResolved);
-        const subdirectoriesForNonAzure = await getSubdirectories(HTTP_SPECS, flagsResolved);
-        const subdirectories =
-            flags.flavor === "azure"
-                ? [...subdirectoriesForAzure, ...subdirectoriesForNonAzure]
-                : subdirectoriesForNonAzure;
-        const cmdList: TspCommand[] = subdirectories.flatMap((subdirectory) =>
-            _getCmdList(subdirectory, flagsResolved),
-        );
-
-        // Create tasks as functions for the pool
-        const tasks: Array<() => Promise<void>> = cmdList.map((tspCommand) => {
-            return () => executeCommand(tspCommand);
-        });
-
-        // Run tasks with a concurrency limit
-        await runTaskPool(tasks, 30);
-    }
+function _getCmdList(spec: string, flags: RegenerateFlags): TspCommand[] {
+    return buildOptions(spec, PLUGIN_DIR, flags, config).map((po) => {
+        const optionsStr = Object.entries(po.options)
+            .flatMap(([k, v]) => {
+                return `--option ${EMITTER_NAME}.${k}=${typeof v === "string" && v.indexOf(" ") > -1 ? `"${v}"` : v}`;
+            })
+            .join(" ");
+        return {
+            outputDir: po.outputDir,
+            command: `tsp compile ${spec} --emit=${toPosix(PLUGIN_DIR)} ${optionsStr}`,
+        };
+    });
 }
 
+const config: RegenerateConfig = {
+    azureHttpSpecs: AZURE_HTTP_SPECS,
+    httpSpecs: HTTP_SPECS,
+    emitterOptions: EMITTER_OPTIONS,
+    azureEmitterOptions: AZURE_EMITTER_OPTIONS,
+    preprocess,
+    getCmdList: _getCmdList,
+    executeCommand,
+};
+
 // PARSE INPUT ARGUMENTS
 const argv = yargs(hideBin(process.argv))
     .option("flavor", {
@@ -525,6 +154,6 @@ const argv = yargs(hideBin(process.argv))
     }).argv;
 
 const start = performance.now();
-regenerate(argv as RegenerateFlags)
+regenerate(argv as RegenerateFlags, config)
     .then(() => console.log(`Regeneration successful, time taken: ${Math.round((performance.now() - start) / 1000)} s`))
     .catch((error) => console.error(`Regeneration failed: ${error.message}`));