@typespec/http-client-python 0.7.0 → 0.8.0

package/emitter/src/emitter.ts

@@ -16,7 +16,7 @@ import { saveCodeModelAsYaml } from "./external-process.js";
 import { PythonEmitterOptions, PythonSdkContext, reportDiagnostic } from "./lib.js";
 import { runPython3 } from "./run-python3.js";
 import { disableGenerationMap, simpleTypesMap, typesMap } from "./types.js";
-import { removeUnderscoresFromNamespace } from "./utils.js";
+import { md2Rst, removeUnderscoresFromNamespace } from "./utils.js";
 
 export function getModelsMode(context: SdkContext): "dpg" | "none" {
   const specifiedModelsMode = context.emitContext.options["models-mode"];
@@ -80,6 +80,44 @@ async function createPythonSdkContext<TServiceOperation extends SdkServiceOperat
   };
 }
 
+function walkThroughNodes(yamlMap: Record<string, any>): Record<string, any> {
+  const stack = [yamlMap];
+  const seen = new WeakSet();
+
+  while (stack.length > 0) {
+    const current = stack.pop();
+
+    if (seen.has(current!)) {
+      continue;
+    }
+    if (current !== undefined && current !== null) {
+      seen.add(current);
+    }
+
+    if (Array.isArray(current)) {
+      for (let i = 0; i < current.length; i++) {
+        if (current[i] !== undefined && typeof current[i] === "object") {
+          stack.push(current[i]);
+        }
+      }
+    } else {
+      for (const key in current) {
+        if (key === "description" || key === "summary") {
+          if (current[key] !== undefined) {
+            current[key] = md2Rst(current[key]);
+          }
+        } else if (Array.isArray(current[key])) {
+          stack.push(current[key]);
+        } else if (current[key] !== undefined && typeof current[key] === "object") {
+          stack.push(current[key]);
+        }
+      }
+    }
+  }
+
+  return yamlMap;
+}
+
 function cleanAllCache() {
   typesMap.clear();
   simpleTypesMap.clear();
@@ -87,6 +125,23 @@ function cleanAllCache() {
 }
 
 export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
+  try {
+    await onEmitMain(context);
+  } catch (error: any) {
+    const errStackStart =
+      "========================================= error stack start ================================================";
+    const errStackEnd =
+      "========================================= error stack end ================================================";
+    const errStack = error.stack ? `\n${errStackStart}\n${error.stack}\n${errStackEnd}` : "";
+    reportDiagnostic(context.program, {
+      code: "unknown-error",
+      target: NoTarget,
+      format: { stack: errStack },
+    });
+  }
+}
+
+async function onEmitMain(context: EmitContext<PythonEmitterOptions>) {
   // clean all cache to make sure emitter could work in watch mode
   cleanAllCache();
 
@@ -103,7 +158,10 @@ export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
     });
     return;
   }
-  const yamlPath = await saveCodeModelAsYaml("python-yaml-path", yamlMap);
+
+  const parsedYamlMap = walkThroughNodes(yamlMap);
+
+  const yamlPath = await saveCodeModelAsYaml("python-yaml-path", parsedYamlMap);
   const resolvedOptions = sdkContext.emitContext.options;
   const commandArgs: Record<string, string> = {};
   if (resolvedOptions["packaging-files-config"]) {
@@ -147,70 +205,55 @@ export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
       }
     }
 
-    try {
-      if (resolvedOptions["use-pyodide"]) {
-        // here we run with pyodide
-        const pyodide = await setupPyodideCall(root);
-        // create the output folder if not exists
-        if (!fs.existsSync(outputDir)) {
-          fs.mkdirSync(outputDir, { recursive: true });
-        }
-        // mount output folder to pyodide
-        pyodide.FS.mkdirTree("/output");
-        pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: outputDir }, "/output");
-        // mount yaml file to pyodide
-        pyodide.FS.mkdirTree("/yaml");
-        pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: path.dirname(yamlPath) }, "/yaml");
-        const globals = pyodide.toPy({
-          outputFolder: "/output",
-          yamlFile: `/yaml/${path.basename(yamlPath)}`,
-          commandArgs,
-        });
-        const pythonCode = `
+    if (resolvedOptions["use-pyodide"]) {
+      // here we run with pyodide
+      const pyodide = await setupPyodideCall(root);
+      // create the output folder if not exists
+      if (!fs.existsSync(outputDir)) {
+        fs.mkdirSync(outputDir, { recursive: true });
+      }
+      // mount output folder to pyodide
+      pyodide.FS.mkdirTree("/output");
+      pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: outputDir }, "/output");
+      // mount yaml file to pyodide
+      pyodide.FS.mkdirTree("/yaml");
+      pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: path.dirname(yamlPath) }, "/yaml");
+      const globals = pyodide.toPy({
+        outputFolder: "/output",
+        yamlFile: `/yaml/${path.basename(yamlPath)}`,
+        commandArgs,
+      });
+      const pythonCode = `
           async def main():
             import warnings
             with warnings.catch_warnings():
-              warnings.simplefilter("ignore", SyntaxWarning) # bc of m2r2 dep issues
-              from pygen import m2r, preprocess, codegen, black
-            m2r.M2R(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
-            preprocess.PreProcessPlugin(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
-            codegen.CodeGenerator(output_folder=outputFolder, cadl_file=yamlFile, **commandArgs).process()
+              from pygen import preprocess, codegen, black
+            preprocess.PreProcessPlugin(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs).process()
+            codegen.CodeGenerator(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs).process()
             black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process()
       
           await main()`;
-        await pyodide.runPythonAsync(pythonCode, { globals });
+      await pyodide.runPythonAsync(pythonCode, { globals });
+    } else {
+      // here we run with native python
+      let venvPath = path.join(root, "venv");
+      if (fs.existsSync(path.join(venvPath, "bin"))) {
+        venvPath = path.join(venvPath, "bin", "python");
+      } else if (fs.existsSync(path.join(venvPath, "Scripts"))) {
+        venvPath = path.join(venvPath, "Scripts", "python.exe");
       } else {
-        // here we run with native python
-        let venvPath = path.join(root, "venv");
-        if (fs.existsSync(path.join(venvPath, "bin"))) {
-          venvPath = path.join(venvPath, "bin", "python");
-        } else if (fs.existsSync(path.join(venvPath, "Scripts"))) {
-          venvPath = path.join(venvPath, "Scripts", "python.exe");
-        } else {
-          reportDiagnostic(program, {
-            code: "pyodide-flag-conflict",
-            target: NoTarget,
-          });
-        }
-        commandArgs["output-folder"] = outputDir;
-        commandArgs["cadl-file"] = yamlPath;
-        const commandFlags = Object.entries(commandArgs)
-          .map(([key, value]) => `--${key}=${value}`)
-          .join(" ");
-        const command = `${venvPath} ${root}/eng/scripts/setup/run_tsp.py ${commandFlags}`;
-        execSync(command, { stdio: [process.stdin, process.stdout] });
+        reportDiagnostic(program, {
+          code: "pyodide-flag-conflict",
+          target: NoTarget,
+        });
       }
-    } catch (error: any) {
-      const errStackStart =
-        "========================================= error stack start ================================================";
-      const errStackEnd =
-        "========================================= error stack end ================================================";
-      const errStack = error.stack ? `\n${errStackStart}\n${error.stack}\n${errStackEnd}` : "";
-      reportDiagnostic(program, {
-        code: "unknown-error",
-        target: NoTarget,
-        format: { stack: errStack },
-      });
+      commandArgs["output-folder"] = outputDir;
+      commandArgs["tsp-file"] = yamlPath;
+      const commandFlags = Object.entries(commandArgs)
+        .map(([key, value]) => `--${key}=${value}`)
+        .join(" ");
+      const command = `${venvPath} ${root}/eng/scripts/setup/run_tsp.py ${commandFlags}`;
+      execSync(command, { stdio: [process.stdin, process.stdout] });
     }
   }
 }

package/emitter/src/external-process.ts

@@ -5,10 +5,10 @@ import { mkdir, writeFile } from "fs/promises";
 import jsyaml from "js-yaml";
 import os from "os";
 
-const cadlCodeGenTempDir = joinPaths(os.tmpdir(), "cadl-codegen");
+const tspCodeGenTempDir = joinPaths(os.tmpdir(), "tsp-codegen");
 
 export function createTempPath(extension: string, prefix: string = "") {
-  return joinPaths(cadlCodeGenTempDir, prefix + randomUUID() + extension);
+  return joinPaths(tspCodeGenTempDir, prefix + randomUUID() + extension);
 }
 
 /**
@@ -18,7 +18,7 @@ export function createTempPath(extension: string, prefix: string = "") {
  * @return the absolute path to the created codemodel.
  */
 export async function saveCodeModelAsYaml(name: string, codemodel: unknown): Promise<string> {
-  await mkdir(cadlCodeGenTempDir, { recursive: true });
+  await mkdir(tspCodeGenTempDir, { recursive: true });
   const filename = createTempPath(".yaml", name);
   const yamlStr = jsyaml.dump(codemodel);
   await writeFile(filename, yamlStr);

package/emitter/src/http.ts

@@ -1,6 +1,7 @@
+import { NoTarget } from "@typespec/compiler";
+
 import {
   SdkBasicServiceMethod,
-  SdkBodyModelPropertyType,
   SdkBodyParameter,
   SdkClientType,
   SdkHeaderParameter,
@@ -10,16 +11,16 @@ import {
   SdkHttpResponse,
   SdkLroPagingServiceMethod,
   SdkLroServiceMethod,
+  SdkModelPropertyType,
   SdkPagingServiceMethod,
   SdkPathParameter,
   SdkQueryParameter,
   SdkServiceMethod,
   SdkServiceResponseHeader,
-  SdkType,
   UsageFlags,
 } from "@azure-tools/typespec-client-generator-core";
 import { HttpStatusCodeRange } from "@typespec/http";
-import { PythonSdkContext } from "./lib.js";
+import { PythonSdkContext, reportDiagnostic } from "./lib.js";
 import { KnownTypes, getType } from "./types.js";
 import {
   camelToSnakeCase,
@@ -29,6 +30,7 @@ import {
   getImplementation,
   isAbstract,
   isAzureCoreErrorResponse,
+  isContinuationToken,
 } from "./utils.js";
 
 function isContentTypeParameter(parameter: SdkHeaderParameter) {
@@ -96,6 +98,88 @@ function addLroInformation(
   };
 }
 
+function getWireNameFromPropertySegments(segments: SdkModelPropertyType[]): string | undefined {
+  if (segments[0].kind === "property") {
+    return segments
+      .filter((s) => s.kind === "property")
+      .map((s) => s.serializationOptions.json?.name ?? "")
+      .join(".");
+  }
+
+  return undefined;
+}
+
+function getWireNameWithDiagnostics(
+  context: PythonSdkContext<SdkHttpOperation>,
+  segments: SdkModelPropertyType[] | undefined,
+  code: "invalid-paging-items" | "invalid-next-link" | "invalid-lro-result",
+  method?: SdkServiceMethod<SdkHttpOperation>,
+): string | undefined {
+  if (segments && segments.length > 0) {
+    const result = getWireNameFromPropertySegments(segments);
+    if (result) {
+      return result;
+    }
+    const operationId = method ? method.name : "";
+    reportDiagnostic(context.program, {
+      code: code,
+      target: NoTarget,
+      format: { operationId: operationId },
+    });
+  }
+
+  return undefined;
+}
+
+function buildContinuationToken(
+  context: PythonSdkContext<SdkHttpOperation>,
+  method: SdkPagingServiceMethod<SdkHttpOperation> | SdkLroPagingServiceMethod<SdkHttpOperation>,
+  segments: SdkModelPropertyType[],
+  input: boolean = true,
+): Record<string, any> {
+  if (segments[0].kind === "property") {
+    const wireName = getWireNameFromPropertySegments(segments);
+    if (wireName) {
+      return { wireName, location: "body" };
+    }
+  } else if (input) {
+    for (const parameter of method.operation.parameters) {
+      if (isContinuationToken(parameter, method)) {
+        return { wireName: parameter.serializedName, location: parameter.kind };
+      }
+    }
+  } else {
+    for (const response of method.operation.responses) {
+      for (const header of response.headers) {
+        if (isContinuationToken(header, method, false)) {
+          return { wireName: header.serializedName, location: "header" };
+        }
+      }
+    }
+  }
+  reportDiagnostic(context.program, {
+    code: "invalid-continuation-token",
+    target: NoTarget,
+    format: { operationId: method.name, direction: input ? "request" : "response" },
+  });
+  return {};
+}
+
+function buildAllContinuationToken(
+  context: PythonSdkContext<SdkHttpOperation>,
+  method: SdkPagingServiceMethod<SdkHttpOperation> | SdkLroPagingServiceMethod<SdkHttpOperation>,
+): Record<string, any> {
+  const parameterSegments = method.pagingMetadata.continuationTokenParameterSegments ?? [];
+  const responseSegments = method.pagingMetadata.continuationTokenResponseSegments ?? [];
+  if (parameterSegments.length > 0 && responseSegments.length > 0) {
+    return {
+      input: buildContinuationToken(context, method, parameterSegments),
+      output: buildContinuationToken(context, method, responseSegments, false),
+    };
+  }
+  return {};
+}
+
 function addPagingInformation(
   context: PythonSdkContext<SdkHttpOperation>,
   rootClient: SdkClientType<SdkHttpOperation>,
@@ -109,13 +193,17 @@ function addPagingInformation(
   }
   const itemType = getType(context, method.response.type!);
   const base = emitHttpOperation(context, rootClient, operationGroupName, method.operation, method);
-  const itemName = getPropertyWireName(
-    method.operation.responses[0].type,
-    method.response.resultPath,
+  const itemName = getWireNameWithDiagnostics(
+    context,
+    method.response.resultSegments,
+    "invalid-paging-items",
+    method,
   );
-  const continuationTokenName = getPropertyWireName(
-    method.operation.responses[0].type,
-    method.nextLinkPath,
+  const nextLinkName = getWireNameWithDiagnostics(
+    context,
+    method.pagingMetadata.nextLinkSegments,
+    "invalid-next-link",
+    method,
   );
   base.responses.forEach((resp: Record<string, any>) => {
     resp.type = itemType;
@@ -126,23 +214,14 @@ function addPagingInformation(
     discriminator: "paging",
     exposeStreamKeyword: false,
     itemName,
-    continuationTokenName,
+    nextLinkName,
     itemType,
     description: method.doc ?? "",
     summary: method.summary,
+    continuationToken: buildAllContinuationToken(context, method),
   };
 }
 
-function getPropertyWireName(type: SdkType | undefined, path?: string) {
-  if (!path || !type || type.kind !== "model") return path;
-  for (const property of type.properties) {
-    if (property.name === path) {
-      return (property as SdkBodyModelPropertyType).serializedName;
-    }
-  }
-  return path;
-}
-
 export function emitLroHttpMethod(
   context: PythonSdkContext<SdkHttpOperation>,
   rootClient: SdkClientType<SdkHttpOperation>,
@@ -192,7 +271,7 @@ function emitHttpOperation(
   const result = {
     url: operation.path,
     method: operation.verb.toUpperCase(),
-    parameters: emitHttpParameters(context, rootClient, operation),
+    parameters: emitHttpParameters(context, rootClient, operation, method),
     bodyParameter: emitHttpBodyParameter(context, operation.bodyParam),
     responses,
     exceptions,
@@ -277,8 +356,9 @@ function emitHttpPathParameter(
 function emitHttpHeaderParameter(
   context: PythonSdkContext<SdkHttpOperation>,
   parameter: SdkHeaderParameter,
+  method: SdkServiceMethod<SdkHttpOperation>,
 ): Record<string, any> {
-  const base = emitParamBase(context, parameter);
+  const base = emitParamBase(context, parameter, method);
   const [delimiter, explode] = getDelimiterAndExplode(parameter);
   let clientDefaultValue = parameter.clientDefaultValue;
   if (isContentTypeParameter(parameter)) {
@@ -302,8 +382,9 @@ function emitHttpHeaderParameter(
 function emitHttpQueryParameter(
   context: PythonSdkContext<SdkHttpOperation>,
   parameter: SdkQueryParameter,
+  method: SdkServiceMethod<SdkHttpOperation>,
 ): Record<string, any> {
-  const base = emitParamBase(context, parameter);
+  const base = emitParamBase(context, parameter, method);
   const [delimiter, explode] = getDelimiterAndExplode(parameter);
   return {
     ...base,
@@ -320,15 +401,16 @@ function emitHttpParameters(
   context: PythonSdkContext<SdkHttpOperation>,
   rootClient: SdkClientType<SdkHttpOperation>,
   operation: SdkHttpOperation,
+  method: SdkServiceMethod<SdkHttpOperation>,
 ): Record<string, any>[] {
   const parameters: Record<string, any>[] = [...context.__endpointPathParameters];
   for (const parameter of operation.parameters) {
     switch (parameter.kind) {
       case "header":
-        parameters.push(emitHttpHeaderParameter(context, parameter));
+        parameters.push(emitHttpHeaderParameter(context, parameter, method));
         break;
       case "query":
-        parameters.push(emitHttpQueryParameter(context, parameter));
+        parameters.push(emitHttpQueryParameter(context, parameter, method));
         break;
       case "path":
         parameters.push(emitHttpPathParameter(context, parameter));
@@ -387,7 +469,12 @@ function emitHttpResponse(
     type,
     contentTypes: response.contentTypes,
     defaultContentType: response.defaultContentType ?? "application/json",
-    resultProperty: method?.response.resultPath,
+    resultProperty: getWireNameWithDiagnostics(
+      context,
+      method?.response.resultSegments,
+      "invalid-lro-result",
+      method,
+    ),
   };
 }
 

package/emitter/src/lib.ts

@@ -83,6 +83,30 @@ const libDef = {
         default: "Can't generate Python SDK since no client defined in typespec file.",
       },
     },
+    "invalid-paging-items": {
+      severity: "warning",
+      messages: {
+        default: paramMessage`No valid paging items for operation '${"operationId"}'.`,
+      },
+    },
+    "invalid-next-link": {
+      severity: "warning",
+      messages: {
+        default: paramMessage`No valid next link for operation '${"operationId"}'.`,
+      },
+    },
+    "invalid-lro-result": {
+      severity: "warning",
+      messages: {
+        default: paramMessage`No valid LRO result for operation '${"operationId"}'.`,
+      },
+    },
+    "invalid-continuation-token": {
+      severity: "warning",
+      messages: {
+        default: paramMessage`No valid continuation token in '${"direction"}' for operation '${"operationId"}'.`,
+      },
+    },
   },
   emitter: {
     options: EmitterOptionsSchema as JSONSchemaType<PythonEmitterOptions>,

package/emitter/src/types.ts

@@ -294,7 +294,7 @@ function emitModel<TServiceOperation extends SdkServiceOperation>(
     usage: type.usage,
     isXml: type.usage & UsageFlags.Xml ? true : false,
     xmlMetadata: getXmlMetadata(type),
-    clientNamespace: getClientNamespace(context, type.clientNamespace),
+    clientNamespace: getClientNamespace(context, type.namespace),
   };
 
   typesMap.set(type, newValue);
@@ -341,13 +341,16 @@ function emitEnum<TServiceOperation extends SdkServiceOperation>(
     if (!type.isFixed) {
       types.push(emitBuiltInType(type.valueType));
     }
-    return {
+
+    const newValue = {
       description: "",
       internal: true,
       type: "combined",
       types,
       xmlMetadata: {},
     };
+    typesMap.set(type, newValue);
+    return newValue;
   }
   const values: Record<string, any>[] = [];
   const name = type.name;
@@ -361,7 +364,7 @@ function emitEnum<TServiceOperation extends SdkServiceOperation>(
     values,
     xmlMetadata: {},
     crossLanguageDefinitionId: type.crossLanguageDefinitionId,
-    clientNamespace: getClientNamespace(context, type.clientNamespace),
+    clientNamespace: getClientNamespace(context, type.namespace),
   };
   for (const value of type.values) {
     newValue.values.push(emitEnumMember(value, newValue));
@@ -479,7 +482,7 @@ function emitUnion<TServiceOperation extends SdkServiceOperation>(
     type: "combined",
     types: type.variantTypes.map((x) => getType(context, x)),
     xmlMetadata: {},
-    clientNamespace: getClientNamespace(context, type.clientNamespace),
+    clientNamespace: getClientNamespace(context, type.namespace),
   });
 }