@typespec/http-client-python 0.36.0 → 0.37.0

package/emitter/test/streaming.test.ts

@@ -0,0 +1,155 @@
+import { strictEqual } from "assert";
+import { describe, it } from "vitest";
+import {
+  getStructuredStreamKind,
+  isStructuredStreamType,
+  partitionSSEEvents,
+} from "../src/http.js";
+
+describe("typespec-python: structured streaming", () => {
+  it("treats model and union payloads as structured", () => {
+    strictEqual(isStructuredStreamType({ kind: "model" } as any), true);
+    strictEqual(isStructuredStreamType({ kind: "union" } as any), true);
+  });
+
+  it("unwraps nullable payloads", () => {
+    strictEqual(isStructuredStreamType({ kind: "nullable", type: { kind: "model" } } as any), true);
+    strictEqual(
+      isStructuredStreamType({ kind: "nullable", type: { kind: "bytes" } } as any),
+      false,
+    );
+  });
+
+  it("treats bare byte/string payloads as unstructured", () => {
+    strictEqual(isStructuredStreamType({ kind: "bytes" } as any), false);
+    strictEqual(isStructuredStreamType({ kind: "string" } as any), false);
+  });
+
+  it("detects the stream protocol explicitly", () => {
+    strictEqual(getStructuredStreamKind({ sseMetadata: { events: [] } } as any), "sse");
+    strictEqual(
+      getStructuredStreamKind({
+        streamMetadata: { contentTypes: ["text/event-stream; charset=utf-8"] },
+      } as any),
+      "sse",
+    );
+    strictEqual(
+      getStructuredStreamKind({
+        streamMetadata: { contentTypes: ["application/jsonl"] },
+      } as any),
+      "jsonl",
+    );
+    strictEqual(
+      getStructuredStreamKind({
+        streamMetadata: { contentTypes: ["application/json"] },
+      } as any),
+      undefined,
+    );
+  });
+
+  describe("terminal-event partitioning", () => {
+    const identity = (payloadType: any) => payloadType;
+    const model = (name: string) => ({ kind: "model", name });
+    const constant = (value: string) => ({ kind: "constant", value });
+
+    it("keeps a nameless string-constant `[DONE]` as a drop-and-stop sentinel", () => {
+      const created = model("ResponseCreated");
+      const done = constant("[DONE]");
+      const { events, terminalEvent } = partitionSSEEvents(
+        [
+          {
+            eventType: "response.created",
+            isTerminalEvent: false,
+            type: created,
+            payloadType: created,
+          },
+          { eventType: undefined, isTerminalEvent: true, type: done, payloadType: done },
+        ] as any,
+        identity,
+      );
+      // The sentinel is NOT a dispatch event; it only sets `terminalEvent`.
+      strictEqual(terminalEvent, "[DONE]");
+      strictEqual(events.length, 1);
+      strictEqual(events[0].eventType, "response.created");
+      strictEqual(events[0].isTerminal, undefined);
+    });
+
+    it("keeps named / model `@terminalEvent`s in the dispatch table as yield-and-stop events", () => {
+      const created = model("ResponseCreated");
+      const completed = model("ResponseCompleted");
+      const errored = model("StreamError");
+      const { events, terminalEvent } = partitionSSEEvents(
+        [
+          {
+            eventType: "response.created",
+            isTerminalEvent: false,
+            type: created,
+            payloadType: created,
+          },
+          {
+            eventType: "response.completed",
+            isTerminalEvent: true,
+            type: completed,
+            payloadType: completed,
+          },
+          { eventType: "error", isTerminalEvent: true, type: errored, payloadType: errored },
+        ] as any,
+        identity,
+      );
+      // No bare sentinel: the two terminals carry payloads, so they stay in `events`.
+      strictEqual(terminalEvent, undefined);
+      strictEqual(events.length, 3);
+      strictEqual(events[0].isTerminal, undefined);
+      strictEqual(events[1].eventType, "response.completed");
+      strictEqual(events[1].isTerminal, true);
+      strictEqual(events[1].payloadType, completed);
+      strictEqual(events[2].eventType, "error");
+      strictEqual(events[2].isTerminal, true);
+      strictEqual(events[2].payloadType, errored);
+    });
+
+    it("supports a sentinel and named terminals together", () => {
+      const delta = model("ResponseDelta");
+      const completed = model("ResponseCompleted");
+      const done = constant("[DONE]");
+      const { events, terminalEvent } = partitionSSEEvents(
+        [
+          { eventType: "response.delta", isTerminalEvent: false, type: delta, payloadType: delta },
+          {
+            eventType: "response.completed",
+            isTerminalEvent: true,
+            type: completed,
+            payloadType: completed,
+          },
+          { eventType: undefined, isTerminalEvent: true, type: done, payloadType: done },
+        ] as any,
+        identity,
+      );
+      strictEqual(terminalEvent, "[DONE]");
+      strictEqual(events.length, 2);
+      strictEqual(events[0].isTerminal, undefined);
+      strictEqual(events[1].eventType, "response.completed");
+      strictEqual(events[1].isTerminal, true);
+    });
+
+    it("emits the payload metadata for event envelopes", () => {
+      const envelope = model("Envelope");
+      const payload = { kind: "string" };
+      const { events } = partitionSSEEvents(
+        [
+          {
+            eventType: "withEnvelope",
+            isTerminalEvent: false,
+            type: envelope,
+            payloadType: payload,
+            isEventEnvelope: true,
+            payloadContentType: "text/plain",
+          },
+        ] as any,
+        identity,
+      );
+      strictEqual(events[0].payloadType, payload);
+      strictEqual(events[0].payloadContentType, "text/plain");
+    });
+  });
+});

package/emitter/test/{external-process.test.ts → yaml-utils.test.ts}

@@ -1,9 +1,9 @@
 import { ok, strictEqual } from "assert";
 import { load } from "js-yaml";
 import { describe, it } from "vitest";
-import { dumpCodeModelToYaml } from "../src/external-process.js";
+import { dumpCodeModelToYaml } from "../src/yaml-utils.js";
 
-describe("typespec-python: external-process", () => {
+describe("typespec-python: yaml-utils", () => {
   // The Python generator parses the emitted YAML with PyYAML (YAML 1.1), where a plain
   // scalar such as `2020_01_01` is interpreted as the integer `20200101`. js-yaml dumps
   // using YAML 1.2 rules and would otherwise leave such string scalars unquoted, so we

package/eng/scripts/ci/regenerate-common.ts

@@ -69,11 +69,7 @@ export interface BuildTaskGroupsOptions {
 
 // ---- Public constants ----
 
-export const SKIP_SPECS: string[] = [
-  "type/file",
-  // LroPaging.postPagingLroWithBody generates begin_post_paging_lro_with_body with pylint R0915 (52/50).
-  "azure/resource-manager/operation-templates",
-];
+export const SKIP_SPECS: string[] = ["type/file"];
 
 export const SpecialFlags: Record<string, Record<string, any>> = {
   azure: {

package/eng/scripts/ci/run_pylint.py

@@ -52,6 +52,7 @@ def _single_dir_pylint(mod):
     # narrow as possible: only this package, only the offending check.
     per_package_disables = {
         "azure-client-generator-core-exact-name": ["client-incorrect-naming-convention"],
+        "azure-resource-manager-operation-templates": ["too-many-statements"],
     }
     extra_disables = per_package_disables.get(mod.name)
     if extra_disables:

package/generator/build/lib/pygen/codegen/models/code_model.py

@@ -279,12 +279,22 @@ class CodeModel:  # pylint: disable=too-many-public-methods, disable=too-many-in
             self.need_utils_utils(async_mode, client_namespace)
             or self.need_utils_serialization
             or self.options["models-mode"] == "dpg"
+            or self.has_structured_stream
         )
 
     @property
     def need_utils_serialization(self) -> bool:
         return not self.options["client-side-validation"]
 
+    @property
+    def has_structured_stream(self) -> bool:
+        return any(
+            op.has_structured_stream_response
+            for client in self.clients
+            for og in client.operation_groups
+            for op in og.operations
+        )
+
     def need_utils_utils(self, async_mode: bool, client_namespace: str) -> bool:
         return (
             self.need_utils_form_data(async_mode, client_namespace)
@@ -395,22 +405,26 @@ class CodeModel:  # pylint: disable=too-many-public-methods, disable=too-many-in
     def _sort_model_types_helper(
         self,
         current: ModelType,
-        seen_schema_names: set[str],
+        seen_schema_keys: set[tuple[str, str]],
         seen_schema_yaml_ids: set[int],
     ):
         if current.id in seen_schema_yaml_ids:
             return []
-        if current.name in seen_schema_names:
-            raise ValueError(f"We have already generated a schema with name {current.name}")
+        current_schema_key = (current.client_namespace, current.name)
+        if current_schema_key in seen_schema_keys:
+            raise ValueError(
+                f"We have already generated a schema with name {current.name} "
+                f"in namespace {current.client_namespace}"
+            )
         ancestors = [current]
         if current.parents:
             for parent in current.parents:
                 if parent.id in seen_schema_yaml_ids:
                     continue
-                seen_schema_names.add(current.name)
+                seen_schema_keys.add(current_schema_key)
                 seen_schema_yaml_ids.add(current.id)
-                ancestors = self._sort_model_types_helper(parent, seen_schema_names, seen_schema_yaml_ids) + ancestors
-        seen_schema_names.add(current.name)
+                ancestors = self._sort_model_types_helper(parent, seen_schema_keys, seen_schema_yaml_ids) + ancestors
+        seen_schema_keys.add(current_schema_key)
         seen_schema_yaml_ids.add(current.id)
         return ancestors
 
@@ -420,11 +434,11 @@ class CodeModel:  # pylint: disable=too-many-public-methods, disable=too-many-in
         :return: None
         :rtype: None
         """
-        seen_schema_names: set[str] = set()
+        seen_schema_keys: set[tuple[str, str]] = set()
         seen_schema_yaml_ids: set[int] = set()
         sorted_object_schemas: list[ModelType] = []
         for schema in sorted(self.model_types, key=lambda x: x.name.lower()):
-            sorted_object_schemas.extend(self._sort_model_types_helper(schema, seen_schema_names, seen_schema_yaml_ids))
+            sorted_object_schemas.extend(self._sort_model_types_helper(schema, seen_schema_keys, seen_schema_yaml_ids))
         self.model_types = sorted_object_schemas
 
     @property

package/generator/build/lib/pygen/codegen/models/enum_type.py

@@ -203,6 +203,10 @@ class EnumType(BaseType):
                 model_alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace)
                 module_name = f"{model_alias}."
             file_name = f"{self.code_model.enums_filename}." if self.internal else ""
+            if serialize_namespace_type == NamespaceType.TYPES_FILE:
+                # In types.py the enum symbol is imported directly (bare name), so no ``_enums.``
+                # module prefix even for internal enums — the prefix would be an undefined name.
+                file_name = ""
             model_name = module_name + file_name + self.name
             # we don't need quoted annotation in operation files, and need it in model folder files.
             if not kwargs.get("is_operation_file", False):
@@ -305,9 +309,17 @@ class EnumType(BaseType):
                         typing_section=TypingSection.REGULAR,
                     )
                 elif serialize_namespace_type == NamespaceType.TYPES_FILE:
-                    # Import enum name directly to avoid dotted forward refs in TypedDict annotations
+                    # Import the enum symbol directly to avoid dotted forward refs in TypedDict
+                    # annotations. Internal enums are not re-exported from the public ``models``
+                    # package, so import them from the private ``_enums`` submodule instead — the
+                    # bare-symbol import (rather than the ``_enums`` module) also avoids name
+                    # collisions when internal enums come from several sibling namespaces.
+                    module_name = f"models.{self.code_model.enums_filename}" if self.internal else "models"
+                    enums_module = self.code_model.get_relative_import_path(
+                        serialize_namespace, self.client_namespace, module_name=module_name
+                    )
                     file_import.add_submodule_import(
-                        f"{relative_path}models" if relative_path != "." else ".models",
+                        enums_module,
                         self.name,
                         ImportType.LOCAL,
                         typing_section=TypingSection.TYPING,

package/generator/build/lib/pygen/codegen/models/operation.py

@@ -104,6 +104,11 @@ class OperationBase(  # pylint: disable=too-many-public-methods,too-many-instanc
             else self.has_stream_response
         )
 
+    @property
+    def has_structured_stream_response(self) -> bool:
+        """Whether any success response is a structured (JSONL / SSE) stream returning Stream[T]."""
+        return any(getattr(r, "is_structured_stream", False) for r in self.responses)
+
     @property
     def has_form_data_body(self):
         return self.parameters.has_form_data_body
@@ -506,7 +511,7 @@ class OperationBase(  # pylint: disable=too-many-public-methods,too-many-instanc
 
     @property
     def has_stream_response(self) -> bool:
-        return any(r.is_stream_response for r in self.responses)
+        return any(r.is_stream_response or r.is_structured_stream for r in self.responses)
 
     @classmethod
     def get_request_builder(cls, yaml_data: dict[str, Any], client: "Client"):

package/generator/build/lib/pygen/codegen/models/response.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
+from dataclasses import dataclass
 from typing import Optional, Any, TYPE_CHECKING, Union
 
 from .base import BaseModel
@@ -17,6 +18,53 @@ from .combined_type import CombinedType
 if TYPE_CHECKING:
     from .code_model import CodeModel
 
+DEFAULT_SSE_EVENT_TYPE = "message"
+
+
+@dataclass(frozen=True)
+class StreamingEvent:
+    event_type: Optional[str]
+    payload_type: BaseType
+    payload_content_type: Optional[str]
+    is_terminal: bool
+
+
+def get_streaming_event_discriminator(
+    events: list[StreamingEvent],
+) -> Optional[tuple[str, list[tuple[str, StreamingEvent]]]]:
+    discriminator_name: Optional[str] = None
+    variants: list[tuple[str, StreamingEvent]] = []
+    values: set[str] = set()
+    for event in events:
+        discriminator_property = getattr(event.payload_type, "discriminator_property", None)
+        discriminator_value = getattr(event.payload_type, "discriminator_value", None)
+        if discriminator_property is None or discriminator_value is None:
+            return None
+        if discriminator_name is None:
+            discriminator_name = discriminator_property.wire_name
+        elif discriminator_property.wire_name != discriminator_name:
+            return None
+        if discriminator_value in values:
+            return None
+        values.add(discriminator_value)
+        variants.append((discriminator_value, event))
+    if discriminator_name is None:
+        return None
+    return discriminator_name, variants
+
+
+def _get_terminal_event_names(events: list[StreamingEvent]) -> list[str]:
+    unnamed_events = [event for event in events if event.event_type is None]
+    terminal_event_names: list[str] = []
+    for event in events:
+        if not event.is_terminal:
+            continue
+        if event.event_type is not None:
+            terminal_event_names.append(event.event_type)
+        elif len(unnamed_events) == 1:
+            terminal_event_names.append(DEFAULT_SSE_EVENT_TYPE)
+    return list(dict.fromkeys(terminal_event_names))
+
 
 class ResponseHeader(BaseModel):
     def __init__(
@@ -58,6 +106,25 @@ class Response(BaseModel):
         self.type = type
         self.nullable = yaml_data.get("nullable")
         self.default_content_type = yaml_data.get("defaultContentType")
+        streaming = yaml_data.get("streaming")
+        self.streaming_kind: Optional[str] = streaming["kind"] if streaming else None
+        self.streaming_events: list[StreamingEvent] = []
+        self.terminal_event: Optional[str] = streaming.get("terminalEvent") if streaming else None
+        # Named / model ``@terminalEvent`` events: deserialized and yielded like any other event,
+        # then iteration stops. The bare string-constant sentinel (``terminal_event``)
+        # stops WITHOUT yielding and is matched on event ``data`` instead of the event name.
+        self.terminal_event_names: list[str] = []
+        if streaming:
+            self.streaming_events = [
+                StreamingEvent(
+                    event_type=event.get("eventType"),
+                    payload_type=self.code_model.lookup_type(id(event["payloadType"])),
+                    payload_content_type=event.get("payloadContentType"),
+                    is_terminal=event.get("isTerminal", False),
+                )
+                for event in streaming.get("events", [])
+            ]
+            self.terminal_event_names = _get_terminal_event_names(self.streaming_events)
 
     @property
     def result_property(self) -> str:
@@ -92,12 +159,48 @@ class Response(BaseModel):
         )
         return retval
 
+    @property
+    def is_structured_stream(self) -> bool:
+        """Is the response a structured (JSONL / SSE) stream rendered as Stream[T] / AsyncStream[T]."""
+        return self.streaming_kind is not None
+
+    def stream_class_name(self, async_mode: bool) -> str:
+        return "AsyncStream" if async_mode else "Stream"
+
+    @property
+    def stream_item_type(self) -> Optional[BaseType]:
+        event_item_types = list(dict.fromkeys(event.payload_type for event in self.streaming_events))
+        if len(event_item_types) == 1:
+            return event_item_types[0]
+        if event_item_types:
+            return CombinedType({"type": "combined"}, self.code_model, event_item_types)
+        return self.type
+
     def serialization_type(self, **kwargs: Any) -> str:
         if self.type:
             return self.type.serialization_type(**kwargs)
         return "None"
 
+    def stream_item_annotation(self, **kwargs: Any) -> str:
+        """Valid type expression for a structured stream's item type.
+
+        A named ``CombinedType`` (``@events`` union) renders its ``type_annotation`` as the
+        ``_unions.<Name>`` alias, which is a module-level variable and therefore rejected by
+        pyright/mypy inside ``Stream[...]`` ("Variable not allowed in type expression"). Expand
+        the union inline (``Union[Model, ...]`` / the single member) so the annotation is a
+        valid type expression.
+        """
+        item_type = self.stream_item_type
+        if isinstance(item_type, CombinedType):
+            return item_type.type_definition(**kwargs)
+        return item_type.type_annotation(**kwargs) if item_type else "None"
+
     def type_annotation(self, **kwargs: Any) -> str:
+        if self.is_structured_stream and self.type:
+            kwargs["is_operation_file"] = True
+            kwargs["is_response"] = True
+            stream_class = self.stream_class_name(kwargs.get("async_mode", False))
+            return f"{stream_class}[{self.stream_item_annotation(**kwargs)}]"
         if self.type:
             kwargs["is_operation_file"] = True
             kwargs["is_response"] = True
@@ -109,30 +212,63 @@ class Response(BaseModel):
 
     def docstring_text(self, **kwargs: Any) -> str:
         kwargs["is_response"] = True
+        if self.is_structured_stream and self.type:
+            stream_class = self.stream_class_name(kwargs.get("async_mode", False))
+            item_type = self.stream_item_type or self.type
+            return f"An instance of {stream_class} that iterates over {item_type.docstring_text(**kwargs)}"
         if self.nullable and self.type:
             return f"{self.type.docstring_text(**kwargs)} or None"
         return self.type.docstring_text(**kwargs) if self.type else "None"
 
     def docstring_type(self, **kwargs: Any) -> str:
         kwargs["is_response"] = True
+        if self.is_structured_stream and self.type:
+            stream_class = self.stream_class_name(kwargs.get("async_mode", False))
+            item_type = (self.stream_item_type or self.type).docstring_type(**kwargs)
+            return f"~{self.code_model.namespace}.{stream_class}[{item_type}]"
         if self.nullable and self.type:
             return f"{self.type.docstring_type(**kwargs)} or None"
         return self.type.docstring_type(**kwargs) if self.type else "None"
 
     def imports(self, **kwargs: Any) -> FileImport:
         file_import = FileImport(self.code_model)
-        if self.type:
-            file_import.merge(self.type.imports(**kwargs))
+        item_type = self.stream_item_type if self.is_structured_stream else self.type
+        # For a structured stream whose item type is a named ``@events`` union, the annotation
+        # is expanded inline (see ``stream_item_annotation``), so import the union member types
+        # rather than the ``_unions`` alias.
+        if self.is_structured_stream and isinstance(item_type, CombinedType):
+            for member in item_type.types:
+                file_import.merge(member.imports(**kwargs))
+            # ``Union`` is only needed when the inline expansion actually yields a union of
+            # 2+ distinct member types (a single member collapses to that member; a union of
+            # only literals collapses to a single ``Literal[...]``).
+            distinct = list(dict.fromkeys(m.type_annotation(**kwargs) for m in item_type.types))
+            all_constant = all(t.type == "constant" for t in item_type.types)
+            if len(distinct) > 1 and not all_constant:
+                file_import.add_submodule_import("typing", "Union", ImportType.STDLIB)
+        elif item_type:
+            file_import.merge(item_type.imports(**kwargs))
+            if not self.is_structured_stream and isinstance(item_type, CombinedType) and item_type.name:
+                serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace)
+                file_import.add_submodule_import(
+                    self.code_model.get_relative_import_path(serialize_namespace),
+                    "_unions",
+                    ImportType.LOCAL,
+                    TypingSection.TYPING,
+                )
         if self.nullable:
             file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB)
-        if isinstance(self.type, CombinedType) and self.type.name:
+        if self.is_structured_stream:
+            stream_class = self.stream_class_name(kwargs.get("async_mode", False))
             serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace)
-            file_import.add_submodule_import(
-                self.code_model.get_relative_import_path(serialize_namespace),
-                "_unions",
-                ImportType.LOCAL,
-                TypingSection.TYPING,
+            relative_path = self.code_model.get_relative_import_path(
+                serialize_namespace, module_name="_utils.streaming_base"
             )
+            file_import.add_submodule_import(relative_path, stream_class, ImportType.LOCAL)
+            if self.streaming_kind == "sse":
+                file_import.add_import("json", ImportType.STDLIB)
+                for event in self.streaming_events:
+                    file_import.merge(event.payload_type.imports(**kwargs))
         return file_import
 
     def _get_import_type(self, input_path: str) -> ImportType:
@@ -143,6 +279,14 @@ class Response(BaseModel):
 
     @classmethod
     def from_yaml(cls, yaml_data: dict[str, Any], code_model: "CodeModel") -> "Response":
+        streaming = yaml_data.get("streaming")
+        if streaming:
+            return cls(
+                yaml_data=yaml_data,
+                code_model=code_model,
+                headers=[ResponseHeader.from_yaml(header, code_model) for header in yaml_data["headers"]],
+                type=code_model.lookup_type(id(streaming["itemType"])),
+            )
         type = code_model.lookup_type(id(yaml_data["type"])) if yaml_data.get("type") else None
         # use ByteIteratorType if we are returning a binary type
         default_content_type = yaml_data.get("defaultContentType", "application/json")

package/generator/build/lib/pygen/codegen/serializers/__init__.py

@@ -525,6 +525,13 @@ class JinjaSerializer(ReaderAndWriter):
                 general_serializer.serialize_model_base_file(),
             )
 
+        # write _utils/streaming_base.py (vendored Stream/AsyncStream + JSONL/SSE decoders)
+        if self.code_model.has_structured_stream:
+            self.write_file(
+                utils_folder_path / Path("streaming_base.py"),
+                general_serializer.serialize_streaming_base_file(),
+            )
+
     def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str) -> None:
         root_dir = self.code_model.get_root_dir()
         generation_dir = self.code_model.get_generation_dir(namespace)