@typespec/http-client-python 0.36.0 → 0.37.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/emitter/emitter.d.ts.map +1 -1
- package/dist/emitter/emitter.js +6 -5
- package/dist/emitter/emitter.js.map +1 -1
- package/dist/emitter/external-process.d.ts +0 -14
- package/dist/emitter/external-process.d.ts.map +1 -1
- package/dist/emitter/external-process.js +1 -17
- package/dist/emitter/external-process.js.map +1 -1
- package/dist/emitter/http.d.ts +44 -1
- package/dist/emitter/http.d.ts.map +1 -1
- package/dist/emitter/http.js +96 -0
- package/dist/emitter/http.js.map +1 -1
- package/dist/emitter/lib.d.ts +5 -5
- package/dist/emitter/lib.js +2 -2
- package/dist/emitter/lib.js.map +1 -1
- package/dist/emitter/yaml-utils.d.ts +19 -0
- package/dist/emitter/yaml-utils.d.ts.map +1 -0
- package/dist/emitter/yaml-utils.js +22 -0
- package/dist/emitter/yaml-utils.js.map +1 -0
- package/emitter/src/emitter.ts +7 -5
- package/emitter/src/external-process.ts +1 -18
- package/emitter/src/http.ts +154 -0
- package/emitter/src/lib.ts +2 -2
- package/emitter/src/yaml-utils.ts +22 -0
- package/emitter/temp/tsconfig.tsbuildinfo +1 -1
- package/emitter/test/emitter.test.ts +32 -0
- package/emitter/test/streaming.test.ts +155 -0
- package/emitter/test/test-host.ts +83 -0
- package/emitter/test/{external-process.test.ts → yaml-utils.test.ts} +2 -2
- package/eng/scripts/ci/regenerate-common.ts +1 -5
- package/eng/scripts/ci/run_pylint.py +1 -0
- package/eng/scripts/setup/__pycache__/package_manager.cpython-311.pyc +0 -0
- package/eng/scripts/setup/__pycache__/venvtools.cpython-311.pyc +0 -0
- package/generator/build/lib/pygen/codegen/models/code_model.py +22 -8
- package/generator/build/lib/pygen/codegen/models/enum_type.py +14 -2
- package/generator/build/lib/pygen/codegen/models/operation.py +6 -1
- package/generator/build/lib/pygen/codegen/models/response.py +152 -8
- package/generator/build/lib/pygen/codegen/serializers/__init__.py +7 -0
- package/generator/build/lib/pygen/codegen/serializers/builder_serializer.py +197 -58
- package/generator/build/lib/pygen/codegen/serializers/general_serializer.py +4 -0
- package/generator/build/lib/pygen/codegen/serializers/import_serializer.py +27 -0
- package/generator/build/lib/pygen/codegen/serializers/types_serializer.py +40 -3
- package/generator/build/lib/pygen/codegen/templates/streaming_base.py.jinja2 +767 -0
- package/generator/build/lib/pygen/preprocess/__init__.py +2 -0
- package/generator/component-detection-pip-report.json +5 -5
- package/generator/dist/pygen-0.1.0-py3-none-any.whl +0 -0
- package/generator/pygen/codegen/models/code_model.py +22 -8
- package/generator/pygen/codegen/models/enum_type.py +14 -2
- package/generator/pygen/codegen/models/operation.py +6 -1
- package/generator/pygen/codegen/models/response.py +152 -8
- package/generator/pygen/codegen/serializers/__init__.py +7 -0
- package/generator/pygen/codegen/serializers/builder_serializer.py +197 -58
- package/generator/pygen/codegen/serializers/general_serializer.py +4 -0
- package/generator/pygen/codegen/serializers/import_serializer.py +27 -0
- package/generator/pygen/codegen/serializers/types_serializer.py +40 -3
- package/generator/pygen/codegen/templates/streaming_base.py.jinja2 +767 -0
- package/generator/pygen/preprocess/__init__.py +2 -0
- package/generator/pygen.egg-info/SOURCES.txt +1 -0
- package/package.json +6 -6
|
@@ -0,0 +1,767 @@
|
|
|
1
|
+
# coding=utf-8
|
|
2
|
+
{% if code_model.license_header %}
|
|
3
|
+
{{ code_model.license_header }}
|
|
4
|
+
{% endif %}
|
|
5
|
+
# pylint: disable=line-too-long,useless-suppression,unnecessary-ellipsis
|
|
6
|
+
# --------------------------------------------------------------------------
|
|
7
|
+
# This file is vendored from the core streaming runtime
|
|
8
|
+
# ({{ code_model.core_library }}.streaming). It provides the Stream / AsyncStream
|
|
9
|
+
# helpers (plus the JSONL / SSE decoders and event types) used by generated
|
|
10
|
+
# structured-streaming operations, so the generated package does not take a hard
|
|
11
|
+
# dependency on a core runtime that ships the streaming helpers. Do not edit by hand.
|
|
12
|
+
# --------------------------------------------------------------------------
|
|
13
|
+
import codecs
|
|
14
|
+
import json
|
|
15
|
+
from contextlib import aclosing
|
|
16
|
+
from types import TracebackType
|
|
17
|
+
from typing import (
|
|
18
|
+
Any,
|
|
19
|
+
AsyncGenerator,
|
|
20
|
+
AsyncIterator,
|
|
21
|
+
Callable,
|
|
22
|
+
Generator,
|
|
23
|
+
Iterator,
|
|
24
|
+
List,
|
|
25
|
+
Optional,
|
|
26
|
+
Protocol,
|
|
27
|
+
Sequence,
|
|
28
|
+
Type,
|
|
29
|
+
TypeVar,
|
|
30
|
+
cast,
|
|
31
|
+
runtime_checkable,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
from typing_extensions import Self
|
|
35
|
+
|
|
36
|
+
from {{ code_model.core_library }}.rest import AsyncHttpResponse, HttpResponse
|
|
37
|
+
|
|
38
|
+
DecodedType = TypeVar("DecodedType")
|
|
39
|
+
ReturnType_co = TypeVar("ReturnType_co", covariant=True)
|
|
40
|
+
T_co = TypeVar("T_co", covariant=True)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@runtime_checkable
|
|
44
|
+
class StreamDecoder(Protocol[T_co]):
|
|
45
|
+
"""Protocol for stream decoders."""
|
|
46
|
+
|
|
47
|
+
def iter_events(self, iter_bytes: Iterator[bytes]) -> Iterator[T_co]:
|
|
48
|
+
"""Iterate over events from a byte iterator.
|
|
49
|
+
|
|
50
|
+
:param iter_bytes: An iterator of byte chunks.
|
|
51
|
+
:type iter_bytes: Iterator[bytes]
|
|
52
|
+
:return: An iterator of decoded data.
|
|
53
|
+
:rtype: Iterator[DecodedType_co]
|
|
54
|
+
"""
|
|
55
|
+
...
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@runtime_checkable
|
|
59
|
+
class AsyncStreamDecoder(Protocol[T_co]):
|
|
60
|
+
"""Protocol for async stream decoders."""
|
|
61
|
+
|
|
62
|
+
# Why this isn't async def: https://mypy.readthedocs.io/en/stable/more_types.html#asynchronous-iterators
|
|
63
|
+
def aiter_events(self, iter_bytes: AsyncIterator[bytes]) -> AsyncIterator[T_co]:
|
|
64
|
+
"""Asynchronously iterate over events from a byte iterator.
|
|
65
|
+
|
|
66
|
+
:param iter_bytes: An asynchronous iterator of byte chunks.
|
|
67
|
+
:type iter_bytes: AsyncIterator[bytes]
|
|
68
|
+
:return: An asynchronous iterator of decoded data.
|
|
69
|
+
:rtype: AsyncIterator[DecodedType_co]
|
|
70
|
+
"""
|
|
71
|
+
...
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class JSONLEvent:
|
|
75
|
+
"""A single JSON Lines (JSONL) event.
|
|
76
|
+
|
|
77
|
+
:ivar data: The raw JSONL record.
|
|
78
|
+
:vartype data: str or None
|
|
79
|
+
:keyword data: The raw JSONL record.
|
|
80
|
+
:paramtype data: str or None
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
def __init__(
|
|
84
|
+
self,
|
|
85
|
+
*,
|
|
86
|
+
data: Optional[str] = None,
|
|
87
|
+
) -> None:
|
|
88
|
+
self.data = data
|
|
89
|
+
|
|
90
|
+
def json(self) -> Any:
|
|
91
|
+
"""Parse the event data as JSON.
|
|
92
|
+
|
|
93
|
+
:return: The parsed JSON value.
|
|
94
|
+
:rtype: Any
|
|
95
|
+
"""
|
|
96
|
+
return json.loads(cast(str, self.data))
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class _JSONLLineFramer:
|
|
100
|
+
"""Incremental JSONL line framer with linear-time behavior.
|
|
101
|
+
|
|
102
|
+
JSONL records are separated only by ``"\\n"`` (tolerating ``"\\r\\n"``). Unlike
|
|
103
|
+
``str.splitlines()``, other Unicode boundaries (``\\v``, ``\\f``, ``\\x1c``-``\\x1e``,
|
|
104
|
+
``\\x85``, ``\\u2028``, ``\\u2029``) are preserved because they are valid inside a JSONL
|
|
105
|
+
record's string value.
|
|
106
|
+
|
|
107
|
+
Rather than re-concatenating and re-splitting the whole pending record on every network chunk
|
|
108
|
+
(which is O(n^2) for a single long record fragmented across many chunks), the unfinished record
|
|
109
|
+
is held as a list of fragments and joined only when a terminator arrives (or at EOF). Only the
|
|
110
|
+
newly decoded text is scanned per chunk, giving O(total) behavior.
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
def __init__(self) -> None:
|
|
114
|
+
# Fragments of the current, not-yet-terminated record. Never contains a "\n".
|
|
115
|
+
self._parts: List[str] = []
|
|
116
|
+
|
|
117
|
+
def push(self, text: str) -> List[str]:
|
|
118
|
+
"""Feed newly decoded text and return any completed records.
|
|
119
|
+
|
|
120
|
+
:param text: Newly decoded text from a single chunk.
|
|
121
|
+
:type text: str
|
|
122
|
+
:return: Completed records produced by this chunk (may be empty).
|
|
123
|
+
:rtype: list[str]
|
|
124
|
+
"""
|
|
125
|
+
if not text:
|
|
126
|
+
return []
|
|
127
|
+
|
|
128
|
+
segments = text.split("\n")
|
|
129
|
+
# Fast path: no line terminator, so this is a continuation of the current record. Stash the
|
|
130
|
+
# fragment without joining or rescanning the accumulated tail.
|
|
131
|
+
if len(segments) == 1:
|
|
132
|
+
self._parts.append(text)
|
|
133
|
+
return []
|
|
134
|
+
|
|
135
|
+
first = "".join(self._parts) + segments[0]
|
|
136
|
+
# All but the final segment are complete records (terminated by "\n"). Strip a trailing "\r"
|
|
137
|
+
# to tolerate "\r\n" line endings.
|
|
138
|
+
completed = [line[:-1] if line.endswith("\r") else line for line in [first, *segments[1:-1]]]
|
|
139
|
+
self._parts = [segments[-1]]
|
|
140
|
+
return completed
|
|
141
|
+
|
|
142
|
+
def flush(self, extra: str = "") -> List[str]:
|
|
143
|
+
"""Return the final unterminated record, if any, at end of stream.
|
|
144
|
+
|
|
145
|
+
:param extra: Trailing text from finalizing the incremental decoder.
|
|
146
|
+
:type extra: str
|
|
147
|
+
:return: The final record, if non-empty.
|
|
148
|
+
:rtype: list[str]
|
|
149
|
+
"""
|
|
150
|
+
tail = "".join(self._parts) + extra
|
|
151
|
+
self._parts = []
|
|
152
|
+
if not tail:
|
|
153
|
+
return []
|
|
154
|
+
return [tail[:-1] if tail.endswith("\r") else tail]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def iter_lines(iter_bytes: Iterator[bytes]) -> Iterator[str]:
|
|
158
|
+
"""Iterate over lines from a byte iterator.
|
|
159
|
+
|
|
160
|
+
:param iter_bytes: An iterator of byte chunks.
|
|
161
|
+
:type iter_bytes: Iterator[bytes]
|
|
162
|
+
:rtype: Iterator[str]
|
|
163
|
+
:return: An iterator of lines.
|
|
164
|
+
"""
|
|
165
|
+
decoder = codecs.getincrementaldecoder("utf-8")()
|
|
166
|
+
framer = _JSONLLineFramer()
|
|
167
|
+
|
|
168
|
+
for chunk in iter_bytes:
|
|
169
|
+
yield from framer.push(decoder.decode(chunk))
|
|
170
|
+
|
|
171
|
+
yield from framer.flush(decoder.decode(b"", final=True))
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
async def aiter_lines(iter_bytes: AsyncIterator[bytes]) -> AsyncGenerator[str, None]:
|
|
175
|
+
"""Iterate over lines from a byte iterator.
|
|
176
|
+
|
|
177
|
+
:param iter_bytes: An iterator of byte chunks.
|
|
178
|
+
:type iter_bytes: Iterator[bytes]
|
|
179
|
+
:rtype: Iterator[str]
|
|
180
|
+
:return: An iterator of lines.
|
|
181
|
+
"""
|
|
182
|
+
decoder = codecs.getincrementaldecoder("utf-8")()
|
|
183
|
+
framer = _JSONLLineFramer()
|
|
184
|
+
|
|
185
|
+
try:
|
|
186
|
+
async for chunk in iter_bytes:
|
|
187
|
+
for line in framer.push(decoder.decode(chunk)):
|
|
188
|
+
yield line
|
|
189
|
+
finally:
|
|
190
|
+
aclose = getattr(iter_bytes, "aclose", None)
|
|
191
|
+
if aclose is not None:
|
|
192
|
+
await aclose()
|
|
193
|
+
|
|
194
|
+
for line in framer.flush(decoder.decode(b"", final=True)):
|
|
195
|
+
yield line
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class JSONLDecoder:
|
|
199
|
+
"""Decoder for JSON Lines (JSONL) format. https://jsonlines.org/"""
|
|
200
|
+
|
|
201
|
+
def iter_events(self, iter_bytes: Iterator[bytes]) -> Iterator[JSONLEvent]:
|
|
202
|
+
"""Iterate over JSONL events from a byte iterator.
|
|
203
|
+
|
|
204
|
+
:param iter_bytes: An iterator of byte chunks.
|
|
205
|
+
:type iter_bytes: Iterator[bytes]
|
|
206
|
+
:rtype: Iterator[JSONLEvent]
|
|
207
|
+
:return: An iterator of JSONL events.
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
yield from (JSONLEvent(data=line) for line in iter_lines(iter_bytes))
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
class AsyncJSONLDecoder:
|
|
214
|
+
"""Asynchronous decoder for JSON Lines (JSONL) format. https://jsonlines.org/"""
|
|
215
|
+
|
|
216
|
+
# pylint: disable=invalid-overridden-method
|
|
217
|
+
async def aiter_events(self, iter_bytes: AsyncIterator[bytes]) -> AsyncIterator[JSONLEvent]:
|
|
218
|
+
"""Asynchronously iterate over JSONL events from a byte iterator.
|
|
219
|
+
|
|
220
|
+
:param iter_bytes: An asynchronous iterator of byte chunks.
|
|
221
|
+
:type iter_bytes: AsyncIterator[bytes]
|
|
222
|
+
:rtype: AsyncIterator[JSONLEvent]
|
|
223
|
+
:return: An asynchronous iterator of JSONL events.
|
|
224
|
+
"""
|
|
225
|
+
|
|
226
|
+
async with aclosing(aiter_lines(iter_bytes)) as lines:
|
|
227
|
+
async for line in lines:
|
|
228
|
+
yield JSONLEvent(data=line)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
class ServerSentEvent:
|
|
232
|
+
"""A single Server-Sent Event (SSE).
|
|
233
|
+
|
|
234
|
+
https://html.spec.whatwg.org/multipage/server-sent-events.html
|
|
235
|
+
|
|
236
|
+
:ivar event: The event type. Defaults to ``"message"`` when the stream does not
|
|
237
|
+
specify one.
|
|
238
|
+
:vartype event: str
|
|
239
|
+
:ivar data: The event payload. Multiple ``data`` lines are joined with ``"\\n"``.
|
|
240
|
+
Left as a raw string; the caller is responsible for any further parsing.
|
|
241
|
+
:vartype data: str
|
|
242
|
+
:ivar id: The last event ID. Defaults to an empty string until the stream
|
|
243
|
+
provides one.
|
|
244
|
+
:vartype id: str
|
|
245
|
+
:ivar retry: The reconnection time in milliseconds, if the stream provided one.
|
|
246
|
+
:vartype retry: int or None
|
|
247
|
+
:keyword event: The event type. Defaults to ``"message"`` when the stream does not
|
|
248
|
+
specify one.
|
|
249
|
+
:paramtype event: str
|
|
250
|
+
:keyword data: The event payload. Multiple ``data`` lines are joined with ``"\\n"``.
|
|
251
|
+
Left as a raw string; the caller is responsible for any further parsing.
|
|
252
|
+
:paramtype data: str
|
|
253
|
+
:keyword id: The last event ID. Defaults to an empty string until the stream
|
|
254
|
+
provides one.
|
|
255
|
+
:paramtype id: str
|
|
256
|
+
:keyword retry: The reconnection time in milliseconds, if the stream provided one.
|
|
257
|
+
:paramtype retry: int or None
|
|
258
|
+
"""
|
|
259
|
+
|
|
260
|
+
def __init__(
|
|
261
|
+
self,
|
|
262
|
+
*,
|
|
263
|
+
event: str = "message",
|
|
264
|
+
data: str = "",
|
|
265
|
+
id: str = "", # pylint: disable=redefined-builtin
|
|
266
|
+
retry: Optional[int] = None,
|
|
267
|
+
) -> None:
|
|
268
|
+
self.event = event
|
|
269
|
+
self.data = data
|
|
270
|
+
self.id = id
|
|
271
|
+
self.retry = retry
|
|
272
|
+
|
|
273
|
+
def __repr__(self) -> str:
|
|
274
|
+
return f"ServerSentEvent(event={self.event!r}, data={self.data!r}, " f"id={self.id!r}, retry={self.retry!r})"
|
|
275
|
+
|
|
276
|
+
def __eq__(self, other: object) -> bool:
|
|
277
|
+
if not isinstance(other, ServerSentEvent):
|
|
278
|
+
return NotImplemented
|
|
279
|
+
return (self.event, self.data, self.id, self.retry) == (
|
|
280
|
+
other.event,
|
|
281
|
+
other.data,
|
|
282
|
+
other.id,
|
|
283
|
+
other.retry,
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
class _SSELineFramer:
|
|
288
|
+
"""Incremental SSE line framer with linear-time behavior.
|
|
289
|
+
|
|
290
|
+
Per the SSE spec, lines may be separated by ``"\\r\\n"``, ``"\\r"`` or ``"\\n"``. A lone trailing
|
|
291
|
+
``"\\r"`` at a chunk boundary is ambiguous (it may be the first half of a ``"\\r\\n"``) and its
|
|
292
|
+
resolution is deferred until the next chunk (or EOF).
|
|
293
|
+
|
|
294
|
+
Rather than re-concatenating and re-scanning the whole pending line on every network chunk
|
|
295
|
+
(which is O(n^2) for a single long line fragmented across many chunks), the unfinished line is
|
|
296
|
+
held as a list of fragments and joined only when a terminator arrives (or at EOF). Only the
|
|
297
|
+
newly decoded text is scanned per chunk, giving O(total) behavior.
|
|
298
|
+
"""
|
|
299
|
+
|
|
300
|
+
def __init__(self) -> None:
|
|
301
|
+
# Fragments of the current, not-yet-terminated line. Never contains a separator.
|
|
302
|
+
self._parts: List[str] = []
|
|
303
|
+
# True when the previous chunk ended with a lone "\r" whose "\r\n" status is still unknown.
|
|
304
|
+
self._pending_cr = False
|
|
305
|
+
|
|
306
|
+
def _emit_current(self) -> str:
|
|
307
|
+
line = "".join(self._parts)
|
|
308
|
+
self._parts = []
|
|
309
|
+
return line
|
|
310
|
+
|
|
311
|
+
def push(self, text: str) -> List[str]:
|
|
312
|
+
"""Feed newly decoded text and return any completed lines.
|
|
313
|
+
|
|
314
|
+
:param text: Newly decoded text from a single chunk.
|
|
315
|
+
:type text: str
|
|
316
|
+
:return: Completed lines (separators stripped) produced by this chunk.
|
|
317
|
+
:rtype: list[str]
|
|
318
|
+
"""
|
|
319
|
+
if not text:
|
|
320
|
+
return []
|
|
321
|
+
|
|
322
|
+
out: List[str] = []
|
|
323
|
+
if self._pending_cr:
|
|
324
|
+
# The deferred "\r" terminates the current line now that more data is available.
|
|
325
|
+
out.append(self._emit_current())
|
|
326
|
+
self._pending_cr = False
|
|
327
|
+
# A leading "\n" is the second half of that "\r\n" pair: consume it.
|
|
328
|
+
if text[:1] == "\n":
|
|
329
|
+
text = text[1:]
|
|
330
|
+
|
|
331
|
+
n = len(text)
|
|
332
|
+
start = 0
|
|
333
|
+
i = 0
|
|
334
|
+
while i < n:
|
|
335
|
+
char = text[i]
|
|
336
|
+
if char == "\n":
|
|
337
|
+
self._parts.append(text[start:i])
|
|
338
|
+
out.append(self._emit_current())
|
|
339
|
+
i += 1
|
|
340
|
+
start = i
|
|
341
|
+
elif char == "\r":
|
|
342
|
+
if i + 1 < n:
|
|
343
|
+
self._parts.append(text[start:i])
|
|
344
|
+
out.append(self._emit_current())
|
|
345
|
+
i += 2 if text[i + 1] == "\n" else 1
|
|
346
|
+
start = i
|
|
347
|
+
else:
|
|
348
|
+
# Trailing lone "\r": defer resolution until the next chunk.
|
|
349
|
+
self._parts.append(text[start:i])
|
|
350
|
+
self._pending_cr = True
|
|
351
|
+
start = n
|
|
352
|
+
break
|
|
353
|
+
else:
|
|
354
|
+
i += 1
|
|
355
|
+
|
|
356
|
+
if start < n:
|
|
357
|
+
self._parts.append(text[start:n])
|
|
358
|
+
return out
|
|
359
|
+
|
|
360
|
+
def flush(self, extra: str = "") -> List[str]:
|
|
361
|
+
"""Return any remaining lines at end of stream.
|
|
362
|
+
|
|
363
|
+
A lone trailing ``"\\r"`` is treated as a terminator (its ``"\\r\\n"`` half never arrives),
|
|
364
|
+
and a non-empty final unterminated line is emitted; an empty tail is not, so no blank line
|
|
365
|
+
(and therefore no spurious event) is invented at EOF.
|
|
366
|
+
|
|
367
|
+
:param extra: Trailing text from finalizing the incremental decoder.
|
|
368
|
+
:type extra: str
|
|
369
|
+
:return: The remaining lines, if any.
|
|
370
|
+
:rtype: list[str]
|
|
371
|
+
"""
|
|
372
|
+
out: List[str] = []
|
|
373
|
+
if self._pending_cr:
|
|
374
|
+
out.append(self._emit_current())
|
|
375
|
+
self._pending_cr = False
|
|
376
|
+
if extra[:1] == "\n":
|
|
377
|
+
extra = extra[1:]
|
|
378
|
+
|
|
379
|
+
if extra:
|
|
380
|
+
out.extend(self.push(extra))
|
|
381
|
+
|
|
382
|
+
if self._pending_cr:
|
|
383
|
+
# 'extra' ended in a lone "\r": at EOF it is a terminator; emit the preceding content.
|
|
384
|
+
out.append(self._emit_current())
|
|
385
|
+
self._pending_cr = False
|
|
386
|
+
else:
|
|
387
|
+
tail = self._emit_current()
|
|
388
|
+
if tail:
|
|
389
|
+
out.append(tail)
|
|
390
|
+
return out
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _iter_sse_lines(iter_bytes: Iterator[bytes]) -> Iterator[str]:
|
|
394
|
+
"""Iterate over SSE lines (line separators stripped) from a byte iterator.
|
|
395
|
+
|
|
396
|
+
:param iter_bytes: An iterator of byte chunks.
|
|
397
|
+
:type iter_bytes: Iterator[bytes]
|
|
398
|
+
:rtype: Iterator[str]
|
|
399
|
+
:return: An iterator of decoded lines.
|
|
400
|
+
"""
|
|
401
|
+
# SSE is always UTF-8 (WHATWG spec). Use utf-8-sig to drop one leading BOM and
|
|
402
|
+
# errors="replace" so invalid byte sequences become U+FFFD instead of crashing.
|
|
403
|
+
decoder = codecs.getincrementaldecoder("utf-8-sig")(errors="replace")
|
|
404
|
+
framer = _SSELineFramer()
|
|
405
|
+
|
|
406
|
+
for chunk in iter_bytes:
|
|
407
|
+
yield from framer.push(decoder.decode(chunk))
|
|
408
|
+
|
|
409
|
+
yield from framer.flush(decoder.decode(b"", final=True))
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
async def _aiter_sse_lines(iter_bytes: AsyncIterator[bytes]) -> AsyncGenerator[str, None]:
|
|
413
|
+
"""Asynchronously iterate over SSE lines (separators stripped) from a byte iterator.
|
|
414
|
+
|
|
415
|
+
:param iter_bytes: An asynchronous iterator of byte chunks.
|
|
416
|
+
:type iter_bytes: AsyncIterator[bytes]
|
|
417
|
+
:rtype: AsyncIterator[str]
|
|
418
|
+
:return: An asynchronous iterator of decoded lines.
|
|
419
|
+
"""
|
|
420
|
+
# SSE is always UTF-8 (WHATWG spec). Use utf-8-sig to drop one leading BOM and
|
|
421
|
+
# errors="replace" so invalid byte sequences become U+FFFD instead of crashing.
|
|
422
|
+
decoder = codecs.getincrementaldecoder("utf-8-sig")(errors="replace")
|
|
423
|
+
framer = _SSELineFramer()
|
|
424
|
+
|
|
425
|
+
try:
|
|
426
|
+
async for chunk in iter_bytes:
|
|
427
|
+
for line in framer.push(decoder.decode(chunk)):
|
|
428
|
+
yield line
|
|
429
|
+
finally:
|
|
430
|
+
aclose = getattr(iter_bytes, "aclose", None)
|
|
431
|
+
if aclose is not None:
|
|
432
|
+
await aclose()
|
|
433
|
+
|
|
434
|
+
for line in framer.flush(decoder.decode(b"", final=True)):
|
|
435
|
+
yield line
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
class _SSEEventBuilder:
|
|
439
|
+
"""Accumulates SSE field lines and builds :class:`ServerSentEvent` instances."""
|
|
440
|
+
|
|
441
|
+
def __init__(self) -> None:
|
|
442
|
+
self._data: List[str] = []
|
|
443
|
+
self._event_type = ""
|
|
444
|
+
self._last_id = ""
|
|
445
|
+
self._retry: Optional[int] = None
|
|
446
|
+
|
|
447
|
+
def add_line(self, line: str) -> Optional[ServerSentEvent]:
|
|
448
|
+
"""Process a single SSE line, dispatching an event on a blank line.
|
|
449
|
+
|
|
450
|
+
:param line: A single SSE line with its terminator already stripped.
|
|
451
|
+
:type line: str
|
|
452
|
+
:return: A :class:`ServerSentEvent` when ``line`` is blank and an event is
|
|
453
|
+
pending, otherwise ``None``.
|
|
454
|
+
:rtype: ServerSentEvent or None
|
|
455
|
+
"""
|
|
456
|
+
if line == "":
|
|
457
|
+
return self._dispatch()
|
|
458
|
+
if line.startswith(":"):
|
|
459
|
+
# Comment line, ignored.
|
|
460
|
+
return None
|
|
461
|
+
|
|
462
|
+
field, _, value = line.partition(":")
|
|
463
|
+
if value.startswith(" "):
|
|
464
|
+
value = value[1:]
|
|
465
|
+
|
|
466
|
+
if field == "event":
|
|
467
|
+
self._event_type = value
|
|
468
|
+
elif field == "data":
|
|
469
|
+
self._data.append(value)
|
|
470
|
+
elif field == "id":
|
|
471
|
+
if "\x00" not in value:
|
|
472
|
+
self._last_id = value
|
|
473
|
+
elif field == "retry":
|
|
474
|
+
if value.isascii() and value.isdigit():
|
|
475
|
+
try:
|
|
476
|
+
self._retry = int(value)
|
|
477
|
+
except ValueError:
|
|
478
|
+
# All ASCII digits but too long for int() (CPython's int-string
|
|
479
|
+
# conversion limit). Ignore rather than crashing the stream.
|
|
480
|
+
pass
|
|
481
|
+
# Unknown fields are ignored per spec.
|
|
482
|
+
return None
|
|
483
|
+
|
|
484
|
+
def _dispatch(self) -> Optional[ServerSentEvent]:
|
|
485
|
+
if not self._data:
|
|
486
|
+
# No data accumulated: reset and dispatch nothing.
|
|
487
|
+
self._event_type = ""
|
|
488
|
+
return None
|
|
489
|
+
event = ServerSentEvent(
|
|
490
|
+
event=self._event_type or "message",
|
|
491
|
+
data="\n".join(self._data),
|
|
492
|
+
id=self._last_id,
|
|
493
|
+
retry=self._retry,
|
|
494
|
+
)
|
|
495
|
+
self._data = []
|
|
496
|
+
self._event_type = ""
|
|
497
|
+
return event
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
class SSEDecoder:
|
|
501
|
+
"""Decoder for Server-Sent Events (SSE).
|
|
502
|
+
|
|
503
|
+
https://html.spec.whatwg.org/multipage/server-sent-events.html
|
|
504
|
+
"""
|
|
505
|
+
|
|
506
|
+
def iter_events(self, iter_bytes: Iterator[bytes]) -> Iterator[ServerSentEvent]:
|
|
507
|
+
"""Iterate over SSE events from a byte iterator.
|
|
508
|
+
|
|
509
|
+
:param iter_bytes: An iterator of byte chunks.
|
|
510
|
+
:type iter_bytes: Iterator[bytes]
|
|
511
|
+
:rtype: Iterator[ServerSentEvent]
|
|
512
|
+
:return: An iterator of server-sent events.
|
|
513
|
+
"""
|
|
514
|
+
builder = _SSEEventBuilder()
|
|
515
|
+
for line in _iter_sse_lines(iter_bytes):
|
|
516
|
+
event = builder.add_line(line)
|
|
517
|
+
if event is not None:
|
|
518
|
+
yield event
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
class AsyncSSEDecoder:
|
|
522
|
+
"""Asynchronous decoder for Server-Sent Events (SSE).
|
|
523
|
+
|
|
524
|
+
https://html.spec.whatwg.org/multipage/server-sent-events.html
|
|
525
|
+
"""
|
|
526
|
+
|
|
527
|
+
# pylint: disable=invalid-overridden-method
|
|
528
|
+
async def aiter_events(self, iter_bytes: AsyncIterator[bytes]) -> AsyncIterator[ServerSentEvent]:
|
|
529
|
+
"""Asynchronously iterate over SSE events from a byte iterator.
|
|
530
|
+
|
|
531
|
+
:param iter_bytes: An asynchronous iterator of byte chunks.
|
|
532
|
+
:type iter_bytes: AsyncIterator[bytes]
|
|
533
|
+
:rtype: AsyncIterator[ServerSentEvent]
|
|
534
|
+
:return: An asynchronous iterator of server-sent events.
|
|
535
|
+
"""
|
|
536
|
+
builder = _SSEEventBuilder()
|
|
537
|
+
async with aclosing(_aiter_sse_lines(iter_bytes)) as lines:
|
|
538
|
+
async for line in lines:
|
|
539
|
+
event = builder.add_line(line)
|
|
540
|
+
if event is not None:
|
|
541
|
+
yield event
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
class Stream(Iterator[ReturnType_co]):
|
|
545
|
+
"""Stream class for consuming a decoded event stream (e.g. JSONL or SSE).
|
|
546
|
+
|
|
547
|
+
:keyword response: The response object.
|
|
548
|
+
:paramtype response: ~{{ code_model.core_library }}.rest.HttpResponse
|
|
549
|
+
:keyword decoder: A decoder to use for the stream. If omitted, the decoder is
|
|
550
|
+
inferred from the response ``Content-Type`` header.
|
|
551
|
+
:paramtype decoder: StreamDecoder
|
|
552
|
+
:keyword deserialization_callback: A callback that takes the response and the decoded event and
|
|
553
|
+
returns a deserialized object.
|
|
554
|
+
:paramtype deserialization_callback: Callable[[~{{ code_model.core_library }}.rest.HttpResponse, Any], ReturnType]
|
|
555
|
+
:keyword terminal_event: Optional event ``data`` value that terminates the stream (e.g.
|
|
556
|
+
``"[DONE]"``). When an event's ``data`` equals this value, iteration stops and the
|
|
557
|
+
event is not passed to ``deserialization_callback``.
|
|
558
|
+
:paramtype terminal_event: str or None
|
|
559
|
+
:keyword terminal_event_names: Optional event names (the SSE ``event`` field) that terminate
|
|
560
|
+
the stream. Unlike ``terminal_event``, such an event carries a payload: it is passed to
|
|
561
|
+
``deserialization_callback`` and yielded, and iteration stops immediately afterwards.
|
|
562
|
+
:paramtype terminal_event_names: ~typing.Sequence[str] or None
|
|
563
|
+
:keyword terminal_event_predicate: Optional predicate that identifies a payload-bearing
|
|
564
|
+
terminal event. The event is yielded before iteration stops.
|
|
565
|
+
:paramtype terminal_event_predicate: Callable[[Any], bool] or None
|
|
566
|
+
"""
|
|
567
|
+
|
|
568
|
+
def __init__(
|
|
569
|
+
self,
|
|
570
|
+
*,
|
|
571
|
+
response: HttpResponse,
|
|
572
|
+
deserialization_callback: Callable[[HttpResponse, DecodedType], ReturnType_co],
|
|
573
|
+
decoder: Optional[StreamDecoder[DecodedType]] = None,
|
|
574
|
+
terminal_event: Optional[str] = None,
|
|
575
|
+
terminal_event_names: Optional[Sequence[str]] = None,
|
|
576
|
+
terminal_event_predicate: Optional[Callable[[DecodedType], bool]] = None,
|
|
577
|
+
) -> None:
|
|
578
|
+
self._response = response
|
|
579
|
+
content_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip().lower()
|
|
580
|
+
self._decoder: StreamDecoder[Any] = (
|
|
581
|
+
decoder if decoder is not None else (SSEDecoder() if content_type == "text/event-stream" else JSONLDecoder())
|
|
582
|
+
)
|
|
583
|
+
self._deserialization_callback = deserialization_callback
|
|
584
|
+
self._terminal_event = terminal_event
|
|
585
|
+
self._terminal_event_names = frozenset(terminal_event_names or ())
|
|
586
|
+
self._terminal_event_predicate = terminal_event_predicate
|
|
587
|
+
self._last_event_id: Optional[str] = None
|
|
588
|
+
self._retry: Optional[int] = None
|
|
589
|
+
self._iterator = self._iter_results()
|
|
590
|
+
|
|
591
|
+
@property
|
|
592
|
+
def last_event_id(self) -> Optional[str]:
|
|
593
|
+
"""The most recently received SSE event ID, if one was provided."""
|
|
594
|
+
return self._last_event_id
|
|
595
|
+
|
|
596
|
+
@property
|
|
597
|
+
def retry(self) -> Optional[int]:
|
|
598
|
+
"""The most recently received valid SSE retry value, if one was provided."""
|
|
599
|
+
return self._retry
|
|
600
|
+
|
|
601
|
+
def __next__(self) -> ReturnType_co:
|
|
602
|
+
return self._iterator.__next__()
|
|
603
|
+
|
|
604
|
+
def __iter__(self) -> Self:
|
|
605
|
+
return self
|
|
606
|
+
|
|
607
|
+
def _iter_results(self) -> Generator[ReturnType_co, None, None]:
|
|
608
|
+
try:
|
|
609
|
+
for event in self._decoder.iter_events(self._response.iter_bytes()):
|
|
610
|
+
event_id = getattr(event, "id", None)
|
|
611
|
+
if event_id is not None:
|
|
612
|
+
self._last_event_id = event_id
|
|
613
|
+
event_retry = getattr(event, "retry", None)
|
|
614
|
+
if event_retry is not None:
|
|
615
|
+
self._retry = event_retry
|
|
616
|
+
if self._terminal_event is not None and getattr(event, "data", None) == self._terminal_event:
|
|
617
|
+
break
|
|
618
|
+
result = self._deserialization_callback(self._response, event)
|
|
619
|
+
yield result
|
|
620
|
+
if (
|
|
621
|
+
self._terminal_event_names
|
|
622
|
+
and getattr(event, "event", None) in self._terminal_event_names
|
|
623
|
+
) or (
|
|
624
|
+
self._terminal_event_predicate is not None
|
|
625
|
+
and self._terminal_event_predicate(event)
|
|
626
|
+
):
|
|
627
|
+
break
|
|
628
|
+
finally:
|
|
629
|
+
self._response.close()
|
|
630
|
+
|
|
631
|
+
def __exit__(
|
|
632
|
+
self,
|
|
633
|
+
exc_type: Optional[Type[BaseException]] = None,
|
|
634
|
+
exc_value: Optional[BaseException] = None,
|
|
635
|
+
traceback: Optional[TracebackType] = None,
|
|
636
|
+
) -> None:
|
|
637
|
+
self.close()
|
|
638
|
+
|
|
639
|
+
def __enter__(self) -> Self:
|
|
640
|
+
return self
|
|
641
|
+
|
|
642
|
+
def close(self) -> None:
|
|
643
|
+
try:
|
|
644
|
+
self._iterator.close()
|
|
645
|
+
finally:
|
|
646
|
+
self._response.close()
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
class AsyncStream(AsyncIterator[ReturnType_co]):
|
|
650
|
+
"""AsyncStream class for asynchronously consuming a decoded event stream (e.g. JSONL or SSE).
|
|
651
|
+
|
|
652
|
+
:keyword response: The response object.
|
|
653
|
+
:paramtype response: ~{{ code_model.core_library }}.rest.AsyncHttpResponse
|
|
654
|
+
:keyword decoder: A decoder to use for the stream. If omitted, the decoder is
|
|
655
|
+
inferred from the response ``Content-Type`` header.
|
|
656
|
+
:paramtype decoder: AsyncStreamDecoder
|
|
657
|
+
:keyword deserialization_callback: A callback that takes the response and the decoded event and
|
|
658
|
+
returns a deserialized object.
|
|
659
|
+
:paramtype deserialization_callback: Callable[[~{{ code_model.core_library }}.rest.AsyncHttpResponse, Any], ReturnType]
|
|
660
|
+
:keyword terminal_event: Optional event ``data`` value that terminates the stream (e.g.
|
|
661
|
+
``"[DONE]"``). When an event's ``data`` equals this value, iteration stops and the
|
|
662
|
+
event is not passed to ``deserialization_callback``.
|
|
663
|
+
:paramtype terminal_event: str or None
|
|
664
|
+
:keyword terminal_event_names: Optional event names (the SSE ``event`` field) that terminate
|
|
665
|
+
the stream. Unlike ``terminal_event``, such an event carries a payload: it is passed to
|
|
666
|
+
``deserialization_callback`` and yielded, and iteration stops immediately afterwards.
|
|
667
|
+
:paramtype terminal_event_names: ~typing.Sequence[str] or None
|
|
668
|
+
:keyword terminal_event_predicate: Optional predicate that identifies a payload-bearing
|
|
669
|
+
terminal event. The event is yielded before iteration stops.
|
|
670
|
+
:paramtype terminal_event_predicate: Callable[[Any], bool] or None
|
|
671
|
+
"""
|
|
672
|
+
|
|
673
|
+
def __init__(
|
|
674
|
+
self,
|
|
675
|
+
*,
|
|
676
|
+
response: AsyncHttpResponse,
|
|
677
|
+
deserialization_callback: Callable[[AsyncHttpResponse, DecodedType], ReturnType_co],
|
|
678
|
+
decoder: Optional[AsyncStreamDecoder[DecodedType]] = None,
|
|
679
|
+
terminal_event: Optional[str] = None,
|
|
680
|
+
terminal_event_names: Optional[Sequence[str]] = None,
|
|
681
|
+
terminal_event_predicate: Optional[Callable[[DecodedType], bool]] = None,
|
|
682
|
+
) -> None:
|
|
683
|
+
self._response = response
|
|
684
|
+
content_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip().lower()
|
|
685
|
+
self._decoder: AsyncStreamDecoder[Any] = (
|
|
686
|
+
decoder
|
|
687
|
+
if decoder is not None
|
|
688
|
+
else (AsyncSSEDecoder() if content_type == "text/event-stream" else AsyncJSONLDecoder())
|
|
689
|
+
)
|
|
690
|
+
self._deserialization_callback = deserialization_callback
|
|
691
|
+
self._terminal_event = terminal_event
|
|
692
|
+
self._terminal_event_names = frozenset(terminal_event_names or ())
|
|
693
|
+
self._terminal_event_predicate = terminal_event_predicate
|
|
694
|
+
self._last_event_id: Optional[str] = None
|
|
695
|
+
self._retry: Optional[int] = None
|
|
696
|
+
self._iterator = self._iter_results()
|
|
697
|
+
|
|
698
|
+
@property
|
|
699
|
+
def last_event_id(self) -> Optional[str]:
|
|
700
|
+
"""The most recently received SSE event ID, if one was provided."""
|
|
701
|
+
return self._last_event_id
|
|
702
|
+
|
|
703
|
+
@property
|
|
704
|
+
def retry(self) -> Optional[int]:
|
|
705
|
+
"""The most recently received valid SSE retry value, if one was provided."""
|
|
706
|
+
return self._retry
|
|
707
|
+
|
|
708
|
+
async def __anext__(self) -> ReturnType_co:
|
|
709
|
+
return await self._iterator.__anext__()
|
|
710
|
+
|
|
711
|
+
def __aiter__(self) -> Self:
|
|
712
|
+
return self
|
|
713
|
+
|
|
714
|
+
async def _iter_results(self) -> AsyncGenerator[ReturnType_co, None]:
|
|
715
|
+
events = self._decoder.aiter_events(self._response.iter_bytes())
|
|
716
|
+
try:
|
|
717
|
+
async for event in events:
|
|
718
|
+
event_id = getattr(event, "id", None)
|
|
719
|
+
if event_id is not None:
|
|
720
|
+
self._last_event_id = event_id
|
|
721
|
+
event_retry = getattr(event, "retry", None)
|
|
722
|
+
if event_retry is not None:
|
|
723
|
+
self._retry = event_retry
|
|
724
|
+
if self._terminal_event is not None and getattr(event, "data", None) == self._terminal_event:
|
|
725
|
+
break
|
|
726
|
+
result = self._deserialization_callback(self._response, event)
|
|
727
|
+
yield result
|
|
728
|
+
if (
|
|
729
|
+
self._terminal_event_names
|
|
730
|
+
and getattr(event, "event", None) in self._terminal_event_names
|
|
731
|
+
) or (
|
|
732
|
+
self._terminal_event_predicate is not None
|
|
733
|
+
and self._terminal_event_predicate(event)
|
|
734
|
+
):
|
|
735
|
+
break
|
|
736
|
+
finally:
|
|
737
|
+
try:
|
|
738
|
+
aclose = getattr(events, "aclose", None)
|
|
739
|
+
if aclose is not None:
|
|
740
|
+
await aclose()
|
|
741
|
+
finally:
|
|
742
|
+
await self._response.close()
|
|
743
|
+
|
|
744
|
+
async def __aexit__(
|
|
745
|
+
self,
|
|
746
|
+
exc_type: Optional[Type[BaseException]] = None,
|
|
747
|
+
exc_value: Optional[BaseException] = None,
|
|
748
|
+
traceback: Optional[TracebackType] = None,
|
|
749
|
+
) -> None:
|
|
750
|
+
await self.close()
|
|
751
|
+
|
|
752
|
+
async def __aenter__(self) -> Self:
|
|
753
|
+
return self
|
|
754
|
+
|
|
755
|
+
async def close(self) -> None:
|
|
756
|
+
try:
|
|
757
|
+
await self._iterator.aclose()
|
|
758
|
+
finally:
|
|
759
|
+
await self._response.close()
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
__all__ = [
|
|
763
|
+
"Stream",
|
|
764
|
+
"AsyncStream",
|
|
765
|
+
"JSONLEvent",
|
|
766
|
+
"ServerSentEvent",
|
|
767
|
+
]
|