@gajae-code/ai 0.17.2 → 0.17.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +103 -0
- package/dist/types/auth-gateway/server.d.ts +23 -1
- package/dist/types/auth-storage.d.ts +12 -1
- package/dist/types/providers/anthropic.d.ts +1 -1
- package/dist/types/providers/cursor.d.ts +10 -0
- package/dist/types/providers/openai-completions.d.ts +9 -1
- package/dist/types/types.d.ts +16 -0
- package/dist/types/utils/discovery/openai-compatible.d.ts +10 -0
- package/dist/types/utils/fallback-transport.d.ts +4 -0
- package/dist/types/utils/h2-fetch.d.ts +8 -7
- package/dist/types/utils/stream-repetition-guard.d.ts +107 -0
- package/dist/types/utils/tool-call-healing.d.ts +4 -0
- package/dist/types/utils/tool-fence-strip.d.ts +27 -0
- package/package.json +3 -3
- package/src/auth-gateway/server.ts +48 -9
- package/src/auth-storage.ts +185 -48
- package/src/model-pricing.ts +22 -0
- package/src/model-thinking.ts +40 -4
- package/src/models.json +191 -15
- package/src/providers/anthropic.d.ts +1 -1
- package/src/providers/anthropic.ts +1 -1
- package/src/providers/cursor.d.ts +10 -0
- package/src/providers/cursor.ts +144 -24
- package/src/providers/openai-completions.d.ts +9 -1
- package/src/providers/openai-completions.ts +371 -126
- package/src/stream.ts +7 -0
- package/src/types.d.ts +16 -0
- package/src/types.ts +17 -0
- package/src/utils/discovery/openai-compatible.ts +16 -2
- package/src/utils/fallback-transport.d.ts +4 -0
- package/src/utils/fallback-transport.ts +11 -0
- package/src/utils/h2-fetch.ts +65 -26
- package/src/utils/http-inspector.ts +2 -0
- package/src/utils/idle-iterator.ts +109 -96
- package/src/utils/json-parse.ts +12 -4
- package/src/utils/stream-repetition-guard.d.ts +107 -0
- package/src/utils/stream-repetition-guard.ts +290 -0
- package/src/utils/tool-call-healing.d.ts +4 -0
- package/src/utils/tool-call-healing.ts +4 -0
- package/src/utils/tool-fence-strip.d.ts +27 -0
- package/src/utils/tool-fence-strip.ts +64 -0
package/src/types.ts
CHANGED
|
@@ -556,6 +556,15 @@ export interface AttemptScopeRef {
|
|
|
556
556
|
readonly lineage: string;
|
|
557
557
|
}
|
|
558
558
|
|
|
559
|
+
/**
|
|
560
|
+
* Runaway-repetition guard thresholds, per stream channel. A number sets the
|
|
561
|
+
* consecutive-repeat threshold; `false` disables that channel's guard.
|
|
562
|
+
*/
|
|
563
|
+
export interface RepetitionGuardOptions {
|
|
564
|
+
thinking?: number | false;
|
|
565
|
+
text?: number | false;
|
|
566
|
+
}
|
|
567
|
+
|
|
559
568
|
// Unified options with reasoning passed to streamSimple() and completeSimple()
|
|
560
569
|
export interface SimpleStreamOptions extends StreamOptions {
|
|
561
570
|
reasoning?: Effort;
|
|
@@ -592,6 +601,14 @@ export interface SimpleStreamOptions extends StreamOptions {
|
|
|
592
601
|
syntheticApiFormat?: "openai" | "anthropic";
|
|
593
602
|
/** Hint that websocket transport should be preferred when supported by the provider implementation. */
|
|
594
603
|
preferWebsockets?: boolean;
|
|
604
|
+
/**
|
|
605
|
+
* Runaway-repetition guard thresholds, per stream channel. Honoured by the
|
|
606
|
+
* openai-completions transport; ignored by providers without a guard.
|
|
607
|
+
* Defaults: thinking = DEFAULT_REPETITION_THRESHOLD, text = false — visible
|
|
608
|
+
* output is a deliverable and intentional repetition there (logs, fixtures,
|
|
609
|
+
* tables, generated code) must survive byte for byte (#5627).
|
|
610
|
+
*/
|
|
611
|
+
repetitionGuard?: RepetitionGuardOptions;
|
|
595
612
|
}
|
|
596
613
|
|
|
597
614
|
// Generic StreamFunction with typed options
|
|
@@ -5,6 +5,15 @@ import type { Api, FetchImpl, Model, Provider } from "../../types";
|
|
|
5
5
|
import { toNumber } from "../../utils";
|
|
6
6
|
|
|
7
7
|
const MODELS_PATH = "/models";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Shared `/models` request deadline for setup-time probing and runtime
|
|
11
|
+
* discovery. One policy so an endpoint that passes setup validation cannot
|
|
12
|
+
* be unreachable under the runtime deadline (previously 10s vs 5s: a 6s
|
|
13
|
+
* endpoint passed setup, saved discovery-only config, then stayed
|
|
14
|
+
* unavailable at runtime with no recovery hint).
|
|
15
|
+
*/
|
|
16
|
+
export const MODELS_LIST_REQUEST_TIMEOUT_MS = 10_000;
|
|
8
17
|
const MAX_MODELS_RESPONSE_BYTES = 1_000_000;
|
|
9
18
|
const MAX_CATALOG_MODEL_ID_LENGTH = 200;
|
|
10
19
|
|
|
@@ -233,8 +242,8 @@ export async function fetchOpenAICompatibleModels<TApi extends Api>(
|
|
|
233
242
|
method: "GET",
|
|
234
243
|
headers: requestHeaders,
|
|
235
244
|
signal: options.signal
|
|
236
|
-
? AbortSignal.any([options.signal, AbortSignal.timeout(
|
|
237
|
-
: AbortSignal.timeout(
|
|
245
|
+
? AbortSignal.any([options.signal, AbortSignal.timeout(MODELS_LIST_REQUEST_TIMEOUT_MS)])
|
|
246
|
+
: AbortSignal.timeout(MODELS_LIST_REQUEST_TIMEOUT_MS),
|
|
238
247
|
});
|
|
239
248
|
} catch {
|
|
240
249
|
return null;
|
|
@@ -308,6 +317,11 @@ export async function fetchOpenAICompatibleModels<TApi extends Api>(
|
|
|
308
317
|
return Array.from(deduped.values()).sort((left, right) => left.id.localeCompare(right.id));
|
|
309
318
|
}
|
|
310
319
|
|
|
320
|
+
/** Bounded JSON body reader for `/models` responses (shared by runtime and setup probe). */
|
|
321
|
+
export async function readBoundedModelsJson(response: Response): Promise<unknown> {
|
|
322
|
+
return JSON.parse(await readModelsResponse(response));
|
|
323
|
+
}
|
|
324
|
+
|
|
311
325
|
async function readModelsResponse(response: Response): Promise<string> {
|
|
312
326
|
const contentLength = Number(response.headers.get("content-length"));
|
|
313
327
|
if (Number.isFinite(contentLength) && contentLength > MAX_MODELS_RESPONSE_BYTES) {
|
|
@@ -43,6 +43,10 @@ export type TransportHeaders = Headers | Record<string, string | undefined>;
|
|
|
43
43
|
*/
|
|
44
44
|
export interface TransportFailureFacts {
|
|
45
45
|
kind: "transport";
|
|
46
|
+
/** Diagnostic HTTP/2 reset code; not HTTP status or retry authority. */
|
|
47
|
+
http2RstCode?: number;
|
|
48
|
+
/** Native HTTP/2 error code; diagnostic only, not retry authority. */
|
|
49
|
+
nativeErrorCode?: string;
|
|
46
50
|
status?: number;
|
|
47
51
|
/** Canonical provider error code used for fallback classification. */
|
|
48
52
|
providerCode?: string;
|
|
@@ -48,6 +48,10 @@ export type TransportHeaders = Headers | Record<string, string | undefined>;
|
|
|
48
48
|
*/
|
|
49
49
|
export interface TransportFailureFacts {
|
|
50
50
|
kind: "transport";
|
|
51
|
+
/** Diagnostic HTTP/2 reset code; not HTTP status or retry authority. */
|
|
52
|
+
http2RstCode?: number;
|
|
53
|
+
/** Native HTTP/2 error code; diagnostic only, not retry authority. */
|
|
54
|
+
nativeErrorCode?: string;
|
|
51
55
|
status?: number;
|
|
52
56
|
/** Canonical provider error code used for fallback classification. */
|
|
53
57
|
providerCode?: string;
|
|
@@ -235,6 +239,9 @@ export function transportFailureFacts(
|
|
|
235
239
|
// (consumers deliberately re-run transportFailureFacts on embedded facts).
|
|
236
240
|
const headers = retainedHeaderRecord(rawHeaders);
|
|
237
241
|
const normalizedCode = providerCode?.toLowerCase();
|
|
242
|
+
const http2RstCode = finiteNonNegativeInteger(propertyOf(value, "http2RstCode"));
|
|
243
|
+
const nativeCode = stringValue(propertyOf(value, "nativeErrorCode")) ?? stringValue(propertyOf(value, "code"));
|
|
244
|
+
const nativeErrorCode = nativeCode && /^ERR_HTTP2_[A-Z_]+$/.test(nativeCode) ? nativeCode : undefined;
|
|
238
245
|
const requestBytes = finiteNonNegativeInteger(propertyOf(value, "requestBytes"));
|
|
239
246
|
const firstEventElapsedMs = finiteNonNegativeInteger(propertyOf(value, "firstEventElapsedMs"));
|
|
240
247
|
const firstEventTimeoutMs = finiteNonNegativeInteger(propertyOf(value, "firstEventTimeoutMs"));
|
|
@@ -244,6 +251,8 @@ export function transportFailureFacts(
|
|
|
244
251
|
endpointClassValue === "canonical" || endpointClassValue === "custom" ? endpointClassValue : undefined;
|
|
245
252
|
const credentialModelUnavailable = propertyOf(value, "credentialModelUnavailable") === true;
|
|
246
253
|
if (
|
|
254
|
+
http2RstCode === undefined &&
|
|
255
|
+
nativeErrorCode === undefined &&
|
|
247
256
|
status === undefined &&
|
|
248
257
|
headers === undefined &&
|
|
249
258
|
!isQuotaCode(normalizedCode) &&
|
|
@@ -268,6 +277,8 @@ export function transportFailureFacts(
|
|
|
268
277
|
}
|
|
269
278
|
return {
|
|
270
279
|
kind: "transport",
|
|
280
|
+
...(http2RstCode === undefined ? {} : { http2RstCode }),
|
|
281
|
+
...(nativeErrorCode === undefined ? {} : { nativeErrorCode }),
|
|
271
282
|
status,
|
|
272
283
|
providerCode,
|
|
273
284
|
anthropicErrorType,
|
package/src/utils/h2-fetch.ts
CHANGED
|
@@ -11,13 +11,14 @@
|
|
|
11
11
|
* Some HTTPS endpoints (e.g. corporate API gateways behind reverse proxies)
|
|
12
12
|
* advertise h2 via ALPN but then refuse or reset the connection at the HTTP/2
|
|
13
13
|
* framing layer. Bun surfaces these as `ConnectionRefused`, `ConnectionReset`,
|
|
14
|
-
* or `
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
14
|
+
* `ConnectionClosed`, or `HTTP2StreamReset` rather than `HTTP2Unsupported`.
|
|
15
|
+
* `ConnectionRefused` is raised before the request is written, while
|
|
16
|
+
* `HTTP2RefusedStream` is the explicit HTTP/2 promise that a stream was never
|
|
17
|
+
* processed (including a stream above a graceful GOAWAY last-stream-id).
|
|
18
|
+
* `ConnectionReset`, `ConnectionClosed`, and a generic `HTTP2StreamReset` do
|
|
19
|
+
* not carry that promise: the peer may have consumed the body before the
|
|
20
|
+
* connection failed. Retrying those codes would duplicate non-idempotent side
|
|
21
|
+
* effects, so they preserve the original error instead of falling back.
|
|
21
22
|
*
|
|
22
23
|
* ALPN-refusing hosts (notably zcode.z.ai, the GLM ZCode OAuth broker) abort
|
|
23
24
|
* the TLS handshake entirely when the client offers ALPN h2. Bun reports that
|
|
@@ -46,23 +47,21 @@ export function installH2Fetch(): void {
|
|
|
46
47
|
const h2FallbackCodes: ReadonlySet<string> = new Set([
|
|
47
48
|
"HTTP2Unsupported", // Server selected h1 in ALPN
|
|
48
49
|
"ConnectionRefused", // Server refused the h2 connection
|
|
49
|
-
"
|
|
50
|
-
"ConnectionClosed", // Server closed before h2 response
|
|
50
|
+
"HTTP2RefusedStream", // REFUSED_STREAM / never-processed h2 stream
|
|
51
51
|
// Bun's h2 client reports an ALPN-refusing host's TLS abort with this
|
|
52
52
|
// code; the h1 fallback below re-verifies the certificate itself.
|
|
53
53
|
"UNKNOWN_CERTIFICATE_VERIFICATION_ERROR",
|
|
54
54
|
]);
|
|
55
|
-
/** Fallback codes that may fire *after* the peer consumed the body — replay only when safe. */
|
|
56
|
-
const replayGatedCodes: ReadonlySet<string> = new Set(["ConnectionReset", "ConnectionClosed"]);
|
|
57
55
|
const wrapper = async function h2fetch(input: string | URL | Request, init?: RequestInit): Promise<Response> {
|
|
58
56
|
if (!isHttps(input)) return original(input, init);
|
|
57
|
+
const snapshot = snapshotRequest(input, init);
|
|
59
58
|
try {
|
|
60
|
-
return await original(input, { ...init, protocol: "http2" });
|
|
59
|
+
return await original(snapshot.input, { ...snapshot.init, protocol: "http2" });
|
|
61
60
|
} catch (err) {
|
|
62
61
|
const code = (err as { code?: string }).code ?? "";
|
|
63
62
|
if (!h2FallbackCodes.has(code)) throw err;
|
|
64
|
-
if (
|
|
65
|
-
return original(input, init);
|
|
63
|
+
if (code === "HTTP2RefusedStream" && !isReplayableRequest(snapshot.input, snapshot.init)) throw err;
|
|
64
|
+
return original(snapshot.input, snapshot.init);
|
|
66
65
|
}
|
|
67
66
|
} as typeof fetch & PatchedFetch;
|
|
68
67
|
|
|
@@ -72,19 +71,31 @@ export function installH2Fetch(): void {
|
|
|
72
71
|
globalThis.fetch = wrapper;
|
|
73
72
|
}
|
|
74
73
|
|
|
75
|
-
|
|
76
|
-
|
|
74
|
+
function snapshotRequest(
|
|
75
|
+
input: string | URL | Request,
|
|
76
|
+
init?: RequestInit,
|
|
77
|
+
): { input: string | URL | Request; init?: RequestInit } {
|
|
78
|
+
const snapshotInput = input instanceof URL ? new URL(input.href) : input;
|
|
79
|
+
const requestLike = typeof input !== "string" && !(input instanceof URL);
|
|
80
|
+
if (init === undefined && !requestLike) {
|
|
81
|
+
return { input: snapshotInput };
|
|
82
|
+
}
|
|
77
83
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
84
|
+
const snapshotInit: RequestInit = { ...(init ?? {}) };
|
|
85
|
+
if (init?.headers !== undefined) {
|
|
86
|
+
snapshotInit.headers = snapshotHeaders(init.headers);
|
|
87
|
+
} else if (requestLike) {
|
|
88
|
+
snapshotInit.headers = new Headers(input.headers);
|
|
89
|
+
}
|
|
90
|
+
return { input: snapshotInput, init: snapshotInit };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function snapshotHeaders(headers: NonNullable<RequestInit["headers"]>): NonNullable<RequestInit["headers"]> {
|
|
94
|
+
if (headers instanceof Headers) return new Headers(headers);
|
|
95
|
+
if (Array.isArray(headers)) {
|
|
96
|
+
return headers.map(([name, value]) => [name, value] as [string, string]);
|
|
97
|
+
}
|
|
98
|
+
return { ...headers };
|
|
88
99
|
}
|
|
89
100
|
|
|
90
101
|
function isHttps(input: string | URL | Request): boolean {
|
|
@@ -92,3 +103,31 @@ function isHttps(input: string | URL | Request): boolean {
|
|
|
92
103
|
if (input instanceof URL) return input.protocol === "https:";
|
|
93
104
|
return input.url.startsWith("https:");
|
|
94
105
|
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Whether the request body can be handed to fetch a second time. This is only
|
|
109
|
+
* used after `HTTP2RefusedStream`, which proves that the peer did not consume
|
|
110
|
+
* request bytes; a one-shot ReadableStream is still not reusable locally.
|
|
111
|
+
*/
|
|
112
|
+
function isReplayableRequest(input: string | URL | Request, init?: RequestInit): boolean {
|
|
113
|
+
try {
|
|
114
|
+
if (init?.body !== undefined) return isReplayableBody(init.body);
|
|
115
|
+
if (typeof input === "string" || input instanceof URL) return true;
|
|
116
|
+
// Request bodies are exposed as ReadableStreams. The same Request object
|
|
117
|
+
// cannot be used for the second fetch once the first attempt touched it.
|
|
118
|
+
return input.body === null;
|
|
119
|
+
} catch {
|
|
120
|
+
// Cross-realm or proxy Request objects may throw while exposing their
|
|
121
|
+
// method/body. Do not retry when replayability cannot be established.
|
|
122
|
+
return false;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function isReplayableBody(body: RequestInit["body"]): boolean {
|
|
127
|
+
if (body === null || typeof body === "string") return true;
|
|
128
|
+
if (typeof Blob !== "undefined" && body instanceof Blob) return true;
|
|
129
|
+
// ArrayBuffer/views, FormData, and URLSearchParams are mutable caller-owned
|
|
130
|
+
// values. Fail closed instead of replaying a potentially changed payload if
|
|
131
|
+
// the asynchronous refusal arrives after the caller mutates one.
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
@@ -255,11 +255,19 @@ export async function* iterateWithIdleTimeout<T>(
|
|
|
255
255
|
const firstItemTimeoutMs = options.firstItemTimeoutMs ?? options.idleTimeoutMs;
|
|
256
256
|
const abortSignal = options.abortSignal;
|
|
257
257
|
const iterator = iterable[Symbol.asyncIterator]();
|
|
258
|
+
let naturallyExhausted = false;
|
|
259
|
+
let iteratorClosed = false;
|
|
258
260
|
|
|
259
261
|
const closeIterator = (): void => {
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
262
|
+
if (iteratorClosed) return;
|
|
263
|
+
iteratorClosed = true;
|
|
264
|
+
try {
|
|
265
|
+
const returnPromise = iterator.return?.();
|
|
266
|
+
if (returnPromise) {
|
|
267
|
+
void returnPromise.catch(() => {});
|
|
268
|
+
}
|
|
269
|
+
} catch {
|
|
270
|
+
// Cleanup must not replace the source error or timeout that triggered it.
|
|
263
271
|
}
|
|
264
272
|
};
|
|
265
273
|
|
|
@@ -296,109 +304,114 @@ export async function* iterateWithIdleTimeout<T>(
|
|
|
296
304
|
(firstItemTimeoutMs === undefined || firstItemTimeoutMs <= 0) &&
|
|
297
305
|
(options.idleTimeoutMs === undefined || options.idleTimeoutMs <= 0);
|
|
298
306
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
activeTimeoutMs
|
|
307
|
+
try {
|
|
308
|
+
while (true) {
|
|
309
|
+
let activeTimeoutMs: number | undefined;
|
|
310
|
+
if (awaitingFirstItem) {
|
|
311
|
+
activeTimeoutMs =
|
|
312
|
+
firstItemDeadlineAt === undefined ? undefined : Math.max(0, firstItemDeadlineAt - Date.now());
|
|
313
|
+
} else if (options.idleTimeoutMs !== undefined && options.idleTimeoutMs > 0) {
|
|
314
|
+
activeTimeoutMs = options.idleTimeoutMs - (Date.now() - lastProgressAt);
|
|
315
|
+
// The idle deadline may already have elapsed because the *consumer*
|
|
316
|
+
// was slow, not because the provider stalled — and the next item may
|
|
317
|
+
// already be buffered and ready to deliver. Clamp to 0 instead of
|
|
318
|
+
// throwing eagerly so the next() race below still gets a chance to
|
|
319
|
+
// win (it settles on a microtask, ahead of the 0ms timer). Only a
|
|
320
|
+
// genuinely hung iterator loses that race and surfaces as a stall.
|
|
321
|
+
if (activeTimeoutMs < 0) {
|
|
322
|
+
activeTimeoutMs = 0;
|
|
323
|
+
}
|
|
314
324
|
}
|
|
315
|
-
}
|
|
316
325
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
326
|
+
const racers: Array<
|
|
327
|
+
Promise<
|
|
328
|
+
| { kind: "next"; result: IteratorResult<T> }
|
|
329
|
+
| { kind: "error"; error: unknown }
|
|
330
|
+
| { kind: "timeout" }
|
|
331
|
+
| { kind: "abort" }
|
|
332
|
+
>
|
|
333
|
+
> = [];
|
|
325
334
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
+
let timer: NodeJS.Timeout | undefined;
|
|
336
|
+
let resolveTimeout: ((value: { kind: "timeout" }) => void) | undefined;
|
|
337
|
+
const enforceTimeout = !noTimeoutEnforced && activeTimeoutMs !== undefined && activeTimeoutMs >= 0;
|
|
338
|
+
if (enforceTimeout) {
|
|
339
|
+
const { promise, resolve } = Promise.withResolvers<{ kind: "timeout" }>();
|
|
340
|
+
resolveTimeout = resolve;
|
|
341
|
+
timer = setTimeout(() => resolve({ kind: "timeout" }), activeTimeoutMs);
|
|
342
|
+
racers.push(promise);
|
|
343
|
+
}
|
|
335
344
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
+
let abortListener: (() => void) | undefined;
|
|
346
|
+
let resolveAbort: ((value: { kind: "abort" }) => void) | undefined;
|
|
347
|
+
if (abortSignal) {
|
|
348
|
+
const { promise, resolve } = Promise.withResolvers<{ kind: "abort" }>();
|
|
349
|
+
resolveAbort = resolve;
|
|
350
|
+
abortListener = () => resolve({ kind: "abort" });
|
|
351
|
+
abortSignal.addEventListener("abort", abortListener, { once: true });
|
|
352
|
+
racers.push(promise);
|
|
353
|
+
}
|
|
345
354
|
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
355
|
+
// Arm timeout/abort races before asking the source for its next item. A
|
|
356
|
+
// periodic keepalive iterator commonly registers its own timer inside
|
|
357
|
+
// `next()`; registering that first lets equal-deadline keepalives win every
|
|
358
|
+
// race and extend the idle window forever. Already-buffered items still
|
|
359
|
+
// settle as microtasks before a 0ms watchdog.
|
|
360
|
+
racers.unshift(withRacy(iterator.next()));
|
|
352
361
|
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
362
|
+
try {
|
|
363
|
+
const outcome = await Promise.race(racers);
|
|
364
|
+
if (outcome.kind === "abort") {
|
|
365
|
+
closeIterator();
|
|
366
|
+
throw abortReason(abortSignal!);
|
|
367
|
+
}
|
|
368
|
+
if (outcome.kind === "timeout") {
|
|
369
|
+
if (!awaitingFirstItem) {
|
|
370
|
+
options.onIdle?.();
|
|
371
|
+
} else {
|
|
372
|
+
options.onFirstItemTimeout?.();
|
|
373
|
+
}
|
|
374
|
+
closeIterator();
|
|
375
|
+
throw awaitingFirstItem
|
|
376
|
+
? new FirstEventTimeoutError(options.firstItemErrorMessage ?? options.errorMessage)
|
|
377
|
+
: new Error(options.errorMessage);
|
|
378
|
+
}
|
|
379
|
+
if (outcome.kind === "error") {
|
|
380
|
+
throw outcome.error;
|
|
381
|
+
}
|
|
382
|
+
if (awaitingFirstItem && firstItemDeadlineAt !== undefined && Date.now() >= firstItemDeadlineAt) {
|
|
363
383
|
options.onFirstItemTimeout?.();
|
|
384
|
+
closeIterator();
|
|
385
|
+
throw new FirstEventTimeoutError(options.firstItemErrorMessage ?? options.errorMessage);
|
|
364
386
|
}
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
}
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
markFirstItemReceived();
|
|
390
|
-
lastProgressAt = Date.now();
|
|
391
|
-
}
|
|
392
|
-
yield item;
|
|
393
|
-
} finally {
|
|
394
|
-
if (timer !== undefined) clearTimeout(timer);
|
|
395
|
-
// Resolve dangling promises so the racers don't leak (Promise.race is one-shot).
|
|
396
|
-
resolveTimeout?.({ kind: "timeout" });
|
|
397
|
-
if (abortListener && abortSignal) {
|
|
398
|
-
abortSignal.removeEventListener("abort", abortListener);
|
|
387
|
+
if (outcome.result.done) {
|
|
388
|
+
naturallyExhausted = true;
|
|
389
|
+
markFirstItemReceived();
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
const item = outcome.result.value;
|
|
393
|
+
// Non-progress items (e.g. provider keepalives, synthetic `start` events that
|
|
394
|
+
// arrive before the model has produced any tokens) MUST NOT flip us out of
|
|
395
|
+
// `awaitingFirstItem`. Otherwise the next iteration switches from the (longer)
|
|
396
|
+
// first-item watchdog to the (shorter) idle watchdog while we're still waiting
|
|
397
|
+
// on the model's first real output.
|
|
398
|
+
if (isProgressItem(item)) {
|
|
399
|
+
markFirstItemReceived();
|
|
400
|
+
lastProgressAt = Date.now();
|
|
401
|
+
}
|
|
402
|
+
yield item;
|
|
403
|
+
} finally {
|
|
404
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
405
|
+
// Resolve dangling promises so the racers don't leak (Promise.race is one-shot).
|
|
406
|
+
resolveTimeout?.({ kind: "timeout" });
|
|
407
|
+
if (abortListener && abortSignal) {
|
|
408
|
+
abortSignal.removeEventListener("abort", abortListener);
|
|
409
|
+
}
|
|
410
|
+
resolveAbort?.({ kind: "abort" });
|
|
399
411
|
}
|
|
400
|
-
resolveAbort?.({ kind: "abort" });
|
|
401
412
|
}
|
|
413
|
+
} finally {
|
|
414
|
+
if (!naturallyExhausted) closeIterator();
|
|
402
415
|
}
|
|
403
416
|
}
|
|
404
417
|
|
package/src/utils/json-parse.ts
CHANGED
|
@@ -249,6 +249,8 @@ export function findUnnecessaryUnicodeEscape(json: string): string | undefined {
|
|
|
249
249
|
const len = json.length;
|
|
250
250
|
let inString = false;
|
|
251
251
|
let i = 0;
|
|
252
|
+
let nextQuote = -1;
|
|
253
|
+
let nextBackslash = -1;
|
|
252
254
|
|
|
253
255
|
const hexAt = (start: number): number | undefined => {
|
|
254
256
|
if (start + 3 >= len) return undefined;
|
|
@@ -268,10 +270,16 @@ export function findUnnecessaryUnicodeEscape(json: string): string | undefined {
|
|
|
268
270
|
// Jump straight to the next quote or backslash. A per-character walk costs
|
|
269
271
|
// ~40ms on a 1MB literal-UTF-8 payload (a large `write`), and every byte in
|
|
270
272
|
// between is by definition uninteresting.
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
273
|
+
if (nextQuote < i) {
|
|
274
|
+
const offset = json.indexOf('"', i);
|
|
275
|
+
nextQuote = offset === -1 ? len : offset;
|
|
276
|
+
}
|
|
277
|
+
if (nextBackslash < i) {
|
|
278
|
+
const offset = json.indexOf("\\", i);
|
|
279
|
+
nextBackslash = offset === -1 ? len : offset;
|
|
280
|
+
}
|
|
281
|
+
i = Math.min(nextQuote, nextBackslash);
|
|
282
|
+
if (i === len) return undefined;
|
|
275
283
|
|
|
276
284
|
if (json.charCodeAt(i) === QUOTE) {
|
|
277
285
|
inString = false;
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runaway-repetition detector for a model's streamed text or thinking channel.
|
|
3
|
+
*
|
|
4
|
+
* Some models fall into a decode loop and emit the same sentence — or the same
|
|
5
|
+
* short token run — until the turn's budget runs out. Nothing errors: the tool
|
|
6
|
+
* calls in the same message still execute, and the transcript just fills with
|
|
7
|
+
* dozens of identical lines (#5624).
|
|
8
|
+
*
|
|
9
|
+
* This is a pure state machine with no provider knowledge: `feed()` takes a
|
|
10
|
+
* chunk of streamed text and returns the prefix that is safe to emit. Until it
|
|
11
|
+
* trips, that prefix is the chunk itself, so a healthy stream passes through
|
|
12
|
+
* byte for byte. Once it trips, it emits nothing further and {@link takeTrip}
|
|
13
|
+
* hands the caller a one-shot signal to abort the request.
|
|
14
|
+
*
|
|
15
|
+
* Use one instance per stream **per channel**. Interleaving the visible-text
|
|
16
|
+
* and reasoning channels through a single instance would splice unrelated
|
|
17
|
+
* tokens into the same window and manufacture patterns that were never
|
|
18
|
+
* streamed.
|
|
19
|
+
*/
|
|
20
|
+
/** Consecutive repeats of one unit that trip the guard. */
|
|
21
|
+
export declare const DEFAULT_REPETITION_THRESHOLD = 12;
|
|
22
|
+
/**
|
|
23
|
+
* Largest accepted repetition threshold.
|
|
24
|
+
*
|
|
25
|
+
* Token retention is `MAX_NGRAM_TOKENS * (threshold + 1)`, so capping the
|
|
26
|
+
* threshold is what makes the guard's memory bounded: at this cap it retains
|
|
27
|
+
* at most 64 * 129 = 8256 tokens, against 832 at the default. That is roughly
|
|
28
|
+
* ten times the default's headroom — generous for a caller who genuinely wants
|
|
29
|
+
* a laxer guard — while keeping a hostile or buggy option from turning the
|
|
30
|
+
* detector into an unbounded buffer (#5627 review r6).
|
|
31
|
+
*/
|
|
32
|
+
export declare const MAX_REPETITION_THRESHOLD = 128;
|
|
33
|
+
/**
|
|
34
|
+
* `errorCode` stamped on a turn this guard stopped. A bounded classifier, never
|
|
35
|
+
* raw model text — consumers branch on it to tell a local decode-loop stop from
|
|
36
|
+
* a client cancellation or a transport fault (#5627).
|
|
37
|
+
*/
|
|
38
|
+
export declare const REPETITION_GUARD_ERROR_CODE = "repetition_guard_tripped";
|
|
39
|
+
/**
|
|
40
|
+
* Wire-safe `errorMessage` for a turn this guard stopped. A literal with zero
|
|
41
|
+
* interpolation — not the sample, not the channel, not the repeat count.
|
|
42
|
+
*
|
|
43
|
+
* The auth gateway forwards `errorMessage` to API clients on the streaming path
|
|
44
|
+
* (`redactGatewayMessage` only strips credential-shaped text), so anything
|
|
45
|
+
* interpolated here is raw model output published verbatim. It also reaches
|
|
46
|
+
* `classifyGatewayError`, which keyword-matches on message text, so a repeated
|
|
47
|
+
* `quota` or `forbidden` in a sample could pick the HTTP status (#5627 r5).
|
|
48
|
+
*
|
|
49
|
+
* The repeated unit is not logged either: the provider logs bounded metadata
|
|
50
|
+
* only, because the default log transport persists metadata verbatim to a
|
|
51
|
+
* rotating file on disk (#5627 review r6). {@link StreamRepetitionTrip.sample}
|
|
52
|
+
* stays in memory for callers that want it.
|
|
53
|
+
*/
|
|
54
|
+
export declare const REPETITION_GUARD_STOP_MESSAGE = "Stopped the turn: the model produced runaway repeated output.";
|
|
55
|
+
export type RepetitionUnitKind = "line" | "ngram";
|
|
56
|
+
export interface StreamRepetitionTrip {
|
|
57
|
+
/** Whether the repeats were whole lines or an n-gram inside one line. */
|
|
58
|
+
readonly kind: RepetitionUnitKind;
|
|
59
|
+
/** Consecutive repeats observed when the guard tripped. */
|
|
60
|
+
readonly repeats: number;
|
|
61
|
+
/** Normalized, truncated sample of the repeated unit, for diagnostics. */
|
|
62
|
+
readonly sample: string;
|
|
63
|
+
}
|
|
64
|
+
export interface StreamRepetitionGuardOptions {
|
|
65
|
+
/**
|
|
66
|
+
* Consecutive repeats that trip the guard. Defaults to 12. Normalized by
|
|
67
|
+
* {@link normalizeThreshold} — non-finite values fall back to the default,
|
|
68
|
+
* fractional values are floored, and the result is clamped into
|
|
69
|
+
* `[2, MAX_REPETITION_THRESHOLD]`.
|
|
70
|
+
*/
|
|
71
|
+
readonly threshold?: number;
|
|
72
|
+
}
|
|
73
|
+
export declare class StreamRepetitionGuard {
|
|
74
|
+
#private;
|
|
75
|
+
constructor(options?: StreamRepetitionGuardOptions);
|
|
76
|
+
/**
|
|
77
|
+
* The threshold actually in force — the caller's option after
|
|
78
|
+
* {@link normalizeThreshold}, which may differ from what was passed.
|
|
79
|
+
*/
|
|
80
|
+
get threshold(): number;
|
|
81
|
+
get tripped(): boolean;
|
|
82
|
+
get trip(): StreamRepetitionTrip | undefined;
|
|
83
|
+
/**
|
|
84
|
+
* Returns the trip exactly once, then `undefined` forever. Callers drive a
|
|
85
|
+
* one-shot side effect (aborting the request) off this, so the once-only
|
|
86
|
+
* latch lives here rather than being re-implemented at each call site.
|
|
87
|
+
*/
|
|
88
|
+
takeTrip(): StreamRepetitionTrip | undefined;
|
|
89
|
+
/**
|
|
90
|
+
* Feed a chunk of streamed text. Returns the portion safe to emit: the whole
|
|
91
|
+
* chunk while healthy, the prefix up to the repeat that tripped the guard on
|
|
92
|
+
* the chunk that trips it, and nothing at all after that.
|
|
93
|
+
*/
|
|
94
|
+
feed(text: string): string;
|
|
95
|
+
/**
|
|
96
|
+
* Close the in-progress unit at end of stream and run detection once more.
|
|
97
|
+
*
|
|
98
|
+
* `feed()` only closes a token on whitespace and a line on `\n`, so a stream
|
|
99
|
+
* whose final repeat arrives without a trailing newline left the last copy
|
|
100
|
+
* uncounted and the turn read as a healthy completion (#5627 review r5).
|
|
101
|
+
*
|
|
102
|
+
* Emits nothing — everything `feed()` returned has already been rendered by
|
|
103
|
+
* the time this runs. A trip found here therefore classifies the turn while
|
|
104
|
+
* the last copy is already on screen; that is intended. Idempotent.
|
|
105
|
+
*/
|
|
106
|
+
finalize(): void;
|
|
107
|
+
}
|