@ssweens/pi-vertex 1.1.12 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +115 -0
- package/README.md +66 -35
- package/TEST_COVERAGE.md +22 -2
- package/auth.ts +1 -1
- package/config.ts +1 -1
- package/index.ts +16 -7
- package/models/claude.ts +213 -20
- package/models/gemini.ts +171 -42
- package/models/maas.ts +127 -261
- package/package.json +15 -6
- package/streaming/gemini.ts +69 -34
- package/streaming/index.ts +30 -1
- package/streaming/maas.ts +71 -33
- package/streaming/sse-comment-filter.ts +169 -0
- package/tests/first-byte-timeout-check.mjs +53 -0
- package/types.ts +22 -2
- package/utils.ts +5 -3
package/streaming/index.ts
CHANGED
|
@@ -3,15 +3,44 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import type { VertexModelConfig, Context, StreamOptions } from "../types.js";
|
|
6
|
-
import type { AssistantMessageEventStream } from "@
|
|
6
|
+
import type { AssistantMessageEventStream } from "@earendil-works/pi-ai";
|
|
7
|
+
import {
|
|
8
|
+
getCurrentSystemMessage,
|
|
9
|
+
getCurrentTools,
|
|
10
|
+
getSystemMessageText,
|
|
11
|
+
} from "@earendil-works/pi-ai";
|
|
7
12
|
import { streamGemini } from "./gemini.js";
|
|
8
13
|
import { streamMaaS } from "./maas.js";
|
|
14
|
+
import { installSseCommentFilter } from "./sse-comment-filter.js";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Pi 0.99 folds the system prompt and tool definitions into system messages
|
|
18
|
+
* (`toolsAdded`/`toolsRemoved` deltas replayed in order); the agent loop no
|
|
19
|
+
* longer populates `context.systemPrompt`/`context.tools`, which are legacy
|
|
20
|
+
* shorthand. The streaming paths read the shorthand fields, so resolve them
|
|
21
|
+
* from the transcript here. Explicit fields still win when a caller sets them.
|
|
22
|
+
*/
|
|
23
|
+
function resolveContext(context: Context): Context {
|
|
24
|
+
if (context.systemPrompt !== undefined && context.tools !== undefined) return context;
|
|
25
|
+
const systemMessage = getCurrentSystemMessage(context.messages);
|
|
26
|
+
return {
|
|
27
|
+
...context,
|
|
28
|
+
systemPrompt: context.systemPrompt ?? (systemMessage ? getSystemMessageText(systemMessage) : undefined),
|
|
29
|
+
tools: context.tools ?? getCurrentTools(context.messages),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
9
32
|
|
|
10
33
|
export function streamVertex(
|
|
11
34
|
model: VertexModelConfig,
|
|
12
35
|
context: Context,
|
|
13
36
|
options?: StreamOptions
|
|
14
37
|
): AssistantMessageEventStream {
|
|
38
|
+
// Vertex injects keepalive lines (": keepalive" / "data: : keepalive") on slow
|
|
39
|
+
// streams; SDK parsers choke on them. Strip at the fetch layer (installed once).
|
|
40
|
+
installSseCommentFilter();
|
|
41
|
+
|
|
42
|
+
context = resolveContext(context);
|
|
43
|
+
|
|
15
44
|
switch (model.endpointType) {
|
|
16
45
|
case "gemini":
|
|
17
46
|
return streamGemini(model, context, options);
|
package/streaming/maas.ts
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* MaaS streaming handler for Claude and all other models.
|
|
3
3
|
*
|
|
4
4
|
* - Anthropic models: native AnthropicVertex SDK streaming
|
|
5
|
-
* - Other MaaS models: Vertex OpenAI-compatible Chat Completions endpoint
|
|
5
|
+
* - Other MaaS models: Vertex OpenAI-compatible Chat Completions endpoint, or the
|
|
6
|
+
* publisher's OpenAI-compatible Responses API for models with `maasApi: "responses"`
|
|
6
7
|
*/
|
|
7
8
|
|
|
8
9
|
import type { VertexModelConfig, Context, StreamOptions } from "../types.js";
|
|
@@ -11,10 +12,9 @@ import {
|
|
|
11
12
|
createAssistantMessageEventStream,
|
|
12
13
|
type AssistantMessageEventStream,
|
|
13
14
|
type Model,
|
|
14
|
-
|
|
15
|
+
type SimpleStreamOptions,
|
|
15
16
|
calculateCost,
|
|
16
|
-
} from "@
|
|
17
|
-
import { AnthropicVertex } from "@anthropic-ai/vertex-sdk";
|
|
17
|
+
} from "@earendil-works/pi-ai";
|
|
18
18
|
|
|
19
19
|
function mapAnthropicEffort(reasoning?: string): "low" | "medium" | "high" | "max" | undefined {
|
|
20
20
|
if (!reasoning) return undefined;
|
|
@@ -52,6 +52,7 @@ async function streamAnthropic(
|
|
|
52
52
|
): Promise<void> {
|
|
53
53
|
const location = resolveLocation(model.region);
|
|
54
54
|
const auth = getAuthConfig(location);
|
|
55
|
+
const { AnthropicVertex } = await import("@anthropic-ai/vertex-sdk");
|
|
55
56
|
|
|
56
57
|
// Use regional pricing when the resolved endpoint is not the global one.
|
|
57
58
|
// Models without costRegional (e.g. Opus 4.1, Sonnet 4) have uniform pricing.
|
|
@@ -328,16 +329,69 @@ async function streamAnthropic(
|
|
|
328
329
|
stream.push({ type: "done", reason: output.stopReason, message: output });
|
|
329
330
|
}
|
|
330
331
|
|
|
332
|
+
/** Chat Completions model on the Vertex OpenAI-compatible endpoint (`/endpoints/openapi`). */
|
|
333
|
+
function chatCompletionsModel(model: VertexModelConfig, baseUrl: string): Model<"openai-completions"> {
|
|
334
|
+
return {
|
|
335
|
+
id: model.apiId.includes("/") ? model.apiId : `${model.publisher}/${model.apiId}`,
|
|
336
|
+
name: model.name,
|
|
337
|
+
api: "openai-completions",
|
|
338
|
+
provider: "vertex",
|
|
339
|
+
baseUrl: `${baseUrl}/endpoints/openapi`,
|
|
340
|
+
reasoning: model.reasoning,
|
|
341
|
+
thinkingLevelMap: model.thinkingLevelMap,
|
|
342
|
+
input: model.input,
|
|
343
|
+
cost: model.cost,
|
|
344
|
+
contextWindow: model.contextWindow,
|
|
345
|
+
maxTokens: model.maxTokens,
|
|
346
|
+
headers: {},
|
|
347
|
+
compat: {
|
|
348
|
+
supportsStore: false,
|
|
349
|
+
supportsDeveloperRole: false,
|
|
350
|
+
// DeepSeek on Vertex thinks only when `reasoning_effort` is present.
|
|
351
|
+
supportsReasoningEffort: model.publisher === "deepseek-ai",
|
|
352
|
+
maxTokensField: "max_tokens",
|
|
353
|
+
thinkingFormat: model.publisher === "zai-org" ? "zai" : "openai",
|
|
354
|
+
},
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Responses model on the publisher's OpenAI-compatible endpoint
|
|
360
|
+
* (`/publishers/{publisher}/v1/responses`), addressed by bare model id.
|
|
361
|
+
* pi-ai always sends `store: false`, which these endpoints require.
|
|
362
|
+
*/
|
|
363
|
+
function responsesModel(model: VertexModelConfig, baseUrl: string): Model<"openai-responses"> {
|
|
364
|
+
return {
|
|
365
|
+
id: model.apiId,
|
|
366
|
+
name: model.name,
|
|
367
|
+
api: "openai-responses",
|
|
368
|
+
provider: "vertex",
|
|
369
|
+
baseUrl: `${baseUrl}/publishers/${model.publisher}/v1`,
|
|
370
|
+
reasoning: model.reasoning,
|
|
371
|
+
// Effort `none` is rejected; `off: null` makes pi-ai omit `reasoning` when no
|
|
372
|
+
// effort is requested, so the model runs at its default.
|
|
373
|
+
thinkingLevelMap: { ...model.thinkingLevelMap, off: null },
|
|
374
|
+
input: model.input,
|
|
375
|
+
cost: model.cost,
|
|
376
|
+
contextWindow: model.contextWindow,
|
|
377
|
+
maxTokens: model.maxTokens,
|
|
378
|
+
headers: {},
|
|
379
|
+
compat: {
|
|
380
|
+
// No 24h retention on Vertex: never send `prompt_cache_retention`.
|
|
381
|
+
supportsLongCacheRetention: false,
|
|
382
|
+
},
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
|
|
331
386
|
export function streamMaaS(
|
|
332
387
|
model: VertexModelConfig,
|
|
333
388
|
context: Context,
|
|
334
389
|
options?: StreamOptions,
|
|
335
390
|
): AssistantMessageEventStream {
|
|
336
391
|
const stream = createAssistantMessageEventStream();
|
|
392
|
+
const useResponses = model.maasApi === "responses";
|
|
337
393
|
|
|
338
394
|
(async () => {
|
|
339
|
-
const apiModelId = model.apiId.includes("/") ? model.apiId : `${model.publisher}/${model.apiId}`;
|
|
340
|
-
|
|
341
395
|
try {
|
|
342
396
|
if (model.publisher === "anthropic") {
|
|
343
397
|
await streamAnthropic(model, context, options, stream);
|
|
@@ -345,40 +399,24 @@ export function streamMaaS(
|
|
|
345
399
|
return;
|
|
346
400
|
}
|
|
347
401
|
|
|
348
|
-
// Non-Anthropic MaaS models: Vertex OpenAI-compatible
|
|
402
|
+
// Non-Anthropic MaaS models: Vertex OpenAI-compatible endpoints via pi-ai.
|
|
349
403
|
const location = resolveLocation(model.region);
|
|
350
404
|
const auth = getAuthConfig(location);
|
|
351
405
|
const accessToken = await getAccessToken();
|
|
352
406
|
const baseUrl = buildBaseUrl(auth.projectId, auth.location);
|
|
353
|
-
const
|
|
354
|
-
|
|
355
|
-
const modelForPi: Model<"openai-completions"> = {
|
|
356
|
-
id: apiModelId,
|
|
357
|
-
name: model.name,
|
|
358
|
-
api: "openai-completions",
|
|
359
|
-
provider: "vertex",
|
|
360
|
-
baseUrl: endpoint,
|
|
361
|
-
reasoning: model.reasoning,
|
|
362
|
-
input: model.input,
|
|
363
|
-
cost: model.cost,
|
|
364
|
-
contextWindow: model.contextWindow,
|
|
365
|
-
maxTokens: model.maxTokens,
|
|
366
|
-
headers: {},
|
|
367
|
-
compat: {
|
|
368
|
-
supportsStore: false,
|
|
369
|
-
supportsDeveloperRole: false,
|
|
370
|
-
supportsReasoningEffort: false,
|
|
371
|
-
maxTokensField: "max_tokens",
|
|
372
|
-
thinkingFormat: model.publisher === "qwen" ? "qwen" : model.publisher === "zai-org" ? "zai" : "openai",
|
|
373
|
-
},
|
|
374
|
-
};
|
|
375
|
-
|
|
376
|
-
const innerStream = streamSimpleOpenAICompletions(modelForPi, context as any, {
|
|
407
|
+
const innerOptions: SimpleStreamOptions = {
|
|
377
408
|
...options,
|
|
378
409
|
apiKey: accessToken,
|
|
379
410
|
maxTokens: options?.maxTokens || model.maxTokens,
|
|
380
411
|
temperature: options?.temperature,
|
|
381
|
-
}
|
|
412
|
+
};
|
|
413
|
+
|
|
414
|
+
// Pi 0.99 moved per-API stream functions off the pi-ai root.
|
|
415
|
+
// Pi's extension loader maps the compat entry at runtime.
|
|
416
|
+
const { openAICompletionsApi, openAIResponsesApi } = await import("@earendil-works/pi-ai/compat");
|
|
417
|
+
const innerStream = useResponses
|
|
418
|
+
? openAIResponsesApi().streamSimple(responsesModel(model, baseUrl), context as any, innerOptions)
|
|
419
|
+
: openAICompletionsApi().streamSimple(chatCompletionsModel(model, baseUrl), context as any, innerOptions);
|
|
382
420
|
|
|
383
421
|
for await (const event of innerStream) {
|
|
384
422
|
if ("partial" in event && event.partial) event.partial.model = model.id;
|
|
@@ -399,7 +437,7 @@ export function streamMaaS(
|
|
|
399
437
|
error: {
|
|
400
438
|
role: "assistant",
|
|
401
439
|
content: [],
|
|
402
|
-
api: model.publisher === "anthropic" ? "anthropic-messages" : "openai-completions",
|
|
440
|
+
api: model.publisher === "anthropic" ? "anthropic-messages" : useResponses ? "openai-responses" : "openai-completions",
|
|
403
441
|
provider: "vertex",
|
|
404
442
|
model: model.id,
|
|
405
443
|
usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } },
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fetch wrapper that strips keepalive noise from Vertex AI streaming responses.
|
|
3
|
+
*
|
|
4
|
+
* On slow streams (long prefill, hidden reasoning, large context) Vertex AI
|
|
5
|
+
* injects keepalive lines that downstream SDK parsers choke on:
|
|
6
|
+
*
|
|
7
|
+
* 1. `: keepalive` — SSE comment (Gemini streamGenerateContent).
|
|
8
|
+
* Spec-ignorable, but @google/genai's hand-rolled parser doesn't skip
|
|
9
|
+
* comments: "Unexpected token ':'..." / "Incomplete JSON segment at the
|
|
10
|
+
* end" / silently dropped data events.
|
|
11
|
+
*
|
|
12
|
+
* 2. `data: : keepalive` — a *data* event whose payload is a pseudo-comment
|
|
13
|
+
* (MaaS /endpoints/openapi chat completions, observed with e.g. Grok).
|
|
14
|
+
* Not a comment at all, so even spec-correct SSE parsers (openai SDK)
|
|
15
|
+
* hand `: keepalive` to JSON.parse:
|
|
16
|
+
* "Unexpected token ':', \": keepalive\" is not valid JSON"
|
|
17
|
+
*
|
|
18
|
+
* Both are safe to drop: a valid JSON payload can never start with ':'.
|
|
19
|
+
* Scoped to Vertex AI (aiplatform.googleapis.com) streaming URLs only.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
let installed = false;
|
|
23
|
+
|
|
24
|
+
/** True for SSE comments and for data events carrying a pseudo-comment payload. */
|
|
25
|
+
function isKeepaliveLine(line: string): boolean {
|
|
26
|
+
if (line.startsWith(":")) return true; // SSE comment
|
|
27
|
+
if (line.startsWith("data:")) {
|
|
28
|
+
const payload = line.substring(5).trimStart();
|
|
29
|
+
return payload.startsWith(":"); // e.g. "data: : keepalive"
|
|
30
|
+
}
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* TransformStream that removes keepalive lines from a byte stream while
|
|
36
|
+
* preserving all other bytes (including event-delimiting blank lines).
|
|
37
|
+
*/
|
|
38
|
+
function createKeepaliveStripper(): TransformStream<Uint8Array, Uint8Array> {
|
|
39
|
+
const decoder = new TextDecoder("utf-8");
|
|
40
|
+
const encoder = new TextEncoder();
|
|
41
|
+
let pending = "";
|
|
42
|
+
|
|
43
|
+
const filter = (text: string, isFlush: boolean): string => {
|
|
44
|
+
pending += text;
|
|
45
|
+
let output = "";
|
|
46
|
+
let start = 0;
|
|
47
|
+
while (true) {
|
|
48
|
+
const nl = pending.indexOf("\n", start);
|
|
49
|
+
if (nl === -1) break;
|
|
50
|
+
const line = pending.substring(start, nl + 1);
|
|
51
|
+
// Strip \r\n / \n terminated keepalive lines; keep everything else.
|
|
52
|
+
const content = line.endsWith("\r\n") ? line.slice(0, -2) : line.slice(0, -1);
|
|
53
|
+
if (!isKeepaliveLine(content)) output += line;
|
|
54
|
+
start = nl + 1;
|
|
55
|
+
}
|
|
56
|
+
pending = pending.substring(start);
|
|
57
|
+
if (isFlush && pending.length > 0) {
|
|
58
|
+
// Trailing data without a newline: drop only if it's a keepalive.
|
|
59
|
+
if (!isKeepaliveLine(pending)) output += pending;
|
|
60
|
+
pending = "";
|
|
61
|
+
}
|
|
62
|
+
return output;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
return new TransformStream<Uint8Array, Uint8Array>({
|
|
66
|
+
transform(chunk, controller) {
|
|
67
|
+
const out = filter(decoder.decode(chunk, { stream: true }), false);
|
|
68
|
+
if (out.length > 0) controller.enqueue(encoder.encode(out));
|
|
69
|
+
},
|
|
70
|
+
flush(controller) {
|
|
71
|
+
const out = filter(decoder.decode(), true);
|
|
72
|
+
if (out.length > 0) controller.enqueue(encoder.encode(out));
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Some Vertex deployments accept a streaming request, answer 200, and then send nothing at
|
|
79
|
+
* all \u2014 observed on `zai-org/glm-5.2-maas` (global) for roughly half of all requests, while
|
|
80
|
+
* glm-4.7 and deepseek on the same endpoint and credentials never did it. undici only gives up
|
|
81
|
+
* after its 300 s body timeout, so every dead request costs five minutes before a retry starts.
|
|
82
|
+
*
|
|
83
|
+
* Bound the wait for the *first* byte only. A stream that has begun is left alone: a genuinely
|
|
84
|
+
* slow one keeps the socket alive with keepalive lines (stripped further down, but counted here
|
|
85
|
+
* because this sits upstream of the stripper), and mid-stream gaps stay undici's business.
|
|
86
|
+
*/
|
|
87
|
+
const FIRST_BYTE_TIMEOUT_MS = Number(process.env.PI_VERTEX_FIRST_BYTE_TIMEOUT_MS ?? 90_000);
|
|
88
|
+
|
|
89
|
+
function withFirstByteTimeout(body: ReadableStream<Uint8Array>, url: string, timeoutMs: number): ReadableStream<Uint8Array> {
|
|
90
|
+
if (!(timeoutMs > 0)) return body;
|
|
91
|
+
const reader = body.getReader();
|
|
92
|
+
let started = false;
|
|
93
|
+
return new ReadableStream<Uint8Array>({
|
|
94
|
+
async pull(controller) {
|
|
95
|
+
if (started) {
|
|
96
|
+
const { done, value } = await reader.read();
|
|
97
|
+
if (done) controller.close();
|
|
98
|
+
else controller.enqueue(value);
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
102
|
+
try {
|
|
103
|
+
const first = await Promise.race([
|
|
104
|
+
reader.read(),
|
|
105
|
+
new Promise<never>((_, reject) => {
|
|
106
|
+
timer = setTimeout(
|
|
107
|
+
() => reject(new Error(`Vertex sent no data for ${Math.round(timeoutMs / 1000)}s after accepting this streaming request (${url}). The endpoint is not producing output; retry or use another offering. Raise PI_VERTEX_FIRST_BYTE_TIMEOUT_MS to wait longer, 0 to disable.`)),
|
|
108
|
+
timeoutMs,
|
|
109
|
+
);
|
|
110
|
+
}),
|
|
111
|
+
]);
|
|
112
|
+
started = true;
|
|
113
|
+
if (first.done) controller.close();
|
|
114
|
+
else controller.enqueue(first.value);
|
|
115
|
+
} catch (error) {
|
|
116
|
+
await reader.cancel(error).catch(() => {});
|
|
117
|
+
controller.error(error);
|
|
118
|
+
} finally {
|
|
119
|
+
clearTimeout(timer);
|
|
120
|
+
}
|
|
121
|
+
},
|
|
122
|
+
cancel(reason) { return reader.cancel(reason); },
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function requestUrl(input: RequestInfo | URL): string {
|
|
127
|
+
if (typeof input === "string") return input;
|
|
128
|
+
if (input instanceof URL) return input.href;
|
|
129
|
+
return input.url;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Vertex AI streaming endpoints: Gemini SSE, MaaS chat completions, publisher
|
|
134
|
+
* Responses API (Meta Muse), Anthropic raw predict.
|
|
135
|
+
*/
|
|
136
|
+
function isVertexStreamingUrl(url: string): boolean {
|
|
137
|
+
if (!url.includes("aiplatform.googleapis.com")) return false;
|
|
138
|
+
return (
|
|
139
|
+
url.includes("streamGenerateContent") ||
|
|
140
|
+
url.includes("/endpoints/openapi/") ||
|
|
141
|
+
url.includes("/v1/responses") ||
|
|
142
|
+
url.includes("streamRawPredict")
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Install a global fetch wrapper (once) that pipes Vertex AI streaming
|
|
148
|
+
* response bodies through the keepalive stripper. All other requests pass
|
|
149
|
+
* through untouched.
|
|
150
|
+
*/
|
|
151
|
+
export function installSseCommentFilter(): void {
|
|
152
|
+
if (installed) return;
|
|
153
|
+
installed = true;
|
|
154
|
+
|
|
155
|
+
const originalFetch = globalThis.fetch;
|
|
156
|
+
globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
|
157
|
+
const response = await originalFetch(input, init);
|
|
158
|
+
const url = requestUrl(input);
|
|
159
|
+
if (!isVertexStreamingUrl(url) || !response.body) {
|
|
160
|
+
return response;
|
|
161
|
+
}
|
|
162
|
+
const live = withFirstByteTimeout(response.body, url, FIRST_BYTE_TIMEOUT_MS);
|
|
163
|
+
return new Response(live.pipeThrough(createKeepaliveStripper()), {
|
|
164
|
+
headers: response.headers,
|
|
165
|
+
status: response.status,
|
|
166
|
+
statusText: response.statusText,
|
|
167
|
+
});
|
|
168
|
+
};
|
|
169
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/** The keepalive filter must also give up on a Vertex stream that never starts. */
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { once } from "node:events";
|
|
5
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
6
|
+
|
|
7
|
+
process.env.PI_VERTEX_FIRST_BYTE_TIMEOUT_MS = "300";
|
|
8
|
+
const { installSseCommentFilter } = await import("../streaming/sse-comment-filter.ts");
|
|
9
|
+
|
|
10
|
+
const open = new Set();
|
|
11
|
+
const server = createServer(async (req, res) => {
|
|
12
|
+
open.add(res); res.on("close", () => open.delete(res));
|
|
13
|
+
res.writeHead(200, { "content-type": "text/event-stream" });
|
|
14
|
+
res.flushHeaders(); // 200 arrives, body may not: exactly what the failing deployment does
|
|
15
|
+
if (req.url.endsWith("/dead")) return; // accepted, then nothing: the glm-5.2 failure shape
|
|
16
|
+
if (req.url.endsWith("/slow-start")) { await sleep(150); }
|
|
17
|
+
res.write(": keepalive\n");
|
|
18
|
+
await sleep(500); // a mid-stream gap longer than the first-byte budget must survive
|
|
19
|
+
res.write(`data: {"choices":[{"delta":{"content":"LATE-BUT-ALIVE"}}]}\n\n`);
|
|
20
|
+
res.end("data: [DONE]\n\n");
|
|
21
|
+
});
|
|
22
|
+
server.listen(0, "127.0.0.1"); await once(server, "listening");
|
|
23
|
+
const { port } = server.address();
|
|
24
|
+
// The wrapper matches Vertex streaming URLs by substring, so a local URL carrying those
|
|
25
|
+
// markers in its path exercises the real wrapper without DNS or credentials.
|
|
26
|
+
const base = `http://127.0.0.1:${port}/aiplatform.googleapis.com/endpoints/openapi`;
|
|
27
|
+
// Meta Muse streams from the publisher Responses API instead of /endpoints/openapi.
|
|
28
|
+
const responsesBase = `http://127.0.0.1:${port}/aiplatform.googleapis.com/publishers/meta/v1/responses`;
|
|
29
|
+
installSseCommentFilter();
|
|
30
|
+
|
|
31
|
+
async function read(path, root = base) {
|
|
32
|
+
const response = await fetch(`${root}${path}`);
|
|
33
|
+
let body = "";
|
|
34
|
+
for await (const chunk of response.body) body += new TextDecoder().decode(chunk);
|
|
35
|
+
return body;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const started = Date.now();
|
|
39
|
+
await assert.rejects(read("/dead"), /sent no data for 0s after accepting this streaming request/);
|
|
40
|
+
const elapsed = Date.now() - started;
|
|
41
|
+
assert(elapsed < 3000, `a dead stream must fail fast, took ${elapsed}ms`);
|
|
42
|
+
assert(elapsed >= 300, `must wait its budget first, gave up after ${elapsed}ms`);
|
|
43
|
+
await assert.rejects(read("/dead", responsesBase), /sent no data/, "a dead Responses API stream must fail fast too");
|
|
44
|
+
|
|
45
|
+
for (const path of ["/live", "/slow-start"]) {
|
|
46
|
+
const body = await read(path);
|
|
47
|
+
assert.match(body, /LATE-BUT-ALIVE/, `${path} must survive a mid-stream gap longer than the first-byte budget`);
|
|
48
|
+
assert.doesNotMatch(body, /keepalive/, `${path} must still have keepalive lines stripped`);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
for (const res of open) res.destroy();
|
|
52
|
+
server.close(); await once(server, "close");
|
|
53
|
+
console.log("PASS first-byte timeout fails a dead Vertex stream fast and leaves live ones alone");
|
package/types.ts
CHANGED
|
@@ -22,13 +22,26 @@ export type {
|
|
|
22
22
|
ToolResultMessage,
|
|
23
23
|
Usage,
|
|
24
24
|
UserMessage,
|
|
25
|
-
} from "@
|
|
25
|
+
} from "@earendil-works/pi-ai";
|
|
26
|
+
|
|
27
|
+
import type { ThinkingLevel, ThinkingLevelMap } from "@earendil-works/pi-ai";
|
|
26
28
|
|
|
27
29
|
// Vertex-specific types
|
|
28
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Mirrors pi-ai's `Model["input"]`, which is `("text" | "image")[]`.
|
|
33
|
+
* Do not widen: pi-coding-agent rejects any other modality at registration.
|
|
34
|
+
*/
|
|
29
35
|
export type ModelInputType = "text" | "image";
|
|
30
36
|
export type EndpointType = "gemini" | "maas";
|
|
31
37
|
|
|
38
|
+
/**
|
|
39
|
+
* Wire API for non-Anthropic MaaS models. Most use the OpenAI-compatible Chat
|
|
40
|
+
* Completions endpoint (`/endpoints/openapi`); some publishers (Meta Muse) only
|
|
41
|
+
* serve the OpenAI-compatible Responses API (`/publishers/{publisher}/v1/responses`).
|
|
42
|
+
*/
|
|
43
|
+
export type MaasApi = "chat-completions" | "responses";
|
|
44
|
+
|
|
32
45
|
export interface ModelCost {
|
|
33
46
|
input: number;
|
|
34
47
|
output: number;
|
|
@@ -59,6 +72,13 @@ export interface VertexModelConfig {
|
|
|
59
72
|
*/
|
|
60
73
|
costRegional?: ModelCost;
|
|
61
74
|
region: string;
|
|
75
|
+
/**
|
|
76
|
+
* Pi thinking level → provider level. Passed to Pi so the selector offers only
|
|
77
|
+
* supported levels: `null` hides a level, and `xhigh`/`max` appear only when set.
|
|
78
|
+
*/
|
|
79
|
+
thinkingLevelMap?: ThinkingLevelMap;
|
|
80
|
+
/** Non-Anthropic MaaS wire API. Default: "chat-completions". */
|
|
81
|
+
maasApi?: MaasApi;
|
|
62
82
|
}
|
|
63
83
|
|
|
64
84
|
export interface AuthConfig {
|
|
@@ -70,6 +90,6 @@ export interface AuthConfig {
|
|
|
70
90
|
export interface StreamOptions {
|
|
71
91
|
maxTokens?: number;
|
|
72
92
|
temperature?: number;
|
|
73
|
-
reasoning?:
|
|
93
|
+
reasoning?: ThinkingLevel;
|
|
74
94
|
signal?: AbortSignal;
|
|
75
95
|
}
|
package/utils.ts
CHANGED
|
@@ -80,7 +80,9 @@ function supportsMultimodalFunctionResponse(modelId: string): boolean {
|
|
|
80
80
|
*/
|
|
81
81
|
export function convertToGeminiMessages(messages: Message[], modelId: string): GeminiContent[] {
|
|
82
82
|
const result: GeminiContent[] = [];
|
|
83
|
-
|
|
83
|
+
// Gemini 3 and 4 require thoughtSignature on all functionCall parts.
|
|
84
|
+
const requiresThoughtSignature =
|
|
85
|
+
modelId.startsWith("gemini-3") || modelId.startsWith("gemini-4");
|
|
84
86
|
let pendingToolCalls: ToolCall[] = [];
|
|
85
87
|
let existingToolResultIds = new Set<string>();
|
|
86
88
|
|
|
@@ -225,8 +227,8 @@ export function convertToGeminiMessages(messages: Message[], modelId: string): G
|
|
|
225
227
|
};
|
|
226
228
|
if (thoughtSig) {
|
|
227
229
|
part.thoughtSignature = thoughtSig;
|
|
228
|
-
} else if (
|
|
229
|
-
// Gemini 3
|
|
230
|
+
} else if (requiresThoughtSignature) {
|
|
231
|
+
// Gemini 3/4 require thoughtSignature on all functionCall parts.
|
|
230
232
|
// For cross-provider tool calls (or rare same-provider calls without signatures),
|
|
231
233
|
// use the documented escape hatch to bypass validation.
|
|
232
234
|
// See: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/thought-signatures
|