@ssweens/pi-vertex 1.1.13 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +125 -0
- package/README.md +86 -36
- package/TEST_COVERAGE.md +36 -2
- package/auth.ts +1 -1
- package/config.ts +1 -1
- package/index.ts +16 -7
- package/models/claude.ts +188 -22
- package/models/gemini.ts +200 -42
- package/models/maas.ts +241 -201
- package/package.json +15 -6
- package/streaming/gemini.ts +71 -35
- package/streaming/index.ts +30 -1
- package/streaming/maas.ts +90 -33
- package/streaming/sse-comment-filter.ts +169 -0
- package/tests/first-byte-timeout-check.mjs +56 -0
- package/tests/model-catalog-check.mjs +122 -0
- package/types.ts +22 -2
- package/utils.ts +5 -3
package/streaming/gemini.ts
CHANGED
|
@@ -4,59 +4,69 @@
|
|
|
4
4
|
* Aligned with pi-mono's google-vertex.ts for consistent handling of:
|
|
5
5
|
* - Thinking content (thought blocks with signatures)
|
|
6
6
|
* - Tool calls with unique IDs and deduplication
|
|
7
|
-
* - Thinking configuration (levels for Gemini 3, budgets for Gemini 2.5)
|
|
7
|
+
* - Thinking configuration (levels for Gemini 3/4, budgets for Gemini 2.5)
|
|
8
8
|
* - Usage tracking including thinking tokens
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import { GoogleGenAI, FinishReason, ThinkingLevel } from "@google/genai";
|
|
12
11
|
import type { VertexModelConfig, Context, StreamOptions, AssistantMessage } from "../types.js";
|
|
13
12
|
import { getAuthConfig, resolveLocation } from "../auth.js";
|
|
14
13
|
import { sanitizeText, convertToGeminiMessages, convertToolsForGemini, retainThoughtSignature, calculateCost } from "../utils.js";
|
|
15
|
-
import { createAssistantMessageEventStream, type AssistantMessageEventStream } from "@
|
|
14
|
+
import { createAssistantMessageEventStream, type AssistantMessageEventStream, type JsonObject } from "@earendil-works/pi-ai";
|
|
16
15
|
|
|
17
16
|
// Module-level counter for generating unique tool call IDs (matches pi-mono pattern)
|
|
18
17
|
let toolCallCounter = 0;
|
|
19
18
|
|
|
20
|
-
const THINKING_LEVEL_MAP: Record<string, ThinkingLevel> = {
|
|
21
|
-
minimal: ThinkingLevel.MINIMAL,
|
|
22
|
-
low: ThinkingLevel.LOW,
|
|
23
|
-
medium: ThinkingLevel.MEDIUM,
|
|
24
|
-
high: ThinkingLevel.HIGH,
|
|
25
|
-
};
|
|
26
|
-
|
|
27
19
|
interface GeminiThinkingConfig {
|
|
28
20
|
includeThoughts?: boolean;
|
|
29
21
|
thinkingBudget?: number;
|
|
30
|
-
thinkingLevel?:
|
|
22
|
+
thinkingLevel?: string;
|
|
31
23
|
}
|
|
32
24
|
|
|
33
25
|
function isGemini3ProModel(modelId: string): boolean {
|
|
34
26
|
return /gemini-3(?:\.\d+)?-pro/.test(modelId.toLowerCase());
|
|
35
27
|
}
|
|
36
28
|
|
|
37
|
-
function
|
|
38
|
-
return /gemini-3
|
|
29
|
+
function isGemini38FlashModel(modelId: string): boolean {
|
|
30
|
+
return /gemini-3\.8-flash/.test(modelId.toLowerCase());
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function isGemini4Model(modelId: string): boolean {
|
|
34
|
+
return modelId.toLowerCase().startsWith("gemini-4");
|
|
39
35
|
}
|
|
40
36
|
|
|
41
37
|
function isGemini25ProModel(modelId: string): boolean {
|
|
42
38
|
return /gemini-2\.5-pro/.test(modelId.toLowerCase());
|
|
43
39
|
}
|
|
44
40
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
41
|
+
// Gemini 3 and 4 models are configured with thinking levels; Gemini 2.5 uses budgets.
|
|
42
|
+
function usesThinkingLevels(apiId: string): boolean {
|
|
43
|
+
return apiId.startsWith("gemini-3") || apiId.startsWith("gemini-4");
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function getGeminiThinkingLevel(
|
|
47
|
+
effort: string,
|
|
48
|
+
modelId: string,
|
|
49
|
+
levels: { LOW: string; MEDIUM: string; HIGH: string },
|
|
50
|
+
levelByEffort: Record<string, string>,
|
|
51
|
+
): string {
|
|
52
|
+
if (isGemini3ProModel(modelId) || isGemini4Model(modelId) ||
|
|
53
|
+
(isGemini38FlashModel(modelId) && effort === "minimal")) {
|
|
54
|
+
// Gemini 3 Pro, Gemini 4, and Gemini 3.8 Flash reject MINIMAL (HTTP 400) — floor to LOW
|
|
55
|
+
if (effort === "minimal" || effort === "low") return levels.LOW;
|
|
56
|
+
if (effort === "medium") return levels.MEDIUM;
|
|
57
|
+
return levels.HIGH;
|
|
51
58
|
}
|
|
52
|
-
return
|
|
59
|
+
return levelByEffort[effort];
|
|
53
60
|
}
|
|
54
61
|
|
|
55
|
-
function mapGeminiStopReason(
|
|
62
|
+
function mapGeminiStopReason(
|
|
63
|
+
reason: string,
|
|
64
|
+
finishReason: { STOP: string; MAX_TOKENS: string },
|
|
65
|
+
): "stop" | "length" | "toolUse" | "error" {
|
|
56
66
|
switch (reason) {
|
|
57
|
-
case
|
|
67
|
+
case finishReason.STOP:
|
|
58
68
|
return "stop";
|
|
59
|
-
case
|
|
69
|
+
case finishReason.MAX_TOKENS:
|
|
60
70
|
return "length";
|
|
61
71
|
default:
|
|
62
72
|
return "error";
|
|
@@ -93,6 +103,14 @@ export function streamGemini(
|
|
|
93
103
|
// Priority: config file > env var > model region > default
|
|
94
104
|
const location = resolveLocation(model.region);
|
|
95
105
|
const auth = getAuthConfig(location);
|
|
106
|
+
const cost = auth.location !== "global" && model.costRegional ? model.costRegional : model.cost;
|
|
107
|
+
const { GoogleGenAI, FinishReason, ThinkingLevel } = await import("@google/genai");
|
|
108
|
+
const thinkingLevelByEffort = {
|
|
109
|
+
minimal: ThinkingLevel.MINIMAL,
|
|
110
|
+
low: ThinkingLevel.LOW,
|
|
111
|
+
medium: ThinkingLevel.MEDIUM,
|
|
112
|
+
high: ThinkingLevel.HIGH,
|
|
113
|
+
};
|
|
96
114
|
|
|
97
115
|
// Create client with explicit API version (matches pi-mono)
|
|
98
116
|
const client = new GoogleGenAI({
|
|
@@ -126,13 +144,18 @@ export function streamGemini(
|
|
|
126
144
|
// Add thinking configuration (matches pi-mono's buildParams logic).
|
|
127
145
|
if (model.reasoning) {
|
|
128
146
|
if (options?.reasoning) {
|
|
129
|
-
|
|
130
|
-
|
|
147
|
+
// XHIGH goes only to models that declare it (Neon and Hydrogen); every
|
|
148
|
+
// other Gemini model answers XHIGH with HTTP 400, so it caps at HIGH.
|
|
149
|
+
const supportsXhigh = model.thinkingLevelMap?.xhigh != null;
|
|
150
|
+
const effort = options.reasoning === "xhigh" || options.reasoning === "max" ? "high" : options.reasoning;
|
|
151
|
+
const usesLevels = usesThinkingLevels(model.apiId);
|
|
131
152
|
const thinkingConfig: GeminiThinkingConfig = { includeThoughts: true };
|
|
132
153
|
|
|
133
|
-
if (
|
|
134
|
-
|
|
135
|
-
|
|
154
|
+
if (usesLevels) {
|
|
155
|
+
thinkingConfig.thinkingLevel = supportsXhigh && options.reasoning === "xhigh"
|
|
156
|
+
// @google/genai's ThinkingLevel enum predates XHIGH; the SDK forwards the string.
|
|
157
|
+
? "XHIGH"
|
|
158
|
+
: getGeminiThinkingLevel(effort, model.apiId, ThinkingLevel, thinkingLevelByEffort);
|
|
136
159
|
} else {
|
|
137
160
|
// Gemini 2.5 models use thinking budgets (token counts)
|
|
138
161
|
const budgets: Record<string, number> = {
|
|
@@ -147,12 +170,12 @@ export function streamGemini(
|
|
|
147
170
|
config.thinkingConfig = thinkingConfig;
|
|
148
171
|
} else {
|
|
149
172
|
// If no reasoning level is specified:
|
|
150
|
-
// - For Gemini 3.x/3.5 models, omit thinkingConfig entirely so Vertex AI uses
|
|
173
|
+
// - For Gemini 3.x/3.5/4 models, omit thinkingConfig entirely so Vertex AI uses
|
|
151
174
|
// the model's native default level (e.g. MEDIUM for 3.5, HIGH for others).
|
|
152
175
|
// - For Gemini 2.5 models, apply a healthy thinking budget floor (thinking is
|
|
153
176
|
// disabled by default on 2.5).
|
|
154
|
-
const
|
|
155
|
-
if (!
|
|
177
|
+
const usesLevels = usesThinkingLevels(model.apiId);
|
|
178
|
+
if (!usesLevels) {
|
|
156
179
|
config.thinkingConfig = {
|
|
157
180
|
includeThoughts: true,
|
|
158
181
|
thinkingBudget: model.apiId.includes("2.5-pro") ? 2048 : 1024,
|
|
@@ -252,7 +275,8 @@ export function streamGemini(
|
|
|
252
275
|
type: "toolCall" as const,
|
|
253
276
|
id: toolCallId,
|
|
254
277
|
name: part.functionCall.name || "",
|
|
255
|
-
|
|
278
|
+
// Gemini returns tool args as parsed JSON, so they are always JSON values.
|
|
279
|
+
arguments: (part.functionCall.args as JsonObject | undefined) ?? {},
|
|
256
280
|
...(part.thoughtSignature && { thoughtSignature: part.thoughtSignature }),
|
|
257
281
|
};
|
|
258
282
|
|
|
@@ -267,7 +291,7 @@ export function streamGemini(
|
|
|
267
291
|
|
|
268
292
|
// Handle finish reason
|
|
269
293
|
if (candidate?.finishReason) {
|
|
270
|
-
output.stopReason = mapGeminiStopReason(candidate.finishReason);
|
|
294
|
+
output.stopReason = mapGeminiStopReason(candidate.finishReason, FinishReason);
|
|
271
295
|
if (candidate.finishReason === FinishReason.SAFETY) {
|
|
272
296
|
output.errorMessage = "Content blocked by safety filters";
|
|
273
297
|
}
|
|
@@ -296,7 +320,7 @@ export function streamGemini(
|
|
|
296
320
|
totalTokens: meta.totalTokenCount || 0,
|
|
297
321
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
298
322
|
};
|
|
299
|
-
calculateCost(
|
|
323
|
+
calculateCost(cost.input, cost.output, cost.cacheRead, cost.cacheWrite, output.usage);
|
|
300
324
|
}
|
|
301
325
|
}
|
|
302
326
|
|
|
@@ -311,7 +335,19 @@ export function streamGemini(
|
|
|
311
335
|
throw new Error("Request was aborted");
|
|
312
336
|
}
|
|
313
337
|
|
|
314
|
-
|
|
338
|
+
// A "done" event only carries successful stop reasons. Safety blocks and
|
|
339
|
+
// unmapped finish reasons surface as an error event instead, matching the
|
|
340
|
+
// catch block below.
|
|
341
|
+
const finalReason = output.stopReason;
|
|
342
|
+
if (finalReason === "stop" || finalReason === "length" || finalReason === "toolUse") {
|
|
343
|
+
stream.push({ type: "done", reason: finalReason, message: output });
|
|
344
|
+
} else {
|
|
345
|
+
// Error events only carry "aborted" | "error"; anything else that is
|
|
346
|
+
// not a successful stop (Pi 0.99 added "pending"/"deferred") is an error.
|
|
347
|
+
const errorReason = finalReason === "aborted" ? "aborted" : "error";
|
|
348
|
+
output.stopReason = errorReason;
|
|
349
|
+
stream.push({ type: "error", reason: errorReason, error: output });
|
|
350
|
+
}
|
|
315
351
|
stream.end();
|
|
316
352
|
} catch (error) {
|
|
317
353
|
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
package/streaming/index.ts
CHANGED
|
@@ -3,15 +3,44 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import type { VertexModelConfig, Context, StreamOptions } from "../types.js";
|
|
6
|
-
import type { AssistantMessageEventStream } from "@
|
|
6
|
+
import type { AssistantMessageEventStream } from "@earendil-works/pi-ai";
|
|
7
|
+
import {
|
|
8
|
+
getCurrentSystemMessage,
|
|
9
|
+
getCurrentTools,
|
|
10
|
+
getSystemMessageText,
|
|
11
|
+
} from "@earendil-works/pi-ai";
|
|
7
12
|
import { streamGemini } from "./gemini.js";
|
|
8
13
|
import { streamMaaS } from "./maas.js";
|
|
14
|
+
import { installSseCommentFilter } from "./sse-comment-filter.js";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Pi 0.99 folds the system prompt and tool definitions into system messages
|
|
18
|
+
* (`toolsAdded`/`toolsRemoved` deltas replayed in order); the agent loop no
|
|
19
|
+
* longer populates `context.systemPrompt`/`context.tools`, which are legacy
|
|
20
|
+
* shorthand. The streaming paths read the shorthand fields, so resolve them
|
|
21
|
+
* from the transcript here. Explicit fields still win when a caller sets them.
|
|
22
|
+
*/
|
|
23
|
+
function resolveContext(context: Context): Context {
|
|
24
|
+
if (context.systemPrompt !== undefined && context.tools !== undefined) return context;
|
|
25
|
+
const systemMessage = getCurrentSystemMessage(context.messages);
|
|
26
|
+
return {
|
|
27
|
+
...context,
|
|
28
|
+
systemPrompt: context.systemPrompt ?? (systemMessage ? getSystemMessageText(systemMessage) : undefined),
|
|
29
|
+
tools: context.tools ?? getCurrentTools(context.messages),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
9
32
|
|
|
10
33
|
export function streamVertex(
|
|
11
34
|
model: VertexModelConfig,
|
|
12
35
|
context: Context,
|
|
13
36
|
options?: StreamOptions
|
|
14
37
|
): AssistantMessageEventStream {
|
|
38
|
+
// Vertex injects keepalive lines (": keepalive" / "data: : keepalive") on slow
|
|
39
|
+
// streams; SDK parsers choke on them. Strip at the fetch layer (installed once).
|
|
40
|
+
installSseCommentFilter();
|
|
41
|
+
|
|
42
|
+
context = resolveContext(context);
|
|
43
|
+
|
|
15
44
|
switch (model.endpointType) {
|
|
16
45
|
case "gemini":
|
|
17
46
|
return streamGemini(model, context, options);
|
package/streaming/maas.ts
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* MaaS streaming handler for Claude and all other models.
|
|
3
3
|
*
|
|
4
4
|
* - Anthropic models: native AnthropicVertex SDK streaming
|
|
5
|
-
* - Other MaaS models: Vertex OpenAI-compatible Chat Completions endpoint
|
|
5
|
+
* - Other MaaS models: Vertex OpenAI-compatible Chat Completions endpoint, or the
|
|
6
|
+
* publisher's OpenAI-compatible Responses API for models with `maasApi: "responses"`
|
|
6
7
|
*/
|
|
7
8
|
|
|
8
9
|
import type { VertexModelConfig, Context, StreamOptions } from "../types.js";
|
|
@@ -11,10 +12,9 @@ import {
|
|
|
11
12
|
createAssistantMessageEventStream,
|
|
12
13
|
type AssistantMessageEventStream,
|
|
13
14
|
type Model,
|
|
14
|
-
|
|
15
|
+
type SimpleStreamOptions,
|
|
15
16
|
calculateCost,
|
|
16
|
-
} from "@
|
|
17
|
-
import { AnthropicVertex } from "@anthropic-ai/vertex-sdk";
|
|
17
|
+
} from "@earendil-works/pi-ai";
|
|
18
18
|
|
|
19
19
|
function mapAnthropicEffort(reasoning?: string): "low" | "medium" | "high" | "max" | undefined {
|
|
20
20
|
if (!reasoning) return undefined;
|
|
@@ -52,6 +52,7 @@ async function streamAnthropic(
|
|
|
52
52
|
): Promise<void> {
|
|
53
53
|
const location = resolveLocation(model.region);
|
|
54
54
|
const auth = getAuthConfig(location);
|
|
55
|
+
const { AnthropicVertex } = await import("@anthropic-ai/vertex-sdk");
|
|
55
56
|
|
|
56
57
|
// Use regional pricing when the resolved endpoint is not the global one.
|
|
57
58
|
// Models without costRegional (e.g. Opus 4.1, Sonnet 4) have uniform pricing.
|
|
@@ -328,16 +329,69 @@ async function streamAnthropic(
|
|
|
328
329
|
stream.push({ type: "done", reason: output.stopReason, message: output });
|
|
329
330
|
}
|
|
330
331
|
|
|
332
|
+
/** Chat Completions model on the Vertex OpenAI-compatible endpoint (`/endpoints/openapi`). */
|
|
333
|
+
function chatCompletionsModel(model: VertexModelConfig, baseUrl: string): Model<"openai-completions"> {
|
|
334
|
+
return {
|
|
335
|
+
id: model.apiId.includes("/") ? model.apiId : `${model.publisher}/${model.apiId}`,
|
|
336
|
+
name: model.name,
|
|
337
|
+
api: "openai-completions",
|
|
338
|
+
provider: "vertex",
|
|
339
|
+
baseUrl: `${baseUrl}/endpoints/openapi`,
|
|
340
|
+
reasoning: model.reasoning,
|
|
341
|
+
thinkingLevelMap: model.thinkingLevelMap,
|
|
342
|
+
input: model.input,
|
|
343
|
+
cost: model.cost,
|
|
344
|
+
contextWindow: model.contextWindow,
|
|
345
|
+
maxTokens: model.maxTokens,
|
|
346
|
+
headers: {},
|
|
347
|
+
compat: {
|
|
348
|
+
supportsStore: false,
|
|
349
|
+
supportsDeveloperRole: false,
|
|
350
|
+
// DeepSeek on Vertex thinks only when `reasoning_effort` is present.
|
|
351
|
+
supportsReasoningEffort: model.publisher === "deepseek-ai",
|
|
352
|
+
maxTokensField: "max_tokens",
|
|
353
|
+
thinkingFormat: model.publisher === "zai-org" ? "zai" : "openai",
|
|
354
|
+
},
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Responses model on the publisher's OpenAI-compatible endpoint
|
|
360
|
+
* (`/publishers/{publisher}/v1/responses`), addressed by bare model id.
|
|
361
|
+
* pi-ai sends `store: false` for stateless replay (required by Muse).
|
|
362
|
+
*/
|
|
363
|
+
function responsesModel(model: VertexModelConfig, baseUrl: string): Model<"openai-responses"> {
|
|
364
|
+
return {
|
|
365
|
+
id: model.apiId,
|
|
366
|
+
name: model.name,
|
|
367
|
+
api: "openai-responses",
|
|
368
|
+
provider: "vertex",
|
|
369
|
+
baseUrl: `${baseUrl}/publishers/${model.publisher}/v1`,
|
|
370
|
+
reasoning: model.reasoning,
|
|
371
|
+
// Default to omitting effort at off; only models declaring an off mapping
|
|
372
|
+
// (GPT Terra/Luna accept `none`) override this.
|
|
373
|
+
thinkingLevelMap: { off: null, ...model.thinkingLevelMap },
|
|
374
|
+
input: model.input,
|
|
375
|
+
cost: model.cost,
|
|
376
|
+
contextWindow: model.contextWindow,
|
|
377
|
+
maxTokens: model.maxTokens,
|
|
378
|
+
headers: {},
|
|
379
|
+
compat: {
|
|
380
|
+
// No 24h retention on Vertex: never send `prompt_cache_retention`.
|
|
381
|
+
supportsLongCacheRetention: false,
|
|
382
|
+
},
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
|
|
331
386
|
export function streamMaaS(
|
|
332
387
|
model: VertexModelConfig,
|
|
333
388
|
context: Context,
|
|
334
389
|
options?: StreamOptions,
|
|
335
390
|
): AssistantMessageEventStream {
|
|
336
391
|
const stream = createAssistantMessageEventStream();
|
|
392
|
+
const useResponses = model.maasApi === "responses";
|
|
337
393
|
|
|
338
394
|
(async () => {
|
|
339
|
-
const apiModelId = model.apiId.includes("/") ? model.apiId : `${model.publisher}/${model.apiId}`;
|
|
340
|
-
|
|
341
395
|
try {
|
|
342
396
|
if (model.publisher === "anthropic") {
|
|
343
397
|
await streamAnthropic(model, context, options, stream);
|
|
@@ -345,40 +399,43 @@ export function streamMaaS(
|
|
|
345
399
|
return;
|
|
346
400
|
}
|
|
347
401
|
|
|
348
|
-
// Non-Anthropic MaaS models: Vertex OpenAI-compatible
|
|
402
|
+
// Non-Anthropic MaaS models: Vertex OpenAI-compatible endpoints via pi-ai.
|
|
349
403
|
const location = resolveLocation(model.region);
|
|
350
404
|
const auth = getAuthConfig(location);
|
|
351
405
|
const accessToken = await getAccessToken();
|
|
352
406
|
const baseUrl = buildBaseUrl(auth.projectId, auth.location);
|
|
353
|
-
const
|
|
354
|
-
|
|
355
|
-
const modelForPi: Model<"openai-completions"> = {
|
|
356
|
-
id: apiModelId,
|
|
357
|
-
name: model.name,
|
|
358
|
-
api: "openai-completions",
|
|
359
|
-
provider: "vertex",
|
|
360
|
-
baseUrl: endpoint,
|
|
361
|
-
reasoning: model.reasoning,
|
|
362
|
-
input: model.input,
|
|
363
|
-
cost: model.cost,
|
|
364
|
-
contextWindow: model.contextWindow,
|
|
365
|
-
maxTokens: model.maxTokens,
|
|
366
|
-
headers: {},
|
|
367
|
-
compat: {
|
|
368
|
-
supportsStore: false,
|
|
369
|
-
supportsDeveloperRole: false,
|
|
370
|
-
supportsReasoningEffort: false,
|
|
371
|
-
maxTokensField: "max_tokens",
|
|
372
|
-
thinkingFormat: model.publisher === "qwen" ? "qwen" : model.publisher === "zai-org" ? "zai" : "openai",
|
|
373
|
-
},
|
|
374
|
-
};
|
|
375
|
-
|
|
376
|
-
const innerStream = streamSimpleOpenAICompletions(modelForPi, context as any, {
|
|
407
|
+
const innerOptions: SimpleStreamOptions = {
|
|
377
408
|
...options,
|
|
378
409
|
apiKey: accessToken,
|
|
379
410
|
maxTokens: options?.maxTokens || model.maxTokens,
|
|
380
411
|
temperature: options?.temperature,
|
|
381
|
-
}
|
|
412
|
+
};
|
|
413
|
+
|
|
414
|
+
if (useResponses && model.publisher === "openai") {
|
|
415
|
+
const onPayload = innerOptions.onPayload;
|
|
416
|
+
innerOptions.onPayload = async (payload, requestModel) => {
|
|
417
|
+
const replacement = await onPayload?.(payload, requestModel);
|
|
418
|
+
const body = (replacement ?? payload) as {
|
|
419
|
+
reasoning?: { summary?: string };
|
|
420
|
+
include?: string[];
|
|
421
|
+
};
|
|
422
|
+
// Vertex's GPT preview does not support reasoning summaries. Keep
|
|
423
|
+
// encrypted reasoning available for stateless replay, even at off.
|
|
424
|
+
if (body.reasoning) delete body.reasoning.summary;
|
|
425
|
+
body.include ??= [];
|
|
426
|
+
if (!body.include.includes("reasoning.encrypted_content")) {
|
|
427
|
+
body.include.push("reasoning.encrypted_content");
|
|
428
|
+
}
|
|
429
|
+
return body;
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
// Pi 0.99 moved per-API stream functions off the pi-ai root.
|
|
434
|
+
// Pi's extension loader maps the compat entry at runtime.
|
|
435
|
+
const { openAICompletionsApi, openAIResponsesApi } = await import("@earendil-works/pi-ai/compat");
|
|
436
|
+
const innerStream = useResponses
|
|
437
|
+
? openAIResponsesApi().streamSimple(responsesModel(model, baseUrl), context as any, innerOptions)
|
|
438
|
+
: openAICompletionsApi().streamSimple(chatCompletionsModel(model, baseUrl), context as any, innerOptions);
|
|
382
439
|
|
|
383
440
|
for await (const event of innerStream) {
|
|
384
441
|
if ("partial" in event && event.partial) event.partial.model = model.id;
|
|
@@ -399,7 +456,7 @@ export function streamMaaS(
|
|
|
399
456
|
error: {
|
|
400
457
|
role: "assistant",
|
|
401
458
|
content: [],
|
|
402
|
-
api: model.publisher === "anthropic" ? "anthropic-messages" : "openai-completions",
|
|
459
|
+
api: model.publisher === "anthropic" ? "anthropic-messages" : useResponses ? "openai-responses" : "openai-completions",
|
|
403
460
|
provider: "vertex",
|
|
404
461
|
model: model.id,
|
|
405
462
|
usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } },
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fetch wrapper that strips keepalive noise from Vertex AI streaming responses.
|
|
3
|
+
*
|
|
4
|
+
* On slow streams (long prefill, hidden reasoning, large context) Vertex AI
|
|
5
|
+
* injects keepalive lines that downstream SDK parsers choke on:
|
|
6
|
+
*
|
|
7
|
+
* 1. `: keepalive` — SSE comment (Gemini streamGenerateContent).
|
|
8
|
+
* Spec-ignorable, but @google/genai's hand-rolled parser doesn't skip
|
|
9
|
+
* comments: "Unexpected token ':'..." / "Incomplete JSON segment at the
|
|
10
|
+
* end" / silently dropped data events.
|
|
11
|
+
*
|
|
12
|
+
* 2. `data: : keepalive` — a *data* event whose payload is a pseudo-comment
|
|
13
|
+
* (MaaS /endpoints/openapi chat completions, observed with e.g. Grok).
|
|
14
|
+
* Not a comment at all, so even spec-correct SSE parsers (openai SDK)
|
|
15
|
+
* hand `: keepalive` to JSON.parse:
|
|
16
|
+
* "Unexpected token ':', \": keepalive\" is not valid JSON"
|
|
17
|
+
*
|
|
18
|
+
* Both are safe to drop: a valid JSON payload can never start with ':'.
|
|
19
|
+
* Scoped to Vertex AI (aiplatform.googleapis.com) streaming URLs only.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
let installed = false;
|
|
23
|
+
|
|
24
|
+
/** True for SSE comments and for data events carrying a pseudo-comment payload. */
|
|
25
|
+
function isKeepaliveLine(line: string): boolean {
|
|
26
|
+
if (line.startsWith(":")) return true; // SSE comment
|
|
27
|
+
if (line.startsWith("data:")) {
|
|
28
|
+
const payload = line.substring(5).trimStart();
|
|
29
|
+
return payload.startsWith(":"); // e.g. "data: : keepalive"
|
|
30
|
+
}
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* TransformStream that removes keepalive lines from a byte stream while
|
|
36
|
+
* preserving all other bytes (including event-delimiting blank lines).
|
|
37
|
+
*/
|
|
38
|
+
function createKeepaliveStripper(): TransformStream<Uint8Array, Uint8Array> {
|
|
39
|
+
const decoder = new TextDecoder("utf-8");
|
|
40
|
+
const encoder = new TextEncoder();
|
|
41
|
+
let pending = "";
|
|
42
|
+
|
|
43
|
+
const filter = (text: string, isFlush: boolean): string => {
|
|
44
|
+
pending += text;
|
|
45
|
+
let output = "";
|
|
46
|
+
let start = 0;
|
|
47
|
+
while (true) {
|
|
48
|
+
const nl = pending.indexOf("\n", start);
|
|
49
|
+
if (nl === -1) break;
|
|
50
|
+
const line = pending.substring(start, nl + 1);
|
|
51
|
+
// Strip \r\n / \n terminated keepalive lines; keep everything else.
|
|
52
|
+
const content = line.endsWith("\r\n") ? line.slice(0, -2) : line.slice(0, -1);
|
|
53
|
+
if (!isKeepaliveLine(content)) output += line;
|
|
54
|
+
start = nl + 1;
|
|
55
|
+
}
|
|
56
|
+
pending = pending.substring(start);
|
|
57
|
+
if (isFlush && pending.length > 0) {
|
|
58
|
+
// Trailing data without a newline: drop only if it's a keepalive.
|
|
59
|
+
if (!isKeepaliveLine(pending)) output += pending;
|
|
60
|
+
pending = "";
|
|
61
|
+
}
|
|
62
|
+
return output;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
return new TransformStream<Uint8Array, Uint8Array>({
|
|
66
|
+
transform(chunk, controller) {
|
|
67
|
+
const out = filter(decoder.decode(chunk, { stream: true }), false);
|
|
68
|
+
if (out.length > 0) controller.enqueue(encoder.encode(out));
|
|
69
|
+
},
|
|
70
|
+
flush(controller) {
|
|
71
|
+
const out = filter(decoder.decode(), true);
|
|
72
|
+
if (out.length > 0) controller.enqueue(encoder.encode(out));
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Some Vertex deployments accept a streaming request, answer 200, and then send nothing at
|
|
79
|
+
* all \u2014 observed on `zai-org/glm-5.2-maas` (global) for roughly half of all requests, while
|
|
80
|
+
* glm-4.7 and deepseek on the same endpoint and credentials never did it. undici only gives up
|
|
81
|
+
* after its 300 s body timeout, so every dead request costs five minutes before a retry starts.
|
|
82
|
+
*
|
|
83
|
+
* Bound the wait for the *first* byte only. A stream that has begun is left alone: a genuinely
|
|
84
|
+
* slow one keeps the socket alive with keepalive lines (stripped further down, but counted here
|
|
85
|
+
* because this sits upstream of the stripper), and mid-stream gaps stay undici's business.
|
|
86
|
+
*/
|
|
87
|
+
const FIRST_BYTE_TIMEOUT_MS = Number(process.env.PI_VERTEX_FIRST_BYTE_TIMEOUT_MS ?? 90_000);
|
|
88
|
+
|
|
89
|
+
function withFirstByteTimeout(body: ReadableStream<Uint8Array>, url: string, timeoutMs: number): ReadableStream<Uint8Array> {
|
|
90
|
+
if (!(timeoutMs > 0)) return body;
|
|
91
|
+
const reader = body.getReader();
|
|
92
|
+
let started = false;
|
|
93
|
+
return new ReadableStream<Uint8Array>({
|
|
94
|
+
async pull(controller) {
|
|
95
|
+
if (started) {
|
|
96
|
+
const { done, value } = await reader.read();
|
|
97
|
+
if (done) controller.close();
|
|
98
|
+
else controller.enqueue(value);
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
102
|
+
try {
|
|
103
|
+
const first = await Promise.race([
|
|
104
|
+
reader.read(),
|
|
105
|
+
new Promise<never>((_, reject) => {
|
|
106
|
+
timer = setTimeout(
|
|
107
|
+
() => reject(new Error(`Vertex sent no data for ${Math.round(timeoutMs / 1000)}s after accepting this streaming request (${url}). The endpoint is not producing output; retry or use another offering. Raise PI_VERTEX_FIRST_BYTE_TIMEOUT_MS to wait longer, 0 to disable.`)),
|
|
108
|
+
timeoutMs,
|
|
109
|
+
);
|
|
110
|
+
}),
|
|
111
|
+
]);
|
|
112
|
+
started = true;
|
|
113
|
+
if (first.done) controller.close();
|
|
114
|
+
else controller.enqueue(first.value);
|
|
115
|
+
} catch (error) {
|
|
116
|
+
await reader.cancel(error).catch(() => {});
|
|
117
|
+
controller.error(error);
|
|
118
|
+
} finally {
|
|
119
|
+
clearTimeout(timer);
|
|
120
|
+
}
|
|
121
|
+
},
|
|
122
|
+
cancel(reason) { return reader.cancel(reason); },
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function requestUrl(input: RequestInfo | URL): string {
|
|
127
|
+
if (typeof input === "string") return input;
|
|
128
|
+
if (input instanceof URL) return input.href;
|
|
129
|
+
return input.url;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Vertex AI streaming endpoints: Gemini SSE, MaaS chat completions, publisher
|
|
134
|
+
* Responses API (Meta Muse), Anthropic raw predict.
|
|
135
|
+
*/
|
|
136
|
+
function isVertexStreamingUrl(url: string): boolean {
|
|
137
|
+
if (!url.includes("aiplatform.googleapis.com")) return false;
|
|
138
|
+
return (
|
|
139
|
+
url.includes("streamGenerateContent") ||
|
|
140
|
+
url.includes("/endpoints/openapi/") ||
|
|
141
|
+
url.includes("/v1/responses") ||
|
|
142
|
+
url.includes("streamRawPredict")
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Install a global fetch wrapper (once) that pipes Vertex AI streaming
|
|
148
|
+
* response bodies through the keepalive stripper. All other requests pass
|
|
149
|
+
* through untouched.
|
|
150
|
+
*/
|
|
151
|
+
export function installSseCommentFilter(): void {
|
|
152
|
+
if (installed) return;
|
|
153
|
+
installed = true;
|
|
154
|
+
|
|
155
|
+
const originalFetch = globalThis.fetch;
|
|
156
|
+
globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
|
157
|
+
const response = await originalFetch(input, init);
|
|
158
|
+
const url = requestUrl(input);
|
|
159
|
+
if (!isVertexStreamingUrl(url) || !response.body) {
|
|
160
|
+
return response;
|
|
161
|
+
}
|
|
162
|
+
const live = withFirstByteTimeout(response.body, url, FIRST_BYTE_TIMEOUT_MS);
|
|
163
|
+
return new Response(live.pipeThrough(createKeepaliveStripper()), {
|
|
164
|
+
headers: response.headers,
|
|
165
|
+
status: response.status,
|
|
166
|
+
statusText: response.statusText,
|
|
167
|
+
});
|
|
168
|
+
};
|
|
169
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/** The keepalive filter must also give up on a Vertex stream that never starts. */
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { once } from "node:events";
|
|
5
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
6
|
+
|
|
7
|
+
process.env.PI_VERTEX_FIRST_BYTE_TIMEOUT_MS = "300";
|
|
8
|
+
const { installSseCommentFilter } = await import("../streaming/sse-comment-filter.ts");
|
|
9
|
+
|
|
10
|
+
const open = new Set();
|
|
11
|
+
const server = createServer(async (req, res) => {
|
|
12
|
+
open.add(res); res.on("close", () => open.delete(res));
|
|
13
|
+
res.writeHead(200, { "content-type": "text/event-stream" });
|
|
14
|
+
res.flushHeaders(); // 200 arrives, body may not: exactly what the failing deployment does
|
|
15
|
+
if (req.url.endsWith("/dead")) return; // accepted, then nothing: the glm-5.2 failure shape
|
|
16
|
+
if (req.url.endsWith("/slow-start")) { await sleep(150); }
|
|
17
|
+
res.write(": keepalive\n");
|
|
18
|
+
await sleep(500); // a mid-stream gap longer than the first-byte budget must survive
|
|
19
|
+
res.write(`data: {"choices":[{"delta":{"content":"LATE-BUT-ALIVE"}}]}\n\n`);
|
|
20
|
+
res.end("data: [DONE]\n\n");
|
|
21
|
+
});
|
|
22
|
+
server.listen(0, "127.0.0.1"); await once(server, "listening");
|
|
23
|
+
const { port } = server.address();
|
|
24
|
+
// The wrapper matches Vertex streaming URLs by substring, so a local URL carrying those
|
|
25
|
+
// markers in its path exercises the real wrapper without DNS or credentials.
|
|
26
|
+
const base = `http://127.0.0.1:${port}/aiplatform.googleapis.com/endpoints/openapi`;
|
|
27
|
+
// Muse and GPT stream from publisher Responses APIs instead of /endpoints/openapi.
|
|
28
|
+
const responsesBases = ["meta", "openai"].map(publisher =>
|
|
29
|
+
`http://127.0.0.1:${port}/aiplatform.googleapis.com/publishers/${publisher}/v1/responses`);
|
|
30
|
+
installSseCommentFilter();
|
|
31
|
+
|
|
32
|
+
async function read(path, root = base) {
|
|
33
|
+
const response = await fetch(`${root}${path}`);
|
|
34
|
+
let body = "";
|
|
35
|
+
for await (const chunk of response.body) body += new TextDecoder().decode(chunk);
|
|
36
|
+
return body;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const started = Date.now();
|
|
40
|
+
await assert.rejects(read("/dead"), /sent no data for 0s after accepting this streaming request/);
|
|
41
|
+
const elapsed = Date.now() - started;
|
|
42
|
+
assert(elapsed < 3000, `a dead stream must fail fast, took ${elapsed}ms`);
|
|
43
|
+
assert(elapsed >= 300, `must wait its budget first, gave up after ${elapsed}ms`);
|
|
44
|
+
for (const root of responsesBases) {
|
|
45
|
+
await assert.rejects(read("/dead", root), /sent no data/, "a dead Responses API stream must fail fast too");
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
for (const path of ["/live", "/slow-start"]) {
|
|
49
|
+
const body = await read(path);
|
|
50
|
+
assert.match(body, /LATE-BUT-ALIVE/, `${path} must survive a mid-stream gap longer than the first-byte budget`);
|
|
51
|
+
assert.doesNotMatch(body, /keepalive/, `${path} must still have keepalive lines stripped`);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
for (const res of open) res.destroy();
|
|
55
|
+
server.close(); await once(server, "close");
|
|
56
|
+
console.log("PASS first-byte timeout fails a dead Vertex stream fast and leaves live ones alone");
|