@ssweens/pi-vertex 1.1.13 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ssweens/pi-vertex",
3
- "version": "1.1.13",
3
+ "version": "1.3.0",
4
4
  "description": "Google Vertex AI provider for Pi coding agent - supports Gemini, Claude, and all MaaS models",
5
5
  "type": "module",
6
6
  "main": "index.ts",
@@ -16,21 +16,30 @@
16
16
  "CHANGELOG.md",
17
17
  "TEST_COVERAGE.md",
18
18
  "LICENSE",
19
- "screenshot.png"
19
+ "screenshot.png",
20
+ "tests/"
20
21
  ],
21
22
  "scripts": {
22
23
  "clean": "echo 'nothing to clean'",
23
24
  "build": "echo 'nothing to build'",
24
- "check": "echo 'nothing to check'"
25
+ "check": "echo 'nothing to check'",
26
+ "test": "node --experimental-strip-types --test tests/*.mjs"
25
27
  },
26
28
  "dependencies": {
27
29
  "@anthropic-ai/vertex-sdk": "^0.14.4",
28
- "@google/genai": "^1.42.0",
30
+ "@google/genai": "^2.22.0",
29
31
  "google-auth-library": "^9.0.0"
30
32
  },
31
33
  "peerDependencies": {
32
- "@mariozechner/pi-ai": "*",
33
- "@mariozechner/pi-coding-agent": "*"
34
+ "@earendil-works/pi-ai": ">=0.99.1 <0.100.0",
35
+ "@earendil-works/pi-coding-agent": ">=0.99.1 <0.100.0"
36
+ },
37
+ "devDependencies": {
38
+ "@earendil-works/pi-ai": "0.99.1",
39
+ "@earendil-works/pi-coding-agent": "0.99.1"
40
+ },
41
+ "overrides": {
42
+ "@earendil-works/pi-ai": "$@earendil-works/pi-ai"
34
43
  },
35
44
  "pi": {
36
45
  "extensions": [
@@ -4,59 +4,69 @@
4
4
  * Aligned with pi-mono's google-vertex.ts for consistent handling of:
5
5
  * - Thinking content (thought blocks with signatures)
6
6
  * - Tool calls with unique IDs and deduplication
7
- * - Thinking configuration (levels for Gemini 3, budgets for Gemini 2.5)
7
+ * - Thinking configuration (levels for Gemini 3/4, budgets for Gemini 2.5)
8
8
  * - Usage tracking including thinking tokens
9
9
  */
10
10
 
11
- import { GoogleGenAI, FinishReason, ThinkingLevel } from "@google/genai";
12
11
  import type { VertexModelConfig, Context, StreamOptions, AssistantMessage } from "../types.js";
13
12
  import { getAuthConfig, resolveLocation } from "../auth.js";
14
13
  import { sanitizeText, convertToGeminiMessages, convertToolsForGemini, retainThoughtSignature, calculateCost } from "../utils.js";
15
- import { createAssistantMessageEventStream, type AssistantMessageEventStream } from "@mariozechner/pi-ai";
14
+ import { createAssistantMessageEventStream, type AssistantMessageEventStream, type JsonObject } from "@earendil-works/pi-ai";
16
15
 
17
16
  // Module-level counter for generating unique tool call IDs (matches pi-mono pattern)
18
17
  let toolCallCounter = 0;
19
18
 
20
- const THINKING_LEVEL_MAP: Record<string, ThinkingLevel> = {
21
- minimal: ThinkingLevel.MINIMAL,
22
- low: ThinkingLevel.LOW,
23
- medium: ThinkingLevel.MEDIUM,
24
- high: ThinkingLevel.HIGH,
25
- };
26
-
27
19
  interface GeminiThinkingConfig {
28
20
  includeThoughts?: boolean;
29
21
  thinkingBudget?: number;
30
- thinkingLevel?: ThinkingLevel;
22
+ thinkingLevel?: string;
31
23
  }
32
24
 
33
25
  function isGemini3ProModel(modelId: string): boolean {
34
26
  return /gemini-3(?:\.\d+)?-pro/.test(modelId.toLowerCase());
35
27
  }
36
28
 
37
- function isGemini3FlashModel(modelId: string): boolean {
38
- return /gemini-3(?:\.\d+)?-flash/.test(modelId.toLowerCase());
29
+ function isGemini38FlashModel(modelId: string): boolean {
30
+ return /gemini-3\.8-flash/.test(modelId.toLowerCase());
31
+ }
32
+
33
+ function isGemini4Model(modelId: string): boolean {
34
+ return modelId.toLowerCase().startsWith("gemini-4");
39
35
  }
40
36
 
41
37
  function isGemini25ProModel(modelId: string): boolean {
42
38
  return /gemini-2\.5-pro/.test(modelId.toLowerCase());
43
39
  }
44
40
 
45
- function getGemini3ThinkingLevel(effort: string, modelId: string): ThinkingLevel {
46
- if (isGemini3ProModel(modelId)) {
47
- // Pro only supports LOW/MEDIUM/HIGH — floor minimal/low to LOW
48
- if (effort === "minimal" || effort === "low") return ThinkingLevel.LOW;
49
- if (effort === "medium") return ThinkingLevel.MEDIUM;
50
- return ThinkingLevel.HIGH;
41
+ // Gemini 3 and 4 models are configured with thinking levels; Gemini 2.5 uses budgets.
42
+ function usesThinkingLevels(apiId: string): boolean {
43
+ return apiId.startsWith("gemini-3") || apiId.startsWith("gemini-4");
44
+ }
45
+
46
+ function getGeminiThinkingLevel(
47
+ effort: string,
48
+ modelId: string,
49
+ levels: { LOW: string; MEDIUM: string; HIGH: string },
50
+ levelByEffort: Record<string, string>,
51
+ ): string {
52
+ if (isGemini3ProModel(modelId) || isGemini4Model(modelId) ||
53
+ (isGemini38FlashModel(modelId) && effort === "minimal")) {
54
+ // Gemini 3 Pro, Gemini 4, and Gemini 3.8 Flash reject MINIMAL (HTTP 400) — floor to LOW
55
+ if (effort === "minimal" || effort === "low") return levels.LOW;
56
+ if (effort === "medium") return levels.MEDIUM;
57
+ return levels.HIGH;
51
58
  }
52
- return THINKING_LEVEL_MAP[effort];
59
+ return levelByEffort[effort];
53
60
  }
54
61
 
55
- function mapGeminiStopReason(reason: string): "stop" | "length" | "toolUse" | "error" {
62
+ function mapGeminiStopReason(
63
+ reason: string,
64
+ finishReason: { STOP: string; MAX_TOKENS: string },
65
+ ): "stop" | "length" | "toolUse" | "error" {
56
66
  switch (reason) {
57
- case FinishReason.STOP:
67
+ case finishReason.STOP:
58
68
  return "stop";
59
- case FinishReason.MAX_TOKENS:
69
+ case finishReason.MAX_TOKENS:
60
70
  return "length";
61
71
  default:
62
72
  return "error";
@@ -93,6 +103,13 @@ export function streamGemini(
93
103
  // Priority: config file > env var > model region > default
94
104
  const location = resolveLocation(model.region);
95
105
  const auth = getAuthConfig(location);
106
+ const { GoogleGenAI, FinishReason, ThinkingLevel } = await import("@google/genai");
107
+ const thinkingLevelByEffort = {
108
+ minimal: ThinkingLevel.MINIMAL,
109
+ low: ThinkingLevel.LOW,
110
+ medium: ThinkingLevel.MEDIUM,
111
+ high: ThinkingLevel.HIGH,
112
+ };
96
113
 
97
114
  // Create client with explicit API version (matches pi-mono)
98
115
  const client = new GoogleGenAI({
@@ -126,13 +143,18 @@ export function streamGemini(
126
143
  // Add thinking configuration (matches pi-mono's buildParams logic).
127
144
  if (model.reasoning) {
128
145
  if (options?.reasoning) {
129
- const effort = options.reasoning === "xhigh" ? "high" : options.reasoning;
130
- const isGemini3 = model.apiId.startsWith("gemini-3");
146
+ // XHIGH goes only to models that declare it (Gemini 4 Hydrogen); every
147
+ // other Gemini model answers XHIGH with HTTP 400, so it caps at HIGH.
148
+ const supportsXhigh = model.thinkingLevelMap?.xhigh != null;
149
+ const effort = options.reasoning === "xhigh" || options.reasoning === "max" ? "high" : options.reasoning;
150
+ const usesLevels = usesThinkingLevels(model.apiId);
131
151
  const thinkingConfig: GeminiThinkingConfig = { includeThoughts: true };
132
152
 
133
- if (isGemini3) {
134
- // Gemini 3 Pro doesn't support MINIMAL; Flash models do.
135
- thinkingConfig.thinkingLevel = getGemini3ThinkingLevel(effort, model.apiId);
153
+ if (usesLevels) {
154
+ thinkingConfig.thinkingLevel = supportsXhigh && options.reasoning === "xhigh"
155
+ // @google/genai's ThinkingLevel enum predates XHIGH; the SDK forwards the string.
156
+ ? "XHIGH"
157
+ : getGeminiThinkingLevel(effort, model.apiId, ThinkingLevel, thinkingLevelByEffort);
136
158
  } else {
137
159
  // Gemini 2.5 models use thinking budgets (token counts)
138
160
  const budgets: Record<string, number> = {
@@ -147,12 +169,12 @@ export function streamGemini(
147
169
  config.thinkingConfig = thinkingConfig;
148
170
  } else {
149
171
  // If no reasoning level is specified:
150
- // - For Gemini 3.x/3.5 models, omit thinkingConfig entirely so Vertex AI uses
172
+ // - For Gemini 3.x/3.5/4 models, omit thinkingConfig entirely so Vertex AI uses
151
173
  // the model's native default level (e.g. MEDIUM for 3.5, HIGH for others).
152
174
  // - For Gemini 2.5 models, apply a healthy thinking budget floor (thinking is
153
175
  // disabled by default on 2.5).
154
- const isGemini3 = model.apiId.startsWith("gemini-3");
155
- if (!isGemini3) {
176
+ const usesLevels = usesThinkingLevels(model.apiId);
177
+ if (!usesLevels) {
156
178
  config.thinkingConfig = {
157
179
  includeThoughts: true,
158
180
  thinkingBudget: model.apiId.includes("2.5-pro") ? 2048 : 1024,
@@ -252,7 +274,8 @@ export function streamGemini(
252
274
  type: "toolCall" as const,
253
275
  id: toolCallId,
254
276
  name: part.functionCall.name || "",
255
- arguments: (part.functionCall.args as Record<string, unknown>) ?? {},
277
+ // Gemini returns tool args as parsed JSON, so they are always JSON values.
278
+ arguments: (part.functionCall.args as JsonObject | undefined) ?? {},
256
279
  ...(part.thoughtSignature && { thoughtSignature: part.thoughtSignature }),
257
280
  };
258
281
 
@@ -267,7 +290,7 @@ export function streamGemini(
267
290
 
268
291
  // Handle finish reason
269
292
  if (candidate?.finishReason) {
270
- output.stopReason = mapGeminiStopReason(candidate.finishReason);
293
+ output.stopReason = mapGeminiStopReason(candidate.finishReason, FinishReason);
271
294
  if (candidate.finishReason === FinishReason.SAFETY) {
272
295
  output.errorMessage = "Content blocked by safety filters";
273
296
  }
@@ -311,7 +334,19 @@ export function streamGemini(
311
334
  throw new Error("Request was aborted");
312
335
  }
313
336
 
314
- stream.push({ type: "done", reason: output.stopReason, message: output });
337
+ // A "done" event only carries successful stop reasons. Safety blocks and
338
+ // unmapped finish reasons surface as an error event instead, matching the
339
+ // catch block below.
340
+ const finalReason = output.stopReason;
341
+ if (finalReason === "stop" || finalReason === "length" || finalReason === "toolUse") {
342
+ stream.push({ type: "done", reason: finalReason, message: output });
343
+ } else {
344
+ // Error events only carry "aborted" | "error"; anything else that is
345
+ // not a successful stop (Pi 0.99 added "pending"/"deferred") is an error.
346
+ const errorReason = finalReason === "aborted" ? "aborted" : "error";
347
+ output.stopReason = errorReason;
348
+ stream.push({ type: "error", reason: errorReason, error: output });
349
+ }
315
350
  stream.end();
316
351
  } catch (error) {
317
352
  output.stopReason = options?.signal?.aborted ? "aborted" : "error";
@@ -3,15 +3,44 @@
3
3
  */
4
4
 
5
5
  import type { VertexModelConfig, Context, StreamOptions } from "../types.js";
6
- import type { AssistantMessageEventStream } from "@mariozechner/pi-ai";
6
+ import type { AssistantMessageEventStream } from "@earendil-works/pi-ai";
7
+ import {
8
+ getCurrentSystemMessage,
9
+ getCurrentTools,
10
+ getSystemMessageText,
11
+ } from "@earendil-works/pi-ai";
7
12
  import { streamGemini } from "./gemini.js";
8
13
  import { streamMaaS } from "./maas.js";
14
+ import { installSseCommentFilter } from "./sse-comment-filter.js";
15
+
16
+ /**
17
+ * Pi 0.99 folds the system prompt and tool definitions into system messages
18
+ * (`toolsAdded`/`toolsRemoved` deltas replayed in order); the agent loop no
19
+ * longer populates `context.systemPrompt`/`context.tools`, which are legacy
20
+ * shorthand. The streaming paths read the shorthand fields, so resolve them
21
+ * from the transcript here. Explicit fields still win when a caller sets them.
22
+ */
23
+ function resolveContext(context: Context): Context {
24
+ if (context.systemPrompt !== undefined && context.tools !== undefined) return context;
25
+ const systemMessage = getCurrentSystemMessage(context.messages);
26
+ return {
27
+ ...context,
28
+ systemPrompt: context.systemPrompt ?? (systemMessage ? getSystemMessageText(systemMessage) : undefined),
29
+ tools: context.tools ?? getCurrentTools(context.messages),
30
+ };
31
+ }
9
32
 
10
33
  export function streamVertex(
11
34
  model: VertexModelConfig,
12
35
  context: Context,
13
36
  options?: StreamOptions
14
37
  ): AssistantMessageEventStream {
38
+ // Vertex injects keepalive lines (": keepalive" / "data: : keepalive") on slow
39
+ // streams; SDK parsers choke on them. Strip at the fetch layer (installed once).
40
+ installSseCommentFilter();
41
+
42
+ context = resolveContext(context);
43
+
15
44
  switch (model.endpointType) {
16
45
  case "gemini":
17
46
  return streamGemini(model, context, options);
package/streaming/maas.ts CHANGED
@@ -2,7 +2,8 @@
2
2
  * MaaS streaming handler for Claude and all other models.
3
3
  *
4
4
  * - Anthropic models: native AnthropicVertex SDK streaming
5
- * - Other MaaS models: Vertex OpenAI-compatible Chat Completions endpoint
5
+ * - Other MaaS models: Vertex OpenAI-compatible Chat Completions endpoint, or the
6
+ * publisher's OpenAI-compatible Responses API for models with `maasApi: "responses"`
6
7
  */
7
8
 
8
9
  import type { VertexModelConfig, Context, StreamOptions } from "../types.js";
@@ -11,10 +12,9 @@ import {
11
12
  createAssistantMessageEventStream,
12
13
  type AssistantMessageEventStream,
13
14
  type Model,
14
- streamSimpleOpenAICompletions,
15
+ type SimpleStreamOptions,
15
16
  calculateCost,
16
- } from "@mariozechner/pi-ai";
17
- import { AnthropicVertex } from "@anthropic-ai/vertex-sdk";
17
+ } from "@earendil-works/pi-ai";
18
18
 
19
19
  function mapAnthropicEffort(reasoning?: string): "low" | "medium" | "high" | "max" | undefined {
20
20
  if (!reasoning) return undefined;
@@ -52,6 +52,7 @@ async function streamAnthropic(
52
52
  ): Promise<void> {
53
53
  const location = resolveLocation(model.region);
54
54
  const auth = getAuthConfig(location);
55
+ const { AnthropicVertex } = await import("@anthropic-ai/vertex-sdk");
55
56
 
56
57
  // Use regional pricing when the resolved endpoint is not the global one.
57
58
  // Models without costRegional (e.g. Opus 4.1, Sonnet 4) have uniform pricing.
@@ -328,16 +329,69 @@ async function streamAnthropic(
328
329
  stream.push({ type: "done", reason: output.stopReason, message: output });
329
330
  }
330
331
 
332
+ /** Chat Completions model on the Vertex OpenAI-compatible endpoint (`/endpoints/openapi`). */
333
+ function chatCompletionsModel(model: VertexModelConfig, baseUrl: string): Model<"openai-completions"> {
334
+ return {
335
+ id: model.apiId.includes("/") ? model.apiId : `${model.publisher}/${model.apiId}`,
336
+ name: model.name,
337
+ api: "openai-completions",
338
+ provider: "vertex",
339
+ baseUrl: `${baseUrl}/endpoints/openapi`,
340
+ reasoning: model.reasoning,
341
+ thinkingLevelMap: model.thinkingLevelMap,
342
+ input: model.input,
343
+ cost: model.cost,
344
+ contextWindow: model.contextWindow,
345
+ maxTokens: model.maxTokens,
346
+ headers: {},
347
+ compat: {
348
+ supportsStore: false,
349
+ supportsDeveloperRole: false,
350
+ // DeepSeek on Vertex thinks only when `reasoning_effort` is present.
351
+ supportsReasoningEffort: model.publisher === "deepseek-ai",
352
+ maxTokensField: "max_tokens",
353
+ thinkingFormat: model.publisher === "zai-org" ? "zai" : "openai",
354
+ },
355
+ };
356
+ }
357
+
358
+ /**
359
+ * Responses model on the publisher's OpenAI-compatible endpoint
360
+ * (`/publishers/{publisher}/v1/responses`), addressed by bare model id.
361
+ * pi-ai always sends `store: false`, which these endpoints require.
362
+ */
363
+ function responsesModel(model: VertexModelConfig, baseUrl: string): Model<"openai-responses"> {
364
+ return {
365
+ id: model.apiId,
366
+ name: model.name,
367
+ api: "openai-responses",
368
+ provider: "vertex",
369
+ baseUrl: `${baseUrl}/publishers/${model.publisher}/v1`,
370
+ reasoning: model.reasoning,
371
+ // Effort `none` is rejected; `off: null` makes pi-ai omit `reasoning` when no
372
+ // effort is requested, so the model runs at its default.
373
+ thinkingLevelMap: { ...model.thinkingLevelMap, off: null },
374
+ input: model.input,
375
+ cost: model.cost,
376
+ contextWindow: model.contextWindow,
377
+ maxTokens: model.maxTokens,
378
+ headers: {},
379
+ compat: {
380
+ // No 24h retention on Vertex: never send `prompt_cache_retention`.
381
+ supportsLongCacheRetention: false,
382
+ },
383
+ };
384
+ }
385
+
331
386
  export function streamMaaS(
332
387
  model: VertexModelConfig,
333
388
  context: Context,
334
389
  options?: StreamOptions,
335
390
  ): AssistantMessageEventStream {
336
391
  const stream = createAssistantMessageEventStream();
392
+ const useResponses = model.maasApi === "responses";
337
393
 
338
394
  (async () => {
339
- const apiModelId = model.apiId.includes("/") ? model.apiId : `${model.publisher}/${model.apiId}`;
340
-
341
395
  try {
342
396
  if (model.publisher === "anthropic") {
343
397
  await streamAnthropic(model, context, options, stream);
@@ -345,40 +399,24 @@ export function streamMaaS(
345
399
  return;
346
400
  }
347
401
 
348
- // Non-Anthropic MaaS models: Vertex OpenAI-compatible endpoint.
402
+ // Non-Anthropic MaaS models: Vertex OpenAI-compatible endpoints via pi-ai.
349
403
  const location = resolveLocation(model.region);
350
404
  const auth = getAuthConfig(location);
351
405
  const accessToken = await getAccessToken();
352
406
  const baseUrl = buildBaseUrl(auth.projectId, auth.location);
353
- const endpoint = `${baseUrl}/endpoints/openapi`;
354
-
355
- const modelForPi: Model<"openai-completions"> = {
356
- id: apiModelId,
357
- name: model.name,
358
- api: "openai-completions",
359
- provider: "vertex",
360
- baseUrl: endpoint,
361
- reasoning: model.reasoning,
362
- input: model.input,
363
- cost: model.cost,
364
- contextWindow: model.contextWindow,
365
- maxTokens: model.maxTokens,
366
- headers: {},
367
- compat: {
368
- supportsStore: false,
369
- supportsDeveloperRole: false,
370
- supportsReasoningEffort: false,
371
- maxTokensField: "max_tokens",
372
- thinkingFormat: model.publisher === "qwen" ? "qwen" : model.publisher === "zai-org" ? "zai" : "openai",
373
- },
374
- };
375
-
376
- const innerStream = streamSimpleOpenAICompletions(modelForPi, context as any, {
407
+ const innerOptions: SimpleStreamOptions = {
377
408
  ...options,
378
409
  apiKey: accessToken,
379
410
  maxTokens: options?.maxTokens || model.maxTokens,
380
411
  temperature: options?.temperature,
381
- });
412
+ };
413
+
414
+ // Pi 0.99 moved per-API stream functions off the pi-ai root.
415
+ // Pi's extension loader maps the compat entry at runtime.
416
+ const { openAICompletionsApi, openAIResponsesApi } = await import("@earendil-works/pi-ai/compat");
417
+ const innerStream = useResponses
418
+ ? openAIResponsesApi().streamSimple(responsesModel(model, baseUrl), context as any, innerOptions)
419
+ : openAICompletionsApi().streamSimple(chatCompletionsModel(model, baseUrl), context as any, innerOptions);
382
420
 
383
421
  for await (const event of innerStream) {
384
422
  if ("partial" in event && event.partial) event.partial.model = model.id;
@@ -399,7 +437,7 @@ export function streamMaaS(
399
437
  error: {
400
438
  role: "assistant",
401
439
  content: [],
402
- api: model.publisher === "anthropic" ? "anthropic-messages" : "openai-completions",
440
+ api: model.publisher === "anthropic" ? "anthropic-messages" : useResponses ? "openai-responses" : "openai-completions",
403
441
  provider: "vertex",
404
442
  model: model.id,
405
443
  usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } },
@@ -0,0 +1,169 @@
1
+ /**
2
+ * Fetch wrapper that strips keepalive noise from Vertex AI streaming responses.
3
+ *
4
+ * On slow streams (long prefill, hidden reasoning, large context) Vertex AI
5
+ * injects keepalive lines that downstream SDK parsers choke on:
6
+ *
7
+ * 1. `: keepalive` — SSE comment (Gemini streamGenerateContent).
8
+ * Spec-ignorable, but @google/genai's hand-rolled parser doesn't skip
9
+ * comments: "Unexpected token ':'..." / "Incomplete JSON segment at the
10
+ * end" / silently dropped data events.
11
+ *
12
+ * 2. `data: : keepalive` — a *data* event whose payload is a pseudo-comment
13
+ * (MaaS /endpoints/openapi chat completions, observed with e.g. Grok).
14
+ * Not a comment at all, so even spec-correct SSE parsers (openai SDK)
15
+ * hand `: keepalive` to JSON.parse:
16
+ * "Unexpected token ':', \": keepalive\" is not valid JSON"
17
+ *
18
+ * Both are safe to drop: a valid JSON payload can never start with ':'.
19
+ * Scoped to Vertex AI (aiplatform.googleapis.com) streaming URLs only.
20
+ */
21
+
22
+ let installed = false;
23
+
24
+ /** True for SSE comments and for data events carrying a pseudo-comment payload. */
25
+ function isKeepaliveLine(line: string): boolean {
26
+ if (line.startsWith(":")) return true; // SSE comment
27
+ if (line.startsWith("data:")) {
28
+ const payload = line.substring(5).trimStart();
29
+ return payload.startsWith(":"); // e.g. "data: : keepalive"
30
+ }
31
+ return false;
32
+ }
33
+
34
+ /**
35
+ * TransformStream that removes keepalive lines from a byte stream while
36
+ * preserving all other bytes (including event-delimiting blank lines).
37
+ */
38
+ function createKeepaliveStripper(): TransformStream<Uint8Array, Uint8Array> {
39
+ const decoder = new TextDecoder("utf-8");
40
+ const encoder = new TextEncoder();
41
+ let pending = "";
42
+
43
+ const filter = (text: string, isFlush: boolean): string => {
44
+ pending += text;
45
+ let output = "";
46
+ let start = 0;
47
+ while (true) {
48
+ const nl = pending.indexOf("\n", start);
49
+ if (nl === -1) break;
50
+ const line = pending.substring(start, nl + 1);
51
+ // Strip \r\n / \n terminated keepalive lines; keep everything else.
52
+ const content = line.endsWith("\r\n") ? line.slice(0, -2) : line.slice(0, -1);
53
+ if (!isKeepaliveLine(content)) output += line;
54
+ start = nl + 1;
55
+ }
56
+ pending = pending.substring(start);
57
+ if (isFlush && pending.length > 0) {
58
+ // Trailing data without a newline: drop only if it's a keepalive.
59
+ if (!isKeepaliveLine(pending)) output += pending;
60
+ pending = "";
61
+ }
62
+ return output;
63
+ };
64
+
65
+ return new TransformStream<Uint8Array, Uint8Array>({
66
+ transform(chunk, controller) {
67
+ const out = filter(decoder.decode(chunk, { stream: true }), false);
68
+ if (out.length > 0) controller.enqueue(encoder.encode(out));
69
+ },
70
+ flush(controller) {
71
+ const out = filter(decoder.decode(), true);
72
+ if (out.length > 0) controller.enqueue(encoder.encode(out));
73
+ },
74
+ });
75
+ }
76
+
77
+ /**
78
+ * Some Vertex deployments accept a streaming request, answer 200, and then send nothing at
79
+ * all \u2014 observed on `zai-org/glm-5.2-maas` (global) for roughly half of all requests, while
80
+ * glm-4.7 and deepseek on the same endpoint and credentials never did it. undici only gives up
81
+ * after its 300 s body timeout, so every dead request costs five minutes before a retry starts.
82
+ *
83
+ * Bound the wait for the *first* byte only. A stream that has begun is left alone: a genuinely
84
+ * slow one keeps the socket alive with keepalive lines (stripped further down, but counted here
85
+ * because this sits upstream of the stripper), and mid-stream gaps stay undici's business.
86
+ */
87
+ const FIRST_BYTE_TIMEOUT_MS = Number(process.env.PI_VERTEX_FIRST_BYTE_TIMEOUT_MS ?? 90_000);
88
+
89
+ function withFirstByteTimeout(body: ReadableStream<Uint8Array>, url: string, timeoutMs: number): ReadableStream<Uint8Array> {
90
+ if (!(timeoutMs > 0)) return body;
91
+ const reader = body.getReader();
92
+ let started = false;
93
+ return new ReadableStream<Uint8Array>({
94
+ async pull(controller) {
95
+ if (started) {
96
+ const { done, value } = await reader.read();
97
+ if (done) controller.close();
98
+ else controller.enqueue(value);
99
+ return;
100
+ }
101
+ let timer: ReturnType<typeof setTimeout> | undefined;
102
+ try {
103
+ const first = await Promise.race([
104
+ reader.read(),
105
+ new Promise<never>((_, reject) => {
106
+ timer = setTimeout(
107
+ () => reject(new Error(`Vertex sent no data for ${Math.round(timeoutMs / 1000)}s after accepting this streaming request (${url}). The endpoint is not producing output; retry or use another offering. Raise PI_VERTEX_FIRST_BYTE_TIMEOUT_MS to wait longer, 0 to disable.`)),
108
+ timeoutMs,
109
+ );
110
+ }),
111
+ ]);
112
+ started = true;
113
+ if (first.done) controller.close();
114
+ else controller.enqueue(first.value);
115
+ } catch (error) {
116
+ await reader.cancel(error).catch(() => {});
117
+ controller.error(error);
118
+ } finally {
119
+ clearTimeout(timer);
120
+ }
121
+ },
122
+ cancel(reason) { return reader.cancel(reason); },
123
+ });
124
+ }
125
+
126
+ function requestUrl(input: RequestInfo | URL): string {
127
+ if (typeof input === "string") return input;
128
+ if (input instanceof URL) return input.href;
129
+ return input.url;
130
+ }
131
+
132
+ /**
133
+ * Vertex AI streaming endpoints: Gemini SSE, MaaS chat completions, publisher
134
+ * Responses API (Meta Muse), Anthropic raw predict.
135
+ */
136
+ function isVertexStreamingUrl(url: string): boolean {
137
+ if (!url.includes("aiplatform.googleapis.com")) return false;
138
+ return (
139
+ url.includes("streamGenerateContent") ||
140
+ url.includes("/endpoints/openapi/") ||
141
+ url.includes("/v1/responses") ||
142
+ url.includes("streamRawPredict")
143
+ );
144
+ }
145
+
146
+ /**
147
+ * Install a global fetch wrapper (once) that pipes Vertex AI streaming
148
+ * response bodies through the keepalive stripper. All other requests pass
149
+ * through untouched.
150
+ */
151
+ export function installSseCommentFilter(): void {
152
+ if (installed) return;
153
+ installed = true;
154
+
155
+ const originalFetch = globalThis.fetch;
156
+ globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
157
+ const response = await originalFetch(input, init);
158
+ const url = requestUrl(input);
159
+ if (!isVertexStreamingUrl(url) || !response.body) {
160
+ return response;
161
+ }
162
+ const live = withFirstByteTimeout(response.body, url, FIRST_BYTE_TIMEOUT_MS);
163
+ return new Response(live.pipeThrough(createKeepaliveStripper()), {
164
+ headers: response.headers,
165
+ status: response.status,
166
+ statusText: response.statusText,
167
+ });
168
+ };
169
+ }
@@ -0,0 +1,53 @@
1
+ /** The keepalive filter must also give up on a Vertex stream that never starts. */
2
+ import assert from "node:assert/strict";
3
+ import { createServer } from "node:http";
4
+ import { once } from "node:events";
5
+ import { setTimeout as sleep } from "node:timers/promises";
6
+
7
+ process.env.PI_VERTEX_FIRST_BYTE_TIMEOUT_MS = "300";
8
+ const { installSseCommentFilter } = await import("../streaming/sse-comment-filter.ts");
9
+
10
+ const open = new Set();
11
+ const server = createServer(async (req, res) => {
12
+ open.add(res); res.on("close", () => open.delete(res));
13
+ res.writeHead(200, { "content-type": "text/event-stream" });
14
+ res.flushHeaders(); // 200 arrives, body may not: exactly what the failing deployment does
15
+ if (req.url.endsWith("/dead")) return; // accepted, then nothing: the glm-5.2 failure shape
16
+ if (req.url.endsWith("/slow-start")) { await sleep(150); }
17
+ res.write(": keepalive\n");
18
+ await sleep(500); // a mid-stream gap longer than the first-byte budget must survive
19
+ res.write(`data: {"choices":[{"delta":{"content":"LATE-BUT-ALIVE"}}]}\n\n`);
20
+ res.end("data: [DONE]\n\n");
21
+ });
22
+ server.listen(0, "127.0.0.1"); await once(server, "listening");
23
+ const { port } = server.address();
24
+ // The wrapper matches Vertex streaming URLs by substring, so a local URL carrying those
25
+ // markers in its path exercises the real wrapper without DNS or credentials.
26
+ const base = `http://127.0.0.1:${port}/aiplatform.googleapis.com/endpoints/openapi`;
27
+ // Meta Muse streams from the publisher Responses API instead of /endpoints/openapi.
28
+ const responsesBase = `http://127.0.0.1:${port}/aiplatform.googleapis.com/publishers/meta/v1/responses`;
29
+ installSseCommentFilter();
30
+
31
+ async function read(path, root = base) {
32
+ const response = await fetch(`${root}${path}`);
33
+ let body = "";
34
+ for await (const chunk of response.body) body += new TextDecoder().decode(chunk);
35
+ return body;
36
+ }
37
+
38
+ const started = Date.now();
39
+ await assert.rejects(read("/dead"), /sent no data for 0s after accepting this streaming request/);
40
+ const elapsed = Date.now() - started;
41
+ assert(elapsed < 3000, `a dead stream must fail fast, took ${elapsed}ms`);
42
+ assert(elapsed >= 300, `must wait its budget first, gave up after ${elapsed}ms`);
43
+ await assert.rejects(read("/dead", responsesBase), /sent no data/, "a dead Responses API stream must fail fast too");
44
+
45
+ for (const path of ["/live", "/slow-start"]) {
46
+ const body = await read(path);
47
+ assert.match(body, /LATE-BUT-ALIVE/, `${path} must survive a mid-stream gap longer than the first-byte budget`);
48
+ assert.doesNotMatch(body, /keepalive/, `${path} must still have keepalive lines stripped`);
49
+ }
50
+
51
+ for (const res of open) res.destroy();
52
+ server.close(); await once(server, "close");
53
+ console.log("PASS first-byte timeout fails a dead Vertex stream fast and leaves live ones alone");