@aliou/pi-neuralwatt 0.6.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aliou/pi-neuralwatt",
3
- "version": "0.6.2",
3
+ "version": "0.7.0",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "private": false,
@@ -33,7 +33,8 @@
33
33
  ],
34
34
  "dependencies": {
35
35
  "@aliou/pi-utils-settings": "^0.17.0",
36
- "@aliou/pi-utils-ui": "^0.4.0"
36
+ "@aliou/pi-utils-ui": "^0.4.0",
37
+ "@earendil-works/pi-ai": "$@earendil-works/pi-coding-agent"
37
38
  },
38
39
  "peerDependencies": {
39
40
  "@earendil-works/pi-coding-agent": "*",
@@ -1,3 +1,4 @@
1
+ import { getApiProvider } from "@earendil-works/pi-ai";
1
2
  import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
2
3
  import {
3
4
  configLoader,
@@ -19,13 +20,23 @@ import { fetchQuotas } from "../../utils/quotas";
19
20
  import { normalizeNeuralwattContextOverflowError } from "./context-overflow";
20
21
  import { getNeuralwattModels } from "./models";
21
22
  import { buildQuotasFromHeaders, fetchRequestedQuotas } from "./quota-store";
23
+ import {
24
+ type NeuralwattRateLimitInfo,
25
+ normalizeNeuralwattRateLimitError,
26
+ parseRateLimitHeaders,
27
+ } from "./rate-limit-error";
28
+ import { updateQuotasFromSseComment } from "./sse-quotas";
29
+ import { wrapNeuralwattStreamSimple } from "./stream-simple";
22
30
 
23
31
  const HEADER_EMIT_THROTTLE_MS = 5_000;
24
32
 
25
- function registerNeuralwattProvider(pi: ExtensionAPI): void {
33
+ function registerNeuralwattProvider(
34
+ pi: ExtensionAPI,
35
+ onSseQuota: (line: string) => void,
36
+ ): void {
26
37
  const { includeLegacyModelIds } = configLoader.getConfig();
27
38
 
28
- pi.registerProvider("neuralwatt", {
39
+ const config: Parameters<ExtensionAPI["registerProvider"]>[1] = {
29
40
  baseUrl: "https://api.neuralwatt.com/v1",
30
41
  apiKey: "$NEURALWATT_API_KEY",
31
42
  api: "openai-completions",
@@ -37,13 +48,36 @@ function registerNeuralwattProvider(pi: ExtensionAPI): void {
37
48
  models: getNeuralwattModels({
38
49
  includeLegacyModelIds,
39
50
  }),
40
- });
51
+ };
52
+
53
+ const provider = getApiProvider("openai-completions");
54
+ const baseStreamSimple = provider?.streamSimple;
55
+ if (baseStreamSimple) {
56
+ config.streamSimple = wrapNeuralwattStreamSimple(
57
+ baseStreamSimple as never,
58
+ onSseQuota,
59
+ ) as never;
60
+ }
61
+
62
+ pi.registerProvider("neuralwatt", config);
41
63
  }
42
64
 
43
65
  export default async function (pi: ExtensionAPI) {
44
66
  await configLoader.load();
45
67
 
46
- registerNeuralwattProvider(pi);
68
+ let latestQuotas: NeuralwattQuotas | undefined;
69
+
70
+ const handleSseQuota = (line: string) => {
71
+ const quotas = updateQuotasFromSseComment(latestQuotas, line);
72
+ if (!quotas || quotas === latestQuotas) return;
73
+ latestQuotas = quotas;
74
+ pi.events.emit(NEURALWATT_QUOTAS_UPDATED_EVENT, {
75
+ quotas,
76
+ source: "sse",
77
+ });
78
+ };
79
+
80
+ registerNeuralwattProvider(pi, handleSseQuota);
47
81
 
48
82
  const loadedFeatures = new Set<NeuralwattFeatureId>();
49
83
 
@@ -53,7 +87,7 @@ export default async function (pi: ExtensionAPI) {
53
87
  });
54
88
 
55
89
  pi.events.on(NEURALWATT_CONFIG_UPDATED_EVENT, () => {
56
- registerNeuralwattProvider(pi);
90
+ registerNeuralwattProvider(pi, handleSseQuota);
57
91
  });
58
92
 
59
93
  let lastHeaderEmitAt = 0;
@@ -67,20 +101,67 @@ export default async function (pi: ExtensionAPI) {
67
101
  if (source === "header" && now - lastHeaderEmitAt < HEADER_EMIT_THROTTLE_MS)
68
102
  return;
69
103
  if (source === "header") lastHeaderEmitAt = now;
104
+ latestQuotas = quotas;
70
105
  pi.events.emit(NEURALWATT_QUOTAS_UPDATED_EVENT, { quotas, source });
71
106
  }
72
107
 
108
+ // Stored rate-limit info from the most recent 429 response.
109
+ // Used in message_end to rewrite the generic error text with
110
+ // actionable details from Neuralwatt's response headers.
111
+ let pendingRateLimitInfo: NeuralwattRateLimitInfo | undefined;
112
+
73
113
  pi.on("message_end", (event, ctx) => {
74
- const message = normalizeNeuralwattContextOverflowError(
114
+ // Rewrite rate-limit errors with layer-specific details
115
+ if (
116
+ pendingRateLimitInfo &&
117
+ event.message.role === "assistant" &&
118
+ event.message.stopReason === "error" &&
119
+ (event.message.provider === "neuralwatt" ||
120
+ ctx.model?.provider === "neuralwatt")
121
+ ) {
122
+ const message = normalizeNeuralwattRateLimitError(
123
+ event.message,
124
+ pendingRateLimitInfo,
125
+ );
126
+ pendingRateLimitInfo = undefined;
127
+ return { message };
128
+ }
129
+
130
+ if (
131
+ event.message.role === "assistant" &&
132
+ event.message.stopReason === "error" &&
133
+ (event.message.provider === "neuralwatt" ||
134
+ ctx.model?.provider === "neuralwatt") &&
135
+ event.message.errorMessage?.includes("429")
136
+ ) {
137
+ return {
138
+ message: normalizeNeuralwattRateLimitError(event.message, {
139
+ layer: "unknown",
140
+ detail:
141
+ "Neuralwatt rate limit reached, but Pi did not receive layer-specific rate-limit headers. Retry shortly.",
142
+ }),
143
+ };
144
+ }
145
+
146
+ // Rewrite context overflow errors for Pi's native compaction
147
+ const overflowMessage = normalizeNeuralwattContextOverflowError(
75
148
  event.message,
76
149
  ctx.model?.provider,
77
150
  );
78
- if (!message) return;
79
- return { message };
151
+ if (!overflowMessage) return;
152
+ return { message: overflowMessage };
80
153
  });
81
154
 
82
155
  pi.on("after_provider_response", (event, ctx) => {
83
156
  if (ctx.model?.provider !== "neuralwatt") return;
157
+
158
+ // Capture rate-limit headers from 429 responses for message_end rewriting
159
+ if (event.status === 429) {
160
+ pendingRateLimitInfo = parseRateLimitHeaders(event.headers);
161
+ } else {
162
+ pendingRateLimitInfo = undefined;
163
+ }
164
+
84
165
  const quotas = buildQuotasFromHeaders(event.headers);
85
166
  if (!quotas) return;
86
167
  emitQuotas(quotas, "header");
@@ -91,11 +172,7 @@ export default async function (pi: ExtensionAPI) {
91
172
  quotaRequestInFlight = true;
92
173
  try {
93
174
  const quotas = await fetchRequestedQuotas(data);
94
- if (quotas)
95
- pi.events.emit(NEURALWATT_QUOTAS_UPDATED_EVENT, {
96
- quotas,
97
- source: "api",
98
- });
175
+ if (quotas) emitQuotas(quotas, "api");
99
176
  } finally {
100
177
  quotaRequestInFlight = false;
101
178
  }
@@ -107,6 +184,7 @@ export default async function (pi: ExtensionAPI) {
107
184
  });
108
185
 
109
186
  pi.on("session_start", async (_event, ctx) => {
187
+ pendingRateLimitInfo = undefined;
110
188
  for (const message of configLoader.drainMessages()) {
111
189
  ctx.ui.notify(message, "warning");
112
190
  }
@@ -5,11 +5,12 @@
5
5
  import type { ProviderModelConfig } from "@earendil-works/pi-coding-agent";
6
6
 
7
7
  export const NEURALWATT_MODELS: ProviderModelConfig[] = [
8
- // GLM-5 Fast - ZhipuAI
8
+ // GLM-5.1 (200K vLLM deployment) - ZhipuAI
9
+ // Legacy id previously aliased to glm-5.1; now serving a GLM-5.2 test build.
9
10
  {
10
- id: "glm-5-fast",
11
- name: "GLM-5 Fast",
12
- reasoning: false,
11
+ id: "zai-org/GLM-5.1-FP8",
12
+ name: "GLM-5.2 (test)",
13
+ reasoning: true,
13
14
  input: ["text"],
14
15
  cost: {
15
16
  input: 1.1,
@@ -17,14 +18,23 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
17
18
  cacheRead: 0,
18
19
  cacheWrite: 0,
19
20
  },
20
- contextWindow: 202736,
21
+ contextWindow: 1048560,
21
22
  maxTokens: 65536,
23
+ thinkingLevelMap: {
24
+ minimal: null,
25
+ low: null,
26
+ medium: "medium",
27
+ high: null,
28
+ xhigh: null,
29
+ },
22
30
  compat: {
23
31
  supportsDeveloperRole: false,
24
32
  maxTokensField: "max_tokens",
33
+ requiresReasoningContentOnAssistantMessages: true,
25
34
  },
26
35
  },
27
36
  // GLM-5.1 - ZhipuAI
37
+ // Backed by the 1048K GLM-5.2 deployment (GLM-5.1 redirect in effect). Deprecated.
28
38
  {
29
39
  id: "glm-5.1",
30
40
  name: "GLM-5.1",
@@ -36,7 +46,7 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
36
46
  cacheRead: 0,
37
47
  cacheWrite: 0,
38
48
  },
39
- contextWindow: 202736,
49
+ contextWindow: 1048560,
40
50
  maxTokens: 65536,
41
51
  thinkingLevelMap: {
42
52
  minimal: null,
@@ -63,7 +73,7 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
63
73
  cacheRead: 0,
64
74
  cacheWrite: 0,
65
75
  },
66
- contextWindow: 202736,
76
+ contextWindow: 1048560,
67
77
  maxTokens: 65536,
68
78
  compat: {
69
79
  supportsDeveloperRole: false,
@@ -84,12 +94,15 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
84
94
  },
85
95
  contextWindow: 1048560,
86
96
  maxTokens: 65536,
97
+ // GLM-5.2 has two native reasoning depths (high, max) plus thinking-off.
98
+ // Pi levels below high disable thinking; high -> high, xhigh -> max.
99
+ // See https://portal.neuralwatt.com/docs/api/chat-completions#reasoning-effort
87
100
  thinkingLevelMap: {
88
- minimal: "low",
89
- low: "low",
90
- medium: "medium",
101
+ minimal: null,
102
+ low: null,
103
+ medium: null,
91
104
  high: "high",
92
- xhigh: null,
105
+ xhigh: "max",
93
106
  },
94
107
  compat: {
95
108
  supportsDeveloperRole: false,
@@ -97,6 +110,25 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
97
110
  requiresReasoningContentOnAssistantMessages: true,
98
111
  },
99
112
  },
113
+ // GLM-5.2 Fast - ZhipuAI
114
+ {
115
+ id: "glm-5.2-fast",
116
+ name: "GLM-5.2 Fast",
117
+ reasoning: false,
118
+ input: ["text"],
119
+ cost: {
120
+ input: 1.45,
121
+ output: 4.5,
122
+ cacheRead: 0,
123
+ cacheWrite: 0,
124
+ },
125
+ contextWindow: 1048560,
126
+ maxTokens: 65536,
127
+ compat: {
128
+ supportsDeveloperRole: false,
129
+ maxTokensField: "max_tokens",
130
+ },
131
+ },
100
132
  // Kimi K2.5 - MoonshotAI
101
133
  {
102
134
  id: "moonshotai/Kimi-K2.5",
@@ -264,7 +296,7 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
264
296
  },
265
297
  // Kimi K2.7 Code - MoonshotAI
266
298
  {
267
- id: "moonshotai/Kimi-K2.7-Code",
299
+ id: "kimi-k2.7-code",
268
300
  name: "Kimi K2.7 Code",
269
301
  reasoning: true,
270
302
  input: ["text", "image"],
@@ -312,7 +344,6 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
312
344
  ];
313
345
 
314
346
  const LEGACY_MODEL_ALIAS_MAP = {
315
- "zai-org/GLM-5.1-FP8": "glm-5.1",
316
347
  "moonshotai/Kimi-K2.6": "kimi-k2.6",
317
348
  "Qwen/Qwen3.5-397B-A17B-FP8": "qwen3.5-397b",
318
349
  "Qwen/Qwen3.6-35B-A3B": "qwen3.6-35b",
@@ -0,0 +1,150 @@
1
+ interface AssistantErrorLike {
2
+ role: string;
3
+ stopReason?: string;
4
+ provider?: string;
5
+ errorMessage?: string;
6
+ }
7
+
8
+ /**
9
+ * Parsed rate-limit info from Neuralwatt 429 response headers.
10
+ *
11
+ * Neuralwatt applies three independent rate-limit layers plus a legacy RPM
12
+ * layer. Each sets unique headers so the client can tell which layer
13
+ * triggered the rejection.
14
+ *
15
+ * @see https://portal.neuralwatt.com/docs/guides/rate-limits
16
+ */
17
+ export interface NeuralwattRateLimitInfo {
18
+ /** Which rate-limit layer triggered the 429 */
19
+ layer: "concurrent" | "tpm" | "admission" | "rpm" | "unknown";
20
+ /** Seconds the server recommends waiting before retrying */
21
+ retryAfter?: number;
22
+ /** Human-readable details (varies per layer) */
23
+ detail: string;
24
+ }
25
+
26
+ /** Case-insensitive header lookup */
27
+ function getHeader(
28
+ headers: Record<string, string>,
29
+ name: string,
30
+ ): string | undefined {
31
+ const entry = Object.entries(headers).find(
32
+ ([key]) => key.toLowerCase() === name.toLowerCase(),
33
+ );
34
+ return entry?.[1];
35
+ }
36
+
37
+ /**
38
+ * Parse Neuralwatt rate-limit headers from a 429 response.
39
+ *
40
+ * Returns `undefined` if no rate-limit-specific headers are found (e.g. the
41
+ * 429 came from a different proxy or middleware that doesn't set these
42
+ * headers).
43
+ */
44
+ export function parseRateLimitHeaders(
45
+ headers: Record<string, string>,
46
+ ): NeuralwattRateLimitInfo | undefined {
47
+ const retryAfterRaw = getHeader(headers, "Retry-After");
48
+ const retryAfter = retryAfterRaw
49
+ ? Number.parseInt(retryAfterRaw, 10)
50
+ : undefined;
51
+
52
+ // 1. Concurrent-request limit
53
+ const concurrentDimension = getHeader(
54
+ headers,
55
+ "X-Concurrent-Limit-Dimension",
56
+ );
57
+ if (concurrentDimension) {
58
+ const active = getHeader(headers, "X-Concurrent-Limit-Active") ?? "?";
59
+ const max = getHeader(headers, "X-Concurrent-Limit-Max") ?? "?";
60
+ return {
61
+ layer: "concurrent",
62
+ retryAfter,
63
+ detail: `Concurrent request limit reached (${active}/${max} active, ${concurrentDimension}-scoped). Wait for an in-flight request to complete before retrying.`,
64
+ };
65
+ }
66
+
67
+ // 2. Input TPM limit
68
+ const tpmDimension = getHeader(headers, "X-TPM-Limit-Dimension");
69
+ if (tpmDimension) {
70
+ const tokens = getHeader(headers, "X-TPM-Limit-Tokens") ?? "?";
71
+ const max = getHeader(headers, "X-TPM-Limit-Max") ?? "?";
72
+ return {
73
+ layer: "tpm",
74
+ retryAfter,
75
+ detail: `Input token rate exceeded (${tokens}/${max} tokens/min, ${tpmDimension}-scoped). Wait before sending more requests.`,
76
+ };
77
+ }
78
+
79
+ // 3. Admission control
80
+ const admissionDimension = getHeader(headers, "X-Admission-Dimension");
81
+ if (admissionDimension) {
82
+ const inFlight = getHeader(headers, "X-Admission-InFlight") ?? "?";
83
+ const threshold = getHeader(headers, "X-Admission-Threshold") ?? "?";
84
+ return {
85
+ layer: "admission",
86
+ retryAfter,
87
+ detail: `Backend at capacity (${inFlight}/${threshold} in-flight tokens, ${admissionDimension}-scoped). The server is busy — retry shortly.`,
88
+ };
89
+ }
90
+
91
+ // 4. Legacy RPM limit
92
+ const rpmLimit = getHeader(headers, "X-RateLimit-Limit");
93
+ if (rpmLimit) {
94
+ const remaining = getHeader(headers, "X-RateLimit-Remaining") ?? "?";
95
+ return {
96
+ layer: "rpm",
97
+ retryAfter,
98
+ detail: `Requests per minute exceeded (${remaining}/${rpmLimit} remaining). Wait before sending more requests.`,
99
+ };
100
+ }
101
+
102
+ // Generic 429 with Retry-After but no layer-specific headers
103
+ if (retryAfter !== undefined) {
104
+ return {
105
+ layer: "unknown",
106
+ retryAfter,
107
+ detail: "Rate limited by the server. Wait before retrying.",
108
+ };
109
+ }
110
+
111
+ return undefined;
112
+ }
113
+
114
+ /**
115
+ * Build a user-facing error message from a NeuralwattRateLimitInfo.
116
+ */
117
+ function formatRateLimitError(info: NeuralwattRateLimitInfo): string {
118
+ const parts = [info.detail];
119
+
120
+ if (info.retryAfter !== undefined && info.retryAfter > 0) {
121
+ if (info.retryAfter < 60) {
122
+ parts.push(`Retry-After: ${info.retryAfter}s.`);
123
+ } else {
124
+ const mins = Math.ceil(info.retryAfter / 60);
125
+ parts.push(`Retry-After: ~${mins} min.`);
126
+ }
127
+ } else if (info.retryAfter === 0 && info.layer === "concurrent") {
128
+ // Retry-After: 0 means: retry as soon as a slot frees
129
+ parts.push("Retry immediately after an in-flight request completes.");
130
+ }
131
+
132
+ return `429 rate limit: ${parts.join(" ")}`;
133
+ }
134
+
135
+ /**
136
+ * Normalize Neuralwatt rate-limit errors so the user sees which layer
137
+ * triggered the 429 and what to do about it.
138
+ *
139
+ * Without this, Pi shows a generic "Too Many Requests" because the
140
+ * Neuralwatt 429 response body is empty — all diagnostics are in the
141
+ * response headers.
142
+ */
143
+ export function normalizeNeuralwattRateLimitError<
144
+ TMessage extends AssistantErrorLike,
145
+ >(message: TMessage, rateLimitInfo: NeuralwattRateLimitInfo): TMessage {
146
+ return {
147
+ ...message,
148
+ errorMessage: formatRateLimitError(rateLimitInfo),
149
+ };
150
+ }
@@ -0,0 +1,84 @@
1
+ import type { NeuralwattQuotas } from "../../types/quota-api";
2
+
3
+ const JOULES_PER_KWH = 3_600_000;
4
+
5
+ export function updateQuotasFromSseComment(
6
+ quotas: NeuralwattQuotas | undefined,
7
+ line: string,
8
+ ): NeuralwattQuotas | undefined {
9
+ if (!quotas) return;
10
+ const trimmed = line.trim();
11
+ const next = structuredClone(quotas);
12
+
13
+ try {
14
+ if (trimmed.startsWith(": energy ")) {
15
+ const energy = JSON.parse(trimmed.slice(9)) as { energy_joules?: number };
16
+ const energyKwh = (energy.energy_joules ?? 0) / JOULES_PER_KWH;
17
+ if (energyKwh <= 0) return quotas;
18
+ next.usage.current_month.energy_kwh += energyKwh;
19
+ next.usage.lifetime.energy_kwh += energyKwh;
20
+ if (next.subscription) {
21
+ next.subscription.kwh_used += energyKwh;
22
+ next.subscription.kwh_remaining = Math.max(
23
+ 0,
24
+ next.subscription.kwh_remaining - energyKwh,
25
+ );
26
+ }
27
+ next.snapshot_at = new Date().toISOString();
28
+ return next;
29
+ }
30
+
31
+ if (trimmed.startsWith(": cost ")) {
32
+ const cost = JSON.parse(trimmed.slice(7)) as {
33
+ request_cost_usd?: number;
34
+ };
35
+ const requestCostUsd = cost.request_cost_usd ?? 0;
36
+ if (requestCostUsd <= 0) return quotas;
37
+ next.balance.credits_remaining_usd = Math.max(
38
+ 0,
39
+ next.balance.credits_remaining_usd - requestCostUsd,
40
+ );
41
+ next.balance.credits_used_usd += requestCostUsd;
42
+ next.usage.current_month.cost_usd += requestCostUsd;
43
+ next.usage.lifetime.cost_usd += requestCostUsd;
44
+ next.snapshot_at = new Date().toISOString();
45
+ return next;
46
+ }
47
+ } catch {
48
+ return quotas;
49
+ }
50
+
51
+ return quotas;
52
+ }
53
+
54
+ export async function readQuotaCommentsFromTee(
55
+ body: ReadableStream<Uint8Array>,
56
+ onComment: (line: string) => void,
57
+ ): Promise<void> {
58
+ const reader = body.getReader();
59
+ const decoder = new TextDecoder();
60
+ let buffer = "";
61
+
62
+ try {
63
+ while (true) {
64
+ const { done, value } = await reader.read();
65
+ if (done) break;
66
+ buffer += decoder.decode(value, { stream: true });
67
+ const lines = buffer.split("\n");
68
+ buffer = lines.pop() ?? "";
69
+ for (const line of lines) onComment(line);
70
+ }
71
+
72
+ const final = decoder.decode(new Uint8Array(0), { stream: false });
73
+ const remaining = (buffer + final).trim();
74
+ if (remaining) onComment(remaining);
75
+ } catch {
76
+ // The SDK side may abort the tee; quota comments are best-effort.
77
+ } finally {
78
+ try {
79
+ reader.releaseLock();
80
+ } catch {
81
+ // ignore
82
+ }
83
+ }
84
+ }
@@ -0,0 +1,138 @@
1
+ // Neuralwatt is OpenAI-compatible, but the OpenAI SDK throws on non-2xx
2
+ // responses before Pi's after_provider_response hook can see the raw headers.
3
+ // We wrap the built-in openai-completions streamSimple so 429 rate-limit
4
+ // headers can be captured before the SDK turns them into a generic error, while
5
+ // still delegating normal streaming behavior to Pi's provider implementation.
6
+ //
7
+ // The SSE tee used for live quota comments is inspired by:
8
+ // https://github.com/monotykamary/pi-neuralwatt-provider
9
+
10
+ import {
11
+ type AssistantMessageEventStream,
12
+ type Context,
13
+ createAssistantMessageEventStream,
14
+ type Model,
15
+ type SimpleStreamOptions,
16
+ } from "@earendil-works/pi-ai";
17
+ import {
18
+ type NeuralwattRateLimitInfo,
19
+ normalizeNeuralwattRateLimitError,
20
+ parseRateLimitHeaders,
21
+ } from "./rate-limit-error";
22
+ import { readQuotaCommentsFromTee } from "./sse-quotas";
23
+
24
+ export type AnyStreamSimple = (
25
+ model: Model<string>,
26
+ context: Context,
27
+ options?: SimpleStreamOptions,
28
+ ) => AssistantMessageEventStream;
29
+
30
+ function headersToRecord(headers: Headers): Record<string, string> {
31
+ const record: Record<string, string> = {};
32
+ headers.forEach((value, key) => {
33
+ record[key] = value;
34
+ });
35
+ return record;
36
+ }
37
+
38
+ function isProviderChatCompletionsUrl(
39
+ input: RequestInfo | URL,
40
+ providerOrigin: string,
41
+ ): boolean {
42
+ const rawUrl =
43
+ typeof input === "string"
44
+ ? input
45
+ : input instanceof URL
46
+ ? input.toString()
47
+ : input.url;
48
+
49
+ try {
50
+ const url = new URL(rawUrl);
51
+ return (
52
+ url.origin === providerOrigin &&
53
+ url.pathname.endsWith("/chat/completions")
54
+ );
55
+ } catch {
56
+ return false;
57
+ }
58
+ }
59
+
60
+ async function forwardStream(
61
+ stream: AssistantMessageEventStream,
62
+ outer: AssistantMessageEventStream,
63
+ getRateLimitInfo: () => NeuralwattRateLimitInfo | undefined,
64
+ restoreFetch: () => void,
65
+ ): Promise<void> {
66
+ try {
67
+ for await (const event of stream) {
68
+ const rateLimitInfo = getRateLimitInfo();
69
+ if (event.type === "error" && rateLimitInfo) {
70
+ outer.push({
71
+ ...event,
72
+ error: normalizeNeuralwattRateLimitError(event.error, rateLimitInfo),
73
+ });
74
+ } else {
75
+ outer.push(event);
76
+ }
77
+ }
78
+ } finally {
79
+ restoreFetch();
80
+ outer.end();
81
+ }
82
+ }
83
+
84
+ export function wrapNeuralwattStreamSimple(
85
+ base: AnyStreamSimple,
86
+ onSseQuota: (line: string) => void,
87
+ ): AnyStreamSimple {
88
+ return (model, context, options = {}) => {
89
+ let rateLimitInfo: NeuralwattRateLimitInfo | undefined;
90
+ let sseQuotaTask: Promise<void> | undefined;
91
+ const outer = createAssistantMessageEventStream();
92
+ const providerOrigin = new URL(
93
+ model.baseUrl ?? "https://api.neuralwatt.com/v1",
94
+ ).origin;
95
+ const originalFetch = globalThis.fetch;
96
+ const wrappedFetch: typeof fetch = async (input, init) => {
97
+ const response = await originalFetch(input, init);
98
+
99
+ if (!isProviderChatCompletionsUrl(input, providerOrigin)) return response;
100
+
101
+ const headers = headersToRecord(response.headers);
102
+ if (response.status === 429) {
103
+ rateLimitInfo = parseRateLimitHeaders(headers);
104
+ return response;
105
+ }
106
+
107
+ if (response.ok && response.body) {
108
+ const [sdkBody, quotaBody] = response.body.tee();
109
+ sseQuotaTask = readQuotaCommentsFromTee(quotaBody, onSseQuota);
110
+ return new Response(sdkBody, {
111
+ headers: response.headers,
112
+ status: response.status,
113
+ statusText: response.statusText,
114
+ });
115
+ }
116
+
117
+ return response;
118
+ };
119
+
120
+ globalThis.fetch = wrappedFetch;
121
+
122
+ const restoreFetch = () => {
123
+ if (globalThis.fetch === wrappedFetch) globalThis.fetch = originalFetch;
124
+ sseQuotaTask?.catch(() => {});
125
+ };
126
+
127
+ const stream = base(model, context, options);
128
+ const originalOuterEnd = outer.end.bind(outer);
129
+ outer.end = (result?: Parameters<typeof originalOuterEnd>[0]) => {
130
+ restoreFetch();
131
+ originalOuterEnd(result);
132
+ };
133
+
134
+ void forwardStream(stream, outer, () => rateLimitInfo, restoreFetch);
135
+
136
+ return outer;
137
+ };
138
+ }
@@ -1,6 +1,6 @@
1
1
  import type { NeuralwattQuotas } from "./quota-api";
2
2
 
3
- export type QuotaSource = "header" | "api";
3
+ export type QuotaSource = "header" | "api" | "sse";
4
4
 
5
5
  export const NEURALWATT_QUOTAS_UPDATED_EVENT =
6
6
  "neuralwatt:quotas:updated" as const;