@aliou/pi-neuralwatt 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -2
- package/src/extensions/provider/index.ts +91 -13
- package/src/extensions/provider/models.ts +44 -13
- package/src/extensions/provider/rate-limit-error.ts +150 -0
- package/src/extensions/provider/sse-quotas.ts +84 -0
- package/src/extensions/provider/stream-simple.ts +138 -0
- package/src/types/quota-events.ts +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aliou/pi-neuralwatt",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"private": false,
|
|
@@ -33,7 +33,8 @@
|
|
|
33
33
|
],
|
|
34
34
|
"dependencies": {
|
|
35
35
|
"@aliou/pi-utils-settings": "^0.17.0",
|
|
36
|
-
"@aliou/pi-utils-ui": "^0.4.0"
|
|
36
|
+
"@aliou/pi-utils-ui": "^0.4.0",
|
|
37
|
+
"@earendil-works/pi-ai": "$@earendil-works/pi-coding-agent"
|
|
37
38
|
},
|
|
38
39
|
"peerDependencies": {
|
|
39
40
|
"@earendil-works/pi-coding-agent": "*",
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { getApiProvider } from "@earendil-works/pi-ai";
|
|
1
2
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
3
|
import {
|
|
3
4
|
configLoader,
|
|
@@ -19,13 +20,23 @@ import { fetchQuotas } from "../../utils/quotas";
|
|
|
19
20
|
import { normalizeNeuralwattContextOverflowError } from "./context-overflow";
|
|
20
21
|
import { getNeuralwattModels } from "./models";
|
|
21
22
|
import { buildQuotasFromHeaders, fetchRequestedQuotas } from "./quota-store";
|
|
23
|
+
import {
|
|
24
|
+
type NeuralwattRateLimitInfo,
|
|
25
|
+
normalizeNeuralwattRateLimitError,
|
|
26
|
+
parseRateLimitHeaders,
|
|
27
|
+
} from "./rate-limit-error";
|
|
28
|
+
import { updateQuotasFromSseComment } from "./sse-quotas";
|
|
29
|
+
import { wrapNeuralwattStreamSimple } from "./stream-simple";
|
|
22
30
|
|
|
23
31
|
const HEADER_EMIT_THROTTLE_MS = 5_000;
|
|
24
32
|
|
|
25
|
-
function registerNeuralwattProvider(
|
|
33
|
+
function registerNeuralwattProvider(
|
|
34
|
+
pi: ExtensionAPI,
|
|
35
|
+
onSseQuota: (line: string) => void,
|
|
36
|
+
): void {
|
|
26
37
|
const { includeLegacyModelIds } = configLoader.getConfig();
|
|
27
38
|
|
|
28
|
-
|
|
39
|
+
const config: Parameters<ExtensionAPI["registerProvider"]>[1] = {
|
|
29
40
|
baseUrl: "https://api.neuralwatt.com/v1",
|
|
30
41
|
apiKey: "$NEURALWATT_API_KEY",
|
|
31
42
|
api: "openai-completions",
|
|
@@ -37,13 +48,36 @@ function registerNeuralwattProvider(pi: ExtensionAPI): void {
|
|
|
37
48
|
models: getNeuralwattModels({
|
|
38
49
|
includeLegacyModelIds,
|
|
39
50
|
}),
|
|
40
|
-
}
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
const provider = getApiProvider("openai-completions");
|
|
54
|
+
const baseStreamSimple = provider?.streamSimple;
|
|
55
|
+
if (baseStreamSimple) {
|
|
56
|
+
config.streamSimple = wrapNeuralwattStreamSimple(
|
|
57
|
+
baseStreamSimple as never,
|
|
58
|
+
onSseQuota,
|
|
59
|
+
) as never;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
pi.registerProvider("neuralwatt", config);
|
|
41
63
|
}
|
|
42
64
|
|
|
43
65
|
export default async function (pi: ExtensionAPI) {
|
|
44
66
|
await configLoader.load();
|
|
45
67
|
|
|
46
|
-
|
|
68
|
+
let latestQuotas: NeuralwattQuotas | undefined;
|
|
69
|
+
|
|
70
|
+
const handleSseQuota = (line: string) => {
|
|
71
|
+
const quotas = updateQuotasFromSseComment(latestQuotas, line);
|
|
72
|
+
if (!quotas || quotas === latestQuotas) return;
|
|
73
|
+
latestQuotas = quotas;
|
|
74
|
+
pi.events.emit(NEURALWATT_QUOTAS_UPDATED_EVENT, {
|
|
75
|
+
quotas,
|
|
76
|
+
source: "sse",
|
|
77
|
+
});
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
registerNeuralwattProvider(pi, handleSseQuota);
|
|
47
81
|
|
|
48
82
|
const loadedFeatures = new Set<NeuralwattFeatureId>();
|
|
49
83
|
|
|
@@ -53,7 +87,7 @@ export default async function (pi: ExtensionAPI) {
|
|
|
53
87
|
});
|
|
54
88
|
|
|
55
89
|
pi.events.on(NEURALWATT_CONFIG_UPDATED_EVENT, () => {
|
|
56
|
-
registerNeuralwattProvider(pi);
|
|
90
|
+
registerNeuralwattProvider(pi, handleSseQuota);
|
|
57
91
|
});
|
|
58
92
|
|
|
59
93
|
let lastHeaderEmitAt = 0;
|
|
@@ -67,20 +101,67 @@ export default async function (pi: ExtensionAPI) {
|
|
|
67
101
|
if (source === "header" && now - lastHeaderEmitAt < HEADER_EMIT_THROTTLE_MS)
|
|
68
102
|
return;
|
|
69
103
|
if (source === "header") lastHeaderEmitAt = now;
|
|
104
|
+
latestQuotas = quotas;
|
|
70
105
|
pi.events.emit(NEURALWATT_QUOTAS_UPDATED_EVENT, { quotas, source });
|
|
71
106
|
}
|
|
72
107
|
|
|
108
|
+
// Stored rate-limit info from the most recent 429 response.
|
|
109
|
+
// Used in message_end to rewrite the generic error text with
|
|
110
|
+
// actionable details from Neuralwatt's response headers.
|
|
111
|
+
let pendingRateLimitInfo: NeuralwattRateLimitInfo | undefined;
|
|
112
|
+
|
|
73
113
|
pi.on("message_end", (event, ctx) => {
|
|
74
|
-
|
|
114
|
+
// Rewrite rate-limit errors with layer-specific details
|
|
115
|
+
if (
|
|
116
|
+
pendingRateLimitInfo &&
|
|
117
|
+
event.message.role === "assistant" &&
|
|
118
|
+
event.message.stopReason === "error" &&
|
|
119
|
+
(event.message.provider === "neuralwatt" ||
|
|
120
|
+
ctx.model?.provider === "neuralwatt")
|
|
121
|
+
) {
|
|
122
|
+
const message = normalizeNeuralwattRateLimitError(
|
|
123
|
+
event.message,
|
|
124
|
+
pendingRateLimitInfo,
|
|
125
|
+
);
|
|
126
|
+
pendingRateLimitInfo = undefined;
|
|
127
|
+
return { message };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if (
|
|
131
|
+
event.message.role === "assistant" &&
|
|
132
|
+
event.message.stopReason === "error" &&
|
|
133
|
+
(event.message.provider === "neuralwatt" ||
|
|
134
|
+
ctx.model?.provider === "neuralwatt") &&
|
|
135
|
+
event.message.errorMessage?.includes("429")
|
|
136
|
+
) {
|
|
137
|
+
return {
|
|
138
|
+
message: normalizeNeuralwattRateLimitError(event.message, {
|
|
139
|
+
layer: "unknown",
|
|
140
|
+
detail:
|
|
141
|
+
"Neuralwatt rate limit reached, but Pi did not receive layer-specific rate-limit headers. Retry shortly.",
|
|
142
|
+
}),
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Rewrite context overflow errors for Pi's native compaction
|
|
147
|
+
const overflowMessage = normalizeNeuralwattContextOverflowError(
|
|
75
148
|
event.message,
|
|
76
149
|
ctx.model?.provider,
|
|
77
150
|
);
|
|
78
|
-
if (!
|
|
79
|
-
return { message };
|
|
151
|
+
if (!overflowMessage) return;
|
|
152
|
+
return { message: overflowMessage };
|
|
80
153
|
});
|
|
81
154
|
|
|
82
155
|
pi.on("after_provider_response", (event, ctx) => {
|
|
83
156
|
if (ctx.model?.provider !== "neuralwatt") return;
|
|
157
|
+
|
|
158
|
+
// Capture rate-limit headers from 429 responses for message_end rewriting
|
|
159
|
+
if (event.status === 429) {
|
|
160
|
+
pendingRateLimitInfo = parseRateLimitHeaders(event.headers);
|
|
161
|
+
} else {
|
|
162
|
+
pendingRateLimitInfo = undefined;
|
|
163
|
+
}
|
|
164
|
+
|
|
84
165
|
const quotas = buildQuotasFromHeaders(event.headers);
|
|
85
166
|
if (!quotas) return;
|
|
86
167
|
emitQuotas(quotas, "header");
|
|
@@ -91,11 +172,7 @@ export default async function (pi: ExtensionAPI) {
|
|
|
91
172
|
quotaRequestInFlight = true;
|
|
92
173
|
try {
|
|
93
174
|
const quotas = await fetchRequestedQuotas(data);
|
|
94
|
-
if (quotas)
|
|
95
|
-
pi.events.emit(NEURALWATT_QUOTAS_UPDATED_EVENT, {
|
|
96
|
-
quotas,
|
|
97
|
-
source: "api",
|
|
98
|
-
});
|
|
175
|
+
if (quotas) emitQuotas(quotas, "api");
|
|
99
176
|
} finally {
|
|
100
177
|
quotaRequestInFlight = false;
|
|
101
178
|
}
|
|
@@ -107,6 +184,7 @@ export default async function (pi: ExtensionAPI) {
|
|
|
107
184
|
});
|
|
108
185
|
|
|
109
186
|
pi.on("session_start", async (_event, ctx) => {
|
|
187
|
+
pendingRateLimitInfo = undefined;
|
|
110
188
|
for (const message of configLoader.drainMessages()) {
|
|
111
189
|
ctx.ui.notify(message, "warning");
|
|
112
190
|
}
|
|
@@ -5,11 +5,12 @@
|
|
|
5
5
|
import type { ProviderModelConfig } from "@earendil-works/pi-coding-agent";
|
|
6
6
|
|
|
7
7
|
export const NEURALWATT_MODELS: ProviderModelConfig[] = [
|
|
8
|
-
// GLM-5
|
|
8
|
+
// GLM-5.1 (200K vLLM deployment) - ZhipuAI
|
|
9
|
+
// Legacy id previously aliased to glm-5.1; now serving a GLM-5.2 test build.
|
|
9
10
|
{
|
|
10
|
-
id: "
|
|
11
|
-
name: "GLM-5
|
|
12
|
-
reasoning:
|
|
11
|
+
id: "zai-org/GLM-5.1-FP8",
|
|
12
|
+
name: "GLM-5.2 (test)",
|
|
13
|
+
reasoning: true,
|
|
13
14
|
input: ["text"],
|
|
14
15
|
cost: {
|
|
15
16
|
input: 1.1,
|
|
@@ -17,14 +18,23 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
|
|
|
17
18
|
cacheRead: 0,
|
|
18
19
|
cacheWrite: 0,
|
|
19
20
|
},
|
|
20
|
-
contextWindow:
|
|
21
|
+
contextWindow: 1048560,
|
|
21
22
|
maxTokens: 65536,
|
|
23
|
+
thinkingLevelMap: {
|
|
24
|
+
minimal: null,
|
|
25
|
+
low: null,
|
|
26
|
+
medium: "medium",
|
|
27
|
+
high: null,
|
|
28
|
+
xhigh: null,
|
|
29
|
+
},
|
|
22
30
|
compat: {
|
|
23
31
|
supportsDeveloperRole: false,
|
|
24
32
|
maxTokensField: "max_tokens",
|
|
33
|
+
requiresReasoningContentOnAssistantMessages: true,
|
|
25
34
|
},
|
|
26
35
|
},
|
|
27
36
|
// GLM-5.1 - ZhipuAI
|
|
37
|
+
// Backed by the 1048K GLM-5.2 deployment (GLM-5.1 redirect in effect). Deprecated.
|
|
28
38
|
{
|
|
29
39
|
id: "glm-5.1",
|
|
30
40
|
name: "GLM-5.1",
|
|
@@ -36,7 +46,7 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
|
|
|
36
46
|
cacheRead: 0,
|
|
37
47
|
cacheWrite: 0,
|
|
38
48
|
},
|
|
39
|
-
contextWindow:
|
|
49
|
+
contextWindow: 1048560,
|
|
40
50
|
maxTokens: 65536,
|
|
41
51
|
thinkingLevelMap: {
|
|
42
52
|
minimal: null,
|
|
@@ -63,7 +73,7 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
|
|
|
63
73
|
cacheRead: 0,
|
|
64
74
|
cacheWrite: 0,
|
|
65
75
|
},
|
|
66
|
-
contextWindow:
|
|
76
|
+
contextWindow: 1048560,
|
|
67
77
|
maxTokens: 65536,
|
|
68
78
|
compat: {
|
|
69
79
|
supportsDeveloperRole: false,
|
|
@@ -84,12 +94,15 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
|
|
|
84
94
|
},
|
|
85
95
|
contextWindow: 1048560,
|
|
86
96
|
maxTokens: 65536,
|
|
97
|
+
// GLM-5.2 has two native reasoning depths (high, max) plus thinking-off.
|
|
98
|
+
// Pi levels below high disable thinking; high -> high, xhigh -> max.
|
|
99
|
+
// See https://portal.neuralwatt.com/docs/api/chat-completions#reasoning-effort
|
|
87
100
|
thinkingLevelMap: {
|
|
88
|
-
minimal:
|
|
89
|
-
low:
|
|
90
|
-
medium:
|
|
101
|
+
minimal: null,
|
|
102
|
+
low: null,
|
|
103
|
+
medium: null,
|
|
91
104
|
high: "high",
|
|
92
|
-
xhigh:
|
|
105
|
+
xhigh: "max",
|
|
93
106
|
},
|
|
94
107
|
compat: {
|
|
95
108
|
supportsDeveloperRole: false,
|
|
@@ -97,6 +110,25 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
|
|
|
97
110
|
requiresReasoningContentOnAssistantMessages: true,
|
|
98
111
|
},
|
|
99
112
|
},
|
|
113
|
+
// GLM-5.2 Fast - ZhipuAI
|
|
114
|
+
{
|
|
115
|
+
id: "glm-5.2-fast",
|
|
116
|
+
name: "GLM-5.2 Fast",
|
|
117
|
+
reasoning: false,
|
|
118
|
+
input: ["text"],
|
|
119
|
+
cost: {
|
|
120
|
+
input: 1.45,
|
|
121
|
+
output: 4.5,
|
|
122
|
+
cacheRead: 0,
|
|
123
|
+
cacheWrite: 0,
|
|
124
|
+
},
|
|
125
|
+
contextWindow: 1048560,
|
|
126
|
+
maxTokens: 65536,
|
|
127
|
+
compat: {
|
|
128
|
+
supportsDeveloperRole: false,
|
|
129
|
+
maxTokensField: "max_tokens",
|
|
130
|
+
},
|
|
131
|
+
},
|
|
100
132
|
// Kimi K2.5 - MoonshotAI
|
|
101
133
|
{
|
|
102
134
|
id: "moonshotai/Kimi-K2.5",
|
|
@@ -264,7 +296,7 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
|
|
|
264
296
|
},
|
|
265
297
|
// Kimi K2.7 Code - MoonshotAI
|
|
266
298
|
{
|
|
267
|
-
id: "
|
|
299
|
+
id: "kimi-k2.7-code",
|
|
268
300
|
name: "Kimi K2.7 Code",
|
|
269
301
|
reasoning: true,
|
|
270
302
|
input: ["text", "image"],
|
|
@@ -312,7 +344,6 @@ export const NEURALWATT_MODELS: ProviderModelConfig[] = [
|
|
|
312
344
|
];
|
|
313
345
|
|
|
314
346
|
const LEGACY_MODEL_ALIAS_MAP = {
|
|
315
|
-
"zai-org/GLM-5.1-FP8": "glm-5.1",
|
|
316
347
|
"moonshotai/Kimi-K2.6": "kimi-k2.6",
|
|
317
348
|
"Qwen/Qwen3.5-397B-A17B-FP8": "qwen3.5-397b",
|
|
318
349
|
"Qwen/Qwen3.6-35B-A3B": "qwen3.6-35b",
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
interface AssistantErrorLike {
|
|
2
|
+
role: string;
|
|
3
|
+
stopReason?: string;
|
|
4
|
+
provider?: string;
|
|
5
|
+
errorMessage?: string;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Parsed rate-limit info from Neuralwatt 429 response headers.
|
|
10
|
+
*
|
|
11
|
+
* Neuralwatt applies three independent rate-limit layers plus a legacy RPM
|
|
12
|
+
* layer. Each sets unique headers so the client can tell which layer
|
|
13
|
+
* triggered the rejection.
|
|
14
|
+
*
|
|
15
|
+
* @see https://portal.neuralwatt.com/docs/guides/rate-limits
|
|
16
|
+
*/
|
|
17
|
+
export interface NeuralwattRateLimitInfo {
|
|
18
|
+
/** Which rate-limit layer triggered the 429 */
|
|
19
|
+
layer: "concurrent" | "tpm" | "admission" | "rpm" | "unknown";
|
|
20
|
+
/** Seconds the server recommends waiting before retrying */
|
|
21
|
+
retryAfter?: number;
|
|
22
|
+
/** Human-readable details (varies per layer) */
|
|
23
|
+
detail: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Case-insensitive header lookup */
|
|
27
|
+
function getHeader(
|
|
28
|
+
headers: Record<string, string>,
|
|
29
|
+
name: string,
|
|
30
|
+
): string | undefined {
|
|
31
|
+
const entry = Object.entries(headers).find(
|
|
32
|
+
([key]) => key.toLowerCase() === name.toLowerCase(),
|
|
33
|
+
);
|
|
34
|
+
return entry?.[1];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Parse Neuralwatt rate-limit headers from a 429 response.
|
|
39
|
+
*
|
|
40
|
+
* Returns `undefined` if no rate-limit-specific headers are found (e.g. the
|
|
41
|
+
* 429 came from a different proxy or middleware that doesn't set these
|
|
42
|
+
* headers).
|
|
43
|
+
*/
|
|
44
|
+
export function parseRateLimitHeaders(
|
|
45
|
+
headers: Record<string, string>,
|
|
46
|
+
): NeuralwattRateLimitInfo | undefined {
|
|
47
|
+
const retryAfterRaw = getHeader(headers, "Retry-After");
|
|
48
|
+
const retryAfter = retryAfterRaw
|
|
49
|
+
? Number.parseInt(retryAfterRaw, 10)
|
|
50
|
+
: undefined;
|
|
51
|
+
|
|
52
|
+
// 1. Concurrent-request limit
|
|
53
|
+
const concurrentDimension = getHeader(
|
|
54
|
+
headers,
|
|
55
|
+
"X-Concurrent-Limit-Dimension",
|
|
56
|
+
);
|
|
57
|
+
if (concurrentDimension) {
|
|
58
|
+
const active = getHeader(headers, "X-Concurrent-Limit-Active") ?? "?";
|
|
59
|
+
const max = getHeader(headers, "X-Concurrent-Limit-Max") ?? "?";
|
|
60
|
+
return {
|
|
61
|
+
layer: "concurrent",
|
|
62
|
+
retryAfter,
|
|
63
|
+
detail: `Concurrent request limit reached (${active}/${max} active, ${concurrentDimension}-scoped). Wait for an in-flight request to complete before retrying.`,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// 2. Input TPM limit
|
|
68
|
+
const tpmDimension = getHeader(headers, "X-TPM-Limit-Dimension");
|
|
69
|
+
if (tpmDimension) {
|
|
70
|
+
const tokens = getHeader(headers, "X-TPM-Limit-Tokens") ?? "?";
|
|
71
|
+
const max = getHeader(headers, "X-TPM-Limit-Max") ?? "?";
|
|
72
|
+
return {
|
|
73
|
+
layer: "tpm",
|
|
74
|
+
retryAfter,
|
|
75
|
+
detail: `Input token rate exceeded (${tokens}/${max} tokens/min, ${tpmDimension}-scoped). Wait before sending more requests.`,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// 3. Admission control
|
|
80
|
+
const admissionDimension = getHeader(headers, "X-Admission-Dimension");
|
|
81
|
+
if (admissionDimension) {
|
|
82
|
+
const inFlight = getHeader(headers, "X-Admission-InFlight") ?? "?";
|
|
83
|
+
const threshold = getHeader(headers, "X-Admission-Threshold") ?? "?";
|
|
84
|
+
return {
|
|
85
|
+
layer: "admission",
|
|
86
|
+
retryAfter,
|
|
87
|
+
detail: `Backend at capacity (${inFlight}/${threshold} in-flight tokens, ${admissionDimension}-scoped). The server is busy — retry shortly.`,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// 4. Legacy RPM limit
|
|
92
|
+
const rpmLimit = getHeader(headers, "X-RateLimit-Limit");
|
|
93
|
+
if (rpmLimit) {
|
|
94
|
+
const remaining = getHeader(headers, "X-RateLimit-Remaining") ?? "?";
|
|
95
|
+
return {
|
|
96
|
+
layer: "rpm",
|
|
97
|
+
retryAfter,
|
|
98
|
+
detail: `Requests per minute exceeded (${remaining}/${rpmLimit} remaining). Wait before sending more requests.`,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Generic 429 with Retry-After but no layer-specific headers
|
|
103
|
+
if (retryAfter !== undefined) {
|
|
104
|
+
return {
|
|
105
|
+
layer: "unknown",
|
|
106
|
+
retryAfter,
|
|
107
|
+
detail: "Rate limited by the server. Wait before retrying.",
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return undefined;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Build a user-facing error message from a NeuralwattRateLimitInfo.
|
|
116
|
+
*/
|
|
117
|
+
function formatRateLimitError(info: NeuralwattRateLimitInfo): string {
|
|
118
|
+
const parts = [info.detail];
|
|
119
|
+
|
|
120
|
+
if (info.retryAfter !== undefined && info.retryAfter > 0) {
|
|
121
|
+
if (info.retryAfter < 60) {
|
|
122
|
+
parts.push(`Retry-After: ${info.retryAfter}s.`);
|
|
123
|
+
} else {
|
|
124
|
+
const mins = Math.ceil(info.retryAfter / 60);
|
|
125
|
+
parts.push(`Retry-After: ~${mins} min.`);
|
|
126
|
+
}
|
|
127
|
+
} else if (info.retryAfter === 0 && info.layer === "concurrent") {
|
|
128
|
+
// Retry-After: 0 means: retry as soon as a slot frees
|
|
129
|
+
parts.push("Retry immediately after an in-flight request completes.");
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
return `429 rate limit: ${parts.join(" ")}`;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Normalize Neuralwatt rate-limit errors so the user sees which layer
|
|
137
|
+
* triggered the 429 and what to do about it.
|
|
138
|
+
*
|
|
139
|
+
* Without this, Pi shows a generic "Too Many Requests" because the
|
|
140
|
+
* Neuralwatt 429 response body is empty — all diagnostics are in the
|
|
141
|
+
* response headers.
|
|
142
|
+
*/
|
|
143
|
+
export function normalizeNeuralwattRateLimitError<
|
|
144
|
+
TMessage extends AssistantErrorLike,
|
|
145
|
+
>(message: TMessage, rateLimitInfo: NeuralwattRateLimitInfo): TMessage {
|
|
146
|
+
return {
|
|
147
|
+
...message,
|
|
148
|
+
errorMessage: formatRateLimitError(rateLimitInfo),
|
|
149
|
+
};
|
|
150
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import type { NeuralwattQuotas } from "../../types/quota-api";
|
|
2
|
+
|
|
3
|
+
const JOULES_PER_KWH = 3_600_000;
|
|
4
|
+
|
|
5
|
+
export function updateQuotasFromSseComment(
|
|
6
|
+
quotas: NeuralwattQuotas | undefined,
|
|
7
|
+
line: string,
|
|
8
|
+
): NeuralwattQuotas | undefined {
|
|
9
|
+
if (!quotas) return;
|
|
10
|
+
const trimmed = line.trim();
|
|
11
|
+
const next = structuredClone(quotas);
|
|
12
|
+
|
|
13
|
+
try {
|
|
14
|
+
if (trimmed.startsWith(": energy ")) {
|
|
15
|
+
const energy = JSON.parse(trimmed.slice(9)) as { energy_joules?: number };
|
|
16
|
+
const energyKwh = (energy.energy_joules ?? 0) / JOULES_PER_KWH;
|
|
17
|
+
if (energyKwh <= 0) return quotas;
|
|
18
|
+
next.usage.current_month.energy_kwh += energyKwh;
|
|
19
|
+
next.usage.lifetime.energy_kwh += energyKwh;
|
|
20
|
+
if (next.subscription) {
|
|
21
|
+
next.subscription.kwh_used += energyKwh;
|
|
22
|
+
next.subscription.kwh_remaining = Math.max(
|
|
23
|
+
0,
|
|
24
|
+
next.subscription.kwh_remaining - energyKwh,
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
next.snapshot_at = new Date().toISOString();
|
|
28
|
+
return next;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (trimmed.startsWith(": cost ")) {
|
|
32
|
+
const cost = JSON.parse(trimmed.slice(7)) as {
|
|
33
|
+
request_cost_usd?: number;
|
|
34
|
+
};
|
|
35
|
+
const requestCostUsd = cost.request_cost_usd ?? 0;
|
|
36
|
+
if (requestCostUsd <= 0) return quotas;
|
|
37
|
+
next.balance.credits_remaining_usd = Math.max(
|
|
38
|
+
0,
|
|
39
|
+
next.balance.credits_remaining_usd - requestCostUsd,
|
|
40
|
+
);
|
|
41
|
+
next.balance.credits_used_usd += requestCostUsd;
|
|
42
|
+
next.usage.current_month.cost_usd += requestCostUsd;
|
|
43
|
+
next.usage.lifetime.cost_usd += requestCostUsd;
|
|
44
|
+
next.snapshot_at = new Date().toISOString();
|
|
45
|
+
return next;
|
|
46
|
+
}
|
|
47
|
+
} catch {
|
|
48
|
+
return quotas;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
return quotas;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export async function readQuotaCommentsFromTee(
|
|
55
|
+
body: ReadableStream<Uint8Array>,
|
|
56
|
+
onComment: (line: string) => void,
|
|
57
|
+
): Promise<void> {
|
|
58
|
+
const reader = body.getReader();
|
|
59
|
+
const decoder = new TextDecoder();
|
|
60
|
+
let buffer = "";
|
|
61
|
+
|
|
62
|
+
try {
|
|
63
|
+
while (true) {
|
|
64
|
+
const { done, value } = await reader.read();
|
|
65
|
+
if (done) break;
|
|
66
|
+
buffer += decoder.decode(value, { stream: true });
|
|
67
|
+
const lines = buffer.split("\n");
|
|
68
|
+
buffer = lines.pop() ?? "";
|
|
69
|
+
for (const line of lines) onComment(line);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const final = decoder.decode(new Uint8Array(0), { stream: false });
|
|
73
|
+
const remaining = (buffer + final).trim();
|
|
74
|
+
if (remaining) onComment(remaining);
|
|
75
|
+
} catch {
|
|
76
|
+
// The SDK side may abort the tee; quota comments are best-effort.
|
|
77
|
+
} finally {
|
|
78
|
+
try {
|
|
79
|
+
reader.releaseLock();
|
|
80
|
+
} catch {
|
|
81
|
+
// ignore
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
// Neuralwatt is OpenAI-compatible, but the OpenAI SDK throws on non-2xx
|
|
2
|
+
// responses before Pi's after_provider_response hook can see the raw headers.
|
|
3
|
+
// We wrap the built-in openai-completions streamSimple so 429 rate-limit
|
|
4
|
+
// headers can be captured before the SDK turns them into a generic error, while
|
|
5
|
+
// still delegating normal streaming behavior to Pi's provider implementation.
|
|
6
|
+
//
|
|
7
|
+
// The SSE tee used for live quota comments is inspired by:
|
|
8
|
+
// https://github.com/monotykamary/pi-neuralwatt-provider
|
|
9
|
+
|
|
10
|
+
import {
|
|
11
|
+
type AssistantMessageEventStream,
|
|
12
|
+
type Context,
|
|
13
|
+
createAssistantMessageEventStream,
|
|
14
|
+
type Model,
|
|
15
|
+
type SimpleStreamOptions,
|
|
16
|
+
} from "@earendil-works/pi-ai";
|
|
17
|
+
import {
|
|
18
|
+
type NeuralwattRateLimitInfo,
|
|
19
|
+
normalizeNeuralwattRateLimitError,
|
|
20
|
+
parseRateLimitHeaders,
|
|
21
|
+
} from "./rate-limit-error";
|
|
22
|
+
import { readQuotaCommentsFromTee } from "./sse-quotas";
|
|
23
|
+
|
|
24
|
+
export type AnyStreamSimple = (
|
|
25
|
+
model: Model<string>,
|
|
26
|
+
context: Context,
|
|
27
|
+
options?: SimpleStreamOptions,
|
|
28
|
+
) => AssistantMessageEventStream;
|
|
29
|
+
|
|
30
|
+
function headersToRecord(headers: Headers): Record<string, string> {
|
|
31
|
+
const record: Record<string, string> = {};
|
|
32
|
+
headers.forEach((value, key) => {
|
|
33
|
+
record[key] = value;
|
|
34
|
+
});
|
|
35
|
+
return record;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function isProviderChatCompletionsUrl(
|
|
39
|
+
input: RequestInfo | URL,
|
|
40
|
+
providerOrigin: string,
|
|
41
|
+
): boolean {
|
|
42
|
+
const rawUrl =
|
|
43
|
+
typeof input === "string"
|
|
44
|
+
? input
|
|
45
|
+
: input instanceof URL
|
|
46
|
+
? input.toString()
|
|
47
|
+
: input.url;
|
|
48
|
+
|
|
49
|
+
try {
|
|
50
|
+
const url = new URL(rawUrl);
|
|
51
|
+
return (
|
|
52
|
+
url.origin === providerOrigin &&
|
|
53
|
+
url.pathname.endsWith("/chat/completions")
|
|
54
|
+
);
|
|
55
|
+
} catch {
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async function forwardStream(
|
|
61
|
+
stream: AssistantMessageEventStream,
|
|
62
|
+
outer: AssistantMessageEventStream,
|
|
63
|
+
getRateLimitInfo: () => NeuralwattRateLimitInfo | undefined,
|
|
64
|
+
restoreFetch: () => void,
|
|
65
|
+
): Promise<void> {
|
|
66
|
+
try {
|
|
67
|
+
for await (const event of stream) {
|
|
68
|
+
const rateLimitInfo = getRateLimitInfo();
|
|
69
|
+
if (event.type === "error" && rateLimitInfo) {
|
|
70
|
+
outer.push({
|
|
71
|
+
...event,
|
|
72
|
+
error: normalizeNeuralwattRateLimitError(event.error, rateLimitInfo),
|
|
73
|
+
});
|
|
74
|
+
} else {
|
|
75
|
+
outer.push(event);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
} finally {
|
|
79
|
+
restoreFetch();
|
|
80
|
+
outer.end();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function wrapNeuralwattStreamSimple(
|
|
85
|
+
base: AnyStreamSimple,
|
|
86
|
+
onSseQuota: (line: string) => void,
|
|
87
|
+
): AnyStreamSimple {
|
|
88
|
+
return (model, context, options = {}) => {
|
|
89
|
+
let rateLimitInfo: NeuralwattRateLimitInfo | undefined;
|
|
90
|
+
let sseQuotaTask: Promise<void> | undefined;
|
|
91
|
+
const outer = createAssistantMessageEventStream();
|
|
92
|
+
const providerOrigin = new URL(
|
|
93
|
+
model.baseUrl ?? "https://api.neuralwatt.com/v1",
|
|
94
|
+
).origin;
|
|
95
|
+
const originalFetch = globalThis.fetch;
|
|
96
|
+
const wrappedFetch: typeof fetch = async (input, init) => {
|
|
97
|
+
const response = await originalFetch(input, init);
|
|
98
|
+
|
|
99
|
+
if (!isProviderChatCompletionsUrl(input, providerOrigin)) return response;
|
|
100
|
+
|
|
101
|
+
const headers = headersToRecord(response.headers);
|
|
102
|
+
if (response.status === 429) {
|
|
103
|
+
rateLimitInfo = parseRateLimitHeaders(headers);
|
|
104
|
+
return response;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (response.ok && response.body) {
|
|
108
|
+
const [sdkBody, quotaBody] = response.body.tee();
|
|
109
|
+
sseQuotaTask = readQuotaCommentsFromTee(quotaBody, onSseQuota);
|
|
110
|
+
return new Response(sdkBody, {
|
|
111
|
+
headers: response.headers,
|
|
112
|
+
status: response.status,
|
|
113
|
+
statusText: response.statusText,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
return response;
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
globalThis.fetch = wrappedFetch;
|
|
121
|
+
|
|
122
|
+
const restoreFetch = () => {
|
|
123
|
+
if (globalThis.fetch === wrappedFetch) globalThis.fetch = originalFetch;
|
|
124
|
+
sseQuotaTask?.catch(() => {});
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
const stream = base(model, context, options);
|
|
128
|
+
const originalOuterEnd = outer.end.bind(outer);
|
|
129
|
+
outer.end = (result?: Parameters<typeof originalOuterEnd>[0]) => {
|
|
130
|
+
restoreFetch();
|
|
131
|
+
originalOuterEnd(result);
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
void forwardStream(stream, outer, () => rateLimitInfo, restoreFetch);
|
|
135
|
+
|
|
136
|
+
return outer;
|
|
137
|
+
};
|
|
138
|
+
}
|