@gtrabanco/pi-nan-provider 0.6.4 → 0.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +31 -1
- package/README.es.md +37 -1
- package/README.md +37 -1
- package/package.json +1 -1
- package/scripts/generate-models.ts +45 -14
- package/scripts/manual-overrides.ts +5 -1
- package/scripts/models.generated.ts +32 -31
- package/src/context-overflow-classifier.ts +181 -0
- package/src/fetch-models.ts +10 -6
- package/src/openai-compat-sanitizer.ts +24 -5
- package/src/provider-factory.ts +29 -2
package/AGENTS.md
CHANGED
|
@@ -111,7 +111,9 @@ Every PR that changes code MUST bump `package.json` version in the same PR; CI p
|
|
|
111
111
|
`MANUAL_OVERRIDES` (mandatory provenance note) in `scripts/manual-overrides.ts`,
|
|
112
112
|
applied by `scripts/generate-models.ts` — never hand-edited into
|
|
113
113
|
`scripts/models.generated.ts` and never invented. e.g. deepseek-v4-flash
|
|
114
|
-
image input (Vision-Exp variant; models.dev lists text
|
|
114
|
+
image input (Vision-Exp variant; models.dev now also lists text+image,
|
|
115
|
+
checked 2026-09-13, so the override is kept as a pin rather than a
|
|
116
|
+
divergence). Re-verify
|
|
115
117
|
overrides when the sources update: the qwen3.8-flash contextWindow 1,000,000
|
|
116
118
|
override (maintainer-confirmed 2026-09-05) was withdrawn 2026-09-07 — the
|
|
117
119
|
updated NaN docs still say 262K "the model's native window" and models.dev
|
|
@@ -121,10 +123,38 @@ Every PR that changes code MUST bump `package.json` version in the same PR; CI p
|
|
|
121
123
|
community `deepseek-v4-flash`, `mimo-v2.5`, `qwen3.8-flash`, `glm5.3-flash`,
|
|
122
124
|
`qwen3.6`, `gemma4` (all text+image vision) + premium-tier `glm5.3`
|
|
123
125
|
(~753B MoE, text-only input, 1M context, 400M tokens/rolling 4h window).
|
|
126
|
+
glm5.3 is now documented by models.dev too (1M context / 131,072 max output,
|
|
127
|
+
checked 2026-09-13) but stays out of the static catalog via
|
|
128
|
+
`LIVE_ONLY_MODEL_IDS` (premium tier) so a non-premium key never sees a
|
|
129
|
+
model it cannot call when the live `/models` fetch is unavailable; premium
|
|
130
|
+
keys still receive it live with conservative placeholder limits.
|
|
124
131
|
`glm5.2` was removed by the provider (2026-09-05); models.dev may still list
|
|
125
132
|
it — the generator excludes it via `PROVIDER_REMOVED_MODEL_IDS`. Non-chat
|
|
126
133
|
endpoints: qwen3-embedding, rerank, kokoro (TTS), whisper (STT),
|
|
127
134
|
flux-2-klein (images) — MCP-bridge territory, not chat catalog models.
|
|
135
|
+
- NaN can close an SSE stream **before** `finish_reason`. The catalog sets
|
|
136
|
+
`supportsFinishReason: true` so pi-ai raises the retryable
|
|
137
|
+
`Stream ended without finish_reason` (pi-ai's `RETRYABLE_PROVIDER_ERROR_PATTERN`
|
|
138
|
+
matches `"ended without"`, so the turn is retried) instead of silently
|
|
139
|
+
synthesizing `stop`/`toolUse`. `supportsUsageInStreaming` stays `false` by
|
|
140
|
+
default (NaN's published schema does not document `stream_options`), but the
|
|
141
|
+
sanitizer gates its `stream_options` removal on the model's effective
|
|
142
|
+
`compat.supportsUsageInStreaming`, so a confirmed per-model `models.json`
|
|
143
|
+
override now yields real usage instead of being silently undone
|
|
144
|
+
(`test/issue-4-token-usage.test.ts`; issues #2, #4). Regression tests:
|
|
145
|
+
`test/issue-2-truncated-stream.test.ts`, `test/issue-4-token-usage.test.ts`;
|
|
146
|
+
issues #2 and #4.
|
|
147
|
+
- A NaN request that still exceeds the destination model's context window (the
|
|
148
|
+
cross-model thinking guard is disabled with `NAN_THINKING_GUARD=0`, the
|
|
149
|
+
inflation is not a `thinking` block, or the window is smaller) gets NaN's
|
|
150
|
+
generic 400 `Invalid request. Check your request parameters.`, which pi-ai's
|
|
151
|
+
`isContextOverflow()` does NOT match — so pi never compacts and the session
|
|
152
|
+
wedges (upstream `earendil-works/pi#9409`). `src/context-overflow-classifier.ts`
|
|
153
|
+
re-checks the request size at the provider boundary and rewrites that error
|
|
154
|
+
into a pi-recognizable overflow message (chars/3.47 estimate; conservative:
|
|
155
|
+
only when estimated over the window). Wired in `src/provider-factory.ts` after
|
|
156
|
+
the sanitizer. Regression tests: `test/issue-3-model-switch-overflow.test.ts`,
|
|
157
|
+
`test/context-overflow-classifier.test.ts`; issue #3.
|
|
128
158
|
- Relative imports inside this package use `.ts` extensions (pi's official
|
|
129
159
|
extension examples do the same; pi transpiles extension sources).
|
|
130
160
|
- pi intentionally has NO built-in MCP client (docs/usage.md). MCP integration
|
package/README.es.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# @gtrabanco/pi-nan-provider
|
|
2
2
|
|
|
3
3
|
[](https://opensource.org/licenses/MIT)
|
|
4
|
-
[](https://github.com/gtrabanco/pi-nan-provider/releases)
|
|
5
5
|
|
|
6
6
|
[NaN Builders](https://nan.builders) model provider + MCP bridges para [pi](https://github.com/earendil-works/pi).
|
|
7
7
|
|
|
@@ -49,6 +49,42 @@ Al cambiar de modelo, pi-ai reenvía el razonamiento del modelo anterior como te
|
|
|
49
49
|
|
|
50
50
|
Este paquete **elimina todos los bloques de razonamiento cross-model reenviados**, de modo que cambiar de un modelo de 1M de contexto a uno de 262K (`qwen3.6`) ya no desborda la ventana. Las respuestas y los tool results de los modelos no se tocan — solo se quitan sus trazas internas de razonamiento, así que `qwen3.6` puede seguir respondiendo sobre lo que hizo otro modelo. El razonamiento del mismo modelo no se toca nunca, y el guard solo actúa sobre peticiones dirigidas a los proveedores de este paquete. Pon `NAN_THINKING_GUARD=0` para desactivarlo.
|
|
51
51
|
|
|
52
|
+
Si una petición sigue desbordando — guard desactivado, inflado que no es un bloque de razonamiento (tool outputs grandes, imágenes) o una ventana de destino más pequeña — NaN responde con el mismo 400 genérico en lugar de nombrar el desbordamiento, y la auto-compactación de pi no lo reconoce, así que la sesión se queda atascada en el techo. Por eso el proveedor **vuelve a comprobar el tamaño de la petición a la salida**: cuando llega ese 400 genérico para una petición estimada por encima de la ventana del modelo, el error se reescribe como un mensaje de context overflow que pi reconoce, de modo que compacta y reintenta en vez de bloquearse. Un 400 genérico en una petición dentro de la ventana no se toca, así que nunca se etiquetan mal errores no relacionados.
|
|
53
|
+
|
|
54
|
+
### ⏱️ Streams truncados intermitentes (auto-retry, sin stall silencioso)
|
|
55
|
+
|
|
56
|
+
El gateway LiteLLM de NaN cierra ocasionalmente un stream SSE **antes** de emitir el chunk final `finish_reason` (observado en `glm5.3-flash`; [issue #2](https://github.com/gtrabanco/pi-nan-provider/issues/2)). El catálogo declara `supportsFinishReason: true`, así que pi-ai lo convierte en el error `Stream ended without finish_reason` — que coincide con el patrón de errores reintentables de pi y se **reintenta automáticamente**, en vez de aceptar en silencio una respuesta a medias. Si una versión del gateway nunca manda `finish_reason`, el turno ahora falla de forma visible al agotar los reintentos.
|
|
57
|
+
|
|
58
|
+
Puedes sobrescribir el `compat` de cualquier modelo en `~/.pi/agent/models.json` (docs de pi → Per-model Overrides); los overrides se componen por encima del proveedor registrado. Ejemplo (forzando el comportamiento de retry explícitamente):
|
|
59
|
+
|
|
60
|
+
```json
|
|
61
|
+
{
|
|
62
|
+
"providers": {
|
|
63
|
+
"nan": {
|
|
64
|
+
"modelOverrides": {
|
|
65
|
+
"glm5.3-flash": { "compat": { "supportsFinishReason": true } }
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
> Poner `supportsFinishReason: false` restaura el antiguo stall silencioso — no recomendado.
|
|
73
|
+
|
|
74
|
+
**Uso de tokens en streaming:** `supportsUsageInStreaming` es `false` por defecto porque el esquema publicado de NaN no documenta `stream_options`; sin él, el usage aparece a cero. Si has confirmado que tu modelo devuelve el chunk de usage en streaming, activalo por modelo — el sanitizer de peticiones entonces reenvía `stream_options: { "include_usage": true }` y pi muestra los tokens reales en lugar de ceros:
|
|
75
|
+
|
|
76
|
+
```json
|
|
77
|
+
{
|
|
78
|
+
"providers": {
|
|
79
|
+
"nan": {
|
|
80
|
+
"modelOverrides": {
|
|
81
|
+
"qwen3.6": { "compat": { "supportsUsageInStreaming": true } }
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
```
|
|
87
|
+
|
|
52
88
|
## 🔑 Autenticación
|
|
53
89
|
|
|
54
90
|
`resolve()` comprueba primero la credencial almacenada y después recurre a la variable de entorno correspondiente.
|
package/README.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# @gtrabanco/pi-nan-provider
|
|
2
2
|
|
|
3
3
|
[](https://opensource.org/licenses/MIT)
|
|
4
|
-
[](https://github.com/gtrabanco/pi-nan-provider/releases)
|
|
5
5
|
|
|
6
6
|
[NaN Builders](https://nan.builders) model provider + MCP bridges for [pi](https://github.com/earendil-works/pi).
|
|
7
7
|
|
|
@@ -49,6 +49,42 @@ When you switch models, pi-ai replays the previous model's reasoning as plain as
|
|
|
49
49
|
|
|
50
50
|
This package **drops every replayed cross-model reasoning block**, so switching from a 1M-context model to a 262K one (`qwen3.6`) no longer overflows the window. The models' answers and tool results are untouched — only their internal reasoning traces are removed, so `qwen3.6` can still answer about what another model did. Same-model reasoning is never altered, and the guard only acts on requests targeting this package's providers. Set `NAN_THINKING_GUARD=0` to disable it.
|
|
51
51
|
|
|
52
|
+
If a request still overflows — the guard is disabled, the inflation is not a reasoning block (large tool outputs, images), or the destination window is simply smaller — NaN answers the same generic 400 instead of naming the overflow, and pi's auto-compaction does not recognize it, so the session wedges at the ceiling. The provider therefore **re-checks the request size on the way out**: when that generic 400 arrives for a request estimated over the model's window, the error is rewritten into a context-overflow message pi recognizes, so it compacts and retries instead of stalling. A generic 400 on a within-window request is left untouched, so unrelated errors are never mislabelled.
|
|
53
|
+
|
|
54
|
+
### ⏱️ Intermittent truncated streams (auto-retry, no silent stall)
|
|
55
|
+
|
|
56
|
+
NaN's LiteLLM gateway occasionally closes an SSE stream **before** emitting the final `finish_reason` chunk (observed on `glm5.3-flash`; [issue #2](https://github.com/gtrabanco/pi-nan-provider/issues/2)). The catalog declares `supportsFinishReason: true`, so pi-ai turns that into the error `Stream ended without finish_reason` — which matches pi's retryable-provider pattern and is **retried automatically**, instead of silently accepting a half-finished answer. If a gateway version never sends `finish_reason`, the turn now fails visibly once the retry budget is exhausted.
|
|
57
|
+
|
|
58
|
+
You can override any model's `compat` per-model in `~/.pi/agent/models.json` (pi's `docs/models.md` → Per-model Overrides); overrides compose above the registered provider. Example (forcing the retry behavior explicitly):
|
|
59
|
+
|
|
60
|
+
```json
|
|
61
|
+
{
|
|
62
|
+
"providers": {
|
|
63
|
+
"nan": {
|
|
64
|
+
"modelOverrides": {
|
|
65
|
+
"glm5.3-flash": { "compat": { "supportsFinishReason": true } }
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
> Setting `supportsFinishReason: false` restores the old silent-stall behavior — not recommended.
|
|
73
|
+
|
|
74
|
+
**Streaming token usage:** `supportsUsageInStreaming` is `false` by default because NaN's published schema does not document `stream_options`; without it usage reads as zero. If you have confirmed that your model returns a streaming usage chunk, opt in per model — the request sanitizer then forwards `stream_options: { "include_usage": true }` and pi reports real token counts instead of zeros:
|
|
75
|
+
|
|
76
|
+
```json
|
|
77
|
+
{
|
|
78
|
+
"providers": {
|
|
79
|
+
"nan": {
|
|
80
|
+
"modelOverrides": {
|
|
81
|
+
"qwen3.6": { "compat": { "supportsUsageInStreaming": true } }
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
```
|
|
87
|
+
|
|
52
88
|
## 🔑 Authentication
|
|
53
89
|
|
|
54
90
|
`resolve()` checks the stored credential first, then falls back to the matching environment variable.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gtrabanco/pi-nan-provider",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.7",
|
|
4
4
|
"description": "NaN Builders (api.nan.builders) model provider for pi - OpenAI-compatible registration with a models.dev-generated fallback, tier-aware live catalog, and MCP bridges (official web search + optional community media server)",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi",
|
|
@@ -57,15 +57,19 @@ const PROVIDER_REMOVED_MODEL_IDS: Record<string, string> = {
|
|
|
57
57
|
};
|
|
58
58
|
|
|
59
59
|
/**
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
60
|
+
* Premium/tier-gated models that models.dev documents but that are deliberately
|
|
61
|
+
* kept OUT of the static fallback catalog. The static baseline is what pi
|
|
62
|
+
* registers with zero network, and it must not advertise a premium model to a
|
|
63
|
+
* key that cannot call it: NaN's live `/models` response is the tier
|
|
64
|
+
* authority, so premium keys still receive these models through the live
|
|
65
|
+
* refresh with conservative placeholder limits (UNKNOWN_MODEL_LIMITS).
|
|
66
|
+
* Excluded at generation time with the recorded reason so a regeneration
|
|
67
|
+
* cannot resurrect them into the baseline.
|
|
65
68
|
*/
|
|
66
|
-
const
|
|
67
|
-
"glm5.3:
|
|
68
|
-
|
|
69
|
+
const LIVE_ONLY_MODEL_IDS: Record<string, string> = {
|
|
70
|
+
"glm5.3":
|
|
71
|
+
"premium-tier model (models.dev now documents it with 1M context / 131,072 max output; NaN docs https://nan.builders/docs/models + https://nan.builders/openapi.json, checked 2026-09-13) kept live-only so a non-premium key never sees a model it cannot call when the live /models fetch is unavailable; premium keys still get it via the /models refresh with conservative placeholder limits",
|
|
72
|
+
};
|
|
69
73
|
|
|
70
74
|
/**
|
|
71
75
|
* LiteLLM compat confirmed against the live api.nan.builders gateway by the
|
|
@@ -79,13 +83,26 @@ const KNOWN_UNEMITTABLE_MODEL_NOTES: readonly string[] = [
|
|
|
79
83
|
const NAN_COMPAT = {
|
|
80
84
|
supportsDeveloperRole: false,
|
|
81
85
|
supportsReasoningEffort: true,
|
|
82
|
-
|
|
83
|
-
|
|
86
|
+
// pi-ai only sends `stream_options: { include_usage: true }` when this is
|
|
87
|
+
// not false. NaN's published schema does not document `stream_options`, so
|
|
88
|
+
// the conservative default is false: the sanitizer strips it and usage
|
|
89
|
+
// stays zero. A user who confirms their gateway reports streaming usage
|
|
90
|
+
// can opt in per model with a models.json compat override
|
|
91
|
+
// (`supportsUsageInStreaming: true`); the sanitizer then forwards
|
|
92
|
+
// `stream_options` instead of deleting it (issue #4, 2026-09-13).
|
|
93
|
+
supportsUsageInStreaming: false,
|
|
94
|
+
// The NaN/LiteLLM gateway intermittently closes SSE streams before emitting
|
|
95
|
+
// `finish_reason`. With true, pi-ai raises "Stream ended without
|
|
96
|
+
// finish_reason", which its retryable-provider pattern ("ended without")
|
|
97
|
+
// matches, so the turn is retried automatically. With false, pi-ai
|
|
98
|
+
// silently synthesizes stop/toolUse and the turn stalls mid-answer
|
|
99
|
+
// (observed 2026-09-13 on glm5.3-flash; issue #2).
|
|
100
|
+
supportsFinishReason: true,
|
|
84
101
|
maxTokensField: "max_tokens" as const,
|
|
85
102
|
};
|
|
86
103
|
|
|
87
104
|
const NAN_COMPAT_NOTE =
|
|
88
|
-
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true,
|
|
105
|
+
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true, maxTokensField max_tokens. NaN's docs example sets only supportsDeveloperRole: true and is not battle-tested. supportsFinishReason true (2026-09-13, issue #2): the LiteLLM gateway intermittently closes SSE streams before emitting finish_reason; with true pi-ai raises 'Stream ended without finish_reason', which matches pi-ai's retryable-provider pattern ('ended without') and is retried automatically, whereas false silently synthesized stop/toolUse and stalled the turn mid-answer. supportsUsageInStreaming false (2026-09-13): pi-ai only sends stream_options when this is not false, and NaN's published schema does not document it, so the conservative default stays false (sanitizer removes stream_options, usage reads zero). A user who has confirmed that their model returns a streaming usage chunk can opt in per model with a models.json compat override (supportsUsageInStreaming: true); the sanitizer then forwards stream_options instead of deleting it (issue #4).";
|
|
89
106
|
|
|
90
107
|
interface ModelsDevModel {
|
|
91
108
|
id?: string;
|
|
@@ -144,6 +161,10 @@ function convertModel(modelId: string, m: ModelsDevModel): GeneratedModel | { sk
|
|
|
144
161
|
if (removedReason) {
|
|
145
162
|
return { skip: `provider-removed: "${modelId}" excluded from the catalog (${removedReason})` };
|
|
146
163
|
}
|
|
164
|
+
const liveOnlyReason = LIVE_ONLY_MODEL_IDS[modelId];
|
|
165
|
+
if (liveOnlyReason) {
|
|
166
|
+
return { skip: `live-only: "${modelId}" kept out of the static catalog (${liveOnlyReason})` };
|
|
167
|
+
}
|
|
147
168
|
const contextWindow = m.limit?.context;
|
|
148
169
|
const maxTokens = m.limit?.output;
|
|
149
170
|
if (typeof contextWindow !== "number" || contextWindow <= 0) {
|
|
@@ -225,9 +246,19 @@ async function main(): Promise<void> {
|
|
|
225
246
|
|
|
226
247
|
const fetchedAt = new Date().toISOString();
|
|
227
248
|
const allNotes = [
|
|
228
|
-
...
|
|
229
|
-
|
|
230
|
-
|
|
249
|
+
...new Set([
|
|
250
|
+
...skipped,
|
|
251
|
+
// Record every exclusion unconditionally (not only when models.dev
|
|
252
|
+
// still lists the id) so a regeneration can never drop the reason a
|
|
253
|
+
// model is absent from the catalog.
|
|
254
|
+
...Object.entries(PROVIDER_REMOVED_MODEL_IDS).map(
|
|
255
|
+
([id, reason]) => `provider-removed: "${id}" excluded from the catalog (${reason})`,
|
|
256
|
+
),
|
|
257
|
+
...Object.entries(LIVE_ONLY_MODEL_IDS).map(
|
|
258
|
+
([id, reason]) => `live-only: "${id}" kept out of the static catalog (${reason})`,
|
|
259
|
+
),
|
|
260
|
+
...entries.flatMap((entry) => entry.notes ?? []),
|
|
261
|
+
]),
|
|
231
262
|
];
|
|
232
263
|
|
|
233
264
|
const generated = `// This file is auto-generated by scripts/generate-models.ts
|
|
@@ -16,6 +16,10 @@
|
|
|
16
16
|
*
|
|
17
17
|
* Fields override the models.dev-derived entry one-for-one:
|
|
18
18
|
* name, reasoning, input, cost, contextWindow, maxTokens.
|
|
19
|
+
*
|
|
20
|
+
* An override may also act as a pin — a value models.dev already agrees with,
|
|
21
|
+
* kept so an upstream regression cannot silently drop a confirmed capability.
|
|
22
|
+
* A pin's note must say it is a pin; never present it as a divergence.
|
|
19
23
|
*/
|
|
20
24
|
|
|
21
25
|
export interface ManualModelOverride {
|
|
@@ -38,6 +42,6 @@ export interface ManualModelOverride {
|
|
|
38
42
|
export const MANUAL_OVERRIDES: Record<string, ManualModelOverride> = {
|
|
39
43
|
"deepseek-v4-flash": {
|
|
40
44
|
input: ["text", "image"],
|
|
41
|
-
note: "input includes image: NaN serves the Vision-Exp variant ('takes images as input', https://nan.builders/docs/models
|
|
45
|
+
note: "input includes image: NaN serves the Vision-Exp variant ('takes images as input', https://nan.builders/docs/models; the image_url content-parts in https://nan.builders/openapi.json list deepseek-v4-flash among the vision models). models.dev provider nan also lists text+image now (DeepSeek V4.1 Flash entry, checked 2026-09-13; its 2026-09-07 snapshot listed text only), so this override is kept as a pin for the vision capability rather than as a divergence.",
|
|
42
46
|
},
|
|
43
47
|
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// This file is auto-generated by scripts/generate-models.ts
|
|
2
2
|
// Do not edit manually — run `bun run generate-models` to update.
|
|
3
3
|
//
|
|
4
|
-
// Source: https://models.dev/api.json (provider "nan"), fetched 2026-09-
|
|
4
|
+
// Source: https://models.dev/api.json (provider "nan"), fetched 2026-09-13T18:08:44.200Z
|
|
5
5
|
// Provenance: every contextWindow/maxTokens/input/cost value traces to
|
|
6
6
|
// models.dev or to the per-entry notes below. Nothing is invented; entries
|
|
7
7
|
// models.dev documents incompletely are omitted and flagged instead.
|
|
@@ -14,7 +14,7 @@ import type { GeneratedModelEntry } from "../src/fetch-models.ts";
|
|
|
14
14
|
export const NAN_GENERATED_MODELS: readonly GeneratedModelEntry[] = [
|
|
15
15
|
{
|
|
16
16
|
"id": "deepseek-v4-flash",
|
|
17
|
-
"name": "DeepSeek V4 Flash",
|
|
17
|
+
"name": "DeepSeek V4.1 Flash",
|
|
18
18
|
"reasoning": true,
|
|
19
19
|
"input": [
|
|
20
20
|
"text",
|
|
@@ -31,31 +31,32 @@ export const NAN_GENERATED_MODELS: readonly GeneratedModelEntry[] = [
|
|
|
31
31
|
"compat": {
|
|
32
32
|
"supportsDeveloperRole": false,
|
|
33
33
|
"supportsReasoningEffort": true,
|
|
34
|
-
"supportsUsageInStreaming":
|
|
35
|
-
"supportsFinishReason":
|
|
34
|
+
"supportsUsageInStreaming": false,
|
|
35
|
+
"supportsFinishReason": true,
|
|
36
36
|
"maxTokensField": "max_tokens"
|
|
37
37
|
},
|
|
38
38
|
"notes": [
|
|
39
|
-
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true,
|
|
40
|
-
"input includes image: NaN serves the Vision-Exp variant ('takes images as input', https://nan.builders/docs/models
|
|
39
|
+
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true, maxTokensField max_tokens. NaN's docs example sets only supportsDeveloperRole: true and is not battle-tested. supportsFinishReason true (2026-09-13, issue #2): the LiteLLM gateway intermittently closes SSE streams before emitting finish_reason; with true pi-ai raises 'Stream ended without finish_reason', which matches pi-ai's retryable-provider pattern ('ended without') and is retried automatically, whereas false silently synthesized stop/toolUse and stalled the turn mid-answer. supportsUsageInStreaming false (2026-09-13): pi-ai only sends stream_options when this is not false, and NaN's published schema does not document it, so the conservative default stays false (sanitizer removes stream_options, usage reads zero). A user who has confirmed that their model returns a streaming usage chunk can opt in per model with a models.json compat override (supportsUsageInStreaming: true); the sanitizer then forwards stream_options instead of deleting it (issue #4).",
|
|
40
|
+
"input includes image: NaN serves the Vision-Exp variant ('takes images as input', https://nan.builders/docs/models; the image_url content-parts in https://nan.builders/openapi.json list deepseek-v4-flash among the vision models). models.dev provider nan also lists text+image now (DeepSeek V4.1 Flash entry, checked 2026-09-13; its 2026-09-07 snapshot listed text only), so this override is kept as a pin for the vision capability rather than as a divergence."
|
|
41
41
|
],
|
|
42
42
|
"extras": {
|
|
43
43
|
"id": "deepseek-v4-flash",
|
|
44
|
-
"name": "DeepSeek V4 Flash",
|
|
45
|
-
"description": "
|
|
44
|
+
"name": "DeepSeek V4.1 Flash",
|
|
45
|
+
"description": "DeepSeek V4.1 Flash model for reasoning and agentic coding",
|
|
46
46
|
"family": "deepseek-flash",
|
|
47
|
-
"attachment":
|
|
47
|
+
"attachment": true,
|
|
48
48
|
"reasoning": true,
|
|
49
49
|
"reasoning_options": [],
|
|
50
50
|
"tool_call": true,
|
|
51
51
|
"structured_output": true,
|
|
52
52
|
"temperature": true,
|
|
53
53
|
"knowledge": "2025-05",
|
|
54
|
-
"release_date": "2026-
|
|
55
|
-
"last_updated": "2026-
|
|
54
|
+
"release_date": "2026-09-10",
|
|
55
|
+
"last_updated": "2026-09-10",
|
|
56
56
|
"modalities": {
|
|
57
57
|
"input": [
|
|
58
|
-
"text"
|
|
58
|
+
"text",
|
|
59
|
+
"image"
|
|
59
60
|
],
|
|
60
61
|
"output": [
|
|
61
62
|
"text"
|
|
@@ -91,12 +92,12 @@ export const NAN_GENERATED_MODELS: readonly GeneratedModelEntry[] = [
|
|
|
91
92
|
"compat": {
|
|
92
93
|
"supportsDeveloperRole": false,
|
|
93
94
|
"supportsReasoningEffort": true,
|
|
94
|
-
"supportsUsageInStreaming":
|
|
95
|
-
"supportsFinishReason":
|
|
95
|
+
"supportsUsageInStreaming": false,
|
|
96
|
+
"supportsFinishReason": true,
|
|
96
97
|
"maxTokensField": "max_tokens"
|
|
97
98
|
},
|
|
98
99
|
"notes": [
|
|
99
|
-
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true,
|
|
100
|
+
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true, maxTokensField max_tokens. NaN's docs example sets only supportsDeveloperRole: true and is not battle-tested. supportsFinishReason true (2026-09-13, issue #2): the LiteLLM gateway intermittently closes SSE streams before emitting finish_reason; with true pi-ai raises 'Stream ended without finish_reason', which matches pi-ai's retryable-provider pattern ('ended without') and is retried automatically, whereas false silently synthesized stop/toolUse and stalled the turn mid-answer. supportsUsageInStreaming false (2026-09-13): pi-ai only sends stream_options when this is not false, and NaN's published schema does not document it, so the conservative default stays false (sanitizer removes stream_options, usage reads zero). A user who has confirmed that their model returns a streaming usage chunk can opt in per model with a models.json compat override (supportsUsageInStreaming: true); the sanitizer then forwards stream_options instead of deleting it (issue #4)."
|
|
100
101
|
],
|
|
101
102
|
"extras": {
|
|
102
103
|
"id": "gemma4",
|
|
@@ -154,12 +155,12 @@ export const NAN_GENERATED_MODELS: readonly GeneratedModelEntry[] = [
|
|
|
154
155
|
"compat": {
|
|
155
156
|
"supportsDeveloperRole": false,
|
|
156
157
|
"supportsReasoningEffort": true,
|
|
157
|
-
"supportsUsageInStreaming":
|
|
158
|
-
"supportsFinishReason":
|
|
158
|
+
"supportsUsageInStreaming": false,
|
|
159
|
+
"supportsFinishReason": true,
|
|
159
160
|
"maxTokensField": "max_tokens"
|
|
160
161
|
},
|
|
161
162
|
"notes": [
|
|
162
|
-
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true,
|
|
163
|
+
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true, maxTokensField max_tokens. NaN's docs example sets only supportsDeveloperRole: true and is not battle-tested. supportsFinishReason true (2026-09-13, issue #2): the LiteLLM gateway intermittently closes SSE streams before emitting finish_reason; with true pi-ai raises 'Stream ended without finish_reason', which matches pi-ai's retryable-provider pattern ('ended without') and is retried automatically, whereas false silently synthesized stop/toolUse and stalled the turn mid-answer. supportsUsageInStreaming false (2026-09-13): pi-ai only sends stream_options when this is not false, and NaN's published schema does not document it, so the conservative default stays false (sanitizer removes stream_options, usage reads zero). A user who has confirmed that their model returns a streaming usage chunk can opt in per model with a models.json compat override (supportsUsageInStreaming: true); the sanitizer then forwards stream_options instead of deleting it (issue #4)."
|
|
163
164
|
],
|
|
164
165
|
"extras": {
|
|
165
166
|
"id": "glm5.3-flash",
|
|
@@ -213,12 +214,12 @@ export const NAN_GENERATED_MODELS: readonly GeneratedModelEntry[] = [
|
|
|
213
214
|
"compat": {
|
|
214
215
|
"supportsDeveloperRole": false,
|
|
215
216
|
"supportsReasoningEffort": true,
|
|
216
|
-
"supportsUsageInStreaming":
|
|
217
|
-
"supportsFinishReason":
|
|
217
|
+
"supportsUsageInStreaming": false,
|
|
218
|
+
"supportsFinishReason": true,
|
|
218
219
|
"maxTokensField": "max_tokens"
|
|
219
220
|
},
|
|
220
221
|
"notes": [
|
|
221
|
-
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true,
|
|
222
|
+
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true, maxTokensField max_tokens. NaN's docs example sets only supportsDeveloperRole: true and is not battle-tested. supportsFinishReason true (2026-09-13, issue #2): the LiteLLM gateway intermittently closes SSE streams before emitting finish_reason; with true pi-ai raises 'Stream ended without finish_reason', which matches pi-ai's retryable-provider pattern ('ended without') and is retried automatically, whereas false silently synthesized stop/toolUse and stalled the turn mid-answer. supportsUsageInStreaming false (2026-09-13): pi-ai only sends stream_options when this is not false, and NaN's published schema does not document it, so the conservative default stays false (sanitizer removes stream_options, usage reads zero). A user who has confirmed that their model returns a streaming usage chunk can opt in per model with a models.json compat override (supportsUsageInStreaming: true); the sanitizer then forwards stream_options instead of deleting it (issue #4)."
|
|
222
223
|
],
|
|
223
224
|
"extras": {
|
|
224
225
|
"id": "mimo-v2.5",
|
|
@@ -273,12 +274,12 @@ export const NAN_GENERATED_MODELS: readonly GeneratedModelEntry[] = [
|
|
|
273
274
|
"compat": {
|
|
274
275
|
"supportsDeveloperRole": false,
|
|
275
276
|
"supportsReasoningEffort": true,
|
|
276
|
-
"supportsUsageInStreaming":
|
|
277
|
-
"supportsFinishReason":
|
|
277
|
+
"supportsUsageInStreaming": false,
|
|
278
|
+
"supportsFinishReason": true,
|
|
278
279
|
"maxTokensField": "max_tokens"
|
|
279
280
|
},
|
|
280
281
|
"notes": [
|
|
281
|
-
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true,
|
|
282
|
+
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true, maxTokensField max_tokens. NaN's docs example sets only supportsDeveloperRole: true and is not battle-tested. supportsFinishReason true (2026-09-13, issue #2): the LiteLLM gateway intermittently closes SSE streams before emitting finish_reason; with true pi-ai raises 'Stream ended without finish_reason', which matches pi-ai's retryable-provider pattern ('ended without') and is retried automatically, whereas false silently synthesized stop/toolUse and stalled the turn mid-answer. supportsUsageInStreaming false (2026-09-13): pi-ai only sends stream_options when this is not false, and NaN's published schema does not document it, so the conservative default stays false (sanitizer removes stream_options, usage reads zero). A user who has confirmed that their model returns a streaming usage chunk can opt in per model with a models.json compat override (supportsUsageInStreaming: true); the sanitizer then forwards stream_options instead of deleting it (issue #4)."
|
|
282
283
|
],
|
|
283
284
|
"extras": {
|
|
284
285
|
"id": "qwen3.6",
|
|
@@ -336,12 +337,12 @@ export const NAN_GENERATED_MODELS: readonly GeneratedModelEntry[] = [
|
|
|
336
337
|
"compat": {
|
|
337
338
|
"supportsDeveloperRole": false,
|
|
338
339
|
"supportsReasoningEffort": true,
|
|
339
|
-
"supportsUsageInStreaming":
|
|
340
|
-
"supportsFinishReason":
|
|
340
|
+
"supportsUsageInStreaming": false,
|
|
341
|
+
"supportsFinishReason": true,
|
|
341
342
|
"maxTokensField": "max_tokens"
|
|
342
343
|
},
|
|
343
344
|
"notes": [
|
|
344
|
-
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true,
|
|
345
|
+
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true, maxTokensField max_tokens. NaN's docs example sets only supportsDeveloperRole: true and is not battle-tested. supportsFinishReason true (2026-09-13, issue #2): the LiteLLM gateway intermittently closes SSE streams before emitting finish_reason; with true pi-ai raises 'Stream ended without finish_reason', which matches pi-ai's retryable-provider pattern ('ended without') and is retried automatically, whereas false silently synthesized stop/toolUse and stalled the turn mid-answer. supportsUsageInStreaming false (2026-09-13): pi-ai only sends stream_options when this is not false, and NaN's published schema does not document it, so the conservative default stays false (sanitizer removes stream_options, usage reads zero). A user who has confirmed that their model returns a streaming usage chunk can opt in per model with a models.json compat override (supportsUsageInStreaming: true); the sanitizer then forwards stream_options instead of deleting it (issue #4).",
|
|
345
346
|
"contextWindow 262,144: the earlier 1,000,000 override (maintainer-confirmed 2026-09-05) was withdrawn 2026-09-07 — the updated https://nan.builders/docs/models still states '262K token context, the model's native window' and models.dev agrees at 262,144; NaN docs are treated as the most reliable source (maintainer instruction, 2026-09-07)."
|
|
346
347
|
],
|
|
347
348
|
"extras": {
|
|
@@ -381,14 +382,14 @@ export const NAN_GENERATED_MODELS: readonly GeneratedModelEntry[] = [
|
|
|
381
382
|
export const GENERATED_CATALOG_META = {
|
|
382
383
|
source: "https://models.dev/api.json",
|
|
383
384
|
modelsDevProvider: "nan",
|
|
384
|
-
fetchedAt: "2026-09-
|
|
385
|
+
fetchedAt: "2026-09-13T18:08:44.200Z",
|
|
385
386
|
modelCount: 6,
|
|
386
387
|
models: ["deepseek-v4-flash","gemma4","glm5.3-flash","mimo-v2.5","qwen3.6","qwen3.8-flash"],
|
|
387
388
|
notes: [
|
|
389
|
+
"live-only: \"glm5.3\" kept out of the static catalog (premium-tier model (models.dev now documents it with 1M context / 131,072 max output; NaN docs https://nan.builders/docs/models + https://nan.builders/openapi.json, checked 2026-09-13) kept live-only so a non-premium key never sees a model it cannot call when the live /models fetch is unavailable; premium keys still get it via the /models refresh with conservative placeholder limits)",
|
|
388
390
|
"provider-removed: \"glm5.2\" excluded from the catalog (removed by NaN (2026-09-05); absent from the official chat model list in https://nan.builders/openapi.json and https://nan.builders/docs/models (checked 2026-09-07) while models.dev provider nan still listed it — excluded so regeneration does not resurrect it)",
|
|
389
|
-
"
|
|
390
|
-
"
|
|
391
|
-
"input includes image: NaN serves the Vision-Exp variant ('takes images as input', https://nan.builders/docs/models, checked 2026-09-07; the image_url content-parts in https://nan.builders/openapi.json list deepseek-v4-flash among the vision models); models.dev provider nan lists text only.",
|
|
391
|
+
"compat matches the maintainer's working ~/.pi/agent/models.json LiteLLM config for api.nan.builders (2026-09-04): supportsDeveloperRole false, supportsReasoningEffort true, maxTokensField max_tokens. NaN's docs example sets only supportsDeveloperRole: true and is not battle-tested. supportsFinishReason true (2026-09-13, issue #2): the LiteLLM gateway intermittently closes SSE streams before emitting finish_reason; with true pi-ai raises 'Stream ended without finish_reason', which matches pi-ai's retryable-provider pattern ('ended without') and is retried automatically, whereas false silently synthesized stop/toolUse and stalled the turn mid-answer. supportsUsageInStreaming false (2026-09-13): pi-ai only sends stream_options when this is not false, and NaN's published schema does not document it, so the conservative default stays false (sanitizer removes stream_options, usage reads zero). A user who has confirmed that their model returns a streaming usage chunk can opt in per model with a models.json compat override (supportsUsageInStreaming: true); the sanitizer then forwards stream_options instead of deleting it (issue #4).",
|
|
392
|
+
"input includes image: NaN serves the Vision-Exp variant ('takes images as input', https://nan.builders/docs/models; the image_url content-parts in https://nan.builders/openapi.json list deepseek-v4-flash among the vision models). models.dev provider nan also lists text+image now (DeepSeek V4.1 Flash entry, checked 2026-09-13; its 2026-09-07 snapshot listed text only), so this override is kept as a pin for the vision capability rather than as a divergence.",
|
|
392
393
|
"contextWindow 262,144: the earlier 1,000,000 override (maintainer-confirmed 2026-09-05) was withdrawn 2026-09-07 — the updated https://nan.builders/docs/models still states '262K token context, the model's native window' and models.dev agrees at 262,144; NaN docs are treated as the most reliable source (maintainer instruction, 2026-09-07)."
|
|
393
394
|
],
|
|
394
395
|
} as const;
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NaN-aware context-overflow classification.
|
|
3
|
+
*
|
|
4
|
+
* Root cause (pi-ai, still present on 0.85.1 / main): when history is replayed
|
|
5
|
+
* into a DIFFERENT model, `transformMessages` downgrades every non-redacted
|
|
6
|
+
* `thinking` block to plain `text` verbatim, with no size bound — and nothing
|
|
7
|
+
* bounds the SUM across messages. Switching from a 1M-context model
|
|
8
|
+
* (`glm5.3-flash`, `deepseek-v4-flash`) to a 262K one (`qwen3.6`) can therefore
|
|
9
|
+
* push the request past the destination window (see
|
|
10
|
+
* `src/cross-model-thinking-guard.ts` for the primary mitigation, which drops
|
|
11
|
+
* that replayed reasoning before pi-ai converts it).
|
|
12
|
+
*
|
|
13
|
+
* When the request still overflows — the guard is disabled
|
|
14
|
+
* (`NAN_THINKING_GUARD=0`), the trace is not a `thinking` block (large tool
|
|
15
|
+
* outputs, images), or the destination window is simply smaller — NaN's LiteLLM
|
|
16
|
+
* gateway answers the generic
|
|
17
|
+
* HTTP 400 `Invalid request. Check your request parameters.`
|
|
18
|
+
* instead of naming the context overflow. pi's auto-compaction keys on pi-ai's
|
|
19
|
+
* `isContextOverflow()`, whose documented patterns do NOT match that text, so
|
|
20
|
+
* the session wedges permanently at the ceiling (the issue's upstream
|
|
21
|
+
* earendil-works/pi#9409, "Sessions wedge permanently at the context ceiling on
|
|
22
|
+
* reasoning models").
|
|
23
|
+
*
|
|
24
|
+
* This module makes the package **re-check the request size on the way out**
|
|
25
|
+
* (one of the two upstream fixes the tracking issue names). After the request is
|
|
26
|
+
* sent and the gateway answers, if
|
|
27
|
+
* - the terminal assistant message is an error whose text carries NaN's
|
|
28
|
+
* generic 400 marker, AND
|
|
29
|
+
* - the request we sent was estimated to exceed the model's context window,
|
|
30
|
+
* then the error message is rewritten into a form that matches pi-ai's
|
|
31
|
+
* documented overflow patterns, so pi compacts and retries instead of wedging.
|
|
32
|
+
* The original provider text is preserved in the rewritten message. The
|
|
33
|
+
* reclassification is deliberately conservative: a generic 400 on a
|
|
34
|
+
* within-window request is left untouched, so unrelated 400s are never
|
|
35
|
+
* mislabelled as overflow.
|
|
36
|
+
*
|
|
37
|
+
* Estimation ratio: the same chars/3.47 the issue's offline session measurements
|
|
38
|
+
* use (`chars/3.47 + ~31K tools + ~12K system`). The system prompt and tool
|
|
39
|
+
* schemas are counted explicitly, so the constant is only applied to the
|
|
40
|
+
* message/character budget. This is a heuristic for deciding whether a generic
|
|
41
|
+
* 400 is plausibly an overflow — it never fabricates model metadata.
|
|
42
|
+
*/
|
|
43
|
+
|
|
44
|
+
import type {
|
|
45
|
+
AssistantMessage,
|
|
46
|
+
AssistantMessageEventStream,
|
|
47
|
+
Context,
|
|
48
|
+
Model,
|
|
49
|
+
ProviderStreams,
|
|
50
|
+
} from "@earendil-works/pi-ai";
|
|
51
|
+
|
|
52
|
+
/** The exact generic body NaN's gateway returns for an over-window request. */
|
|
53
|
+
export const NAN_GENERIC_BAD_REQUEST = "Invalid request. Check your request parameters.";
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Characters per token used for the request-size estimate. Same ratio as the
|
|
57
|
+
* issue's offline measurements (chars/3.47); a conservative, provider-agnostic
|
|
58
|
+
* heuristic, never a capability claim.
|
|
59
|
+
*/
|
|
60
|
+
export const ESTIMATED_CHARS_PER_TOKEN = 3.47;
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Estimate the input tokens of an outgoing request from its character size.
|
|
64
|
+
* Returns 0 when the context cannot be serialized (nothing is reclassified).
|
|
65
|
+
*/
|
|
66
|
+
export function estimateRequestTokens(
|
|
67
|
+
context: Pick<Context, "systemPrompt" | "messages" | "tools"> | undefined,
|
|
68
|
+
): number {
|
|
69
|
+
if (!context) return 0;
|
|
70
|
+
const systemChars = typeof context.systemPrompt === "string" ? context.systemPrompt.length : 0;
|
|
71
|
+
const toolsChars = safeJsonLength(context.tools);
|
|
72
|
+
const messagesChars = safeJsonLength(context.messages);
|
|
73
|
+
const total = systemChars + toolsChars + messagesChars;
|
|
74
|
+
if (!Number.isFinite(total) || total <= 0) return 0;
|
|
75
|
+
return Math.ceil(total / ESTIMATED_CHARS_PER_TOKEN);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function safeJsonLength(value: unknown): number {
|
|
79
|
+
if (value === undefined || value === null) return 0;
|
|
80
|
+
if (Array.isArray(value) && value.length === 0) return 0;
|
|
81
|
+
try {
|
|
82
|
+
return JSON.stringify(value)?.length ?? 0;
|
|
83
|
+
} catch {
|
|
84
|
+
return 0;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* True when a terminal error message carries NaN's generic-400 marker (with or
|
|
90
|
+
* without the `400: {...}` wrapper pi-ai adds).
|
|
91
|
+
*/
|
|
92
|
+
export function isGenericNanBadRequest(message: Pick<AssistantMessage, "errorMessage"> | undefined): boolean {
|
|
93
|
+
return typeof message?.errorMessage === "string" && message.errorMessage.includes(NAN_GENERIC_BAD_REQUEST);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Reclassify a terminal assistant error as a context overflow when — and only
|
|
98
|
+
* when — NaN returned its generic 400 for a request estimated to exceed the
|
|
99
|
+
* model's context window. Mutates and returns the message (the same object
|
|
100
|
+
* flows through the error event and `result()`).
|
|
101
|
+
*/
|
|
102
|
+
export function classifyContextOverflowError<T extends AssistantMessage>(
|
|
103
|
+
message: T,
|
|
104
|
+
model: Pick<Model<"openai-completions">, "contextWindow"> | { contextWindow?: number } | undefined,
|
|
105
|
+
estimatedInputTokens: number,
|
|
106
|
+
): T {
|
|
107
|
+
if (message.stopReason !== "error") return message;
|
|
108
|
+
if (!isGenericNanBadRequest(message)) return message;
|
|
109
|
+
const contextWindow = model?.contextWindow ?? 0;
|
|
110
|
+
if (!Number.isFinite(contextWindow) || contextWindow <= 0) return message;
|
|
111
|
+
if (!Number.isFinite(estimatedInputTokens) || estimatedInputTokens <= contextWindow) return message;
|
|
112
|
+
|
|
113
|
+
message.errorMessage =
|
|
114
|
+
`Requested token count exceeds the model's maximum context length of ${contextWindow} tokens ` +
|
|
115
|
+
`(estimated ${Math.ceil(estimatedInputTokens)} input tokens). ` +
|
|
116
|
+
`NaN's gateway answered HTTP 400 "${NAN_GENERIC_BAD_REQUEST}" instead of naming the overflow, ` +
|
|
117
|
+
`so the context must be compacted before retrying.`;
|
|
118
|
+
return message;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Wrap a provider stream so its terminal error is classified against the size
|
|
123
|
+
* of the request that produced it. Both the `error` event and `result()` yield
|
|
124
|
+
* the same (mutated) message, so iterating consumers and `result()`-only
|
|
125
|
+
* consumers agree.
|
|
126
|
+
*/
|
|
127
|
+
export function classifyStreamContextOverflow(
|
|
128
|
+
stream: AssistantMessageEventStream,
|
|
129
|
+
model: { contextWindow?: number } | undefined,
|
|
130
|
+
estimatedInputTokens: number,
|
|
131
|
+
): AssistantMessageEventStream {
|
|
132
|
+
const rewrite = (message: AssistantMessage | undefined): AssistantMessage | undefined => {
|
|
133
|
+
if (message) classifyContextOverflowError(message, model, estimatedInputTokens);
|
|
134
|
+
return message;
|
|
135
|
+
};
|
|
136
|
+
const resultPromise = stream.result().then(rewrite);
|
|
137
|
+
|
|
138
|
+
return new Proxy(stream, {
|
|
139
|
+
get(target, property, receiver) {
|
|
140
|
+
if (property === "result") return () => resultPromise;
|
|
141
|
+
if (property === Symbol.asyncIterator) {
|
|
142
|
+
return () => {
|
|
143
|
+
const iterator = target[Symbol.asyncIterator]();
|
|
144
|
+
return {
|
|
145
|
+
async next() {
|
|
146
|
+
const step = await iterator.next();
|
|
147
|
+
if (!step.done && step.value?.type === "error") rewrite(step.value.error);
|
|
148
|
+
return step;
|
|
149
|
+
},
|
|
150
|
+
async return(value?: unknown) {
|
|
151
|
+
return iterator.return ? iterator.return(value) : { done: true, value };
|
|
152
|
+
},
|
|
153
|
+
async throw(error?: unknown) {
|
|
154
|
+
if (iterator.throw) return iterator.throw(error);
|
|
155
|
+
throw error;
|
|
156
|
+
},
|
|
157
|
+
[Symbol.asyncIterator]() {
|
|
158
|
+
return this;
|
|
159
|
+
},
|
|
160
|
+
};
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
const value = Reflect.get(target, property, receiver);
|
|
164
|
+
return typeof value === "function" ? value.bind(target) : value;
|
|
165
|
+
},
|
|
166
|
+
}) as AssistantMessageEventStream;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Wrap every `stream`/`streamSimple` call so a NaN over-window request whose
|
|
171
|
+
* gateway answer is the opaque generic 400 is surfaced as a context overflow.
|
|
172
|
+
*/
|
|
173
|
+
export function withContextOverflowClassification(api: ProviderStreams): ProviderStreams {
|
|
174
|
+
return {
|
|
175
|
+
...api,
|
|
176
|
+
stream: (model, context, options) =>
|
|
177
|
+
classifyStreamContextOverflow(api.stream(model, context, options), model, estimateRequestTokens(context)),
|
|
178
|
+
streamSimple: (model, context, options) =>
|
|
179
|
+
classifyStreamContextOverflow(api.streamSimple(model, context, options), model, estimateRequestTokens(context)),
|
|
180
|
+
};
|
|
181
|
+
}
|
package/src/fetch-models.ts
CHANGED
|
@@ -201,11 +201,15 @@ export function mergeLiveWithGenerated(
|
|
|
201
201
|
} else {
|
|
202
202
|
// Conservative placeholder for allowlisted uncatalogued live ids
|
|
203
203
|
// (e.g. premium glm5.3): limits are the documented safe envelope and
|
|
204
|
-
// capabilities stay "unknown".
|
|
205
|
-
//
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
// ended without finish_reason"
|
|
204
|
+
// capabilities stay "unknown". The compat keys are the same
|
|
205
|
+
// gateway-behavior flags applied to every catalog model in
|
|
206
|
+
// NAN_COMPAT: the NaN/LiteLLM gateway cuts SSE streams before
|
|
207
|
+
// finish_reason, so supportsFinishReason must stay true (pi-ai then
|
|
208
|
+
// raises the retryable "Stream ended without finish_reason" instead
|
|
209
|
+
// of silently stalling), and supportsUsageInStreaming stays false
|
|
210
|
+
// because NaN's schema does not document `stream_options`: the
|
|
211
|
+
// sanitizer strips it unless the model's effective compat opts in
|
|
212
|
+
// (issue #4), so false matches the default wire payload.
|
|
209
213
|
models.push({
|
|
210
214
|
id,
|
|
211
215
|
name: id,
|
|
@@ -217,7 +221,7 @@ export function mergeLiveWithGenerated(
|
|
|
217
221
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
218
222
|
contextWindow: UNKNOWN_MODEL_LIMITS.contextWindow,
|
|
219
223
|
maxTokens: UNKNOWN_MODEL_LIMITS.maxTokens,
|
|
220
|
-
compat: { supportsFinishReason: false },
|
|
224
|
+
compat: { supportsFinishReason: true, supportsUsageInStreaming: false },
|
|
221
225
|
});
|
|
222
226
|
unknown.push(id);
|
|
223
227
|
}
|
|
@@ -34,9 +34,13 @@
|
|
|
34
34
|
* 4. Top-level fields NaN's schema does not list: `store` and
|
|
35
35
|
* `stream_options`. These are opt-in/usage fields pi-ai sends by default
|
|
36
36
|
* for a "standard" provider; NaN does not document them, so they are
|
|
37
|
-
* removed.
|
|
38
|
-
*
|
|
39
|
-
*
|
|
37
|
+
* removed by default. `stream_options` is the one exception that can be
|
|
38
|
+
* explicitly opted into: when the model's effective
|
|
39
|
+
* `compat.supportsUsageInStreaming` is true (a catalog flag or a user
|
|
40
|
+
* `models.json` override), the caller has confirmed the gateway reports
|
|
41
|
+
* usage and `stream_options` is preserved — deleting it unconditionally
|
|
42
|
+
* would silently zero out `message.usage` (issue #4). `store` is always
|
|
43
|
+
* removed.
|
|
40
44
|
* 5. An EMPTY `tools` array. Verified against the live gateway (2026-09-09):
|
|
41
45
|
* NaN rejects `tools: []` with the same 400, while `stream: true`, a
|
|
42
46
|
* `system` message, string content, and a `tool` role message are all
|
|
@@ -183,12 +187,23 @@ function sanitizeMessage(message: unknown): unknown {
|
|
|
183
187
|
return out;
|
|
184
188
|
}
|
|
185
189
|
|
|
190
|
+
export interface SanitizerOptions {
|
|
191
|
+
/**
|
|
192
|
+
* Keep `stream_options` in the payload. Set only when the model's effective
|
|
193
|
+
* `compat.supportsUsageInStreaming` is true: pi-ai then asks the gateway for
|
|
194
|
+
* `stream_options: { include_usage: true }` and expects the terminal usage
|
|
195
|
+
* chunk. When false/absent, `stream_options` is stripped from the strict
|
|
196
|
+
* NaN payload as before.
|
|
197
|
+
*/
|
|
198
|
+
preserveStreamOptions?: boolean;
|
|
199
|
+
}
|
|
200
|
+
|
|
186
201
|
/**
|
|
187
202
|
* Rewrite an OpenAI-compatible `/chat/completions` payload so every field
|
|
188
203
|
* conforms to NaN's published schema. Returns the updated payload; if the
|
|
189
204
|
* payload has no `messages` array it is returned unchanged.
|
|
190
205
|
*/
|
|
191
|
-
export function sanitizeOpenAICompatPayload(payload: unknown): unknown {
|
|
206
|
+
export function sanitizeOpenAICompatPayload(payload: unknown, options: SanitizerOptions = {}): unknown {
|
|
192
207
|
if (!isObject(payload) || !Array.isArray(payload.messages)) return payload;
|
|
193
208
|
|
|
194
209
|
const messages = payload.messages.map(sanitizeMessage);
|
|
@@ -221,7 +236,11 @@ export function sanitizeOpenAICompatPayload(payload: unknown): unknown {
|
|
|
221
236
|
|
|
222
237
|
// Top-level fields absent from NaN's schema.
|
|
223
238
|
delete out.store;
|
|
224
|
-
|
|
239
|
+
// `stream_options` is preserved only when the model declares that the
|
|
240
|
+
// gateway reports usage in the stream (issue #4). Otherwise it is stripped:
|
|
241
|
+
// pi-ai would not send it for such a model anyway, and a hand-built payload
|
|
242
|
+
// must stay schema-strict.
|
|
243
|
+
if (options.preserveStreamOptions !== true) delete out.stream_options;
|
|
225
244
|
// NaN documents `max_tokens`, not `max_completion_tokens`.
|
|
226
245
|
if ("max_completion_tokens" in out && !("max_tokens" in out)) {
|
|
227
246
|
out.max_tokens = out.max_completion_tokens;
|
package/src/provider-factory.ts
CHANGED
|
@@ -26,6 +26,7 @@ import type {
|
|
|
26
26
|
ProviderStreams,
|
|
27
27
|
RefreshModelsContext,
|
|
28
28
|
} from "@earendil-works/pi-ai";
|
|
29
|
+
import { withContextOverflowClassification } from "./context-overflow-classifier.ts";
|
|
29
30
|
import {
|
|
30
31
|
baselineModels,
|
|
31
32
|
DEFAULT_MODELS_TIMEOUT_MS,
|
|
@@ -92,6 +93,12 @@ export async function resolveOpenAICompletionsApi(): Promise<OpenAICompletionsAp
|
|
|
92
93
|
* which pi-ai version the runtime bundles, so the fix is not tied to a
|
|
93
94
|
* specific upstream build.
|
|
94
95
|
*
|
|
96
|
+
* `stream_options` is the one field whose removal is conditional: when the
|
|
97
|
+
* model's effective `compat.supportsUsageInStreaming` is true (catalog value
|
|
98
|
+
* or user `models.json` override), pi-ai requested usage and the gateway will
|
|
99
|
+
* return it — stripping the field would silently zero `message.usage`
|
|
100
|
+
* (issue #4). Every other model keeps the strict payload.
|
|
101
|
+
*
|
|
95
102
|
* Any caller-supplied `onPayload` (e.g. pi's own debug/passthrough hook) is
|
|
96
103
|
* preserved and chained AFTER sanitization, so the final payload is always
|
|
97
104
|
* schema-valid.
|
|
@@ -100,13 +107,27 @@ function isObject(value: unknown): value is Record<string, unknown> {
|
|
|
100
107
|
return typeof value === "object" && value !== null;
|
|
101
108
|
}
|
|
102
109
|
|
|
110
|
+
/**
|
|
111
|
+
* Whether the model's effective compat asks pi-ai for streaming usage. pi-ai
|
|
112
|
+
* emits `stream_options: { include_usage: true }` when this is not false, and
|
|
113
|
+
* `models.json` overrides compose above the registered catalog, so this reads
|
|
114
|
+
* the user's confirmed value rather than the generated default.
|
|
115
|
+
*/
|
|
116
|
+
function modelSupportsUsageInStreaming(model: unknown): boolean {
|
|
117
|
+
if (!isObject(model)) return false;
|
|
118
|
+
const compat = model.compat;
|
|
119
|
+
return isObject(compat) && compat.supportsUsageInStreaming === true;
|
|
120
|
+
}
|
|
121
|
+
|
|
103
122
|
export function wrapApiForStrictSanitization(api: ProviderStreams): ProviderStreams {
|
|
104
123
|
const withSanitizer = <TOptions extends object | undefined>(options: TOptions): TOptions => {
|
|
105
124
|
const userOnPayload = isObject(options) ? (options.onPayload as unknown) : undefined;
|
|
106
125
|
return {
|
|
107
126
|
...((options ?? {}) as Record<string, unknown>),
|
|
108
127
|
onPayload: async (payload: unknown, model: unknown) => {
|
|
109
|
-
const sanitized = sanitizeOpenAICompatPayload(payload
|
|
128
|
+
const sanitized = sanitizeOpenAICompatPayload(payload, {
|
|
129
|
+
preserveStreamOptions: modelSupportsUsageInStreaming(model),
|
|
130
|
+
});
|
|
110
131
|
if (typeof userOnPayload === "function") {
|
|
111
132
|
const userResult = await (userOnPayload as (p: unknown, m: unknown) => unknown)(sanitized, model);
|
|
112
133
|
return userResult ?? sanitized;
|
|
@@ -173,6 +194,12 @@ export async function createNanCompatibleProvider(
|
|
|
173
194
|
const current = liveIds;
|
|
174
195
|
return current ? models.filter((model) => current.has(model.id)) : models;
|
|
175
196
|
},
|
|
176
|
-
|
|
197
|
+
// Sanitize the payload for NaN's strict schema, then classify an opaque
|
|
198
|
+
// generic 400 as a context overflow when the request we just sent was over
|
|
199
|
+
// the model's window. The second layer keeps a replayed cross-model
|
|
200
|
+
// reasoning trace (or any other over-window request the context-hook guard
|
|
201
|
+
// cannot reach, e.g. NAN_THINKING_GUARD=0) recoverable instead of wedging
|
|
202
|
+
// the session — see src/context-overflow-classifier.ts.
|
|
203
|
+
api: withContextOverflowClassification(wrapApiForStrictSanitization(apiFactory())),
|
|
177
204
|
});
|
|
178
205
|
}
|