@goodandready/dsh-clinebot 0.5.17 → 0.5.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/lib/account-pool.js +14 -11
  3. package/lib/client.js +145 -50
  4. package/lib/config.js +3 -0
  5. package/lib/proxy-token.js +78 -0
  6. package/lib/routes/proxy.js +408 -391
  7. package/lib/session-router.js +43 -6
  8. package/package.json +1 -1
package/lib/routes/proxy.js CHANGED
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Transparent Loopback Proxy & Zero-Downtime Failover (Issues #127, #145, #147)
2
+ * Transparent Loopback Proxy & Zero-Downtime Failover (Issues #127, #145, #147, #211, #212)
3
3
  *
4
4
  * Exposes local OpenAI-compatible endpoints:
5
5
  * - POST /dsh-clinebot/v1/chat/completions
@@ -10,44 +10,26 @@
10
10
  * 2. Origin & Sec-Fetch-Site rejection to prevent cross-site/browser CSRF attacks (Issue #145).
11
11
  * 3. Dynamic User-Agent extraction from package.json (Issue #147).
12
12
  * 4. Configurable upstream timeout with multi-account failover and 504 handling (Issue #147).
13
- * 5. Sticky session routing with least-used quota allocation.
14
- * 6. Streaming SSE token usage interception & persistent telemetry recording.
13
+ * 5. Sticky session routing with least-used and round-robin quota allocation (Issue #211).
14
+ * 6. Pre-first-token failover: commits SSE headers only after first valid token/chunk (Issue #212).
15
+ * 7. Streaming SSE token usage interception & persistent telemetry recording.
15
16
  */
16
17
 
17
- import { createRequire } from 'node:module'
18
18
  import { writeJson, readBody, isLoopbackAddress } from '../http.js'
19
19
  import { publicConfig, plainConfig } from '../config.js'
20
20
  import { resolveAccountPool } from '../account-pool.js'
21
21
  import { resolveSessionAccount, markAccountCooldown } from '../session-router.js'
22
22
  import { recordUsage } from '../stats-storage.js'
23
23
  import { getAllModels, DEFAULT_MODEL_ID } from '../models.js'
24
- import { getLocalProxyToken } from '../proxy-token.js'
25
-
26
- const require = createRequire(import.meta.url)
27
- let packageVersion = '0.5.17'
28
- try {
29
- packageVersion = require('../../package.json').version || '0.5.17'
30
- } catch (err) {
31
- /* fallback to default version if manifest is unreadable */
32
- }
33
-
34
- export const USER_AGENT = `@goodandready/dsh-clinebot/${packageVersion}`
35
-
36
- export function hasDisallowedBrowserOrigin(req) {
37
- const secFetchSite = req.headers?.['sec-fetch-site']
38
- return secFetchSite === 'cross-site' || secFetchSite === 'same-site' || Boolean(req.headers?.['origin'])
39
- }
24
+ import {
25
+ getLocalProxyToken,
26
+ hasDisallowedBrowserOrigin,
27
+ isAuthorizedProxyRequest,
28
+ USER_AGENT,
29
+ sendProxyTerminalError,
30
+ } from '../proxy-token.js'
40
31
 
41
- export function isAuthorizedProxyRequest(req, accountPool = []) {
42
- const auth = req.headers?.['authorization'] || ''
43
- const match = auth.match(/^Bearer\s+(.+)$/i)
44
- if (!match) return false
45
- const token = match[1].trim()
46
- if (!token) return false
47
- const localToken = getLocalProxyToken()
48
- if (localToken && token === localToken) return true
49
- return Array.isArray(accountPool) && accountPool.some((acc) => acc.value && acc.value === token)
50
- }
32
+ export { hasDisallowedBrowserOrigin, isAuthorizedProxyRequest, USER_AGENT }
51
33
 
52
34
  export function createChatCompletionsHandler(ctx, { live, getSettingsApi, syncProviderState }) {
53
35
  return async (req, res) => {
@@ -55,236 +37,335 @@ export function createChatCompletionsHandler(ctx, { live, getSettingsApi, syncPr
55
37
  return writeJson(res, 405, { ok: false, error: 'POST only' })
56
38
  }
57
39
 
58
- if (!isLoopbackAddress(req.socket?.remoteAddress)) {
59
- return writeJson(res, 403, { ok: false, error: 'Loopback access only' })
60
- }
40
+ if (!isLoopbackAddress(req.socket?.remoteAddress)) {
41
+ return writeJson(res, 403, { ok: false, error: 'Loopback access only' })
42
+ }
61
43
 
62
- if (hasDisallowedBrowserOrigin(req)) {
63
- return writeJson(res, 403, { ok: false, error: 'Cross-site or browser requests to proxy are forbidden' })
64
- }
44
+ if (hasDisallowedBrowserOrigin(req)) {
45
+ return writeJson(res, 403, { ok: false, error: 'Cross-site or browser requests to proxy are forbidden' })
46
+ }
65
47
 
66
- const pool = await resolveAccountPool(ctx, live())
67
- if (!isAuthorizedProxyRequest(req, pool)) {
68
- return writeJson(res, 401, { ok: false, error: 'Unauthorized: missing or invalid bearer token' })
69
- }
48
+ const pool = await resolveAccountPool(ctx, live())
49
+ if (!isAuthorizedProxyRequest(req, pool)) {
50
+ return writeJson(res, 401, { ok: false, error: 'Unauthorized: missing or invalid bearer token' })
51
+ }
70
52
 
71
- let bodyBuf
72
- try {
73
- bodyBuf = await readBody(req, 64 * 1024 * 1024)
74
- } catch (err) {
75
- return writeJson(res, 400, { ok: false, error: String(err?.message || err) })
76
- }
53
+ let bodyBuf
54
+ try {
55
+ bodyBuf = await readBody(req, 64 * 1024 * 1024)
56
+ } catch (err) {
57
+ return writeJson(res, 400, { ok: false, error: String(err?.message || err) })
58
+ }
77
59
 
78
- let body
79
- try {
80
- body = JSON.parse(bodyBuf.toString('utf8') || '{}')
81
- } catch {
82
- return writeJson(res, 400, { ok: false, error: 'Invalid JSON request body' })
83
- }
60
+ let body
61
+ try {
62
+ body = JSON.parse(bodyBuf.toString('utf8') || '{}')
63
+ } catch {
64
+ return writeJson(res, 400, { ok: false, error: 'Invalid JSON request body' })
65
+ }
66
+
67
+ const cfg = plainConfig(live())
68
+ const pub = publicConfig(cfg)
69
+
70
+ const h = req.headers || {}
71
+ const sessionId = String(
72
+ h['x-dsh-session-id'] || h['x-session-id'] || h['session-id'] || h['session_id'] ||
73
+ h['x-session-affinity'] || h['x-client-request-id'] ||
74
+ body?.session_id || body?.conversation_id || body?.prompt_cache_key || body?.chat_id || body?.user || ''
75
+ ).trim()
76
+
77
+ if (sessionId) {
78
+ if (!body.session_id) body.session_id = sessionId
79
+ if (!body.conversation_id) body.conversation_id = sessionId
80
+ }
84
81
 
85
- const cfg = plainConfig(live())
86
- const pub = publicConfig(cfg)
82
+ body.model = String(body.model || pub.defaultModel || DEFAULT_MODEL_ID).trim()
83
+ if (pub.modelReasoningDefaults?.[body.model] && !body.reasoning_effort) {
84
+ body.reasoning_effort = pub.modelReasoningDefaults[body.model]
85
+ }
87
86
 
88
- const h = req.headers || {}
89
- const sessionId = String(
90
- h['x-dsh-session-id'] || h['x-session-id'] || h['session-id'] || h['session_id'] ||
91
- h['x-session-affinity'] || h['x-client-request-id'] ||
92
- body?.session_id || body?.conversation_id || body?.prompt_cache_key || body?.chat_id || body?.user || ''
93
- ).trim()
87
+ const isStreaming = Boolean(body.stream)
88
+ const configuredAccounts = pool.filter(a => a.present && a.value)
94
89
 
95
- if (sessionId) {
96
- if (!body.session_id) body.session_id = sessionId
97
- if (!body.conversation_id) body.conversation_id = sessionId
90
+ const clientAbortCtrl = new AbortController()
91
+ let clientClosed = false
92
+ const onClientClose = () => {
93
+ clientClosed = true
94
+ if (!res.writableFinished) {
95
+ clientAbortCtrl.abort()
98
96
  }
97
+ }
98
+ if (typeof res.on === 'function') res.on('close', onClientClose)
99
+ if (typeof req.on === 'function') req.on('close', onClientClose)
100
+
101
+ const cleanupClientListeners = () => {
102
+ if (typeof res.off === 'function') res.off('close', onClientClose)
103
+ else if (typeof res.removeListener === 'function') res.removeListener('close', onClientClose)
104
+ if (typeof req.off === 'function') req.off('close', onClientClose)
105
+ else if (typeof req.removeListener === 'function') req.removeListener('close', onClientClose)
106
+ }
99
107
 
100
- body.model = String(body.model || pub.defaultModel || DEFAULT_MODEL_ID).trim()
101
- if (pub.modelReasoningDefaults?.[body.model] && !body.reasoning_effort) {
102
- body.reasoning_effort = pub.modelReasoningDefaults[body.model]
108
+ const maxRetries = Math.max(1, configuredAccounts.length)
109
+ let attempt = 0
110
+ let lastError = null
111
+ let hadTimeout = false
112
+ let lastTargetAccount = null
113
+ let lastRetryAfter = null
114
+
115
+ while (attempt < maxRetries) {
116
+ attempt += 1
117
+
118
+ if (clientAbortCtrl.signal.aborted || clientClosed) {
119
+ cleanupClientListeners()
120
+ return
103
121
  }
104
122
 
105
- const isStreaming = Boolean(body.stream)
106
- const configuredAccounts = pool.filter(a => a.present && a.value)
123
+ const routeResult = resolveSessionAccount(sessionId, pool, pub.activeAccount, pub.accountMode)
124
+ const targetAccount = routeResult.account
125
+ lastTargetAccount = targetAccount
107
126
 
108
- const clientAbortCtrl = new AbortController()
109
- let clientClosed = false
110
- const onClientClose = () => {
111
- clientClosed = true
112
- if (!res.writableFinished) {
113
- clientAbortCtrl.abort()
114
- }
127
+ if (!targetAccount || !targetAccount.value) {
128
+ cleanupClientListeners()
129
+ return writeJson(res, 503, {
130
+ error: {
131
+ message: 'No available ClinePass API keys configured in pool',
132
+ type: 'service_unavailable',
133
+ code: 'no_keys_available'
134
+ }
135
+ })
115
136
  }
116
- if (typeof res.on === 'function') res.on('close', onClientClose)
117
- if (typeof req.on === 'function') req.on('close', onClientClose)
118
-
119
- const cleanupClientListeners = () => {
120
- if (typeof res.off === 'function') res.off('close', onClientClose)
121
- else if (typeof res.removeListener === 'function') res.removeListener('close', onClientClose)
122
- if (typeof req.off === 'function') req.off('close', onClientClose)
123
- else if (typeof req.removeListener === 'function') req.removeListener('close', onClientClose)
137
+
138
+ const accountId = targetAccount.apiKeyEnv || targetAccount.id || targetAccount.label
139
+ const upstreamUrl = `${pub.baseUrl}/chat/completions`
140
+ const requestHeaders = {
141
+ 'Content-Type': 'application/json',
142
+ 'Authorization': `Bearer ${targetAccount.value}`,
143
+ 'Accept': isStreaming ? 'text/event-stream' : 'application/json',
144
+ 'User-Agent': USER_AGENT
124
145
  }
125
146
 
126
- const maxRetries = Math.max(1, configuredAccounts.length)
127
- let attempt = 0
128
- let lastError = null
129
- let hadTimeout = false
130
- let lastTargetAccount = null
131
- let lastRetryAfter = null
147
+ const connectTimeoutMs = Math.max(50, Number(pub.timeoutMs) || 15000)
148
+ const streamIdleTimeoutMs = Math.max(50, Number(pub.streamIdleTimeoutMs) || 30000)
149
+
150
+ const connectAbortCtrl = new AbortController()
151
+ const bodyAbortCtrl = new AbortController()
152
+ let connectTimedOut = false
153
+ const connectTimer = setTimeout(() => {
154
+ connectTimedOut = true
155
+ connectAbortCtrl.abort(new Error(`Upstream connect timeout after ${connectTimeoutMs}ms`))
156
+ }, connectTimeoutMs)
132
157
 
133
- while (attempt < maxRetries) {
134
- attempt += 1
158
+ const fetchSignal = AbortSignal.any([clientAbortCtrl.signal, connectAbortCtrl.signal, bodyAbortCtrl.signal])
135
159
 
160
+ let upstreamRes
161
+ try {
162
+ upstreamRes = await fetch(upstreamUrl, {
163
+ method: 'POST',
164
+ headers: requestHeaders,
165
+ body: JSON.stringify(body),
166
+ signal: fetchSignal,
167
+ keepalive: true
168
+ })
169
+ } catch (fetchErr) {
170
+ clearTimeout(connectTimer)
136
171
  if (clientAbortCtrl.signal.aborted || clientClosed) {
137
172
  cleanupClientListeners()
138
173
  return
139
174
  }
175
+ lastError = fetchErr
176
+ const isTimeout = connectTimedOut || connectAbortCtrl.signal.aborted || fetchErr.name === 'TimeoutError' || String(fetchErr).includes('timeout')
177
+ if (isTimeout) {
178
+ hadTimeout = true
179
+ markAccountCooldown(accountId, 60000, null, 'upstream_timeout')
180
+ if (targetAccount.id && targetAccount.id !== accountId) markAccountCooldown(targetAccount.id, 60000, null, 'upstream_timeout')
181
+ ctx?.logger?.warn?.(`[dsh-clinebot:proxy] Account "${accountId}" upstream connect timed out after ${connectTimeoutMs}ms, attempting failover (attempt ${attempt}/${maxRetries})`)
182
+ } else {
183
+ markAccountCooldown(accountId, 60000, null, 'upstream_fetch_error')
184
+ if (targetAccount.id && targetAccount.id !== accountId) markAccountCooldown(targetAccount.id, 60000, null, 'upstream_fetch_error')
185
+ ctx?.logger?.warn?.(`[dsh-clinebot:proxy] Account "${accountId}" connect error (${fetchErr.message}), attempting failover (attempt ${attempt}/${maxRetries})`)
186
+ }
187
+ continue
188
+ }
189
+ clearTimeout(connectTimer)
140
190
 
141
- const routeResult = resolveSessionAccount(sessionId, pool, pub.activeAccount)
142
- const targetAccount = routeResult.account
143
- lastTargetAccount = targetAccount
191
+ if (clientAbortCtrl.signal.aborted || clientClosed) {
192
+ cleanupClientListeners()
193
+ return
194
+ }
144
195
 
145
- if (!targetAccount || !targetAccount.value) {
146
- cleanupClientListeners()
147
- return writeJson(res, 503, {
148
- error: {
149
- message: 'No available ClinePass API keys configured in pool',
150
- type: 'service_unavailable',
151
- code: 'no_keys_available'
152
- }
153
- })
154
- }
196
+ // Pre-first-token failover: check for retryable non-200 responses before sending ANY headers
197
+ if (upstreamRes.status !== 200) {
198
+ const isRateLimit = upstreamRes.status === 429 || upstreamRes.status === 402
199
+ const isServerErr = upstreamRes.status >= 500 && upstreamRes.status <= 599
200
+ const isAuthErr = upstreamRes.status === 401 || upstreamRes.status === 403
155
201
 
156
- const upstreamUrl = `${pub.baseUrl}/chat/completions`
157
- const requestHeaders = {
158
- 'Content-Type': 'application/json',
159
- 'Authorization': `Bearer ${targetAccount.value}`,
160
- 'Accept': isStreaming ? 'text/event-stream' : 'application/json',
161
- 'User-Agent': USER_AGENT
202
+ if (isRateLimit || isServerErr || isAuthErr) {
203
+ const reason = isRateLimit ? 'upstream_429' : (isAuthErr ? 'upstream_auth' : `upstream_status_${upstreamRes.status}`)
204
+ markAccountCooldown(accountId, 60000, null, reason)
205
+ if (targetAccount.id && targetAccount.id !== accountId) markAccountCooldown(targetAccount.id, 60000, null, reason)
206
+ lastError = new Error(`Upstream rejected with HTTP ${upstreamRes.status}`)
207
+ lastError.status = upstreamRes.status
208
+ const retryAfter = upstreamRes.headers?.get?.('retry-after')
209
+ if (retryAfter) lastRetryAfter = retryAfter
210
+ if (upstreamRes.body) {
211
+ try {
212
+ if (typeof upstreamRes.body.cancel === 'function') upstreamRes.body.cancel().catch(() => {})
213
+ else if (typeof upstreamRes.body.destroy === 'function') upstreamRes.body.destroy()
214
+ } catch {}
215
+ }
216
+ ctx?.logger?.warn?.(`[dsh-clinebot:proxy] Account "${accountId}" returned HTTP ${upstreamRes.status}, attempting failover (attempt ${attempt}/${maxRetries})`)
217
+ continue
162
218
  }
163
219
 
164
- const connectTimeoutMs = Math.max(50, Number(pub.timeoutMs) || 15000)
165
- const streamIdleTimeoutMs = Math.max(50, Number(pub.streamIdleTimeoutMs) || 30000)
220
+ // Non-retryable client error (e.g. 400 Bad Request)
221
+ cleanupClientListeners()
222
+ const errText = await upstreamRes.text().catch(() => '')
223
+ res.statusCode = upstreamRes.status
224
+ res.setHeader('Content-Type', 'application/json; charset=utf-8')
225
+ res.end(errText || JSON.stringify({ error: { message: `Upstream error ${upstreamRes.status}` } }))
226
+ return
227
+ }
166
228
 
167
- const connectAbortCtrl = new AbortController()
168
- const bodyAbortCtrl = new AbortController()
169
- let connectTimedOut = false
170
- let bodyTimedOut = false
171
- const connectTimer = setTimeout(() => {
172
- connectTimedOut = true
173
- connectAbortCtrl.abort(new Error(`Upstream connect timeout after ${connectTimeoutMs}ms`))
174
- }, connectTimeoutMs)
229
+ // Streaming response with pre-first-token buffering (#212)
230
+ if (isStreaming && upstreamRes.status === 200 && upstreamRes.body) {
231
+ const reader = upstreamRes.body.getReader()
232
+ const decoder = new TextDecoder('utf8')
233
+
234
+ let firstReadResult = null
235
+ let preTokenError = null
175
236
 
176
- const fetchSignal = AbortSignal.any([clientAbortCtrl.signal, connectAbortCtrl.signal, bodyAbortCtrl.signal])
237
+ const ttftAbortCtrl = new AbortController()
238
+ const ttftTimer = setTimeout(() => {
239
+ ttftAbortCtrl.abort(new Error(`TTFT timeout waiting for first stream chunk after ${connectTimeoutMs}ms`))
240
+ }, connectTimeoutMs)
177
241
 
178
- let upstreamRes
179
242
  try {
180
- upstreamRes = await fetch(upstreamUrl, {
181
- method: 'POST',
182
- headers: requestHeaders,
183
- body: JSON.stringify(body),
184
- signal: fetchSignal,
185
- keepalive: true
243
+ const firstReadPromise = reader.read()
244
+ const timeoutPromise = new Promise((_, reject) => {
245
+ ttftAbortCtrl.signal.addEventListener('abort', () => reject(ttftAbortCtrl.signal.reason), { once: true })
186
246
  })
187
- } catch (fetchErr) {
188
- clearTimeout(connectTimer)
189
- if (clientAbortCtrl.signal.aborted || clientClosed) {
190
- cleanupClientListeners()
191
- return
192
- }
193
- lastError = fetchErr
194
- const isTimeout = connectTimedOut || connectAbortCtrl.signal.aborted || fetchErr.name === 'TimeoutError' || String(fetchErr).includes('timeout')
195
- if (isTimeout) {
196
- hadTimeout = true
197
- const accountId = targetAccount.apiKeyEnv || targetAccount.id || targetAccount.label
198
- markAccountCooldown(accountId, 60000, null, 'upstream_timeout')
199
- if (targetAccount.id && targetAccount.id !== accountId) markAccountCooldown(targetAccount.id, 60000, null, 'upstream_timeout')
200
- ctx?.logger?.warn?.(`[dsh-clinebot:proxy] Account "${accountId}" upstream connect timed out after ${connectTimeoutMs}ms, attempting failover (attempt ${attempt}/${maxRetries})`)
201
- }
202
- continue
247
+ firstReadResult = await Promise.race([firstReadPromise, timeoutPromise])
248
+ } catch (err) {
249
+ preTokenError = err
250
+ } finally {
251
+ clearTimeout(ttftTimer)
203
252
  }
204
- clearTimeout(connectTimer)
205
253
 
206
- // If upstream rejected with 429 or 402 quota exhaustion before streaming, failover!
207
- if (upstreamRes.status === 429 || upstreamRes.status === 402) {
208
- const accountId = targetAccount.apiKeyEnv || targetAccount.id || targetAccount.label
209
- markAccountCooldown(accountId, 60000, null, 'upstream_429')
210
- if (targetAccount.id && targetAccount.id !== accountId) markAccountCooldown(targetAccount.id, 60000, null, 'upstream_429')
211
- lastError = new Error(`Upstream rejected with HTTP ${upstreamRes.status}`)
212
- lastError.status = upstreamRes.status
213
- const retryAfter = upstreamRes.headers?.get?.('retry-after')
214
- if (retryAfter) {
215
- lastRetryAfter = retryAfter
216
- }
217
- if (upstreamRes.body) {
254
+ if (clientAbortCtrl.signal.aborted || clientClosed) {
255
+ cleanupClientListeners()
256
+ try { reader.cancel('client aborted').catch(() => {}) } catch {}
257
+ return
258
+ }
259
+
260
+ let earlyError = null
261
+ if (!preTokenError && firstReadResult?.value) {
262
+ const textSample = decoder.decode(firstReadResult.value, { stream: true })
263
+ const trimmed = textSample.trim()
264
+ if (trimmed.startsWith('{') && trimmed.includes('"error"')) {
218
265
  try {
219
- if (typeof upstreamRes.body.cancel === 'function') {
220
- upstreamRes.body.cancel().catch(() => {})
221
- } else if (typeof upstreamRes.body.destroy === 'function') {
222
- upstreamRes.body.destroy()
223
- }
224
- } catch {
225
- /* ignore cancel error */
226
- }
266
+ const parsed = JSON.parse(trimmed)
267
+ if (parsed.error && !parsed.choices) earlyError = new Error(parsed.error.message || 'Upstream error')
268
+ } catch {}
269
+ } else if (trimmed.startsWith('data:') && trimmed.includes('"error"')) {
270
+ try {
271
+ const parsed = JSON.parse(trimmed.slice(5).trim())
272
+ if (parsed.error && !parsed.choices) earlyError = new Error(parsed.error.message || 'Upstream stream error')
273
+ } catch {}
227
274
  }
228
- ctx?.logger?.warn?.(`[dsh-clinebot:proxy] Account "${accountId}" hit 429, attempting failover (attempt ${attempt}/${maxRetries})`)
275
+ } else if (!preTokenError && firstReadResult?.done) {
276
+ earlyError = new Error('Upstream stream completed with zero chunks')
277
+ }
278
+
279
+ if (preTokenError || earlyError) {
280
+ if (ttftAbortCtrl.signal.aborted || String(preTokenError?.message).includes('timeout')) hadTimeout = true
281
+ try { reader.cancel('pre-first-token error').catch(() => {}) } catch {}
282
+ markAccountCooldown(accountId, 60000, null, 'upstream_pre_token_error')
283
+ if (targetAccount.id && targetAccount.id !== accountId) markAccountCooldown(targetAccount.id, 60000, null, 'upstream_pre_token_error')
284
+ lastError = preTokenError || earlyError
285
+ ctx?.logger?.warn?.(`[dsh-clinebot:proxy] Account "${accountId}" failed before first token (${lastError.message}), attempting failover (attempt ${attempt}/${maxRetries})`)
229
286
  continue
230
287
  }
231
288
 
232
- // Upstream responded with non-retryable response: stream or write JSON to client
233
- if (isStreaming && upstreamRes.status === 200 && upstreamRes.body) {
234
- res.statusCode = 200
235
- res.setHeader('Content-Type', 'text/event-stream; charset=utf-8')
236
- res.setHeader('Cache-Control', 'no-cache, no-transform')
237
- res.setHeader('Connection', 'keep-alive')
238
- if (typeof res.flushHeaders === 'function') {
239
- res.flushHeaders()
240
- }
289
+ // First token received: commit headers to client now
290
+ res.statusCode = 200
291
+ res.setHeader('Content-Type', 'text/event-stream; charset=utf-8')
292
+ res.setHeader('Cache-Control', 'no-cache, no-transform')
293
+ res.setHeader('Connection', 'keep-alive')
294
+ if (typeof res.flushHeaders === 'function') {
295
+ res.flushHeaders()
296
+ }
297
+
298
+ let promptTokens = 0
299
+ let completionTokens = 0
300
+ let totalTokens = 0
301
+ let buffer = ''
302
+ let streamError = null
303
+ let streamCompletedCleanly = false
241
304
 
242
- let promptTokens = 0
243
- let completionTokens = 0
244
- let totalTokens = 0
245
- let buffer = ''
246
- let streamError = null
247
- let streamCompletedCleanly = false
305
+ let streamAborted = false
306
+ const onStreamAbort = () => {
307
+ streamAborted = true
308
+ try {
309
+ reader.cancel('client closed response').catch(() => {})
310
+ } catch {}
311
+ }
248
312
 
249
- const reader = upstreamRes.body.getReader()
250
- const decoder = new TextDecoder('utf8')
313
+ if (clientAbortCtrl.signal.aborted || clientClosed) {
314
+ onStreamAbort()
315
+ } else {
316
+ clientAbortCtrl.signal.addEventListener('abort', onStreamAbort, { once: true })
317
+ }
251
318
 
252
- let streamAborted = false
253
- const onStreamAbort = () => {
254
- streamAborted = true
319
+ let idleTimer = null
320
+ let idleTimedOut = false
321
+ const resetIdleTimer = () => {
322
+ if (idleTimer) clearTimeout(idleTimer)
323
+ idleTimer = setTimeout(() => {
324
+ idleTimedOut = true
325
+ streamError = new Error(`Stream idle timeout after ${streamIdleTimeoutMs}ms`)
255
326
  try {
256
- reader.cancel('client closed response').catch(() => {})
257
- } catch { /* reader already cancelled or closed */ }
258
- }
327
+ reader.cancel('stream idle timeout').catch(() => {})
328
+ } catch {}
329
+ }, streamIdleTimeoutMs)
330
+ }
259
331
 
260
- if (clientAbortCtrl.signal.aborted || clientClosed) {
261
- onStreamAbort()
262
- } else {
263
- clientAbortCtrl.signal.addEventListener('abort', onStreamAbort, { once: true })
332
+ const clearIdleTimer = () => {
333
+ if (idleTimer) {
334
+ clearTimeout(idleTimer)
335
+ idleTimer = null
264
336
  }
337
+ }
265
338
 
266
- let idleTimer = null
267
- let idleTimedOut = false
268
- const resetIdleTimer = () => {
269
- if (idleTimer) clearTimeout(idleTimer)
270
- idleTimer = setTimeout(() => {
271
- idleTimedOut = true
272
- streamError = new Error(`Stream idle timeout after ${streamIdleTimeoutMs}ms`)
273
- try {
274
- reader.cancel('stream idle timeout').catch(() => {})
275
- } catch { /* reader already cancelled or closed */ }
276
- }, streamIdleTimeoutMs)
339
+ const processChunk = (val) => {
340
+ if (!res.destroyed && !res.writableEnded) {
341
+ res.write(val)
277
342
  }
278
-
279
- const clearIdleTimer = () => {
280
- if (idleTimer) {
281
- clearTimeout(idleTimer)
282
- idleTimer = null
343
+ const chunkStr = decoder.decode(val, { stream: true })
344
+ buffer += chunkStr
345
+ const lines = buffer.split('\n')
346
+ buffer = lines.pop() || ''
347
+ for (const line of lines) {
348
+ const trimmed = line.trim()
349
+ if (trimmed.startsWith('data:') && trimmed !== 'data: [DONE]') {
350
+ try {
351
+ const parsed = JSON.parse(trimmed.slice(5).trim())
352
+ if (parsed.usage) {
353
+ promptTokens = parsed.usage.prompt_tokens || promptTokens
354
+ completionTokens = parsed.usage.completion_tokens || completionTokens
355
+ totalTokens = parsed.usage.total_tokens || totalTokens
356
+ }
357
+ } catch {}
283
358
  }
284
359
  }
360
+ }
285
361
 
362
+ if (firstReadResult?.value) {
363
+ processChunk(firstReadResult.value)
364
+ }
286
365
 
287
-
366
+ if (firstReadResult?.done) {
367
+ streamCompletedCleanly = true
368
+ } else {
288
369
  try {
289
370
  resetIdleTimer()
290
371
  while (true) {
@@ -297,213 +378,152 @@ export function createChatCompletionsHandler(ctx, { live, getSettingsApi, syncPr
297
378
  }
298
379
  break
299
380
  }
300
-
301
381
  resetIdleTimer()
302
-
303
- const chunkStr = decoder.decode(value, { stream: true })
304
- if (!res.destroyed && !res.writableEnded) {
305
- res.write(value)
306
- }
382
+ processChunk(value)
307
383
  if (streamAborted || clientAbortCtrl.signal.aborted || clientClosed) break
308
-
309
- // Parse SSE lines for usage metrics
310
- buffer += chunkStr
311
- const lines = buffer.split('\n')
312
- buffer = lines.pop() || ''
313
-
314
- for (const line of lines) {
315
- const trimmed = line.trim()
316
- if (trimmed.startsWith('data:') && trimmed !== 'data: [DONE]') {
317
- try {
318
- const parsed = JSON.parse(trimmed.slice(5).trim())
319
- if (parsed.usage) {
320
- promptTokens = parsed.usage.prompt_tokens || promptTokens
321
- completionTokens = parsed.usage.completion_tokens || completionTokens
322
- totalTokens = parsed.usage.total_tokens || totalTokens
323
- }
324
- } catch {
325
- /* ignore JSON chunk parse error */
326
- }
327
- }
328
- }
329
384
  }
330
- } catch (streamErr) {
331
- streamError = streamErr
332
- ctx?.logger?.warn?.('[dsh-clinebot:proxy] Stream read error: ' + (streamErr?.message || streamErr))
385
+ } catch (err) {
386
+ streamError = err
333
387
  } finally {
334
388
  clearIdleTimer()
335
-
336
- clientAbortCtrl.signal.removeEventListener('abort', onStreamAbort)
337
- cleanupClientListeners()
338
- try {
339
- reader.releaseLock()
340
- } catch (err) { /* ignore lock release error if reader closed */ }
341
-
342
- const isStreamFailed = Boolean(streamError || idleTimedOut || clientClosed || clientAbortCtrl.signal.aborted || !streamCompletedCleanly)
343
-
344
- if (!res.destroyed && !res.writableEnded) {
345
- if (isStreamFailed && (idleTimedOut || streamError)) {
346
- try {
347
- const errPayload = JSON.stringify({
348
- error: {
349
- message: streamError?.message || 'Stream interrupted or timed out',
350
- type: 'stream_error',
351
- code: idleTimedOut ? 'stream_idle_timeout' : 'stream_error'
352
- }
353
- })
354
- res.write(`data: ${errPayload}\n\n`)
355
- } catch { /* reader already cancelled or closed */ }
356
- }
357
- res.end()
358
- }
359
-
360
- recordUsage({
361
- model: body.model,
362
- accountId: targetAccount.id || targetAccount.label,
363
- promptTokens,
364
- completionTokens,
365
- totalTokens: totalTokens || (promptTokens + completionTokens),
366
- isError: isStreamFailed,
367
- statsPath: pub.statsPath
368
- })
369
389
  }
370
- return
371
- }
372
-
373
- // Non-streaming JSON response with deadline and fail-closed error handling (#185, #198, #202)
374
- const bodyTimeoutMs = Math.max(50, Number(pub.timeoutMs) || 15000)
375
- let bodyTimer = null
376
- const bodyTimeoutPromise = new Promise((_, reject) => {
377
- bodyTimer = setTimeout(() => {
378
- bodyTimedOut = true
379
- bodyAbortCtrl.abort(new Error(`Upstream body read timed out after ${bodyTimeoutMs}ms`))
380
- reject(new Error(`Upstream body read timed out after ${bodyTimeoutMs}ms`))
381
- }, bodyTimeoutMs)
382
- })
383
-
384
- const MAX_BODY_BYTES = 10 * 1024 * 1024
385
- const contentLength = Number(upstreamRes.headers.get('content-length'))
386
- if (contentLength > MAX_BODY_BYTES) {
387
- clearTimeout(bodyTimer)
388
- cleanupClientListeners()
389
- writeJson(res, 502, { ok: false, error: 'Upstream response exceeded maximum allowed size' })
390
- return
391
- }
392
-
393
- let resText = ''
394
- let readError = null
395
- try {
396
- resText = await Promise.race([
397
- upstreamRes.text(),
398
- bodyTimeoutPromise,
399
- ])
400
- if (resText.length > MAX_BODY_BYTES) {
401
- throw new Error('Upstream response exceeded maximum allowed size')
402
- }
403
- } catch (err) {
404
- readError = err
405
- } finally {
406
- clearTimeout(bodyTimer)
407
390
  }
408
391
 
392
+ clearIdleTimer()
393
+ clientAbortCtrl.signal.removeEventListener('abort', onStreamAbort)
409
394
  cleanupClientListeners()
395
+ try {
396
+ reader.releaseLock()
397
+ } catch {}
410
398
 
411
- if (clientAbortCtrl.signal.aborted || clientClosed) {
412
- return
413
- }
399
+ const isStreamFailed = Boolean(streamError || idleTimedOut || clientClosed || clientAbortCtrl.signal.aborted || !streamCompletedCleanly)
414
400
 
415
- if (bodyTimedOut || readError) {
416
- const isTimeout = bodyTimedOut || bodyAbortCtrl.signal.aborted
417
- recordUsage({
418
- model: body.model,
419
- accountId: targetAccount.id || targetAccount.label,
420
- isError: true,
421
- statsPath: pub.statsPath
422
- })
423
- const statusCode = isTimeout ? 504 : 502
424
- const errMessage = isTimeout
425
- ? `Gateway Timeout: upstream body read timed out after ${bodyTimeoutMs}ms`
426
- : `Bad Gateway: upstream response body read error: ${readError?.message || readError}`
427
- return writeJson(res, statusCode, {
428
- error: {
429
- message: errMessage,
430
- type: isTimeout ? 'gateway_timeout' : 'bad_gateway',
431
- code: isTimeout ? 'body_read_timeout' : 'body_read_error'
432
- }
433
- })
434
- }
435
-
436
- let resJson = null
437
- try {
438
- resJson = JSON.parse(resText)
439
- } catch {
440
- resJson = null
401
+ if (!res.destroyed && !res.writableEnded) {
402
+ if (isStreamFailed && (idleTimedOut || streamError)) {
403
+ try {
404
+ const errPayload = JSON.stringify({
405
+ error: {
406
+ message: streamError?.message || 'Stream interrupted or timed out',
407
+ type: 'stream_error',
408
+ code: idleTimedOut ? 'stream_idle_timeout' : 'stream_error'
409
+ }
410
+ })
411
+ res.write(`data: ${errPayload}\n\n`)
412
+ } catch {}
413
+ }
414
+ res.end()
441
415
  }
442
416
 
443
417
  recordUsage({
444
418
  model: body.model,
445
419
  accountId: targetAccount.id || targetAccount.label,
446
- promptTokens: resJson?.usage?.prompt_tokens || 0,
447
- completionTokens: resJson?.usage?.completion_tokens || 0,
448
- totalTokens: resJson?.usage?.total_tokens || 0,
449
- isError: upstreamRes.status >= 400,
420
+ promptTokens,
421
+ completionTokens,
422
+ totalTokens: totalTokens || (promptTokens + completionTokens),
423
+ isError: isStreamFailed,
450
424
  statsPath: pub.statsPath
451
425
  })
452
-
453
- res.statusCode = upstreamRes.status
454
- res.setHeader('Content-Type', 'application/json; charset=utf-8')
455
- res.end(resText)
456
426
  return
457
427
  }
458
428
 
459
- cleanupClientListeners()
460
- // Exhausted all retries
461
- const is429Exhausted = !hadTimeout && (lastError?.status === 429 || lastError?.status === 402)
462
- recordUsage({
463
- model: body.model,
464
- accountId: lastTargetAccount?.id || lastTargetAccount?.label || 'default',
465
- isError: true,
466
- is429: is429Exhausted,
467
- statsPath: pub.statsPath
429
+ // Non-streaming JSON response with deadline and failover (#185, #198, #202, #212)
430
+ const bodyTimeoutMs = Math.max(50, Number(pub.timeoutMs) || 15000)
431
+ let bodyTimedOut = false
432
+ let bodyTimer = null
433
+ const bodyTimeoutPromise = new Promise((_, reject) => {
434
+ bodyTimer = setTimeout(() => {
435
+ bodyTimedOut = true
436
+ bodyAbortCtrl.abort(new Error(`Upstream body read timed out after ${bodyTimeoutMs}ms`))
437
+ reject(new Error(`Upstream body read timed out after ${bodyTimeoutMs}ms`))
438
+ }, bodyTimeoutMs)
468
439
  })
469
- if (hadTimeout) {
470
- return writeJson(res, 504, {
471
- error: {
472
- message: 'Gateway Timeout: upstream provider timed out across all available accounts',
473
- type: 'gateway_timeout',
474
- code: 'upstream_timeout',
475
- details: lastError?.message || null
476
- }
477
- })
440
+
441
+ const MAX_BODY_BYTES = 10 * 1024 * 1024
442
+ const contentLength = Number(upstreamRes.headers.get('content-length'))
443
+ if (contentLength > MAX_BODY_BYTES) {
444
+ clearTimeout(bodyTimer)
445
+ cleanupClientListeners()
446
+ writeJson(res, 502, { ok: false, error: 'Upstream response exceeded maximum allowed size' })
447
+ return
478
448
  }
479
449
 
480
- if (is429Exhausted) {
481
- if (lastRetryAfter) {
482
- res.setHeader('Retry-After', lastRetryAfter)
450
+ let resText = ''
451
+ let readError = null
452
+ try {
453
+ resText = await Promise.race([
454
+ upstreamRes.text(),
455
+ bodyTimeoutPromise,
456
+ ])
457
+ if (resText.length > MAX_BODY_BYTES) {
458
+ throw new Error('Upstream response exceeded maximum allowed size')
483
459
  }
484
- return writeJson(res, 429, {
485
- error: {
486
- message: 'All configured accounts in pool exceeded rate limits or quota',
487
- type: 'rate_limit_exceeded',
488
- code: 'rate_limit_exceeded',
489
- details: lastError?.message || null
490
- }
491
- })
460
+ } catch (err) {
461
+ readError = err
462
+ } finally {
463
+ clearTimeout(bodyTimer)
492
464
  }
493
465
 
494
- const status = typeof lastError?.status === 'number' && lastError.status >= 500 && lastError.status <= 599
495
- ? lastError.status
496
- : 502
497
- return writeJson(res, status, {
498
- error: {
499
- message: `Bad Gateway: upstream network error or unavailable: ${lastError?.message || 'unknown error'}`,
500
- type: 'bad_gateway',
501
- code: 'upstream_unavailable',
502
- details: lastError?.message || null
503
- }
466
+ if (clientAbortCtrl.signal.aborted || clientClosed) {
467
+ cleanupClientListeners()
468
+ return
469
+ }
470
+
471
+ if (bodyTimedOut || readError) {
472
+ const isTimeout = bodyTimedOut || bodyAbortCtrl.signal.aborted
473
+ if (isTimeout) hadTimeout = true
474
+ markAccountCooldown(accountId, 60000, null, isTimeout ? 'upstream_timeout' : 'upstream_read_error')
475
+ if (targetAccount.id && targetAccount.id !== accountId) markAccountCooldown(targetAccount.id, 60000, null, isTimeout ? 'upstream_timeout' : 'upstream_read_error')
476
+ lastError = readError || new Error(`Upstream body read timed out after ${bodyTimeoutMs}ms`)
477
+ lastError.code = isTimeout ? 'body_read_timeout' : 'body_read_error'
478
+ ctx?.logger?.warn?.(`[dsh-clinebot:proxy] Account "${accountId}" body read failed (${lastError.message}), attempting failover (attempt ${attempt}/${maxRetries})`)
479
+ continue
480
+ }
481
+
482
+ let resJson = null
483
+ try {
484
+ resJson = JSON.parse(resText)
485
+ } catch {
486
+ resJson = null
487
+ }
488
+
489
+ if (resJson?.error && !resJson?.choices) {
490
+ markAccountCooldown(accountId, 60000, null, 'upstream_error_payload')
491
+ if (targetAccount.id && targetAccount.id !== accountId) markAccountCooldown(targetAccount.id, 60000, null, 'upstream_error_payload')
492
+ lastError = new Error(resJson.error.message || 'Upstream error response')
493
+ ctx?.logger?.warn?.(`[dsh-clinebot:proxy] Account "${accountId}" returned 200 with error (${lastError.message}), attempting failover (attempt ${attempt}/${maxRetries})`)
494
+ continue
495
+ }
496
+
497
+ cleanupClientListeners()
498
+ recordUsage({
499
+ model: body.model,
500
+ accountId: targetAccount.id || targetAccount.label,
501
+ promptTokens: resJson?.usage?.prompt_tokens || 0,
502
+ completionTokens: resJson?.usage?.completion_tokens || 0,
503
+ totalTokens: resJson?.usage?.total_tokens || 0,
504
+ isError: false,
505
+ statsPath: pub.statsPath
504
506
  })
507
+
508
+ res.statusCode = 200
509
+ res.setHeader('Content-Type', 'application/json; charset=utf-8')
510
+ res.end(resText)
511
+ return
505
512
  }
513
+
514
+ cleanupClientListeners()
515
+ // Exhausted all retries
516
+ const is429Exhausted = !hadTimeout && (lastError?.status === 429 || lastError?.status === 402)
517
+ recordUsage({
518
+ model: body.model,
519
+ accountId: lastTargetAccount?.id || lastTargetAccount?.label || 'default',
520
+ isError: true,
521
+ is429: is429Exhausted,
522
+ statsPath: pub.statsPath
523
+ })
524
+ return sendProxyTerminalError(res, { hadTimeout, is429Exhausted, lastError, lastRetryAfter })
506
525
  }
526
+ }
507
527
 
508
528
  export function createModelsHandler(ctx, { live }) {
509
529
  return async (req, res) => {
@@ -551,18 +571,15 @@ export function registerProxyRoutes(ctx, routeEnv) {
551
571
  const chatHandler = createChatCompletionsHandler(ctx, routeEnv)
552
572
  const modelsHandler = createModelsHandler(ctx, routeEnv)
553
573
 
554
- // 1. POST /dsh-clinebot/v1/chat/completions
555
574
  ctx.effect(() => ctx.webServer.register({
556
575
  kind: 'exact',
557
576
  path: '/dsh-clinebot/v1/chat/completions',
558
577
  handler: chatHandler,
559
578
  }), 'dsh-clinebot: /v1/chat/completions')
560
579
 
561
- // 2. GET /dsh-clinebot/v1/models (OpenAI standard)
562
580
  ctx.effect(() => ctx.webServer.register({
563
581
  kind: 'exact',
564
582
  path: '/dsh-clinebot/v1/models',
565
583
  handler: modelsHandler,
566
584
  }), 'dsh-clinebot: /v1/models')
567
- }
568
-
585
+ }