@gemus/mcp-proxy 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/failFast.test.ts +254 -0
- package/src/failFast.mjs +266 -0
- package/src/proxy.mjs +38 -1
package/package.json
CHANGED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @gemus/mcp-proxy fail-fast (#2068) — single-pass upstream-disconnect correlation.
|
|
3
|
+
*
|
|
4
|
+
* Pure logic; the socket is an injected `realFetch` returning controllable `Response`s. Cases that
|
|
5
|
+
* involve the final-safety timer use fake timers with advanceTimersByTimeAsync (lesson: fake timers
|
|
6
|
+
* + real reader.read() promises need the async advance to flush continuations).
|
|
7
|
+
*/
|
|
8
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
|
9
|
+
import { parseRequestIds, isCodexOriginated, createSseIdScanner, createFailFast, isJsonRpcResponse } from '../failFast.mjs'
|
|
10
|
+
|
|
11
|
+
const enc = new TextEncoder()
|
|
12
|
+
|
|
13
|
+
/** A response body that emits the given SSE frames then closes. */
|
|
14
|
+
function bodyOf(...frames: string[]): ReadableStream<Uint8Array> {
|
|
15
|
+
return new ReadableStream<Uint8Array>({
|
|
16
|
+
start(c) {
|
|
17
|
+
for (const f of frames) c.enqueue(enc.encode(f))
|
|
18
|
+
c.close()
|
|
19
|
+
},
|
|
20
|
+
})
|
|
21
|
+
}
|
|
22
|
+
/** A response body that never emits and never closes (zombie stream). */
|
|
23
|
+
function neverBody(): ReadableStream<Uint8Array> {
|
|
24
|
+
return new ReadableStream<Uint8Array>({ start() {} })
|
|
25
|
+
}
|
|
26
|
+
function sseRes(body: ReadableStream<Uint8Array> | null, init?: { status?: number; contentType?: string }) {
|
|
27
|
+
const status = init?.status ?? 200
|
|
28
|
+
return new Response(body, { status, headers: { 'content-type': init?.contentType ?? 'text/event-stream' } })
|
|
29
|
+
}
|
|
30
|
+
function resultFrame(id: number | string): string {
|
|
31
|
+
return `event: message\ndata: ${JSON.stringify({ jsonrpc: '2.0', id, result: { ok: true } })}\n\n`
|
|
32
|
+
}
|
|
33
|
+
function post(id: number | string, method = 'tools/call') {
|
|
34
|
+
return { method: 'POST', body: JSON.stringify({ jsonrpc: '2.0', id, method, params: {} }) }
|
|
35
|
+
}
|
|
36
|
+
async function drain(res: Response): Promise<void> {
|
|
37
|
+
const r = res.body!.getReader()
|
|
38
|
+
for (;;) { const { done } = await r.read(); if (done) break }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
describe('parseRequestIds', () => {
|
|
42
|
+
it('extracts the id from a POST JSON-RPC request', () => {
|
|
43
|
+
expect(parseRequestIds(post(42))).toEqual([42])
|
|
44
|
+
})
|
|
45
|
+
it('extracts all ids from a batch', () => {
|
|
46
|
+
const body = JSON.stringify([
|
|
47
|
+
{ jsonrpc: '2.0', id: 1, method: 'a' },
|
|
48
|
+
{ jsonrpc: '2.0', id: 2, method: 'b' },
|
|
49
|
+
])
|
|
50
|
+
expect(parseRequestIds({ method: 'POST', body })).toEqual([1, 2])
|
|
51
|
+
})
|
|
52
|
+
it('returns [] for GET / DELETE / bodyless requests', () => {
|
|
53
|
+
expect(parseRequestIds({ method: 'GET' })).toEqual([])
|
|
54
|
+
expect(parseRequestIds({ method: 'DELETE' })).toEqual([])
|
|
55
|
+
expect(parseRequestIds({ method: 'POST' })).toEqual([])
|
|
56
|
+
})
|
|
57
|
+
it('returns [] for non-JSON body and for notifications (no id)', () => {
|
|
58
|
+
expect(parseRequestIds({ method: 'POST', body: 'not json' })).toEqual([])
|
|
59
|
+
expect(parseRequestIds({ method: 'POST', body: JSON.stringify({ jsonrpc: '2.0', method: 'x' }) })).toEqual([])
|
|
60
|
+
})
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
describe('isCodexOriginated', () => {
|
|
64
|
+
it.each([
|
|
65
|
+
[42, true],
|
|
66
|
+
['abc', true],
|
|
67
|
+
['gemus-bf-3', false],
|
|
68
|
+
['gemus-ping-9', false],
|
|
69
|
+
[null, false],
|
|
70
|
+
[undefined, false],
|
|
71
|
+
])('id=%s → %s', (id, expected) => {
|
|
72
|
+
expect(isCodexOriginated(id as never)).toBe(expected)
|
|
73
|
+
})
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
describe('isJsonRpcResponse', () => {
|
|
77
|
+
it('is true for a result/error response, false for a request or notification', () => {
|
|
78
|
+
expect(isJsonRpcResponse({ id: 1, result: {} })).toBe(true)
|
|
79
|
+
expect(isJsonRpcResponse({ id: 1, error: {} })).toBe(true)
|
|
80
|
+
expect(isJsonRpcResponse({ id: 1, method: 'elicitation/create', params: {} })).toBe(false) // server→client request
|
|
81
|
+
expect(isJsonRpcResponse({ method: 'notifications/x' })).toBe(false)
|
|
82
|
+
expect(isJsonRpcResponse(null)).toBe(false)
|
|
83
|
+
})
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
describe('createSseIdScanner', () => {
|
|
87
|
+
it('yields the id of a complete RESPONSE frame; ignores comment lines', () => {
|
|
88
|
+
const s = createSseIdScanner()
|
|
89
|
+
expect(s.push(enc.encode(': keepalive\n\n'))).toEqual([])
|
|
90
|
+
expect(s.push(enc.encode(resultFrame(7)))).toEqual([7])
|
|
91
|
+
})
|
|
92
|
+
it('does NOT yield a server→client REQUEST id (has a method) — avoids id-collision premature settle', () => {
|
|
93
|
+
const s = createSseIdScanner()
|
|
94
|
+
const reqFrame = `event: message\ndata: ${JSON.stringify({ jsonrpc: '2.0', id: 3, method: 'elicitation/create', params: {} })}\n\n`
|
|
95
|
+
expect(s.push(enc.encode(reqFrame))).toEqual([])
|
|
96
|
+
})
|
|
97
|
+
it('buffers a half-frame split across chunks until its boundary arrives', () => {
|
|
98
|
+
const s = createSseIdScanner()
|
|
99
|
+
const frame = resultFrame(5)
|
|
100
|
+
const mid = Math.floor(frame.length / 2)
|
|
101
|
+
expect(s.push(enc.encode(frame.slice(0, mid)))).toEqual([])
|
|
102
|
+
expect(s.push(enc.encode(frame.slice(mid)))).toEqual([5])
|
|
103
|
+
})
|
|
104
|
+
it('handles CRLF line endings (normalizes to LF)', () => {
|
|
105
|
+
const s = createSseIdScanner()
|
|
106
|
+
const crlf = `event: message\r\ndata: ${JSON.stringify({ jsonrpc: '2.0', id: 8, result: {} })}\r\n\r\n`
|
|
107
|
+
expect(s.push(enc.encode(crlf))).toEqual([8])
|
|
108
|
+
})
|
|
109
|
+
it('ignores non-JSON data payloads', () => {
|
|
110
|
+
const s = createSseIdScanner()
|
|
111
|
+
expect(s.push(enc.encode('data: not-json\n\n'))).toEqual([])
|
|
112
|
+
})
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
describe('createFailFast.fetch', () => {
|
|
116
|
+
it('injects a JSON-RPC error when the stream closes before the result', async () => {
|
|
117
|
+
const onLost = vi.fn()
|
|
118
|
+
const realFetch = vi.fn(async () => sseRes(bodyOf(': keepalive\n\n'))) // no result frame → lost
|
|
119
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
120
|
+
|
|
121
|
+
const res = await ff.fetch('u', post(42))
|
|
122
|
+
await drain(res)
|
|
123
|
+
|
|
124
|
+
expect(onLost).toHaveBeenCalledTimes(1)
|
|
125
|
+
expect(onLost.mock.calls[0][0]).toBe(42)
|
|
126
|
+
expect(ff.failedIds.has('42')).toBe(true)
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
it('does NOT inject an error when the result crosses the wire before close', async () => {
|
|
130
|
+
const onLost = vi.fn()
|
|
131
|
+
const realFetch = vi.fn(async () => sseRes(bodyOf(resultFrame(42))))
|
|
132
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
133
|
+
|
|
134
|
+
const res = await ff.fetch('u', post(42))
|
|
135
|
+
await drain(res)
|
|
136
|
+
|
|
137
|
+
expect(onLost).not.toHaveBeenCalled()
|
|
138
|
+
expect(ff.failedIds.has('42')).toBe(false)
|
|
139
|
+
})
|
|
140
|
+
|
|
141
|
+
it('fails fast immediately when realFetch throws before any response', async () => {
|
|
142
|
+
const onLost = vi.fn()
|
|
143
|
+
const realFetch = vi.fn(async () => { throw new Error('ECONNRESET') })
|
|
144
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
145
|
+
|
|
146
|
+
await expect(ff.fetch('u', post(7))).rejects.toThrow('ECONNRESET')
|
|
147
|
+
expect(onLost).toHaveBeenCalledWith(7, expect.stringContaining('upstream fetch failed'))
|
|
148
|
+
expect(realFetch).toHaveBeenCalledTimes(1) // must NOT retry — a 2nd fetch could execute the tool
|
|
149
|
+
})
|
|
150
|
+
|
|
151
|
+
it('fails fast on a non-ok HTTP response', async () => {
|
|
152
|
+
const onLost = vi.fn()
|
|
153
|
+
const realFetch = vi.fn(async () => sseRes(null, { status: 404, contentType: 'application/json' }))
|
|
154
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
155
|
+
|
|
156
|
+
await ff.fetch('u', post(7))
|
|
157
|
+
expect(onLost).toHaveBeenCalledWith(7, 'upstream HTTP 404')
|
|
158
|
+
})
|
|
159
|
+
|
|
160
|
+
it('passes GET and DELETE through untouched (no body → no monitoring, no onLost)', async () => {
|
|
161
|
+
const onLost = vi.fn()
|
|
162
|
+
const getRes = sseRes(bodyOf())
|
|
163
|
+
const realFetch = vi.fn(async () => getRes)
|
|
164
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
165
|
+
|
|
166
|
+
const g = await ff.fetch('u', { method: 'GET' })
|
|
167
|
+
const d = await ff.fetch('u', { method: 'DELETE' })
|
|
168
|
+
expect(g).toBe(getRes)
|
|
169
|
+
expect(d).toBe(getRes)
|
|
170
|
+
expect(onLost).not.toHaveBeenCalled()
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
it('does not track internal gemus-bf-* / gemus-ping-* ids', async () => {
|
|
174
|
+
const onLost = vi.fn()
|
|
175
|
+
const realFetch = vi.fn(async () => sseRes(bodyOf(': keepalive\n\n'))) // closes with no result
|
|
176
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
177
|
+
|
|
178
|
+
await drain(await ff.fetch('u', post('gemus-bf-1')))
|
|
179
|
+
await drain(await ff.fetch('u', post('gemus-ping-2')))
|
|
180
|
+
expect(onLost).not.toHaveBeenCalled()
|
|
181
|
+
})
|
|
182
|
+
|
|
183
|
+
it('degrades to plain passthrough if internal logic throws', async () => {
|
|
184
|
+
const onLost = vi.fn()
|
|
185
|
+
const badRes = { ok: true, status: 200, headers: { get() { throw new Error('boom') } }, body: null } as unknown as Response
|
|
186
|
+
const realFetch = vi.fn(async () => badRes)
|
|
187
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
188
|
+
|
|
189
|
+
const res = await ff.fetch('u', post(1))
|
|
190
|
+
expect(res).toBe(badRes) // returns the response we ALREADY have — no re-fetch
|
|
191
|
+
expect(realFetch).toHaveBeenCalledTimes(1)
|
|
192
|
+
expect(onLost).not.toHaveBeenCalled()
|
|
193
|
+
ff.dispose()
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
it('isolates concurrent requests — disconnecting one does not affect the other', async () => {
|
|
197
|
+
const onLost = vi.fn()
|
|
198
|
+
const realFetch = vi
|
|
199
|
+
.fn()
|
|
200
|
+
.mockResolvedValueOnce(sseRes(bodyOf(': keepalive\n\n'))) // id 1 → dies with no result
|
|
201
|
+
.mockResolvedValueOnce(sseRes(bodyOf(resultFrame(2)))) // id 2 → delivers
|
|
202
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
203
|
+
|
|
204
|
+
await drain(await ff.fetch('u', post(1)))
|
|
205
|
+
await drain(await ff.fetch('u', post(2)))
|
|
206
|
+
|
|
207
|
+
expect(onLost).toHaveBeenCalledTimes(1)
|
|
208
|
+
expect(onLost.mock.calls[0][0]).toBe(1)
|
|
209
|
+
})
|
|
210
|
+
|
|
211
|
+
describe('idle safety timeout', () => {
|
|
212
|
+
beforeEach(() => vi.useFakeTimers())
|
|
213
|
+
afterEach(() => vi.useRealTimers())
|
|
214
|
+
|
|
215
|
+
it('injects an error for a zombie connection that never emits a byte', async () => {
|
|
216
|
+
const onLost = vi.fn()
|
|
217
|
+
const realFetch = vi.fn(async () => sseRes(neverBody()))
|
|
218
|
+
const ff = createFailFast({ onLost, realFetch, idleTimeoutMs: 120_000 })
|
|
219
|
+
|
|
220
|
+
await ff.fetch('u', post(99)) // never emits → idle timer is never reset
|
|
221
|
+
expect(onLost).not.toHaveBeenCalled()
|
|
222
|
+
await vi.advanceTimersByTimeAsync(120_000)
|
|
223
|
+
expect(onLost).toHaveBeenCalledWith(99, expect.stringContaining('idle'))
|
|
224
|
+
ff.dispose()
|
|
225
|
+
})
|
|
226
|
+
|
|
227
|
+
it('does NOT kill a healthy-but-slow stream — heartbeat bytes reset the idle timer', async () => {
|
|
228
|
+
const onLost = vi.fn()
|
|
229
|
+
let ctl!: ReadableStreamDefaultController<Uint8Array>
|
|
230
|
+
const body = new ReadableStream<Uint8Array>({ start(c) { ctl = c } })
|
|
231
|
+
const realFetch = vi.fn(async () => sseRes(body))
|
|
232
|
+
const ff = createFailFast({ onLost, realFetch, idleTimeoutMs: 100 })
|
|
233
|
+
|
|
234
|
+
const res = await ff.fetch('u', post(1))
|
|
235
|
+
const reader = res.body!.getReader()
|
|
236
|
+
|
|
237
|
+
// Keepalive every 60ms (< 100ms idle) for 300ms total — each read touches → never fires.
|
|
238
|
+
for (let i = 0; i < 5; i++) {
|
|
239
|
+
ctl.enqueue(enc.encode(': keepalive\n\n'))
|
|
240
|
+
await reader.read()
|
|
241
|
+
await vi.advanceTimersByTimeAsync(60)
|
|
242
|
+
}
|
|
243
|
+
expect(onLost).not.toHaveBeenCalled() // survived 300ms > idleTimeout because bytes kept flowing
|
|
244
|
+
|
|
245
|
+
// Now the result arrives and the stream closes → still no fail-fast.
|
|
246
|
+
ctl.enqueue(enc.encode(resultFrame(1)))
|
|
247
|
+
await reader.read()
|
|
248
|
+
ctl.close()
|
|
249
|
+
await reader.read()
|
|
250
|
+
expect(onLost).not.toHaveBeenCalled()
|
|
251
|
+
ff.dispose()
|
|
252
|
+
})
|
|
253
|
+
})
|
|
254
|
+
})
|
package/src/failFast.mjs
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
// @gemus/mcp-proxy fail-fast (#2068) — surface upstream stream death to Codex immediately.
|
|
2
|
+
//
|
|
3
|
+
// StreamableHTTPClientTransport.send() resolves as soon as it DISPATCHES the SSE read loop,
|
|
4
|
+
// not when the tool result arrives. When the upstream POST SSE body dies before the result,
|
|
5
|
+
// the SDK only fires a global `onerror` with NO request id, and never reconnects a POST (no
|
|
6
|
+
// priming event). So Codex's pending request never settles → it hangs to its own 300s timeout.
|
|
7
|
+
//
|
|
8
|
+
// The only interception point the SDK offers is the constructor `fetch` option: it receives the
|
|
9
|
+
// exact (url, init) per request, so we parse the JSON-RPC id from `init.body` and wrap the
|
|
10
|
+
// response body. We use a SINGLE-PASS observe-while-forward wrapper (not tee()+grace): the same
|
|
11
|
+
// read loop that forwards bytes to the transport also scans the SSE frames for the tracked id's
|
|
12
|
+
// result/error. "Detect order == forward order" on one reader, so there is NO race between "result
|
|
13
|
+
// delivered" and "stream closed" — no wall-clock grace window (which CLAUDE.md forbids). If the
|
|
14
|
+
// stream reaches done/error while a tracked id was never seen on the wire, that result is lost →
|
|
15
|
+
// inject a JSON-RPC error keyed to that id.
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* A Codex-originated id is anything the AGENT issued. We exclude our own internal namespaces:
|
|
19
|
+
* - `gemus-bf-*` footprint-0 backfill injections (self-consumed by upstream.onmessage)
|
|
20
|
+
* - `gemus-ping-*` liveness heartbeat pongs (self-consumed)
|
|
21
|
+
* Numeric ids (Codex initialize / tools/list / tools/call) are all tracked.
|
|
22
|
+
*/
|
|
23
|
+
export function isCodexOriginated(id) {
|
|
24
|
+
if (id === undefined || id === null) return false
|
|
25
|
+
return !(typeof id === 'string' && (id.startsWith('gemus-bf-') || id.startsWith('gemus-ping-')))
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Extract JSON-RPC request ids from a fetch init. Only POST requests with a JSON body carry a
|
|
30
|
+
* request whose response we must guarantee; GET (standalone SSE) and DELETE (terminateSession)
|
|
31
|
+
* have no body → []. Never throws — a body we can't parse yields [].
|
|
32
|
+
*/
|
|
33
|
+
export function parseRequestIds(init) {
|
|
34
|
+
try {
|
|
35
|
+
const method = (init?.method || 'GET').toUpperCase()
|
|
36
|
+
if (method !== 'POST' || init.body == null) return []
|
|
37
|
+
const body = typeof init.body === 'string' ? init.body : String(init.body)
|
|
38
|
+
const parsed = JSON.parse(body)
|
|
39
|
+
const msgs = Array.isArray(parsed) ? parsed : [parsed]
|
|
40
|
+
const ids = []
|
|
41
|
+
for (const m of msgs) {
|
|
42
|
+
// A request has both an id and a method; responses/notifications are not our concern here.
|
|
43
|
+
if (m && m.id !== undefined && m.id !== null && typeof m.method === 'string') ids.push(m.id)
|
|
44
|
+
}
|
|
45
|
+
return ids
|
|
46
|
+
} catch {
|
|
47
|
+
return []
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** True for a JSON-RPC RESPONSE (result/error) — has an id and no `method`. A server→client REQUEST
|
|
52
|
+
* (elicitation/sampling) also has an id but carries a `method`, and its id (the server's own counter)
|
|
53
|
+
* can COLLIDE with an in-flight Codex tools/call id — settling on it would disarm the fail-fast guard
|
|
54
|
+
* for the very tool that stream carries. So we settle only on responses, never on relayed requests. */
|
|
55
|
+
export function isJsonRpcResponse(msg) {
|
|
56
|
+
return !!msg && msg.id !== undefined && msg.id !== null && msg.method === undefined
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Minimal SSE frame scanner. Accumulates decoded text, splits on the event boundary, and for each
|
|
61
|
+
* complete event yields the JSON-RPC id of any `data:` payload that is a RESPONSE (see
|
|
62
|
+
* isJsonRpcResponse). Comment lines (`:` — including our injected `: keepalive`) are ignored, so
|
|
63
|
+
* they never count as a delivered result. Half-frames straddling chunk boundaries are buffered until
|
|
64
|
+
* their terminating blank line arrives (no early/missed detection). CRLF line endings are normalized
|
|
65
|
+
* to LF so an intermediary that rewrites them can't wedge the scanner.
|
|
66
|
+
*/
|
|
67
|
+
export function createSseIdScanner() {
|
|
68
|
+
const decoder = new TextDecoder()
|
|
69
|
+
let buf = ''
|
|
70
|
+
return {
|
|
71
|
+
/** Push a chunk; returns the JSON-RPC ids of any RESPONSE frames that crossed the wire in it. */
|
|
72
|
+
push(chunk) {
|
|
73
|
+
buf += decoder.decode(chunk, { stream: true }).replace(/\r\n/g, '\n')
|
|
74
|
+
const ids = []
|
|
75
|
+
let sep
|
|
76
|
+
while ((sep = buf.indexOf('\n\n')) !== -1) {
|
|
77
|
+
const block = buf.slice(0, sep)
|
|
78
|
+
buf = buf.slice(sep + 2)
|
|
79
|
+
const dataLines = []
|
|
80
|
+
for (const line of block.split('\n')) {
|
|
81
|
+
if (line.startsWith('data:')) dataLines.push(line.slice(5).trimStart())
|
|
82
|
+
}
|
|
83
|
+
if (!dataLines.length) continue
|
|
84
|
+
try {
|
|
85
|
+
const msg = JSON.parse(dataLines.join('\n'))
|
|
86
|
+
if (isJsonRpcResponse(msg)) ids.push(msg.id)
|
|
87
|
+
} catch {
|
|
88
|
+
/* not JSON-RPC (or partial) — ignore */
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return ids
|
|
92
|
+
},
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Build the fail-fast fetch wrapper + settle/dispose controls.
|
|
98
|
+
*
|
|
99
|
+
* @param {object} opts
|
|
100
|
+
* @param {(id:any, reason:string)=>void} opts.onLost called once per lost id (must be idempotent-safe on caller side)
|
|
101
|
+
* @param {number} [opts.idleTimeoutMs] IDLE backstop reset on every upstream byte (incl. server
|
|
102
|
+
* heartbeats every ~12s), so it trips ONLY on a genuinely silent/zombie connection — never on a
|
|
103
|
+
* healthy-but-slow tool. Must be < Codex 300s; default 120_000.
|
|
104
|
+
* @param {(dir:string, m:any)=>void} [opts.log]
|
|
105
|
+
* @param {typeof fetch} [opts.realFetch] injectable for tests; defaults to global fetch
|
|
106
|
+
* @returns {{ fetch, settle, touch, forget, dispose, failedIds }}
|
|
107
|
+
*/
|
|
108
|
+
export function createFailFast(opts) {
|
|
109
|
+
const { onLost, log = () => {} } = opts
|
|
110
|
+
const idleTimeoutMs = Number(opts.idleTimeoutMs) || 120_000
|
|
111
|
+
const realFetch = opts.realFetch || fetch
|
|
112
|
+
const FAILED_IDS_CAP = 512
|
|
113
|
+
|
|
114
|
+
/** idStr → { timer } — Codex-originated in-flight requests awaiting a result. */
|
|
115
|
+
const inFlight = new Map()
|
|
116
|
+
/** ids already failed by onLost — lets onmessage drop a late real result's forward (still taps it). */
|
|
117
|
+
const failedIds = new Set()
|
|
118
|
+
|
|
119
|
+
function armTimer(entry, id) {
|
|
120
|
+
entry.timer = setTimeout(() => lose(id, 'proxy idle timeout (no upstream bytes)'), idleTimeoutMs)
|
|
121
|
+
if (typeof entry.timer.unref === 'function') entry.timer.unref()
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function register(id) {
|
|
125
|
+
const key = String(id)
|
|
126
|
+
if (inFlight.has(key)) return
|
|
127
|
+
const entry = {}
|
|
128
|
+
inFlight.set(key, entry)
|
|
129
|
+
armTimer(entry, id)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Reset the idle timer on any upstream byte — a healthy stream (heartbeats + data) never trips it. */
|
|
133
|
+
function touch(id) {
|
|
134
|
+
const entry = inFlight.get(String(id))
|
|
135
|
+
if (!entry) return
|
|
136
|
+
clearTimeout(entry.timer)
|
|
137
|
+
armTimer(entry, id)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function settle(id) {
|
|
141
|
+
const key = String(id)
|
|
142
|
+
const entry = inFlight.get(key)
|
|
143
|
+
if (!entry) return
|
|
144
|
+
clearTimeout(entry.timer)
|
|
145
|
+
inFlight.delete(key)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Drop an id from failedIds once its late result has been handled (bounds the set). */
|
|
149
|
+
function forget(id) {
|
|
150
|
+
failedIds.delete(String(id))
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function lose(id, reason) {
|
|
154
|
+
const key = String(id)
|
|
155
|
+
if (!inFlight.has(key)) return // already settled or already lost
|
|
156
|
+
settle(id)
|
|
157
|
+
failedIds.add(key)
|
|
158
|
+
// Bound the set: disconnect-case entries never see a late result to forget(), so cap FIFO.
|
|
159
|
+
if (failedIds.size > FAILED_IDS_CAP) failedIds.delete(failedIds.values().next().value)
|
|
160
|
+
log('FAILFAST', { id, reason })
|
|
161
|
+
onLost(id, reason)
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function observeAndForward(body, codexIds) {
|
|
165
|
+
const reader = body.getReader()
|
|
166
|
+
const scanner = createSseIdScanner()
|
|
167
|
+
let ended = false
|
|
168
|
+
const onEnd = () => {
|
|
169
|
+
if (ended) return
|
|
170
|
+
ended = true
|
|
171
|
+
for (const id of codexIds) lose(id, 'upstream stream closed before result')
|
|
172
|
+
}
|
|
173
|
+
return new ReadableStream({
|
|
174
|
+
async pull(controller) {
|
|
175
|
+
let r
|
|
176
|
+
try {
|
|
177
|
+
r = await reader.read()
|
|
178
|
+
} catch (e) {
|
|
179
|
+
try {
|
|
180
|
+
controller.error(e)
|
|
181
|
+
} catch {
|
|
182
|
+
/* already closed */
|
|
183
|
+
}
|
|
184
|
+
onEnd()
|
|
185
|
+
return
|
|
186
|
+
}
|
|
187
|
+
if (r.done) {
|
|
188
|
+
try {
|
|
189
|
+
controller.close()
|
|
190
|
+
} catch {
|
|
191
|
+
/* already closed */
|
|
192
|
+
}
|
|
193
|
+
onEnd()
|
|
194
|
+
return
|
|
195
|
+
}
|
|
196
|
+
try {
|
|
197
|
+
controller.enqueue(r.value) // forward bytes unchanged to the transport
|
|
198
|
+
} catch {
|
|
199
|
+
/* downstream gone */
|
|
200
|
+
}
|
|
201
|
+
// Any byte (data OR heartbeat) proves the stream is alive → reset the idle backstop.
|
|
202
|
+
for (const id of codexIds) touch(id)
|
|
203
|
+
// Same loop: any tracked id whose RESPONSE crossed the wire is now delivered.
|
|
204
|
+
for (const id of scanner.push(r.value)) settle(id)
|
|
205
|
+
},
|
|
206
|
+
cancel(reason) {
|
|
207
|
+
reader.cancel(reason).catch(() => {})
|
|
208
|
+
onEnd()
|
|
209
|
+
},
|
|
210
|
+
})
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
async function wrappedFetch(url, init) {
|
|
214
|
+
// Parse ids + register. Both are internally safe (parseRequestIds never throws), but guard
|
|
215
|
+
// defensively so a future change can't break the chain — degrade to "no tracking".
|
|
216
|
+
let codexIds = []
|
|
217
|
+
try {
|
|
218
|
+
codexIds = parseRequestIds(init).filter(isCodexOriginated)
|
|
219
|
+
for (const id of codexIds) register(id)
|
|
220
|
+
} catch (e) {
|
|
221
|
+
log('FAILFAST-WRAP-ERR', String(e))
|
|
222
|
+
codexIds = []
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// The network call. Its failure MUST propagate to the SDK unchanged — do NOT retry here
|
|
226
|
+
// (a second fetch could execute the tool server-side after Codex already got the error).
|
|
227
|
+
let res
|
|
228
|
+
try {
|
|
229
|
+
res = await realFetch(url, init)
|
|
230
|
+
} catch (e) {
|
|
231
|
+
// Connection failed before any response (DNS / TLS / RST) — fail fast, don't wait 270s.
|
|
232
|
+
for (const id of codexIds) lose(id, `upstream fetch failed: ${e}`)
|
|
233
|
+
throw e // preserve the SDK's existing onerror/logging path (no re-fetch)
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// Response handling + monitoring. On any internal bug, return the response we ALREADY have,
|
|
237
|
+
// unmonitored — never re-fetch (wrappedFetch sits on 100% of upstream traffic incl. GET
|
|
238
|
+
// elicitation + DELETE teardown, so it must never break the chain, but also never duplicate).
|
|
239
|
+
try {
|
|
240
|
+
if (!codexIds.length) return res // GET / DELETE / notification / internal id → passthrough
|
|
241
|
+
if (!res.ok) {
|
|
242
|
+
// Non-SSE HTTP error (404 session-gone / 403 / 429 / …). send() throws before onmessage,
|
|
243
|
+
// and that rejection is swallowed by upstream.send().catch — so nothing ever settles the id.
|
|
244
|
+
for (const id of codexIds) lose(id, `upstream HTTP ${res.status}`)
|
|
245
|
+
return res
|
|
246
|
+
}
|
|
247
|
+
const ct = res.headers.get('content-type') || ''
|
|
248
|
+
if (!ct.includes('text/event-stream') || !res.body) return res // 202 etc. — no id to monitor
|
|
249
|
+
return new Response(observeAndForward(res.body, codexIds), {
|
|
250
|
+
status: res.status,
|
|
251
|
+
statusText: res.statusText,
|
|
252
|
+
headers: res.headers,
|
|
253
|
+
})
|
|
254
|
+
} catch (e) {
|
|
255
|
+
log('FAILFAST-WRAP-ERR', String(e))
|
|
256
|
+
return res
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function dispose() {
|
|
261
|
+
for (const { timer } of inFlight.values()) clearTimeout(timer)
|
|
262
|
+
inFlight.clear()
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
return { fetch: wrappedFetch, settle, touch, forget, dispose, failedIds }
|
|
266
|
+
}
|
package/src/proxy.mjs
CHANGED
|
@@ -26,6 +26,7 @@ import { randomBytes } from 'node:crypto'
|
|
|
26
26
|
import { createBackfiller, injectSteering } from './backfill.mjs'
|
|
27
27
|
import { unwrapMcpContent } from '@gemus/codex-backfill-core'
|
|
28
28
|
import { buildProxyUpstreamHeaders } from './mcpHeaders.mjs'
|
|
29
|
+
import { createFailFast, isJsonRpcResponse } from './failFast.mjs'
|
|
29
30
|
|
|
30
31
|
const KEY = process.env.GEMUS_KEY || ''
|
|
31
32
|
const GEMUS_URL = process.env.GEMUS_URL || 'https://gemus.ai/api/mcp'
|
|
@@ -244,10 +245,33 @@ hookServer.listen(0, '127.0.0.1', () => {
|
|
|
244
245
|
if (typeof hookServer.unref === 'function') hookServer.unref()
|
|
245
246
|
|
|
246
247
|
// ── transparent JSON-RPC passthrough (raw send/onmessage) ──────────────────────────────────────
|
|
248
|
+
const stdio = new StdioServerTransport()
|
|
249
|
+
|
|
250
|
+
// #2068 fail-fast: StreamableHTTPClientTransport.send() resolves before the upstream SSE result
|
|
251
|
+
// arrives and only surfaces a stream death via a global, id-less `onerror` (no reconnect for POST).
|
|
252
|
+
// The custom `fetch` wrapper parses each request's JSON-RPC id and, if the response stream dies
|
|
253
|
+
// before that id's result crosses the wire, injects a JSON-RPC error so Codex fails in <2s instead
|
|
254
|
+
// of hanging to its 300s timeout. onLost only injects to stdio — it does NOT tap: tapping here would
|
|
255
|
+
// delete the proxy's `inflight` entry, so the DROP-AFTER-FAILFAST branch below could no longer let
|
|
256
|
+
// backfill observe a late real result (env delivery on the still-open, idle-timed-out case).
|
|
257
|
+
const failFast = createFailFast({
|
|
258
|
+
idleTimeoutMs: Number(process.env.PROXY_FAILFAST_TIMEOUT_MS) || 120_000,
|
|
259
|
+
log,
|
|
260
|
+
onLost: (id, reason) => {
|
|
261
|
+
const errObj = {
|
|
262
|
+
jsonrpc: '2.0',
|
|
263
|
+
id,
|
|
264
|
+
error: { code: -32000, message: 'Gemus upstream stream disconnected before the tool result was delivered' },
|
|
265
|
+
}
|
|
266
|
+
log('FAILFAST-INJECT', { id, reason })
|
|
267
|
+
stdio.send(errObj).catch((e) => log('ERR-stdio-send', String(e)))
|
|
268
|
+
},
|
|
269
|
+
})
|
|
270
|
+
|
|
247
271
|
const upstream = new StreamableHTTPClientTransport(new URL(GEMUS_URL), {
|
|
248
272
|
requestInit: { headers: buildProxyUpstreamHeaders(KEY) },
|
|
273
|
+
fetch: failFast.fetch,
|
|
249
274
|
})
|
|
250
|
-
const stdio = new StdioServerTransport()
|
|
251
275
|
|
|
252
276
|
stdio.onmessage = (m) => { tapClientToServer(m); log('C->G', m); upstream.send(m).catch((e) => log('ERR-http-send', String(e))) }
|
|
253
277
|
upstream.onmessage = (m) => {
|
|
@@ -262,6 +286,18 @@ upstream.onmessage = (m) => {
|
|
|
262
286
|
// #1756 liveness heartbeat: our own `ping` pongs (gemus-ping-*) are self-consumed — Codex never
|
|
263
287
|
// issued them, so it must never see the responses.
|
|
264
288
|
if (typeof m?.id === 'string' && m.id.startsWith('gemus-ping-')) return
|
|
289
|
+
// #2068 fail-fast bookkeeping. Gate on RESPONSE only (isJsonRpcResponse) — a server→client request
|
|
290
|
+
// (elicitation/sampling) also carries an id that can collide with an in-flight tools/call id, and
|
|
291
|
+
// must be forwarded to Codex untouched, never settled/dropped.
|
|
292
|
+
if (isJsonRpcResponse(m)) {
|
|
293
|
+
if (failFast.failedIds.has(String(m.id))) {
|
|
294
|
+
// Already fail-fasted (idle timeout on a still-open stream) → Codex got the error. Still tap so
|
|
295
|
+
// backfill (#1751) observes the late real result (inflight survives — onLost didn't tap), but
|
|
296
|
+
// do NOT re-forward it to Codex (no double-send). forget() bounds failedIds.
|
|
297
|
+
tapServerToClient(m); failFast.forget(m.id); log('DROP-AFTER-FAILFAST', m.id); return
|
|
298
|
+
}
|
|
299
|
+
failFast.settle(m.id)
|
|
300
|
+
}
|
|
265
301
|
tapServerToClient(m); steerOnInit(m); log('G->C', m); stdio.send(m).catch((e) => log('ERR-stdio-send', String(e)))
|
|
266
302
|
}
|
|
267
303
|
|
|
@@ -293,6 +329,7 @@ async function shutdown() {
|
|
|
293
329
|
// Settle any in-flight footprint-0 injections so a mid-delivery shutdown doesn't hang their awaits.
|
|
294
330
|
for (const p of pendingInjections.values()) p.settle({ ok: false, error: 'proxy shutting down' })
|
|
295
331
|
pendingInjections.clear()
|
|
332
|
+
failFast.dispose() // #2068: clear fail-fast final-safety timers
|
|
296
333
|
try { hookServer.close() } catch { /* ignore */ }
|
|
297
334
|
if (discoveryFile) { try { fs.unlinkSync(discoveryFile) } catch { /* ignore */ } }
|
|
298
335
|
try { await upstream.terminateSession() } catch { /* ignore */ }
|