@gemus/mcp-proxy 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/failFast.test.ts +254 -0
- package/src/__tests__/mcpHeaders.test.ts +22 -1
- package/src/failFast.mjs +266 -0
- package/src/mcpHeaders.mjs +28 -1
- package/src/proxy.mjs +38 -1
package/package.json
CHANGED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @gemus/mcp-proxy fail-fast (#2068) — single-pass upstream-disconnect correlation.
|
|
3
|
+
*
|
|
4
|
+
* Pure logic; the socket is an injected `realFetch` returning controllable `Response`s. Cases that
|
|
5
|
+
* involve the final-safety timer use fake timers with advanceTimersByTimeAsync (lesson: fake timers
|
|
6
|
+
* + real reader.read() promises need the async advance to flush continuations).
|
|
7
|
+
*/
|
|
8
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
|
9
|
+
import { parseRequestIds, isCodexOriginated, createSseIdScanner, createFailFast, isJsonRpcResponse } from '../failFast.mjs'
|
|
10
|
+
|
|
11
|
+
const enc = new TextEncoder()
|
|
12
|
+
|
|
13
|
+
/** A response body that emits the given SSE frames then closes. */
|
|
14
|
+
function bodyOf(...frames: string[]): ReadableStream<Uint8Array> {
|
|
15
|
+
return new ReadableStream<Uint8Array>({
|
|
16
|
+
start(c) {
|
|
17
|
+
for (const f of frames) c.enqueue(enc.encode(f))
|
|
18
|
+
c.close()
|
|
19
|
+
},
|
|
20
|
+
})
|
|
21
|
+
}
|
|
22
|
+
/** A response body that never emits and never closes (zombie stream). */
|
|
23
|
+
function neverBody(): ReadableStream<Uint8Array> {
|
|
24
|
+
return new ReadableStream<Uint8Array>({ start() {} })
|
|
25
|
+
}
|
|
26
|
+
function sseRes(body: ReadableStream<Uint8Array> | null, init?: { status?: number; contentType?: string }) {
|
|
27
|
+
const status = init?.status ?? 200
|
|
28
|
+
return new Response(body, { status, headers: { 'content-type': init?.contentType ?? 'text/event-stream' } })
|
|
29
|
+
}
|
|
30
|
+
function resultFrame(id: number | string): string {
|
|
31
|
+
return `event: message\ndata: ${JSON.stringify({ jsonrpc: '2.0', id, result: { ok: true } })}\n\n`
|
|
32
|
+
}
|
|
33
|
+
function post(id: number | string, method = 'tools/call') {
|
|
34
|
+
return { method: 'POST', body: JSON.stringify({ jsonrpc: '2.0', id, method, params: {} }) }
|
|
35
|
+
}
|
|
36
|
+
async function drain(res: Response): Promise<void> {
|
|
37
|
+
const r = res.body!.getReader()
|
|
38
|
+
for (;;) { const { done } = await r.read(); if (done) break }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
describe('parseRequestIds', () => {
|
|
42
|
+
it('extracts the id from a POST JSON-RPC request', () => {
|
|
43
|
+
expect(parseRequestIds(post(42))).toEqual([42])
|
|
44
|
+
})
|
|
45
|
+
it('extracts all ids from a batch', () => {
|
|
46
|
+
const body = JSON.stringify([
|
|
47
|
+
{ jsonrpc: '2.0', id: 1, method: 'a' },
|
|
48
|
+
{ jsonrpc: '2.0', id: 2, method: 'b' },
|
|
49
|
+
])
|
|
50
|
+
expect(parseRequestIds({ method: 'POST', body })).toEqual([1, 2])
|
|
51
|
+
})
|
|
52
|
+
it('returns [] for GET / DELETE / bodyless requests', () => {
|
|
53
|
+
expect(parseRequestIds({ method: 'GET' })).toEqual([])
|
|
54
|
+
expect(parseRequestIds({ method: 'DELETE' })).toEqual([])
|
|
55
|
+
expect(parseRequestIds({ method: 'POST' })).toEqual([])
|
|
56
|
+
})
|
|
57
|
+
it('returns [] for non-JSON body and for notifications (no id)', () => {
|
|
58
|
+
expect(parseRequestIds({ method: 'POST', body: 'not json' })).toEqual([])
|
|
59
|
+
expect(parseRequestIds({ method: 'POST', body: JSON.stringify({ jsonrpc: '2.0', method: 'x' }) })).toEqual([])
|
|
60
|
+
})
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
describe('isCodexOriginated', () => {
|
|
64
|
+
it.each([
|
|
65
|
+
[42, true],
|
|
66
|
+
['abc', true],
|
|
67
|
+
['gemus-bf-3', false],
|
|
68
|
+
['gemus-ping-9', false],
|
|
69
|
+
[null, false],
|
|
70
|
+
[undefined, false],
|
|
71
|
+
])('id=%s → %s', (id, expected) => {
|
|
72
|
+
expect(isCodexOriginated(id as never)).toBe(expected)
|
|
73
|
+
})
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
describe('isJsonRpcResponse', () => {
|
|
77
|
+
it('is true for a result/error response, false for a request or notification', () => {
|
|
78
|
+
expect(isJsonRpcResponse({ id: 1, result: {} })).toBe(true)
|
|
79
|
+
expect(isJsonRpcResponse({ id: 1, error: {} })).toBe(true)
|
|
80
|
+
expect(isJsonRpcResponse({ id: 1, method: 'elicitation/create', params: {} })).toBe(false) // server→client request
|
|
81
|
+
expect(isJsonRpcResponse({ method: 'notifications/x' })).toBe(false)
|
|
82
|
+
expect(isJsonRpcResponse(null)).toBe(false)
|
|
83
|
+
})
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
describe('createSseIdScanner', () => {
|
|
87
|
+
it('yields the id of a complete RESPONSE frame; ignores comment lines', () => {
|
|
88
|
+
const s = createSseIdScanner()
|
|
89
|
+
expect(s.push(enc.encode(': keepalive\n\n'))).toEqual([])
|
|
90
|
+
expect(s.push(enc.encode(resultFrame(7)))).toEqual([7])
|
|
91
|
+
})
|
|
92
|
+
it('does NOT yield a server→client REQUEST id (has a method) — avoids id-collision premature settle', () => {
|
|
93
|
+
const s = createSseIdScanner()
|
|
94
|
+
const reqFrame = `event: message\ndata: ${JSON.stringify({ jsonrpc: '2.0', id: 3, method: 'elicitation/create', params: {} })}\n\n`
|
|
95
|
+
expect(s.push(enc.encode(reqFrame))).toEqual([])
|
|
96
|
+
})
|
|
97
|
+
it('buffers a half-frame split across chunks until its boundary arrives', () => {
|
|
98
|
+
const s = createSseIdScanner()
|
|
99
|
+
const frame = resultFrame(5)
|
|
100
|
+
const mid = Math.floor(frame.length / 2)
|
|
101
|
+
expect(s.push(enc.encode(frame.slice(0, mid)))).toEqual([])
|
|
102
|
+
expect(s.push(enc.encode(frame.slice(mid)))).toEqual([5])
|
|
103
|
+
})
|
|
104
|
+
it('handles CRLF line endings (normalizes to LF)', () => {
|
|
105
|
+
const s = createSseIdScanner()
|
|
106
|
+
const crlf = `event: message\r\ndata: ${JSON.stringify({ jsonrpc: '2.0', id: 8, result: {} })}\r\n\r\n`
|
|
107
|
+
expect(s.push(enc.encode(crlf))).toEqual([8])
|
|
108
|
+
})
|
|
109
|
+
it('ignores non-JSON data payloads', () => {
|
|
110
|
+
const s = createSseIdScanner()
|
|
111
|
+
expect(s.push(enc.encode('data: not-json\n\n'))).toEqual([])
|
|
112
|
+
})
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
describe('createFailFast.fetch', () => {
|
|
116
|
+
it('injects a JSON-RPC error when the stream closes before the result', async () => {
|
|
117
|
+
const onLost = vi.fn()
|
|
118
|
+
const realFetch = vi.fn(async () => sseRes(bodyOf(': keepalive\n\n'))) // no result frame → lost
|
|
119
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
120
|
+
|
|
121
|
+
const res = await ff.fetch('u', post(42))
|
|
122
|
+
await drain(res)
|
|
123
|
+
|
|
124
|
+
expect(onLost).toHaveBeenCalledTimes(1)
|
|
125
|
+
expect(onLost.mock.calls[0][0]).toBe(42)
|
|
126
|
+
expect(ff.failedIds.has('42')).toBe(true)
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
it('does NOT inject an error when the result crosses the wire before close', async () => {
|
|
130
|
+
const onLost = vi.fn()
|
|
131
|
+
const realFetch = vi.fn(async () => sseRes(bodyOf(resultFrame(42))))
|
|
132
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
133
|
+
|
|
134
|
+
const res = await ff.fetch('u', post(42))
|
|
135
|
+
await drain(res)
|
|
136
|
+
|
|
137
|
+
expect(onLost).not.toHaveBeenCalled()
|
|
138
|
+
expect(ff.failedIds.has('42')).toBe(false)
|
|
139
|
+
})
|
|
140
|
+
|
|
141
|
+
it('fails fast immediately when realFetch throws before any response', async () => {
|
|
142
|
+
const onLost = vi.fn()
|
|
143
|
+
const realFetch = vi.fn(async () => { throw new Error('ECONNRESET') })
|
|
144
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
145
|
+
|
|
146
|
+
await expect(ff.fetch('u', post(7))).rejects.toThrow('ECONNRESET')
|
|
147
|
+
expect(onLost).toHaveBeenCalledWith(7, expect.stringContaining('upstream fetch failed'))
|
|
148
|
+
expect(realFetch).toHaveBeenCalledTimes(1) // must NOT retry — a 2nd fetch could execute the tool
|
|
149
|
+
})
|
|
150
|
+
|
|
151
|
+
it('fails fast on a non-ok HTTP response', async () => {
|
|
152
|
+
const onLost = vi.fn()
|
|
153
|
+
const realFetch = vi.fn(async () => sseRes(null, { status: 404, contentType: 'application/json' }))
|
|
154
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
155
|
+
|
|
156
|
+
await ff.fetch('u', post(7))
|
|
157
|
+
expect(onLost).toHaveBeenCalledWith(7, 'upstream HTTP 404')
|
|
158
|
+
})
|
|
159
|
+
|
|
160
|
+
it('passes GET and DELETE through untouched (no body → no monitoring, no onLost)', async () => {
|
|
161
|
+
const onLost = vi.fn()
|
|
162
|
+
const getRes = sseRes(bodyOf())
|
|
163
|
+
const realFetch = vi.fn(async () => getRes)
|
|
164
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
165
|
+
|
|
166
|
+
const g = await ff.fetch('u', { method: 'GET' })
|
|
167
|
+
const d = await ff.fetch('u', { method: 'DELETE' })
|
|
168
|
+
expect(g).toBe(getRes)
|
|
169
|
+
expect(d).toBe(getRes)
|
|
170
|
+
expect(onLost).not.toHaveBeenCalled()
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
it('does not track internal gemus-bf-* / gemus-ping-* ids', async () => {
|
|
174
|
+
const onLost = vi.fn()
|
|
175
|
+
const realFetch = vi.fn(async () => sseRes(bodyOf(': keepalive\n\n'))) // closes with no result
|
|
176
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
177
|
+
|
|
178
|
+
await drain(await ff.fetch('u', post('gemus-bf-1')))
|
|
179
|
+
await drain(await ff.fetch('u', post('gemus-ping-2')))
|
|
180
|
+
expect(onLost).not.toHaveBeenCalled()
|
|
181
|
+
})
|
|
182
|
+
|
|
183
|
+
it('degrades to plain passthrough if internal logic throws', async () => {
|
|
184
|
+
const onLost = vi.fn()
|
|
185
|
+
const badRes = { ok: true, status: 200, headers: { get() { throw new Error('boom') } }, body: null } as unknown as Response
|
|
186
|
+
const realFetch = vi.fn(async () => badRes)
|
|
187
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
188
|
+
|
|
189
|
+
const res = await ff.fetch('u', post(1))
|
|
190
|
+
expect(res).toBe(badRes) // returns the response we ALREADY have — no re-fetch
|
|
191
|
+
expect(realFetch).toHaveBeenCalledTimes(1)
|
|
192
|
+
expect(onLost).not.toHaveBeenCalled()
|
|
193
|
+
ff.dispose()
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
it('isolates concurrent requests — disconnecting one does not affect the other', async () => {
|
|
197
|
+
const onLost = vi.fn()
|
|
198
|
+
const realFetch = vi
|
|
199
|
+
.fn()
|
|
200
|
+
.mockResolvedValueOnce(sseRes(bodyOf(': keepalive\n\n'))) // id 1 → dies with no result
|
|
201
|
+
.mockResolvedValueOnce(sseRes(bodyOf(resultFrame(2)))) // id 2 → delivers
|
|
202
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
203
|
+
|
|
204
|
+
await drain(await ff.fetch('u', post(1)))
|
|
205
|
+
await drain(await ff.fetch('u', post(2)))
|
|
206
|
+
|
|
207
|
+
expect(onLost).toHaveBeenCalledTimes(1)
|
|
208
|
+
expect(onLost.mock.calls[0][0]).toBe(1)
|
|
209
|
+
})
|
|
210
|
+
|
|
211
|
+
describe('idle safety timeout', () => {
|
|
212
|
+
beforeEach(() => vi.useFakeTimers())
|
|
213
|
+
afterEach(() => vi.useRealTimers())
|
|
214
|
+
|
|
215
|
+
it('injects an error for a zombie connection that never emits a byte', async () => {
|
|
216
|
+
const onLost = vi.fn()
|
|
217
|
+
const realFetch = vi.fn(async () => sseRes(neverBody()))
|
|
218
|
+
const ff = createFailFast({ onLost, realFetch, idleTimeoutMs: 120_000 })
|
|
219
|
+
|
|
220
|
+
await ff.fetch('u', post(99)) // never emits → idle timer is never reset
|
|
221
|
+
expect(onLost).not.toHaveBeenCalled()
|
|
222
|
+
await vi.advanceTimersByTimeAsync(120_000)
|
|
223
|
+
expect(onLost).toHaveBeenCalledWith(99, expect.stringContaining('idle'))
|
|
224
|
+
ff.dispose()
|
|
225
|
+
})
|
|
226
|
+
|
|
227
|
+
it('does NOT kill a healthy-but-slow stream — heartbeat bytes reset the idle timer', async () => {
|
|
228
|
+
const onLost = vi.fn()
|
|
229
|
+
let ctl!: ReadableStreamDefaultController<Uint8Array>
|
|
230
|
+
const body = new ReadableStream<Uint8Array>({ start(c) { ctl = c } })
|
|
231
|
+
const realFetch = vi.fn(async () => sseRes(body))
|
|
232
|
+
const ff = createFailFast({ onLost, realFetch, idleTimeoutMs: 100 })
|
|
233
|
+
|
|
234
|
+
const res = await ff.fetch('u', post(1))
|
|
235
|
+
const reader = res.body!.getReader()
|
|
236
|
+
|
|
237
|
+
// Keepalive every 60ms (< 100ms idle) for 300ms total — each read touches → never fires.
|
|
238
|
+
for (let i = 0; i < 5; i++) {
|
|
239
|
+
ctl.enqueue(enc.encode(': keepalive\n\n'))
|
|
240
|
+
await reader.read()
|
|
241
|
+
await vi.advanceTimersByTimeAsync(60)
|
|
242
|
+
}
|
|
243
|
+
expect(onLost).not.toHaveBeenCalled() // survived 300ms > idleTimeout because bytes kept flowing
|
|
244
|
+
|
|
245
|
+
// Now the result arrives and the stream closes → still no fail-fast.
|
|
246
|
+
ctl.enqueue(enc.encode(resultFrame(1)))
|
|
247
|
+
await reader.read()
|
|
248
|
+
ctl.close()
|
|
249
|
+
await reader.read()
|
|
250
|
+
expect(onLost).not.toHaveBeenCalled()
|
|
251
|
+
ff.dispose()
|
|
252
|
+
})
|
|
253
|
+
})
|
|
254
|
+
})
|
|
@@ -1,11 +1,32 @@
|
|
|
1
1
|
import { describe, expect, it } from 'vitest'
|
|
2
|
+
import { CLIENT_CAPABILITIES } from '@gemus/codex-backfill-core'
|
|
2
3
|
import { buildProxyUpstreamHeaders } from '../mcpHeaders.mjs'
|
|
3
4
|
|
|
4
5
|
describe('buildProxyUpstreamHeaders', () => {
|
|
5
|
-
it('advertises the
|
|
6
|
+
it('advertises only the capabilities the proxy can actually fulfil', () => {
|
|
6
7
|
expect(buildProxyUpstreamHeaders('mak_test123')).toEqual({
|
|
7
8
|
Authorization: 'Bearer mak_test123',
|
|
8
9
|
'X-Gemus-Client-Capabilities': 'agent-authored-prompts,codex-imagen',
|
|
9
10
|
})
|
|
10
11
|
})
|
|
12
|
+
|
|
13
|
+
it('never advertises codex-instant-delivery — the proxy has no ImageScheduler to renew the lease (#2049)', () => {
|
|
14
|
+
// 该能力门控服务端的 AWAITING_CLIENT execute 分支,要求客户端有 scheduler 每 60s renewLease。
|
|
15
|
+
// 只有 bridge 有(且 probe 门控,见 bridge/src/cli.ts)。proxy 没有 —— 发了它就等于承诺一个
|
|
16
|
+
// 永不到来的续租 → 5min 后 CLIENT_LEASE_EXPIRED → 图 orphan 成孤立 image-upload 节点。
|
|
17
|
+
// absent → 服务端回退经典两阶段 client-delivered 协议,proxy 完全支持(backfill.mjs)。
|
|
18
|
+
const header = buildProxyUpstreamHeaders('mak_test123')['X-Gemus-Client-Capabilities']
|
|
19
|
+
expect(header).not.toContain('codex-instant-delivery')
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
it('stays a strict subset of the shared allowlist — a new capability must not auto-inherit (#2049)', () => {
|
|
23
|
+
// 回归门:本 bug 的成因正是 proxy 发「整个 CLIENT_CAPABILITIES 数组」—— #1969 往数组里加了
|
|
24
|
+
// codex-instant-delivery,proxy 就自动继承了一个它做不到的承诺,而 #1990 把红掉的断言改成
|
|
25
|
+
// 迎合它。此断言让「加能力」与「proxy 声称支持它」变成两个独立决策,不再自动传染。
|
|
26
|
+
const advertised = buildProxyUpstreamHeaders('mak_test123')['X-Gemus-Client-Capabilities'].split(',')
|
|
27
|
+
for (const capability of advertised) {
|
|
28
|
+
expect(CLIENT_CAPABILITIES).toContain(capability)
|
|
29
|
+
}
|
|
30
|
+
expect(advertised.length).toBeLessThan(CLIENT_CAPABILITIES.length)
|
|
31
|
+
})
|
|
11
32
|
})
|
package/src/failFast.mjs
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
// @gemus/mcp-proxy fail-fast (#2068) — surface upstream stream death to Codex immediately.
|
|
2
|
+
//
|
|
3
|
+
// StreamableHTTPClientTransport.send() resolves as soon as it DISPATCHES the SSE read loop,
|
|
4
|
+
// not when the tool result arrives. When the upstream POST SSE body dies before the result,
|
|
5
|
+
// the SDK only fires a global `onerror` with NO request id, and never reconnects a POST (no
|
|
6
|
+
// priming event). So Codex's pending request never settles → it hangs to its own 300s timeout.
|
|
7
|
+
//
|
|
8
|
+
// The only interception point the SDK offers is the constructor `fetch` option: it receives the
|
|
9
|
+
// exact (url, init) per request, so we parse the JSON-RPC id from `init.body` and wrap the
|
|
10
|
+
// response body. We use a SINGLE-PASS observe-while-forward wrapper (not tee()+grace): the same
|
|
11
|
+
// read loop that forwards bytes to the transport also scans the SSE frames for the tracked id's
|
|
12
|
+
// result/error. "Detect order == forward order" on one reader, so there is NO race between "result
|
|
13
|
+
// delivered" and "stream closed" — no wall-clock grace window (which CLAUDE.md forbids). If the
|
|
14
|
+
// stream reaches done/error while a tracked id was never seen on the wire, that result is lost →
|
|
15
|
+
// inject a JSON-RPC error keyed to that id.
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* A Codex-originated id is anything the AGENT issued. We exclude our own internal namespaces:
|
|
19
|
+
* - `gemus-bf-*` footprint-0 backfill injections (self-consumed by upstream.onmessage)
|
|
20
|
+
* - `gemus-ping-*` liveness heartbeat pongs (self-consumed)
|
|
21
|
+
* Numeric ids (Codex initialize / tools/list / tools/call) are all tracked.
|
|
22
|
+
*/
|
|
23
|
+
export function isCodexOriginated(id) {
|
|
24
|
+
if (id === undefined || id === null) return false
|
|
25
|
+
return !(typeof id === 'string' && (id.startsWith('gemus-bf-') || id.startsWith('gemus-ping-')))
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Extract JSON-RPC request ids from a fetch init. Only POST requests with a JSON body carry a
|
|
30
|
+
* request whose response we must guarantee; GET (standalone SSE) and DELETE (terminateSession)
|
|
31
|
+
* have no body → []. Never throws — a body we can't parse yields [].
|
|
32
|
+
*/
|
|
33
|
+
export function parseRequestIds(init) {
|
|
34
|
+
try {
|
|
35
|
+
const method = (init?.method || 'GET').toUpperCase()
|
|
36
|
+
if (method !== 'POST' || init.body == null) return []
|
|
37
|
+
const body = typeof init.body === 'string' ? init.body : String(init.body)
|
|
38
|
+
const parsed = JSON.parse(body)
|
|
39
|
+
const msgs = Array.isArray(parsed) ? parsed : [parsed]
|
|
40
|
+
const ids = []
|
|
41
|
+
for (const m of msgs) {
|
|
42
|
+
// A request has both an id and a method; responses/notifications are not our concern here.
|
|
43
|
+
if (m && m.id !== undefined && m.id !== null && typeof m.method === 'string') ids.push(m.id)
|
|
44
|
+
}
|
|
45
|
+
return ids
|
|
46
|
+
} catch {
|
|
47
|
+
return []
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** True for a JSON-RPC RESPONSE (result/error) — has an id and no `method`. A server→client REQUEST
|
|
52
|
+
* (elicitation/sampling) also has an id but carries a `method`, and its id (the server's own counter)
|
|
53
|
+
* can COLLIDE with an in-flight Codex tools/call id — settling on it would disarm the fail-fast guard
|
|
54
|
+
* for the very tool that stream carries. So we settle only on responses, never on relayed requests. */
|
|
55
|
+
export function isJsonRpcResponse(msg) {
|
|
56
|
+
return !!msg && msg.id !== undefined && msg.id !== null && msg.method === undefined
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Minimal SSE frame scanner. Accumulates decoded text, splits on the event boundary, and for each
|
|
61
|
+
* complete event yields the JSON-RPC id of any `data:` payload that is a RESPONSE (see
|
|
62
|
+
* isJsonRpcResponse). Comment lines (`:` — including our injected `: keepalive`) are ignored, so
|
|
63
|
+
* they never count as a delivered result. Half-frames straddling chunk boundaries are buffered until
|
|
64
|
+
* their terminating blank line arrives (no early/missed detection). CRLF line endings are normalized
|
|
65
|
+
* to LF so an intermediary that rewrites them can't wedge the scanner.
|
|
66
|
+
*/
|
|
67
|
+
export function createSseIdScanner() {
|
|
68
|
+
const decoder = new TextDecoder()
|
|
69
|
+
let buf = ''
|
|
70
|
+
return {
|
|
71
|
+
/** Push a chunk; returns the JSON-RPC ids of any RESPONSE frames that crossed the wire in it. */
|
|
72
|
+
push(chunk) {
|
|
73
|
+
buf += decoder.decode(chunk, { stream: true }).replace(/\r\n/g, '\n')
|
|
74
|
+
const ids = []
|
|
75
|
+
let sep
|
|
76
|
+
while ((sep = buf.indexOf('\n\n')) !== -1) {
|
|
77
|
+
const block = buf.slice(0, sep)
|
|
78
|
+
buf = buf.slice(sep + 2)
|
|
79
|
+
const dataLines = []
|
|
80
|
+
for (const line of block.split('\n')) {
|
|
81
|
+
if (line.startsWith('data:')) dataLines.push(line.slice(5).trimStart())
|
|
82
|
+
}
|
|
83
|
+
if (!dataLines.length) continue
|
|
84
|
+
try {
|
|
85
|
+
const msg = JSON.parse(dataLines.join('\n'))
|
|
86
|
+
if (isJsonRpcResponse(msg)) ids.push(msg.id)
|
|
87
|
+
} catch {
|
|
88
|
+
/* not JSON-RPC (or partial) — ignore */
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return ids
|
|
92
|
+
},
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Build the fail-fast fetch wrapper + settle/dispose controls.
|
|
98
|
+
*
|
|
99
|
+
* @param {object} opts
|
|
100
|
+
* @param {(id:any, reason:string)=>void} opts.onLost called once per lost id (must be idempotent-safe on caller side)
|
|
101
|
+
* @param {number} [opts.idleTimeoutMs] IDLE backstop reset on every upstream byte (incl. server
|
|
102
|
+
* heartbeats every ~12s), so it trips ONLY on a genuinely silent/zombie connection — never on a
|
|
103
|
+
* healthy-but-slow tool. Must be < Codex 300s; default 120_000.
|
|
104
|
+
* @param {(dir:string, m:any)=>void} [opts.log]
|
|
105
|
+
* @param {typeof fetch} [opts.realFetch] injectable for tests; defaults to global fetch
|
|
106
|
+
* @returns {{ fetch, settle, touch, forget, dispose, failedIds }}
|
|
107
|
+
*/
|
|
108
|
+
export function createFailFast(opts) {
|
|
109
|
+
const { onLost, log = () => {} } = opts
|
|
110
|
+
const idleTimeoutMs = Number(opts.idleTimeoutMs) || 120_000
|
|
111
|
+
const realFetch = opts.realFetch || fetch
|
|
112
|
+
const FAILED_IDS_CAP = 512
|
|
113
|
+
|
|
114
|
+
/** idStr → { timer } — Codex-originated in-flight requests awaiting a result. */
|
|
115
|
+
const inFlight = new Map()
|
|
116
|
+
/** ids already failed by onLost — lets onmessage drop a late real result's forward (still taps it). */
|
|
117
|
+
const failedIds = new Set()
|
|
118
|
+
|
|
119
|
+
function armTimer(entry, id) {
|
|
120
|
+
entry.timer = setTimeout(() => lose(id, 'proxy idle timeout (no upstream bytes)'), idleTimeoutMs)
|
|
121
|
+
if (typeof entry.timer.unref === 'function') entry.timer.unref()
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function register(id) {
|
|
125
|
+
const key = String(id)
|
|
126
|
+
if (inFlight.has(key)) return
|
|
127
|
+
const entry = {}
|
|
128
|
+
inFlight.set(key, entry)
|
|
129
|
+
armTimer(entry, id)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Reset the idle timer on any upstream byte — a healthy stream (heartbeats + data) never trips it. */
|
|
133
|
+
function touch(id) {
|
|
134
|
+
const entry = inFlight.get(String(id))
|
|
135
|
+
if (!entry) return
|
|
136
|
+
clearTimeout(entry.timer)
|
|
137
|
+
armTimer(entry, id)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function settle(id) {
|
|
141
|
+
const key = String(id)
|
|
142
|
+
const entry = inFlight.get(key)
|
|
143
|
+
if (!entry) return
|
|
144
|
+
clearTimeout(entry.timer)
|
|
145
|
+
inFlight.delete(key)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Drop an id from failedIds once its late result has been handled (bounds the set). */
|
|
149
|
+
function forget(id) {
|
|
150
|
+
failedIds.delete(String(id))
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function lose(id, reason) {
|
|
154
|
+
const key = String(id)
|
|
155
|
+
if (!inFlight.has(key)) return // already settled or already lost
|
|
156
|
+
settle(id)
|
|
157
|
+
failedIds.add(key)
|
|
158
|
+
// Bound the set: disconnect-case entries never see a late result to forget(), so cap FIFO.
|
|
159
|
+
if (failedIds.size > FAILED_IDS_CAP) failedIds.delete(failedIds.values().next().value)
|
|
160
|
+
log('FAILFAST', { id, reason })
|
|
161
|
+
onLost(id, reason)
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function observeAndForward(body, codexIds) {
|
|
165
|
+
const reader = body.getReader()
|
|
166
|
+
const scanner = createSseIdScanner()
|
|
167
|
+
let ended = false
|
|
168
|
+
const onEnd = () => {
|
|
169
|
+
if (ended) return
|
|
170
|
+
ended = true
|
|
171
|
+
for (const id of codexIds) lose(id, 'upstream stream closed before result')
|
|
172
|
+
}
|
|
173
|
+
return new ReadableStream({
|
|
174
|
+
async pull(controller) {
|
|
175
|
+
let r
|
|
176
|
+
try {
|
|
177
|
+
r = await reader.read()
|
|
178
|
+
} catch (e) {
|
|
179
|
+
try {
|
|
180
|
+
controller.error(e)
|
|
181
|
+
} catch {
|
|
182
|
+
/* already closed */
|
|
183
|
+
}
|
|
184
|
+
onEnd()
|
|
185
|
+
return
|
|
186
|
+
}
|
|
187
|
+
if (r.done) {
|
|
188
|
+
try {
|
|
189
|
+
controller.close()
|
|
190
|
+
} catch {
|
|
191
|
+
/* already closed */
|
|
192
|
+
}
|
|
193
|
+
onEnd()
|
|
194
|
+
return
|
|
195
|
+
}
|
|
196
|
+
try {
|
|
197
|
+
controller.enqueue(r.value) // forward bytes unchanged to the transport
|
|
198
|
+
} catch {
|
|
199
|
+
/* downstream gone */
|
|
200
|
+
}
|
|
201
|
+
// Any byte (data OR heartbeat) proves the stream is alive → reset the idle backstop.
|
|
202
|
+
for (const id of codexIds) touch(id)
|
|
203
|
+
// Same loop: any tracked id whose RESPONSE crossed the wire is now delivered.
|
|
204
|
+
for (const id of scanner.push(r.value)) settle(id)
|
|
205
|
+
},
|
|
206
|
+
cancel(reason) {
|
|
207
|
+
reader.cancel(reason).catch(() => {})
|
|
208
|
+
onEnd()
|
|
209
|
+
},
|
|
210
|
+
})
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
async function wrappedFetch(url, init) {
|
|
214
|
+
// Parse ids + register. Both are internally safe (parseRequestIds never throws), but guard
|
|
215
|
+
// defensively so a future change can't break the chain — degrade to "no tracking".
|
|
216
|
+
let codexIds = []
|
|
217
|
+
try {
|
|
218
|
+
codexIds = parseRequestIds(init).filter(isCodexOriginated)
|
|
219
|
+
for (const id of codexIds) register(id)
|
|
220
|
+
} catch (e) {
|
|
221
|
+
log('FAILFAST-WRAP-ERR', String(e))
|
|
222
|
+
codexIds = []
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// The network call. Its failure MUST propagate to the SDK unchanged — do NOT retry here
|
|
226
|
+
// (a second fetch could execute the tool server-side after Codex already got the error).
|
|
227
|
+
let res
|
|
228
|
+
try {
|
|
229
|
+
res = await realFetch(url, init)
|
|
230
|
+
} catch (e) {
|
|
231
|
+
// Connection failed before any response (DNS / TLS / RST) — fail fast, don't wait 270s.
|
|
232
|
+
for (const id of codexIds) lose(id, `upstream fetch failed: ${e}`)
|
|
233
|
+
throw e // preserve the SDK's existing onerror/logging path (no re-fetch)
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// Response handling + monitoring. On any internal bug, return the response we ALREADY have,
|
|
237
|
+
// unmonitored — never re-fetch (wrappedFetch sits on 100% of upstream traffic incl. GET
|
|
238
|
+
// elicitation + DELETE teardown, so it must never break the chain, but also never duplicate).
|
|
239
|
+
try {
|
|
240
|
+
if (!codexIds.length) return res // GET / DELETE / notification / internal id → passthrough
|
|
241
|
+
if (!res.ok) {
|
|
242
|
+
// Non-SSE HTTP error (404 session-gone / 403 / 429 / …). send() throws before onmessage,
|
|
243
|
+
// and that rejection is swallowed by upstream.send().catch — so nothing ever settles the id.
|
|
244
|
+
for (const id of codexIds) lose(id, `upstream HTTP ${res.status}`)
|
|
245
|
+
return res
|
|
246
|
+
}
|
|
247
|
+
const ct = res.headers.get('content-type') || ''
|
|
248
|
+
if (!ct.includes('text/event-stream') || !res.body) return res // 202 etc. — no id to monitor
|
|
249
|
+
return new Response(observeAndForward(res.body, codexIds), {
|
|
250
|
+
status: res.status,
|
|
251
|
+
statusText: res.statusText,
|
|
252
|
+
headers: res.headers,
|
|
253
|
+
})
|
|
254
|
+
} catch (e) {
|
|
255
|
+
log('FAILFAST-WRAP-ERR', String(e))
|
|
256
|
+
return res
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function dispose() {
|
|
261
|
+
for (const { timer } of inFlight.values()) clearTimeout(timer)
|
|
262
|
+
inFlight.clear()
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
return { fetch: wrappedFetch, settle, touch, forget, dispose, failedIds }
|
|
266
|
+
}
|
package/src/mcpHeaders.mjs
CHANGED
|
@@ -4,9 +4,36 @@ import {
|
|
|
4
4
|
serializeClientCapabilities,
|
|
5
5
|
} from '@gemus/codex-backfill-core'
|
|
6
6
|
|
|
7
|
+
/**
|
|
8
|
+
* #2049:proxy 只 advertise **它自己能履行**的能力 —— 不是 `CLIENT_CAPABILITIES` 全集。
|
|
9
|
+
*
|
|
10
|
+
* `CLIENT_CAPABILITIES` 是**所有已知能力的 allowlist**(共享 SSOT),不是「每个客户端都支持」的清单。
|
|
11
|
+
* 各客户端各自声明自己能兑现的子集:bridge 见 `bridge/src/cli.ts`(`imageScheduler` 与
|
|
12
|
+
* `activeCapabilities` **成对注入** —— 有 scheduler 才发 capability,且 probe 门控)。
|
|
13
|
+
*
|
|
14
|
+
* **`codex-instant-delivery` 必须排除**:它门控服务端的 AWAITING_CLIENT execute 分支,
|
|
15
|
+
* 要求客户端有 ImageScheduler 每 60s `renewLease`。**proxy 没有 scheduler**(本包与
|
|
16
|
+
* `@gemus/codex-backfill-core` 里 `renew`/`lease` 零命中)。发了它 = 承诺一个永不到来的续租:
|
|
17
|
+
* execute 建 AWAITING_CLIENT job(lease=+5min)→ 无人续 → `CLIENT_LEASE_EXPIRED` → 节点 FAILED
|
|
18
|
+
* → Codex 的图回来时目标已死 → `backfill.mjs` orphan 兜底 → 孤立 `image-upload` 节点。
|
|
19
|
+
* 真机实测见 #2049。
|
|
20
|
+
* **不发它 → 服务端回退经典两阶段 client-delivered 协议,proxy 完全支持**(`backfill.mjs`
|
|
21
|
+
* tap execute + 注入交付)—— 这正是 `codex-backfill-core/src/mcp.ts` 那条注释写的
|
|
22
|
+
* "absent → the classic dryRun/Phase-2 two-phase client-delivered protocol is used unchanged"。
|
|
23
|
+
*
|
|
24
|
+
* ⚠️ **往 `CLIENT_CAPABILITIES` 加新能力时**:默认**不会**、也**不该**自动出现在这里。先问
|
|
25
|
+
* 「proxy 拿什么兑现它?」—— 本 bug 的成因正是「发全集」让新能力自动传染(#1969 加,proxy 白捡,
|
|
26
|
+
* #1990 把红掉的断言改成迎合)。`__tests__/mcpHeaders.test.ts` 有严格子集断言守着这条。
|
|
27
|
+
*/
|
|
28
|
+
const PROXY_UNSUPPORTED_CAPABILITIES = new Set(['codex-instant-delivery'])
|
|
29
|
+
|
|
30
|
+
const PROXY_CAPABILITIES = Object.freeze(
|
|
31
|
+
CLIENT_CAPABILITIES.filter((capability) => !PROXY_UNSUPPORTED_CAPABILITIES.has(capability)),
|
|
32
|
+
)
|
|
33
|
+
|
|
7
34
|
export function buildProxyUpstreamHeaders(apiKey) {
|
|
8
35
|
return {
|
|
9
36
|
Authorization: `Bearer ${apiKey}`,
|
|
10
|
-
[CLIENT_CAPABILITIES_HEADER]: serializeClientCapabilities(
|
|
37
|
+
[CLIENT_CAPABILITIES_HEADER]: serializeClientCapabilities(PROXY_CAPABILITIES),
|
|
11
38
|
}
|
|
12
39
|
}
|
package/src/proxy.mjs
CHANGED
|
@@ -26,6 +26,7 @@ import { randomBytes } from 'node:crypto'
|
|
|
26
26
|
import { createBackfiller, injectSteering } from './backfill.mjs'
|
|
27
27
|
import { unwrapMcpContent } from '@gemus/codex-backfill-core'
|
|
28
28
|
import { buildProxyUpstreamHeaders } from './mcpHeaders.mjs'
|
|
29
|
+
import { createFailFast, isJsonRpcResponse } from './failFast.mjs'
|
|
29
30
|
|
|
30
31
|
const KEY = process.env.GEMUS_KEY || ''
|
|
31
32
|
const GEMUS_URL = process.env.GEMUS_URL || 'https://gemus.ai/api/mcp'
|
|
@@ -244,10 +245,33 @@ hookServer.listen(0, '127.0.0.1', () => {
|
|
|
244
245
|
if (typeof hookServer.unref === 'function') hookServer.unref()
|
|
245
246
|
|
|
246
247
|
// ── transparent JSON-RPC passthrough (raw send/onmessage) ──────────────────────────────────────
|
|
248
|
+
const stdio = new StdioServerTransport()
|
|
249
|
+
|
|
250
|
+
// #2068 fail-fast: StreamableHTTPClientTransport.send() resolves before the upstream SSE result
|
|
251
|
+
// arrives and only surfaces a stream death via a global, id-less `onerror` (no reconnect for POST).
|
|
252
|
+
// The custom `fetch` wrapper parses each request's JSON-RPC id and, if the response stream dies
|
|
253
|
+
// before that id's result crosses the wire, injects a JSON-RPC error so Codex fails in <2s instead
|
|
254
|
+
// of hanging to its 300s timeout. onLost only injects to stdio — it does NOT tap: tapping here would
|
|
255
|
+
// delete the proxy's `inflight` entry, so the DROP-AFTER-FAILFAST branch below could no longer let
|
|
256
|
+
// backfill observe a late real result (env delivery on the still-open, idle-timed-out case).
|
|
257
|
+
const failFast = createFailFast({
|
|
258
|
+
idleTimeoutMs: Number(process.env.PROXY_FAILFAST_TIMEOUT_MS) || 120_000,
|
|
259
|
+
log,
|
|
260
|
+
onLost: (id, reason) => {
|
|
261
|
+
const errObj = {
|
|
262
|
+
jsonrpc: '2.0',
|
|
263
|
+
id,
|
|
264
|
+
error: { code: -32000, message: 'Gemus upstream stream disconnected before the tool result was delivered' },
|
|
265
|
+
}
|
|
266
|
+
log('FAILFAST-INJECT', { id, reason })
|
|
267
|
+
stdio.send(errObj).catch((e) => log('ERR-stdio-send', String(e)))
|
|
268
|
+
},
|
|
269
|
+
})
|
|
270
|
+
|
|
247
271
|
const upstream = new StreamableHTTPClientTransport(new URL(GEMUS_URL), {
|
|
248
272
|
requestInit: { headers: buildProxyUpstreamHeaders(KEY) },
|
|
273
|
+
fetch: failFast.fetch,
|
|
249
274
|
})
|
|
250
|
-
const stdio = new StdioServerTransport()
|
|
251
275
|
|
|
252
276
|
stdio.onmessage = (m) => { tapClientToServer(m); log('C->G', m); upstream.send(m).catch((e) => log('ERR-http-send', String(e))) }
|
|
253
277
|
upstream.onmessage = (m) => {
|
|
@@ -262,6 +286,18 @@ upstream.onmessage = (m) => {
|
|
|
262
286
|
// #1756 liveness heartbeat: our own `ping` pongs (gemus-ping-*) are self-consumed — Codex never
|
|
263
287
|
// issued them, so it must never see the responses.
|
|
264
288
|
if (typeof m?.id === 'string' && m.id.startsWith('gemus-ping-')) return
|
|
289
|
+
// #2068 fail-fast bookkeeping. Gate on RESPONSE only (isJsonRpcResponse) — a server→client request
|
|
290
|
+
// (elicitation/sampling) also carries an id that can collide with an in-flight tools/call id, and
|
|
291
|
+
// must be forwarded to Codex untouched, never settled/dropped.
|
|
292
|
+
if (isJsonRpcResponse(m)) {
|
|
293
|
+
if (failFast.failedIds.has(String(m.id))) {
|
|
294
|
+
// Already fail-fasted (idle timeout on a still-open stream) → Codex got the error. Still tap so
|
|
295
|
+
// backfill (#1751) observes the late real result (inflight survives — onLost didn't tap), but
|
|
296
|
+
// do NOT re-forward it to Codex (no double-send). forget() bounds failedIds.
|
|
297
|
+
tapServerToClient(m); failFast.forget(m.id); log('DROP-AFTER-FAILFAST', m.id); return
|
|
298
|
+
}
|
|
299
|
+
failFast.settle(m.id)
|
|
300
|
+
}
|
|
265
301
|
tapServerToClient(m); steerOnInit(m); log('G->C', m); stdio.send(m).catch((e) => log('ERR-stdio-send', String(e)))
|
|
266
302
|
}
|
|
267
303
|
|
|
@@ -293,6 +329,7 @@ async function shutdown() {
|
|
|
293
329
|
// Settle any in-flight footprint-0 injections so a mid-delivery shutdown doesn't hang their awaits.
|
|
294
330
|
for (const p of pendingInjections.values()) p.settle({ ok: false, error: 'proxy shutting down' })
|
|
295
331
|
pendingInjections.clear()
|
|
332
|
+
failFast.dispose() // #2068: clear fail-fast final-safety timers
|
|
296
333
|
try { hookServer.close() } catch { /* ignore */ }
|
|
297
334
|
if (discoveryFile) { try { fs.unlinkSync(discoveryFile) } catch { /* ignore */ } }
|
|
298
335
|
try { await upstream.terminateSession() } catch { /* ignore */ }
|