@gemus/mcp-proxy 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/package.json +3 -2
- package/src/__tests__/failFast.test.ts +108 -6
- package/src/__tests__/fixtures/proxy-runtime-loader.mjs +86 -6
- package/src/__tests__/proxyMessages.test.ts +81 -0
- package/src/__tests__/startup.test.ts +197 -3
- package/src/__tests__/upstreamHttp.real.test.ts +37 -0
- package/src/__tests__/upstreamHttp.test.ts +151 -0
- package/src/failFast.mjs +23 -10
- package/src/proxy.mjs +34 -11
- package/src/proxyMessages.mjs +38 -0
- package/src/startup.mjs +16 -0
- package/src/upstreamHttp.mjs +69 -0
package/README.md
CHANGED
|
@@ -15,6 +15,40 @@ then delivers an unambiguous image to its planned node or conservatively creates
|
|
|
15
15
|
`image-upload`. MCP initialization advertises the shared Codex capability contract used by the
|
|
16
16
|
server blueprint policy.
|
|
17
17
|
|
|
18
|
+
## Upstream connection reliability (#2328)
|
|
19
|
+
|
|
20
|
+
The proxy uses one owned Undici `Agent` for all upstream MCP traffic. By default its TCP
|
|
21
|
+
address-family racing policy sets `autoSelectFamily: true` with a
|
|
22
|
+
`PROXY_CONNECT_ATTEMPT_TIMEOUT_MS=1000` per-address-family attempt window. An operator may set a
|
|
23
|
+
positive numeric override; invalid, empty, and non-positive values fall back to `1000`, while
|
|
24
|
+
values below `10` ms clamp to `10` ms. The transport's request method, body, and auth headers are
|
|
25
|
+
preserved; the proxy adds only that owned dispatcher.
|
|
26
|
+
|
|
27
|
+
This is a connection-attempt policy, not an MCP retry policy. For a post-start request, a rejected
|
|
28
|
+
`POST` before response headers produces one request only: the affected JSON-RPC id receives `-32000`
|
|
29
|
+
with `Gemus upstream connection failed before a response was received`. The raw network reason is
|
|
30
|
+
never returned to Codex. During `initialize`, the same classification writes one sanitized fatal
|
|
31
|
+
diagnostic and exits non-zero. Once an SSE response has started, a body that ends before its result
|
|
32
|
+
remains the distinct #2068 failure:
|
|
33
|
+
`Gemus upstream stream disconnected before the tool result was delivered`.
|
|
34
|
+
|
|
35
|
+
On graceful or fatal shutdown, the proxy bounds session `DELETE`, then transport close, before it
|
|
36
|
+
closes the owned Agent. Each stage has a 2-second bound; an Agent that does not close in its bound
|
|
37
|
+
has its retained per-origin dispatchers force-destroyed before the Agent is destroyed. Repeated
|
|
38
|
+
shutdown signals share the same cleanup. See the [transport reliability
|
|
39
|
+
runbook](../../docs/operations/mcp-transport-reliability.md) for the network boundaries and the
|
|
40
|
+
[Companion architecture](../../docs/architecture/agent%20architecture/codex-desktop-companion.md)
|
|
41
|
+
for the request path.
|
|
42
|
+
|
|
43
|
+
## Release gate (#2330)
|
|
44
|
+
|
|
45
|
+
This package is prepared as `@gemus/mcp-proxy@0.1.9`; it is not published by this change. Release
|
|
46
|
+
Issue [#2330](https://github.com/Gemus-AI/Gemus/issues/2330) owns the post-merge gate: publish and
|
|
47
|
+
verify the npm package, then in a follow-up change pin the plugin to exact `0.1.9`, forward
|
|
48
|
+
`PROXY_CONNECT_ATTEMPT_TIMEOUT_MS`, bump and sync the plugin, refresh the installed Codex plugin
|
|
49
|
+
cache, and run fresh ESA and direct-domain smoke tests. Until that gate completes, the plugin
|
|
50
|
+
contract stays byte-for-byte at exact `@gemus/mcp-proxy@0.1.8`.
|
|
51
|
+
|
|
18
52
|
## What was validated end-to-end (2026-07-06, codex-cli 0.142.2 → local dev `/api/mcp`)
|
|
19
53
|
|
|
20
54
|
- **Transparent passthrough** — initialize + tools/list relay all **12 gemus tools** (incl.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gemus/mcp-proxy",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.9",
|
|
4
4
|
"description": "Local stdio<->HTTP MCP proxy for Codex desktop: transparent passthrough + execute tap + imagegen backfill (Issue #1751, P1).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -25,10 +25,11 @@
|
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
28
|
+
"undici": "^6.28.0",
|
|
28
29
|
"@gemus/codex-backfill-core": "0.1.3"
|
|
29
30
|
},
|
|
30
31
|
"engines": {
|
|
31
|
-
"node": ">=18"
|
|
32
|
+
"node": ">=18.17"
|
|
32
33
|
},
|
|
33
34
|
"files": [
|
|
34
35
|
"src/",
|
|
@@ -122,7 +122,7 @@ describe('createFailFast.fetch', () => {
|
|
|
122
122
|
await drain(res)
|
|
123
123
|
|
|
124
124
|
expect(onLost).toHaveBeenCalledTimes(1)
|
|
125
|
-
expect(onLost
|
|
125
|
+
expect(onLost).toHaveBeenCalledWith(42, { kind: 'stream' })
|
|
126
126
|
expect(ff.failedIds.has('42')).toBe(true)
|
|
127
127
|
})
|
|
128
128
|
|
|
@@ -140,21 +140,90 @@ describe('createFailFast.fetch', () => {
|
|
|
140
140
|
|
|
141
141
|
it('fails fast immediately when realFetch throws before any response', async () => {
|
|
142
142
|
const onLost = vi.fn()
|
|
143
|
-
const realFetch = vi.fn(async () => {
|
|
143
|
+
const realFetch = vi.fn(async () => {
|
|
144
|
+
throw new Error('socket reset', { cause: Object.assign(new Error('inner socket failure'), { code: 'ECONNRESET' }) })
|
|
145
|
+
})
|
|
144
146
|
const ff = createFailFast({ onLost, realFetch })
|
|
145
147
|
|
|
146
|
-
await expect(ff.fetch('u', post(7))).rejects.toThrow('
|
|
147
|
-
expect(onLost).toHaveBeenCalledWith(7,
|
|
148
|
+
await expect(ff.fetch('u', post(7))).rejects.toThrow('socket reset')
|
|
149
|
+
expect(onLost).toHaveBeenCalledWith(7, { kind: 'connection', code: 'ECONNRESET' })
|
|
148
150
|
expect(realFetch).toHaveBeenCalledTimes(1) // must NOT retry — a 2nd fetch could execute the tool
|
|
149
151
|
})
|
|
150
152
|
|
|
153
|
+
it('allowlists a nested connection code without exposing hostile raw diagnostics or retrying', async () => {
|
|
154
|
+
const hostile =
|
|
155
|
+
'Authorization: Bearer arbitrary-secret at '
|
|
156
|
+
+ 'https://alice:password@example.test/api/mcp?token=query-secret '
|
|
157
|
+
+ 'via 203.0.113.1,2001:db8::1'
|
|
158
|
+
const onLost = vi.fn()
|
|
159
|
+
const log = vi.fn()
|
|
160
|
+
const rejection = new Error(`outer ${hostile}`, {
|
|
161
|
+
cause: new Error(`middle ${hostile}`, {
|
|
162
|
+
cause: Object.assign(new Error(`inner ${hostile}`), {
|
|
163
|
+
code: 'UND_ERR_CONNECT_TIMEOUT',
|
|
164
|
+
}),
|
|
165
|
+
}),
|
|
166
|
+
})
|
|
167
|
+
const realFetch = vi.fn(async () => { throw rejection })
|
|
168
|
+
const ff = createFailFast({ onLost, log, realFetch })
|
|
169
|
+
|
|
170
|
+
await expect(ff.fetch('u', post(71))).rejects.toBe(rejection)
|
|
171
|
+
|
|
172
|
+
expect(realFetch).toHaveBeenCalledTimes(1)
|
|
173
|
+
expect(onLost).toHaveBeenCalledTimes(1)
|
|
174
|
+
expect(onLost).toHaveBeenCalledWith(71, {
|
|
175
|
+
kind: 'connection',
|
|
176
|
+
code: 'UND_ERR_CONNECT_TIMEOUT',
|
|
177
|
+
})
|
|
178
|
+
const diagnostics = JSON.stringify(log.mock.calls)
|
|
179
|
+
expect(diagnostics).not.toMatch(
|
|
180
|
+
/arbitrary-secret|alice|password|query-secret|203\.0\.113\.1|2001:db8/,
|
|
181
|
+
)
|
|
182
|
+
})
|
|
183
|
+
|
|
184
|
+
it('classifies an established SSE reader error before its result as a stream failure', async () => {
|
|
185
|
+
const readerError = new Error('established stream failed')
|
|
186
|
+
let pulls = 0
|
|
187
|
+
const body = new ReadableStream<Uint8Array>({
|
|
188
|
+
pull(controller) {
|
|
189
|
+
if (pulls++ === 0) {
|
|
190
|
+
controller.enqueue(enc.encode(': established\n\n'))
|
|
191
|
+
} else {
|
|
192
|
+
controller.error(readerError)
|
|
193
|
+
}
|
|
194
|
+
},
|
|
195
|
+
})
|
|
196
|
+
const onLost = vi.fn()
|
|
197
|
+
const realFetch = vi.fn(async () => sseRes(body))
|
|
198
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
199
|
+
|
|
200
|
+
const res = await ff.fetch('u', post(72))
|
|
201
|
+
await expect(drain(res)).rejects.toBe(readerError)
|
|
202
|
+
|
|
203
|
+
expect(onLost).toHaveBeenCalledTimes(1)
|
|
204
|
+
expect(onLost).toHaveBeenCalledWith(72, { kind: 'stream' })
|
|
205
|
+
})
|
|
206
|
+
|
|
207
|
+
it('classifies downstream cancellation before its result as a stream failure', async () => {
|
|
208
|
+
const onLost = vi.fn()
|
|
209
|
+
const realFetch = vi.fn(async () => sseRes(neverBody()))
|
|
210
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
211
|
+
const res = await ff.fetch('u', post(73))
|
|
212
|
+
const reader = res.body!.getReader()
|
|
213
|
+
|
|
214
|
+
await reader.cancel('downstream closed')
|
|
215
|
+
|
|
216
|
+
expect(onLost).toHaveBeenCalledTimes(1)
|
|
217
|
+
expect(onLost).toHaveBeenCalledWith(73, { kind: 'stream' })
|
|
218
|
+
})
|
|
219
|
+
|
|
151
220
|
it('fails fast on a non-ok HTTP response', async () => {
|
|
152
221
|
const onLost = vi.fn()
|
|
153
222
|
const realFetch = vi.fn(async () => sseRes(null, { status: 404, contentType: 'application/json' }))
|
|
154
223
|
const ff = createFailFast({ onLost, realFetch })
|
|
155
224
|
|
|
156
225
|
await ff.fetch('u', post(7))
|
|
157
|
-
expect(onLost).toHaveBeenCalledWith(7, '
|
|
226
|
+
expect(onLost).toHaveBeenCalledWith(7, { kind: 'http', status: 404 })
|
|
158
227
|
})
|
|
159
228
|
|
|
160
229
|
it('passes GET and DELETE through untouched (no body → no monitoring, no onLost)', async () => {
|
|
@@ -208,6 +277,39 @@ describe('createFailFast.fetch', () => {
|
|
|
208
277
|
expect(onLost.mock.calls[0][0]).toBe(1)
|
|
209
278
|
})
|
|
210
279
|
|
|
280
|
+
it('does not settle a healthy concurrent request when another connection rejects before response', async () => {
|
|
281
|
+
let healthyController!: ReadableStreamDefaultController<Uint8Array>
|
|
282
|
+
const healthyBody = new ReadableStream<Uint8Array>({
|
|
283
|
+
start(controller) { healthyController = controller },
|
|
284
|
+
})
|
|
285
|
+
const connectionError = Object.assign(new Error('connection rejected'), {
|
|
286
|
+
code: 'ECONNREFUSED',
|
|
287
|
+
})
|
|
288
|
+
const onLost = vi.fn()
|
|
289
|
+
const realFetch = vi.fn(async (_url: unknown, init: { body?: unknown }) => {
|
|
290
|
+
const [{ id }] = [JSON.parse(String(init.body))]
|
|
291
|
+
if (id === 74) throw connectionError
|
|
292
|
+
return sseRes(healthyBody)
|
|
293
|
+
})
|
|
294
|
+
const ff = createFailFast({ onLost, realFetch })
|
|
295
|
+
|
|
296
|
+
const rejecting = ff.fetch('u', post(74))
|
|
297
|
+
const healthy = ff.fetch('u', post(75))
|
|
298
|
+
await expect(rejecting).rejects.toBe(connectionError)
|
|
299
|
+
const healthyResponse = await healthy
|
|
300
|
+
|
|
301
|
+
expect(onLost).toHaveBeenCalledTimes(1)
|
|
302
|
+
expect(onLost).toHaveBeenCalledWith(74, {
|
|
303
|
+
kind: 'connection',
|
|
304
|
+
code: 'ECONNREFUSED',
|
|
305
|
+
})
|
|
306
|
+
healthyController.enqueue(enc.encode(resultFrame(75)))
|
|
307
|
+
healthyController.close()
|
|
308
|
+
await drain(healthyResponse)
|
|
309
|
+
expect(onLost).toHaveBeenCalledTimes(1)
|
|
310
|
+
expect(ff.failedIds.has('75')).toBe(false)
|
|
311
|
+
})
|
|
312
|
+
|
|
211
313
|
describe('idle safety timeout', () => {
|
|
212
314
|
beforeEach(() => vi.useFakeTimers())
|
|
213
315
|
afterEach(() => vi.useRealTimers())
|
|
@@ -220,7 +322,7 @@ describe('createFailFast.fetch', () => {
|
|
|
220
322
|
await ff.fetch('u', post(99)) // never emits → idle timer is never reset
|
|
221
323
|
expect(onLost).not.toHaveBeenCalled()
|
|
222
324
|
await vi.advanceTimersByTimeAsync(120_000)
|
|
223
|
-
expect(onLost).toHaveBeenCalledWith(99,
|
|
325
|
+
expect(onLost).toHaveBeenCalledWith(99, { kind: 'idle' })
|
|
224
326
|
ff.dispose()
|
|
225
327
|
})
|
|
226
328
|
|
|
@@ -47,14 +47,31 @@ const modules = new Map([
|
|
|
47
47
|
}
|
|
48
48
|
process.stdin.resume()
|
|
49
49
|
process.stdin.once('end', () => this.onclose?.())
|
|
50
|
+
if (process.env.PROXY_TEST_SIGNAL_AFTER_START === 'SIGTERM') {
|
|
51
|
+
setTimeout(() => {
|
|
52
|
+
if (process.platform === 'win32') {
|
|
53
|
+
process.emit('SIGTERM')
|
|
54
|
+
} else {
|
|
55
|
+
process.kill(process.pid, 'SIGTERM')
|
|
56
|
+
}
|
|
57
|
+
}, 0)
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
async send(message) {
|
|
61
|
+
if (process.env.PROXY_TEST_STDIO_CAPTURE_SEND === '1') {
|
|
62
|
+
process.stdout.write(JSON.stringify(message) + '\\n')
|
|
63
|
+
}
|
|
50
64
|
}
|
|
51
|
-
async send() {}
|
|
52
65
|
}
|
|
53
66
|
`],
|
|
54
67
|
['@modelcontextprotocol/sdk/client/streamableHttp.js', `
|
|
55
68
|
${markerPrelude}
|
|
56
69
|
export class StreamableHTTPClientTransport {
|
|
57
|
-
constructor() {
|
|
70
|
+
constructor(url, options) {
|
|
71
|
+
mark('upstream-transport')
|
|
72
|
+
this.url = url
|
|
73
|
+
this.fetch = options.fetch
|
|
74
|
+
}
|
|
58
75
|
async start() {}
|
|
59
76
|
async send(message) {
|
|
60
77
|
if (
|
|
@@ -68,9 +85,72 @@ const modules = new Map([
|
|
|
68
85
|
+ process.env.GEMUS_URL
|
|
69
86
|
)
|
|
70
87
|
}
|
|
88
|
+
if (
|
|
89
|
+
process.env.PROXY_TEST_UPSTREAM_INITIALIZE_FETCH_FAILURE === '1'
|
|
90
|
+
&& message?.method === 'initialize'
|
|
91
|
+
) {
|
|
92
|
+
await this.fetch(this.url, {
|
|
93
|
+
method: 'POST',
|
|
94
|
+
headers: { Authorization: 'Bearer ' + process.env.GEMUS_KEY },
|
|
95
|
+
body: JSON.stringify(message),
|
|
96
|
+
})
|
|
97
|
+
}
|
|
98
|
+
if (
|
|
99
|
+
process.env.PROXY_TEST_LATE_RESULT === '1'
|
|
100
|
+
&& message?.method === 'tools/call'
|
|
101
|
+
) {
|
|
102
|
+
await this.fetch(this.url, {
|
|
103
|
+
method: 'POST',
|
|
104
|
+
body: JSON.stringify(message),
|
|
105
|
+
})
|
|
106
|
+
setTimeout(() => {
|
|
107
|
+
this.onmessage?.({
|
|
108
|
+
jsonrpc: '2.0',
|
|
109
|
+
id: message.id,
|
|
110
|
+
result: { content: [{ type: 'text', text: 'late-real-result' }] },
|
|
111
|
+
})
|
|
112
|
+
}, 80)
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
async terminateSession() {
|
|
116
|
+
mark('upstream-terminate-session')
|
|
117
|
+
if (process.env.PROXY_TEST_TERMINATE_SESSION_HANG === '1') {
|
|
118
|
+
return await new Promise(() => {})
|
|
119
|
+
}
|
|
120
|
+
if (process.env.PROXY_TEST_TERMINATE_SESSION_REJECT === '1') {
|
|
121
|
+
throw new Error('terminate session rejected')
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
async close() { mark('upstream-transport-close') }
|
|
125
|
+
}
|
|
126
|
+
`],
|
|
127
|
+
['undici', `
|
|
128
|
+
${markerPrelude}
|
|
129
|
+
export class Agent {
|
|
130
|
+
constructor() { mark('upstream-http-agent') }
|
|
131
|
+
async close() { mark('upstream-http-shutdown') }
|
|
132
|
+
destroy() { mark('upstream-http-destroy') }
|
|
133
|
+
}
|
|
134
|
+
export class Pool {
|
|
135
|
+
destroy() { mark('upstream-origin-destroy') }
|
|
136
|
+
}
|
|
137
|
+
export async function fetch() {
|
|
138
|
+
mark('undici-fetch')
|
|
139
|
+
if (process.env.PROXY_TEST_LATE_RESULT === '1') {
|
|
140
|
+
return new Response(
|
|
141
|
+
new ReadableStream({ start() {} }),
|
|
142
|
+
{ headers: { 'content-type': 'text/event-stream' } },
|
|
143
|
+
)
|
|
71
144
|
}
|
|
72
|
-
|
|
73
|
-
|
|
145
|
+
const error = new Error(
|
|
146
|
+
'connect failed Authorization: Bearer '
|
|
147
|
+
+ process.env.GEMUS_KEY
|
|
148
|
+
+ ' at '
|
|
149
|
+
+ process.env.GEMUS_URL
|
|
150
|
+
+ ' via 203.0.113.1,2001:db8::1'
|
|
151
|
+
)
|
|
152
|
+
error.cause = { code: 'UND_ERR_CONNECT_TIMEOUT' }
|
|
153
|
+
throw error
|
|
74
154
|
}
|
|
75
155
|
`],
|
|
76
156
|
['node:http', `
|
|
@@ -115,8 +195,8 @@ const modules = new Map([
|
|
|
115
195
|
export function createBackfiller() {
|
|
116
196
|
mark('backfiller')
|
|
117
197
|
return {
|
|
118
|
-
observeCall() {},
|
|
119
|
-
observeResult() {},
|
|
198
|
+
observeCall() { mark('backfiller-observe-call') },
|
|
199
|
+
observeResult() { mark('backfiller-observe-result') },
|
|
120
200
|
async backfillTurn() {},
|
|
121
201
|
}
|
|
122
202
|
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest'
|
|
2
|
+
import { createProxyFailure, publicMessageForFailure } from '../proxyMessages.mjs'
|
|
3
|
+
|
|
4
|
+
describe('publicMessageForFailure', () => {
|
|
5
|
+
it.each([
|
|
6
|
+
[
|
|
7
|
+
{ kind: 'connection', code: 'UND_ERR_CONNECT_TIMEOUT' },
|
|
8
|
+
'Gemus upstream connection failed before a response was received',
|
|
9
|
+
],
|
|
10
|
+
[
|
|
11
|
+
{ kind: 'http', status: 503 },
|
|
12
|
+
'Gemus upstream returned an HTTP error before the tool result was delivered',
|
|
13
|
+
],
|
|
14
|
+
[
|
|
15
|
+
{ kind: 'stream' },
|
|
16
|
+
'Gemus upstream stream disconnected before the tool result was delivered',
|
|
17
|
+
],
|
|
18
|
+
[
|
|
19
|
+
{ kind: 'idle' },
|
|
20
|
+
'Gemus upstream stream became unresponsive before the tool result was delivered',
|
|
21
|
+
],
|
|
22
|
+
])('maps $kind to its delivery-stage message', (failure, expected) => {
|
|
23
|
+
expect(publicMessageForFailure(failure)).toBe(expected)
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
it('fails closed for an unknown failure kind', () => {
|
|
27
|
+
expect(publicMessageForFailure({ kind: 'hostile-secret-kind' })).toBe(
|
|
28
|
+
'Gemus upstream failed before the tool result was delivered',
|
|
29
|
+
)
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
it.each(['toString', 'constructor', '__proto__'])(
|
|
33
|
+
'fails closed for the Object.prototype property name %s',
|
|
34
|
+
(kind) => {
|
|
35
|
+
expect(publicMessageForFailure({ kind })).toBe(
|
|
36
|
+
'Gemus upstream failed before the tool result was delivered',
|
|
37
|
+
)
|
|
38
|
+
},
|
|
39
|
+
)
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
describe('createProxyFailure', () => {
|
|
43
|
+
it('preserves the original JSON-RPC id in the public error envelope', () => {
|
|
44
|
+
expect(createProxyFailure('request-17', { kind: 'stream' }).response).toEqual({
|
|
45
|
+
jsonrpc: '2.0',
|
|
46
|
+
id: 'request-17',
|
|
47
|
+
error: {
|
|
48
|
+
code: -32000,
|
|
49
|
+
message: 'Gemus upstream stream disconnected before the tool result was delivered',
|
|
50
|
+
},
|
|
51
|
+
})
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
it.each([
|
|
55
|
+
[
|
|
56
|
+
{
|
|
57
|
+
kind: 'connection',
|
|
58
|
+
code: 'UND_ERR_CONNECT_TIMEOUT',
|
|
59
|
+
reason: 'Authorization: Bearer secret at https://alice:password@example.test/api/mcp?token=secret',
|
|
60
|
+
addresses: ['203.0.113.1', '2001:db8::1'],
|
|
61
|
+
},
|
|
62
|
+
{ id: 41, kind: 'connection', code: 'UND_ERR_CONNECT_TIMEOUT' },
|
|
63
|
+
],
|
|
64
|
+
[
|
|
65
|
+
{
|
|
66
|
+
kind: 'http',
|
|
67
|
+
status: 503,
|
|
68
|
+
reason: 'https://alice:password@example.test/api/mcp?token=secret',
|
|
69
|
+
},
|
|
70
|
+
{ id: 41, kind: 'http', status: 503 },
|
|
71
|
+
],
|
|
72
|
+
])('keeps only safe diagnostic fields for $kind failures', (failure, expected) => {
|
|
73
|
+
const result = createProxyFailure(41, failure)
|
|
74
|
+
const serialized = JSON.stringify(result)
|
|
75
|
+
|
|
76
|
+
expect(result.diagnostic).toEqual(expected)
|
|
77
|
+
expect(serialized).not.toMatch(
|
|
78
|
+
/Authorization|Bearer|secret|alice|password|example\.test|203\.0\.113\.1|2001:db8/,
|
|
79
|
+
)
|
|
80
|
+
})
|
|
81
|
+
})
|
|
@@ -37,7 +37,7 @@ async function runProxy(
|
|
|
37
37
|
closeStdinAfterMs = 600,
|
|
38
38
|
input,
|
|
39
39
|
}: {
|
|
40
|
-
closeStdinAfterMs?: number
|
|
40
|
+
closeStdinAfterMs?: number | null
|
|
41
41
|
input?: Record<string, unknown>
|
|
42
42
|
} = {},
|
|
43
43
|
) {
|
|
@@ -69,10 +69,12 @@ async function runProxy(
|
|
|
69
69
|
child.on('error', reject)
|
|
70
70
|
if (input) child.stdin.write(`${JSON.stringify(input)}\n`)
|
|
71
71
|
|
|
72
|
-
const closeTimer =
|
|
72
|
+
const closeTimer = closeStdinAfterMs === null
|
|
73
|
+
? undefined
|
|
74
|
+
: setTimeout(() => child.stdin.end(), closeStdinAfterMs)
|
|
73
75
|
const killTimer = setTimeout(() => child.kill(), 5_000)
|
|
74
76
|
child.on('close', (code, signal) => {
|
|
75
|
-
clearTimeout(closeTimer)
|
|
77
|
+
if (closeTimer) clearTimeout(closeTimer)
|
|
76
78
|
clearTimeout(killTimer)
|
|
77
79
|
resolve({ code, signal, stdout, stderr })
|
|
78
80
|
})
|
|
@@ -337,6 +339,142 @@ describe('proxy process startup', () => {
|
|
|
337
339
|
}
|
|
338
340
|
})
|
|
339
341
|
|
|
342
|
+
it('shuts down the upstream transport before closing its HTTP adapter exactly once', async () => {
|
|
343
|
+
const tempRoot = await mkdtemp(path.join(tmpdir(), 'gemus-proxy-cleanup-'))
|
|
344
|
+
const constructionLogPath = path.join(tempRoot, 'construction.log')
|
|
345
|
+
try {
|
|
346
|
+
const result = await runProxy({
|
|
347
|
+
GEMUS_KEY: 'configured-key',
|
|
348
|
+
PROXY_LOG: undefined,
|
|
349
|
+
PROXY_TEST_CONSTRUCTION_LOG: constructionLogPath,
|
|
350
|
+
})
|
|
351
|
+
const events = (await readFile(constructionLogPath, 'utf8')).trim().split(/\r?\n/)
|
|
352
|
+
|
|
353
|
+
expect(result.code).toBe(0)
|
|
354
|
+
expect(result.signal).toBeNull()
|
|
355
|
+
expect(events.filter((event) => event === 'upstream-http-agent')).toHaveLength(1)
|
|
356
|
+
expect(events.filter((event) => event === 'upstream-http-shutdown')).toHaveLength(1)
|
|
357
|
+
expect(events.indexOf('upstream-terminate-session')).toBeLessThan(
|
|
358
|
+
events.indexOf('upstream-transport-close'),
|
|
359
|
+
)
|
|
360
|
+
expect(events.indexOf('upstream-transport-close')).toBeLessThan(
|
|
361
|
+
events.indexOf('upstream-http-shutdown'),
|
|
362
|
+
)
|
|
363
|
+
} finally {
|
|
364
|
+
await rm(tempRoot, { recursive: true, force: true })
|
|
365
|
+
}
|
|
366
|
+
})
|
|
367
|
+
|
|
368
|
+
it('bounds a stuck session DELETE before transport, adapter, and process shutdown', async () => {
|
|
369
|
+
const tempRoot = await mkdtemp(path.join(tmpdir(), 'gemus-proxy-bounded-cleanup-'))
|
|
370
|
+
const constructionLogPath = path.join(tempRoot, 'construction.log')
|
|
371
|
+
try {
|
|
372
|
+
const result = await runProxy({
|
|
373
|
+
GEMUS_KEY: 'configured-key',
|
|
374
|
+
PROXY_LOG: undefined,
|
|
375
|
+
PROXY_TEST_CONSTRUCTION_LOG: constructionLogPath,
|
|
376
|
+
PROXY_TEST_TERMINATE_SESSION_HANG: '1',
|
|
377
|
+
})
|
|
378
|
+
const events = (await readFile(constructionLogPath, 'utf8')).trim().split(/\r?\n/)
|
|
379
|
+
|
|
380
|
+
expect(result.code).toBe(0)
|
|
381
|
+
expect(result.signal).toBeNull()
|
|
382
|
+
expect(events.indexOf('upstream-terminate-session')).toBeLessThan(
|
|
383
|
+
events.indexOf('upstream-transport-close'),
|
|
384
|
+
)
|
|
385
|
+
expect(events.indexOf('upstream-transport-close')).toBeLessThan(
|
|
386
|
+
events.indexOf('upstream-http-shutdown'),
|
|
387
|
+
)
|
|
388
|
+
expect(events.filter((event) => event === 'upstream-http-shutdown')).toHaveLength(1)
|
|
389
|
+
} finally {
|
|
390
|
+
await rm(tempRoot, { recursive: true, force: true })
|
|
391
|
+
}
|
|
392
|
+
}, 10_000)
|
|
393
|
+
|
|
394
|
+
it('taps and drops a late real result without forwarding a second stdio response', async () => {
|
|
395
|
+
const tempRoot = await mkdtemp(path.join(tmpdir(), 'gemus-proxy-late-result-'))
|
|
396
|
+
const logPath = path.join(tempRoot, 'proxy.log')
|
|
397
|
+
const constructionLogPath = path.join(tempRoot, 'construction.log')
|
|
398
|
+
try {
|
|
399
|
+
const result = await runProxy(
|
|
400
|
+
{
|
|
401
|
+
GEMUS_KEY: 'configured-key',
|
|
402
|
+
PROXY_LOG: logPath,
|
|
403
|
+
PROXY_FAILFAST_TIMEOUT_MS: '25',
|
|
404
|
+
PROXY_TEST_CONSTRUCTION_LOG: constructionLogPath,
|
|
405
|
+
PROXY_TEST_FORWARD_STDIN: '1',
|
|
406
|
+
PROXY_TEST_LATE_RESULT: '1',
|
|
407
|
+
PROXY_TEST_STDIO_CAPTURE_SEND: '1',
|
|
408
|
+
},
|
|
409
|
+
{
|
|
410
|
+
input: {
|
|
411
|
+
jsonrpc: '2.0',
|
|
412
|
+
id: 76,
|
|
413
|
+
method: 'tools/call',
|
|
414
|
+
params: { name: 'execute', arguments: {} },
|
|
415
|
+
},
|
|
416
|
+
},
|
|
417
|
+
)
|
|
418
|
+
const log = await readFile(logPath, 'utf8')
|
|
419
|
+
const events = (await readFile(constructionLogPath, 'utf8')).trim().split(/\r?\n/)
|
|
420
|
+
const stdoutMessages = result.stdout.trim().split(/\r?\n/).map((line) => JSON.parse(line))
|
|
421
|
+
|
|
422
|
+
expect(result.code).toBe(0)
|
|
423
|
+
expect(result.signal).toBeNull()
|
|
424
|
+
expect(stdoutMessages).toEqual([
|
|
425
|
+
{
|
|
426
|
+
jsonrpc: '2.0',
|
|
427
|
+
id: 76,
|
|
428
|
+
error: {
|
|
429
|
+
code: -32000,
|
|
430
|
+
message: 'Gemus upstream stream became unresponsive before the tool result was delivered',
|
|
431
|
+
},
|
|
432
|
+
},
|
|
433
|
+
])
|
|
434
|
+
expect(events.filter((event) => event === 'backfiller-observe-call')).toHaveLength(1)
|
|
435
|
+
expect(events.filter((event) => event === 'backfiller-observe-result')).toHaveLength(1)
|
|
436
|
+
expect(log.match(/DROP-AFTER-FAILFAST/g)).toHaveLength(1)
|
|
437
|
+
expect(log).not.toContain('late-real-result')
|
|
438
|
+
} finally {
|
|
439
|
+
await rm(tempRoot, { recursive: true, force: true })
|
|
440
|
+
}
|
|
441
|
+
})
|
|
442
|
+
|
|
443
|
+
it('continues transport and adapter shutdown after terminateSession rejects on SIGTERM', async () => {
|
|
444
|
+
const tempRoot = await mkdtemp(path.join(tmpdir(), 'gemus-proxy-signal-cleanup-'))
|
|
445
|
+
const constructionLogPath = path.join(tempRoot, 'construction.log')
|
|
446
|
+
try {
|
|
447
|
+
const result = await runProxy(
|
|
448
|
+
{
|
|
449
|
+
GEMUS_KEY: 'configured-key',
|
|
450
|
+
PROXY_LOG: undefined,
|
|
451
|
+
PROXY_TEST_CONSTRUCTION_LOG: constructionLogPath,
|
|
452
|
+
PROXY_TEST_SIGNAL_AFTER_START: 'SIGTERM',
|
|
453
|
+
PROXY_TEST_TERMINATE_SESSION_REJECT: '1',
|
|
454
|
+
},
|
|
455
|
+
{
|
|
456
|
+
closeStdinAfterMs: null,
|
|
457
|
+
},
|
|
458
|
+
)
|
|
459
|
+
const events = (await readFile(constructionLogPath, 'utf8')).trim().split(/\r?\n/)
|
|
460
|
+
|
|
461
|
+
expect(result.code).toBe(0)
|
|
462
|
+
expect(result.signal).toBeNull()
|
|
463
|
+
expect(result.stderr).toBe('')
|
|
464
|
+
expect(events.filter((event) => event === 'upstream-terminate-session')).toHaveLength(1)
|
|
465
|
+
expect(events.filter((event) => event === 'upstream-transport-close')).toHaveLength(1)
|
|
466
|
+
expect(events.filter((event) => event === 'upstream-http-shutdown')).toHaveLength(1)
|
|
467
|
+
expect(events.indexOf('upstream-terminate-session')).toBeLessThan(
|
|
468
|
+
events.indexOf('upstream-transport-close'),
|
|
469
|
+
)
|
|
470
|
+
expect(events.indexOf('upstream-transport-close')).toBeLessThan(
|
|
471
|
+
events.indexOf('upstream-http-shutdown'),
|
|
472
|
+
)
|
|
473
|
+
} finally {
|
|
474
|
+
await rm(tempRoot, { recursive: true, force: true })
|
|
475
|
+
}
|
|
476
|
+
})
|
|
477
|
+
|
|
340
478
|
it('keeps exit code 1 when stdio onclose races with a startup error', async () => {
|
|
341
479
|
const secret = 'arbitrary-secret'
|
|
342
480
|
const result = await runProxy({
|
|
@@ -390,6 +528,62 @@ describe('proxy process startup', () => {
|
|
|
390
528
|
)
|
|
391
529
|
})
|
|
392
530
|
|
|
531
|
+
it('classifies an initialize fetch rejection without exposing its raw network error', async () => {
|
|
532
|
+
const tempRoot = await mkdtemp(path.join(tmpdir(), 'gemus-proxy-initialize-fetch-'))
|
|
533
|
+
const logPath = path.join(tempRoot, 'proxy.log')
|
|
534
|
+
const constructionLogPath = path.join(tempRoot, 'construction.log')
|
|
535
|
+
const secret = 'arbitrary-secret'
|
|
536
|
+
const endpoint =
|
|
537
|
+
'https://alice:password@example.test/api/mcp?token=query-secret#fragment-secret'
|
|
538
|
+
try {
|
|
539
|
+
const result = await runProxy(
|
|
540
|
+
{
|
|
541
|
+
GEMUS_KEY: secret,
|
|
542
|
+
GEMUS_URL: endpoint,
|
|
543
|
+
PROXY_LOG: logPath,
|
|
544
|
+
PROXY_TEST_CONSTRUCTION_LOG: constructionLogPath,
|
|
545
|
+
PROXY_TEST_FORWARD_STDIN: '1',
|
|
546
|
+
PROXY_TEST_UPSTREAM_INITIALIZE_FETCH_FAILURE: '1',
|
|
547
|
+
},
|
|
548
|
+
{
|
|
549
|
+
input: {
|
|
550
|
+
jsonrpc: '2.0',
|
|
551
|
+
id: 29,
|
|
552
|
+
method: 'initialize',
|
|
553
|
+
params: {
|
|
554
|
+
protocolVersion: '2025-06-18',
|
|
555
|
+
capabilities: {},
|
|
556
|
+
clientInfo: { name: 'startup-test', version: '1.0.0' },
|
|
557
|
+
},
|
|
558
|
+
},
|
|
559
|
+
},
|
|
560
|
+
)
|
|
561
|
+
const log = await readFileOrEmpty(logPath)
|
|
562
|
+
const events = (await readFile(constructionLogPath, 'utf8')).trim().split(/\r?\n/)
|
|
563
|
+
const fatalLines = log.split(/\r?\n/).filter((line) => line.includes(' FATAL '))
|
|
564
|
+
|
|
565
|
+
expect(result.code).toBe(1)
|
|
566
|
+
expect(result.signal).toBeNull()
|
|
567
|
+
expect(result.stderr).toBe(
|
|
568
|
+
'Gemus MCP proxy failed to start: Gemus upstream connection failed before a response was received\n',
|
|
569
|
+
)
|
|
570
|
+
expect(result.stderr.trim().split(/\r?\n/)).toHaveLength(1)
|
|
571
|
+
expect(events.filter((event) => event === 'upstream-http-agent')).toHaveLength(1)
|
|
572
|
+
expect(events.filter((event) => event === 'undici-fetch')).toHaveLength(1)
|
|
573
|
+
expect(events.filter((event) => event === 'upstream-http-shutdown')).toHaveLength(1)
|
|
574
|
+
expect(log).toContain('"kind":"connection","code":"UND_ERR_CONNECT_TIMEOUT"')
|
|
575
|
+
expect(fatalLines).toHaveLength(1)
|
|
576
|
+
for (const output of [result.stderr, log]) {
|
|
577
|
+
expect(output).not.toContain(endpoint)
|
|
578
|
+
expect(output).not.toMatch(
|
|
579
|
+
/arbitrary-secret|alice|password|query-secret|fragment-secret|203\.0\.113\.1|2001:db8/,
|
|
580
|
+
)
|
|
581
|
+
}
|
|
582
|
+
} finally {
|
|
583
|
+
await rm(tempRoot, { recursive: true, force: true })
|
|
584
|
+
}
|
|
585
|
+
})
|
|
586
|
+
|
|
393
587
|
it.each(['ftp', 'ws'])(
|
|
394
588
|
'sanitizes a credential-bearing %s endpoint in a real initialize failure',
|
|
395
589
|
async (scheme) => {
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { createServer, type Server } from 'node:http'
|
|
2
|
+
import type { Socket } from 'node:net'
|
|
3
|
+
import { afterEach, describe, expect, it } from 'vitest'
|
|
4
|
+
import { createUpstreamHttp } from '../upstreamHttp.mjs'
|
|
5
|
+
|
|
6
|
+
let server: Server | undefined
|
|
7
|
+
|
|
8
|
+
afterEach(async () => {
|
|
9
|
+
server?.closeAllConnections()
|
|
10
|
+
if (server?.listening) {
|
|
11
|
+
await new Promise<void>((resolve) => server?.close(() => resolve()))
|
|
12
|
+
}
|
|
13
|
+
server = undefined
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
describe('createUpstreamHttp with real Undici dispatchers', () => {
|
|
17
|
+
it('force-closes an established hanging response after the graceful budget expires', async () => {
|
|
18
|
+
let activeSocket!: Socket
|
|
19
|
+
server = createServer((request, response) => {
|
|
20
|
+
activeSocket = request.socket
|
|
21
|
+
response.writeHead(200, { 'content-type': 'text/event-stream' })
|
|
22
|
+
response.write(': heartbeat\n\n')
|
|
23
|
+
})
|
|
24
|
+
await new Promise<void>((resolve) => server?.listen(0, '127.0.0.1', resolve))
|
|
25
|
+
const address = server.address()
|
|
26
|
+
if (!address || typeof address === 'string') throw new Error('expected a TCP listener')
|
|
27
|
+
|
|
28
|
+
const upstream = createUpstreamHttp()
|
|
29
|
+
const response = await upstream.fetch(`http://127.0.0.1:${address.port}/api/mcp`)
|
|
30
|
+
expect(response.ok).toBe(true)
|
|
31
|
+
expect(activeSocket.destroyed).toBe(false)
|
|
32
|
+
|
|
33
|
+
await upstream.shutdown()
|
|
34
|
+
|
|
35
|
+
await expect.poll(() => activeSocket.destroyed, { timeout: 500 }).toBe(true)
|
|
36
|
+
}, 5000)
|
|
37
|
+
})
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
|
2
|
+
|
|
3
|
+
const undiciFetch = vi.fn()
|
|
4
|
+
const agents: Array<{ options: unknown; close: ReturnType<typeof vi.fn>; destroy: ReturnType<typeof vi.fn> }> = []
|
|
5
|
+
const pools: Array<{ origin: unknown; options: unknown; destroy: ReturnType<typeof vi.fn> }> = []
|
|
6
|
+
|
|
7
|
+
const Agent = vi.fn(function Agent(options) {
|
|
8
|
+
const agent = {
|
|
9
|
+
options,
|
|
10
|
+
close: vi.fn(() => Promise.resolve()),
|
|
11
|
+
destroy: vi.fn(),
|
|
12
|
+
}
|
|
13
|
+
agents.push(agent)
|
|
14
|
+
return agent
|
|
15
|
+
})
|
|
16
|
+
|
|
17
|
+
const Pool = vi.fn(function Pool(origin, options) {
|
|
18
|
+
const pool = {
|
|
19
|
+
origin,
|
|
20
|
+
options,
|
|
21
|
+
destroy: vi.fn(() => Promise.resolve()),
|
|
22
|
+
}
|
|
23
|
+
pools.push(pool)
|
|
24
|
+
return pool
|
|
25
|
+
})
|
|
26
|
+
|
|
27
|
+
vi.mock('undici', () => ({ Agent, Pool, fetch: undiciFetch }))
|
|
28
|
+
|
|
29
|
+
async function createAdapter() {
|
|
30
|
+
return (await import('../upstreamHttp.mjs')).createUpstreamHttp()
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
beforeEach(() => {
|
|
34
|
+
agents.length = 0
|
|
35
|
+
pools.length = 0
|
|
36
|
+
Agent.mockClear()
|
|
37
|
+
Pool.mockClear()
|
|
38
|
+
undiciFetch.mockReset()
|
|
39
|
+
delete process.env.PROXY_CONNECT_ATTEMPT_TIMEOUT_MS
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
afterEach(() => {
|
|
43
|
+
vi.useRealTimers()
|
|
44
|
+
delete process.env.PROXY_CONNECT_ATTEMPT_TIMEOUT_MS
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
describe('createUpstreamHttp', () => {
|
|
48
|
+
it('creates one Agent with the 1000ms address-family attempt window by default', async () => {
|
|
49
|
+
const upstream = await createAdapter()
|
|
50
|
+
|
|
51
|
+
expect(upstream.attemptTimeoutMs).toBe(1000)
|
|
52
|
+
expect(Agent).toHaveBeenCalledTimes(1)
|
|
53
|
+
expect(agents[0].options).toEqual({
|
|
54
|
+
factory: expect.any(Function),
|
|
55
|
+
connect: {
|
|
56
|
+
autoSelectFamily: true,
|
|
57
|
+
autoSelectFamilyAttemptTimeout: 1000,
|
|
58
|
+
},
|
|
59
|
+
})
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
it.each([
|
|
63
|
+
['', 1000],
|
|
64
|
+
['not-a-number', 1000],
|
|
65
|
+
['Infinity', 1000],
|
|
66
|
+
['0', 1000],
|
|
67
|
+
['-1', 1000],
|
|
68
|
+
['9.9', 10],
|
|
69
|
+
['10.9', 10],
|
|
70
|
+
['251.9', 251],
|
|
71
|
+
])('normalizes PROXY_CONNECT_ATTEMPT_TIMEOUT_MS=%j to %i milliseconds', async (value, expected) => {
|
|
72
|
+
process.env.PROXY_CONNECT_ATTEMPT_TIMEOUT_MS = value
|
|
73
|
+
|
|
74
|
+
const upstream = await createAdapter()
|
|
75
|
+
|
|
76
|
+
expect(upstream.attemptTimeoutMs).toBe(expected)
|
|
77
|
+
expect(agents[0].options).toEqual({
|
|
78
|
+
factory: expect.any(Function),
|
|
79
|
+
connect: {
|
|
80
|
+
autoSelectFamily: true,
|
|
81
|
+
autoSelectFamilyAttemptTimeout: expected,
|
|
82
|
+
},
|
|
83
|
+
})
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
it('preserves transport request init while injecting its dispatcher', async () => {
|
|
87
|
+
const upstream = await createAdapter()
|
|
88
|
+
const signal = new AbortController().signal
|
|
89
|
+
const init = {
|
|
90
|
+
method: 'POST',
|
|
91
|
+
headers: { authorization: 'Bearer example', accept: 'text/event-stream' },
|
|
92
|
+
body: '{"jsonrpc":"2.0","id":7}',
|
|
93
|
+
signal,
|
|
94
|
+
cache: 'no-store' as RequestCache,
|
|
95
|
+
redirect: 'manual' as RequestRedirect,
|
|
96
|
+
}
|
|
97
|
+
undiciFetch.mockResolvedValue(new Response('ok'))
|
|
98
|
+
|
|
99
|
+
await upstream.fetch('https://example.test/api/mcp', init)
|
|
100
|
+
|
|
101
|
+
expect(undiciFetch).toHaveBeenCalledTimes(1)
|
|
102
|
+
expect(undiciFetch).toHaveBeenCalledWith('https://example.test/api/mcp', {
|
|
103
|
+
...init,
|
|
104
|
+
dispatcher: agents[0],
|
|
105
|
+
})
|
|
106
|
+
expect(init).not.toHaveProperty('dispatcher')
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
it('propagates a failed request after one fetch attempt', async () => {
|
|
110
|
+
const upstream = await createAdapter()
|
|
111
|
+
undiciFetch.mockRejectedValue(new Error('connect failed'))
|
|
112
|
+
|
|
113
|
+
await expect(upstream.fetch('https://example.test/api/mcp', { method: 'POST', body: 'side-effecting' })).rejects.toThrow('connect failed')
|
|
114
|
+
|
|
115
|
+
expect(undiciFetch).toHaveBeenCalledTimes(1)
|
|
116
|
+
})
|
|
117
|
+
|
|
118
|
+
it('shares one graceful shutdown across repeated calls', async () => {
|
|
119
|
+
const upstream = await createAdapter()
|
|
120
|
+
let finishClose!: () => void
|
|
121
|
+
agents[0].close.mockImplementation(() => new Promise<void>((resolve) => { finishClose = resolve }))
|
|
122
|
+
|
|
123
|
+
const first = upstream.shutdown()
|
|
124
|
+
const second = upstream.shutdown()
|
|
125
|
+
expect(second).toBe(first)
|
|
126
|
+
expect(agents[0].close).toHaveBeenCalledTimes(1)
|
|
127
|
+
|
|
128
|
+
finishClose()
|
|
129
|
+
await first
|
|
130
|
+
await upstream.shutdown()
|
|
131
|
+
expect(agents[0].close).toHaveBeenCalledTimes(1)
|
|
132
|
+
expect(agents[0].destroy).not.toHaveBeenCalled()
|
|
133
|
+
})
|
|
134
|
+
|
|
135
|
+
it('destroys retained origin dispatchers and the Agent when graceful close exceeds 2000ms', async () => {
|
|
136
|
+
vi.useFakeTimers()
|
|
137
|
+
const upstream = await createAdapter()
|
|
138
|
+
agents[0].close.mockImplementation(() => new Promise<void>(() => {}))
|
|
139
|
+
const factory = (agents[0].options as { factory: (origin: string, options: object) => unknown }).factory
|
|
140
|
+
factory('https://example.test', { connections: 2 })
|
|
141
|
+
|
|
142
|
+
const shutdown = upstream.shutdown()
|
|
143
|
+
await vi.advanceTimersByTimeAsync(2000)
|
|
144
|
+
await shutdown
|
|
145
|
+
|
|
146
|
+
expect(agents[0].close).toHaveBeenCalledTimes(1)
|
|
147
|
+
expect(agents[0].destroy).toHaveBeenCalledTimes(1)
|
|
148
|
+
expect(pools[0].destroy).toHaveBeenCalledTimes(1)
|
|
149
|
+
expect(upstream.shutdown()).toBe(shutdown)
|
|
150
|
+
})
|
|
151
|
+
})
|
package/src/failFast.mjs
CHANGED
|
@@ -93,11 +93,24 @@ export function createSseIdScanner() {
|
|
|
93
93
|
}
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
/** Extract an allowlisted network error code without exposing the raw error text. */
|
|
97
|
+
function failureCode(error) {
|
|
98
|
+
let current = error
|
|
99
|
+
for (let depth = 0; depth < 8 && current && typeof current === 'object'; depth++) {
|
|
100
|
+
if (typeof current.code === 'string') {
|
|
101
|
+
const code = current.code.toUpperCase()
|
|
102
|
+
if (/^[A-Z0-9_]{1,64}$/.test(code)) return code
|
|
103
|
+
}
|
|
104
|
+
current = current.cause
|
|
105
|
+
}
|
|
106
|
+
return 'UNKNOWN'
|
|
107
|
+
}
|
|
108
|
+
|
|
96
109
|
/**
|
|
97
110
|
* Build the fail-fast fetch wrapper + settle/dispose controls.
|
|
98
111
|
*
|
|
99
112
|
* @param {object} opts
|
|
100
|
-
* @param {(id:any,
|
|
113
|
+
* @param {(id:any, failure:{kind:string, code?:string, status?:number})=>void} opts.onLost called once per lost id (must be idempotent-safe on caller side)
|
|
101
114
|
* @param {number} [opts.idleTimeoutMs] IDLE backstop reset on every upstream byte (incl. server
|
|
102
115
|
* heartbeats every ~12s), so it trips ONLY on a genuinely silent/zombie connection — never on a
|
|
103
116
|
* healthy-but-slow tool. Must be < Codex 300s; default 120_000.
|
|
@@ -117,7 +130,7 @@ export function createFailFast(opts) {
|
|
|
117
130
|
const failedIds = new Set()
|
|
118
131
|
|
|
119
132
|
function armTimer(entry, id) {
|
|
120
|
-
entry.timer = setTimeout(() => lose(id, '
|
|
133
|
+
entry.timer = setTimeout(() => lose(id, { kind: 'idle' }), idleTimeoutMs)
|
|
121
134
|
if (typeof entry.timer.unref === 'function') entry.timer.unref()
|
|
122
135
|
}
|
|
123
136
|
|
|
@@ -150,15 +163,15 @@ export function createFailFast(opts) {
|
|
|
150
163
|
failedIds.delete(String(id))
|
|
151
164
|
}
|
|
152
165
|
|
|
153
|
-
function lose(id,
|
|
166
|
+
function lose(id, failure) {
|
|
154
167
|
const key = String(id)
|
|
155
168
|
if (!inFlight.has(key)) return // already settled or already lost
|
|
156
169
|
settle(id)
|
|
157
170
|
failedIds.add(key)
|
|
158
171
|
// Bound the set: disconnect-case entries never see a late result to forget(), so cap FIFO.
|
|
159
172
|
if (failedIds.size > FAILED_IDS_CAP) failedIds.delete(failedIds.values().next().value)
|
|
160
|
-
log('FAILFAST', { id,
|
|
161
|
-
onLost(id,
|
|
173
|
+
log('FAILFAST', { id, ...failure })
|
|
174
|
+
onLost(id, failure)
|
|
162
175
|
}
|
|
163
176
|
|
|
164
177
|
function observeAndForward(body, codexIds) {
|
|
@@ -168,7 +181,7 @@ export function createFailFast(opts) {
|
|
|
168
181
|
const onEnd = () => {
|
|
169
182
|
if (ended) return
|
|
170
183
|
ended = true
|
|
171
|
-
for (const id of codexIds) lose(id, '
|
|
184
|
+
for (const id of codexIds) lose(id, { kind: 'stream' })
|
|
172
185
|
}
|
|
173
186
|
return new ReadableStream({
|
|
174
187
|
async pull(controller) {
|
|
@@ -218,7 +231,7 @@ export function createFailFast(opts) {
|
|
|
218
231
|
codexIds = parseRequestIds(init).filter(isCodexOriginated)
|
|
219
232
|
for (const id of codexIds) register(id)
|
|
220
233
|
} catch (e) {
|
|
221
|
-
log('FAILFAST-WRAP-ERR',
|
|
234
|
+
log('FAILFAST-WRAP-ERR', { code: failureCode(e) })
|
|
222
235
|
codexIds = []
|
|
223
236
|
}
|
|
224
237
|
|
|
@@ -229,7 +242,7 @@ export function createFailFast(opts) {
|
|
|
229
242
|
res = await realFetch(url, init)
|
|
230
243
|
} catch (e) {
|
|
231
244
|
// Connection failed before any response (DNS / TLS / RST) — fail fast, don't wait 270s.
|
|
232
|
-
for (const id of codexIds) lose(id,
|
|
245
|
+
for (const id of codexIds) lose(id, { kind: 'connection', code: failureCode(e) })
|
|
233
246
|
throw e // preserve the SDK's existing onerror/logging path (no re-fetch)
|
|
234
247
|
}
|
|
235
248
|
|
|
@@ -241,7 +254,7 @@ export function createFailFast(opts) {
|
|
|
241
254
|
if (!res.ok) {
|
|
242
255
|
// Non-SSE HTTP error (404 session-gone / 403 / 429 / …). send() throws before onmessage,
|
|
243
256
|
// and that rejection is swallowed by upstream.send().catch — so nothing ever settles the id.
|
|
244
|
-
for (const id of codexIds) lose(id,
|
|
257
|
+
for (const id of codexIds) lose(id, { kind: 'http', status: res.status })
|
|
245
258
|
return res
|
|
246
259
|
}
|
|
247
260
|
const ct = res.headers.get('content-type') || ''
|
|
@@ -252,7 +265,7 @@ export function createFailFast(opts) {
|
|
|
252
265
|
headers: res.headers,
|
|
253
266
|
})
|
|
254
267
|
} catch (e) {
|
|
255
|
-
log('FAILFAST-WRAP-ERR',
|
|
268
|
+
log('FAILFAST-WRAP-ERR', { code: failureCode(e) })
|
|
256
269
|
return res
|
|
257
270
|
}
|
|
258
271
|
}
|
package/src/proxy.mjs
CHANGED
|
@@ -28,11 +28,14 @@ import { createBackfiller, injectSteering } from './backfill.mjs'
|
|
|
28
28
|
import { unwrapMcpContent } from '@gemus/codex-backfill-core'
|
|
29
29
|
import { buildProxyUpstreamHeaders } from './mcpHeaders.mjs'
|
|
30
30
|
import { createFailFast, isJsonRpcResponse } from './failFast.mjs'
|
|
31
|
+
import { createProxyFailure } from './proxyMessages.mjs'
|
|
32
|
+
import { createUpstreamHttp } from './upstreamHttp.mjs'
|
|
31
33
|
import {
|
|
32
34
|
assertGemusKey,
|
|
33
35
|
createFatalReporter,
|
|
34
36
|
createShutdownCoordinator,
|
|
35
37
|
safeEndpointLabel,
|
|
38
|
+
settleWithin,
|
|
36
39
|
startProxy,
|
|
37
40
|
} from './startup.mjs'
|
|
38
41
|
|
|
@@ -284,6 +287,8 @@ if (typeof hookServer.unref === 'function') hookServer.unref()
|
|
|
284
287
|
|
|
285
288
|
// ── transparent JSON-RPC passthrough (raw send/onmessage) ──────────────────────────────────────
|
|
286
289
|
const stdio = new StdioServerTransport()
|
|
290
|
+
const initializingIds = new Set()
|
|
291
|
+
const classifiedInitializeFailures = new Set()
|
|
287
292
|
|
|
288
293
|
// #2068 fail-fast: StreamableHTTPClientTransport.send() resolves before the upstream SSE result
|
|
289
294
|
// arrives and only surfaces a stream death via a global, id-less `onerror` (no reconnect for POST).
|
|
@@ -292,17 +297,21 @@ const stdio = new StdioServerTransport()
|
|
|
292
297
|
// of hanging to its 300s timeout. onLost only injects to stdio — it does NOT tap: tapping here would
|
|
293
298
|
// delete the proxy's `inflight` entry, so the DROP-AFTER-FAILFAST branch below could no longer let
|
|
294
299
|
// backfill observe a late real result (env delivery on the still-open, idle-timed-out case).
|
|
300
|
+
const upstreamHttp = createUpstreamHttp()
|
|
295
301
|
const failFast = createFailFast({
|
|
296
302
|
idleTimeoutMs: Number(process.env.PROXY_FAILFAST_TIMEOUT_MS) || 120_000,
|
|
303
|
+
realFetch: upstreamHttp.fetch,
|
|
297
304
|
log,
|
|
298
|
-
onLost: (id,
|
|
299
|
-
const
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
305
|
+
onLost: (id, failure) => {
|
|
306
|
+
const { response, diagnostic } = createProxyFailure(id, failure)
|
|
307
|
+
const key = String(id)
|
|
308
|
+
log('FAILFAST-INJECT', diagnostic)
|
|
309
|
+
if (initializingIds.has(key)) {
|
|
310
|
+
classifiedInitializeFailures.add(key)
|
|
311
|
+
void terminateFatally(new Error(response.error.message))
|
|
312
|
+
return
|
|
303
313
|
}
|
|
304
|
-
log('
|
|
305
|
-
stdio.send(errObj).catch((e) => log('ERR-stdio-send', String(e)))
|
|
314
|
+
stdio.send(response).catch((e) => log('ERR-stdio-send', String(e)))
|
|
306
315
|
},
|
|
307
316
|
})
|
|
308
317
|
|
|
@@ -314,12 +323,17 @@ const upstream = new StreamableHTTPClientTransport(new URL(GEMUS_URL), {
|
|
|
314
323
|
stdio.onmessage = (m) => {
|
|
315
324
|
tapClientToServer(m)
|
|
316
325
|
log('C->G', m)
|
|
326
|
+
const key = String(m?.id)
|
|
327
|
+
if (m?.method === 'initialize') initializingIds.add(key)
|
|
317
328
|
upstream.send(m).catch((error) => {
|
|
329
|
+
const classified = classifiedInitializeFailures.delete(key)
|
|
330
|
+
initializingIds.delete(key)
|
|
318
331
|
if (m?.method === 'initialize') {
|
|
332
|
+
if (classified) return
|
|
319
333
|
void terminateFatally(error)
|
|
320
334
|
return
|
|
321
335
|
}
|
|
322
|
-
log('ERR-http-send',
|
|
336
|
+
log('ERR-http-send', { kind: 'transport' })
|
|
323
337
|
})
|
|
324
338
|
}
|
|
325
339
|
upstream.onmessage = (m) => {
|
|
@@ -338,6 +352,7 @@ upstream.onmessage = (m) => {
|
|
|
338
352
|
// (elicitation/sampling) also carries an id that can collide with an in-flight tools/call id, and
|
|
339
353
|
// must be forwarded to Codex untouched, never settled/dropped.
|
|
340
354
|
if (isJsonRpcResponse(m)) {
|
|
355
|
+
initializingIds.delete(String(m.id))
|
|
341
356
|
if (failFast.failedIds.has(String(m.id))) {
|
|
342
357
|
// Already fail-fasted (idle timeout on a still-open stream) → Codex got the error. Still tap so
|
|
343
358
|
// backfill (#1751) observes the late real result (inflight survives — onLost didn't tap), but
|
|
@@ -358,6 +373,7 @@ upstream.onmessage = (m) => {
|
|
|
358
373
|
// **independent of Codex tool activity**, so a proxy mid-imagegen (silent to Codex for minutes) still
|
|
359
374
|
// proves liveness → the platform reclaims only genuinely-dead sessions (short floor), never a live one.
|
|
360
375
|
const HEARTBEAT_MS = Number(process.env.PROXY_HEARTBEAT_MS) || 30_000
|
|
376
|
+
const SHUTDOWN_STEP_TIMEOUT_MS = 2000
|
|
361
377
|
let pingSeq = 0
|
|
362
378
|
let heartbeatTimer
|
|
363
379
|
function startHeartbeat() {
|
|
@@ -377,8 +393,15 @@ async function cleanup() {
|
|
|
377
393
|
failFast.dispose() // #2068: clear fail-fast final-safety timers
|
|
378
394
|
try { hookServer.close() } catch { /* ignore */ }
|
|
379
395
|
if (discoveryFile) { try { fs.unlinkSync(discoveryFile) } catch { /* ignore */ } }
|
|
380
|
-
|
|
381
|
-
|
|
396
|
+
await settleWithin(
|
|
397
|
+
Promise.resolve().then(() => upstream.terminateSession()),
|
|
398
|
+
SHUTDOWN_STEP_TIMEOUT_MS,
|
|
399
|
+
)
|
|
400
|
+
await settleWithin(
|
|
401
|
+
Promise.resolve().then(() => upstream.close()),
|
|
402
|
+
SHUTDOWN_STEP_TIMEOUT_MS,
|
|
403
|
+
)
|
|
404
|
+
try { await upstreamHttp.shutdown() } catch { /* ignore */ }
|
|
382
405
|
}
|
|
383
406
|
shutdown = createShutdownCoordinator({
|
|
384
407
|
cleanup,
|
|
@@ -391,7 +414,7 @@ function shutdownGracefully() {
|
|
|
391
414
|
stdio.onclose = () => { log('stdio-close', ''); void shutdownGracefully() }
|
|
392
415
|
upstream.onclose = () => { log('http-close', '') }
|
|
393
416
|
stdio.onerror = (e) => log('stdio-err', String(e))
|
|
394
|
-
upstream.onerror = (
|
|
417
|
+
upstream.onerror = () => log('http-err', { kind: 'transport' })
|
|
395
418
|
process.on('SIGTERM', () => { void shutdownGracefully() })
|
|
396
419
|
process.on('SIGINT', () => { void shutdownGracefully() })
|
|
397
420
|
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
const PUBLIC_MESSAGES = {
|
|
2
|
+
connection: 'Gemus upstream connection failed before a response was received',
|
|
3
|
+
http: 'Gemus upstream returned an HTTP error before the tool result was delivered',
|
|
4
|
+
stream: 'Gemus upstream stream disconnected before the tool result was delivered',
|
|
5
|
+
idle: 'Gemus upstream stream became unresponsive before the tool result was delivered',
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
const FALLBACK_MESSAGE = 'Gemus upstream failed before the tool result was delivered'
|
|
9
|
+
const SAFE_CODE = /^[A-Z0-9_]{1,64}$/
|
|
10
|
+
|
|
11
|
+
export function publicMessageForFailure(failure) {
|
|
12
|
+
return Object.hasOwn(PUBLIC_MESSAGES, failure?.kind)
|
|
13
|
+
? PUBLIC_MESSAGES[failure.kind]
|
|
14
|
+
: FALLBACK_MESSAGE
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function createProxyFailure(id, failure) {
|
|
18
|
+
const knownKind = Object.hasOwn(PUBLIC_MESSAGES, failure?.kind)
|
|
19
|
+
const kind = knownKind ? failure.kind : 'unknown'
|
|
20
|
+
const diagnostic = { id, kind }
|
|
21
|
+
|
|
22
|
+
if (kind === 'connection') {
|
|
23
|
+
diagnostic.code = typeof failure.code === 'string' && SAFE_CODE.test(failure.code)
|
|
24
|
+
? failure.code
|
|
25
|
+
: 'UNKNOWN'
|
|
26
|
+
} else if (kind === 'http' && Number.isInteger(failure.status)) {
|
|
27
|
+
diagnostic.status = failure.status
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return {
|
|
31
|
+
response: {
|
|
32
|
+
jsonrpc: '2.0',
|
|
33
|
+
id,
|
|
34
|
+
error: { code: -32000, message: publicMessageForFailure(failure) },
|
|
35
|
+
},
|
|
36
|
+
diagnostic,
|
|
37
|
+
}
|
|
38
|
+
}
|
package/src/startup.mjs
CHANGED
|
@@ -92,3 +92,19 @@ export function createShutdownCoordinator({ cleanup, exit }) {
|
|
|
92
92
|
return cleanupPromise
|
|
93
93
|
}
|
|
94
94
|
}
|
|
95
|
+
|
|
96
|
+
export function settleWithin(promise, timeoutMs) {
|
|
97
|
+
return new Promise((resolve) => {
|
|
98
|
+
const timer = setTimeout(() => resolve('timed_out'), timeoutMs)
|
|
99
|
+
Promise.resolve(promise).then(
|
|
100
|
+
() => {
|
|
101
|
+
clearTimeout(timer)
|
|
102
|
+
resolve('settled')
|
|
103
|
+
},
|
|
104
|
+
() => {
|
|
105
|
+
clearTimeout(timer)
|
|
106
|
+
resolve('settled')
|
|
107
|
+
},
|
|
108
|
+
)
|
|
109
|
+
})
|
|
110
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { Agent, Pool, fetch as undiciFetch } from 'undici'
|
|
2
|
+
|
|
3
|
+
const DEFAULT_ATTEMPT_TIMEOUT_MS = 1000
|
|
4
|
+
const MIN_ATTEMPT_TIMEOUT_MS = 10
|
|
5
|
+
const SHUTDOWN_TIMEOUT_MS = 2000
|
|
6
|
+
|
|
7
|
+
function attemptTimeoutFromEnvironment(value) {
|
|
8
|
+
const parsed = Number(value)
|
|
9
|
+
if (!value || !Number.isFinite(parsed) || parsed <= 0) return DEFAULT_ATTEMPT_TIMEOUT_MS
|
|
10
|
+
return Math.max(MIN_ATTEMPT_TIMEOUT_MS, Math.trunc(parsed))
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function settleClose(closePromise) {
|
|
14
|
+
return new Promise((resolve) => {
|
|
15
|
+
const timer = setTimeout(() => resolve('timed_out'), SHUTDOWN_TIMEOUT_MS)
|
|
16
|
+
Promise.resolve(closePromise).then(
|
|
17
|
+
() => {
|
|
18
|
+
clearTimeout(timer)
|
|
19
|
+
resolve('settled')
|
|
20
|
+
},
|
|
21
|
+
() => {
|
|
22
|
+
clearTimeout(timer)
|
|
23
|
+
resolve('rejected')
|
|
24
|
+
},
|
|
25
|
+
)
|
|
26
|
+
})
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Owns the MCP upstream dispatcher and its bounded shutdown. */
|
|
30
|
+
export function createUpstreamHttp() {
|
|
31
|
+
const attemptTimeoutMs = attemptTimeoutFromEnvironment(process.env.PROXY_CONNECT_ATTEMPT_TIMEOUT_MS)
|
|
32
|
+
const originDispatchers = new Set()
|
|
33
|
+
const dispatcher = new Agent({
|
|
34
|
+
factory(origin, options) {
|
|
35
|
+
const originDispatcher = new Pool(origin, options)
|
|
36
|
+
originDispatchers.add(originDispatcher)
|
|
37
|
+
return originDispatcher
|
|
38
|
+
},
|
|
39
|
+
connect: {
|
|
40
|
+
autoSelectFamily: true,
|
|
41
|
+
autoSelectFamilyAttemptTimeout: attemptTimeoutMs,
|
|
42
|
+
},
|
|
43
|
+
})
|
|
44
|
+
let shutdownPromise
|
|
45
|
+
|
|
46
|
+
function upstreamFetch(input, init) {
|
|
47
|
+
return undiciFetch(input, { ...init, dispatcher })
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function shutdown() {
|
|
51
|
+
if (!shutdownPromise) {
|
|
52
|
+
shutdownPromise = (async () => {
|
|
53
|
+
const closePromise = dispatcher.close()
|
|
54
|
+
const outcome = await settleClose(closePromise)
|
|
55
|
+
if (outcome !== 'settled') {
|
|
56
|
+
const error = new Error('MCP upstream dispatcher shutdown timed out')
|
|
57
|
+
await Promise.allSettled([
|
|
58
|
+
dispatcher.destroy(error),
|
|
59
|
+
...Array.from(originDispatchers, (originDispatcher) => originDispatcher.destroy(error)),
|
|
60
|
+
])
|
|
61
|
+
}
|
|
62
|
+
originDispatchers.clear()
|
|
63
|
+
})()
|
|
64
|
+
}
|
|
65
|
+
return shutdownPromise
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return { fetch: upstreamFetch, shutdown, attemptTimeoutMs }
|
|
69
|
+
}
|