gogcli-mcp 3.0.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/.claude-plugin/marketplace.json +2 -2
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/dist/index.js +273 -748
  4. package/dist/lib.js +375 -785
  5. package/manifest.json +2 -2
  6. package/mint.yaml +37 -33
  7. package/package.json +3 -3
  8. package/server.json +2 -2
  9. package/src/attachments.ts +28 -34
  10. package/src/blob-upload.ts +165 -134
  11. package/src/blob-urls.ts +3 -5
  12. package/src/bootstrap-auth.ts +97 -0
  13. package/src/index.ts +3 -4
  14. package/src/lib.ts +14 -9
  15. package/src/runner.ts +23 -175
  16. package/src/tools/appscript.ts +1 -1
  17. package/src/tools/auth.ts +5 -5
  18. package/src/tools/drive.ts +1 -3
  19. package/src/tools/gmail.ts +9 -9
  20. package/src/tools/utils.ts +12 -58
  21. package/tests/attachments.test.ts +11 -14
  22. package/tests/blob-upload.test.ts +235 -160
  23. package/tests/bootstrap-auth.test.ts +245 -0
  24. package/tests/runner-file-args.test.ts +1 -13
  25. package/tests/runner.test.ts +8 -95
  26. package/tests/sdk-single-copy.test.ts +4 -12
  27. package/tests/tools/auth-401-shapes.test.ts +2 -3
  28. package/tests/tools/auth.test.ts +5 -4
  29. package/tests/tools/drive.test.ts +11 -3
  30. package/tests/tools/gmail.test.ts +2 -2
  31. package/tests/tools/utils.test.ts +1 -50
  32. package/tests/zod-single-copy.test.ts +6 -14
  33. package/tsconfig.json +1 -2
  34. package/vitest.config.ts +2 -12
  35. package/src/auth-log.ts +0 -205
  36. package/src/connector-auth.ts +0 -319
  37. package/src/connector-login.ts +0 -87
  38. package/src/connector-runtime.ts +0 -910
  39. package/src/google-probe.ts +0 -113
  40. package/src/google-token.ts +0 -391
  41. package/src/remote-runner.ts +0 -77
  42. package/src/worker.ts +0 -117
  43. package/tests/auth-log.test.ts +0 -530
  44. package/tests/connector-auth.test.ts +0 -559
  45. package/tests/connector-login.test.ts +0 -151
  46. package/tests/connector-runtime.test.ts +0 -1664
  47. package/tests/google-probe.test.ts +0 -116
  48. package/tests/google-token.test.ts +0 -425
  49. package/tests/remote-runner.test.ts +0 -202
  50. package/tests/worker.test.ts +0 -167
@@ -1,1664 +0,0 @@
1
- import { describe, it, expect, vi, afterEach } from 'vitest';
2
- import { run, runExecutor } from '../src/runner.js';
3
- import type { GogArg, GogExecutor, GogFileArg } from '../src/runner.js';
4
- import { createFlyExecutorResolver, makeFlyExecutor, wrapServer, RunnerTransportError, isRunnerTransportError } from '../src/connector-runtime.js';
5
- import { runOrDiagnose } from '../src/tools/utils.js';
6
- import { makeAccessTokenSource, clearAccessTokenCache } from '../src/google-token.js';
7
-
8
- afterEach(() => {
9
- vi.unstubAllEnvs();
10
- vi.unstubAllGlobals();
11
- vi.restoreAllMocks();
12
- });
13
-
14
- describe('createFlyExecutorResolver', () => {
15
- it('reuses one executor for the same endpoint and key while isolating different credentials', () => {
16
- const factory = vi.fn((_endpoint: string, _key: string): GogExecutor => vi.fn());
17
- const resolve = createFlyExecutorResolver(factory);
18
-
19
- const first = resolve('https://runner.example', 'key-a');
20
- expect(resolve('https://runner.example', 'key-a')).toBe(first);
21
- expect(resolve('https://runner.example', 'key-b')).not.toBe(first);
22
- expect(resolve('https://other.example', 'key-a')).not.toBe(first);
23
- expect(factory).toHaveBeenCalledTimes(3);
24
- });
25
-
26
- it('uses makeFlyExecutor by default', () => {
27
- const resolve = createFlyExecutorResolver();
28
- expect(resolve('https://runner.example', 'key-a')).toBe(
29
- resolve('https://runner.example', 'key-a'),
30
- );
31
- });
32
- });
33
-
34
- describe('wrapServer', () => {
35
- // Neutralize ambient GOG_ACCOUNT / GOG_READONLY so run()'s assembled arg list
36
- // is deterministic regardless of the shell the suite runs in.
37
- function stubEnv() {
38
- vi.stubEnv('GOG_ACCOUNT', '');
39
- vi.stubEnv('GOG_READONLY', '');
40
- }
41
-
42
- it('scopes each registerTool handler so run() forwards to the injected executor', async () => {
43
- stubEnv();
44
- let captured: ((...a: unknown[]) => unknown) | undefined;
45
- const server = {
46
- registerTool(_name: string, _config: unknown, handler: (...a: unknown[]) => unknown) {
47
- captured = handler;
48
- return 'registered';
49
- },
50
- };
51
- const executor: GogExecutor = vi.fn(async () => 'MOCK_STDOUT');
52
-
53
- const wrapped = wrapServer(server, executor);
54
- // The unchanged registrar registers a handler that calls run() — exactly
55
- // what the real base registrars do.
56
- const ret = wrapped.registerTool('gog_sheets_get', {}, async () =>
57
- run(['sheets', 'get', 'A1']),
58
- );
59
- expect(ret).toBe('registered'); // the original return flows back through
60
-
61
- // Invoking the wrapped handler must resolve run()'s executor to ours.
62
- const out = await captured!({ some: 'args' }, { extra: true });
63
- expect(out).toBe('MOCK_STDOUT');
64
- expect(executor).toHaveBeenCalledTimes(1);
65
- expect(executor).toHaveBeenCalledWith(
66
- ['--json', '--color=never', '--no-input', 'sheets', 'get', 'A1'],
67
- expect.anything(),
68
- );
69
- });
70
-
71
- it('also intercepts the low-level `tool` registration method', async () => {
72
- stubEnv();
73
- let captured: ((...a: unknown[]) => unknown) | undefined;
74
- const server = {
75
- tool(_name: string, handler: (...a: unknown[]) => unknown) {
76
- captured = handler;
77
- },
78
- };
79
- const executor: GogExecutor = vi.fn(async () => 'VIA_TOOL');
80
- const wrapped = wrapServer(server, executor);
81
- wrapped.tool('t', async () => run(['gmail', 'search', 'q']));
82
- const out = await captured!();
83
- expect(out).toBe('VIA_TOOL');
84
- expect(executor).toHaveBeenCalledWith(
85
- ['--json', '--color=never', '--no-input', 'gmail', 'search', 'q'],
86
- expect.anything(),
87
- );
88
- });
89
-
90
- it('passes a non-function trailing arg straight through (no wrapping)', () => {
91
- const calls: unknown[][] = [];
92
- const server = {
93
- registerTool(...args: unknown[]) {
94
- calls.push(args);
95
- return 'ok';
96
- },
97
- };
98
- const executor: GogExecutor = vi.fn();
99
- const wrapped = wrapServer(server, executor);
100
- // Only a name, no handler — nothing to wrap.
101
- expect(wrapped.registerTool('just-a-name')).toBe('ok');
102
- expect(calls).toEqual([['just-a-name']]);
103
- expect(executor).not.toHaveBeenCalled();
104
- });
105
-
106
- it('proxies non-registration properties and methods through unchanged', () => {
107
- const server = {
108
- answer: 42,
109
- greet() {
110
- return 'hi';
111
- },
112
- registerTool() {},
113
- };
114
- const wrapped = wrapServer(server, vi.fn());
115
- expect(wrapped.answer).toBe(42);
116
- expect(wrapped.greet()).toBe('hi');
117
- });
118
- });
119
-
120
- describe('makeFlyExecutor', () => {
121
- const ENDPOINT = 'https://runner.example';
122
- const KEY = 'secret-key';
123
-
124
- it('POSTs the arg-array to /run with the bearer and returns stdout', async () => {
125
- const fetchMock = vi.fn(async () => ({
126
- ok: true,
127
- json: async () => ({ stdout: 'gog output' }),
128
- }));
129
- vi.stubGlobal('fetch', fetchMock);
130
-
131
- const exec = makeFlyExecutor(ENDPOINT, KEY);
132
- const out = await exec(['sheets', 'get', 'A1'], {});
133
- expect(out).toBe('gog output');
134
-
135
- const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit];
136
- expect(url).toBe('https://runner.example/run');
137
- expect(init.method).toBe('POST');
138
- expect(init.headers).toEqual({
139
- Authorization: 'Bearer secret-key',
140
- 'Content-Type': 'application/json',
141
- });
142
- expect(init.body).toBe(JSON.stringify({ args: ['sheets', 'get', 'A1'] }));
143
- });
144
-
145
- // A hosted gog authenticates as whoever seeded the Fly volume, not as the
146
- // person calling it (#230). mcp-host can give each caller their own child
147
- // carrying their own GOG_ACCESS_TOKEN, so the missing link is the child
148
- // handing that token to the backend for ITS call only — never as something
149
- // ambient on the box, which would be the same shared identity from the other
150
- // direction.
151
- describe('per-request access token', () => {
152
- function okFetch() {
153
- const fetchMock = vi.fn(async () => ({ ok: true, json: async () => ({ stdout: 'ok' }) }));
154
- vi.stubGlobal('fetch', fetchMock);
155
- return fetchMock;
156
- }
157
- function bodyOf(fetchMock: { mock: { calls: unknown[][] } }, i = 0): Record<string, unknown> {
158
- const [, init] = fetchMock.mock.calls[i] as [string, RequestInit];
159
- return JSON.parse(init.body as string) as Record<string, unknown>;
160
- }
161
-
162
- it('sends the token with the request when one is available', async () => {
163
- const fetchMock = okFetch();
164
- const exec = makeFlyExecutor(ENDPOINT, KEY, () => 'ya29.caller-token');
165
- await exec(['auth', 'status'], {});
166
- expect(bodyOf(fetchMock)).toEqual({ args: ['auth', 'status'], accessToken: 'ya29.caller-token' });
167
- });
168
-
169
- it('omits the field entirely when there is no token', async () => {
170
- // Absent, not null or "": the backend distinguishes "act as the caller"
171
- // from "act as the box", and a present-but-empty field would be a third
172
- // state neither side has a meaning for.
173
- const fetchMock = okFetch();
174
- for (const provider of [undefined, () => undefined, () => '']) {
175
- vi.clearAllMocks();
176
- const exec = makeFlyExecutor(ENDPOINT, KEY, provider as (() => string | undefined) | undefined);
177
- await exec(['auth', 'status'], {});
178
- expect(bodyOf(fetchMock)).toEqual({ args: ['auth', 'status'] });
179
- }
180
- });
181
-
182
- it('reads the token per CALL, not once when the executor is built', async () => {
183
- // The whole contract is "this token belongs to this request". Reading it
184
- // once at construction would pin the first caller's identity onto an
185
- // executor that outlives them — exactly the bug this closes, rebuilt.
186
- const fetchMock = okFetch();
187
- const tokens = ['first', 'second'];
188
- const exec = makeFlyExecutor(ENDPOINT, KEY, () => tokens.shift());
189
- await exec(['auth', 'status'], {});
190
- await exec(['auth', 'status'], {});
191
- expect(bodyOf(fetchMock, 0).accessToken).toBe('first');
192
- expect(bodyOf(fetchMock, 1).accessToken).toBe('second');
193
- });
194
- });
195
-
196
- // Everything below is the file-arg wire contract. The Worker has no filesystem
197
- // and no gog binary, so a GogFileArg must cross the wire STRUCTURED and be
198
- // materialized on the Fly runner. If this layer ever flattened it back into an
199
- // argv string, the >4 KiB payload it exists to carry would hit the arg cap again.
200
- describe('GogFileArg forwarding', () => {
201
- function bodyOf(fetchMock: { mock: { calls: unknown[][] } }): { args: GogArg[] } {
202
- const [, init] = fetchMock.mock.calls[0] as [string, RequestInit];
203
- return JSON.parse(init.body as string) as { args: GogArg[] };
204
- }
205
-
206
- function okFetch() {
207
- const fetchMock = vi.fn(async () => ({ ok: true, json: async () => ({ stdout: 'ok' }) }));
208
- vi.stubGlobal('fetch', fetchMock);
209
- return fetchMock;
210
- }
211
-
212
- it('serializes a file arg into the request body with its contents intact', async () => {
213
- const fetchMock = okFetch();
214
- // Comfortably past the runner's 4 KiB single-arg cap — the whole point.
215
- const html = '<p>' + 'x'.repeat(10_000) + '</p>';
216
- const fileArg: GogFileArg = {
217
- kind: 'file',
218
- flag: 'body-html-file',
219
- contents: html,
220
- ext: 'html',
221
- };
222
-
223
- const exec = makeFlyExecutor(ENDPOINT, KEY);
224
- await exec(['gmail', 'drafts', 'create', fileArg], {});
225
-
226
- const { args } = bodyOf(fetchMock);
227
- expect(args[3]).toEqual(fileArg);
228
- // Byte-for-byte: no truncation, no re-encoding, no flattening to argv.
229
- expect((args[3] as GogFileArg).contents).toBe(html);
230
- expect((args[3] as GogFileArg).contents).toHaveLength(html.length);
231
- expect(typeof args[3]).toBe('object');
232
- });
233
-
234
- it('preserves order across a mixed array of strings and file args', async () => {
235
- const fetchMock = okFetch();
236
- const body: GogFileArg = { kind: 'file', flag: 'body-file', contents: 'plain body' };
237
- const notes: GogFileArg = {
238
- kind: 'file',
239
- flag: 'signature-file',
240
- contents: 'sig',
241
- ext: 'html',
242
- };
243
- const sent: GogArg[] = [
244
- '--json',
245
- 'gmail',
246
- 'send',
247
- '--to=a@example.com',
248
- body,
249
- '--subject=Hi',
250
- notes,
251
- '--no-input',
252
- ];
253
-
254
- const exec = makeFlyExecutor(ENDPOINT, KEY);
255
- await exec(sent, {});
256
-
257
- // Order is load-bearing: gog parses positionally, so a reordered array is
258
- // a different command.
259
- expect(bodyOf(fetchMock).args).toEqual(sent);
260
- });
261
-
262
- it('round-trips UTF-8 payloads — multibyte, emoji, and interior newlines', async () => {
263
- const fetchMock = okFetch();
264
- // Multibyte scripts, an astral-plane emoji (surrogate pair), and combining
265
- // marks: the classes most likely to be mangled by a naive re-encode.
266
- const contents = 'héllo wörld — naïve café\n日本語のテキスト\n🎉 emoji + ZWJ 👩‍💻\né\n';
267
- const fileArg: GogFileArg = { kind: 'file', flag: 'body-file', contents };
268
-
269
- const exec = makeFlyExecutor(ENDPOINT, KEY);
270
- await exec([fileArg], {});
271
-
272
- const got = bodyOf(fetchMock).args[0] as GogFileArg;
273
- expect(got.contents).toBe(contents);
274
- // Interior newlines survive; only gog's own per-command trailing-newline
275
- // trimming (on the runner) may alter the tail.
276
- expect(got.contents.split('\n')).toHaveLength(contents.split('\n').length);
277
- });
278
-
279
- it('omits ext when the caller omitted it, leaving the default to the runner', async () => {
280
- const fetchMock = okFetch();
281
- const exec = makeFlyExecutor(ENDPOINT, KEY);
282
- await exec([{ kind: 'file', flag: 'note-file', contents: 'n' }], {});
283
-
284
- const got = bodyOf(fetchMock).args[0] as GogFileArg;
285
- // Thresholding and defaulting live in ONE place each; this layer must not
286
- // invent an ext, or two boxes would disagree about the temp filename.
287
- expect('ext' in got).toBe(false);
288
- });
289
-
290
- it('still errors deterministically when a file-arg call fails on the runner', async () => {
291
- // Classification must not vary with arg shape: a 422 is "gog ran and
292
- // failed" whether or not the call carried a file arg.
293
- vi.stubGlobal(
294
- 'fetch',
295
- vi.fn(async () => ({
296
- ok: false,
297
- status: 422,
298
- json: async () => ({ error: 'use only one of --body-html or --body-html-file' }),
299
- })),
300
- );
301
- const exec = makeFlyExecutor(ENDPOINT, KEY);
302
- const err = (await exec(
303
- [{ kind: 'file', flag: 'body-html-file', contents: 'x'.repeat(9000) }],
304
- {},
305
- ).catch((e: Error) => e)) as Error;
306
- expect(err.message).toContain('use only one of --body-html');
307
- expect(err.message).not.toMatch(/retry|transient/i);
308
- });
309
-
310
- it('gives a large payload no extra deadline beyond the standard grace', async () => {
311
- // Deliberate: upload is a datacenter hop measured in milliseconds, so the
312
- // runner must still win the race and return a real error.
313
- const timeoutSpy = vi.spyOn(AbortSignal, 'timeout');
314
- okFetch();
315
- const exec = makeFlyExecutor(ENDPOINT, KEY);
316
- await exec([{ kind: 'file', flag: 'body-file', contents: 'y'.repeat(500_000) }], {});
317
- expect(timeoutSpy).toHaveBeenCalledWith(35_000);
318
- timeoutSpy.mockRestore();
319
- });
320
- });
321
-
322
- // A scale-to-zero Fly backend that never answers would otherwise hang the MCP
323
- // request forever: Workers' fetch has no default deadline, and the stdio path's
324
- // 30s kill lives in the child process we are NOT spawning here.
325
- it('arms a client-side deadline so a cold or wedged backend cannot hang forever', async () => {
326
- const fetchMock = vi.fn(async () => ({ ok: true, json: async () => ({ stdout: '' }) }));
327
- vi.stubGlobal('fetch', fetchMock);
328
-
329
- const exec = makeFlyExecutor(ENDPOINT, KEY);
330
- await exec(['x'], {});
331
-
332
- const [, init] = fetchMock.mock.calls[0] as [string, RequestInit];
333
- expect(init.signal).toBeInstanceOf(AbortSignal);
334
- });
335
-
336
- it('gives the backend its own timeout plus headroom, so the server wins when it can', async () => {
337
- const timeoutSpy = vi.spyOn(AbortSignal, 'timeout');
338
- vi.stubGlobal('fetch', vi.fn(async () => ({ ok: true, json: async () => ({ stdout: '' }) })));
339
-
340
- const exec = makeFlyExecutor(ENDPOINT, KEY);
341
- await exec(['x'], { timeout: 60_000 });
342
-
343
- expect(timeoutSpy).toHaveBeenCalledWith(65_000);
344
- timeoutSpy.mockRestore();
345
- });
346
-
347
- it('defaults to the stdio path\'s 30s budget (plus headroom) when no timeout is given', async () => {
348
- const timeoutSpy = vi.spyOn(AbortSignal, 'timeout');
349
- vi.stubGlobal('fetch', vi.fn(async () => ({ ok: true, json: async () => ({ stdout: '' }) })));
350
-
351
- const exec = makeFlyExecutor(ENDPOINT, KEY);
352
- await exec(['x'], {});
353
-
354
- expect(timeoutSpy).toHaveBeenCalledWith(35_000);
355
- timeoutSpy.mockRestore();
356
- });
357
-
358
- it('reports an actionable timeout rather than a bare AbortError', async () => {
359
- vi.stubGlobal(
360
- 'fetch',
361
- vi.fn(async () => {
362
- throw Object.assign(new Error('The operation was aborted'), { name: 'TimeoutError' });
363
- }),
364
- );
365
- const exec = makeFlyExecutor(ENDPOINT, KEY);
366
- await expect(exec(['x'], {})).rejects.toThrow(/gog-runner did not respond within 35000ms/);
367
- });
368
-
369
- it('rethrows a non-timeout fetch failure verbatim, not as a timeout', async () => {
370
- // A real network error (DNS failure, connection refused) rejects with a
371
- // TypeError named 'TypeError' — neither TimeoutError nor AbortError — so it
372
- // must pass through untouched rather than be relabelled a timeout.
373
- const networkErr = new TypeError('fetch failed');
374
- vi.stubGlobal('fetch', vi.fn(async () => { throw networkErr; }));
375
- const exec = makeFlyExecutor(ENDPOINT, KEY);
376
- await expect(exec(['x'], {})).rejects.toBe(networkErr);
377
- });
378
-
379
- it('rethrows a non-Error rejection verbatim', async () => {
380
- // Guards the `err instanceof Error ? err.name : ''` false branch: if fetch
381
- // ever rejects with a non-Error value, it is rethrown unchanged.
382
- vi.stubGlobal('fetch', vi.fn(async () => { throw 'kaboom'; }));
383
- const exec = makeFlyExecutor(ENDPOINT, KEY);
384
- await expect(exec(['x'], {})).rejects.toBe('kaboom');
385
- });
386
-
387
- // The runner reports "gog ran and exited non-zero" as 422 — deliberately NOT
388
- // a 5xx, so it can never be confused with Fly's edge failing to reach the
389
- // Machine, and so it never matches TRANSIENT_ERROR_PATTERN (/\b5\d\d\b/).
390
- it('surfaces gog stderr verbatim for a 422 and never advises a retry', async () => {
391
- vi.stubGlobal(
392
- 'fetch',
393
- vi.fn(async () => ({
394
- ok: false,
395
- status: 422,
396
- json: async () => ({
397
- error: 'gog exited with code 1',
398
- stderr: 'invalid attachment id',
399
- retryable: false,
400
- }),
401
- })),
402
- );
403
-
404
- const exec = makeFlyExecutor(ENDPOINT, KEY);
405
- const err = (await exec(['gmail', 'attachment'], {}).catch((e: Error) => e)) as Error;
406
- expect(err.message).toContain('gog exited with code 1');
407
- expect(err.message).toContain('invalid attachment id');
408
- // Deterministic: retrying cannot help, so nothing may invite it.
409
- expect(err.message).not.toMatch(/retry/i);
410
- expect(err.message).not.toMatch(/transient/i);
411
- // It reached gog — the executor must not claim otherwise.
412
- expect(err.message).not.toMatch(/never reached gog/i);
413
- });
414
-
415
- // The live repro that started this: an --out path from the caller's sandbox
416
- // does not exist on the runner, so gog cannot create it. No number of retries
417
- // makes the directory appear.
418
- it('reports an unwritable --out path as a plain deterministic gog error', async () => {
419
- vi.stubGlobal(
420
- 'fetch',
421
- vi.fn(async () => ({
422
- ok: false,
423
- status: 422,
424
- json: async () => ({
425
- error: 'mkdir /home/claude: operation not supported',
426
- stderr: 'mkdir /home/claude: operation not supported',
427
- retryable: false,
428
- }),
429
- })),
430
- );
431
-
432
- const exec = makeFlyExecutor(ENDPOINT, KEY);
433
- const err = (await exec(['gmail', 'attachment'], {}).catch((e: Error) => e)) as Error;
434
- expect(err.message).toContain('mkdir /home/claude');
435
- expect(err.message).not.toMatch(/retry|transient/i);
436
- // stderr duplicating error must not be echoed twice.
437
- expect(err.message.match(/mkdir \/home\/claude/g)).toHaveLength(1);
438
- });
439
-
440
- it('marks the runner drain 503 as retryable', async () => {
441
- vi.stubGlobal(
442
- 'fetch',
443
- vi.fn(async () => ({
444
- ok: false,
445
- status: 503,
446
- json: async () => ({ error: 'gog-runner is shutting down', retryable: true }),
447
- })),
448
- );
449
-
450
- const exec = makeFlyExecutor(ENDPOINT, KEY);
451
- await expect(exec(['gmail', 'attachment'], {})).rejects.toThrow(/restarting; retry/i);
452
- });
453
-
454
- it('still advises a retry on a drain 503 whose body is unreadable', async () => {
455
- // Covers the no-detail arm: a drain that races the response body away is
456
- // still the runner refusing work on purpose, so it stays retryable.
457
- vi.stubGlobal(
458
- 'fetch',
459
- vi.fn(async () => ({
460
- ok: false,
461
- status: 503,
462
- json: async () => {
463
- throw new Error('not json');
464
- },
465
- })),
466
- );
467
- const exec = makeFlyExecutor(ENDPOINT, KEY);
468
- const err = (await exec(['x'], {}).catch((e: Error) => e)) as Error;
469
- expect(err.message).toBe('gog-runner is restarting; retry this call.');
470
- });
471
-
472
- // Fly's edge could not reach the Machine: no runner body at all (an HTML error
473
- // page or an empty response). This is the ONLY case that is genuinely transient.
474
- it('names the gateway hop when a 502 carries no runner body', async () => {
475
- vi.stubGlobal(
476
- 'fetch',
477
- vi.fn(async () => ({
478
- ok: false,
479
- status: 502,
480
- json: async () => {
481
- throw new Error('Fly returned HTML, not JSON');
482
- },
483
- })),
484
- );
485
-
486
- const exec = makeFlyExecutor(ENDPOINT, KEY);
487
- await expect(exec(['gmail', 'attachment'], {})).rejects.toThrow(
488
- /never reached gog.*transient/s,
489
- );
490
- });
491
-
492
- // Belt and braces for the rollout window (and any future runner that answers
493
- // 5xx with real detail): if the runner did speak, repeat its words rather than
494
- // asserting the request never arrived.
495
- it('repeats the runner detail on a 5xx that does carry a body', async () => {
496
- vi.stubGlobal(
497
- 'fetch',
498
- vi.fn(async () => ({
499
- ok: false,
500
- status: 502,
501
- json: async () => ({ error: 'gog exited with code 1', stderr: 'bad flag' }),
502
- })),
503
- );
504
-
505
- const exec = makeFlyExecutor(ENDPOINT, KEY);
506
- const err = (await exec(['bogus'], {}).catch((e: Error) => e)) as Error;
507
- expect(err.message).toContain('gog exited with code 1');
508
- expect(err.message).toContain('bad flag');
509
- expect(err.message).not.toMatch(/never reached gog/i);
510
- // A runner body proves gog ran, so this is deterministic — nothing may
511
- // invite a retry, exactly as for the 422 path.
512
- expect(err.message).not.toMatch(/retry|transient/i);
513
- // And the status digits must not leak into the message: a literal "502"
514
- // matches TRANSIENT_ERROR_PATTERN downstream and re-attaches the hint.
515
- expect(err.message).not.toMatch(/\b5\d\d\b/);
516
- });
517
-
518
- it('falls back to an HTTP-status message when the error body is unreadable', async () => {
519
- vi.stubGlobal(
520
- 'fetch',
521
- vi.fn(async () => ({
522
- ok: false,
523
- status: 500,
524
- json: async () => {
525
- throw new Error('not json');
526
- },
527
- })),
528
- );
529
- const exec = makeFlyExecutor(ENDPOINT, KEY);
530
- await expect(exec(['x'], {})).rejects.toThrow('gog-runner HTTP 500');
531
- });
532
-
533
- it('handles a 422 whose body is unreadable without pretending it never ran', async () => {
534
- vi.stubGlobal(
535
- 'fetch',
536
- vi.fn(async () => ({
537
- ok: false,
538
- status: 422,
539
- json: async () => {
540
- throw new Error('not json');
541
- },
542
- })),
543
- );
544
- const exec = makeFlyExecutor(ENDPOINT, KEY);
545
- await expect(exec(['x'], {})).rejects.toThrow(/gog failed on the runner/i);
546
- });
547
- });
548
-
549
- // A failure the RUNNER authored — its own bearer check, its own request
550
- // validation, its own drain — never reached gog and never showed a credential
551
- // to Google. Those failures must be distinguishable by TYPE, because the layer
552
- // that diagnoses them (tools/utils.ts) can only otherwise guess from prose, and
553
- // guessing is what turned the runner's bare `unauthorized` body into "your
554
- // Google sign-in expired, re-authorize".
555
- describe('makeFlyExecutor runner-authored transport failures', () => {
556
- const ENDPOINT = 'https://gogcli-gog-runner.fly.dev';
557
- const KEY = 'k';
558
-
559
- function stubStatus(status: number, body?: unknown) {
560
- vi.stubGlobal(
561
- 'fetch',
562
- vi.fn(async () => ({
563
- ok: false,
564
- status,
565
- json: async () => {
566
- if (body === undefined) throw new Error('not json');
567
- return body;
568
- },
569
- })),
570
- );
571
- }
572
-
573
- async function thrownBy(status: number, body?: unknown): Promise<unknown> {
574
- stubStatus(status, body);
575
- const exec = makeFlyExecutor(ENDPOINT, KEY);
576
- return exec(['gmail', 'search', 'q'], {}).catch((e: unknown) => e);
577
- }
578
-
579
- // The exact body fly-gog-runner/server.mjs sends when its bearer check fails
580
- // (server.mjs:450 /health, :460 /run) — nothing to do with Google.
581
- const RUNNER_401 = { error: 'unauthorized' };
582
-
583
- it('types the runner bearer rejection as transport auth, not a gog failure', async () => {
584
- const err = await thrownBy(401, RUNNER_401);
585
- expect(isRunnerTransportError(err)).toBe(true);
586
- expect((err as RunnerTransportError).kind).toBe('transport-auth');
587
- expect((err as RunnerTransportError).status).toBe(401);
588
- });
589
-
590
- it('names the real cause — the runner key mismatch — and never the Google account', async () => {
591
- const err = (await thrownBy(401, RUNNER_401)) as Error;
592
- expect(err.message).toContain('GOG_RUNNER_KEY');
593
- expect(err.message).toContain('RUNNER_KEY');
594
- expect(err.message).not.toMatch(/gog_auth_add/i);
595
- // The runner's own body is the single word "unauthorized". Repeating it is
596
- // what fed DEFINITE_AUTH_PATTERN in tools/utils.ts; the status digits can do
597
- // the same whenever a status word sits near them. Neither may appear, so that
598
- // even if the TYPE is lost at some future boundary the prose cannot be
599
- // misread as Google's.
600
- expect(err.message).not.toMatch(/unauthorized/i);
601
- expect(err.message).not.toMatch(/\b401\b/);
602
- });
603
-
604
- it('types the runner request-validation 400s as transport-request, keeping their detail', async () => {
605
- for (const detail of [
606
- 'request body too large',
607
- 'failed to read request body',
608
- 'body must be valid JSON',
609
- 'args must be an array',
610
- 'accessToken must not contain whitespace or control characters',
611
- ]) {
612
- const err = await thrownBy(400, { error: detail });
613
- expect(isRunnerTransportError(err)).toBe(true);
614
- expect((err as RunnerTransportError).kind).toBe('transport-request');
615
- expect((err as RunnerTransportError).status).toBe(400);
616
- expect((err as Error).message).toBe(detail);
617
- }
618
- });
619
-
620
- it('still says something when a 400 body is unreadable', async () => {
621
- const err = await thrownBy(400);
622
- expect(isRunnerTransportError(err)).toBe(true);
623
- expect((err as RunnerTransportError).kind).toBe('transport-request');
624
- expect((err as Error).message).toMatch(/gog-runner rejected the request/i);
625
- });
626
-
627
- it('types the drain 503 as retryable transport, message unchanged', async () => {
628
- const err = await thrownBy(503, { error: 'gog-runner is shutting down', retryable: true });
629
- expect(isRunnerTransportError(err)).toBe(true);
630
- expect((err as RunnerTransportError).kind).toBe('transport-retryable');
631
- expect((err as RunnerTransportError).status).toBe(503);
632
- expect((err as Error).message).toBe('gog-runner is restarting; retry this call. gog-runner is shutting down');
633
- });
634
-
635
- it('types a retryable-flagged 500 (a materialization failure) as retryable transport', async () => {
636
- const err = await thrownBy(500, {
637
- error: 'failed to write a file arg to disk: ENOSPC: no space left on device',
638
- retryable: true,
639
- });
640
- expect(isRunnerTransportError(err)).toBe(true);
641
- expect((err as RunnerTransportError).kind).toBe('transport-retryable');
642
- expect((err as RunnerTransportError).status).toBe(500);
643
- });
644
-
645
- it('types a bodiless gateway failure as retryable transport', async () => {
646
- const err = await thrownBy(502);
647
- expect(isRunnerTransportError(err)).toBe(true);
648
- expect((err as RunnerTransportError).kind).toBe('transport-retryable');
649
- expect((err as RunnerTransportError).status).toBe(502);
650
- });
651
-
652
- it('types the client-side deadline as retryable transport with no status', async () => {
653
- vi.stubGlobal(
654
- 'fetch',
655
- vi.fn(async () => {
656
- throw Object.assign(new Error('The operation was aborted'), { name: 'TimeoutError' });
657
- }),
658
- );
659
- const exec = makeFlyExecutor(ENDPOINT, KEY);
660
- const err = await exec(['x'], {}).catch((e: unknown) => e);
661
- expect(isRunnerTransportError(err)).toBe(true);
662
- expect((err as RunnerTransportError).kind).toBe('transport-retryable');
663
- expect((err as RunnerTransportError).status).toBeUndefined();
664
- });
665
-
666
- // The other half of the contract: an error that carries gog's/Google's OWN
667
- // words must stay untyped, so the prose classifier still gets to read it.
668
- it('leaves a 422 (gog ran and failed) untyped, for the prose classifier', async () => {
669
- const err = await thrownBy(422, { error: 'gog failed', stderr: 'Error 401: invalid_grant' });
670
- expect(err).toBeInstanceOf(Error);
671
- expect(isRunnerTransportError(err)).toBe(false);
672
- });
673
-
674
- it('leaves a detail-bearing non-2xx untyped', async () => {
675
- const err = await thrownBy(502, { error: 'gog exited with code 1', stderr: 'bad flag' });
676
- expect(isRunnerTransportError(err)).toBe(false);
677
- });
678
-
679
- it('recognises only branded errors', () => {
680
- expect(isRunnerTransportError(new Error('unauthorized'))).toBe(false);
681
- expect(isRunnerTransportError('unauthorized')).toBe(false);
682
- expect(isRunnerTransportError(new RunnerTransportError('x', 'transport-auth', 401))).toBe(true);
683
- });
684
-
685
- // The whole chain, end to end, with the REAL executor, the REAL run() and the
686
- // REAL diagnose(): the incident was a user told all session to re-authorize a
687
- // Google account that was never asked for a credential.
688
- it('does not tell the caller to re-authorize Google when the RUNNER rejected our bearer', async () => {
689
- vi.stubEnv('GOG_ACCOUNT', '');
690
- vi.stubEnv('GOG_READONLY', '');
691
- stubStatus(401, RUNNER_401);
692
- const executor = makeFlyExecutor(ENDPOINT, KEY);
693
- const result = await runExecutor.run({ executor }, () =>
694
- runOrDiagnose(['sheets', 'get', 'A1'], {}),
695
- );
696
- const text = result.content[0].text as string;
697
- expect(result.isError).toBe(true);
698
- expect(text).not.toMatch(/gog_auth_add/);
699
- expect(text).not.toMatch(/re-authorize the account/i);
700
- expect(text).toContain('GOG_RUNNER_KEY');
701
- });
702
- });
703
-
704
- // A Google 401 on the ACCESS token and a dead REFRESH token are opposite
705
- // failures that used to be told to the user identically.
706
- //
707
- // * The access token is ours to replace. Google rejecting it means "mint
708
- // another and try again" — automatic, no human, no re-authorization. Before
709
- // this, the rejected token stayed cached for the rest of its nominal hour
710
- // and NOTHING retried, so every call in that window failed the same way and
711
- // only reconnecting (a fresh isolate, an empty cache) appeared to help.
712
- // * invalid_grant means the refresh token is gone. No re-mint is possible and
713
- // a human must re-authorize. Retrying that is a loop against a credential
714
- // that can never work.
715
- //
716
- // So the re-mint is gated hard: gog must actually have run (422), the token must
717
- // be one WE supplied, gog's own stderr must say Google rejected it, the call
718
- // must be safe to replay, and our cache must still hold exactly that token.
719
- describe('makeFlyExecutor re-mints a rejected access token and replays once', () => {
720
- const ENDPOINT = 'https://gogcli-gog-runner.fly.dev';
721
- const KEY = 'k';
722
-
723
- // gog v0.34.1's real words when the access token it was handed is rejected.
724
- const GOOGLE_401_STDERR =
725
- 'Note: Using direct access token (expires in ~1 hour; no auto-refresh)\n' +
726
- 'Google API error (401 authError): Request had invalid authentication credentials. ' +
727
- 'Expected OAuth 2 access token, login cookie or other valid authentication credential.';
728
-
729
- // What the executor really receives: run()'s assembleArgs puts the global
730
- // flags in front of the service and subcommand.
731
- const READ = ['--json', '--color=never', '--no-input', 'gmail', 'search', 'q'];
732
- const WRITE = ['--json', '--color=never', '--no-input', 'gmail', 'send', '--to', 'a@b.c'];
733
-
734
- function gogFailed(stderr: string) {
735
- return {
736
- ok: false,
737
- status: 422,
738
- // `error` is Node's execFile message, which embeds the whole command line
739
- // AND stderr — see the command-line test below for why that matters.
740
- json: async () => ({
741
- error: `Command failed: gog gmail search q\n${stderr}`,
742
- stderr,
743
- retryable: false,
744
- }),
745
- };
746
- }
747
- function ok(stdout: string) {
748
- return { ok: true, json: async () => ({ stdout }) };
749
- }
750
- function bodyOf(fetchMock: { mock: { calls: unknown[][] } }, i: number): Record<string, unknown> {
751
- const [, init] = fetchMock.mock.calls[i] as [string, RequestInit];
752
- return JSON.parse(init.body as string) as Record<string, unknown>;
753
- }
754
- /** A token source that hands out `tokens` in order and can be invalidated. */
755
- function source(tokens: (string | undefined)[], evicted = true) {
756
- const fn = vi.fn(async () => tokens.shift());
757
- return Object.assign(fn, { invalidate: vi.fn(async () => evicted) });
758
- }
759
-
760
- it('mints a new token and replays the call, invisibly to the caller', async () => {
761
- const fetchMock = vi
762
- .fn()
763
- .mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
764
- .mockResolvedValueOnce(ok('thread json'));
765
- vi.stubGlobal('fetch', fetchMock);
766
- const readToken = source(['ya29.stale', 'ya29.fresh']);
767
-
768
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
769
- await expect(exec(READ, {})).resolves.toBe('thread json');
770
-
771
- expect(fetchMock).toHaveBeenCalledTimes(2);
772
- expect(bodyOf(fetchMock, 0).accessToken).toBe('ya29.stale');
773
- expect(bodyOf(fetchMock, 1).accessToken).toBe('ya29.fresh');
774
- // Evicted by VALUE — only the token that was actually rejected.
775
- expect(readToken.invalidate).toHaveBeenCalledTimes(1);
776
- expect(readToken.invalidate).toHaveBeenCalledWith('ya29.stale');
777
- });
778
-
779
- it('replays exactly once, so a genuinely dead credential cannot loop', async () => {
780
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
781
- vi.stubGlobal('fetch', fetchMock);
782
- const readToken = source(['ya29.stale', 'ya29.fresh', 'ya29.third']);
783
-
784
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
785
- await expect(exec(READ, {})).rejects.toThrow(/Google API error \(401/);
786
-
787
- expect(fetchMock).toHaveBeenCalledTimes(2);
788
- // Two evictions, one per token Google refused — the replay's included. The
789
- // ATTEMPT count is what "exactly once" is about, and it is still two.
790
- expect(readToken.invalidate).toHaveBeenCalledTimes(2);
791
- });
792
-
793
- it('does not replay when gog reported invalid_grant — only a human can fix that', async () => {
794
- // The stderr deliberately matches BOTH shapes, so this pins the exclusion
795
- // rather than the absence of a 401.
796
- const fetchMock = vi.fn(async () => gogFailed(`${GOOGLE_401_STDERR}\noauth2: "invalid_grant"`));
797
- vi.stubGlobal('fetch', fetchMock);
798
- const readToken = source(['ya29.stale', 'ya29.fresh']);
799
-
800
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
801
- await expect(exec(READ, {})).rejects.toThrow(/invalid_grant/);
802
- expect(fetchMock).toHaveBeenCalledTimes(1);
803
- expect(readToken.invalidate).not.toHaveBeenCalled();
804
- });
805
-
806
- it('does not replay a call that supplied no token of ours', async () => {
807
- // Nothing was minted, so there is nothing to re-mint: gog used whatever
808
- // identity the backend volume holds, and only an operator can change that.
809
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
810
- vi.stubGlobal('fetch', fetchMock);
811
- const exec = makeFlyExecutor(ENDPOINT, KEY);
812
- await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
813
- // Counted by ENDPOINT, not in total: this same refusal now also takes a
814
- // read of the Google layer (`/health/google`, see the refusal-probe block
815
- // below), and that reading is a diagnostic, not a second attempt. What
816
- // "does not replay" means is that gog ran exactly once.
817
- const runs = fetchMock.mock.calls.filter(([url]) => String(url).endsWith('/run'));
818
- expect(runs).toHaveLength(1);
819
- });
820
-
821
- it('does not replay when the token source cannot invalidate', async () => {
822
- // A bare `() => token` (the #230 direct-token wiring) has no cache behind
823
- // it, so replaying would re-send the identical rejected token.
824
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
825
- vi.stubGlobal('fetch', fetchMock);
826
- const exec = makeFlyExecutor(ENDPOINT, KEY, () => 'ya29.direct');
827
- await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
828
- expect(fetchMock).toHaveBeenCalledTimes(1);
829
- });
830
-
831
- it('does not replay when the cache no longer held the rejected token', async () => {
832
- // Another caller already refreshed it; the token we would send is the one
833
- // that is already in use, so a replay proves nothing.
834
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
835
- vi.stubGlobal('fetch', fetchMock);
836
- const readToken = source(['ya29.stale', 'ya29.fresh'], false);
837
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
838
- await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
839
- expect(fetchMock).toHaveBeenCalledTimes(1);
840
- expect(readToken.invalidate).toHaveBeenCalledTimes(1);
841
- });
842
-
843
- // A subcommand is only a leaf VERB some of the time. `gog tasks lists` is a
844
- // NAMESPACE — `lists list` reads, `lists create` writes — and the allow-list
845
- // is consulted with the namespace word, never the verb under it. So a
846
- // namespace word in the set hands the replay to every child it will ever
847
- // grow, including the ones that write.
848
- //
849
- // `tasks lists create <title> ...` is real in gog v0.34.1 today, and reachable
850
- // without any new gog: `gog_tasks_run({subcommand: 'lists', args: ['create',
851
- // 'A', 'B']})` assembles exactly this argv. It is variadic, so one invocation
852
- // makes N Google calls and a 401 on the second means the first already landed
853
- // — the precise double-apply the write rule exists to prevent.
854
- it('does not replay `tasks lists create`, a WRITE under a namespace-shaped word', async () => {
855
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
856
- vi.stubGlobal('fetch', fetchMock);
857
- const readToken = source(['ya29.stale', 'ya29.fresh']);
858
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
859
- await expect(
860
- exec(['--json', '--color=never', '--no-input', 'tasks', 'lists', 'create', 'A', 'B'], {}),
861
- ).rejects.toThrow(/Google API error/);
862
- expect(fetchMock).toHaveBeenCalledTimes(1);
863
- // The eviction still happens — it runs before the allow-list is consulted.
864
- expect(readToken.invalidate).toHaveBeenCalledTimes(1);
865
- });
866
-
867
- // The invariant the allow-list's comment states, pinned as behaviour so the
868
- // comment is no longer the only thing enforcing it. Every word here is a gog
869
- // NAMESPACE that already has, or can grow, a mutating child; none of them may
870
- // ever earn a replay, whichever verb follows.
871
- it.each([
872
- ['tasks', 'lists'],
873
- ['gmail', 'labels'],
874
- ['gmail', 'drafts'],
875
- ['gmail', 'filters'],
876
- ['gmail', 'sendas'],
877
- ['drive', 'permissions'],
878
- ['drive', 'revisions'],
879
- ['docs', 'comments'],
880
- ['docs', 'replies'],
881
- ])('never replays the namespace word %s %s, whatever verb follows it', async (service, namespace) => {
882
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
883
- vi.stubGlobal('fetch', fetchMock);
884
- const readToken = source(['ya29.stale', 'ya29.fresh']);
885
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
886
- await expect(
887
- exec(['--json', '--color=never', '--no-input', service, namespace, 'create', 'x'], {}),
888
- ).rejects.toThrow(/Google API error/);
889
- expect(fetchMock).toHaveBeenCalledTimes(1);
890
- // The eviction is unaffected: it runs before the allow-list is consulted.
891
- expect(readToken.invalidate).toHaveBeenCalledTimes(1);
892
- });
893
-
894
- // Symmetry with the rule the whole fix is built on: a token Google has
895
- // refused must not stay cached, and the very same 401 refused the replay's
896
- // token. This is NOT a round-trip saving — measured through the real chain
897
- // under a sustained non-invalid_grant refusal, a steady-state call costs two
898
- // /run round-trips either way, and this eviction adds a mint (2 rather than
899
- // 1) by emptying a cache the next call would have hit. What it buys is a
900
- // bound on how long a KNOWN-REFUSED token can be served: left cached it is
901
- // re-served for the rest of its nominal hour, and a write — which gets the
902
- // eviction and no replay — would be sent with it and fail on contact.
903
- it('evicts the replayed token too when Google refuses that one as well', async () => {
904
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
905
- vi.stubGlobal('fetch', fetchMock);
906
- const readToken = source(['ya29.stale', 'ya29.fresh']);
907
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
908
- await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
909
- expect(readToken.invalidate.mock.calls).toEqual([['ya29.stale'], ['ya29.fresh']]);
910
- });
911
-
912
- // The other half of that rule, and the one that keeps this branch honest.
913
- //
914
- // A replay can fail without Google ever seeing the token: the Machine starts
915
- // draining between the two attempts, the client-side deadline fires, the
916
- // runner's own bearer is rotated mid-call. Evicting on THOSE would throw away
917
- // a token nothing has refused and — worse — emit `token.evicted` with
918
- // "Google rejected this access token", which is a lie about a service that
919
- // was never consulted. Misattributing a runner-side failure to Google is the
920
- // exact defect this branch exists to remove; re-introducing it in the log
921
- // would just move it from the user's screen to the operator's query.
922
- it('does not evict the replayed token when the replay never reached Google', async () => {
923
- const fetchMock = vi
924
- .fn()
925
- .mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
926
- // The runner drains between the two attempts — a transport failure, not
927
- // a verdict on the freshly minted token.
928
- .mockResolvedValueOnce({
929
- ok: false,
930
- status: 503,
931
- json: async () => ({ error: 'gog-runner is shutting down', retryable: true }),
932
- });
933
- vi.stubGlobal('fetch', fetchMock);
934
- const readToken = source(['ya29.stale', 'ya29.fresh']);
935
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
936
- const err = await exec(READ, {}).catch((e: unknown) => e);
937
- // The replay's own error reaches the caller, and here that is the RIGHT
938
- // one to surface: "the runner is restarting, retry" is actionable, where
939
- // re-raising the superseded Google 401 would send the user off to
940
- // re-authorize an account that is fine.
941
- expect(isRunnerTransportError(err)).toBe(true);
942
- expect((err as RunnerTransportError).kind).toBe('transport-retryable');
943
- // Only the token Google actually refused was dropped. `ya29.fresh` stays
944
- // cached: it is unproven, not refused, and the next call may well succeed
945
- // with it once the new Machine is up.
946
- expect(readToken.invalidate.mock.calls).toEqual([['ya29.stale']]);
947
- });
948
-
949
- // The mint is a first-class error surface on this branch, so the error it
950
- // raises has to reach the caller with the SAME specificity a gog-authored
951
- // invalid_grant gets: the 7-day Testing-mode cause and the headless re-auth
952
- // pair. It only does if the message carries the literal `invalid_grant`,
953
- // which is what tools/utils.ts keys the richer hint on.
954
- it('gives a mint-path invalid_grant the full re-auth guidance, not the generic hint', async () => {
955
- vi.stubEnv('GOG_ACCOUNT', '');
956
- vi.stubEnv('GOG_READONLY', '');
957
- clearAccessTokenCache();
958
- const fetchMock = vi.fn(async (url: unknown) => {
959
- if (String(url).includes('oauth2.googleapis.com')) {
960
- return new Response(JSON.stringify({ error: 'invalid_grant' }), { status: 400 });
961
- }
962
- return gogFailed('unreachable');
963
- });
964
- vi.stubGlobal('fetch', fetchMock);
965
- const executor = makeFlyExecutor(
966
- ENDPOINT,
967
- KEY,
968
- makeAccessTokenSource({
969
- GOG_CLIENT_ID: 'cid',
970
- GOG_CLIENT_SECRET: 'cs',
971
- GOG_REFRESH_TOKEN: 'rt-dead',
972
- }),
973
- );
974
-
975
- const result = await runExecutor.run({ executor }, () =>
976
- runOrDiagnose(['gmail', 'search', 'q'], {}),
977
- );
978
- expect(result.isError).toBe(true);
979
- const text = result.content[0].text as string;
980
- // Text unique to INVALID_GRANT_HINT — the durable fix. The generic
981
- // AUTH_HINT (which is what a message omitting `invalid_grant` earns) says
982
- // only "Authentication may have expired".
983
- expect(text).toContain('publish the OAuth consent screen to "In production"');
984
- expect(text).not.toContain('Authentication may have expired');
985
- clearAccessTokenCache();
986
- });
987
-
988
- it('does not replay a WRITE, but still evicts the token Google rejected', async () => {
989
- // The one case an automatic replay must never take. A `gog gmail send` that
990
- // failed AFTER the message went out would send it twice.
991
- //
992
- // The EVICTION is a different question, and the write rule must not answer
993
- // it. Refusing to evict is what leaves Google's rejected token in the cache
994
- // for the rest of its nominal hour, so every following call — write or read
995
- // — re-sends it and fails identically until the isolate is replaced. That
996
- // is DEFECT 2 itself, and it is the half a write path used to keep.
997
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
998
- vi.stubGlobal('fetch', fetchMock);
999
- const readToken = source(['ya29.stale', 'ya29.fresh']);
1000
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1001
- await expect(exec(WRITE, {})).rejects.toThrow(/Google API error/);
1002
- expect(fetchMock).toHaveBeenCalledTimes(1);
1003
- expect(readToken.invalidate).toHaveBeenCalledTimes(1);
1004
- expect(readToken.invalidate).toHaveBeenCalledWith('ya29.stale');
1005
- });
1006
-
1007
- it('evicts for a READ whose subcommand is outside the allow-list', async () => {
1008
- // `gog gmail labels list` arrives here as the subcommand `labels`, which is
1009
- // deliberately not in READ_ONLY_SUBCOMMANDS (a later `labels create` would
1010
- // inherit the replay). Costing that call its replay is the intended price;
1011
- // costing it the eviction would poison every call after it.
1012
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
1013
- vi.stubGlobal('fetch', fetchMock);
1014
- const readToken = source(['ya29.stale', 'ya29.fresh']);
1015
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1016
- await expect(
1017
- exec(['--json', '--color=never', '--no-input', 'gmail', 'labels', 'list'], {}),
1018
- ).rejects.toThrow(/Google API error/);
1019
- expect(fetchMock).toHaveBeenCalledTimes(1);
1020
- expect(readToken.invalidate).toHaveBeenCalledTimes(1);
1021
- });
1022
-
1023
- it('lets a second WRITE mint a fresh token, through the REAL token source', async () => {
1024
- // The incident, reproduced end to end with nothing stubbed but the network:
1025
- // two consecutive `gog gmail send` calls after Google has rejected the
1026
- // cached token. Before the eviction was moved ahead of the write rule, BOTH
1027
- // shipped `ya29.t1` and both failed, for up to ~58 minutes, and only a
1028
- // reconnect helped.
1029
- clearAccessTokenCache();
1030
- let minted = 0;
1031
- const fetchMock = vi.fn(async (url: unknown) => {
1032
- if (String(url).includes('oauth2.googleapis.com')) {
1033
- return new Response(
1034
- JSON.stringify({ access_token: `ya29.t${(minted += 1)}`, expires_in: 3600 }),
1035
- { status: 200, headers: { 'content-type': 'application/json' } },
1036
- );
1037
- }
1038
- return gogFailed(GOOGLE_401_STDERR);
1039
- });
1040
- vi.stubGlobal('fetch', fetchMock);
1041
-
1042
- const exec = makeFlyExecutor(
1043
- ENDPOINT,
1044
- KEY,
1045
- makeAccessTokenSource({
1046
- GOG_CLIENT_ID: 'cid',
1047
- GOG_CLIENT_SECRET: 'cs',
1048
- GOG_REFRESH_TOKEN: 'rt-1',
1049
- }),
1050
- );
1051
- await expect(exec(WRITE, {})).rejects.toThrow(/Google API error/);
1052
- await expect(exec(WRITE, {})).rejects.toThrow(/Google API error/);
1053
-
1054
- const runBodies = fetchMock.mock.calls
1055
- .filter(([url]) => !String(url).includes('oauth2.googleapis.com'))
1056
- .map(([, init]) => JSON.parse((init as RequestInit).body as string) as { accessToken: string });
1057
- expect(runBodies.map((b) => b.accessToken)).toEqual(['ya29.t1', 'ya29.t2']);
1058
- clearAccessTokenCache();
1059
- });
1060
-
1061
- it('replays with what is LEFT of the deadline, not a second full one', async () => {
1062
- // 30s default + 5s grace is one tool call's whole budget. Handing the
1063
- // replay a fresh copy of it makes the worst case ~70s of wall clock, which
1064
- // can outlast the MCP client's own request timeout and turn a self-healing
1065
- // read into a client-side hang.
1066
- const budgets: number[] = [];
1067
- const realTimeout = AbortSignal.timeout.bind(AbortSignal);
1068
- vi.spyOn(AbortSignal, 'timeout').mockImplementation((ms: number) => {
1069
- budgets.push(ms);
1070
- return realTimeout(ms);
1071
- });
1072
- const fetchMock = vi
1073
- .fn()
1074
- .mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
1075
- .mockResolvedValueOnce(ok('threads'));
1076
- vi.stubGlobal('fetch', fetchMock);
1077
- const base = Date.now();
1078
- // The clock is read once to fix the deadline, then once more to size the
1079
- // replay; 5s of the budget is gone by then.
1080
- vi.spyOn(Date, 'now').mockReturnValueOnce(base).mockReturnValue(base + 5_000);
1081
-
1082
- const exec = makeFlyExecutor(ENDPOINT, KEY, source(['ya29.stale', 'ya29.fresh']));
1083
- await expect(exec(READ, {})).resolves.toBe('threads');
1084
- expect(budgets).toEqual([35_000, 30_000]);
1085
- });
1086
-
1087
- it('skips the replay when the first attempt used the whole deadline', async () => {
1088
- // With no budget left, a replay can only end in an abort, and that
1089
- // TimeoutError would REPLACE gog's own 401 — trading an actionable error
1090
- // for an opaque one. The eviction is the durable half of the repair and it
1091
- // has already happened, so the caller's own next call is the fresh one.
1092
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
1093
- vi.stubGlobal('fetch', fetchMock);
1094
- const readToken = source(['ya29.stale', 'ya29.fresh']);
1095
- const base = Date.now();
1096
- vi.spyOn(Date, 'now').mockReturnValueOnce(base).mockReturnValue(base + 34_900);
1097
-
1098
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1099
- await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
1100
- expect(fetchMock).toHaveBeenCalledTimes(1);
1101
- expect(readToken.invalidate).toHaveBeenCalledTimes(1);
1102
- });
1103
-
1104
- it('finds the subcommand past --account, whose value is not a subcommand', async () => {
1105
- const fetchMock = vi
1106
- .fn()
1107
- .mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
1108
- .mockResolvedValueOnce(ok('[]'));
1109
- vi.stubGlobal('fetch', fetchMock);
1110
- const readToken = source(['ya29.stale', 'ya29.fresh']);
1111
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1112
- await expect(
1113
- exec(['--json', '--color=never', '--no-input', '--account', 'me@example.com', 'drive', 'ls'], {}),
1114
- ).resolves.toBe('[]');
1115
- expect(fetchMock).toHaveBeenCalledTimes(2);
1116
- });
1117
-
1118
- it('does not replay an invocation with no subcommand to judge', async () => {
1119
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
1120
- vi.stubGlobal('fetch', fetchMock);
1121
- const readToken = source(['ya29.stale', 'ya29.fresh']);
1122
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1123
- await expect(exec(['--json', '--color=never', 'gmail'], {})).rejects.toThrow(/Google API error/);
1124
- expect(fetchMock).toHaveBeenCalledTimes(1);
1125
- });
1126
-
1127
- it('reads gog stderr, not the command line the runner echoed back', async () => {
1128
- // execFile's message embeds the whole argv, so a caller's own text lands in
1129
- // `error`. Classifying on that would let `--subject "invoice 401"` trigger a
1130
- // replay of a call that failed for an unrelated reason — and if that call
1131
- // were a write, replay it after it had partly applied.
1132
- const fetchMock = vi.fn(async () => ({
1133
- ok: false,
1134
- status: 422,
1135
- json: async () => ({
1136
- error: 'Command failed: gog gmail search "Google API error (401 authError)"\nno results',
1137
- stderr: 'no results',
1138
- retryable: false,
1139
- }),
1140
- }));
1141
- vi.stubGlobal('fetch', fetchMock);
1142
- const readToken = source(['ya29.stale', 'ya29.fresh']);
1143
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1144
- await expect(exec(READ, {})).rejects.toThrow(/no results/);
1145
- expect(fetchMock).toHaveBeenCalledTimes(1);
1146
- expect(readToken.invalidate).not.toHaveBeenCalled();
1147
- });
1148
-
1149
- it('does not replay a gog failure that has nothing to do with auth', async () => {
1150
- const fetchMock = vi.fn(async () => gogFailed('invalid attachment id'));
1151
- vi.stubGlobal('fetch', fetchMock);
1152
- const readToken = source(['ya29.stale', 'ya29.fresh']);
1153
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1154
- await expect(exec(READ, {})).rejects.toThrow(/invalid attachment id/);
1155
- expect(fetchMock).toHaveBeenCalledTimes(1);
1156
- });
1157
-
1158
- it('surfaces the original failure when the re-mint yields no token', async () => {
1159
- // Sending the request without a token would run it as the BACKEND's
1160
- // identity and hand this caller someone else's mailbox — never that.
1161
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
1162
- vi.stubGlobal('fetch', fetchMock);
1163
- const readToken = source(['ya29.stale', undefined]);
1164
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1165
- await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
1166
- expect(fetchMock).toHaveBeenCalledTimes(1);
1167
- });
1168
-
1169
- it('surfaces the re-mint failure, which is the more actionable one', async () => {
1170
- // Evicting revealed that the refresh token is dead too. That message names
1171
- // the real repair (re-authorize); gog's 401 does not.
1172
- const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
1173
- vi.stubGlobal('fetch', fetchMock);
1174
- const readToken = Object.assign(
1175
- vi
1176
- .fn<() => Promise<string | undefined>>()
1177
- .mockResolvedValueOnce('ya29.stale')
1178
- .mockRejectedValueOnce(new Error('the stored refresh token has expired or been revoked')),
1179
- { invalidate: vi.fn(async () => true) },
1180
- );
1181
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1182
- await expect(exec(READ, {})).rejects.toThrow(/refresh token has expired or been revoked/);
1183
- expect(fetchMock).toHaveBeenCalledTimes(1);
1184
- });
1185
-
1186
- it('never re-mints for the RUNNER\'s own 401, which never reached Google', async () => {
1187
- // The defect-1 failure. Its body is the bare word "unauthorized" and no
1188
- // Google credential was even read, so minting a new one is pure waste — and
1189
- // replaying would double every call during a key mismatch.
1190
- const fetchMock = vi.fn(async () => ({
1191
- ok: false,
1192
- status: 401,
1193
- json: async () => ({ error: 'unauthorized' }),
1194
- }));
1195
- vi.stubGlobal('fetch', fetchMock);
1196
- const readToken = source(['ya29.stale', 'ya29.fresh']);
1197
- const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1198
- const err = await exec(READ, {}).catch((e: unknown) => e);
1199
- expect(isRunnerTransportError(err)).toBe(true);
1200
- expect(fetchMock).toHaveBeenCalledTimes(1);
1201
- expect(readToken.invalidate).not.toHaveBeenCalled();
1202
- });
1203
-
1204
- // End to end through the REAL run() and the REAL diagnose(): the user-visible
1205
- // property is that a stale access token produces no error and no advice at
1206
- // all, because it healed itself.
1207
- it('turns a stale-token failure into a plain success, with no re-auth advice', async () => {
1208
- vi.stubEnv('GOG_ACCOUNT', '');
1209
- vi.stubEnv('GOG_READONLY', '');
1210
- const fetchMock = vi
1211
- .fn()
1212
- .mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
1213
- .mockResolvedValueOnce(ok('{"threads":[]}'));
1214
- vi.stubGlobal('fetch', fetchMock);
1215
- const executor = makeFlyExecutor(ENDPOINT, KEY, source(['ya29.stale', 'ya29.fresh']));
1216
-
1217
- const result = await runExecutor.run({ executor }, () =>
1218
- runOrDiagnose(['gmail', 'search', 'q'], {}),
1219
- );
1220
- expect(result.isError).toBeFalsy();
1221
- expect(result.content[0].text).toBe('{"threads":[]}');
1222
- });
1223
- });
1224
-
1225
- /**
1226
- * DEFECT 3, the half of it that survived grounding.
1227
- *
1228
- * `worker.ts` builds `makeFlyExecutor(FLY_ENDPOINT, key)` with NO token source,
1229
- * so the eviction + replay machinery is inert on the hosted path: `gog` runs as
1230
- * the Fly volume's own identity, and a Google 401 stops at the "no access token
1231
- * was supplied" guard. That much is by design and stays.
1232
- *
1233
- * What did NOT survive is the idea of building a replay for it. `gog` is spawned
1234
- * fresh per `/run` and re-reads the keyring every time, so there is no
1235
- * cross-spawn in-memory token that could go stale — a Google 401 here means the
1236
- * stored credential itself was refused, and no retry can fix that. Building a
1237
- * retry would have been the fifth plausible theory in a row.
1238
- *
1239
- * So this path gets INSTRUMENTATION instead. The one thing nobody could answer
1240
- * after the incident was: at the moment Google refused that call, was the
1241
- * refresh token on the volume alive or dead? `replay.declined` records only that
1242
- * WE did nothing. These tests pin a record of what GOOGLE said, measured at the
1243
- * moment of the refusal with the probe `/health/google` — and pin that the
1244
- * measurement changes nothing the caller sees.
1245
- */
1246
- describe('the Google-layer measurement taken when a hosted call is refused', () => {
1247
- const ENDPOINT = 'https://gogcli-gog-runner.fly.dev';
1248
- const KEY = 'k';
1249
- const PREFIX = 'gog-auth ';
1250
-
1251
- const GOOGLE_401_STDERR =
1252
- 'Google API error (401 authError): Request had invalid authentication credentials.';
1253
- const READ = ['--json', '--color=never', '--no-input', 'gmail', 'search', 'q'];
1254
-
1255
- function gogFailed(stderr: string) {
1256
- return {
1257
- ok: false,
1258
- status: 422,
1259
- json: async () => ({ error: `Command failed: gog gmail search q\n${stderr}`, stderr }),
1260
- };
1261
- }
1262
- const probeBody = (body: unknown, status = 200) => ({
1263
- ok: status >= 200 && status < 300,
1264
- status,
1265
- json: async () => body,
1266
- });
1267
-
1268
- /** A `fetch` that answers `/run` and `/health/google` separately. */
1269
- function routedFetch(run: () => unknown, probe: () => unknown) {
1270
- return vi.fn(async (url: string) => {
1271
- if (url.endsWith('/run')) return run();
1272
- if (url.endsWith('/health/google')) return probe();
1273
- throw new Error(`unexpected fetch: ${url}`);
1274
- });
1275
- }
1276
- const urls = (f: { mock: { calls: unknown[][] } }) => f.mock.calls.map((c) => c[0] as string);
1277
- const runCalls = (f: { mock: { calls: unknown[][] } }) =>
1278
- urls(f).filter((u) => u.endsWith('/run')).length;
1279
- const probeCalls = (f: { mock: { calls: unknown[][] } }) =>
1280
- urls(f).filter((u) => u.endsWith('/health/google')).length;
1281
-
1282
- function captureLog() {
1283
- const emitted: Array<{ method: 'warn' | 'error'; line: string }> = [];
1284
- const toStdout: string[] = [];
1285
- for (const method of ['warn', 'error'] as const) {
1286
- vi.spyOn(console, method).mockImplementation((...args: unknown[]) => {
1287
- emitted.push({ method, line: args.map(String).join(' ') });
1288
- });
1289
- }
1290
- for (const method of ['log', 'info', 'debug', 'trace'] as const) {
1291
- vi.spyOn(console, method).mockImplementation((...args: unknown[]) => {
1292
- toStdout.push(args.map(String).join(' '));
1293
- });
1294
- }
1295
- return {
1296
- emitted,
1297
- toStdout,
1298
- records(): Record<string, unknown>[] {
1299
- return emitted.map((e) => {
1300
- expect(e.line.startsWith(PREFIX)).toBe(true);
1301
- return JSON.parse(e.line.slice(PREFIX.length)) as Record<string, unknown>;
1302
- });
1303
- },
1304
- byEvent(event: string): Record<string, unknown> | undefined {
1305
- return this.records().find((r) => r.event === event);
1306
- },
1307
- };
1308
- }
1309
-
1310
- it('asks the runner whether Google still accepts the credential, with the same bearer', async () => {
1311
- const log = captureLog();
1312
- const fetchMock = routedFetch(
1313
- () => gogFailed(GOOGLE_401_STDERR),
1314
- () => probeBody({ ok: true, measured: true, accounts: [{ email: 'a@b.c', valid: true }] }),
1315
- );
1316
- vi.stubGlobal('fetch', fetchMock);
1317
-
1318
- const exec = makeFlyExecutor(ENDPOINT, KEY);
1319
- // The caller's error is untouched — this is instrumentation, not recovery.
1320
- await expect(exec(READ, {})).rejects.toThrow(/Google API error \(401/);
1321
-
1322
- expect(runCalls(fetchMock)).toBe(1);
1323
- expect(probeCalls(fetchMock)).toBe(1);
1324
- const [, init] = fetchMock.mock.calls[1] as [string, RequestInit];
1325
- expect(init.headers).toEqual({ Authorization: `Bearer ${KEY}` });
1326
- expect(init.signal).toBeInstanceOf(AbortSignal);
1327
- });
1328
-
1329
- it('records an UNEXPLAINED refusal when Google says the credential is fine', async () => {
1330
- // The narrow theory the plan refused to build a fix for: a stored token
1331
- // refused by Google while the grant behind it is alive. This record is the
1332
- // only thing that can ever prove or kill it, so it is emitted at error
1333
- // level — "we cannot explain this" is the loudest thing a log can say.
1334
- const log = captureLog();
1335
- vi.stubGlobal('fetch', routedFetch(
1336
- () => gogFailed(GOOGLE_401_STDERR),
1337
- () => probeBody({ ok: true, measured: true, accounts: [{ email: 'a@b.c', valid: true }] }),
1338
- ));
1339
-
1340
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1341
-
1342
- const record = log.byEvent('refusal.google-ok');
1343
- expect(record).toBeDefined();
1344
- expect(record!.service).toBe('gmail');
1345
- expect(record!.endpoint).toBe(ENDPOINT);
1346
- expect(log.emitted.find((e) => e.line.includes('refusal.google-ok'))!.method).toBe('error');
1347
- // Still followed by the decision record, so the pair reads: what Google
1348
- // said, then what we did about it.
1349
- expect(log.byEvent('replay.declined')).toBeDefined();
1350
- expect(log.toStdout).toEqual([]);
1351
- });
1352
-
1353
- it('records a dead credential, carrying the runner’s classification and not gog’s words', async () => {
1354
- const log = captureLog();
1355
- const cause =
1356
- 'invalid_grant: the stored Google refresh token is expired or revoked — re-authorize the account';
1357
- vi.stubGlobal('fetch', routedFetch(
1358
- () => gogFailed(GOOGLE_401_STDERR),
1359
- () => probeBody({ ok: false, measured: true, accounts: [], error: cause }),
1360
- ));
1361
-
1362
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1363
-
1364
- const record = log.byEvent('refusal.google-unhealthy');
1365
- expect(record).toBeDefined();
1366
- expect(record!.reason).toBe(cause);
1367
- expect(log.emitted.find((e) => e.line.includes('refusal.google-unhealthy'))!.method).toBe('error');
1368
- });
1369
-
1370
- it('reports an unhealthy layer even when the runner names no cause', async () => {
1371
- const log = captureLog();
1372
- vi.stubGlobal('fetch', routedFetch(
1373
- () => gogFailed(GOOGLE_401_STDERR),
1374
- () => probeBody({ ok: false, measured: true, accounts: [] }),
1375
- ));
1376
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1377
- expect(log.byEvent('refusal.google-unhealthy')!.reason).toMatch(/no cause/);
1378
- });
1379
-
1380
- it('REVIEW DEFECT: a probe that could not RUN never becomes "the credential is refused"', async () => {
1381
- // This is the record that decides an incident: `refusal.google-unhealthy`
1382
- // means "Google refused the call AND the live check agrees the credential is
1383
- // refused". Three of the runner's causes are facts about the probe — it
1384
- // timed out, it could not be run at all (no `gog` on PATH, no
1385
- // `credentials.json` on the volume), its output could not be parsed — and
1386
- // filing those here would tell an operator the refresh token was dead on
1387
- // evidence nobody gathered. Worse, it silently disables `refusal.google-ok`,
1388
- // the ONE record that can prove or kill the narrow theory.
1389
- for (const error of [
1390
- 'the Google probe timed out before gog answered',
1391
- 'the Google probe could not be run',
1392
- 'gog auth list --check returned unrecognized output',
1393
- 'gog did not report token validity',
1394
- 'gog reported an account it explicitly did not check',
1395
- ]) {
1396
- const log = captureLog();
1397
- vi.stubGlobal('fetch', routedFetch(
1398
- () => gogFailed(GOOGLE_401_STDERR),
1399
- () => probeBody({ ok: false, measured: false, accounts: [], error }),
1400
- ));
1401
-
1402
- // The caller's error is untouched, as always.
1403
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/Google API error \(401/);
1404
-
1405
- expect(log.byEvent('refusal.google-unhealthy')).toBeUndefined();
1406
- const record = log.byEvent('refusal.google-unmeasured');
1407
- expect(record).toBeDefined();
1408
- expect(record!.reason).toBe(error);
1409
- expect(log.emitted.find((e) => e.line.includes('refusal.google-unmeasured'))!.method).toBe('warn');
1410
- vi.restoreAllMocks();
1411
- }
1412
- });
1413
-
1414
- it('will not claim the credential is refused from a runner that never said it measured', async () => {
1415
- const log = captureLog();
1416
- vi.stubGlobal('fetch', routedFetch(
1417
- () => gogFailed(GOOGLE_401_STDERR),
1418
- () => probeBody({ ok: false, accounts: [], error: 'something went wrong' }),
1419
- ));
1420
-
1421
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1422
-
1423
- expect(log.byEvent('refusal.google-unhealthy')).toBeUndefined();
1424
- expect(log.byEvent('refusal.google-unmeasured')!.reason).toContain('something went wrong');
1425
- });
1426
-
1427
- it('never turns a probe that could not run into a claim about Google', async () => {
1428
- // A runner deployed before /health/google existed answers 404. "I could not
1429
- // ask" must never be filed as "Google said no" — that is the defect this
1430
- // whole branch exists to delete, with the alarm merely inverted.
1431
- const log = captureLog();
1432
- vi.stubGlobal('fetch', routedFetch(
1433
- () => gogFailed(GOOGLE_401_STDERR),
1434
- () => probeBody({ error: 'not found' }, 404),
1435
- ));
1436
-
1437
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1438
-
1439
- const record = log.byEvent('refusal.google-unmeasured');
1440
- expect(record).toBeDefined();
1441
- expect(record!.reason).toMatch(/404/);
1442
- // NOT an error: the absence of a measurement is not evidence of anything.
1443
- expect(log.emitted.find((e) => e.line.includes('refusal.google-unmeasured'))!.method).toBe('warn');
1444
- expect(log.byEvent('refusal.google-unhealthy')).toBeUndefined();
1445
- });
1446
-
1447
- it('survives a probe that rejects, and still lets the original error through', async () => {
1448
- const log = captureLog();
1449
- vi.stubGlobal('fetch', routedFetch(
1450
- () => gogFailed(GOOGLE_401_STDERR),
1451
- () => { throw new Error('network down'); },
1452
- ));
1453
-
1454
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/Google API error \(401/);
1455
- expect(log.byEvent('refusal.google-unmeasured')!.reason).toMatch(/network down/);
1456
- });
1457
-
1458
- it('survives a probe that rejects with a non-Error', async () => {
1459
- const log = captureLog();
1460
- vi.stubGlobal('fetch', routedFetch(
1461
- () => gogFailed(GOOGLE_401_STDERR),
1462
- () => { throw 'nope'; },
1463
- ));
1464
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1465
- expect(log.byEvent('refusal.google-unmeasured')!.reason).toBe('nope');
1466
- });
1467
-
1468
- it('does not ask a question gog already answered: invalid_grant is not probed', async () => {
1469
- // gog said the grant is dead. Spending a Google API call to be told the same
1470
- // thing buys nothing, and this is the COMMON failure — the 7-day cliff — so
1471
- // probing it would be the one case that costs the most and learns the least.
1472
- const log = captureLog();
1473
- const fetchMock = routedFetch(
1474
- () => gogFailed(`${GOOGLE_401_STDERR}\noauth2: "invalid_grant"`),
1475
- () => probeBody({ ok: true, measured: true, accounts: [] }),
1476
- );
1477
- vi.stubGlobal('fetch', fetchMock);
1478
-
1479
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/invalid_grant/);
1480
- expect(probeCalls(fetchMock)).toBe(0);
1481
- expect(log.byEvent('grant.dead')).toBeDefined();
1482
- });
1483
-
1484
- it('MUST NOT REGRESS: a runner transport failure is never probed for Google health', async () => {
1485
- // The runner's own 401 is about OUR bearer. gog never ran and no Google
1486
- // credential was read, so asking Google anything here would re-create the
1487
- // exact misattribution 2.21.1 fixed — one layer down, in the log.
1488
- const fetchMock = routedFetch(
1489
- () => ({ ok: false, status: 401, json: async () => ({ error: 'unauthorized' }) }),
1490
- () => probeBody({ ok: true, measured: true, accounts: [] }),
1491
- );
1492
- vi.stubGlobal('fetch', fetchMock);
1493
- captureLog();
1494
-
1495
- const err = await makeFlyExecutor(ENDPOINT, KEY)(READ, {}).catch((e: unknown) => e);
1496
- expect(isRunnerTransportError(err)).toBe(true);
1497
- expect(probeCalls(fetchMock)).toBe(0);
1498
- });
1499
-
1500
- it('MUST NOT REGRESS: an ordinary gog failure is never probed', async () => {
1501
- const fetchMock = routedFetch(
1502
- () => gogFailed('row 401 is outside the sheet grid'),
1503
- () => probeBody({ ok: true, measured: true, accounts: [] }),
1504
- );
1505
- vi.stubGlobal('fetch', fetchMock);
1506
- captureLog();
1507
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/outside the sheet grid/);
1508
- expect(probeCalls(fetchMock)).toBe(0);
1509
- });
1510
-
1511
- it('does not probe when the call carried a token of ours — that path repairs itself', async () => {
1512
- const fetchMock = routedFetch(
1513
- () => gogFailed(GOOGLE_401_STDERR),
1514
- () => probeBody({ ok: true, measured: true, accounts: [] }),
1515
- );
1516
- vi.stubGlobal('fetch', fetchMock);
1517
- captureLog();
1518
- const readToken = Object.assign(vi.fn(async () => 'ya29.stale'), {
1519
- invalidate: vi.fn(async () => true),
1520
- });
1521
-
1522
- await expect(makeFlyExecutor(ENDPOINT, KEY, readToken)(READ, {})).rejects.toThrow(/401/);
1523
- // Two /run attempts (the replay), and no probe: the eviction already
1524
- // answered the question the probe would ask.
1525
- expect(runCalls(fetchMock)).toBe(2);
1526
- expect(probeCalls(fetchMock)).toBe(0);
1527
- });
1528
-
1529
- it('will not spend the caller’s remaining deadline on a diagnostic', async () => {
1530
- // The probe shares the tool call's ONE deadline. Below the floor it could
1531
- // only abort, and an abort here would delay the caller's real error for
1532
- // nothing.
1533
- const log = captureLog();
1534
- const fetchMock = routedFetch(
1535
- () => gogFailed(GOOGLE_401_STDERR),
1536
- () => probeBody({ ok: true, measured: true, accounts: [] }),
1537
- );
1538
- vi.stubGlobal('fetch', fetchMock);
1539
-
1540
- // opts.timeout 0 leaves only DEADLINE_GRACE_MS. The clock is read once to
1541
- // fix the deadline and once by the probe; making the second read land a
1542
- // minute later is exactly "the first attempt ran long".
1543
- const start = Date.now();
1544
- let reads = 0;
1545
- vi.spyOn(Date, 'now').mockImplementation(() => (reads++ === 0 ? start : start + 60_000));
1546
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, { timeout: 0 })).rejects.toThrow(/401/);
1547
-
1548
- expect(probeCalls(fetchMock)).toBe(0);
1549
- expect(log.byEvent('refusal.google-unmeasured')!.reason).toMatch(/deadline/);
1550
- });
1551
-
1552
- it('probes at most once per interval, so a retry loop cannot storm the backend', async () => {
1553
- // /health/google spawns a real gog and takes the keyring's exclusive flock
1554
- // (gogcli v0.34.1: auth list → ListTokens → withWriteLock → unix.LOCK_EX;
1555
- // see the sourced note on PROBE_INTERVAL_MS). A model retrying a dead call
1556
- // must not turn one diagnostic into a queue.
1557
- const log = captureLog();
1558
- const fetchMock = routedFetch(
1559
- () => gogFailed(GOOGLE_401_STDERR),
1560
- () => probeBody({ ok: true, measured: true, accounts: [] }),
1561
- );
1562
- vi.stubGlobal('fetch', fetchMock);
1563
-
1564
- const exec = makeFlyExecutor(ENDPOINT, KEY);
1565
- await expect(exec(READ, {})).rejects.toThrow(/401/);
1566
- await expect(exec(READ, {})).rejects.toThrow(/401/);
1567
-
1568
- expect(runCalls(fetchMock)).toBe(2);
1569
- expect(probeCalls(fetchMock)).toBe(1);
1570
- expect(log.byEvent('refusal.google-unmeasured')!.reason).toMatch(/attempted recently/);
1571
- });
1572
-
1573
- // End to end through the REAL run() and the REAL diagnose(), because the claim
1574
- // that matters most about this whole feature is a NEGATIVE one: the caller's
1575
- // result is byte-identical whether the probe ran or not. This also re-pins the
1576
- // #250 shape — `Google API error (401 authError)` still reaching the auth
1577
- // hint — with the probe in the loop.
1578
- it('MUST NOT REGRESS: the caller’s diagnosed result is identical with the probe in the loop', async () => {
1579
- captureLog();
1580
- const withProbe = routedFetch(
1581
- () => gogFailed(GOOGLE_401_STDERR),
1582
- () => probeBody({ ok: true, measured: true, accounts: [] }),
1583
- );
1584
- vi.stubGlobal('fetch', withProbe);
1585
- const probed = await runExecutor.run({ executor: makeFlyExecutor(ENDPOINT, KEY) }, () =>
1586
- runOrDiagnose(['gmail', 'search', 'q'], {}),
1587
- );
1588
-
1589
- // The same failure through an executor whose probe is skipped outright
1590
- // (gog said invalid_grant is a different error, so use the throttle: a
1591
- // second call on the same executor never probes).
1592
- const exec = makeFlyExecutor(ENDPOINT, KEY);
1593
- const twice = routedFetch(
1594
- () => gogFailed(GOOGLE_401_STDERR),
1595
- () => probeBody({ ok: true, measured: true, accounts: [] }),
1596
- );
1597
- vi.stubGlobal('fetch', twice);
1598
- await runExecutor.run({ executor: exec }, () => runOrDiagnose(['gmail', 'search', 'q'], {}));
1599
- const unprobed = await runExecutor.run({ executor: exec }, () =>
1600
- runOrDiagnose(['gmail', 'search', 'q'], {}),
1601
- );
1602
-
1603
- expect(probed.isError).toBe(true);
1604
- expect(probed.content[0].text).toContain('Google API error (401 authError)');
1605
- expect(probed.content[0].text).toContain('gog_auth_add');
1606
- // The whole point: the probe is invisible to the caller.
1607
- expect(unprobed.content[0].text).toBe(probed.content[0].text);
1608
- });
1609
-
1610
- it('MUST NOT CLAIM: the throttle line never asserts a measurement that did not happen', async () => {
1611
- // The whole thesis of this branch is that a log line may not claim health it
1612
- // did not measure. The throttle is the one place that rule can be broken
1613
- // from the inside: `lastProbeAt` is stamped BEFORE the fetch and is not
1614
- // reset when the probe comes back with no verdict, so every refusal for the
1615
- // next PROBE_INTERVAL_MS is explained by a sentence about the previous
1616
- // probe. If that sentence says the layer "was measured", it is describing a
1617
- // measurement that never occurred — here, a runner too old to have
1618
- // /health/google at all.
1619
- //
1620
- // Stamping before the await is correct and stays: it is what stops two
1621
- // overlapping refusals from both spawning a probe, and the backend cost the
1622
- // throttle protects was paid whether or not a verdict came back. So the
1623
- // sentence is what has to be true, not the timestamp.
1624
- const log = captureLog();
1625
- const fetchMock = routedFetch(
1626
- () => gogFailed(GOOGLE_401_STDERR),
1627
- () => probeBody({ error: 'not found' }, 404),
1628
- );
1629
- vi.stubGlobal('fetch', fetchMock);
1630
-
1631
- const exec = makeFlyExecutor(ENDPOINT, KEY);
1632
- await expect(exec(READ, {})).rejects.toThrow(/401/);
1633
- await expect(exec(READ, {})).rejects.toThrow(/401/);
1634
-
1635
- // One probe attempted, and it produced no verdict about Google.
1636
- expect(probeCalls(fetchMock)).toBe(1);
1637
- const unmeasured = log
1638
- .records()
1639
- .filter((r) => r.event === 'refusal.google-unmeasured')
1640
- .map((r) => r.reason as string);
1641
- expect(unmeasured).toHaveLength(2);
1642
- expect(unmeasured[0]).toMatch(/did not answer the Google probe \(HTTP 404\)/);
1643
-
1644
- // The throttled line: it must say a probe was ATTEMPTED, never that the
1645
- // Google layer was measured.
1646
- expect(unmeasured[1]).toMatch(/attempted recently/);
1647
- expect(unmeasured[1]).not.toMatch(/measured recently|was measured|re-measured/);
1648
- });
1649
-
1650
- it('throttles per executor, so one session cannot silence another', async () => {
1651
- const log = captureLog();
1652
- const fetchMock = routedFetch(
1653
- () => gogFailed(GOOGLE_401_STDERR),
1654
- () => probeBody({ ok: true, measured: true, accounts: [] }),
1655
- );
1656
- vi.stubGlobal('fetch', fetchMock);
1657
-
1658
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1659
- await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1660
-
1661
- expect(probeCalls(fetchMock)).toBe(2);
1662
- expect(log.byEvent('refusal.google-unmeasured')).toBeUndefined();
1663
- });
1664
- });