gogcli-mcp 2.21.0 → 2.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/dist/index.js +436 -58
- package/dist/lib.js +437 -59
- package/manifest.json +1 -1
- package/package.json +1 -1
- package/server.json +2 -2
- package/src/auth-log.ts +205 -0
- package/src/connector-auth.ts +265 -8
- package/src/connector-runtime.ts +764 -79
- package/src/google-probe.ts +113 -0
- package/src/google-token.ts +181 -15
- package/src/runner.ts +67 -2
- package/src/timestamps.ts +7 -0
- package/src/tools/auth.ts +8 -2
- package/src/tools/utils.ts +70 -5
- package/src/worker.ts +33 -3
- package/tests/auth-log.test.ts +530 -0
- package/tests/connector-auth.test.ts +539 -8
- package/tests/connector-runtime.test.ts +1121 -2
- package/tests/google-probe.test.ts +116 -0
- package/tests/google-token.test.ts +125 -4
- package/tests/runner.test.ts +21 -1
- package/tests/timestamps.test.ts +52 -0
- package/tests/tools/auth-401-context.test.ts +42 -0
- package/tests/tools/auth-401-shapes.test.ts +50 -0
- package/tests/tools/auth.test.ts +28 -0
- package/tests/tools/utils.test.ts +55 -2
- package/tests/worker.test.ts +33 -8
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { describe, it, expect, vi, afterEach } from 'vitest';
|
|
2
|
-
import { run } from '../src/runner.js';
|
|
2
|
+
import { run, runExecutor } from '../src/runner.js';
|
|
3
3
|
import type { GogArg, GogExecutor, GogFileArg } from '../src/runner.js';
|
|
4
|
-
import { makeFlyExecutor, wrapServer } from '../src/connector-runtime.js';
|
|
4
|
+
import { makeFlyExecutor, wrapServer, RunnerTransportError, isRunnerTransportError } from '../src/connector-runtime.js';
|
|
5
|
+
import { runOrDiagnose } from '../src/tools/utils.js';
|
|
6
|
+
import { makeAccessTokenSource, clearAccessTokenCache } from '../src/google-token.js';
|
|
5
7
|
|
|
6
8
|
afterEach(() => {
|
|
7
9
|
vi.unstubAllEnvs();
|
|
@@ -523,3 +525,1120 @@ describe('makeFlyExecutor', () => {
|
|
|
523
525
|
await expect(exec(['x'], {})).rejects.toThrow(/gog failed on the runner/i);
|
|
524
526
|
});
|
|
525
527
|
});
|
|
528
|
+
|
|
529
|
+
// A failure the RUNNER authored — its own bearer check, its own request
|
|
530
|
+
// validation, its own drain — never reached gog and never showed a credential
|
|
531
|
+
// to Google. Those failures must be distinguishable by TYPE, because the layer
|
|
532
|
+
// that diagnoses them (tools/utils.ts) can only otherwise guess from prose, and
|
|
533
|
+
// guessing is what turned the runner's bare `unauthorized` body into "your
|
|
534
|
+
// Google sign-in expired, re-authorize".
|
|
535
|
+
describe('makeFlyExecutor runner-authored transport failures', () => {
|
|
536
|
+
const ENDPOINT = 'https://gogcli-gog-runner.fly.dev';
|
|
537
|
+
const KEY = 'k';
|
|
538
|
+
|
|
539
|
+
function stubStatus(status: number, body?: unknown) {
|
|
540
|
+
vi.stubGlobal(
|
|
541
|
+
'fetch',
|
|
542
|
+
vi.fn(async () => ({
|
|
543
|
+
ok: false,
|
|
544
|
+
status,
|
|
545
|
+
json: async () => {
|
|
546
|
+
if (body === undefined) throw new Error('not json');
|
|
547
|
+
return body;
|
|
548
|
+
},
|
|
549
|
+
})),
|
|
550
|
+
);
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
async function thrownBy(status: number, body?: unknown): Promise<unknown> {
|
|
554
|
+
stubStatus(status, body);
|
|
555
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY);
|
|
556
|
+
return exec(['gmail', 'search', 'q'], {}).catch((e: unknown) => e);
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
// The exact body fly-gog-runner/server.mjs sends when its bearer check fails
|
|
560
|
+
// (server.mjs:450 /health, :460 /run) — nothing to do with Google.
|
|
561
|
+
const RUNNER_401 = { error: 'unauthorized' };
|
|
562
|
+
|
|
563
|
+
it('types the runner bearer rejection as transport auth, not a gog failure', async () => {
|
|
564
|
+
const err = await thrownBy(401, RUNNER_401);
|
|
565
|
+
expect(isRunnerTransportError(err)).toBe(true);
|
|
566
|
+
expect((err as RunnerTransportError).kind).toBe('transport-auth');
|
|
567
|
+
expect((err as RunnerTransportError).status).toBe(401);
|
|
568
|
+
});
|
|
569
|
+
|
|
570
|
+
it('names the real cause — the runner key mismatch — and never the Google account', async () => {
|
|
571
|
+
const err = (await thrownBy(401, RUNNER_401)) as Error;
|
|
572
|
+
expect(err.message).toContain('GOG_RUNNER_KEY');
|
|
573
|
+
expect(err.message).toContain('RUNNER_KEY');
|
|
574
|
+
expect(err.message).not.toMatch(/gog_auth_add/i);
|
|
575
|
+
// The runner's own body is the single word "unauthorized". Repeating it is
|
|
576
|
+
// what fed DEFINITE_AUTH_PATTERN in tools/utils.ts; the status digits can do
|
|
577
|
+
// the same whenever a status word sits near them. Neither may appear, so that
|
|
578
|
+
// even if the TYPE is lost at some future boundary the prose cannot be
|
|
579
|
+
// misread as Google's.
|
|
580
|
+
expect(err.message).not.toMatch(/unauthorized/i);
|
|
581
|
+
expect(err.message).not.toMatch(/\b401\b/);
|
|
582
|
+
});
|
|
583
|
+
|
|
584
|
+
it('types the runner request-validation 400s as transport-request, keeping their detail', async () => {
|
|
585
|
+
for (const detail of [
|
|
586
|
+
'request body too large',
|
|
587
|
+
'failed to read request body',
|
|
588
|
+
'body must be valid JSON',
|
|
589
|
+
'args must be an array',
|
|
590
|
+
'accessToken must not contain whitespace or control characters',
|
|
591
|
+
]) {
|
|
592
|
+
const err = await thrownBy(400, { error: detail });
|
|
593
|
+
expect(isRunnerTransportError(err)).toBe(true);
|
|
594
|
+
expect((err as RunnerTransportError).kind).toBe('transport-request');
|
|
595
|
+
expect((err as RunnerTransportError).status).toBe(400);
|
|
596
|
+
expect((err as Error).message).toBe(detail);
|
|
597
|
+
}
|
|
598
|
+
});
|
|
599
|
+
|
|
600
|
+
it('still says something when a 400 body is unreadable', async () => {
|
|
601
|
+
const err = await thrownBy(400);
|
|
602
|
+
expect(isRunnerTransportError(err)).toBe(true);
|
|
603
|
+
expect((err as RunnerTransportError).kind).toBe('transport-request');
|
|
604
|
+
expect((err as Error).message).toMatch(/gog-runner rejected the request/i);
|
|
605
|
+
});
|
|
606
|
+
|
|
607
|
+
it('types the drain 503 as retryable transport, message unchanged', async () => {
|
|
608
|
+
const err = await thrownBy(503, { error: 'gog-runner is shutting down', retryable: true });
|
|
609
|
+
expect(isRunnerTransportError(err)).toBe(true);
|
|
610
|
+
expect((err as RunnerTransportError).kind).toBe('transport-retryable');
|
|
611
|
+
expect((err as RunnerTransportError).status).toBe(503);
|
|
612
|
+
expect((err as Error).message).toBe('gog-runner is restarting; retry this call. gog-runner is shutting down');
|
|
613
|
+
});
|
|
614
|
+
|
|
615
|
+
it('types a retryable-flagged 500 (a materialization failure) as retryable transport', async () => {
|
|
616
|
+
const err = await thrownBy(500, {
|
|
617
|
+
error: 'failed to write a file arg to disk: ENOSPC: no space left on device',
|
|
618
|
+
retryable: true,
|
|
619
|
+
});
|
|
620
|
+
expect(isRunnerTransportError(err)).toBe(true);
|
|
621
|
+
expect((err as RunnerTransportError).kind).toBe('transport-retryable');
|
|
622
|
+
expect((err as RunnerTransportError).status).toBe(500);
|
|
623
|
+
});
|
|
624
|
+
|
|
625
|
+
it('types a bodiless gateway failure as retryable transport', async () => {
|
|
626
|
+
const err = await thrownBy(502);
|
|
627
|
+
expect(isRunnerTransportError(err)).toBe(true);
|
|
628
|
+
expect((err as RunnerTransportError).kind).toBe('transport-retryable');
|
|
629
|
+
expect((err as RunnerTransportError).status).toBe(502);
|
|
630
|
+
});
|
|
631
|
+
|
|
632
|
+
it('types the client-side deadline as retryable transport with no status', async () => {
|
|
633
|
+
vi.stubGlobal(
|
|
634
|
+
'fetch',
|
|
635
|
+
vi.fn(async () => {
|
|
636
|
+
throw Object.assign(new Error('The operation was aborted'), { name: 'TimeoutError' });
|
|
637
|
+
}),
|
|
638
|
+
);
|
|
639
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY);
|
|
640
|
+
const err = await exec(['x'], {}).catch((e: unknown) => e);
|
|
641
|
+
expect(isRunnerTransportError(err)).toBe(true);
|
|
642
|
+
expect((err as RunnerTransportError).kind).toBe('transport-retryable');
|
|
643
|
+
expect((err as RunnerTransportError).status).toBeUndefined();
|
|
644
|
+
});
|
|
645
|
+
|
|
646
|
+
// The other half of the contract: an error that carries gog's/Google's OWN
|
|
647
|
+
// words must stay untyped, so the prose classifier still gets to read it.
|
|
648
|
+
it('leaves a 422 (gog ran and failed) untyped, for the prose classifier', async () => {
|
|
649
|
+
const err = await thrownBy(422, { error: 'gog failed', stderr: 'Error 401: invalid_grant' });
|
|
650
|
+
expect(err).toBeInstanceOf(Error);
|
|
651
|
+
expect(isRunnerTransportError(err)).toBe(false);
|
|
652
|
+
});
|
|
653
|
+
|
|
654
|
+
it('leaves a detail-bearing non-2xx untyped', async () => {
|
|
655
|
+
const err = await thrownBy(502, { error: 'gog exited with code 1', stderr: 'bad flag' });
|
|
656
|
+
expect(isRunnerTransportError(err)).toBe(false);
|
|
657
|
+
});
|
|
658
|
+
|
|
659
|
+
it('recognises only branded errors', () => {
|
|
660
|
+
expect(isRunnerTransportError(new Error('unauthorized'))).toBe(false);
|
|
661
|
+
expect(isRunnerTransportError('unauthorized')).toBe(false);
|
|
662
|
+
expect(isRunnerTransportError(new RunnerTransportError('x', 'transport-auth', 401))).toBe(true);
|
|
663
|
+
});
|
|
664
|
+
|
|
665
|
+
// The whole chain, end to end, with the REAL executor, the REAL run() and the
|
|
666
|
+
// REAL diagnose(): the incident was a user told all session to re-authorize a
|
|
667
|
+
// Google account that was never asked for a credential.
|
|
668
|
+
it('does not tell the caller to re-authorize Google when the RUNNER rejected our bearer', async () => {
|
|
669
|
+
vi.stubEnv('GOG_ACCOUNT', '');
|
|
670
|
+
vi.stubEnv('GOG_READONLY', '');
|
|
671
|
+
stubStatus(401, RUNNER_401);
|
|
672
|
+
const executor = makeFlyExecutor(ENDPOINT, KEY);
|
|
673
|
+
const result = await runExecutor.run({ executor }, () =>
|
|
674
|
+
runOrDiagnose(['sheets', 'get', 'A1'], {}),
|
|
675
|
+
);
|
|
676
|
+
const text = result.content[0].text as string;
|
|
677
|
+
expect(result.isError).toBe(true);
|
|
678
|
+
expect(text).not.toMatch(/gog_auth_add/);
|
|
679
|
+
expect(text).not.toMatch(/re-authorize the account/i);
|
|
680
|
+
expect(text).toContain('GOG_RUNNER_KEY');
|
|
681
|
+
});
|
|
682
|
+
});
|
|
683
|
+
|
|
684
|
+
// A Google 401 on the ACCESS token and a dead REFRESH token are opposite
|
|
685
|
+
// failures that used to be told to the user identically.
|
|
686
|
+
//
|
|
687
|
+
// * The access token is ours to replace. Google rejecting it means "mint
|
|
688
|
+
// another and try again" — automatic, no human, no re-authorization. Before
|
|
689
|
+
// this, the rejected token stayed cached for the rest of its nominal hour
|
|
690
|
+
// and NOTHING retried, so every call in that window failed the same way and
|
|
691
|
+
// only reconnecting (a fresh isolate, an empty cache) appeared to help.
|
|
692
|
+
// * invalid_grant means the refresh token is gone. No re-mint is possible and
|
|
693
|
+
// a human must re-authorize. Retrying that is a loop against a credential
|
|
694
|
+
// that can never work.
|
|
695
|
+
//
|
|
696
|
+
// So the re-mint is gated hard: gog must actually have run (422), the token must
|
|
697
|
+
// be one WE supplied, gog's own stderr must say Google rejected it, the call
|
|
698
|
+
// must be safe to replay, and our cache must still hold exactly that token.
|
|
699
|
+
describe('makeFlyExecutor re-mints a rejected access token and replays once', () => {
|
|
700
|
+
const ENDPOINT = 'https://gogcli-gog-runner.fly.dev';
|
|
701
|
+
const KEY = 'k';
|
|
702
|
+
|
|
703
|
+
// gog v0.34.1's real words when the access token it was handed is rejected.
|
|
704
|
+
const GOOGLE_401_STDERR =
|
|
705
|
+
'Note: Using direct access token (expires in ~1 hour; no auto-refresh)\n' +
|
|
706
|
+
'Google API error (401 authError): Request had invalid authentication credentials. ' +
|
|
707
|
+
'Expected OAuth 2 access token, login cookie or other valid authentication credential.';
|
|
708
|
+
|
|
709
|
+
// What the executor really receives: run()'s assembleArgs puts the global
|
|
710
|
+
// flags in front of the service and subcommand.
|
|
711
|
+
const READ = ['--json', '--color=never', '--no-input', 'gmail', 'search', 'q'];
|
|
712
|
+
const WRITE = ['--json', '--color=never', '--no-input', 'gmail', 'send', '--to', 'a@b.c'];
|
|
713
|
+
|
|
714
|
+
function gogFailed(stderr: string) {
|
|
715
|
+
return {
|
|
716
|
+
ok: false,
|
|
717
|
+
status: 422,
|
|
718
|
+
// `error` is Node's execFile message, which embeds the whole command line
|
|
719
|
+
// AND stderr — see the command-line test below for why that matters.
|
|
720
|
+
json: async () => ({
|
|
721
|
+
error: `Command failed: gog gmail search q\n${stderr}`,
|
|
722
|
+
stderr,
|
|
723
|
+
retryable: false,
|
|
724
|
+
}),
|
|
725
|
+
};
|
|
726
|
+
}
|
|
727
|
+
function ok(stdout: string) {
|
|
728
|
+
return { ok: true, json: async () => ({ stdout }) };
|
|
729
|
+
}
|
|
730
|
+
function bodyOf(fetchMock: { mock: { calls: unknown[][] } }, i: number): Record<string, unknown> {
|
|
731
|
+
const [, init] = fetchMock.mock.calls[i] as [string, RequestInit];
|
|
732
|
+
return JSON.parse(init.body as string) as Record<string, unknown>;
|
|
733
|
+
}
|
|
734
|
+
/** A token source that hands out `tokens` in order and can be invalidated. */
|
|
735
|
+
function source(tokens: (string | undefined)[], evicted = true) {
|
|
736
|
+
const fn = vi.fn(async () => tokens.shift());
|
|
737
|
+
return Object.assign(fn, { invalidate: vi.fn(async () => evicted) });
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
it('mints a new token and replays the call, invisibly to the caller', async () => {
|
|
741
|
+
const fetchMock = vi
|
|
742
|
+
.fn()
|
|
743
|
+
.mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
|
|
744
|
+
.mockResolvedValueOnce(ok('thread json'));
|
|
745
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
746
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
747
|
+
|
|
748
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
749
|
+
await expect(exec(READ, {})).resolves.toBe('thread json');
|
|
750
|
+
|
|
751
|
+
expect(fetchMock).toHaveBeenCalledTimes(2);
|
|
752
|
+
expect(bodyOf(fetchMock, 0).accessToken).toBe('ya29.stale');
|
|
753
|
+
expect(bodyOf(fetchMock, 1).accessToken).toBe('ya29.fresh');
|
|
754
|
+
// Evicted by VALUE — only the token that was actually rejected.
|
|
755
|
+
expect(readToken.invalidate).toHaveBeenCalledTimes(1);
|
|
756
|
+
expect(readToken.invalidate).toHaveBeenCalledWith('ya29.stale');
|
|
757
|
+
});
|
|
758
|
+
|
|
759
|
+
it('replays exactly once, so a genuinely dead credential cannot loop', async () => {
|
|
760
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
761
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
762
|
+
const readToken = source(['ya29.stale', 'ya29.fresh', 'ya29.third']);
|
|
763
|
+
|
|
764
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
765
|
+
await expect(exec(READ, {})).rejects.toThrow(/Google API error \(401/);
|
|
766
|
+
|
|
767
|
+
expect(fetchMock).toHaveBeenCalledTimes(2);
|
|
768
|
+
// Two evictions, one per token Google refused — the replay's included. The
|
|
769
|
+
// ATTEMPT count is what "exactly once" is about, and it is still two.
|
|
770
|
+
expect(readToken.invalidate).toHaveBeenCalledTimes(2);
|
|
771
|
+
});
|
|
772
|
+
|
|
773
|
+
it('does not replay when gog reported invalid_grant — only a human can fix that', async () => {
|
|
774
|
+
// The stderr deliberately matches BOTH shapes, so this pins the exclusion
|
|
775
|
+
// rather than the absence of a 401.
|
|
776
|
+
const fetchMock = vi.fn(async () => gogFailed(`${GOOGLE_401_STDERR}\noauth2: "invalid_grant"`));
|
|
777
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
778
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
779
|
+
|
|
780
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
781
|
+
await expect(exec(READ, {})).rejects.toThrow(/invalid_grant/);
|
|
782
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
783
|
+
expect(readToken.invalidate).not.toHaveBeenCalled();
|
|
784
|
+
});
|
|
785
|
+
|
|
786
|
+
it('does not replay a call that supplied no token of ours', async () => {
|
|
787
|
+
// Nothing was minted, so there is nothing to re-mint: gog used whatever
|
|
788
|
+
// identity the backend volume holds, and only an operator can change that.
|
|
789
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
790
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
791
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY);
|
|
792
|
+
await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
|
|
793
|
+
// Counted by ENDPOINT, not in total: this same refusal now also takes a
|
|
794
|
+
// read of the Google layer (`/health/google`, see the refusal-probe block
|
|
795
|
+
// below), and that reading is a diagnostic, not a second attempt. What
|
|
796
|
+
// "does not replay" means is that gog ran exactly once.
|
|
797
|
+
const runs = fetchMock.mock.calls.filter(([url]) => String(url).endsWith('/run'));
|
|
798
|
+
expect(runs).toHaveLength(1);
|
|
799
|
+
});
|
|
800
|
+
|
|
801
|
+
it('does not replay when the token source cannot invalidate', async () => {
|
|
802
|
+
// A bare `() => token` (the #230 direct-token wiring) has no cache behind
|
|
803
|
+
// it, so replaying would re-send the identical rejected token.
|
|
804
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
805
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
806
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, () => 'ya29.direct');
|
|
807
|
+
await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
|
|
808
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
809
|
+
});
|
|
810
|
+
|
|
811
|
+
it('does not replay when the cache no longer held the rejected token', async () => {
|
|
812
|
+
// Another caller already refreshed it; the token we would send is the one
|
|
813
|
+
// that is already in use, so a replay proves nothing.
|
|
814
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
815
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
816
|
+
const readToken = source(['ya29.stale', 'ya29.fresh'], false);
|
|
817
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
818
|
+
await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
|
|
819
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
820
|
+
expect(readToken.invalidate).toHaveBeenCalledTimes(1);
|
|
821
|
+
});
|
|
822
|
+
|
|
823
|
+
// A subcommand is only a leaf VERB some of the time. `gog tasks lists` is a
|
|
824
|
+
// NAMESPACE — `lists list` reads, `lists create` writes — and the allow-list
|
|
825
|
+
// is consulted with the namespace word, never the verb under it. So a
|
|
826
|
+
// namespace word in the set hands the replay to every child it will ever
|
|
827
|
+
// grow, including the ones that write.
|
|
828
|
+
//
|
|
829
|
+
// `tasks lists create <title> ...` is real in gog v0.34.1 today, and reachable
|
|
830
|
+
// without any new gog: `gog_tasks_run({subcommand: 'lists', args: ['create',
|
|
831
|
+
// 'A', 'B']})` assembles exactly this argv. It is variadic, so one invocation
|
|
832
|
+
// makes N Google calls and a 401 on the second means the first already landed
|
|
833
|
+
// — the precise double-apply the write rule exists to prevent.
|
|
834
|
+
it('does not replay `tasks lists create`, a WRITE under a namespace-shaped word', async () => {
|
|
835
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
836
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
837
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
838
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
839
|
+
await expect(
|
|
840
|
+
exec(['--json', '--color=never', '--no-input', 'tasks', 'lists', 'create', 'A', 'B'], {}),
|
|
841
|
+
).rejects.toThrow(/Google API error/);
|
|
842
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
843
|
+
// The eviction still happens — it runs before the allow-list is consulted.
|
|
844
|
+
expect(readToken.invalidate).toHaveBeenCalledTimes(1);
|
|
845
|
+
});
|
|
846
|
+
|
|
847
|
+
// The invariant the allow-list's comment states, pinned as behaviour so the
|
|
848
|
+
// comment is no longer the only thing enforcing it. Every word here is a gog
|
|
849
|
+
// NAMESPACE that already has, or can grow, a mutating child; none of them may
|
|
850
|
+
// ever earn a replay, whichever verb follows.
|
|
851
|
+
it.each([
|
|
852
|
+
['tasks', 'lists'],
|
|
853
|
+
['gmail', 'labels'],
|
|
854
|
+
['gmail', 'drafts'],
|
|
855
|
+
['gmail', 'filters'],
|
|
856
|
+
['gmail', 'sendas'],
|
|
857
|
+
['drive', 'permissions'],
|
|
858
|
+
['drive', 'revisions'],
|
|
859
|
+
['docs', 'comments'],
|
|
860
|
+
['docs', 'replies'],
|
|
861
|
+
])('never replays the namespace word %s %s, whatever verb follows it', async (service, namespace) => {
|
|
862
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
863
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
864
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
865
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
866
|
+
await expect(
|
|
867
|
+
exec(['--json', '--color=never', '--no-input', service, namespace, 'create', 'x'], {}),
|
|
868
|
+
).rejects.toThrow(/Google API error/);
|
|
869
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
870
|
+
// The eviction is unaffected: it runs before the allow-list is consulted.
|
|
871
|
+
expect(readToken.invalidate).toHaveBeenCalledTimes(1);
|
|
872
|
+
});
|
|
873
|
+
|
|
874
|
+
// Symmetry with the rule the whole fix is built on: a token Google has
|
|
875
|
+
// refused must not stay cached, and the very same 401 refused the replay's
|
|
876
|
+
// token. This is NOT a round-trip saving — measured through the real chain
|
|
877
|
+
// under a sustained non-invalid_grant refusal, a steady-state call costs two
|
|
878
|
+
// /run round-trips either way, and this eviction adds a mint (2 rather than
|
|
879
|
+
// 1) by emptying a cache the next call would have hit. What it buys is a
|
|
880
|
+
// bound on how long a KNOWN-REFUSED token can be served: left cached it is
|
|
881
|
+
// re-served for the rest of its nominal hour, and a write — which gets the
|
|
882
|
+
// eviction and no replay — would be sent with it and fail on contact.
|
|
883
|
+
it('evicts the replayed token too when Google refuses that one as well', async () => {
|
|
884
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
885
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
886
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
887
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
888
|
+
await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
|
|
889
|
+
expect(readToken.invalidate.mock.calls).toEqual([['ya29.stale'], ['ya29.fresh']]);
|
|
890
|
+
});
|
|
891
|
+
|
|
892
|
+
// The other half of that rule, and the one that keeps this branch honest.
|
|
893
|
+
//
|
|
894
|
+
// A replay can fail without Google ever seeing the token: the Machine starts
|
|
895
|
+
// draining between the two attempts, the client-side deadline fires, the
|
|
896
|
+
// runner's own bearer is rotated mid-call. Evicting on THOSE would throw away
|
|
897
|
+
// a token nothing has refused and — worse — emit `token.evicted` with
|
|
898
|
+
// "Google rejected this access token", which is a lie about a service that
|
|
899
|
+
// was never consulted. Misattributing a runner-side failure to Google is the
|
|
900
|
+
// exact defect this branch exists to remove; re-introducing it in the log
|
|
901
|
+
// would just move it from the user's screen to the operator's query.
|
|
902
|
+
it('does not evict the replayed token when the replay never reached Google', async () => {
|
|
903
|
+
const fetchMock = vi
|
|
904
|
+
.fn()
|
|
905
|
+
.mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
|
|
906
|
+
// The runner drains between the two attempts — a transport failure, not
|
|
907
|
+
// a verdict on the freshly minted token.
|
|
908
|
+
.mockResolvedValueOnce({
|
|
909
|
+
ok: false,
|
|
910
|
+
status: 503,
|
|
911
|
+
json: async () => ({ error: 'gog-runner is shutting down', retryable: true }),
|
|
912
|
+
});
|
|
913
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
914
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
915
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
916
|
+
const err = await exec(READ, {}).catch((e: unknown) => e);
|
|
917
|
+
// The replay's own error reaches the caller, and here that is the RIGHT
|
|
918
|
+
// one to surface: "the runner is restarting, retry" is actionable, where
|
|
919
|
+
// re-raising the superseded Google 401 would send the user off to
|
|
920
|
+
// re-authorize an account that is fine.
|
|
921
|
+
expect(isRunnerTransportError(err)).toBe(true);
|
|
922
|
+
expect((err as RunnerTransportError).kind).toBe('transport-retryable');
|
|
923
|
+
// Only the token Google actually refused was dropped. `ya29.fresh` stays
|
|
924
|
+
// cached: it is unproven, not refused, and the next call may well succeed
|
|
925
|
+
// with it once the new Machine is up.
|
|
926
|
+
expect(readToken.invalidate.mock.calls).toEqual([['ya29.stale']]);
|
|
927
|
+
});
|
|
928
|
+
|
|
929
|
+
// The mint is a first-class error surface on this branch, so the error it
|
|
930
|
+
// raises has to reach the caller with the SAME specificity a gog-authored
|
|
931
|
+
// invalid_grant gets: the 7-day Testing-mode cause and the headless re-auth
|
|
932
|
+
// pair. It only does if the message carries the literal `invalid_grant`,
|
|
933
|
+
// which is what tools/utils.ts keys the richer hint on.
|
|
934
|
+
it('gives a mint-path invalid_grant the full re-auth guidance, not the generic hint', async () => {
|
|
935
|
+
vi.stubEnv('GOG_ACCOUNT', '');
|
|
936
|
+
vi.stubEnv('GOG_READONLY', '');
|
|
937
|
+
clearAccessTokenCache();
|
|
938
|
+
const fetchMock = vi.fn(async (url: unknown) => {
|
|
939
|
+
if (String(url).includes('oauth2.googleapis.com')) {
|
|
940
|
+
return new Response(JSON.stringify({ error: 'invalid_grant' }), { status: 400 });
|
|
941
|
+
}
|
|
942
|
+
return gogFailed('unreachable');
|
|
943
|
+
});
|
|
944
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
945
|
+
const executor = makeFlyExecutor(
|
|
946
|
+
ENDPOINT,
|
|
947
|
+
KEY,
|
|
948
|
+
makeAccessTokenSource({
|
|
949
|
+
GOG_CLIENT_ID: 'cid',
|
|
950
|
+
GOG_CLIENT_SECRET: 'cs',
|
|
951
|
+
GOG_REFRESH_TOKEN: 'rt-dead',
|
|
952
|
+
}),
|
|
953
|
+
);
|
|
954
|
+
|
|
955
|
+
const result = await runExecutor.run({ executor }, () =>
|
|
956
|
+
runOrDiagnose(['gmail', 'search', 'q'], {}),
|
|
957
|
+
);
|
|
958
|
+
expect(result.isError).toBe(true);
|
|
959
|
+
const text = result.content[0].text as string;
|
|
960
|
+
// Text unique to INVALID_GRANT_HINT — the durable fix. The generic
|
|
961
|
+
// AUTH_HINT (which is what a message omitting `invalid_grant` earns) says
|
|
962
|
+
// only "Authentication may have expired".
|
|
963
|
+
expect(text).toContain('publish the OAuth consent screen to "In production"');
|
|
964
|
+
expect(text).not.toContain('Authentication may have expired');
|
|
965
|
+
clearAccessTokenCache();
|
|
966
|
+
});
|
|
967
|
+
|
|
968
|
+
it('does not replay a WRITE, but still evicts the token Google rejected', async () => {
|
|
969
|
+
// The one case an automatic replay must never take. A `gog gmail send` that
|
|
970
|
+
// failed AFTER the message went out would send it twice.
|
|
971
|
+
//
|
|
972
|
+
// The EVICTION is a different question, and the write rule must not answer
|
|
973
|
+
// it. Refusing to evict is what leaves Google's rejected token in the cache
|
|
974
|
+
// for the rest of its nominal hour, so every following call — write or read
|
|
975
|
+
// — re-sends it and fails identically until the isolate is replaced. That
|
|
976
|
+
// is DEFECT 2 itself, and it is the half a write path used to keep.
|
|
977
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
978
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
979
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
980
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
981
|
+
await expect(exec(WRITE, {})).rejects.toThrow(/Google API error/);
|
|
982
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
983
|
+
expect(readToken.invalidate).toHaveBeenCalledTimes(1);
|
|
984
|
+
expect(readToken.invalidate).toHaveBeenCalledWith('ya29.stale');
|
|
985
|
+
});
|
|
986
|
+
|
|
987
|
+
it('evicts for a READ whose subcommand is outside the allow-list', async () => {
|
|
988
|
+
// `gog gmail labels list` arrives here as the subcommand `labels`, which is
|
|
989
|
+
// deliberately not in READ_ONLY_SUBCOMMANDS (a later `labels create` would
|
|
990
|
+
// inherit the replay). Costing that call its replay is the intended price;
|
|
991
|
+
// costing it the eviction would poison every call after it.
|
|
992
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
993
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
994
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
995
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
996
|
+
await expect(
|
|
997
|
+
exec(['--json', '--color=never', '--no-input', 'gmail', 'labels', 'list'], {}),
|
|
998
|
+
).rejects.toThrow(/Google API error/);
|
|
999
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
1000
|
+
expect(readToken.invalidate).toHaveBeenCalledTimes(1);
|
|
1001
|
+
});
|
|
1002
|
+
|
|
1003
|
+
it('lets a second WRITE mint a fresh token, through the REAL token source', async () => {
|
|
1004
|
+
// The incident, reproduced end to end with nothing stubbed but the network:
|
|
1005
|
+
// two consecutive `gog gmail send` calls after Google has rejected the
|
|
1006
|
+
// cached token. Before the eviction was moved ahead of the write rule, BOTH
|
|
1007
|
+
// shipped `ya29.t1` and both failed, for up to ~58 minutes, and only a
|
|
1008
|
+
// reconnect helped.
|
|
1009
|
+
clearAccessTokenCache();
|
|
1010
|
+
let minted = 0;
|
|
1011
|
+
const fetchMock = vi.fn(async (url: unknown) => {
|
|
1012
|
+
if (String(url).includes('oauth2.googleapis.com')) {
|
|
1013
|
+
return new Response(
|
|
1014
|
+
JSON.stringify({ access_token: `ya29.t${(minted += 1)}`, expires_in: 3600 }),
|
|
1015
|
+
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
1016
|
+
);
|
|
1017
|
+
}
|
|
1018
|
+
return gogFailed(GOOGLE_401_STDERR);
|
|
1019
|
+
});
|
|
1020
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1021
|
+
|
|
1022
|
+
const exec = makeFlyExecutor(
|
|
1023
|
+
ENDPOINT,
|
|
1024
|
+
KEY,
|
|
1025
|
+
makeAccessTokenSource({
|
|
1026
|
+
GOG_CLIENT_ID: 'cid',
|
|
1027
|
+
GOG_CLIENT_SECRET: 'cs',
|
|
1028
|
+
GOG_REFRESH_TOKEN: 'rt-1',
|
|
1029
|
+
}),
|
|
1030
|
+
);
|
|
1031
|
+
await expect(exec(WRITE, {})).rejects.toThrow(/Google API error/);
|
|
1032
|
+
await expect(exec(WRITE, {})).rejects.toThrow(/Google API error/);
|
|
1033
|
+
|
|
1034
|
+
const runBodies = fetchMock.mock.calls
|
|
1035
|
+
.filter(([url]) => !String(url).includes('oauth2.googleapis.com'))
|
|
1036
|
+
.map(([, init]) => JSON.parse((init as RequestInit).body as string) as { accessToken: string });
|
|
1037
|
+
expect(runBodies.map((b) => b.accessToken)).toEqual(['ya29.t1', 'ya29.t2']);
|
|
1038
|
+
clearAccessTokenCache();
|
|
1039
|
+
});
|
|
1040
|
+
|
|
1041
|
+
it('replays with what is LEFT of the deadline, not a second full one', async () => {
|
|
1042
|
+
// 30s default + 5s grace is one tool call's whole budget. Handing the
|
|
1043
|
+
// replay a fresh copy of it makes the worst case ~70s of wall clock, which
|
|
1044
|
+
// can outlast the MCP client's own request timeout and turn a self-healing
|
|
1045
|
+
// read into a client-side hang.
|
|
1046
|
+
const budgets: number[] = [];
|
|
1047
|
+
const realTimeout = AbortSignal.timeout.bind(AbortSignal);
|
|
1048
|
+
vi.spyOn(AbortSignal, 'timeout').mockImplementation((ms: number) => {
|
|
1049
|
+
budgets.push(ms);
|
|
1050
|
+
return realTimeout(ms);
|
|
1051
|
+
});
|
|
1052
|
+
const fetchMock = vi
|
|
1053
|
+
.fn()
|
|
1054
|
+
.mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
|
|
1055
|
+
.mockResolvedValueOnce(ok('threads'));
|
|
1056
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1057
|
+
const base = Date.now();
|
|
1058
|
+
// The clock is read once to fix the deadline, then once more to size the
|
|
1059
|
+
// replay; 5s of the budget is gone by then.
|
|
1060
|
+
vi.spyOn(Date, 'now').mockReturnValueOnce(base).mockReturnValue(base + 5_000);
|
|
1061
|
+
|
|
1062
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, source(['ya29.stale', 'ya29.fresh']));
|
|
1063
|
+
await expect(exec(READ, {})).resolves.toBe('threads');
|
|
1064
|
+
expect(budgets).toEqual([35_000, 30_000]);
|
|
1065
|
+
});
|
|
1066
|
+
|
|
1067
|
+
it('skips the replay when the first attempt used the whole deadline', async () => {
|
|
1068
|
+
// With no budget left, a replay can only end in an abort, and that
|
|
1069
|
+
// TimeoutError would REPLACE gog's own 401 — trading an actionable error
|
|
1070
|
+
// for an opaque one. The eviction is the durable half of the repair and it
|
|
1071
|
+
// has already happened, so the caller's own next call is the fresh one.
|
|
1072
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
1073
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1074
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
1075
|
+
const base = Date.now();
|
|
1076
|
+
vi.spyOn(Date, 'now').mockReturnValueOnce(base).mockReturnValue(base + 34_900);
|
|
1077
|
+
|
|
1078
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
1079
|
+
await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
|
|
1080
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
1081
|
+
expect(readToken.invalidate).toHaveBeenCalledTimes(1);
|
|
1082
|
+
});
|
|
1083
|
+
|
|
1084
|
+
it('finds the subcommand past --account, whose value is not a subcommand', async () => {
|
|
1085
|
+
const fetchMock = vi
|
|
1086
|
+
.fn()
|
|
1087
|
+
.mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
|
|
1088
|
+
.mockResolvedValueOnce(ok('[]'));
|
|
1089
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1090
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
1091
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
1092
|
+
await expect(
|
|
1093
|
+
exec(['--json', '--color=never', '--no-input', '--account', 'me@example.com', 'drive', 'ls'], {}),
|
|
1094
|
+
).resolves.toBe('[]');
|
|
1095
|
+
expect(fetchMock).toHaveBeenCalledTimes(2);
|
|
1096
|
+
});
|
|
1097
|
+
|
|
1098
|
+
it('does not replay an invocation with no subcommand to judge', async () => {
|
|
1099
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
1100
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1101
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
1102
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
1103
|
+
await expect(exec(['--json', '--color=never', 'gmail'], {})).rejects.toThrow(/Google API error/);
|
|
1104
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
1105
|
+
});
|
|
1106
|
+
|
|
1107
|
+
it('reads gog stderr, not the command line the runner echoed back', async () => {
|
|
1108
|
+
// execFile's message embeds the whole argv, so a caller's own text lands in
|
|
1109
|
+
// `error`. Classifying on that would let `--subject "invoice 401"` trigger a
|
|
1110
|
+
// replay of a call that failed for an unrelated reason — and if that call
|
|
1111
|
+
// were a write, replay it after it had partly applied.
|
|
1112
|
+
const fetchMock = vi.fn(async () => ({
|
|
1113
|
+
ok: false,
|
|
1114
|
+
status: 422,
|
|
1115
|
+
json: async () => ({
|
|
1116
|
+
error: 'Command failed: gog gmail search "Google API error (401 authError)"\nno results',
|
|
1117
|
+
stderr: 'no results',
|
|
1118
|
+
retryable: false,
|
|
1119
|
+
}),
|
|
1120
|
+
}));
|
|
1121
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1122
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
1123
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
1124
|
+
await expect(exec(READ, {})).rejects.toThrow(/no results/);
|
|
1125
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
1126
|
+
expect(readToken.invalidate).not.toHaveBeenCalled();
|
|
1127
|
+
});
|
|
1128
|
+
|
|
1129
|
+
it('does not replay a gog failure that has nothing to do with auth', async () => {
|
|
1130
|
+
const fetchMock = vi.fn(async () => gogFailed('invalid attachment id'));
|
|
1131
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1132
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
1133
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
1134
|
+
await expect(exec(READ, {})).rejects.toThrow(/invalid attachment id/);
|
|
1135
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
1136
|
+
});
|
|
1137
|
+
|
|
1138
|
+
it('surfaces the original failure when the re-mint yields no token', async () => {
|
|
1139
|
+
// Sending the request without a token would run it as the BACKEND's
|
|
1140
|
+
// identity and hand this caller someone else's mailbox — never that.
|
|
1141
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
1142
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1143
|
+
const readToken = source(['ya29.stale', undefined]);
|
|
1144
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
1145
|
+
await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
|
|
1146
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
1147
|
+
});
|
|
1148
|
+
|
|
1149
|
+
it('surfaces the re-mint failure, which is the more actionable one', async () => {
|
|
1150
|
+
// Evicting revealed that the refresh token is dead too. That message names
|
|
1151
|
+
// the real repair (re-authorize); gog's 401 does not.
|
|
1152
|
+
const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
|
|
1153
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1154
|
+
const readToken = Object.assign(
|
|
1155
|
+
vi
|
|
1156
|
+
.fn<() => Promise<string | undefined>>()
|
|
1157
|
+
.mockResolvedValueOnce('ya29.stale')
|
|
1158
|
+
.mockRejectedValueOnce(new Error('the stored refresh token has expired or been revoked')),
|
|
1159
|
+
{ invalidate: vi.fn(async () => true) },
|
|
1160
|
+
);
|
|
1161
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
1162
|
+
await expect(exec(READ, {})).rejects.toThrow(/refresh token has expired or been revoked/);
|
|
1163
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
1164
|
+
});
|
|
1165
|
+
|
|
1166
|
+
it('never re-mints for the RUNNER\'s own 401, which never reached Google', async () => {
|
|
1167
|
+
// The defect-1 failure. Its body is the bare word "unauthorized" and no
|
|
1168
|
+
// Google credential was even read, so minting a new one is pure waste — and
|
|
1169
|
+
// replaying would double every call during a key mismatch.
|
|
1170
|
+
const fetchMock = vi.fn(async () => ({
|
|
1171
|
+
ok: false,
|
|
1172
|
+
status: 401,
|
|
1173
|
+
json: async () => ({ error: 'unauthorized' }),
|
|
1174
|
+
}));
|
|
1175
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1176
|
+
const readToken = source(['ya29.stale', 'ya29.fresh']);
|
|
1177
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
|
|
1178
|
+
const err = await exec(READ, {}).catch((e: unknown) => e);
|
|
1179
|
+
expect(isRunnerTransportError(err)).toBe(true);
|
|
1180
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
1181
|
+
expect(readToken.invalidate).not.toHaveBeenCalled();
|
|
1182
|
+
});
|
|
1183
|
+
|
|
1184
|
+
// End to end through the REAL run() and the REAL diagnose(): the user-visible
|
|
1185
|
+
// property is that a stale access token produces no error and no advice at
|
|
1186
|
+
// all, because it healed itself.
|
|
1187
|
+
it('turns a stale-token failure into a plain success, with no re-auth advice', async () => {
|
|
1188
|
+
vi.stubEnv('GOG_ACCOUNT', '');
|
|
1189
|
+
vi.stubEnv('GOG_READONLY', '');
|
|
1190
|
+
const fetchMock = vi
|
|
1191
|
+
.fn()
|
|
1192
|
+
.mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
|
|
1193
|
+
.mockResolvedValueOnce(ok('{"threads":[]}'));
|
|
1194
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1195
|
+
const executor = makeFlyExecutor(ENDPOINT, KEY, source(['ya29.stale', 'ya29.fresh']));
|
|
1196
|
+
|
|
1197
|
+
const result = await runExecutor.run({ executor }, () =>
|
|
1198
|
+
runOrDiagnose(['gmail', 'search', 'q'], {}),
|
|
1199
|
+
);
|
|
1200
|
+
expect(result.isError).toBeFalsy();
|
|
1201
|
+
expect(result.content[0].text).toBe('{"threads":[]}');
|
|
1202
|
+
});
|
|
1203
|
+
});
|
|
1204
|
+
|
|
1205
|
+
/**
|
|
1206
|
+
* DEFECT 3, the half of it that survived grounding.
|
|
1207
|
+
*
|
|
1208
|
+
* `worker.ts` builds `makeFlyExecutor(FLY_ENDPOINT, key)` with NO token source,
|
|
1209
|
+
* so the eviction + replay machinery is inert on the hosted path: `gog` runs as
|
|
1210
|
+
* the Fly volume's own identity, and a Google 401 stops at the "no access token
|
|
1211
|
+
* was supplied" guard. That much is by design and stays.
|
|
1212
|
+
*
|
|
1213
|
+
* What did NOT survive is the idea of building a replay for it. `gog` is spawned
|
|
1214
|
+
* fresh per `/run` and re-reads the keyring every time, so there is no
|
|
1215
|
+
* cross-spawn in-memory token that could go stale — a Google 401 here means the
|
|
1216
|
+
* stored credential itself was refused, and no retry can fix that. Building a
|
|
1217
|
+
* retry would have been the fifth plausible theory in a row.
|
|
1218
|
+
*
|
|
1219
|
+
* So this path gets INSTRUMENTATION instead. The one thing nobody could answer
|
|
1220
|
+
* after the incident was: at the moment Google refused that call, was the
|
|
1221
|
+
* refresh token on the volume alive or dead? `replay.declined` records only that
|
|
1222
|
+
* WE did nothing. These tests pin a record of what GOOGLE said, measured at the
|
|
1223
|
+
* moment of the refusal with the probe `/health/google` — and pin that the
|
|
1224
|
+
* measurement changes nothing the caller sees.
|
|
1225
|
+
*/
|
|
1226
|
+
describe('the Google-layer measurement taken when a hosted call is refused', () => {
|
|
1227
|
+
const ENDPOINT = 'https://gogcli-gog-runner.fly.dev';
|
|
1228
|
+
const KEY = 'k';
|
|
1229
|
+
const PREFIX = 'gog-auth ';
|
|
1230
|
+
|
|
1231
|
+
const GOOGLE_401_STDERR =
|
|
1232
|
+
'Google API error (401 authError): Request had invalid authentication credentials.';
|
|
1233
|
+
const READ = ['--json', '--color=never', '--no-input', 'gmail', 'search', 'q'];
|
|
1234
|
+
|
|
1235
|
+
function gogFailed(stderr: string) {
|
|
1236
|
+
return {
|
|
1237
|
+
ok: false,
|
|
1238
|
+
status: 422,
|
|
1239
|
+
json: async () => ({ error: `Command failed: gog gmail search q\n${stderr}`, stderr }),
|
|
1240
|
+
};
|
|
1241
|
+
}
|
|
1242
|
+
const probeBody = (body: unknown, status = 200) => ({
|
|
1243
|
+
ok: status >= 200 && status < 300,
|
|
1244
|
+
status,
|
|
1245
|
+
json: async () => body,
|
|
1246
|
+
});
|
|
1247
|
+
|
|
1248
|
+
/** A `fetch` that answers `/run` and `/health/google` separately. */
|
|
1249
|
+
function routedFetch(run: () => unknown, probe: () => unknown) {
|
|
1250
|
+
return vi.fn(async (url: string) => {
|
|
1251
|
+
if (url.endsWith('/run')) return run();
|
|
1252
|
+
if (url.endsWith('/health/google')) return probe();
|
|
1253
|
+
throw new Error(`unexpected fetch: ${url}`);
|
|
1254
|
+
});
|
|
1255
|
+
}
|
|
1256
|
+
const urls = (f: { mock: { calls: unknown[][] } }) => f.mock.calls.map((c) => c[0] as string);
|
|
1257
|
+
const runCalls = (f: { mock: { calls: unknown[][] } }) =>
|
|
1258
|
+
urls(f).filter((u) => u.endsWith('/run')).length;
|
|
1259
|
+
const probeCalls = (f: { mock: { calls: unknown[][] } }) =>
|
|
1260
|
+
urls(f).filter((u) => u.endsWith('/health/google')).length;
|
|
1261
|
+
|
|
1262
|
+
function captureLog() {
|
|
1263
|
+
const emitted: Array<{ method: 'warn' | 'error'; line: string }> = [];
|
|
1264
|
+
const toStdout: string[] = [];
|
|
1265
|
+
for (const method of ['warn', 'error'] as const) {
|
|
1266
|
+
vi.spyOn(console, method).mockImplementation((...args: unknown[]) => {
|
|
1267
|
+
emitted.push({ method, line: args.map(String).join(' ') });
|
|
1268
|
+
});
|
|
1269
|
+
}
|
|
1270
|
+
for (const method of ['log', 'info', 'debug', 'trace'] as const) {
|
|
1271
|
+
vi.spyOn(console, method).mockImplementation((...args: unknown[]) => {
|
|
1272
|
+
toStdout.push(args.map(String).join(' '));
|
|
1273
|
+
});
|
|
1274
|
+
}
|
|
1275
|
+
return {
|
|
1276
|
+
emitted,
|
|
1277
|
+
toStdout,
|
|
1278
|
+
records(): Record<string, unknown>[] {
|
|
1279
|
+
return emitted.map((e) => {
|
|
1280
|
+
expect(e.line.startsWith(PREFIX)).toBe(true);
|
|
1281
|
+
return JSON.parse(e.line.slice(PREFIX.length)) as Record<string, unknown>;
|
|
1282
|
+
});
|
|
1283
|
+
},
|
|
1284
|
+
byEvent(event: string): Record<string, unknown> | undefined {
|
|
1285
|
+
return this.records().find((r) => r.event === event);
|
|
1286
|
+
},
|
|
1287
|
+
};
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
it('asks the runner whether Google still accepts the credential, with the same bearer', async () => {
|
|
1291
|
+
const log = captureLog();
|
|
1292
|
+
const fetchMock = routedFetch(
|
|
1293
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1294
|
+
() => probeBody({ ok: true, measured: true, accounts: [{ email: 'a@b.c', valid: true }] }),
|
|
1295
|
+
);
|
|
1296
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1297
|
+
|
|
1298
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY);
|
|
1299
|
+
// The caller's error is untouched — this is instrumentation, not recovery.
|
|
1300
|
+
await expect(exec(READ, {})).rejects.toThrow(/Google API error \(401/);
|
|
1301
|
+
|
|
1302
|
+
expect(runCalls(fetchMock)).toBe(1);
|
|
1303
|
+
expect(probeCalls(fetchMock)).toBe(1);
|
|
1304
|
+
const [, init] = fetchMock.mock.calls[1] as [string, RequestInit];
|
|
1305
|
+
expect(init.headers).toEqual({ Authorization: `Bearer ${KEY}` });
|
|
1306
|
+
expect(init.signal).toBeInstanceOf(AbortSignal);
|
|
1307
|
+
});
|
|
1308
|
+
|
|
1309
|
+
it('records an UNEXPLAINED refusal when Google says the credential is fine', async () => {
|
|
1310
|
+
// The narrow theory the plan refused to build a fix for: a stored token
|
|
1311
|
+
// refused by Google while the grant behind it is alive. This record is the
|
|
1312
|
+
// only thing that can ever prove or kill it, so it is emitted at error
|
|
1313
|
+
// level — "we cannot explain this" is the loudest thing a log can say.
|
|
1314
|
+
const log = captureLog();
|
|
1315
|
+
vi.stubGlobal('fetch', routedFetch(
|
|
1316
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1317
|
+
() => probeBody({ ok: true, measured: true, accounts: [{ email: 'a@b.c', valid: true }] }),
|
|
1318
|
+
));
|
|
1319
|
+
|
|
1320
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
|
|
1321
|
+
|
|
1322
|
+
const record = log.byEvent('refusal.google-ok');
|
|
1323
|
+
expect(record).toBeDefined();
|
|
1324
|
+
expect(record!.service).toBe('gmail');
|
|
1325
|
+
expect(record!.endpoint).toBe(ENDPOINT);
|
|
1326
|
+
expect(log.emitted.find((e) => e.line.includes('refusal.google-ok'))!.method).toBe('error');
|
|
1327
|
+
// Still followed by the decision record, so the pair reads: what Google
|
|
1328
|
+
// said, then what we did about it.
|
|
1329
|
+
expect(log.byEvent('replay.declined')).toBeDefined();
|
|
1330
|
+
expect(log.toStdout).toEqual([]);
|
|
1331
|
+
});
|
|
1332
|
+
|
|
1333
|
+
it('records a dead credential, carrying the runner’s classification and not gog’s words', async () => {
|
|
1334
|
+
const log = captureLog();
|
|
1335
|
+
const cause =
|
|
1336
|
+
'invalid_grant: the stored Google refresh token is expired or revoked — re-authorize the account';
|
|
1337
|
+
vi.stubGlobal('fetch', routedFetch(
|
|
1338
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1339
|
+
() => probeBody({ ok: false, measured: true, accounts: [], error: cause }),
|
|
1340
|
+
));
|
|
1341
|
+
|
|
1342
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
|
|
1343
|
+
|
|
1344
|
+
const record = log.byEvent('refusal.google-unhealthy');
|
|
1345
|
+
expect(record).toBeDefined();
|
|
1346
|
+
expect(record!.reason).toBe(cause);
|
|
1347
|
+
expect(log.emitted.find((e) => e.line.includes('refusal.google-unhealthy'))!.method).toBe('error');
|
|
1348
|
+
});
|
|
1349
|
+
|
|
1350
|
+
it('reports an unhealthy layer even when the runner names no cause', async () => {
|
|
1351
|
+
const log = captureLog();
|
|
1352
|
+
vi.stubGlobal('fetch', routedFetch(
|
|
1353
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1354
|
+
() => probeBody({ ok: false, measured: true, accounts: [] }),
|
|
1355
|
+
));
|
|
1356
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
|
|
1357
|
+
expect(log.byEvent('refusal.google-unhealthy')!.reason).toMatch(/no cause/);
|
|
1358
|
+
});
|
|
1359
|
+
|
|
1360
|
+
it('REVIEW DEFECT: a probe that could not RUN never becomes "the credential is refused"', async () => {
|
|
1361
|
+
// This is the record that decides an incident: `refusal.google-unhealthy`
|
|
1362
|
+
// means "Google refused the call AND the live check agrees the credential is
|
|
1363
|
+
// refused". Three of the runner's causes are facts about the probe — it
|
|
1364
|
+
// timed out, it could not be run at all (no `gog` on PATH, no
|
|
1365
|
+
// `credentials.json` on the volume), its output could not be parsed — and
|
|
1366
|
+
// filing those here would tell an operator the refresh token was dead on
|
|
1367
|
+
// evidence nobody gathered. Worse, it silently disables `refusal.google-ok`,
|
|
1368
|
+
// the ONE record that can prove or kill the narrow theory.
|
|
1369
|
+
for (const error of [
|
|
1370
|
+
'the Google probe timed out before gog answered',
|
|
1371
|
+
'the Google probe could not be run',
|
|
1372
|
+
'gog auth list --check returned unrecognized output',
|
|
1373
|
+
'gog did not report token validity',
|
|
1374
|
+
'gog reported an account it explicitly did not check',
|
|
1375
|
+
]) {
|
|
1376
|
+
const log = captureLog();
|
|
1377
|
+
vi.stubGlobal('fetch', routedFetch(
|
|
1378
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1379
|
+
() => probeBody({ ok: false, measured: false, accounts: [], error }),
|
|
1380
|
+
));
|
|
1381
|
+
|
|
1382
|
+
// The caller's error is untouched, as always.
|
|
1383
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/Google API error \(401/);
|
|
1384
|
+
|
|
1385
|
+
expect(log.byEvent('refusal.google-unhealthy')).toBeUndefined();
|
|
1386
|
+
const record = log.byEvent('refusal.google-unmeasured');
|
|
1387
|
+
expect(record).toBeDefined();
|
|
1388
|
+
expect(record!.reason).toBe(error);
|
|
1389
|
+
expect(log.emitted.find((e) => e.line.includes('refusal.google-unmeasured'))!.method).toBe('warn');
|
|
1390
|
+
vi.restoreAllMocks();
|
|
1391
|
+
}
|
|
1392
|
+
});
|
|
1393
|
+
|
|
1394
|
+
it('will not claim the credential is refused from a runner that never said it measured', async () => {
|
|
1395
|
+
const log = captureLog();
|
|
1396
|
+
vi.stubGlobal('fetch', routedFetch(
|
|
1397
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1398
|
+
() => probeBody({ ok: false, accounts: [], error: 'something went wrong' }),
|
|
1399
|
+
));
|
|
1400
|
+
|
|
1401
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
|
|
1402
|
+
|
|
1403
|
+
expect(log.byEvent('refusal.google-unhealthy')).toBeUndefined();
|
|
1404
|
+
expect(log.byEvent('refusal.google-unmeasured')!.reason).toContain('something went wrong');
|
|
1405
|
+
});
|
|
1406
|
+
|
|
1407
|
+
it('never turns a probe that could not run into a claim about Google', async () => {
|
|
1408
|
+
// A runner deployed before /health/google existed answers 404. "I could not
|
|
1409
|
+
// ask" must never be filed as "Google said no" — that is the defect this
|
|
1410
|
+
// whole branch exists to delete, with the alarm merely inverted.
|
|
1411
|
+
const log = captureLog();
|
|
1412
|
+
vi.stubGlobal('fetch', routedFetch(
|
|
1413
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1414
|
+
() => probeBody({ error: 'not found' }, 404),
|
|
1415
|
+
));
|
|
1416
|
+
|
|
1417
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
|
|
1418
|
+
|
|
1419
|
+
const record = log.byEvent('refusal.google-unmeasured');
|
|
1420
|
+
expect(record).toBeDefined();
|
|
1421
|
+
expect(record!.reason).toMatch(/404/);
|
|
1422
|
+
// NOT an error: the absence of a measurement is not evidence of anything.
|
|
1423
|
+
expect(log.emitted.find((e) => e.line.includes('refusal.google-unmeasured'))!.method).toBe('warn');
|
|
1424
|
+
expect(log.byEvent('refusal.google-unhealthy')).toBeUndefined();
|
|
1425
|
+
});
|
|
1426
|
+
|
|
1427
|
+
it('survives a probe that rejects, and still lets the original error through', async () => {
|
|
1428
|
+
const log = captureLog();
|
|
1429
|
+
vi.stubGlobal('fetch', routedFetch(
|
|
1430
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1431
|
+
() => { throw new Error('network down'); },
|
|
1432
|
+
));
|
|
1433
|
+
|
|
1434
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/Google API error \(401/);
|
|
1435
|
+
expect(log.byEvent('refusal.google-unmeasured')!.reason).toMatch(/network down/);
|
|
1436
|
+
});
|
|
1437
|
+
|
|
1438
|
+
it('survives a probe that rejects with a non-Error', async () => {
|
|
1439
|
+
const log = captureLog();
|
|
1440
|
+
vi.stubGlobal('fetch', routedFetch(
|
|
1441
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1442
|
+
() => { throw 'nope'; },
|
|
1443
|
+
));
|
|
1444
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
|
|
1445
|
+
expect(log.byEvent('refusal.google-unmeasured')!.reason).toBe('nope');
|
|
1446
|
+
});
|
|
1447
|
+
|
|
1448
|
+
it('does not ask a question gog already answered: invalid_grant is not probed', async () => {
|
|
1449
|
+
// gog said the grant is dead. Spending a Google API call to be told the same
|
|
1450
|
+
// thing buys nothing, and this is the COMMON failure — the 7-day cliff — so
|
|
1451
|
+
// probing it would be the one case that costs the most and learns the least.
|
|
1452
|
+
const log = captureLog();
|
|
1453
|
+
const fetchMock = routedFetch(
|
|
1454
|
+
() => gogFailed(`${GOOGLE_401_STDERR}\noauth2: "invalid_grant"`),
|
|
1455
|
+
() => probeBody({ ok: true, measured: true, accounts: [] }),
|
|
1456
|
+
);
|
|
1457
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1458
|
+
|
|
1459
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/invalid_grant/);
|
|
1460
|
+
expect(probeCalls(fetchMock)).toBe(0);
|
|
1461
|
+
expect(log.byEvent('grant.dead')).toBeDefined();
|
|
1462
|
+
});
|
|
1463
|
+
|
|
1464
|
+
it('MUST NOT REGRESS: a runner transport failure is never probed for Google health', async () => {
|
|
1465
|
+
// The runner's own 401 is about OUR bearer. gog never ran and no Google
|
|
1466
|
+
// credential was read, so asking Google anything here would re-create the
|
|
1467
|
+
// exact misattribution 2.21.1 fixed — one layer down, in the log.
|
|
1468
|
+
const fetchMock = routedFetch(
|
|
1469
|
+
() => ({ ok: false, status: 401, json: async () => ({ error: 'unauthorized' }) }),
|
|
1470
|
+
() => probeBody({ ok: true, measured: true, accounts: [] }),
|
|
1471
|
+
);
|
|
1472
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1473
|
+
captureLog();
|
|
1474
|
+
|
|
1475
|
+
const err = await makeFlyExecutor(ENDPOINT, KEY)(READ, {}).catch((e: unknown) => e);
|
|
1476
|
+
expect(isRunnerTransportError(err)).toBe(true);
|
|
1477
|
+
expect(probeCalls(fetchMock)).toBe(0);
|
|
1478
|
+
});
|
|
1479
|
+
|
|
1480
|
+
it('MUST NOT REGRESS: an ordinary gog failure is never probed', async () => {
|
|
1481
|
+
const fetchMock = routedFetch(
|
|
1482
|
+
() => gogFailed('row 401 is outside the sheet grid'),
|
|
1483
|
+
() => probeBody({ ok: true, measured: true, accounts: [] }),
|
|
1484
|
+
);
|
|
1485
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1486
|
+
captureLog();
|
|
1487
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/outside the sheet grid/);
|
|
1488
|
+
expect(probeCalls(fetchMock)).toBe(0);
|
|
1489
|
+
});
|
|
1490
|
+
|
|
1491
|
+
it('does not probe when the call carried a token of ours — that path repairs itself', async () => {
|
|
1492
|
+
const fetchMock = routedFetch(
|
|
1493
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1494
|
+
() => probeBody({ ok: true, measured: true, accounts: [] }),
|
|
1495
|
+
);
|
|
1496
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1497
|
+
captureLog();
|
|
1498
|
+
const readToken = Object.assign(vi.fn(async () => 'ya29.stale'), {
|
|
1499
|
+
invalidate: vi.fn(async () => true),
|
|
1500
|
+
});
|
|
1501
|
+
|
|
1502
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY, readToken)(READ, {})).rejects.toThrow(/401/);
|
|
1503
|
+
// Two /run attempts (the replay), and no probe: the eviction already
|
|
1504
|
+
// answered the question the probe would ask.
|
|
1505
|
+
expect(runCalls(fetchMock)).toBe(2);
|
|
1506
|
+
expect(probeCalls(fetchMock)).toBe(0);
|
|
1507
|
+
});
|
|
1508
|
+
|
|
1509
|
+
it('will not spend the caller’s remaining deadline on a diagnostic', async () => {
|
|
1510
|
+
// The probe shares the tool call's ONE deadline. Below the floor it could
|
|
1511
|
+
// only abort, and an abort here would delay the caller's real error for
|
|
1512
|
+
// nothing.
|
|
1513
|
+
const log = captureLog();
|
|
1514
|
+
const fetchMock = routedFetch(
|
|
1515
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1516
|
+
() => probeBody({ ok: true, measured: true, accounts: [] }),
|
|
1517
|
+
);
|
|
1518
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1519
|
+
|
|
1520
|
+
// opts.timeout 0 leaves only DEADLINE_GRACE_MS. The clock is read once to
|
|
1521
|
+
// fix the deadline and once by the probe; making the second read land a
|
|
1522
|
+
// minute later is exactly "the first attempt ran long".
|
|
1523
|
+
const start = Date.now();
|
|
1524
|
+
let reads = 0;
|
|
1525
|
+
vi.spyOn(Date, 'now').mockImplementation(() => (reads++ === 0 ? start : start + 60_000));
|
|
1526
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, { timeout: 0 })).rejects.toThrow(/401/);
|
|
1527
|
+
|
|
1528
|
+
expect(probeCalls(fetchMock)).toBe(0);
|
|
1529
|
+
expect(log.byEvent('refusal.google-unmeasured')!.reason).toMatch(/deadline/);
|
|
1530
|
+
});
|
|
1531
|
+
|
|
1532
|
+
it('probes at most once per interval, so a retry loop cannot storm the backend', async () => {
|
|
1533
|
+
// /health/google spawns a real gog and takes the keyring's exclusive flock
|
|
1534
|
+
// (gogcli v0.34.1: auth list → ListTokens → withWriteLock → unix.LOCK_EX;
|
|
1535
|
+
// see the sourced note on PROBE_INTERVAL_MS). A model retrying a dead call
|
|
1536
|
+
// must not turn one diagnostic into a queue.
|
|
1537
|
+
const log = captureLog();
|
|
1538
|
+
const fetchMock = routedFetch(
|
|
1539
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1540
|
+
() => probeBody({ ok: true, measured: true, accounts: [] }),
|
|
1541
|
+
);
|
|
1542
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1543
|
+
|
|
1544
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY);
|
|
1545
|
+
await expect(exec(READ, {})).rejects.toThrow(/401/);
|
|
1546
|
+
await expect(exec(READ, {})).rejects.toThrow(/401/);
|
|
1547
|
+
|
|
1548
|
+
expect(runCalls(fetchMock)).toBe(2);
|
|
1549
|
+
expect(probeCalls(fetchMock)).toBe(1);
|
|
1550
|
+
expect(log.byEvent('refusal.google-unmeasured')!.reason).toMatch(/attempted recently/);
|
|
1551
|
+
});
|
|
1552
|
+
|
|
1553
|
+
// End to end through the REAL run() and the REAL diagnose(), because the claim
|
|
1554
|
+
// that matters most about this whole feature is a NEGATIVE one: the caller's
|
|
1555
|
+
// result is byte-identical whether the probe ran or not. This also re-pins the
|
|
1556
|
+
// #250 shape — `Google API error (401 authError)` still reaching the auth
|
|
1557
|
+
// hint — with the probe in the loop.
|
|
1558
|
+
it('MUST NOT REGRESS: the caller’s diagnosed result is identical with the probe in the loop', async () => {
|
|
1559
|
+
captureLog();
|
|
1560
|
+
const withProbe = routedFetch(
|
|
1561
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1562
|
+
() => probeBody({ ok: true, measured: true, accounts: [] }),
|
|
1563
|
+
);
|
|
1564
|
+
vi.stubGlobal('fetch', withProbe);
|
|
1565
|
+
const probed = await runExecutor.run({ executor: makeFlyExecutor(ENDPOINT, KEY) }, () =>
|
|
1566
|
+
runOrDiagnose(['gmail', 'search', 'q'], {}),
|
|
1567
|
+
);
|
|
1568
|
+
|
|
1569
|
+
// The same failure through an executor whose probe is skipped outright
|
|
1570
|
+
// (gog said invalid_grant is a different error, so use the throttle: a
|
|
1571
|
+
// second call on the same executor never probes).
|
|
1572
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY);
|
|
1573
|
+
const twice = routedFetch(
|
|
1574
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1575
|
+
() => probeBody({ ok: true, measured: true, accounts: [] }),
|
|
1576
|
+
);
|
|
1577
|
+
vi.stubGlobal('fetch', twice);
|
|
1578
|
+
await runExecutor.run({ executor: exec }, () => runOrDiagnose(['gmail', 'search', 'q'], {}));
|
|
1579
|
+
const unprobed = await runExecutor.run({ executor: exec }, () =>
|
|
1580
|
+
runOrDiagnose(['gmail', 'search', 'q'], {}),
|
|
1581
|
+
);
|
|
1582
|
+
|
|
1583
|
+
expect(probed.isError).toBe(true);
|
|
1584
|
+
expect(probed.content[0].text).toContain('Google API error (401 authError)');
|
|
1585
|
+
expect(probed.content[0].text).toContain('gog_auth_add');
|
|
1586
|
+
// The whole point: the probe is invisible to the caller.
|
|
1587
|
+
expect(unprobed.content[0].text).toBe(probed.content[0].text);
|
|
1588
|
+
});
|
|
1589
|
+
|
|
1590
|
+
it('MUST NOT CLAIM: the throttle line never asserts a measurement that did not happen', async () => {
|
|
1591
|
+
// The whole thesis of this branch is that a log line may not claim health it
|
|
1592
|
+
// did not measure. The throttle is the one place that rule can be broken
|
|
1593
|
+
// from the inside: `lastProbeAt` is stamped BEFORE the fetch and is not
|
|
1594
|
+
// reset when the probe comes back with no verdict, so every refusal for the
|
|
1595
|
+
// next PROBE_INTERVAL_MS is explained by a sentence about the previous
|
|
1596
|
+
// probe. If that sentence says the layer "was measured", it is describing a
|
|
1597
|
+
// measurement that never occurred — here, a runner too old to have
|
|
1598
|
+
// /health/google at all.
|
|
1599
|
+
//
|
|
1600
|
+
// Stamping before the await is correct and stays: it is what stops two
|
|
1601
|
+
// overlapping refusals from both spawning a probe, and the backend cost the
|
|
1602
|
+
// throttle protects was paid whether or not a verdict came back. So the
|
|
1603
|
+
// sentence is what has to be true, not the timestamp.
|
|
1604
|
+
const log = captureLog();
|
|
1605
|
+
const fetchMock = routedFetch(
|
|
1606
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1607
|
+
() => probeBody({ error: 'not found' }, 404),
|
|
1608
|
+
);
|
|
1609
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1610
|
+
|
|
1611
|
+
const exec = makeFlyExecutor(ENDPOINT, KEY);
|
|
1612
|
+
await expect(exec(READ, {})).rejects.toThrow(/401/);
|
|
1613
|
+
await expect(exec(READ, {})).rejects.toThrow(/401/);
|
|
1614
|
+
|
|
1615
|
+
// One probe attempted, and it produced no verdict about Google.
|
|
1616
|
+
expect(probeCalls(fetchMock)).toBe(1);
|
|
1617
|
+
const unmeasured = log
|
|
1618
|
+
.records()
|
|
1619
|
+
.filter((r) => r.event === 'refusal.google-unmeasured')
|
|
1620
|
+
.map((r) => r.reason as string);
|
|
1621
|
+
expect(unmeasured).toHaveLength(2);
|
|
1622
|
+
expect(unmeasured[0]).toMatch(/did not answer the Google probe \(HTTP 404\)/);
|
|
1623
|
+
|
|
1624
|
+
// The throttled line: it must say a probe was ATTEMPTED, never that the
|
|
1625
|
+
// Google layer was measured.
|
|
1626
|
+
expect(unmeasured[1]).toMatch(/attempted recently/);
|
|
1627
|
+
expect(unmeasured[1]).not.toMatch(/measured recently|was measured|re-measured/);
|
|
1628
|
+
});
|
|
1629
|
+
|
|
1630
|
+
it('throttles per executor, so one session cannot silence another', async () => {
|
|
1631
|
+
const log = captureLog();
|
|
1632
|
+
const fetchMock = routedFetch(
|
|
1633
|
+
() => gogFailed(GOOGLE_401_STDERR),
|
|
1634
|
+
() => probeBody({ ok: true, measured: true, accounts: [] }),
|
|
1635
|
+
);
|
|
1636
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
1637
|
+
|
|
1638
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
|
|
1639
|
+
await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
|
|
1640
|
+
|
|
1641
|
+
expect(probeCalls(fetchMock)).toBe(2);
|
|
1642
|
+
expect(log.byEvent('refusal.google-unmeasured')).toBeUndefined();
|
|
1643
|
+
});
|
|
1644
|
+
});
|