gogcli-mcp 2.21.0 → 2.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,9 @@
1
1
  import { describe, it, expect, vi, afterEach } from 'vitest';
2
- import { run } from '../src/runner.js';
2
+ import { run, runExecutor } from '../src/runner.js';
3
3
  import type { GogArg, GogExecutor, GogFileArg } from '../src/runner.js';
4
- import { makeFlyExecutor, wrapServer } from '../src/connector-runtime.js';
4
+ import { makeFlyExecutor, wrapServer, RunnerTransportError, isRunnerTransportError } from '../src/connector-runtime.js';
5
+ import { runOrDiagnose } from '../src/tools/utils.js';
6
+ import { makeAccessTokenSource, clearAccessTokenCache } from '../src/google-token.js';
5
7
 
6
8
  afterEach(() => {
7
9
  vi.unstubAllEnvs();
@@ -523,3 +525,1120 @@ describe('makeFlyExecutor', () => {
523
525
  await expect(exec(['x'], {})).rejects.toThrow(/gog failed on the runner/i);
524
526
  });
525
527
  });
528
+
529
+ // A failure the RUNNER authored — its own bearer check, its own request
530
+ // validation, its own drain — never reached gog and never showed a credential
531
+ // to Google. Those failures must be distinguishable by TYPE, because the layer
532
+ // that diagnoses them (tools/utils.ts) can only otherwise guess from prose, and
533
+ // guessing is what turned the runner's bare `unauthorized` body into "your
534
+ // Google sign-in expired, re-authorize".
535
+ describe('makeFlyExecutor runner-authored transport failures', () => {
536
+ const ENDPOINT = 'https://gogcli-gog-runner.fly.dev';
537
+ const KEY = 'k';
538
+
539
+ function stubStatus(status: number, body?: unknown) {
540
+ vi.stubGlobal(
541
+ 'fetch',
542
+ vi.fn(async () => ({
543
+ ok: false,
544
+ status,
545
+ json: async () => {
546
+ if (body === undefined) throw new Error('not json');
547
+ return body;
548
+ },
549
+ })),
550
+ );
551
+ }
552
+
553
+ async function thrownBy(status: number, body?: unknown): Promise<unknown> {
554
+ stubStatus(status, body);
555
+ const exec = makeFlyExecutor(ENDPOINT, KEY);
556
+ return exec(['gmail', 'search', 'q'], {}).catch((e: unknown) => e);
557
+ }
558
+
559
+ // The exact body fly-gog-runner/server.mjs sends when its bearer check fails
560
+ // (server.mjs:450 /health, :460 /run) — nothing to do with Google.
561
+ const RUNNER_401 = { error: 'unauthorized' };
562
+
563
+ it('types the runner bearer rejection as transport auth, not a gog failure', async () => {
564
+ const err = await thrownBy(401, RUNNER_401);
565
+ expect(isRunnerTransportError(err)).toBe(true);
566
+ expect((err as RunnerTransportError).kind).toBe('transport-auth');
567
+ expect((err as RunnerTransportError).status).toBe(401);
568
+ });
569
+
570
+ it('names the real cause — the runner key mismatch — and never the Google account', async () => {
571
+ const err = (await thrownBy(401, RUNNER_401)) as Error;
572
+ expect(err.message).toContain('GOG_RUNNER_KEY');
573
+ expect(err.message).toContain('RUNNER_KEY');
574
+ expect(err.message).not.toMatch(/gog_auth_add/i);
575
+ // The runner's own body is the single word "unauthorized". Repeating it is
576
+ // what fed DEFINITE_AUTH_PATTERN in tools/utils.ts; the status digits can do
577
+ // the same whenever a status word sits near them. Neither may appear, so that
578
+ // even if the TYPE is lost at some future boundary the prose cannot be
579
+ // misread as Google's.
580
+ expect(err.message).not.toMatch(/unauthorized/i);
581
+ expect(err.message).not.toMatch(/\b401\b/);
582
+ });
583
+
584
+ it('types the runner request-validation 400s as transport-request, keeping their detail', async () => {
585
+ for (const detail of [
586
+ 'request body too large',
587
+ 'failed to read request body',
588
+ 'body must be valid JSON',
589
+ 'args must be an array',
590
+ 'accessToken must not contain whitespace or control characters',
591
+ ]) {
592
+ const err = await thrownBy(400, { error: detail });
593
+ expect(isRunnerTransportError(err)).toBe(true);
594
+ expect((err as RunnerTransportError).kind).toBe('transport-request');
595
+ expect((err as RunnerTransportError).status).toBe(400);
596
+ expect((err as Error).message).toBe(detail);
597
+ }
598
+ });
599
+
600
+ it('still says something when a 400 body is unreadable', async () => {
601
+ const err = await thrownBy(400);
602
+ expect(isRunnerTransportError(err)).toBe(true);
603
+ expect((err as RunnerTransportError).kind).toBe('transport-request');
604
+ expect((err as Error).message).toMatch(/gog-runner rejected the request/i);
605
+ });
606
+
607
+ it('types the drain 503 as retryable transport, message unchanged', async () => {
608
+ const err = await thrownBy(503, { error: 'gog-runner is shutting down', retryable: true });
609
+ expect(isRunnerTransportError(err)).toBe(true);
610
+ expect((err as RunnerTransportError).kind).toBe('transport-retryable');
611
+ expect((err as RunnerTransportError).status).toBe(503);
612
+ expect((err as Error).message).toBe('gog-runner is restarting; retry this call. gog-runner is shutting down');
613
+ });
614
+
615
+ it('types a retryable-flagged 500 (a materialization failure) as retryable transport', async () => {
616
+ const err = await thrownBy(500, {
617
+ error: 'failed to write a file arg to disk: ENOSPC: no space left on device',
618
+ retryable: true,
619
+ });
620
+ expect(isRunnerTransportError(err)).toBe(true);
621
+ expect((err as RunnerTransportError).kind).toBe('transport-retryable');
622
+ expect((err as RunnerTransportError).status).toBe(500);
623
+ });
624
+
625
+ it('types a bodiless gateway failure as retryable transport', async () => {
626
+ const err = await thrownBy(502);
627
+ expect(isRunnerTransportError(err)).toBe(true);
628
+ expect((err as RunnerTransportError).kind).toBe('transport-retryable');
629
+ expect((err as RunnerTransportError).status).toBe(502);
630
+ });
631
+
632
+ it('types the client-side deadline as retryable transport with no status', async () => {
633
+ vi.stubGlobal(
634
+ 'fetch',
635
+ vi.fn(async () => {
636
+ throw Object.assign(new Error('The operation was aborted'), { name: 'TimeoutError' });
637
+ }),
638
+ );
639
+ const exec = makeFlyExecutor(ENDPOINT, KEY);
640
+ const err = await exec(['x'], {}).catch((e: unknown) => e);
641
+ expect(isRunnerTransportError(err)).toBe(true);
642
+ expect((err as RunnerTransportError).kind).toBe('transport-retryable');
643
+ expect((err as RunnerTransportError).status).toBeUndefined();
644
+ });
645
+
646
+ // The other half of the contract: an error that carries gog's/Google's OWN
647
+ // words must stay untyped, so the prose classifier still gets to read it.
648
+ it('leaves a 422 (gog ran and failed) untyped, for the prose classifier', async () => {
649
+ const err = await thrownBy(422, { error: 'gog failed', stderr: 'Error 401: invalid_grant' });
650
+ expect(err).toBeInstanceOf(Error);
651
+ expect(isRunnerTransportError(err)).toBe(false);
652
+ });
653
+
654
+ it('leaves a detail-bearing non-2xx untyped', async () => {
655
+ const err = await thrownBy(502, { error: 'gog exited with code 1', stderr: 'bad flag' });
656
+ expect(isRunnerTransportError(err)).toBe(false);
657
+ });
658
+
659
+ it('recognises only branded errors', () => {
660
+ expect(isRunnerTransportError(new Error('unauthorized'))).toBe(false);
661
+ expect(isRunnerTransportError('unauthorized')).toBe(false);
662
+ expect(isRunnerTransportError(new RunnerTransportError('x', 'transport-auth', 401))).toBe(true);
663
+ });
664
+
665
+ // The whole chain, end to end, with the REAL executor, the REAL run() and the
666
+ // REAL diagnose(): the incident was a user told all session to re-authorize a
667
+ // Google account that was never asked for a credential.
668
+ it('does not tell the caller to re-authorize Google when the RUNNER rejected our bearer', async () => {
669
+ vi.stubEnv('GOG_ACCOUNT', '');
670
+ vi.stubEnv('GOG_READONLY', '');
671
+ stubStatus(401, RUNNER_401);
672
+ const executor = makeFlyExecutor(ENDPOINT, KEY);
673
+ const result = await runExecutor.run({ executor }, () =>
674
+ runOrDiagnose(['sheets', 'get', 'A1'], {}),
675
+ );
676
+ const text = result.content[0].text as string;
677
+ expect(result.isError).toBe(true);
678
+ expect(text).not.toMatch(/gog_auth_add/);
679
+ expect(text).not.toMatch(/re-authorize the account/i);
680
+ expect(text).toContain('GOG_RUNNER_KEY');
681
+ });
682
+ });
683
+
684
+ // A Google 401 on the ACCESS token and a dead REFRESH token are opposite
685
+ // failures that used to be told to the user identically.
686
+ //
687
+ // * The access token is ours to replace. Google rejecting it means "mint
688
+ // another and try again" — automatic, no human, no re-authorization. Before
689
+ // this, the rejected token stayed cached for the rest of its nominal hour
690
+ // and NOTHING retried, so every call in that window failed the same way and
691
+ // only reconnecting (a fresh isolate, an empty cache) appeared to help.
692
+ // * invalid_grant means the refresh token is gone. No re-mint is possible and
693
+ // a human must re-authorize. Retrying that is a loop against a credential
694
+ // that can never work.
695
+ //
696
+ // So the re-mint is gated hard: gog must actually have run (422), the token must
697
+ // be one WE supplied, gog's own stderr must say Google rejected it, the call
698
+ // must be safe to replay, and our cache must still hold exactly that token.
699
+ describe('makeFlyExecutor re-mints a rejected access token and replays once', () => {
700
+ const ENDPOINT = 'https://gogcli-gog-runner.fly.dev';
701
+ const KEY = 'k';
702
+
703
+ // gog v0.34.1's real words when the access token it was handed is rejected.
704
+ const GOOGLE_401_STDERR =
705
+ 'Note: Using direct access token (expires in ~1 hour; no auto-refresh)\n' +
706
+ 'Google API error (401 authError): Request had invalid authentication credentials. ' +
707
+ 'Expected OAuth 2 access token, login cookie or other valid authentication credential.';
708
+
709
+ // What the executor really receives: run()'s assembleArgs puts the global
710
+ // flags in front of the service and subcommand.
711
+ const READ = ['--json', '--color=never', '--no-input', 'gmail', 'search', 'q'];
712
+ const WRITE = ['--json', '--color=never', '--no-input', 'gmail', 'send', '--to', 'a@b.c'];
713
+
714
+ function gogFailed(stderr: string) {
715
+ return {
716
+ ok: false,
717
+ status: 422,
718
+ // `error` is Node's execFile message, which embeds the whole command line
719
+ // AND stderr — see the command-line test below for why that matters.
720
+ json: async () => ({
721
+ error: `Command failed: gog gmail search q\n${stderr}`,
722
+ stderr,
723
+ retryable: false,
724
+ }),
725
+ };
726
+ }
727
+ function ok(stdout: string) {
728
+ return { ok: true, json: async () => ({ stdout }) };
729
+ }
730
+ function bodyOf(fetchMock: { mock: { calls: unknown[][] } }, i: number): Record<string, unknown> {
731
+ const [, init] = fetchMock.mock.calls[i] as [string, RequestInit];
732
+ return JSON.parse(init.body as string) as Record<string, unknown>;
733
+ }
734
+ /** A token source that hands out `tokens` in order and can be invalidated. */
735
+ function source(tokens: (string | undefined)[], evicted = true) {
736
+ const fn = vi.fn(async () => tokens.shift());
737
+ return Object.assign(fn, { invalidate: vi.fn(async () => evicted) });
738
+ }
739
+
740
+ it('mints a new token and replays the call, invisibly to the caller', async () => {
741
+ const fetchMock = vi
742
+ .fn()
743
+ .mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
744
+ .mockResolvedValueOnce(ok('thread json'));
745
+ vi.stubGlobal('fetch', fetchMock);
746
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
747
+
748
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
749
+ await expect(exec(READ, {})).resolves.toBe('thread json');
750
+
751
+ expect(fetchMock).toHaveBeenCalledTimes(2);
752
+ expect(bodyOf(fetchMock, 0).accessToken).toBe('ya29.stale');
753
+ expect(bodyOf(fetchMock, 1).accessToken).toBe('ya29.fresh');
754
+ // Evicted by VALUE — only the token that was actually rejected.
755
+ expect(readToken.invalidate).toHaveBeenCalledTimes(1);
756
+ expect(readToken.invalidate).toHaveBeenCalledWith('ya29.stale');
757
+ });
758
+
759
+ it('replays exactly once, so a genuinely dead credential cannot loop', async () => {
760
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
761
+ vi.stubGlobal('fetch', fetchMock);
762
+ const readToken = source(['ya29.stale', 'ya29.fresh', 'ya29.third']);
763
+
764
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
765
+ await expect(exec(READ, {})).rejects.toThrow(/Google API error \(401/);
766
+
767
+ expect(fetchMock).toHaveBeenCalledTimes(2);
768
+ // Two evictions, one per token Google refused — the replay's included. The
769
+ // ATTEMPT count is what "exactly once" is about, and it is still two.
770
+ expect(readToken.invalidate).toHaveBeenCalledTimes(2);
771
+ });
772
+
773
+ it('does not replay when gog reported invalid_grant — only a human can fix that', async () => {
774
+ // The stderr deliberately matches BOTH shapes, so this pins the exclusion
775
+ // rather than the absence of a 401.
776
+ const fetchMock = vi.fn(async () => gogFailed(`${GOOGLE_401_STDERR}\noauth2: "invalid_grant"`));
777
+ vi.stubGlobal('fetch', fetchMock);
778
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
779
+
780
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
781
+ await expect(exec(READ, {})).rejects.toThrow(/invalid_grant/);
782
+ expect(fetchMock).toHaveBeenCalledTimes(1);
783
+ expect(readToken.invalidate).not.toHaveBeenCalled();
784
+ });
785
+
786
+ it('does not replay a call that supplied no token of ours', async () => {
787
+ // Nothing was minted, so there is nothing to re-mint: gog used whatever
788
+ // identity the backend volume holds, and only an operator can change that.
789
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
790
+ vi.stubGlobal('fetch', fetchMock);
791
+ const exec = makeFlyExecutor(ENDPOINT, KEY);
792
+ await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
793
+ // Counted by ENDPOINT, not in total: this same refusal now also takes a
794
+ // read of the Google layer (`/health/google`, see the refusal-probe block
795
+ // below), and that reading is a diagnostic, not a second attempt. What
796
+ // "does not replay" means is that gog ran exactly once.
797
+ const runs = fetchMock.mock.calls.filter(([url]) => String(url).endsWith('/run'));
798
+ expect(runs).toHaveLength(1);
799
+ });
800
+
801
+ it('does not replay when the token source cannot invalidate', async () => {
802
+ // A bare `() => token` (the #230 direct-token wiring) has no cache behind
803
+ // it, so replaying would re-send the identical rejected token.
804
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
805
+ vi.stubGlobal('fetch', fetchMock);
806
+ const exec = makeFlyExecutor(ENDPOINT, KEY, () => 'ya29.direct');
807
+ await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
808
+ expect(fetchMock).toHaveBeenCalledTimes(1);
809
+ });
810
+
811
+ it('does not replay when the cache no longer held the rejected token', async () => {
812
+ // Another caller already refreshed it; the token we would send is the one
813
+ // that is already in use, so a replay proves nothing.
814
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
815
+ vi.stubGlobal('fetch', fetchMock);
816
+ const readToken = source(['ya29.stale', 'ya29.fresh'], false);
817
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
818
+ await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
819
+ expect(fetchMock).toHaveBeenCalledTimes(1);
820
+ expect(readToken.invalidate).toHaveBeenCalledTimes(1);
821
+ });
822
+
823
+ // A subcommand is only a leaf VERB some of the time. `gog tasks lists` is a
824
+ // NAMESPACE — `lists list` reads, `lists create` writes — and the allow-list
825
+ // is consulted with the namespace word, never the verb under it. So a
826
+ // namespace word in the set hands the replay to every child it will ever
827
+ // grow, including the ones that write.
828
+ //
829
+ // `tasks lists create <title> ...` is real in gog v0.34.1 today, and reachable
830
+ // without any new gog: `gog_tasks_run({subcommand: 'lists', args: ['create',
831
+ // 'A', 'B']})` assembles exactly this argv. It is variadic, so one invocation
832
+ // makes N Google calls and a 401 on the second means the first already landed
833
+ // — the precise double-apply the write rule exists to prevent.
834
+ it('does not replay `tasks lists create`, a WRITE under a namespace-shaped word', async () => {
835
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
836
+ vi.stubGlobal('fetch', fetchMock);
837
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
838
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
839
+ await expect(
840
+ exec(['--json', '--color=never', '--no-input', 'tasks', 'lists', 'create', 'A', 'B'], {}),
841
+ ).rejects.toThrow(/Google API error/);
842
+ expect(fetchMock).toHaveBeenCalledTimes(1);
843
+ // The eviction still happens — it runs before the allow-list is consulted.
844
+ expect(readToken.invalidate).toHaveBeenCalledTimes(1);
845
+ });
846
+
847
+ // The invariant the allow-list's comment states, pinned as behaviour so the
848
+ // comment is no longer the only thing enforcing it. Every word here is a gog
849
+ // NAMESPACE that already has, or can grow, a mutating child; none of them may
850
+ // ever earn a replay, whichever verb follows.
851
+ it.each([
852
+ ['tasks', 'lists'],
853
+ ['gmail', 'labels'],
854
+ ['gmail', 'drafts'],
855
+ ['gmail', 'filters'],
856
+ ['gmail', 'sendas'],
857
+ ['drive', 'permissions'],
858
+ ['drive', 'revisions'],
859
+ ['docs', 'comments'],
860
+ ['docs', 'replies'],
861
+ ])('never replays the namespace word %s %s, whatever verb follows it', async (service, namespace) => {
862
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
863
+ vi.stubGlobal('fetch', fetchMock);
864
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
865
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
866
+ await expect(
867
+ exec(['--json', '--color=never', '--no-input', service, namespace, 'create', 'x'], {}),
868
+ ).rejects.toThrow(/Google API error/);
869
+ expect(fetchMock).toHaveBeenCalledTimes(1);
870
+ // The eviction is unaffected: it runs before the allow-list is consulted.
871
+ expect(readToken.invalidate).toHaveBeenCalledTimes(1);
872
+ });
873
+
874
+ // Symmetry with the rule the whole fix is built on: a token Google has
875
+ // refused must not stay cached, and the very same 401 refused the replay's
876
+ // token. This is NOT a round-trip saving — measured through the real chain
877
+ // under a sustained non-invalid_grant refusal, a steady-state call costs two
878
+ // /run round-trips either way, and this eviction adds a mint (2 rather than
879
+ // 1) by emptying a cache the next call would have hit. What it buys is a
880
+ // bound on how long a KNOWN-REFUSED token can be served: left cached it is
881
+ // re-served for the rest of its nominal hour, and a write — which gets the
882
+ // eviction and no replay — would be sent with it and fail on contact.
883
+ it('evicts the replayed token too when Google refuses that one as well', async () => {
884
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
885
+ vi.stubGlobal('fetch', fetchMock);
886
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
887
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
888
+ await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
889
+ expect(readToken.invalidate.mock.calls).toEqual([['ya29.stale'], ['ya29.fresh']]);
890
+ });
891
+
892
+ // The other half of that rule, and the one that keeps this branch honest.
893
+ //
894
+ // A replay can fail without Google ever seeing the token: the Machine starts
895
+ // draining between the two attempts, the client-side deadline fires, the
896
+ // runner's own bearer is rotated mid-call. Evicting on THOSE would throw away
897
+ // a token nothing has refused and — worse — emit `token.evicted` with
898
+ // "Google rejected this access token", which is a lie about a service that
899
+ // was never consulted. Misattributing a runner-side failure to Google is the
900
+ // exact defect this branch exists to remove; re-introducing it in the log
901
+ // would just move it from the user's screen to the operator's query.
902
+ it('does not evict the replayed token when the replay never reached Google', async () => {
903
+ const fetchMock = vi
904
+ .fn()
905
+ .mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
906
+ // The runner drains between the two attempts — a transport failure, not
907
+ // a verdict on the freshly minted token.
908
+ .mockResolvedValueOnce({
909
+ ok: false,
910
+ status: 503,
911
+ json: async () => ({ error: 'gog-runner is shutting down', retryable: true }),
912
+ });
913
+ vi.stubGlobal('fetch', fetchMock);
914
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
915
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
916
+ const err = await exec(READ, {}).catch((e: unknown) => e);
917
+ // The replay's own error reaches the caller, and here that is the RIGHT
918
+ // one to surface: "the runner is restarting, retry" is actionable, where
919
+ // re-raising the superseded Google 401 would send the user off to
920
+ // re-authorize an account that is fine.
921
+ expect(isRunnerTransportError(err)).toBe(true);
922
+ expect((err as RunnerTransportError).kind).toBe('transport-retryable');
923
+ // Only the token Google actually refused was dropped. `ya29.fresh` stays
924
+ // cached: it is unproven, not refused, and the next call may well succeed
925
+ // with it once the new Machine is up.
926
+ expect(readToken.invalidate.mock.calls).toEqual([['ya29.stale']]);
927
+ });
928
+
929
+ // The mint is a first-class error surface on this branch, so the error it
930
+ // raises has to reach the caller with the SAME specificity a gog-authored
931
+ // invalid_grant gets: the 7-day Testing-mode cause and the headless re-auth
932
+ // pair. It only does if the message carries the literal `invalid_grant`,
933
+ // which is what tools/utils.ts keys the richer hint on.
934
+ it('gives a mint-path invalid_grant the full re-auth guidance, not the generic hint', async () => {
935
+ vi.stubEnv('GOG_ACCOUNT', '');
936
+ vi.stubEnv('GOG_READONLY', '');
937
+ clearAccessTokenCache();
938
+ const fetchMock = vi.fn(async (url: unknown) => {
939
+ if (String(url).includes('oauth2.googleapis.com')) {
940
+ return new Response(JSON.stringify({ error: 'invalid_grant' }), { status: 400 });
941
+ }
942
+ return gogFailed('unreachable');
943
+ });
944
+ vi.stubGlobal('fetch', fetchMock);
945
+ const executor = makeFlyExecutor(
946
+ ENDPOINT,
947
+ KEY,
948
+ makeAccessTokenSource({
949
+ GOG_CLIENT_ID: 'cid',
950
+ GOG_CLIENT_SECRET: 'cs',
951
+ GOG_REFRESH_TOKEN: 'rt-dead',
952
+ }),
953
+ );
954
+
955
+ const result = await runExecutor.run({ executor }, () =>
956
+ runOrDiagnose(['gmail', 'search', 'q'], {}),
957
+ );
958
+ expect(result.isError).toBe(true);
959
+ const text = result.content[0].text as string;
960
+ // Text unique to INVALID_GRANT_HINT — the durable fix. The generic
961
+ // AUTH_HINT (which is what a message omitting `invalid_grant` earns) says
962
+ // only "Authentication may have expired".
963
+ expect(text).toContain('publish the OAuth consent screen to "In production"');
964
+ expect(text).not.toContain('Authentication may have expired');
965
+ clearAccessTokenCache();
966
+ });
967
+
968
+ it('does not replay a WRITE, but still evicts the token Google rejected', async () => {
969
+ // The one case an automatic replay must never take. A `gog gmail send` that
970
+ // failed AFTER the message went out would send it twice.
971
+ //
972
+ // The EVICTION is a different question, and the write rule must not answer
973
+ // it. Refusing to evict is what leaves Google's rejected token in the cache
974
+ // for the rest of its nominal hour, so every following call — write or read
975
+ // — re-sends it and fails identically until the isolate is replaced. That
976
+ // is DEFECT 2 itself, and it is the half a write path used to keep.
977
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
978
+ vi.stubGlobal('fetch', fetchMock);
979
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
980
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
981
+ await expect(exec(WRITE, {})).rejects.toThrow(/Google API error/);
982
+ expect(fetchMock).toHaveBeenCalledTimes(1);
983
+ expect(readToken.invalidate).toHaveBeenCalledTimes(1);
984
+ expect(readToken.invalidate).toHaveBeenCalledWith('ya29.stale');
985
+ });
986
+
987
+ it('evicts for a READ whose subcommand is outside the allow-list', async () => {
988
+ // `gog gmail labels list` arrives here as the subcommand `labels`, which is
989
+ // deliberately not in READ_ONLY_SUBCOMMANDS (a later `labels create` would
990
+ // inherit the replay). Costing that call its replay is the intended price;
991
+ // costing it the eviction would poison every call after it.
992
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
993
+ vi.stubGlobal('fetch', fetchMock);
994
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
995
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
996
+ await expect(
997
+ exec(['--json', '--color=never', '--no-input', 'gmail', 'labels', 'list'], {}),
998
+ ).rejects.toThrow(/Google API error/);
999
+ expect(fetchMock).toHaveBeenCalledTimes(1);
1000
+ expect(readToken.invalidate).toHaveBeenCalledTimes(1);
1001
+ });
1002
+
1003
+ it('lets a second WRITE mint a fresh token, through the REAL token source', async () => {
1004
+ // The incident, reproduced end to end with nothing stubbed but the network:
1005
+ // two consecutive `gog gmail send` calls after Google has rejected the
1006
+ // cached token. Before the eviction was moved ahead of the write rule, BOTH
1007
+ // shipped `ya29.t1` and both failed, for up to ~58 minutes, and only a
1008
+ // reconnect helped.
1009
+ clearAccessTokenCache();
1010
+ let minted = 0;
1011
+ const fetchMock = vi.fn(async (url: unknown) => {
1012
+ if (String(url).includes('oauth2.googleapis.com')) {
1013
+ return new Response(
1014
+ JSON.stringify({ access_token: `ya29.t${(minted += 1)}`, expires_in: 3600 }),
1015
+ { status: 200, headers: { 'content-type': 'application/json' } },
1016
+ );
1017
+ }
1018
+ return gogFailed(GOOGLE_401_STDERR);
1019
+ });
1020
+ vi.stubGlobal('fetch', fetchMock);
1021
+
1022
+ const exec = makeFlyExecutor(
1023
+ ENDPOINT,
1024
+ KEY,
1025
+ makeAccessTokenSource({
1026
+ GOG_CLIENT_ID: 'cid',
1027
+ GOG_CLIENT_SECRET: 'cs',
1028
+ GOG_REFRESH_TOKEN: 'rt-1',
1029
+ }),
1030
+ );
1031
+ await expect(exec(WRITE, {})).rejects.toThrow(/Google API error/);
1032
+ await expect(exec(WRITE, {})).rejects.toThrow(/Google API error/);
1033
+
1034
+ const runBodies = fetchMock.mock.calls
1035
+ .filter(([url]) => !String(url).includes('oauth2.googleapis.com'))
1036
+ .map(([, init]) => JSON.parse((init as RequestInit).body as string) as { accessToken: string });
1037
+ expect(runBodies.map((b) => b.accessToken)).toEqual(['ya29.t1', 'ya29.t2']);
1038
+ clearAccessTokenCache();
1039
+ });
1040
+
1041
+ it('replays with what is LEFT of the deadline, not a second full one', async () => {
1042
+ // 30s default + 5s grace is one tool call's whole budget. Handing the
1043
+ // replay a fresh copy of it makes the worst case ~70s of wall clock, which
1044
+ // can outlast the MCP client's own request timeout and turn a self-healing
1045
+ // read into a client-side hang.
1046
+ const budgets: number[] = [];
1047
+ const realTimeout = AbortSignal.timeout.bind(AbortSignal);
1048
+ vi.spyOn(AbortSignal, 'timeout').mockImplementation((ms: number) => {
1049
+ budgets.push(ms);
1050
+ return realTimeout(ms);
1051
+ });
1052
+ const fetchMock = vi
1053
+ .fn()
1054
+ .mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
1055
+ .mockResolvedValueOnce(ok('threads'));
1056
+ vi.stubGlobal('fetch', fetchMock);
1057
+ const base = Date.now();
1058
+ // The clock is read once to fix the deadline, then once more to size the
1059
+ // replay; 5s of the budget is gone by then.
1060
+ vi.spyOn(Date, 'now').mockReturnValueOnce(base).mockReturnValue(base + 5_000);
1061
+
1062
+ const exec = makeFlyExecutor(ENDPOINT, KEY, source(['ya29.stale', 'ya29.fresh']));
1063
+ await expect(exec(READ, {})).resolves.toBe('threads');
1064
+ expect(budgets).toEqual([35_000, 30_000]);
1065
+ });
1066
+
1067
+ it('skips the replay when the first attempt used the whole deadline', async () => {
1068
+ // With no budget left, a replay can only end in an abort, and that
1069
+ // TimeoutError would REPLACE gog's own 401 — trading an actionable error
1070
+ // for an opaque one. The eviction is the durable half of the repair and it
1071
+ // has already happened, so the caller's own next call is the fresh one.
1072
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
1073
+ vi.stubGlobal('fetch', fetchMock);
1074
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
1075
+ const base = Date.now();
1076
+ vi.spyOn(Date, 'now').mockReturnValueOnce(base).mockReturnValue(base + 34_900);
1077
+
1078
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1079
+ await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
1080
+ expect(fetchMock).toHaveBeenCalledTimes(1);
1081
+ expect(readToken.invalidate).toHaveBeenCalledTimes(1);
1082
+ });
1083
+
1084
+ it('finds the subcommand past --account, whose value is not a subcommand', async () => {
1085
+ const fetchMock = vi
1086
+ .fn()
1087
+ .mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
1088
+ .mockResolvedValueOnce(ok('[]'));
1089
+ vi.stubGlobal('fetch', fetchMock);
1090
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
1091
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1092
+ await expect(
1093
+ exec(['--json', '--color=never', '--no-input', '--account', 'me@example.com', 'drive', 'ls'], {}),
1094
+ ).resolves.toBe('[]');
1095
+ expect(fetchMock).toHaveBeenCalledTimes(2);
1096
+ });
1097
+
1098
+ it('does not replay an invocation with no subcommand to judge', async () => {
1099
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
1100
+ vi.stubGlobal('fetch', fetchMock);
1101
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
1102
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1103
+ await expect(exec(['--json', '--color=never', 'gmail'], {})).rejects.toThrow(/Google API error/);
1104
+ expect(fetchMock).toHaveBeenCalledTimes(1);
1105
+ });
1106
+
1107
+ it('reads gog stderr, not the command line the runner echoed back', async () => {
1108
+ // execFile's message embeds the whole argv, so a caller's own text lands in
1109
+ // `error`. Classifying on that would let `--subject "invoice 401"` trigger a
1110
+ // replay of a call that failed for an unrelated reason — and if that call
1111
+ // were a write, replay it after it had partly applied.
1112
+ const fetchMock = vi.fn(async () => ({
1113
+ ok: false,
1114
+ status: 422,
1115
+ json: async () => ({
1116
+ error: 'Command failed: gog gmail search "Google API error (401 authError)"\nno results',
1117
+ stderr: 'no results',
1118
+ retryable: false,
1119
+ }),
1120
+ }));
1121
+ vi.stubGlobal('fetch', fetchMock);
1122
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
1123
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1124
+ await expect(exec(READ, {})).rejects.toThrow(/no results/);
1125
+ expect(fetchMock).toHaveBeenCalledTimes(1);
1126
+ expect(readToken.invalidate).not.toHaveBeenCalled();
1127
+ });
1128
+
1129
+ it('does not replay a gog failure that has nothing to do with auth', async () => {
1130
+ const fetchMock = vi.fn(async () => gogFailed('invalid attachment id'));
1131
+ vi.stubGlobal('fetch', fetchMock);
1132
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
1133
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1134
+ await expect(exec(READ, {})).rejects.toThrow(/invalid attachment id/);
1135
+ expect(fetchMock).toHaveBeenCalledTimes(1);
1136
+ });
1137
+
1138
+ it('surfaces the original failure when the re-mint yields no token', async () => {
1139
+ // Sending the request without a token would run it as the BACKEND's
1140
+ // identity and hand this caller someone else's mailbox — never that.
1141
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
1142
+ vi.stubGlobal('fetch', fetchMock);
1143
+ const readToken = source(['ya29.stale', undefined]);
1144
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1145
+ await expect(exec(READ, {})).rejects.toThrow(/Google API error/);
1146
+ expect(fetchMock).toHaveBeenCalledTimes(1);
1147
+ });
1148
+
1149
+ it('surfaces the re-mint failure, which is the more actionable one', async () => {
1150
+ // Evicting revealed that the refresh token is dead too. That message names
1151
+ // the real repair (re-authorize); gog's 401 does not.
1152
+ const fetchMock = vi.fn(async () => gogFailed(GOOGLE_401_STDERR));
1153
+ vi.stubGlobal('fetch', fetchMock);
1154
+ const readToken = Object.assign(
1155
+ vi
1156
+ .fn<() => Promise<string | undefined>>()
1157
+ .mockResolvedValueOnce('ya29.stale')
1158
+ .mockRejectedValueOnce(new Error('the stored refresh token has expired or been revoked')),
1159
+ { invalidate: vi.fn(async () => true) },
1160
+ );
1161
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1162
+ await expect(exec(READ, {})).rejects.toThrow(/refresh token has expired or been revoked/);
1163
+ expect(fetchMock).toHaveBeenCalledTimes(1);
1164
+ });
1165
+
1166
+ it('never re-mints for the RUNNER\'s own 401, which never reached Google', async () => {
1167
+ // The defect-1 failure. Its body is the bare word "unauthorized" and no
1168
+ // Google credential was even read, so minting a new one is pure waste — and
1169
+ // replaying would double every call during a key mismatch.
1170
+ const fetchMock = vi.fn(async () => ({
1171
+ ok: false,
1172
+ status: 401,
1173
+ json: async () => ({ error: 'unauthorized' }),
1174
+ }));
1175
+ vi.stubGlobal('fetch', fetchMock);
1176
+ const readToken = source(['ya29.stale', 'ya29.fresh']);
1177
+ const exec = makeFlyExecutor(ENDPOINT, KEY, readToken);
1178
+ const err = await exec(READ, {}).catch((e: unknown) => e);
1179
+ expect(isRunnerTransportError(err)).toBe(true);
1180
+ expect(fetchMock).toHaveBeenCalledTimes(1);
1181
+ expect(readToken.invalidate).not.toHaveBeenCalled();
1182
+ });
1183
+
1184
+ // End to end through the REAL run() and the REAL diagnose(): the user-visible
1185
+ // property is that a stale access token produces no error and no advice at
1186
+ // all, because it healed itself.
1187
+ it('turns a stale-token failure into a plain success, with no re-auth advice', async () => {
1188
+ vi.stubEnv('GOG_ACCOUNT', '');
1189
+ vi.stubEnv('GOG_READONLY', '');
1190
+ const fetchMock = vi
1191
+ .fn()
1192
+ .mockResolvedValueOnce(gogFailed(GOOGLE_401_STDERR))
1193
+ .mockResolvedValueOnce(ok('{"threads":[]}'));
1194
+ vi.stubGlobal('fetch', fetchMock);
1195
+ const executor = makeFlyExecutor(ENDPOINT, KEY, source(['ya29.stale', 'ya29.fresh']));
1196
+
1197
+ const result = await runExecutor.run({ executor }, () =>
1198
+ runOrDiagnose(['gmail', 'search', 'q'], {}),
1199
+ );
1200
+ expect(result.isError).toBeFalsy();
1201
+ expect(result.content[0].text).toBe('{"threads":[]}');
1202
+ });
1203
+ });
1204
+
1205
+ /**
1206
+ * DEFECT 3, the half of it that survived grounding.
1207
+ *
1208
+ * `worker.ts` builds `makeFlyExecutor(FLY_ENDPOINT, key)` with NO token source,
1209
+ * so the eviction + replay machinery is inert on the hosted path: `gog` runs as
1210
+ * the Fly volume's own identity, and a Google 401 stops at the "no access token
1211
+ * was supplied" guard. That much is by design and stays.
1212
+ *
1213
+ * What did NOT survive is the idea of building a replay for it. `gog` is spawned
1214
+ * fresh per `/run` and re-reads the keyring every time, so there is no
1215
+ * cross-spawn in-memory token that could go stale — a Google 401 here means the
1216
+ * stored credential itself was refused, and no retry can fix that. Building a
1217
+ * retry would have been the fifth plausible theory in a row.
1218
+ *
1219
+ * So this path gets INSTRUMENTATION instead. The one thing nobody could answer
1220
+ * after the incident was: at the moment Google refused that call, was the
1221
+ * refresh token on the volume alive or dead? `replay.declined` records only that
1222
+ * WE did nothing. These tests pin a record of what GOOGLE said, measured at the
1223
+ * moment of the refusal with the probe `/health/google` — and pin that the
1224
+ * measurement changes nothing the caller sees.
1225
+ */
1226
+ describe('the Google-layer measurement taken when a hosted call is refused', () => {
1227
+ const ENDPOINT = 'https://gogcli-gog-runner.fly.dev';
1228
+ const KEY = 'k';
1229
+ const PREFIX = 'gog-auth ';
1230
+
1231
+ const GOOGLE_401_STDERR =
1232
+ 'Google API error (401 authError): Request had invalid authentication credentials.';
1233
+ const READ = ['--json', '--color=never', '--no-input', 'gmail', 'search', 'q'];
1234
+
1235
+ function gogFailed(stderr: string) {
1236
+ return {
1237
+ ok: false,
1238
+ status: 422,
1239
+ json: async () => ({ error: `Command failed: gog gmail search q\n${stderr}`, stderr }),
1240
+ };
1241
+ }
1242
+ const probeBody = (body: unknown, status = 200) => ({
1243
+ ok: status >= 200 && status < 300,
1244
+ status,
1245
+ json: async () => body,
1246
+ });
1247
+
1248
+ /** A `fetch` that answers `/run` and `/health/google` separately. */
1249
+ function routedFetch(run: () => unknown, probe: () => unknown) {
1250
+ return vi.fn(async (url: string) => {
1251
+ if (url.endsWith('/run')) return run();
1252
+ if (url.endsWith('/health/google')) return probe();
1253
+ throw new Error(`unexpected fetch: ${url}`);
1254
+ });
1255
+ }
1256
+ const urls = (f: { mock: { calls: unknown[][] } }) => f.mock.calls.map((c) => c[0] as string);
1257
+ const runCalls = (f: { mock: { calls: unknown[][] } }) =>
1258
+ urls(f).filter((u) => u.endsWith('/run')).length;
1259
+ const probeCalls = (f: { mock: { calls: unknown[][] } }) =>
1260
+ urls(f).filter((u) => u.endsWith('/health/google')).length;
1261
+
1262
+ function captureLog() {
1263
+ const emitted: Array<{ method: 'warn' | 'error'; line: string }> = [];
1264
+ const toStdout: string[] = [];
1265
+ for (const method of ['warn', 'error'] as const) {
1266
+ vi.spyOn(console, method).mockImplementation((...args: unknown[]) => {
1267
+ emitted.push({ method, line: args.map(String).join(' ') });
1268
+ });
1269
+ }
1270
+ for (const method of ['log', 'info', 'debug', 'trace'] as const) {
1271
+ vi.spyOn(console, method).mockImplementation((...args: unknown[]) => {
1272
+ toStdout.push(args.map(String).join(' '));
1273
+ });
1274
+ }
1275
+ return {
1276
+ emitted,
1277
+ toStdout,
1278
+ records(): Record<string, unknown>[] {
1279
+ return emitted.map((e) => {
1280
+ expect(e.line.startsWith(PREFIX)).toBe(true);
1281
+ return JSON.parse(e.line.slice(PREFIX.length)) as Record<string, unknown>;
1282
+ });
1283
+ },
1284
+ byEvent(event: string): Record<string, unknown> | undefined {
1285
+ return this.records().find((r) => r.event === event);
1286
+ },
1287
+ };
1288
+ }
1289
+
1290
+ it('asks the runner whether Google still accepts the credential, with the same bearer', async () => {
1291
+ const log = captureLog();
1292
+ const fetchMock = routedFetch(
1293
+ () => gogFailed(GOOGLE_401_STDERR),
1294
+ () => probeBody({ ok: true, measured: true, accounts: [{ email: 'a@b.c', valid: true }] }),
1295
+ );
1296
+ vi.stubGlobal('fetch', fetchMock);
1297
+
1298
+ const exec = makeFlyExecutor(ENDPOINT, KEY);
1299
+ // The caller's error is untouched — this is instrumentation, not recovery.
1300
+ await expect(exec(READ, {})).rejects.toThrow(/Google API error \(401/);
1301
+
1302
+ expect(runCalls(fetchMock)).toBe(1);
1303
+ expect(probeCalls(fetchMock)).toBe(1);
1304
+ const [, init] = fetchMock.mock.calls[1] as [string, RequestInit];
1305
+ expect(init.headers).toEqual({ Authorization: `Bearer ${KEY}` });
1306
+ expect(init.signal).toBeInstanceOf(AbortSignal);
1307
+ });
1308
+
1309
+ it('records an UNEXPLAINED refusal when Google says the credential is fine', async () => {
1310
+ // The narrow theory the plan refused to build a fix for: a stored token
1311
+ // refused by Google while the grant behind it is alive. This record is the
1312
+ // only thing that can ever prove or kill it, so it is emitted at error
1313
+ // level — "we cannot explain this" is the loudest thing a log can say.
1314
+ const log = captureLog();
1315
+ vi.stubGlobal('fetch', routedFetch(
1316
+ () => gogFailed(GOOGLE_401_STDERR),
1317
+ () => probeBody({ ok: true, measured: true, accounts: [{ email: 'a@b.c', valid: true }] }),
1318
+ ));
1319
+
1320
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1321
+
1322
+ const record = log.byEvent('refusal.google-ok');
1323
+ expect(record).toBeDefined();
1324
+ expect(record!.service).toBe('gmail');
1325
+ expect(record!.endpoint).toBe(ENDPOINT);
1326
+ expect(log.emitted.find((e) => e.line.includes('refusal.google-ok'))!.method).toBe('error');
1327
+ // Still followed by the decision record, so the pair reads: what Google
1328
+ // said, then what we did about it.
1329
+ expect(log.byEvent('replay.declined')).toBeDefined();
1330
+ expect(log.toStdout).toEqual([]);
1331
+ });
1332
+
1333
+ it('records a dead credential, carrying the runner’s classification and not gog’s words', async () => {
1334
+ const log = captureLog();
1335
+ const cause =
1336
+ 'invalid_grant: the stored Google refresh token is expired or revoked — re-authorize the account';
1337
+ vi.stubGlobal('fetch', routedFetch(
1338
+ () => gogFailed(GOOGLE_401_STDERR),
1339
+ () => probeBody({ ok: false, measured: true, accounts: [], error: cause }),
1340
+ ));
1341
+
1342
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1343
+
1344
+ const record = log.byEvent('refusal.google-unhealthy');
1345
+ expect(record).toBeDefined();
1346
+ expect(record!.reason).toBe(cause);
1347
+ expect(log.emitted.find((e) => e.line.includes('refusal.google-unhealthy'))!.method).toBe('error');
1348
+ });
1349
+
1350
+ it('reports an unhealthy layer even when the runner names no cause', async () => {
1351
+ const log = captureLog();
1352
+ vi.stubGlobal('fetch', routedFetch(
1353
+ () => gogFailed(GOOGLE_401_STDERR),
1354
+ () => probeBody({ ok: false, measured: true, accounts: [] }),
1355
+ ));
1356
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1357
+ expect(log.byEvent('refusal.google-unhealthy')!.reason).toMatch(/no cause/);
1358
+ });
1359
+
1360
+ it('REVIEW DEFECT: a probe that could not RUN never becomes "the credential is refused"', async () => {
1361
+ // This is the record that decides an incident: `refusal.google-unhealthy`
1362
+ // means "Google refused the call AND the live check agrees the credential is
1363
+ // refused". Three of the runner's causes are facts about the probe — it
1364
+ // timed out, it could not be run at all (no `gog` on PATH, no
1365
+ // `credentials.json` on the volume), its output could not be parsed — and
1366
+ // filing those here would tell an operator the refresh token was dead on
1367
+ // evidence nobody gathered. Worse, it silently disables `refusal.google-ok`,
1368
+ // the ONE record that can prove or kill the narrow theory.
1369
+ for (const error of [
1370
+ 'the Google probe timed out before gog answered',
1371
+ 'the Google probe could not be run',
1372
+ 'gog auth list --check returned unrecognized output',
1373
+ 'gog did not report token validity',
1374
+ 'gog reported an account it explicitly did not check',
1375
+ ]) {
1376
+ const log = captureLog();
1377
+ vi.stubGlobal('fetch', routedFetch(
1378
+ () => gogFailed(GOOGLE_401_STDERR),
1379
+ () => probeBody({ ok: false, measured: false, accounts: [], error }),
1380
+ ));
1381
+
1382
+ // The caller's error is untouched, as always.
1383
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/Google API error \(401/);
1384
+
1385
+ expect(log.byEvent('refusal.google-unhealthy')).toBeUndefined();
1386
+ const record = log.byEvent('refusal.google-unmeasured');
1387
+ expect(record).toBeDefined();
1388
+ expect(record!.reason).toBe(error);
1389
+ expect(log.emitted.find((e) => e.line.includes('refusal.google-unmeasured'))!.method).toBe('warn');
1390
+ vi.restoreAllMocks();
1391
+ }
1392
+ });
1393
+
1394
+ it('will not claim the credential is refused from a runner that never said it measured', async () => {
1395
+ const log = captureLog();
1396
+ vi.stubGlobal('fetch', routedFetch(
1397
+ () => gogFailed(GOOGLE_401_STDERR),
1398
+ () => probeBody({ ok: false, accounts: [], error: 'something went wrong' }),
1399
+ ));
1400
+
1401
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1402
+
1403
+ expect(log.byEvent('refusal.google-unhealthy')).toBeUndefined();
1404
+ expect(log.byEvent('refusal.google-unmeasured')!.reason).toContain('something went wrong');
1405
+ });
1406
+
1407
+ it('never turns a probe that could not run into a claim about Google', async () => {
1408
+ // A runner deployed before /health/google existed answers 404. "I could not
1409
+ // ask" must never be filed as "Google said no" — that is the defect this
1410
+ // whole branch exists to delete, with the alarm merely inverted.
1411
+ const log = captureLog();
1412
+ vi.stubGlobal('fetch', routedFetch(
1413
+ () => gogFailed(GOOGLE_401_STDERR),
1414
+ () => probeBody({ error: 'not found' }, 404),
1415
+ ));
1416
+
1417
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1418
+
1419
+ const record = log.byEvent('refusal.google-unmeasured');
1420
+ expect(record).toBeDefined();
1421
+ expect(record!.reason).toMatch(/404/);
1422
+ // NOT an error: the absence of a measurement is not evidence of anything.
1423
+ expect(log.emitted.find((e) => e.line.includes('refusal.google-unmeasured'))!.method).toBe('warn');
1424
+ expect(log.byEvent('refusal.google-unhealthy')).toBeUndefined();
1425
+ });
1426
+
1427
+ it('survives a probe that rejects, and still lets the original error through', async () => {
1428
+ const log = captureLog();
1429
+ vi.stubGlobal('fetch', routedFetch(
1430
+ () => gogFailed(GOOGLE_401_STDERR),
1431
+ () => { throw new Error('network down'); },
1432
+ ));
1433
+
1434
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/Google API error \(401/);
1435
+ expect(log.byEvent('refusal.google-unmeasured')!.reason).toMatch(/network down/);
1436
+ });
1437
+
1438
+ it('survives a probe that rejects with a non-Error', async () => {
1439
+ const log = captureLog();
1440
+ vi.stubGlobal('fetch', routedFetch(
1441
+ () => gogFailed(GOOGLE_401_STDERR),
1442
+ () => { throw 'nope'; },
1443
+ ));
1444
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1445
+ expect(log.byEvent('refusal.google-unmeasured')!.reason).toBe('nope');
1446
+ });
1447
+
1448
+ it('does not ask a question gog already answered: invalid_grant is not probed', async () => {
1449
+ // gog said the grant is dead. Spending a Google API call to be told the same
1450
+ // thing buys nothing, and this is the COMMON failure — the 7-day cliff — so
1451
+ // probing it would be the one case that costs the most and learns the least.
1452
+ const log = captureLog();
1453
+ const fetchMock = routedFetch(
1454
+ () => gogFailed(`${GOOGLE_401_STDERR}\noauth2: "invalid_grant"`),
1455
+ () => probeBody({ ok: true, measured: true, accounts: [] }),
1456
+ );
1457
+ vi.stubGlobal('fetch', fetchMock);
1458
+
1459
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/invalid_grant/);
1460
+ expect(probeCalls(fetchMock)).toBe(0);
1461
+ expect(log.byEvent('grant.dead')).toBeDefined();
1462
+ });
1463
+
1464
+ it('MUST NOT REGRESS: a runner transport failure is never probed for Google health', async () => {
1465
+ // The runner's own 401 is about OUR bearer. gog never ran and no Google
1466
+ // credential was read, so asking Google anything here would re-create the
1467
+ // exact misattribution 2.21.1 fixed — one layer down, in the log.
1468
+ const fetchMock = routedFetch(
1469
+ () => ({ ok: false, status: 401, json: async () => ({ error: 'unauthorized' }) }),
1470
+ () => probeBody({ ok: true, measured: true, accounts: [] }),
1471
+ );
1472
+ vi.stubGlobal('fetch', fetchMock);
1473
+ captureLog();
1474
+
1475
+ const err = await makeFlyExecutor(ENDPOINT, KEY)(READ, {}).catch((e: unknown) => e);
1476
+ expect(isRunnerTransportError(err)).toBe(true);
1477
+ expect(probeCalls(fetchMock)).toBe(0);
1478
+ });
1479
+
1480
+ it('MUST NOT REGRESS: an ordinary gog failure is never probed', async () => {
1481
+ const fetchMock = routedFetch(
1482
+ () => gogFailed('row 401 is outside the sheet grid'),
1483
+ () => probeBody({ ok: true, measured: true, accounts: [] }),
1484
+ );
1485
+ vi.stubGlobal('fetch', fetchMock);
1486
+ captureLog();
1487
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/outside the sheet grid/);
1488
+ expect(probeCalls(fetchMock)).toBe(0);
1489
+ });
1490
+
1491
+ it('does not probe when the call carried a token of ours — that path repairs itself', async () => {
1492
+ const fetchMock = routedFetch(
1493
+ () => gogFailed(GOOGLE_401_STDERR),
1494
+ () => probeBody({ ok: true, measured: true, accounts: [] }),
1495
+ );
1496
+ vi.stubGlobal('fetch', fetchMock);
1497
+ captureLog();
1498
+ const readToken = Object.assign(vi.fn(async () => 'ya29.stale'), {
1499
+ invalidate: vi.fn(async () => true),
1500
+ });
1501
+
1502
+ await expect(makeFlyExecutor(ENDPOINT, KEY, readToken)(READ, {})).rejects.toThrow(/401/);
1503
+ // Two /run attempts (the replay), and no probe: the eviction already
1504
+ // answered the question the probe would ask.
1505
+ expect(runCalls(fetchMock)).toBe(2);
1506
+ expect(probeCalls(fetchMock)).toBe(0);
1507
+ });
1508
+
1509
+ it('will not spend the caller’s remaining deadline on a diagnostic', async () => {
1510
+ // The probe shares the tool call's ONE deadline. Below the floor it could
1511
+ // only abort, and an abort here would delay the caller's real error for
1512
+ // nothing.
1513
+ const log = captureLog();
1514
+ const fetchMock = routedFetch(
1515
+ () => gogFailed(GOOGLE_401_STDERR),
1516
+ () => probeBody({ ok: true, measured: true, accounts: [] }),
1517
+ );
1518
+ vi.stubGlobal('fetch', fetchMock);
1519
+
1520
+ // opts.timeout 0 leaves only DEADLINE_GRACE_MS. The clock is read once to
1521
+ // fix the deadline and once by the probe; making the second read land a
1522
+ // minute later is exactly "the first attempt ran long".
1523
+ const start = Date.now();
1524
+ let reads = 0;
1525
+ vi.spyOn(Date, 'now').mockImplementation(() => (reads++ === 0 ? start : start + 60_000));
1526
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, { timeout: 0 })).rejects.toThrow(/401/);
1527
+
1528
+ expect(probeCalls(fetchMock)).toBe(0);
1529
+ expect(log.byEvent('refusal.google-unmeasured')!.reason).toMatch(/deadline/);
1530
+ });
1531
+
1532
+ it('probes at most once per interval, so a retry loop cannot storm the backend', async () => {
1533
+ // /health/google spawns a real gog and takes the keyring's exclusive flock
1534
+ // (gogcli v0.34.1: auth list → ListTokens → withWriteLock → unix.LOCK_EX;
1535
+ // see the sourced note on PROBE_INTERVAL_MS). A model retrying a dead call
1536
+ // must not turn one diagnostic into a queue.
1537
+ const log = captureLog();
1538
+ const fetchMock = routedFetch(
1539
+ () => gogFailed(GOOGLE_401_STDERR),
1540
+ () => probeBody({ ok: true, measured: true, accounts: [] }),
1541
+ );
1542
+ vi.stubGlobal('fetch', fetchMock);
1543
+
1544
+ const exec = makeFlyExecutor(ENDPOINT, KEY);
1545
+ await expect(exec(READ, {})).rejects.toThrow(/401/);
1546
+ await expect(exec(READ, {})).rejects.toThrow(/401/);
1547
+
1548
+ expect(runCalls(fetchMock)).toBe(2);
1549
+ expect(probeCalls(fetchMock)).toBe(1);
1550
+ expect(log.byEvent('refusal.google-unmeasured')!.reason).toMatch(/attempted recently/);
1551
+ });
1552
+
1553
+ // End to end through the REAL run() and the REAL diagnose(), because the claim
1554
+ // that matters most about this whole feature is a NEGATIVE one: the caller's
1555
+ // result is byte-identical whether the probe ran or not. This also re-pins the
1556
+ // #250 shape — `Google API error (401 authError)` still reaching the auth
1557
+ // hint — with the probe in the loop.
1558
+ it('MUST NOT REGRESS: the caller’s diagnosed result is identical with the probe in the loop', async () => {
1559
+ captureLog();
1560
+ const withProbe = routedFetch(
1561
+ () => gogFailed(GOOGLE_401_STDERR),
1562
+ () => probeBody({ ok: true, measured: true, accounts: [] }),
1563
+ );
1564
+ vi.stubGlobal('fetch', withProbe);
1565
+ const probed = await runExecutor.run({ executor: makeFlyExecutor(ENDPOINT, KEY) }, () =>
1566
+ runOrDiagnose(['gmail', 'search', 'q'], {}),
1567
+ );
1568
+
1569
+ // The same failure through an executor whose probe is skipped outright
1570
+ // (gog said invalid_grant is a different error, so use the throttle: a
1571
+ // second call on the same executor never probes).
1572
+ const exec = makeFlyExecutor(ENDPOINT, KEY);
1573
+ const twice = routedFetch(
1574
+ () => gogFailed(GOOGLE_401_STDERR),
1575
+ () => probeBody({ ok: true, measured: true, accounts: [] }),
1576
+ );
1577
+ vi.stubGlobal('fetch', twice);
1578
+ await runExecutor.run({ executor: exec }, () => runOrDiagnose(['gmail', 'search', 'q'], {}));
1579
+ const unprobed = await runExecutor.run({ executor: exec }, () =>
1580
+ runOrDiagnose(['gmail', 'search', 'q'], {}),
1581
+ );
1582
+
1583
+ expect(probed.isError).toBe(true);
1584
+ expect(probed.content[0].text).toContain('Google API error (401 authError)');
1585
+ expect(probed.content[0].text).toContain('gog_auth_add');
1586
+ // The whole point: the probe is invisible to the caller.
1587
+ expect(unprobed.content[0].text).toBe(probed.content[0].text);
1588
+ });
1589
+
1590
+ it('MUST NOT CLAIM: the throttle line never asserts a measurement that did not happen', async () => {
1591
+ // The whole thesis of this branch is that a log line may not claim health it
1592
+ // did not measure. The throttle is the one place that rule can be broken
1593
+ // from the inside: `lastProbeAt` is stamped BEFORE the fetch and is not
1594
+ // reset when the probe comes back with no verdict, so every refusal for the
1595
+ // next PROBE_INTERVAL_MS is explained by a sentence about the previous
1596
+ // probe. If that sentence says the layer "was measured", it is describing a
1597
+ // measurement that never occurred — here, a runner too old to have
1598
+ // /health/google at all.
1599
+ //
1600
+ // Stamping before the await is correct and stays: it is what stops two
1601
+ // overlapping refusals from both spawning a probe, and the backend cost the
1602
+ // throttle protects was paid whether or not a verdict came back. So the
1603
+ // sentence is what has to be true, not the timestamp.
1604
+ const log = captureLog();
1605
+ const fetchMock = routedFetch(
1606
+ () => gogFailed(GOOGLE_401_STDERR),
1607
+ () => probeBody({ error: 'not found' }, 404),
1608
+ );
1609
+ vi.stubGlobal('fetch', fetchMock);
1610
+
1611
+ const exec = makeFlyExecutor(ENDPOINT, KEY);
1612
+ await expect(exec(READ, {})).rejects.toThrow(/401/);
1613
+ await expect(exec(READ, {})).rejects.toThrow(/401/);
1614
+
1615
+ // One probe attempted, and it produced no verdict about Google.
1616
+ expect(probeCalls(fetchMock)).toBe(1);
1617
+ const unmeasured = log
1618
+ .records()
1619
+ .filter((r) => r.event === 'refusal.google-unmeasured')
1620
+ .map((r) => r.reason as string);
1621
+ expect(unmeasured).toHaveLength(2);
1622
+ expect(unmeasured[0]).toMatch(/did not answer the Google probe \(HTTP 404\)/);
1623
+
1624
+ // The throttled line: it must say a probe was ATTEMPTED, never that the
1625
+ // Google layer was measured.
1626
+ expect(unmeasured[1]).toMatch(/attempted recently/);
1627
+ expect(unmeasured[1]).not.toMatch(/measured recently|was measured|re-measured/);
1628
+ });
1629
+
1630
+ it('throttles per executor, so one session cannot silence another', async () => {
1631
+ const log = captureLog();
1632
+ const fetchMock = routedFetch(
1633
+ () => gogFailed(GOOGLE_401_STDERR),
1634
+ () => probeBody({ ok: true, measured: true, accounts: [] }),
1635
+ );
1636
+ vi.stubGlobal('fetch', fetchMock);
1637
+
1638
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1639
+ await expect(makeFlyExecutor(ENDPOINT, KEY)(READ, {})).rejects.toThrow(/401/);
1640
+
1641
+ expect(probeCalls(fetchMock)).toBe(2);
1642
+ expect(log.byEvent('refusal.google-unmeasured')).toBeUndefined();
1643
+ });
1644
+ });