gogcli-mcp 2.21.0 → 2.22.0

package/src/connector-runtime.ts

@@ -1,5 +1,15 @@
-import { runExecutor } from './runner.js';
+import { runExecutor, RunnerTransportError } from './runner.js';
 import type { GogArg, GogExecutor } from './runner.js';
+import { logAuthTransition, type AuthTransition } from './auth-log.js';
+import { readGoogleProbe } from './google-probe.js';
+
+// Re-exported so the runner-transport error type reads as part of THIS module's
+// surface — this is the layer that authors these failures. It is DEFINED in
+// runner.js because runner.js also has to preserve the type across its
+// redacting rewrap, and importing it from here would make the two modules
+// circular.
+export { RunnerTransportError, isRunnerTransportError } from './runner.js';
+export type { RunnerFailureKind } from './runner.js';
 
 // Runtime helpers for the Cloudflare connector (worker.ts), split out here so
 // they can be unit-tested under the node pool — worker.ts itself imports the
@@ -30,6 +40,74 @@ const DEFAULT_TIMEOUT_MS = 30_000;
 // than to inflate the grace, so the runner still gets to answer first.
 const DEADLINE_GRACE_MS = 5_000;
 
+// The smallest remaining budget worth spending on a replay.
+//
+// The replay shares the ORIGINAL call's deadline rather than getting a fresh
+// copy of it (see `deadlineAt` in makeFlyExecutor), so a first attempt that ran
+// long leaves the second one very little. Below this floor the replay can only
+// end in an abort, and that TimeoutError would REPLACE gog's own 401 — trading
+// the error that names the problem for one that names nothing.
+//
+// Declining costs only the automatic repair, never the durable one: the
+// eviction has already happened by the time this is consulted, so the caller's
+// own next call mints a fresh token.
+const MIN_REPLAY_BUDGET_MS = 1_000;
+
+// --- The Google-layer measurement taken when a hosted call is refused --------
+//
+// DEFECT 3 was "the hosted path has no automatic recovery, by design", and the
+// instinct was to build one. It should not be built. `gog` is spawned fresh per
+// `/run` and re-reads the keyring every time, so there is no cross-spawn
+// in-memory token that could go stale: a Google 401 on this path means the
+// STORED credential was refused, and no retry can repair that. What was missing
+// was never a retry — it was an answer to the question nobody could answer after
+// the incident: at the moment Google refused that call, was the refresh token on
+// the volume alive or dead? `replay.declined` records only that WE did nothing.
+//
+// So this asks, once, using the probe the runner now exposes, and writes down
+// what it heard. It never decides anything, never alters the caller's error and
+// never throws.
+
+/**
+ * How long the refusal probe may take.
+ *
+ * Deliberately much shorter than the runner's own budget for the same probe
+ * (`GOOGLE_PROBE_TIMEOUT_MS` in server.mjs): the caller is already holding a
+ * failed tool call, and every millisecond spent here delays the error they
+ * actually need. Timing out is a fine outcome — it records "not measured".
+ */
+const REFUSAL_PROBE_TIMEOUT_MS = 4_000;
+
+/**
+ * The smallest slice of the call's REMAINING deadline worth spending on a
+ * diagnostic. Mirrors MIN_REPLAY_BUDGET_MS and for the same reason: below this
+ * the probe can only abort, so it would buy nothing and cost the caller's error
+ * a delay. Skipping is recorded, not silent.
+ */
+const MIN_PROBE_BUDGET_MS = 1_000;
+
+/**
+ * How rarely one executor will re-measure the same backend.
+ *
+ * `/health/google` spawns a real `gog auth list --check`, which costs a Google
+ * API call and takes the keyring's EXCLUSIVE flock. That adjective was
+ * challenged in review as read-only rhetoric, so it is sourced: `auth list`
+ * reaches the keyring through `listAuthTokensWithFallback` →
+ * `store.ListTokens()` (gogcli `internal/cmd/auth_list_helpers.go`), and
+ * `ListTokens` wraps its read in `withWriteLock`, not `withReadLock`
+ * (`internal/secrets/token.go`) → `withFileLock(true, …)` →
+ * `unix.LOCK_EX | LOCK_NB` (`internal/secrets/keyring_lock_unix.go`). Verified
+ * against v0.34.1, the tag this deployment's Dockerfile pins. The shared lock
+ * exists but is taken by `Keys()`, which `auth list` reaches only on the
+ * fallback path after `ListTokens` has already failed.
+ *
+ * A model retrying a call against a genuinely refused credential would
+ * otherwise turn one diagnostic into a queue of them, on the box that is
+ * already failing. One measurement a minute is plenty: the fact being measured
+ * changes on the order of days.
+ */
+const PROBE_INTERVAL_MS = 60_000;
+
 // Status codes the Fly runner uses to classify its OWN failures. These must stay
 // in sync with fly-gog-runner/server.mjs — they are the contract that lets this
 // side tell "gog ran and failed" apart from "the request never arrived", without
@@ -38,8 +116,351 @@ const DEADLINE_GRACE_MS = 5_000;
 // 422: `gog` executed and exited non-zero. Deterministic — never retry.
 // 503: the runner is draining (SIGINT from Fly's autostop). Transient — retry.
 // Any other non-2xx: infrastructure, i.e. Fly's edge, not us.
+// 400: the runner refused the request shape before `gog` was reached.
+// 401: the runner rejected OUR bearer token — its own transport auth.
+//
+// Both are runner-authored: they are surfaced as RunnerTransportError so the
+// diagnosing layer classifies them by TYPE. Reading them as prose is what made
+// a key mismatch look like a dead Google grant: the runner answers a bad bearer
+// with the single word "unauthorized", which is also exactly how Google phrases
+// a rejected credential.
 const RUNNER_GOG_FAILED = 422;
 const RUNNER_DRAINING = 503;
+const RUNNER_BAD_REQUEST = 400;
+const RUNNER_BAD_KEY = 401;
+
+// How this module reads a token source. Structurally the plain
+// `() => token` of the #230 wiring, plus the optional `invalidate` that
+// google-token.ts's minting source provides — optional precisely so a source
+// that cannot re-mint says so by not having it.
+export type FlyAccessTokenSource = {
+  (): string | undefined | Promise<string | undefined>;
+  invalidate?: (rejected: string) => boolean | Promise<boolean>;
+  /**
+   * The credential's log-safe name (google-token.ts `credentialTag`), so a
+   * record written here names the same credential as the records written
+   * there. Optional for the same reason `invalidate` is: a source holding a
+   * directly-supplied token has no mintable credential to name.
+   */
+  credentialId?: () => Promise<string>;
+};
+
+// `gog` ran on the backend and exited non-zero (the runner's 422). Kept as an
+// ORDINARY Error subclass — its `name` stays 'Error' and its message is still
+// gog's own words — because tools/utils.ts must go on reading that prose. The
+// only thing added is a place to keep gog's stderr APART from the command line
+// the runner echoed alongside it, and a type this module can recognise.
+//
+// Why a type rather than re-reading the message: the re-mint below must fire
+// only for a failure that actually reached Google. Deciding that by pattern
+// would put us back where defect 1 started — inferring the author of a failure
+// from words that several different authors can produce.
+//
+// `instanceof` is safe here (unlike RunnerTransportError, which is branded)
+// because this class is thrown and caught inside this one module: there is no
+// boundary a second copy of it could be created across.
+class GogFailedError extends Error {
+  /** gog's stderr alone, with no echoed argv mixed in. */
+  readonly stderr: string;
+
+  constructor(message: string, stderr: string) {
+    super(message);
+    this.stderr = stderr;
+  }
+}
+
+// Google saying, through gog, that the ACCESS token it was given is no good.
+//
+// Matched against gog's stderr only, never against the runner's `error` field:
+// that field is Node's execFile message, which embeds the entire command line,
+// so a caller's own text (`--subject "invoice 401"`) would otherwise decide
+// whether we replay their call.
+//
+// Narrow on purpose, and narrower than tools/utils.ts. That module picks a
+// HINT, where being over-eager costs a sentence; here it would cost a REPLAYED
+// request, so this only matches the shapes Google and gog actually emit.
+// (tools/utils.ts no longer uses a bare /\b401\b/ — 58d3e5b made it require a
+// status word, and #246 widened the separator so `Google API error (401 …)`,
+// the very shape below, still matches there too.)
+const GOOGLE_TOKEN_REJECTED_PATTERN =
+  /Google API error \(401\b|invalid[ _]authentication[ _]credentials|\bACCESS_TOKEN_EXPIRED\b|\binvalid_token\b/i;
+
+// The refresh token is dead, not the access token. Nothing can be minted from
+// it, so there is nothing to replay — a human has to re-authorize. This
+// outranks the pattern above, because gog reports both in one stderr when the
+// grant is gone.
+const REFRESH_TOKEN_DEAD_PATTERN = /\binvalid_grant\b/i;
+
+// `gog` subcommands that only READ. Everything outside this set is treated as a
+// write, which is the conservative direction: a read that is missing here loses
+// nothing but the automatic replay — the eviction happens BEFORE this set is
+// consulted (see the write rule below), so the caller's own next call mints a
+// fresh token and succeeds — while a write that crept in could be applied
+// twice.
+//
+// Only verbs that cannot plausibly grow mutating children are listed.
+// Namespace-shaped words are deliberately absent: `gog gmail labels list`
+// arrives here as the subcommand `labels`, and if `labels` were listed then a
+// later `labels create` would inherit the replay. Reading `labels` as unsafe
+// costs one message, and nothing else.
+//
+// `lists` used to be here, and was the counter-example that proves the rule is
+// not hypothetical: `gog tasks lists` is a NAMESPACE, and `tasks lists create
+// <title> ...` exists in gog v0.34.1 today — variadic, so one invocation makes
+// N Google calls and a 401 on the second means the first already landed. It is
+// reachable without any new gog, through the `gog_tasks_run` escape hatch
+// (`{subcommand: 'lists', args: ['create', 'A', 'B']}`), so listing the
+// namespace word handed a real double-apply the replay. Its absence costs
+// `gog_tasks_lists` (tools/tasks.ts, argv `tasks lists list`) one automatic
+// replay and nothing more: the eviction runs BEFORE this set is consulted, so
+// that caller's next call still mints a fresh token.
+const READ_ONLY_SUBCOMMANDS = new Set([
+  'cat',
+  'describe',
+  'get',
+  'info',
+  'list',
+  'list-slides',
+  'ls',
+  'metadata',
+  'read-slide',
+  'search',
+  'services',
+  'status',
+  'structure',
+]);
+
+// The gog SERVICE and SUBCOMMAND inside a fully-assembled arg list, either of
+// which may be absent.
+//
+// `run()` (runner.ts assembleArgs) puts global flags in front of the service and
+// subcommand: --json, --color=never, --no-input, --readonly, and --account,
+// which is the only one carrying a SEPARATE value. So the subcommand is the
+// second bare word once flags (and --account's value) are skipped.
+//
+// If a future global flag with a value is added and not handled here, the scan
+// mistakes that value for the service and returns the wrong word — which will
+// almost certainly not be in the read-only set, i.e. it degrades toward NOT
+// replaying. That is the direction a mistake here has to fail in.
+// The service is returned as well as the subcommand because it is what a log
+// record needs: the whole fleet shares one backend and one Google credential,
+// so "which service was this" is the first question asked of an auth record,
+// and re-scanning argv a second time to answer it would be waste on a path that
+// is already handling a failure.
+function gogTarget(args: GogArg[]): { service?: string; subcommand?: string } {
+  // A GogFileArg is a payload, never a verb; dropping the non-strings first
+  // keeps the scan below about words only.
+  const words = args.filter((arg): arg is string => typeof arg === 'string');
+  let service: string | undefined;
+  for (let i = 0; i < words.length; i += 1) {
+    const word = words[i];
+    if (word.startsWith('-')) {
+      if (word === '--account') i += 1;
+      continue;
+    }
+    if (service === undefined) {
+      service = word;
+      continue;
+    }
+    return { service, subcommand: word };
+  }
+  return { service };
+}
+
+// A replay that has been authorized: the freshly minted token to send, plus the
+// identifiers the records around it are tagged with. Carried together rather
+// than recomputed by the caller so the token and the credential a log line
+// names can never come from two different decisions.
+interface Replay {
+  token: string;
+  /**
+   * What is LEFT of the one deadline this tool call was given, in ms. Carried
+   * rather than recomputed by the caller so the budget that was judged
+   * sufficient is exactly the budget that gets spent.
+   */
+  budgetMs: number;
+  /**
+   * The same eviction the rejected token got, carried so the caller can apply
+   * it to THIS token if Google refuses it too — see the symmetric eviction
+   * below. Carried rather than re-read from the source because by the time an
+   * authorization exists it has already been proved present, and re-deriving it
+   * would mean re-asking a question whose answer is what this object IS.
+   */
+  invalidate: NonNullable<FlyAccessTokenSource['invalidate']>;
+  credential: string | undefined;
+  service: string | undefined;
+}
+
+// Decide whether this failure earns exactly one replay with a freshly minted
+// token, and produce that token if so. `undefined` means "do not replay".
+//
+// Every condition is a separate refusal because each is refusing for its own
+// reason, and collapsing them would make the log of WHY unreadable.
+async function remintAfterGoogleRejection(
+  err: unknown,
+  used: string | undefined,
+  args: GogArg[],
+  readAccessToken: FlyAccessTokenSource | undefined,
+  deadlineAt: number,
+  // Take a live reading of the Google layer. Supplied by `makeFlyExecutor` so
+  // this function keeps knowing nothing about the endpoint, the bearer or how
+  // often measuring is affordable — it decides only WHEN a reading is worth
+  // taking, which is the one part of it that belongs to the replay ladder.
+  probeGoogle: (where: { credential?: string; service?: string }) => Promise<void>,
+): Promise<Replay | undefined> {
+  // Only a failure gog itself authored can carry Google's verdict. A runner
+  // transport failure never reached Google, and the runner's own 401 is about
+  // OUR bearer — minting a Google token for either is pure waste.
+  if (!(err instanceof GogFailedError)) return undefined;
+
+  // THE RECORDING GATE, and the reason the two pattern tests moved up here from
+  // the middle of the ladder.
+  //
+  // Everything below this line is a refusal worth WRITING DOWN, because by here
+  // Google has demonstrably refused a credential. Everything above it is an
+  // ordinary gog failure — a bad attachment id, an --out path that does not
+  // exist on the box — and an auth log that also carries those is an auth log
+  // nobody reads.
+  //
+  // Reordering is safe precisely because every check in this function is a pure
+  // predicate whose failure returns `undefined`: none has a side effect until
+  // the `invalidate` call at the bottom, so the answer cannot depend on the
+  // order they are asked in.
+  const grantDead = REFRESH_TOKEN_DEAD_PATTERN.test(err.stderr);
+  if (!grantDead && !GOOGLE_TOKEN_REJECTED_PATTERN.test(err.stderr)) return undefined;
+
+  const { service, subcommand } = gogTarget(args);
+  const credential = await readAccessToken?.credentialId?.();
+  const where = { credential, service };
+
+  // The refresh token is gone: no mint can succeed, and replaying would loop a
+  // caller against a credential that can never work. This is the ONE outcome on
+  // this path that legitimately ends in "a human must re-authorize", which is
+  // exactly why it gets its own transition rather than a declined-replay note.
+  if (grantDead) {
+    logAuthTransition('grant.dead', {
+      ...where,
+      reason:
+        'gog reported invalid_grant: the stored refresh token is dead, so no token can be minted ' +
+        'and this account must be re-authorized',
+    });
+    return undefined;
+  }
+
+  // Only a token WE supplied is ours to replace. Without one, gog acted as
+  // whatever identity the backend volume holds and only an operator can change
+  // that.
+  if (!used) {
+    // THE HOSTED PATH, and the only place a reading is worth paying for.
+    //
+    // We are here because Google refused a call that `gog` made as the BACKEND
+    // VOLUME's own identity — the shape `worker.ts` produces on every hosted
+    // connector. Nothing above this line can say whether the credential behind
+    // that identity is dead or alive, and that is precisely the fact the
+    // incident needed and did not have. So measure first, then record the
+    // decision: the pair reads as "here is what Google said, here is why we did
+    // nothing about it".
+    //
+    // Ordered before the `replay.declined` line rather than after it so the log
+    // tells the story in the order it happened. Awaited rather than fired and
+    // forgotten because a Worker may cancel unawaited work at the end of the
+    // request — an unawaited probe is one that silently does not happen, which
+    // is the failure mode this whole branch exists to delete.
+    await probeGoogle(where);
+    logAuthTransition('replay.declined', {
+      ...where,
+      reason:
+        'no access token was supplied with the call, so gog acted as the backend volume’s own identity',
+    });
+    return undefined;
+  }
+
+  // A source with no cache behind it (a directly-supplied GOG_ACCESS_TOKEN)
+  // would hand back the identical rejected string.
+  if (!readAccessToken?.invalidate) {
+    logAuthTransition('replay.declined', {
+      ...where,
+      reason: 'this token source cannot mint a replacement, so a replay would resend the rejected token',
+    });
+    return undefined;
+  }
+
+  // EVICT FIRST, and unconditionally — every remaining check decides whether to
+  // REPLAY, which is a different question with a different answer.
+  //
+  // A replay re-runs the call and can double-apply; an eviction only drops a
+  // string Google has already refused, and the cache's own read guard is purely
+  // about TIME, so a token left in it is re-served until its nominal expiry.
+  // Gating the eviction behind the replay rules is therefore not a conservative
+  // choice but the caching half of the original defect: after Google rejected
+  // the token, every write (and every read whose subcommand is outside the
+  // allow-list) re-sent that same rejected token for up to ~58 minutes, and
+  // only a reconnect — a fresh isolate with an empty cache — appeared to help.
+  //
+  // Safe to do before the decision because it is idempotent and value-matched:
+  // `invalidate` drops the entry only if it still holds exactly this string, so
+  // a concurrent caller's fresher token is never the casualty, and a second
+  // call for the same token changes nothing.
+  const evicted = await readAccessToken.invalidate(used);
+
+  // THE WRITE RULE. A replay re-runs the whole gog invocation. For a read that
+  // is free; for a write it is only safe if nothing was applied before the
+  // failure, and this layer cannot know that — gog may make several Google
+  // calls in one invocation, and a 401 on a later one would mean the earlier
+  // ones already landed. Re-sending an email is not a cost worth paying for
+  // hiding one error message, so writes get the eviction above and nothing more.
+  if (subcommand === undefined || !READ_ONLY_SUBCOMMANDS.has(subcommand)) {
+    logAuthTransition('replay.declined', {
+      ...where,
+      reason: `not replayable: '${subcommand ?? '(none)'}' is not a known read-only subcommand and a write could double-apply`,
+    });
+    return undefined;
+  }
+
+  // Did the eviction actually drop this token? False means a concurrent caller
+  // already replaced it, so the token a replay would send is the one already in
+  // use and the replay proves nothing.
+  if (!evicted) {
+    logAuthTransition('replay.declined', {
+      ...where,
+      reason: 'the rejected token was already superseded, so the cache holds the token a replay would send',
+    });
+    return undefined;
+  }
+
+  // The mint can throw (invalid_grant, Google unreachable). That error is
+  // allowed to propagate in place of gog's 401, because it is strictly more
+  // actionable: it names the credential that is actually dead and the step that
+  // repairs it, where gog's 401 only says a token was refused. The mint's own
+  // outcome is recorded by google-token.ts, so nothing is logged for it here.
+  const fresh = await readAccessToken();
+
+  // A source that minted a moment ago and now answers nothing must NOT be
+  // replayed without a token: the backend would run the call as its own
+  // identity and hand this caller someone else's account.
+  if (!fresh) {
+    logAuthTransition('replay.declined', {
+      ...where,
+      reason: 'the token source produced no token after eviction; replaying without one would act as the backend',
+    });
+    return undefined;
+  }
+
+  // Whatever is LEFT of the one deadline this call was given — the mint above
+  // spends from it too. A replay with no budget can only abort, and that
+  // timeout would land on the caller in place of gog's own 401.
+  const budgetMs = deadlineAt - Date.now();
+  if (budgetMs < MIN_REPLAY_BUDGET_MS) {
+    logAuthTransition('replay.declined', {
+      ...where,
+      reason:
+        `only ${budgetMs}ms of the call’s deadline remained, so a replay could only time out; ` +
+        'the rejected token was evicted, so the next call mints a fresh one',
+    });
+    return undefined;
+  }
+  return { token: fresh, budgetMs, invalidate: readAccessToken.invalidate, ...where };
+}
 
 // Build a GogExecutor that forwards a fully-assembled `gog` arg-array to the Fly
 // backend's `/run` endpoint.
@@ -78,8 +499,115 @@ const RUNNER_DRAINING = 503;
 export function makeFlyExecutor(
   endpoint: string,
   key: string,
-  readAccessToken?: () => string | undefined | Promise<string | undefined>,
+  readAccessToken?: FlyAccessTokenSource,
 ): GogExecutor {
+  // Throttle state for the refusal probe, held PER EXECUTOR rather than in a
+  // module-level map.
+  //
+  // That is the scope the thing being throttled actually has: on the Worker one
+  // executor is built per agent session (`worker.ts` `init()`), on stdio one per
+  // process (`remote-runner.ts`). So a session that is hammering a refused
+  // credential rate-limits itself without a second, unrelated session's probe
+  // being suppressed by it — a module global would let one caller's retry loop
+  // silence everybody else's first and only measurement. It also means the state
+  // dies with the session instead of accumulating endpoints for the isolate's
+  // lifetime.
+  let lastProbeAt = Number.NEGATIVE_INFINITY;
+
+  /**
+   * Ask the runner whether Google still accepts the credential on its volume,
+   * and record the answer. Resolves in EVERY case: it can neither throw nor
+   * return a value, because nothing may make a decision out of what it finds.
+   * The caller's error is already decided by the time this runs.
+   */
+  const probeGoogleAfterRefusal = async (
+    where: { credential?: string; service?: string },
+    deadlineAt: number,
+  ): Promise<void> => {
+    const record = { ...where, endpoint };
+
+    // ONE reading of the clock for both budget and throttle, so the two
+    // decisions cannot disagree about what time it is.
+    const now = Date.now();
+    const remainingMs = deadlineAt - now;
+    if (remainingMs < MIN_PROBE_BUDGET_MS) {
+      logAuthTransition('refusal.google-unmeasured', {
+        ...record,
+        reason:
+          `only ${remainingMs}ms of the call’s deadline remained, so the Google layer was not ` +
+          'measured rather than delay the caller’s own error',
+      });
+      return;
+    }
+    if (now - lastProbeAt < PROBE_INTERVAL_MS) {
+      logAuthTransition('refusal.google-unmeasured', {
+        ...record,
+        // "attempted", not "measured". `lastProbeAt` is stamped before the
+        // fetch and is deliberately NOT reset when the probe comes back with no
+        // verdict (a 404 from a runner too old to have the endpoint, a timeout,
+        // a dead socket) — the backend cost this throttle exists to bound was
+        // paid either way, and resetting it would let a retry loop storm a
+        // runner that is already unwell. So the timestamp stays and the sentence
+        // has to be the true one: on this branch a log line may not assert a
+        // measurement that never happened, and the previous probe may well have
+        // measured nothing at all.
+        reason:
+          'a Google probe was attempted recently, so another was not sent: this probe spawns ' +
+          'gog on the backend and takes the keyring’s exclusive lock',
+      });
+      return;
+    }
+    // Claimed BEFORE the await, so two overlapping refusals cannot both get
+    // past the check and spawn a probe apiece.
+    lastProbeAt = now;
+
+    let event: AuthTransition;
+    let reason: string;
+    try {
+      const res = await fetch(`${endpoint}/health/google`, {
+        headers: { Authorization: `Bearer ${key}` },
+        // Never more than the probe's own budget, never more than the call has
+        // left. `Math.min` rather than a plain constant because the second
+        // bound is the caller's, and it outranks ours.
+        signal: AbortSignal.timeout(Math.min(REFUSAL_PROBE_TIMEOUT_MS, remainingMs)),
+      });
+      if (!res.ok) {
+        // Includes the 404 from a runner deployed before `/health/google`
+        // existed. "I could not ask" is never filed as "Google said no" — that
+        // is the defect this branch exists to delete, with the alarm inverted.
+        event = 'refusal.google-unmeasured';
+        reason = `the runner did not answer the Google probe (HTTP ${res.status})`;
+      } else {
+        // `readGoogleProbe` is the ONE place that judges a probe body, shared
+        // with the connect-time probe in connector-auth.ts. It reads the
+        // runner's `measured` field BEFORE its `ok` field, which is what keeps a
+        // probe that timed out or could not be run from being filed as
+        // `-unhealthy` — the record an operator reads as "the live check agrees
+        // the credential is refused". Its reason strings come from the runner's
+        // CLOSED vocabulary of causes (PROBE_CAUSES in server.mjs), so they
+        // carry a classification and never gog's own output.
+        const verdict = readGoogleProbe(await res.json());
+        if (verdict.kind === 'ok') {
+          event = 'refusal.google-ok';
+          reason =
+            'Google refused this call, yet a live token check on the same volume succeeded — ' +
+            'so a dead or expired refresh token does not explain this refusal';
+        } else {
+          event = verdict.kind === 'unhealthy' ? 'refusal.google-unhealthy' : 'refusal.google-unmeasured';
+          reason = verdict.reason;
+        }
+      }
+    } catch (err) {
+      // A rejected fetch, an abort at the budget, or a body that is not JSON (a
+      // proxy's HTML error page). None of them are facts about Google.
+      event = 'refusal.google-unmeasured';
+      reason = err instanceof Error ? err.message : String(err);
+    }
+    // `reason` can quote text this layer did not author, so the record goes
+    // through the same redactor as every other auth log line.
+    logAuthTransition(event, { ...record, reason });
+  };
+
   return async (args: GogArg[], opts) => {
     const deadlineMs = (opts?.timeout ?? DEFAULT_TIMEOUT_MS) + DEADLINE_GRACE_MS;
     // Awaited, because the token may have to be MINTED (#241): a refresh token
@@ -88,91 +616,248 @@ export function makeFlyExecutor(
     // fails, because the alternative is running it as the backend's identity
     // and handing this caller someone else's account.
     const accessToken = await readAccessToken?.();
-    let res: Response;
+
+    // ONE deadline for the whole tool call, fixed before the first attempt
+    // rather than re-derived per attempt. A replay is a second `fetch`, and
+    // handing it a fresh copy of the budget made the worst case two full
+    // budgets — ~70s of wall clock for one tool call, which can outlast the MCP
+    // client's own request timeout and turn a self-healing read into a
+    // client-side hang.
+    const deadlineAt = Date.now() + deadlineMs;
+
     try {
-      res = await fetch(endpoint + '/run', {
-        method: 'POST',
-        headers: {
-          Authorization: 'Bearer ' + key,
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify(accessToken ? { args, accessToken } : { args }),
-        signal: AbortSignal.timeout(deadlineMs),
-      });
+      return await attempt(endpoint, key, args, accessToken, deadlineMs);
     } catch (err) {
-      // AbortSignal.timeout rejects with a TimeoutError; a caller-supplied abort
-      // surfaces as AbortError. Either way the bare message ("The operation was
-      // aborted") says nothing about which backend failed to answer.
-      const name = err instanceof Error ? err.name : '';
-      if (name === 'TimeoutError' || name === 'AbortError') {
-        throw new Error(
-          `gog-runner did not respond within ${deadlineMs}ms (${endpoint}) — the Fly backend may be cold or wedged`,
-        );
+      // ONE replay, and only when the token we sent is the reason it failed.
+      // Not a retry loop: an unbounded one against a genuinely dead credential
+      // is exactly the behaviour the "retry the same call" hints already
+      // produced elsewhere, and it never terminates. `remintAfterGoogleRejection`
+      // both decides and mints, so the decision cannot drift from the token.
+      const replay = await remintAfterGoogleRejection(
+        err,
+        accessToken,
+        args,
+        readAccessToken,
+        deadlineAt,
+        (where) => probeGoogleAfterRefusal(where, deadlineAt),
+      );
+      if (replay === undefined) throw err;
+
+      const where = { credential: replay.credential, service: replay.service, endpoint };
+      logAuthTransition('replay.attempted', {
+        ...where,
+        reason: 'Google rejected the access token; replaying this read once with a freshly minted one',
+      });
+      try {
+        const stdout = await attempt(endpoint, key, args, replay.token, replay.budgetMs);
+        // The interesting record of the pair: it says the caller saw a clean
+        // success where they used to see an hour of identical 401s.
+        logAuthTransition('replay.succeeded', where);
+        return stdout;
+      } catch (replayErr) {
+        // `String`, not `instanceof Error ? .message : …` — a rethrown non-Error
+        // rejection is a real possibility here (see attempt's abort handling)
+        // and stringifying uniformly avoids an arm that no test could reach.
+        logAuthTransition('replay.failed', { ...where, reason: String(replayErr) });
+        // SYMMETRY with the eviction that got us here, under the SAME predicate
+        // — a token is dropped because GOOGLE refused it, never merely because
+        // a call carrying it failed.
+        //
+        // The gate is the point. A replay can fail without Google ever seeing
+        // the token: the Machine drains between the two attempts, the
+        // client-side deadline fires, the runner's bearer rotates mid-call.
+        // Evicting on those would discard a token nothing has refused and emit
+        // `token.evicted` reading "Google rejected this access token" about a
+        // service that was never consulted — the same misattribution this
+        // branch exists to delete, moved from the user's screen to the
+        // operator's query. So the second eviction asks exactly what authorized
+        // the first: is this a gog failure whose stderr shows Google refusing
+        // the credential?
+        //
+        // WHAT THIS BUYS, measured rather than assumed. Under a sustained
+        // Google-side refusal that is not invalid_grant (a revoked scope, say),
+        // a steady-state call costs 2 /run round-trips either way — the first
+        // attempt and the replay both always happen — and this eviction in fact
+        // costs one EXTRA mint per call (2 instead of 1), because it empties the
+        // cache the next call would otherwise have hit. It is not a round-trip
+        // saving and must not be justified as one.
+        //
+        // It is worth keeping because it bounds how long a KNOWN-REFUSED token
+        // can be handed out. Left cached, `ya29.fresh` is re-served for the rest
+        // of its nominal hour, and the caller that suffers most is a WRITE: a
+        // write gets the eviction and no replay, so it would be sent with the
+        // token that just failed twice and fail on contact — even after the
+        // underlying refusal has cleared. Trading a mint for that is the right
+        // side of the deal.
+        //
+        // Value-matched and idempotent like the first eviction, so a concurrent
+        // caller's fresher token is never the casualty. Not wrapped in its own
+        // catch: `invalidate` is a map lookup behind a digest, the first
+        // eviction is already un-guarded on this same path, and a guard here
+        // would add an arm no test can reach.
+        if (replayErr instanceof GogFailedError && GOOGLE_TOKEN_REJECTED_PATTERN.test(replayErr.stderr)) {
+          await replay.invalidate(replay.token);
+        }
+        throw replayErr;
       }
-      throw err;
     }
-    if (!res.ok) {
-      // Two very different failures arrive as non-2xx, and collapsing them (as
-      // this used to) is what made a real bug look like random flakiness:
-      //
-      //  a) The runner answered with its own JSON — `gog` actually ran on the
-      //     box and failed. Deterministic: the same call will fail the same way.
-      //  b) The body is NOT the runner's JSON (Fly's HTML error page, or empty).
-      //     Then the request never reached `gog` at all; Fly's edge proxy is
-      //     reporting that it could not reach the Machine — typically because
-      //     the Machine was starting from scale-to-zero, or was mid-shutdown.
-      //     Genuinely transient, and the only case worth retrying.
-      const body = (await res.json().catch(() => null)) as
-        | { error?: string; stderr?: string; retryable?: boolean }
-        | null;
-      const detail =
-        body && typeof body.error === 'string'
-          ? body.stderr && body.stderr.trim() && body.stderr.trim() !== body.error.trim()
-            ? `${body.error}\n${body.stderr}`
-            : body.error
-          : '';
-
-      // 422 is the runner's "gog ran and exited non-zero" status. It is only
-      // ever produced by our own handler, so reaching here proves the request
-      // was delivered and executed. Deterministic — say so, and say nothing
-      // that invites a retry.
-      if (res.status === RUNNER_GOG_FAILED) {
-        throw new Error(detail || 'gog failed on the runner (no detail supplied)');
-      }
+  };
+}
 
-      // The runner's drain response: it is up, but deliberately refusing new
-      // work while it shuts down. The one runner-authored failure worth retrying.
-      if (res.status === RUNNER_DRAINING || body?.retryable === true) {
-        throw new Error(
-          `gog-runner is restarting; retry this call.${detail ? ` ${detail}` : ''}`,
-        );
-      }
+// One request to the runner: send the args (and, when we have one, the identity
+// to act as), and turn whatever comes back into either stdout or a classified
+// failure. Everything about WHICH failure this is lives here; the caller above
+// decides only whether to run it a second time.
+async function attempt(
+  endpoint: string,
+  key: string,
+  args: GogArg[],
+  accessToken: string | undefined,
+  deadlineMs: number,
+): Promise<string> {
+  let res: Response;
+  try {
+    res = await fetch(endpoint + '/run', {
+      method: 'POST',
+      headers: {
+        Authorization: 'Bearer ' + key,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(accessToken ? { args, accessToken } : { args }),
+      signal: AbortSignal.timeout(deadlineMs),
+    });
+  } catch (err) {
+    // AbortSignal.timeout rejects with a TimeoutError; a caller-supplied abort
+    // surfaces as AbortError. Either way the bare message ("The operation was
+    // aborted") says nothing about which backend failed to answer.
+    const name = err instanceof Error ? err.name : '';
+    if (name === 'TimeoutError' || name === 'AbortError') {
+      // No status: nothing answered. Retryable — the usual cause is the Fly
+      // machine waking from scale-to-zero, which succeeds on the next call.
+      throw new RunnerTransportError(
+        `gog-runner did not respond within ${deadlineMs}ms (${endpoint}) — the Fly backend may be cold or wedged`,
+        'transport-retryable',
+      );
+    }
+    throw err;
+  }
+  if (!res.ok) {
+    // Two very different failures arrive as non-2xx, and collapsing them (as
+    // this used to) is what made a real bug look like random flakiness:
+    //
+    //  a) The runner answered with its own JSON — `gog` actually ran on the
+    //     box and failed. Deterministic: the same call will fail the same way.
+    //  b) The body is NOT the runner's JSON (Fly's HTML error page, or empty).
+    //     Then the request never reached `gog` at all; Fly's edge proxy is
+    //     reporting that it could not reach the Machine — typically because
+    //     the Machine was starting from scale-to-zero, or was mid-shutdown.
+    //     Genuinely transient, and the only case worth retrying.
+    const body = (await res.json().catch(() => null)) as
+      | { error?: string; stderr?: string; retryable?: boolean }
+      | null;
+    const detail =
+      body && typeof body.error === 'string'
+        ? body.stderr && body.stderr.trim() && body.stderr.trim() !== body.error.trim()
+          ? `${body.error}\n${body.stderr}`
+          : body.error
+        : '';
 
-      // Anything else non-2xx is infrastructure: Fly's edge could not reach the
-      // Machine, or the Machine answered with something that is not ours. Only
-      // claim the request never arrived when there is genuinely no runner body
-      // — a runner that did answer deserves to have its own words repeated.
-      //
-      // The status is deliberately NOT interpolated here. A runner body proves
-      // gog ran, so this is a deterministic failure; embedding the literal
-      // status would put "502" into the message, which matches
-      // TRANSIENT_ERROR_PATTERN (/\b5\d\d\b/) in tools/utils.ts and re-attaches
-      // the very "this is transient, retry the same call" hint this change
-      // exists to remove — reintroducing the bug during the rollout window this
-      // branch exists to cover. Anything genuinely transient in gog's own text
-      // (a Google 5xx, say) still matches on its own merits, which is correct.
-      if (detail) {
-        throw new Error(detail);
-      }
-      throw new Error(
-        `gog-runner HTTP ${res.status}: the response did not come from the runner, ` +
-        'so the request never reached gog. The backend Machine was most likely starting ' +
-        'or shutting down — this is transient, retry the same call.',
+    // 422 is the runner's "gog ran and exited non-zero" status. It is only
+    // ever produced by our own handler, so reaching here proves the request
+    // was delivered and executed. Deterministic — say so, and say nothing
+    // that invites a retry.
+    if (res.status === RUNNER_GOG_FAILED) {
+      // The message is unchanged — gog's words, exactly as before. `stderr` is
+      // carried alongside rather than folded in, so the replay decision can
+      // read what GOG said without also reading the argv the runner echoed
+      // back inside `error`.
+      throw new GogFailedError(
+        detail || 'gog failed on the runner (no detail supplied)',
+        typeof body?.stderr === 'string' ? body.stderr : '',
       );
     }
-    const { stdout } = (await res.json()) as { stdout: string };
-    return stdout;
-  };
+
+    // The runner's OWN bearer auth failed: the key the Worker sent is not the
+    // key the Fly app expects. `gog` never ran, Google was never contacted,
+    // and no stored credential was even read — so the one thing this must not
+    // do is send the caller to re-authorize an account that is fine.
+    //
+    // The runner's body (the bare word "unauthorized") is deliberately
+    // DROPPED rather than repeated, and the status is deliberately not
+    // interpolated, for the same reason the 5xx fallback below omits its own:
+    // both `unauthorized` and `401` match DEFINITE_AUTH_PATTERN in
+    // tools/utils.ts. The type is what classifies this error now, but the
+    // prose must not be able to re-create the old misdiagnosis at any future
+    // boundary where the type could be lost (a serialized error, a log line a
+    // human reads).
+    if (res.status === RUNNER_BAD_KEY) {
+      // The record a human reads must not re-create defect 1 either, so it
+      // states the negative facts explicitly instead of repeating the runner's
+      // bare "unauthorized".
+      logAuthTransition('runner.auth-failed', {
+        service: gogTarget(args).service,
+        endpoint,
+        reason:
+          'the gog-runner rejected the connector’s bearer token, so gog never ran and no Google ' +
+          'credential was read; GOG_RUNNER_KEY does not match the Fly app’s RUNNER_KEY',
+      });
+      throw new RunnerTransportError(
+        "gog-runner rejected the connector's bearer token, so the request never reached gog and no " +
+        'Google credential was involved. The Worker secret GOG_RUNNER_KEY no longer matches RUNNER_KEY ' +
+        'on the Fly app; set them to the same value (wrangler secret put GOG_RUNNER_KEY / fly secrets ' +
+        'set RUNNER_KEY) and retry.',
+        'transport-auth',
+        res.status,
+      );
+    }
+
+    // The runner refused the request shape (oversized arg, unparseable JSON,
+    // a malformed access token). Its words are about OUR request, never about
+    // Google — deterministic, and no hint applies.
+    if (res.status === RUNNER_BAD_REQUEST) {
+      throw new RunnerTransportError(
+        detail || 'gog-runner rejected the request (no detail supplied)',
+        'transport-request',
+        res.status,
+      );
+    }
+
+    // The runner's drain response: it is up, but deliberately refusing new
+    // work while it shuts down. The one runner-authored failure worth retrying.
+    if (res.status === RUNNER_DRAINING || body?.retryable === true) {
+      throw new RunnerTransportError(
+        `gog-runner is restarting; retry this call.${detail ? ` ${detail}` : ''}`,
+        'transport-retryable',
+        res.status,
+      );
+    }
+
+    // Anything else non-2xx is infrastructure: Fly's edge could not reach the
+    // Machine, or the Machine answered with something that is not ours. Only
+    // claim the request never arrived when there is genuinely no runner body
+    // — a runner that did answer deserves to have its own words repeated.
+    //
+    // The status is deliberately NOT interpolated here. A runner body proves
+    // gog ran, so this is a deterministic failure; embedding the literal
+    // status would put "502" into the message, which matches
+    // TRANSIENT_ERROR_PATTERN (/\b5\d\d\b/) in tools/utils.ts and re-attaches
+    // the very "this is transient, retry the same call" hint this change
+    // exists to remove — reintroducing the bug during the rollout window this
+    // branch exists to cover. Anything genuinely transient in gog's own text
+    // (a Google 5xx, say) still matches on its own merits, which is correct.
+    if (detail) {
+      throw new Error(detail);
+    }
+    throw new RunnerTransportError(
+      `gog-runner HTTP ${res.status}: the response did not come from the runner, ` +
+      'so the request never reached gog. The backend Machine was most likely starting ' +
+      'or shutting down — this is transient, retry the same call.',
+      'transport-retryable',
+      res.status,
+    );
+  }
+  const { stdout } = (await res.json()) as { stdout: string };
+  return stdout;
 }
 
 // Wrap an McpServer in a Proxy whose `registerTool` (and `tool`, if any