@whittlelabs/sifter 0.30.9 → 0.30.11

package/bin.js

@@ -11199,16 +11199,260 @@ var require_client = __commonJS({
   }
 });
 
+// ../../packages/keep/dist/rate-limited.js
+var require_rate_limited = __commonJS({
+  "../../packages/keep/dist/rate-limited.js"(exports2) {
+    "use strict";
+    Object.defineProperty(exports2, "__esModule", { value: true });
+    exports2.RateLimitGate = exports2.RATE_LIMIT_DEFAULT_MS = exports2.RATE_LIMIT_CAP_MS = exports2.RATE_LIMIT_FLOOR_MS = exports2.KeepRateLimitedError = void 0;
+    exports2.retryAfterMsOf = retryAfterMsOf;
+    var KeepRateLimitedError = class extends Error {
+      code = "RATE_LIMITED";
+      /** Host the exchange was sent to. */
+      host;
+      /** Keep's own words for the refusal ("Too many attempts. Please try again later."). */
+      reason;
+      /** How long from now to wait before asking again (ms), already floored and capped. */
+      retryAfterMs;
+      /** When the next exchange may be attempted (epoch ms). */
+      retryAt;
+      constructor(message, detail) {
+        super(message);
+        this.name = "KeepRateLimitedError";
+        this.host = detail.host;
+        this.reason = detail.reason;
+        this.retryAt = detail.retryAt;
+        this.retryAfterMs = Math.max(0, detail.retryAt - Date.now());
+      }
+    };
+    exports2.KeepRateLimitedError = KeepRateLimitedError;
+    exports2.RATE_LIMIT_FLOOR_MS = 5e3;
+    exports2.RATE_LIMIT_CAP_MS = 15 * 6e4;
+    exports2.RATE_LIMIT_DEFAULT_MS = 6e4;
+    function retryAfterMsOf(body, headers, now = Date.now()) {
+      return clamp(bodyRetryAfter(body) ?? headerRetryAfter(headers, now) ?? exports2.RATE_LIMIT_DEFAULT_MS);
+    }
+    function bodyRetryAfter(body) {
+      const value = body?.error?.retryAfter;
+      return typeof value === "number" && Number.isFinite(value) ? value * 1e3 : null;
+    }
+    function headerRetryAfter(headers, now) {
+      const raw = headers?.get("retry-after")?.trim();
+      if (!raw)
+        return null;
+      if (/^\d+$/.test(raw))
+        return Number(raw) * 1e3;
+      const at = Date.parse(raw);
+      return Number.isNaN(at) ? null : at - now;
+    }
+    function clamp(ms) {
+      return Math.min(exports2.RATE_LIMIT_CAP_MS, Math.max(exports2.RATE_LIMIT_FLOOR_MS, ms));
+    }
+    var RateLimitGate = class {
+      closed = null;
+      /** Record a 429 and return the error to throw for it. */
+      close(detail) {
+        const retryAt = Date.now() + detail.retryAfterMs;
+        const when = new Date(retryAt).toISOString().replace(/\.\d{3}Z$/, "Z");
+        this.closed = new KeepRateLimitedError(`[${detail.manager}] Keep at ${detail.host} is rate limiting this client's token exchange (429: ${detail.reason}); not asking again before ${when}`, { host: detail.host, reason: detail.reason, retryAt });
+        return this.closed;
+      }
+      /** Throws the rate-limited error while the gate is closed; a no-op once the time has passed. */
+      check() {
+        const closed = this.closed;
+        if (closed === null)
+          return;
+        if (Date.now() >= closed.retryAt) {
+          this.closed = null;
+          return;
+        }
+        throw new KeepRateLimitedError(closed.message, {
+          host: closed.host,
+          reason: closed.reason,
+          retryAt: closed.retryAt
+        });
+      }
+    };
+    exports2.RateLimitGate = RateLimitGate;
+  }
+});
+
+// ../../packages/keep/dist/exchange.js
+var require_exchange = __commonJS({
+  "../../packages/keep/dist/exchange.js"(exports2) {
+    "use strict";
+    Object.defineProperty(exports2, "__esModule", { value: true });
+    exports2.KeepUnavailableError = void 0;
+    exports2.exchangeCredentials = exchangeCredentials;
+    exports2.hostOf = hostOf;
+    var rate_limited_1 = require_rate_limited();
+    var KeepUnavailableError = class extends Error {
+      code = "SERVICE_UNAVAILABLE";
+      /** Host the exchange was sent to, so a log line names the service that is down. */
+      host;
+      /** The status whatever answered sent, or null when nothing answered at all. */
+      statusCode;
+      constructor(message, detail) {
+        super(message, detail.cause !== void 0 ? { cause: detail.cause } : void 0);
+        this.name = "KeepUnavailableError";
+        this.host = detail.host;
+        this.statusCode = detail.statusCode;
+      }
+    };
+    exports2.KeepUnavailableError = KeepUnavailableError;
+    async function exchangeCredentials(request) {
+      const { manager, url } = request;
+      const host = hostOf(url);
+      let response;
+      try {
+        response = await fetch(url, {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({
+            clientId: request.clientId,
+            clientSecret: request.clientSecret
+          })
+        });
+      } catch (err) {
+        throw new KeepUnavailableError(`[${manager}] could not reach Keep at ${host}: ${describeTransportError(err)}`, { host, statusCode: null, cause: err });
+      }
+      request.onResponse?.(response);
+      const text = await response.text().catch(() => "");
+      let json;
+      try {
+        json = JSON.parse(text);
+      } catch {
+        json = null;
+      }
+      if (response.status >= 500) {
+        const reason = json?.error?.message ?? `${request.exchangeName} failed with status ${response.status}`;
+        throw new KeepUnavailableError(`[${manager}] Keep at ${host} is unavailable (${response.status}): ${reason}`, { host, statusCode: response.status });
+      }
+      if (response.status === 429) {
+        throw request.rateLimit.close({
+          manager,
+          host,
+          reason: json?.error?.message ?? "Too Many Requests",
+          retryAfterMs: (0, rate_limited_1.retryAfterMsOf)(json, response.headers)
+        });
+      }
+      if (json === null) {
+        throw new Error(`[${manager}] Keep at ${host} answered ${response.status} with a body that is not JSON`);
+      }
+      if (!response.ok || !json.success) {
+        const msg = json.error?.message ?? `${request.exchangeName} failed with status ${response.status}`;
+        if (response.status === 401 && request.refused) {
+          throw request.refused(msg);
+        }
+        throw new Error(`[${manager}] ${msg}`);
+      }
+      return json.data;
+    }
+    function hostOf(url) {
+      try {
+        return new URL(url).host;
+      } catch {
+        return url;
+      }
+    }
+    function describeTransportError(err) {
+      if (err instanceof Error) {
+        const cause = err.cause;
+        if (cause instanceof Error && cause.message && cause.message !== err.message) {
+          return `${err.message}: ${cause.message}`;
+        }
+        return err.message;
+      }
+      return String(err);
+    }
+  }
+});
+
+// ../../packages/keep/dist/wait-for-token.js
+var require_wait_for_token = __commonJS({
+  "../../packages/keep/dist/wait-for-token.js"(exports2) {
+    "use strict";
+    Object.defineProperty(exports2, "__esModule", { value: true });
+    exports2.waitForKeepToken = waitForKeepToken;
+    var WAIT_FOR_TOKEN_MAX_MS = 20 * 6e4;
+    var INITIAL_DELAY_MS = 1e3;
+    var MAX_DELAY_MS = 3e4;
+    async function waitForKeepToken(getToken, options) {
+      const logger = options.logger ?? console;
+      const initialDelay = options.initialDelayMs ?? INITIAL_DELAY_MS;
+      const maxDelay = options.maxDelayMs ?? MAX_DELAY_MS;
+      const started = Date.now();
+      const deadline = started + (options.maxWaitMs ?? WAIT_FOR_TOKEN_MAX_MS);
+      const tag = `[${options.label}]`;
+      let delay = initialDelay;
+      let waiting = null;
+      for (; ; ) {
+        let pause;
+        try {
+          const token = await getToken();
+          if (waiting !== null) {
+            logger.info(`${tag} Keep issued a token after ${formatDuration(Date.now() - started)} of waiting`);
+          }
+          return token;
+        } catch (err) {
+          if (isRateLimited(err)) {
+            pause = Math.max(0, err.retryAfterMs);
+            delay = initialDelay;
+            if (waiting !== "rate-limited") {
+              waiting = "rate-limited";
+              logger.warn(`${tag} Keep is rate limiting this client's token exchange (${err.message}). The credentials are not in question; waiting ${formatDuration(pause)} to ask again.`);
+            }
+          } else if (isUnavailable(err)) {
+            pause = delay;
+            delay = Math.min(delay * 2, maxDelay);
+            if (waiting !== "unavailable") {
+              waiting = "unavailable";
+              logger.warn(`${tag} Keep is unavailable (${err.message}); retrying with backoff.`);
+            }
+          } else {
+            throw err;
+          }
+          if (Date.now() + pause > deadline) {
+            logger.error(`${tag} giving up on a Keep token after ${formatDuration(Date.now() - started)}: ${err.message}`);
+            throw err;
+          }
+        }
+        await new Promise((resolve) => setTimeout(resolve, pause));
+      }
+    }
+    function isRateLimited(err) {
+      const e = err;
+      return e?.code === "RATE_LIMITED" && typeof e.retryAfterMs === "number";
+    }
+    function isUnavailable(err) {
+      return err?.code === "SERVICE_UNAVAILABLE";
+    }
+    function formatDuration(ms) {
+      const s = Math.round(ms / 1e3);
+      if (s < 60)
+        return `${s}s`;
+      const m = Math.floor(s / 60);
+      const rest = s % 60;
+      return rest === 0 ? `${m}m` : `${m}m${rest}s`;
+    }
+  }
+});
+
 // ../../packages/keep/dist/service-token.js
 var require_service_token = __commonJS({
   "../../packages/keep/dist/service-token.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.ServiceTokenManager = void 0;
+    var rate_limited_1 = require_rate_limited();
+    var exchange_1 = require_exchange();
+    var wait_for_token_1 = require_wait_for_token();
     var ServiceTokenManager = class {
       token = null;
       expiresAt = 0;
       pendingExchange = null;
+      /** Closed by a 429 until Keep's retry-after: no exchange is sent before it. */
+      rateLimit = new rate_limited_1.RateLimitGate();
       keepApiUrl;
       clientId;
       clientSecret;
@@ -11227,6 +11471,14 @@ var require_service_token = __commonJS({
       /**
        * Returns a valid service token, refreshing if needed.
        * Returns null if Keep credentials are not configured (graceful degradation for local dev).
+       *
+       * Throws `KeepUnavailableError` when Keep did not answer or answered 5xx,
+       * `KeepRateLimitedError` when Keep's exchange limiter locked this client out
+       * (and, until the time Keep named, without asking again), and a plain `Error`
+       * for any other refusal, a 401 above all. A refresh that fails for either
+       * of the first two reasons while the cached token has not yet expired
+       * answers with the cached token instead: the refresh starts 30s early
+       * precisely so a failed one costs nothing.
        */
       async getToken() {
         if (!this.enabled)
@@ -11237,35 +11489,51 @@ var require_service_token = __commonJS({
         if (this.pendingExchange) {
           return this.pendingExchange;
         }
-        this.pendingExchange = this.exchange();
+        this.pendingExchange = this.refresh();
         try {
           return await this.pendingExchange;
         } finally {
           this.pendingExchange = null;
         }
       }
+      /**
+       * `getToken`, for a service that cannot start without a token: waits out a
+       * rate limit (for the time Keep named) and an outage (with backoff), bounded
+       * by `maxWaitMs` (20 minutes by default, one full lockout and change), and
+       * throws a refusal at once. See `waitForKeepToken`.
+       */
+      waitForToken(options) {
+        return (0, wait_for_token_1.waitForKeepToken)(() => this.getToken(), options);
+      }
+      async refresh() {
+        try {
+          this.rateLimit.check();
+          return await this.exchange();
+        } catch (err) {
+          const code = err?.code;
+          if (this.token && Date.now() < this.expiresAt - STALE_MARGIN_MS && (code === "SERVICE_UNAVAILABLE" || code === "RATE_LIMITED")) {
+            return this.token;
+          }
+          throw err;
+        }
+      }
       async exchange() {
-        const url = `${this.keepApiUrl}/api/service-tokens`;
-        const response = await fetch(url, {
-          method: "POST",
-          headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({
-            clientId: this.clientId,
-            clientSecret: this.clientSecret
-          })
+        const data = await (0, exchange_1.exchangeCredentials)({
+          manager: "ServiceTokenManager",
+          url: `${this.keepApiUrl}/api/service-tokens`,
+          clientId: this.clientId,
+          clientSecret: this.clientSecret,
+          rateLimit: this.rateLimit,
+          onResponse: this.onResponse,
+          exchangeName: "Token exchange"
         });
-        this.onResponse?.(response);
-        const json = await response.json();
-        if (!response.ok || !json.success) {
-          const msg = json.error?.message ?? `Token exchange failed with status ${response.status}`;
-          throw new Error(`[ServiceTokenManager] ${msg}`);
-        }
-        this.token = json.data.token;
-        this.expiresAt = Date.now() + json.data.expiresIn * 1e3;
+        this.token = data.token;
+        this.expiresAt = Date.now() + data.expiresIn * 1e3;
         return this.token;
       }
     };
     exports2.ServiceTokenManager = ServiceTokenManager;
+    var STALE_MARGIN_MS = 5e3;
   }
 });
 
@@ -11274,7 +11542,9 @@ var require_agent_token = __commonJS({
   "../../packages/keep/dist/agent-token.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
-    exports2.AgentTokenManager = exports2.KeepUnavailableError = exports2.AgentCredentialRevokedError = void 0;
+    exports2.AgentTokenManager = exports2.AgentCredentialRevokedError = void 0;
+    var rate_limited_1 = require_rate_limited();
+    var exchange_1 = require_exchange();
     var AgentCredentialRevokedError2 = class extends Error {
       code = "PAIRING_REVOKED";
       constructor(message) {
@@ -11283,24 +11553,12 @@ var require_agent_token = __commonJS({
       }
     };
     exports2.AgentCredentialRevokedError = AgentCredentialRevokedError2;
-    var KeepUnavailableError = class extends Error {
-      code = "SERVICE_UNAVAILABLE";
-      /** Host the exchange was sent to, so a log line names the service that is down. */
-      host;
-      /** The status whatever answered sent, or null when nothing answered at all. */
-      statusCode;
-      constructor(message, detail) {
-        super(message, detail.cause !== void 0 ? { cause: detail.cause } : void 0);
-        this.name = "KeepUnavailableError";
-        this.host = detail.host;
-        this.statusCode = detail.statusCode;
-      }
-    };
-    exports2.KeepUnavailableError = KeepUnavailableError;
     var AgentTokenManager = class {
       token = null;
       expiresAt = 0;
       pendingExchange = null;
+      /** Closed by a 429 until Keep's retry-after: no exchange is sent before it. */
+      rateLimit = new rate_limited_1.RateLimitGate();
       keepApiUrl;
       clientId;
       clientSecret;
@@ -11322,6 +11580,7 @@ var require_agent_token = __commonJS({
         if (this.pendingExchange) {
           return this.pendingExchange;
         }
+        this.rateLimit.check();
         this.pendingExchange = this.exchange();
         try {
           return await this.pendingExchange;
@@ -11330,66 +11589,22 @@ var require_agent_token = __commonJS({
         }
       }
       async exchange() {
-        const url = `${this.keepApiUrl}/api/agent-tokens`;
-        const host = hostOf(url);
-        let response;
-        try {
-          response = await fetch(url, {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({
-              clientId: this.clientId,
-              clientSecret: this.clientSecret
-            })
-          });
-        } catch (err) {
-          throw new KeepUnavailableError(`[AgentTokenManager] could not reach Keep at ${host}: ${describeTransportError(err)}`, { host, statusCode: null, cause: err });
-        }
-        this.onResponse?.(response);
-        const text = await response.text().catch(() => "");
-        let json;
-        try {
-          json = JSON.parse(text);
-        } catch {
-          json = null;
-        }
-        if (response.status >= 500) {
-          const reason = json?.error?.message ?? `Agent token exchange failed with status ${response.status}`;
-          throw new KeepUnavailableError(`[AgentTokenManager] Keep at ${host} is unavailable (${response.status}): ${reason}`, { host, statusCode: response.status });
-        }
-        if (json === null) {
-          throw new Error(`[AgentTokenManager] Keep at ${host} answered ${response.status} with a body that is not JSON`);
-        }
-        if (!response.ok || !json.success) {
-          const msg = json.error?.message ?? `Agent token exchange failed with status ${response.status}`;
-          if (response.status === 401) {
-            throw new AgentCredentialRevokedError2(`[AgentTokenManager] Keep refused this agent's credentials (401): the pairing was revoked or the secret rotated. Re-pair to continue.`);
-          }
-          throw new Error(`[AgentTokenManager] ${msg}`);
-        }
-        this.token = json.data.token;
-        this.expiresAt = Date.now() + json.data.expiresIn * 1e3;
+        const data = await (0, exchange_1.exchangeCredentials)({
+          manager: "AgentTokenManager",
+          url: `${this.keepApiUrl}/api/agent-tokens`,
+          clientId: this.clientId,
+          clientSecret: this.clientSecret,
+          rateLimit: this.rateLimit,
+          onResponse: this.onResponse,
+          exchangeName: "Agent token exchange",
+          refused: () => new AgentCredentialRevokedError2(`[AgentTokenManager] Keep refused this agent's credentials (401): the pairing was revoked or the secret rotated. Re-pair to continue.`)
+        });
+        this.token = data.token;
+        this.expiresAt = Date.now() + data.expiresIn * 1e3;
         return this.token;
       }
     };
     exports2.AgentTokenManager = AgentTokenManager;
-    function hostOf(url) {
-      try {
-        return new URL(url).host;
-      } catch {
-        return url;
-      }
-    }
-    function describeTransportError(err) {
-      if (err instanceof Error) {
-        const cause = err.cause;
-        if (cause instanceof Error && cause.message && cause.message !== err.message) {
-          return `${err.message}: ${cause.message}`;
-        }
-        return err.message;
-      }
-      return String(err);
-    }
   }
 });
 
@@ -11398,7 +11613,7 @@ var require_dist2 = __commonJS({
   "../../packages/keep/dist/index.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
-    exports2.KeepUnavailableError = exports2.AgentCredentialRevokedError = exports2.AgentTokenManager = exports2.ServiceTokenManager = exports2.KeepClient = void 0;
+    exports2.waitForKeepToken = exports2.KeepRateLimitedError = exports2.KeepUnavailableError = exports2.AgentCredentialRevokedError = exports2.AgentTokenManager = exports2.ServiceTokenManager = exports2.KeepClient = void 0;
     var client_1 = require_client();
     Object.defineProperty(exports2, "KeepClient", { enumerable: true, get: function() {
       return client_1.KeepClient;
@@ -11414,8 +11629,17 @@ var require_dist2 = __commonJS({
     Object.defineProperty(exports2, "AgentCredentialRevokedError", { enumerable: true, get: function() {
       return agent_token_1.AgentCredentialRevokedError;
     } });
+    var exchange_1 = require_exchange();
     Object.defineProperty(exports2, "KeepUnavailableError", { enumerable: true, get: function() {
-      return agent_token_1.KeepUnavailableError;
+      return exchange_1.KeepUnavailableError;
+    } });
+    var rate_limited_1 = require_rate_limited();
+    Object.defineProperty(exports2, "KeepRateLimitedError", { enumerable: true, get: function() {
+      return rate_limited_1.KeepRateLimitedError;
+    } });
+    var wait_for_token_1 = require_wait_for_token();
+    Object.defineProperty(exports2, "waitForKeepToken", { enumerable: true, get: function() {
+      return wait_for_token_1.waitForKeepToken;
     } });
   }
 });
@@ -12282,6 +12506,10 @@ var require_checks = __commonJS({
           return skip("pairing-live", "no readable pairing; see pairing-present");
         }
         const minted = await probes.mintAgentToken(stored.value);
+        if (!minted.ok && minted.rateLimited) {
+          const at = formatInstant(minted.rateLimited.retryAt);
+          return warn("pairing-live", `Keep is rate limiting this pairing's token exchange (429), so the pairing could not be checked; it asked for no retry before ${at}`, subject.phase === "run" ? `nothing to do; \`run\` waits until ${at}, asks once, and starts claiming when Keep issues a token` : `wait until ${at}, then run doctor again`);
+        }
         if (!minted.ok && minted.unavailable) {
           return warn("pairing-live", `Keep did not answer, so the pairing could not be checked: ${oneLine(minted.error).replace(/^\[AgentTokenManager\]\s*/, "")}`, subject.phase === "run" ? "nothing to do if this is a deployment window; `run` waits for Keep and starts claiming when it answers" : `check ${stored.value.keepApiUrl} is reachable, then run doctor again`);
         }
@@ -12339,6 +12567,9 @@ var require_checks = __commonJS({
         const pairing = stored.value;
         return withConfig("jobs-reachable", probes, async (config) => {
           const result = await probes.reachJobs(config, pairing);
+          if (!result.ok && result.rateLimited) {
+            return warn("jobs-reachable", `${config.auth.jobsApiUrl} was not asked: Keep is rate limiting the token exchange until ${formatInstant(result.rateLimited.retryAt)}`, "see pairing-live; the run loop asks once Keep issues a token");
+          }
           if (!result.ok) {
             return warn("jobs-reachable", `${config.auth.jobsApiUrl} did not answer: ${oneLine(result.error)}`, "nothing to do if this is a deployment window; the run loop will keep trying");
           }
@@ -12620,6 +12851,9 @@ var require_checks = __commonJS({
     function osTmpDirOf(probes) {
       return probes.osTmpDir();
     }
+    function formatInstant(epochMs) {
+      return new Date(epochMs).toISOString().replace(/\.\d{3}Z$/, "Z");
+    }
     function oneLine(message) {
       return message.split("\n").map((line) => line.trim()).filter((line) => line.length > 0).join(" ");
     }
@@ -20320,6 +20554,20 @@ var require_subscribe = __commonJS({
         return true;
       return err instanceof TypeError && err.message === "fetch failed";
     }
+    function isRateLimited(err) {
+      return err?.code === "RATE_LIMITED";
+    }
+    function rateLimitWaitMs(err) {
+      const e = err ?? {};
+      if (typeof e.retryAfterMs === "number" && Number.isFinite(e.retryAfterMs))
+        return Math.max(0, e.retryAfterMs);
+      if (typeof e.retryAt === "number" && Number.isFinite(e.retryAt))
+        return Math.max(0, e.retryAt - Date.now());
+      return 0;
+    }
+    function formatInstant(epochMs) {
+      return new Date(epochMs).toISOString().replace(/\.\d{3}Z$/, "Z");
+    }
     function describeOutage(err) {
       const e = err ?? {};
       if (typeof e.host === "string" && typeof e.statusCode === "number") {
@@ -20386,11 +20634,33 @@ var require_subscribe = __commonJS({
       let outcome = newOutcome();
       let wakeStartup = null;
       let unavailableSince = null;
+      let rateLimited = null;
       function noteReachable() {
         if (unavailableSince !== null) {
           logger.info(`Sift is reachable again after ${formatDuration(Date.now() - unavailableSince)}`);
           unavailableSince = null;
         }
+        if (rateLimited !== null) {
+          logger.info(`${rateLimited.host} stopped rate limiting the Sifter after ${formatDuration(Date.now() - rateLimited.since)}`);
+          rateLimited = null;
+        }
+      }
+      function noteRateLimited(err) {
+        const waitMs = Math.max(pollIntervalMs, rateLimitWaitMs(err));
+        const e = err ?? {};
+        const host = typeof e.host === "string" ? e.host : "the service";
+        if (rateLimited === null) {
+          rateLimited = { since: Date.now(), host };
+          const reason = typeof e.reason === "string" ? `: ${e.reason}` : "";
+          logger.warn(`${host} is rate limiting the Sifter (429${reason}). It will wait and try again at ${formatInstant(Date.now() + waitMs)} (in ${formatDuration(waitMs)}).`);
+        } else {
+          logger.debug?.("still rate limited", {
+            host,
+            rateLimitedFor: formatDuration(Date.now() - rateLimited.since),
+            nextAttemptIn: formatDuration(waitMs)
+          });
+        }
+        return waitMs;
       }
       function noteUnavailable(err) {
         if (unavailableSince === null) {
@@ -20420,16 +20690,22 @@ var require_subscribe = __commonJS({
       }
       async function untilAvailable(call, mySession) {
         for (; ; ) {
+          let pause = pollIntervalMs;
           try {
             const value = await call();
             noteReachable();
             return value;
           } catch (err) {
-            if (isFatal(err) || !isUnavailable(err))
+            if (isFatal(err))
+              throw err;
+            if (isRateLimited(err))
+              pause = noteRateLimited(err);
+            else if (isUnavailable(err))
+              noteUnavailable(err);
+            else
               throw err;
-            noteUnavailable(err);
           }
-          await startupPause(pollIntervalMs);
+          await startupPause(pause);
           if (stopped || mySession !== session)
             return STOPPED;
         }
@@ -20515,6 +20791,7 @@ var require_subscribe = __commonJS({
       async function poll(mySession) {
         if (stopped || mySession !== session)
           return;
+        let nextPollMs = pollIntervalMs;
         if (inflight >= maxConcurrency) {
           schedulePoll(pollIntervalMs);
           return;
@@ -20557,14 +20834,16 @@ var require_subscribe = __commonJS({
             void fail(err);
             return;
           }
-          if (isUnavailable(err)) {
+          if (isRateLimited(err)) {
+            nextPollMs = noteRateLimited(err);
+          } else if (isUnavailable(err)) {
             noteUnavailable(err);
           } else {
             logger.error("poll failed", { error: err instanceof Error ? err.message : String(err) });
           }
         }
         if (!stopped && mySession === session)
-          schedulePoll(pollIntervalMs);
+          schedulePoll(nextPollMs);
       }
       async function runOnce(available) {
         let dispatch;
@@ -21035,6 +21314,21 @@ var require_probes = __commonJS({
           packageName: options.brand.product.packageName
         });
       };
+      const managers = /* @__PURE__ */ new Map();
+      const tokensFor = (pairing) => {
+        const key = JSON.stringify([pairing.keepApiUrl, pairing.credentials.clientId, pairing.credentials.clientSecret]);
+        let tokens = managers.get(key);
+        if (tokens === void 0) {
+          tokens = new keep_1.AgentTokenManager({
+            keepApiUrl: pairing.keepApiUrl,
+            clientId: pairing.credentials.clientId,
+            clientSecret: pairing.credentials.clientSecret,
+            onResponse: enforceFloor
+          });
+          managers.set(key, tokens);
+        }
+        return tokens;
+      };
       let pairingRead;
       let configRead;
       let scaffoldedRead;
@@ -21163,12 +21457,7 @@ var require_probes = __commonJS({
           return scaffoldedRead;
         },
         async mintAgentToken(pairing) {
-          const tokens = new keep_1.AgentTokenManager({
-            keepApiUrl: pairing.keepApiUrl,
-            clientId: pairing.credentials.clientId,
-            clientSecret: pairing.credentials.clientSecret,
-            onResponse: enforceFloor
-          });
+          const tokens = tokensFor(pairing);
           try {
             const token = await tokens.getToken();
             if (token === null) {
@@ -21178,16 +21467,14 @@ var require_probes = __commonJS({
           } catch (err) {
             if (isRevoked(err) || isUpgradeRequired(err))
               throw err;
+            if (isKeepRateLimited(err)) {
+              return { ok: false, error: describe(err), rateLimited: { retryAt: err.retryAt } };
+            }
             return { ok: false, error: describe(err), ...isKeepUnavailable(err) ? { unavailable: true } : {} };
           }
         },
         async reachJobs(config, pairing) {
-          const tokens = new keep_1.AgentTokenManager({
-            keepApiUrl: pairing.keepApiUrl,
-            clientId: pairing.credentials.clientId,
-            clientSecret: pairing.credentials.clientSecret,
-            onResponse: enforceFloor
-          });
+          const tokens = tokensFor(pairing);
           const client = new jobs_1.JobsClient({
             baseUrl: config.auth.jobsApiUrl,
             ...config.auth.apiKey ? { apiKey: config.auth.apiKey } : {},
@@ -21204,6 +21491,9 @@ var require_probes = __commonJS({
           } catch (err) {
             if (isRevoked(err) || isUpgradeRequired(err))
               throw err;
+            if (isKeepRateLimited(err)) {
+              return { ok: false, error: describe(err), rateLimited: { retryAt: err.retryAt } };
+            }
             return { ok: false, error: describe(err) };
           }
         },
@@ -21299,6 +21589,9 @@ var require_probes = __commonJS({
     function isKeepUnavailable(err) {
       return err instanceof Error && err.name === "KeepUnavailableError";
     }
+    function isKeepRateLimited(err) {
+      return err instanceof Error && err.name === "KeepRateLimitedError";
+    }
     function isUpgradeRequired(err) {
       return err instanceof Error && err.name === "UpgradeRequiredError";
     }
@@ -25442,6 +25735,12 @@ var require_shuttle = __commonJS({
     function isKeepUnavailable(err) {
       return err instanceof Error && err.name === "KeepUnavailableError";
     }
+    function isKeepRateLimited(err) {
+      return err instanceof Error && err.name === "KeepRateLimitedError";
+    }
+    function formatInstant(epochMs) {
+      return new Date(epochMs).toISOString().replace(/\.\d{3}Z$/, "Z");
+    }
     async function runOutputGates(dispatch, response, executor, signal, ctx) {
       let spec;
       try {
@@ -25897,30 +26196,44 @@ var require_shuttle = __commonJS({
     }
     async function mintWaitingOutKeep(mint, brand, options) {
       const maxDelay = options.retryMaxMs ?? KEEP_RETRY_MAX_MS;
-      let delay = options.retryInitialMs ?? KEEP_RETRY_INITIAL_MS;
-      let unavailableSince = null;
+      const initialDelay = options.retryInitialMs ?? KEEP_RETRY_INITIAL_MS;
+      let delay = initialDelay;
+      let waiting = null;
       for (; ; ) {
+        let pause;
         try {
           const token = await mint();
-          if (unavailableSince !== null) {
-            options.logger?.info(`Keep is reachable again after ${formatDuration(Date.now() - unavailableSince)}`);
+          if (waiting?.kind === "unavailable") {
+            options.logger?.info(`Keep is reachable again after ${formatDuration(Date.now() - waiting.since)}`);
+          } else if (waiting?.kind === "rate-limited") {
+            options.logger?.info(`Keep issued a token; the rate limit lifted after ${formatDuration(Date.now() - waiting.since)}`);
           }
           return token;
         } catch (err) {
-          if (!isKeepUnavailable(err))
+          if (isKeepRateLimited(err)) {
+            if (waiting?.kind !== "rate-limited") {
+              waiting = { kind: "rate-limited", since: Date.now() };
+              options.logger?.warn(`Keep is rate limiting this ${brand.product.title}'s token exchange (${err.host ?? "Keep"} answered 429: ${err.reason ?? err.message}). The credentials are not in question; the ${brand.product.title} will wait and ask again at ${formatInstant(err.retryAt)} (in ${formatDuration(err.retryAfterMs)}), and start when Keep issues a token.`);
+            }
+            pause = err.retryAfterMs;
+            delay = initialDelay;
+          } else if (isKeepUnavailable(err)) {
+            if (waiting?.kind !== "unavailable") {
+              waiting = { kind: "unavailable", since: Date.now() };
+              const e = err;
+              const what = typeof e.statusCode === "number" ? `${e.host ?? "Keep"} answered ${e.statusCode}` : `${e.host ?? "Keep"} did not answer`;
+              options.logger?.warn(`Keep is unavailable (${what}). This is typical during a deployment window; the ${brand.product.title} will wait for it and start when it answers.`);
+            }
+            pause = delay;
+            delay = Math.min(delay * 2, maxDelay);
+          } else {
             throw err;
-          if (unavailableSince === null) {
-            unavailableSince = Date.now();
-            const e = err;
-            const what = typeof e.statusCode === "number" ? `${e.host ?? "Keep"} answered ${e.statusCode}` : `${e.host ?? "Keep"} did not answer`;
-            options.logger?.warn(`Keep is unavailable (${what}). This is typical during a deployment window; the ${brand.product.title} will wait for it and start when it answers.`);
           }
         }
-        await abortableSleep(delay, options.signal);
+        await abortableSleep(pause, options.signal);
         if (options.signal?.aborted) {
           throw options.signal.reason instanceof Error ? options.signal.reason : new Error("stopped while waiting for Keep");
         }
-        delay = Math.min(delay * 2, maxDelay);
       }
     }
   }
@@ -26548,7 +26861,7 @@ var import_path = require("path");
 var import_promises = require("fs/promises");
 var import_yaml = __toESM(require_dist3());
 var import_shuttle = __toESM(require_dist5());
-var buildVersion = true ? "0.30.9" : pkg.version;
+var buildVersion = true ? "0.30.11" : pkg.version;
 var sifterBrand = {
   product: {
     id: "sifter",