@whittlelabs/sifter 0.30.9 → 0.30.10

package/README.md

@@ -82,6 +82,7 @@ The Sifter subscribes to your job pool and waits for review jobs. Leave it runni
 A running Sifter deals with Sift or `claude` going away on its own, and says what it is doing:
 
 - **Sift can't be reached** (a deploy, a network blip): it logs one warning, keeps trying, and says when Sift is back. This holds at startup too, so starting it during an outage is fine; it begins claiming once Sift answers.
+- **Sift asks it to slow down** (too many sign-in attempts from this machine in a short time, which Sift answers by locking it out for a while): this is not a problem with the pairing, so it logs one warning saying when it will try again, waits until then without asking sooner, and says when it is back. `run` started during this waits the same way instead of failing; `doctor` warns and says when to try again.
 - **`claude` stops working** (signed out, out of plan, or uninstalled): the review it was running fails with Claude's own message, and the Sifter stops taking new reviews so the rest of your queue waits for you instead of failing. It logs one line saying what to fix, checks again by itself (after 15 seconds at first, backing off to every 5 minutes), and starts claiming again once `claude` works. A check costs nothing while you are signed out, and one short turn once you are back.
 - **Sift needs a newer Sifter**: it finishes the review in hand, installs the newest version with `npm install -g`, and restarts itself. If the install fails, it exits with code 2 and prints the command to run.
 - **The pairing is revoked** (removed in Sift, or replaced by a newer pairing): it exits with code 3 and shows the two ways to pair again. A review it was running goes back to the queue for another Sifter.

package/bin.js

@@ -11199,16 +11199,97 @@ var require_client = __commonJS({
   }
 });
 
+// ../../packages/keep/dist/rate-limited.js
+var require_rate_limited = __commonJS({
+  "../../packages/keep/dist/rate-limited.js"(exports2) {
+    "use strict";
+    Object.defineProperty(exports2, "__esModule", { value: true });
+    exports2.RateLimitGate = exports2.RATE_LIMIT_DEFAULT_MS = exports2.RATE_LIMIT_CAP_MS = exports2.RATE_LIMIT_FLOOR_MS = exports2.KeepRateLimitedError = void 0;
+    exports2.retryAfterMsOf = retryAfterMsOf;
+    var KeepRateLimitedError = class extends Error {
+      code = "RATE_LIMITED";
+      /** Host the exchange was sent to. */
+      host;
+      /** Keep's own words for the refusal ("Too many attempts. Please try again later."). */
+      reason;
+      /** How long from now to wait before asking again (ms), already floored and capped. */
+      retryAfterMs;
+      /** When the next exchange may be attempted (epoch ms). */
+      retryAt;
+      constructor(message, detail) {
+        super(message);
+        this.name = "KeepRateLimitedError";
+        this.host = detail.host;
+        this.reason = detail.reason;
+        this.retryAt = detail.retryAt;
+        this.retryAfterMs = Math.max(0, detail.retryAt - Date.now());
+      }
+    };
+    exports2.KeepRateLimitedError = KeepRateLimitedError;
+    exports2.RATE_LIMIT_FLOOR_MS = 5e3;
+    exports2.RATE_LIMIT_CAP_MS = 15 * 6e4;
+    exports2.RATE_LIMIT_DEFAULT_MS = 6e4;
+    function retryAfterMsOf(body, headers, now = Date.now()) {
+      return clamp(bodyRetryAfter(body) ?? headerRetryAfter(headers, now) ?? exports2.RATE_LIMIT_DEFAULT_MS);
+    }
+    function bodyRetryAfter(body) {
+      const value = body?.error?.retryAfter;
+      return typeof value === "number" && Number.isFinite(value) ? value * 1e3 : null;
+    }
+    function headerRetryAfter(headers, now) {
+      const raw = headers?.get("retry-after")?.trim();
+      if (!raw)
+        return null;
+      if (/^\d+$/.test(raw))
+        return Number(raw) * 1e3;
+      const at = Date.parse(raw);
+      return Number.isNaN(at) ? null : at - now;
+    }
+    function clamp(ms) {
+      return Math.min(exports2.RATE_LIMIT_CAP_MS, Math.max(exports2.RATE_LIMIT_FLOOR_MS, ms));
+    }
+    var RateLimitGate = class {
+      closed = null;
+      /** Record a 429 and return the error to throw for it. */
+      close(detail) {
+        const retryAt = Date.now() + detail.retryAfterMs;
+        const when = new Date(retryAt).toISOString().replace(/\.\d{3}Z$/, "Z");
+        this.closed = new KeepRateLimitedError(`[${detail.manager}] Keep at ${detail.host} is rate limiting this client's token exchange (429: ${detail.reason}); not asking again before ${when}`, { host: detail.host, reason: detail.reason, retryAt });
+        return this.closed;
+      }
+      /** Throws the rate-limited error while the gate is closed; a no-op once the time has passed. */
+      check() {
+        const closed = this.closed;
+        if (closed === null)
+          return;
+        if (Date.now() >= closed.retryAt) {
+          this.closed = null;
+          return;
+        }
+        throw new KeepRateLimitedError(closed.message, {
+          host: closed.host,
+          reason: closed.reason,
+          retryAt: closed.retryAt
+        });
+      }
+    };
+    exports2.RateLimitGate = RateLimitGate;
+  }
+});
+
 // ../../packages/keep/dist/service-token.js
 var require_service_token = __commonJS({
   "../../packages/keep/dist/service-token.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.ServiceTokenManager = void 0;
+    var rate_limited_1 = require_rate_limited();
     var ServiceTokenManager = class {
       token = null;
       expiresAt = 0;
       pendingExchange = null;
+      /** Closed by a 429 until Keep's retry-after: no exchange is sent before it. */
+      rateLimit = new rate_limited_1.RateLimitGate();
       keepApiUrl;
       clientId;
       clientSecret;
@@ -11237,6 +11318,7 @@ var require_service_token = __commonJS({
         if (this.pendingExchange) {
           return this.pendingExchange;
         }
+        this.rateLimit.check();
         this.pendingExchange = this.exchange();
         try {
           return await this.pendingExchange;
@@ -11255,6 +11337,15 @@ var require_service_token = __commonJS({
           })
         });
         this.onResponse?.(response);
+        if (response.status === 429) {
+          const body = await response.json().catch(() => null);
+          throw this.rateLimit.close({
+            manager: "ServiceTokenManager",
+            host: hostOf(url),
+            reason: body?.error?.message ?? "Too Many Requests",
+            retryAfterMs: (0, rate_limited_1.retryAfterMsOf)(body, response.headers)
+          });
+        }
         const json = await response.json();
         if (!response.ok || !json.success) {
           const msg = json.error?.message ?? `Token exchange failed with status ${response.status}`;
@@ -11266,6 +11357,13 @@ var require_service_token = __commonJS({
       }
     };
     exports2.ServiceTokenManager = ServiceTokenManager;
+    function hostOf(url) {
+      try {
+        return new URL(url).host;
+      } catch {
+        return url;
+      }
+    }
   }
 });
 
@@ -11275,6 +11373,7 @@ var require_agent_token = __commonJS({
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.AgentTokenManager = exports2.KeepUnavailableError = exports2.AgentCredentialRevokedError = void 0;
+    var rate_limited_1 = require_rate_limited();
     var AgentCredentialRevokedError2 = class extends Error {
       code = "PAIRING_REVOKED";
       constructor(message) {
@@ -11301,6 +11400,8 @@ var require_agent_token = __commonJS({
       token = null;
       expiresAt = 0;
       pendingExchange = null;
+      /** Closed by a 429 until Keep's retry-after: no exchange is sent before it. */
+      rateLimit = new rate_limited_1.RateLimitGate();
       keepApiUrl;
       clientId;
       clientSecret;
@@ -11322,6 +11423,7 @@ var require_agent_token = __commonJS({
         if (this.pendingExchange) {
           return this.pendingExchange;
         }
+        this.rateLimit.check();
         this.pendingExchange = this.exchange();
         try {
           return await this.pendingExchange;
@@ -11357,6 +11459,14 @@ var require_agent_token = __commonJS({
           const reason = json?.error?.message ?? `Agent token exchange failed with status ${response.status}`;
           throw new KeepUnavailableError(`[AgentTokenManager] Keep at ${host} is unavailable (${response.status}): ${reason}`, { host, statusCode: response.status });
         }
+        if (response.status === 429) {
+          throw this.rateLimit.close({
+            manager: "AgentTokenManager",
+            host,
+            reason: json?.error?.message ?? "Too Many Requests",
+            retryAfterMs: (0, rate_limited_1.retryAfterMsOf)(json, response.headers)
+          });
+        }
         if (json === null) {
           throw new Error(`[AgentTokenManager] Keep at ${host} answered ${response.status} with a body that is not JSON`);
         }
@@ -11398,7 +11508,7 @@ var require_dist2 = __commonJS({
   "../../packages/keep/dist/index.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
-    exports2.KeepUnavailableError = exports2.AgentCredentialRevokedError = exports2.AgentTokenManager = exports2.ServiceTokenManager = exports2.KeepClient = void 0;
+    exports2.KeepRateLimitedError = exports2.KeepUnavailableError = exports2.AgentCredentialRevokedError = exports2.AgentTokenManager = exports2.ServiceTokenManager = exports2.KeepClient = void 0;
     var client_1 = require_client();
     Object.defineProperty(exports2, "KeepClient", { enumerable: true, get: function() {
       return client_1.KeepClient;
@@ -11417,6 +11527,10 @@ var require_dist2 = __commonJS({
     Object.defineProperty(exports2, "KeepUnavailableError", { enumerable: true, get: function() {
       return agent_token_1.KeepUnavailableError;
     } });
+    var rate_limited_1 = require_rate_limited();
+    Object.defineProperty(exports2, "KeepRateLimitedError", { enumerable: true, get: function() {
+      return rate_limited_1.KeepRateLimitedError;
+    } });
   }
 });
 
@@ -12282,6 +12396,10 @@ var require_checks = __commonJS({
           return skip("pairing-live", "no readable pairing; see pairing-present");
         }
         const minted = await probes.mintAgentToken(stored.value);
+        if (!minted.ok && minted.rateLimited) {
+          const at = formatInstant(minted.rateLimited.retryAt);
+          return warn("pairing-live", `Keep is rate limiting this pairing's token exchange (429), so the pairing could not be checked; it asked for no retry before ${at}`, subject.phase === "run" ? `nothing to do; \`run\` waits until ${at}, asks once, and starts claiming when Keep issues a token` : `wait until ${at}, then run doctor again`);
+        }
         if (!minted.ok && minted.unavailable) {
           return warn("pairing-live", `Keep did not answer, so the pairing could not be checked: ${oneLine(minted.error).replace(/^\[AgentTokenManager\]\s*/, "")}`, subject.phase === "run" ? "nothing to do if this is a deployment window; `run` waits for Keep and starts claiming when it answers" : `check ${stored.value.keepApiUrl} is reachable, then run doctor again`);
         }
@@ -12339,6 +12457,9 @@ var require_checks = __commonJS({
         const pairing = stored.value;
         return withConfig("jobs-reachable", probes, async (config) => {
           const result = await probes.reachJobs(config, pairing);
+          if (!result.ok && result.rateLimited) {
+            return warn("jobs-reachable", `${config.auth.jobsApiUrl} was not asked: Keep is rate limiting the token exchange until ${formatInstant(result.rateLimited.retryAt)}`, "see pairing-live; the run loop asks once Keep issues a token");
+          }
           if (!result.ok) {
             return warn("jobs-reachable", `${config.auth.jobsApiUrl} did not answer: ${oneLine(result.error)}`, "nothing to do if this is a deployment window; the run loop will keep trying");
           }
@@ -12620,6 +12741,9 @@ var require_checks = __commonJS({
     function osTmpDirOf(probes) {
       return probes.osTmpDir();
     }
+    function formatInstant(epochMs) {
+      return new Date(epochMs).toISOString().replace(/\.\d{3}Z$/, "Z");
+    }
     function oneLine(message) {
       return message.split("\n").map((line) => line.trim()).filter((line) => line.length > 0).join(" ");
     }
@@ -20320,6 +20444,20 @@ var require_subscribe = __commonJS({
         return true;
       return err instanceof TypeError && err.message === "fetch failed";
     }
+    function isRateLimited(err) {
+      return err?.code === "RATE_LIMITED";
+    }
+    function rateLimitWaitMs(err) {
+      const e = err ?? {};
+      if (typeof e.retryAfterMs === "number" && Number.isFinite(e.retryAfterMs))
+        return Math.max(0, e.retryAfterMs);
+      if (typeof e.retryAt === "number" && Number.isFinite(e.retryAt))
+        return Math.max(0, e.retryAt - Date.now());
+      return 0;
+    }
+    function formatInstant(epochMs) {
+      return new Date(epochMs).toISOString().replace(/\.\d{3}Z$/, "Z");
+    }
     function describeOutage(err) {
       const e = err ?? {};
       if (typeof e.host === "string" && typeof e.statusCode === "number") {
@@ -20386,11 +20524,33 @@ var require_subscribe = __commonJS({
       let outcome = newOutcome();
       let wakeStartup = null;
       let unavailableSince = null;
+      let rateLimited = null;
       function noteReachable() {
         if (unavailableSince !== null) {
           logger.info(`Sift is reachable again after ${formatDuration(Date.now() - unavailableSince)}`);
           unavailableSince = null;
         }
+        if (rateLimited !== null) {
+          logger.info(`${rateLimited.host} stopped rate limiting the Sifter after ${formatDuration(Date.now() - rateLimited.since)}`);
+          rateLimited = null;
+        }
+      }
+      function noteRateLimited(err) {
+        const waitMs = Math.max(pollIntervalMs, rateLimitWaitMs(err));
+        const e = err ?? {};
+        const host = typeof e.host === "string" ? e.host : "the service";
+        if (rateLimited === null) {
+          rateLimited = { since: Date.now(), host };
+          const reason = typeof e.reason === "string" ? `: ${e.reason}` : "";
+          logger.warn(`${host} is rate limiting the Sifter (429${reason}). It will wait and try again at ${formatInstant(Date.now() + waitMs)} (in ${formatDuration(waitMs)}).`);
+        } else {
+          logger.debug?.("still rate limited", {
+            host,
+            rateLimitedFor: formatDuration(Date.now() - rateLimited.since),
+            nextAttemptIn: formatDuration(waitMs)
+          });
+        }
+        return waitMs;
       }
       function noteUnavailable(err) {
         if (unavailableSince === null) {
@@ -20420,16 +20580,22 @@ var require_subscribe = __commonJS({
       }
       async function untilAvailable(call, mySession) {
         for (; ; ) {
+          let pause = pollIntervalMs;
           try {
             const value = await call();
             noteReachable();
             return value;
           } catch (err) {
-            if (isFatal(err) || !isUnavailable(err))
+            if (isFatal(err))
+              throw err;
+            if (isRateLimited(err))
+              pause = noteRateLimited(err);
+            else if (isUnavailable(err))
+              noteUnavailable(err);
+            else
               throw err;
-            noteUnavailable(err);
           }
-          await startupPause(pollIntervalMs);
+          await startupPause(pause);
           if (stopped || mySession !== session)
             return STOPPED;
         }
@@ -20515,6 +20681,7 @@ var require_subscribe = __commonJS({
       async function poll(mySession) {
         if (stopped || mySession !== session)
           return;
+        let nextPollMs = pollIntervalMs;
         if (inflight >= maxConcurrency) {
           schedulePoll(pollIntervalMs);
           return;
@@ -20557,14 +20724,16 @@ var require_subscribe = __commonJS({
             void fail(err);
             return;
           }
-          if (isUnavailable(err)) {
+          if (isRateLimited(err)) {
+            nextPollMs = noteRateLimited(err);
+          } else if (isUnavailable(err)) {
             noteUnavailable(err);
           } else {
             logger.error("poll failed", { error: err instanceof Error ? err.message : String(err) });
           }
         }
         if (!stopped && mySession === session)
-          schedulePoll(pollIntervalMs);
+          schedulePoll(nextPollMs);
       }
       async function runOnce(available) {
         let dispatch;
@@ -21035,6 +21204,21 @@ var require_probes = __commonJS({
           packageName: options.brand.product.packageName
         });
       };
+      const managers = /* @__PURE__ */ new Map();
+      const tokensFor = (pairing) => {
+        const key = JSON.stringify([pairing.keepApiUrl, pairing.credentials.clientId, pairing.credentials.clientSecret]);
+        let tokens = managers.get(key);
+        if (tokens === void 0) {
+          tokens = new keep_1.AgentTokenManager({
+            keepApiUrl: pairing.keepApiUrl,
+            clientId: pairing.credentials.clientId,
+            clientSecret: pairing.credentials.clientSecret,
+            onResponse: enforceFloor
+          });
+          managers.set(key, tokens);
+        }
+        return tokens;
+      };
       let pairingRead;
       let configRead;
       let scaffoldedRead;
@@ -21163,12 +21347,7 @@ var require_probes = __commonJS({
           return scaffoldedRead;
         },
         async mintAgentToken(pairing) {
-          const tokens = new keep_1.AgentTokenManager({
-            keepApiUrl: pairing.keepApiUrl,
-            clientId: pairing.credentials.clientId,
-            clientSecret: pairing.credentials.clientSecret,
-            onResponse: enforceFloor
-          });
+          const tokens = tokensFor(pairing);
           try {
             const token = await tokens.getToken();
             if (token === null) {
@@ -21178,16 +21357,14 @@ var require_probes = __commonJS({
           } catch (err) {
             if (isRevoked(err) || isUpgradeRequired(err))
               throw err;
+            if (isKeepRateLimited(err)) {
+              return { ok: false, error: describe(err), rateLimited: { retryAt: err.retryAt } };
+            }
             return { ok: false, error: describe(err), ...isKeepUnavailable(err) ? { unavailable: true } : {} };
           }
         },
         async reachJobs(config, pairing) {
-          const tokens = new keep_1.AgentTokenManager({
-            keepApiUrl: pairing.keepApiUrl,
-            clientId: pairing.credentials.clientId,
-            clientSecret: pairing.credentials.clientSecret,
-            onResponse: enforceFloor
-          });
+          const tokens = tokensFor(pairing);
           const client = new jobs_1.JobsClient({
             baseUrl: config.auth.jobsApiUrl,
             ...config.auth.apiKey ? { apiKey: config.auth.apiKey } : {},
@@ -21204,6 +21381,9 @@ var require_probes = __commonJS({
           } catch (err) {
             if (isRevoked(err) || isUpgradeRequired(err))
               throw err;
+            if (isKeepRateLimited(err)) {
+              return { ok: false, error: describe(err), rateLimited: { retryAt: err.retryAt } };
+            }
             return { ok: false, error: describe(err) };
           }
         },
@@ -21299,6 +21479,9 @@ var require_probes = __commonJS({
     function isKeepUnavailable(err) {
       return err instanceof Error && err.name === "KeepUnavailableError";
     }
+    function isKeepRateLimited(err) {
+      return err instanceof Error && err.name === "KeepRateLimitedError";
+    }
     function isUpgradeRequired(err) {
       return err instanceof Error && err.name === "UpgradeRequiredError";
     }
@@ -25442,6 +25625,12 @@ var require_shuttle = __commonJS({
     function isKeepUnavailable(err) {
       return err instanceof Error && err.name === "KeepUnavailableError";
     }
+    function isKeepRateLimited(err) {
+      return err instanceof Error && err.name === "KeepRateLimitedError";
+    }
+    function formatInstant(epochMs) {
+      return new Date(epochMs).toISOString().replace(/\.\d{3}Z$/, "Z");
+    }
     async function runOutputGates(dispatch, response, executor, signal, ctx) {
       let spec;
       try {
@@ -25897,30 +26086,44 @@ var require_shuttle = __commonJS({
     }
     async function mintWaitingOutKeep(mint, brand, options) {
       const maxDelay = options.retryMaxMs ?? KEEP_RETRY_MAX_MS;
-      let delay = options.retryInitialMs ?? KEEP_RETRY_INITIAL_MS;
-      let unavailableSince = null;
+      const initialDelay = options.retryInitialMs ?? KEEP_RETRY_INITIAL_MS;
+      let delay = initialDelay;
+      let waiting = null;
       for (; ; ) {
+        let pause;
         try {
           const token = await mint();
-          if (unavailableSince !== null) {
-            options.logger?.info(`Keep is reachable again after ${formatDuration(Date.now() - unavailableSince)}`);
+          if (waiting?.kind === "unavailable") {
+            options.logger?.info(`Keep is reachable again after ${formatDuration(Date.now() - waiting.since)}`);
+          } else if (waiting?.kind === "rate-limited") {
+            options.logger?.info(`Keep issued a token; the rate limit lifted after ${formatDuration(Date.now() - waiting.since)}`);
           }
           return token;
         } catch (err) {
-          if (!isKeepUnavailable(err))
+          if (isKeepRateLimited(err)) {
+            if (waiting?.kind !== "rate-limited") {
+              waiting = { kind: "rate-limited", since: Date.now() };
+              options.logger?.warn(`Keep is rate limiting this ${brand.product.title}'s token exchange (${err.host ?? "Keep"} answered 429: ${err.reason ?? err.message}). The credentials are not in question; the ${brand.product.title} will wait and ask again at ${formatInstant(err.retryAt)} (in ${formatDuration(err.retryAfterMs)}), and start when Keep issues a token.`);
+            }
+            pause = err.retryAfterMs;
+            delay = initialDelay;
+          } else if (isKeepUnavailable(err)) {
+            if (waiting?.kind !== "unavailable") {
+              waiting = { kind: "unavailable", since: Date.now() };
+              const e = err;
+              const what = typeof e.statusCode === "number" ? `${e.host ?? "Keep"} answered ${e.statusCode}` : `${e.host ?? "Keep"} did not answer`;
+              options.logger?.warn(`Keep is unavailable (${what}). This is typical during a deployment window; the ${brand.product.title} will wait for it and start when it answers.`);
+            }
+            pause = delay;
+            delay = Math.min(delay * 2, maxDelay);
+          } else {
             throw err;
-          if (unavailableSince === null) {
-            unavailableSince = Date.now();
-            const e = err;
-            const what = typeof e.statusCode === "number" ? `${e.host ?? "Keep"} answered ${e.statusCode}` : `${e.host ?? "Keep"} did not answer`;
-            options.logger?.warn(`Keep is unavailable (${what}). This is typical during a deployment window; the ${brand.product.title} will wait for it and start when it answers.`);
           }
         }
-        await abortableSleep(delay, options.signal);
+        await abortableSleep(pause, options.signal);
         if (options.signal?.aborted) {
           throw options.signal.reason instanceof Error ? options.signal.reason : new Error("stopped while waiting for Keep");
         }
-        delay = Math.min(delay * 2, maxDelay);
       }
     }
   }
@@ -26548,7 +26751,7 @@ var import_path = require("path");
 var import_promises = require("fs/promises");
 var import_yaml = __toESM(require_dist3());
 var import_shuttle = __toESM(require_dist5());
-var buildVersion = true ? "0.30.9" : pkg.version;
+var buildVersion = true ? "0.30.10" : pkg.version;
 var sifterBrand = {
   product: {
     id: "sifter",