agent-relay-orchestrator 0.78.5 → 0.78.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.78.5",
+  "version": "0.78.6",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {

package/src/quota-poller.ts

@@ -25,6 +25,8 @@ import { codexCommandFromEnv, providerHomeRootFromEnv, type OrchestratorConfig }
 
 const QUOTA_LEASE_TTL_MS = 90_000;
 const QUOTA_LEASE_RENEW_MS = 30_000;
+const QUOTA_RETRY_BACKOFF_MAX_MS = 15 * 60_000;
+const QUOTA_RETRY_BACKOFF_MIN_MS = 1_000;
 const CODEX_APP_SERVER_CONNECT_ATTEMPTS = 40;
 const CODEX_APP_SERVER_CONNECT_RETRY_MS = 250;
 
@@ -62,6 +64,7 @@ type QuotaPollState = {
   leaseExpiresAt?: number;
   nextPollAt?: number;
   lastAttemptAt?: number;
+  consecutiveFailures?: number;
   lastLog?: { key: string; at: number };
 };
 
@@ -115,14 +118,18 @@ export class OrchestratorQuotaPoller {
       const { candidates, skips } = await this.discoverCandidates();
       await this.releaseRemovedCandidates(candidates);
       for (const candidate of candidates) {
-        await this.processCandidate(candidate);
+        try {
+          await this.processCandidate(candidate);
+        } catch (error) {
+          await this.handleCandidateFailure(candidate, error);
+        }
       }
       for (const skip of skips) {
         await this.reportSkip(skip);
       }
     } finally {
       this.inFlight = false;
-      this.schedule(this.options.intervalMs ?? QUOTA_LEASE_RENEW_MS);
+      this.schedule(this.nextScheduleDelay(this.options.intervalMs ?? QUOTA_LEASE_RENEW_MS));
     }
   }
 
@@ -136,6 +143,18 @@ export class OrchestratorQuotaPoller {
     }, Math.max(1_000, delayMs));
   }
 
+  private nextScheduleDelay(defaultDelayMs: number): number {
+    const now = this.now();
+    let delayMs = defaultDelayMs;
+    for (const state of this.states.values()) {
+      if (state.nextPollAt !== undefined) delayMs = Math.min(delayMs, state.nextPollAt - now);
+      if (state.leaseToken && state.leaseExpiresAt !== undefined) {
+        delayMs = Math.min(delayMs, state.leaseExpiresAt - now - QUOTA_LEASE_RENEW_MS);
+      }
+    }
+    return Math.max(QUOTA_RETRY_BACKOFF_MIN_MS, delayMs);
+  }
+
   // Refresh per-provider quota config (#605). Best-effort: on failure we keep the
   // last known config (defaults for any unset provider), so a transient relay blip
   // never silently stops collection.
@@ -270,16 +289,14 @@ export class OrchestratorQuotaPoller {
       await this.relay.reportProviderQuota(update);
       state.lastAttemptAt = update.lastAttemptAt;
       state.nextPollAt = now + pollIntervalMs;
+      state.consecutiveFailures = 0;
     } catch (error) {
       const retryAfterMs = quotaRetryAfterMs(error);
       const lastError = providerQuotaErrorFromCollectorError(error, retryAfterMs);
-      const retryDelayMs = retryAfterMs ?? (state.lastAttemptAt ? pollIntervalMs : QUOTA_FAST_RETRY_MS);
+      const retryDelayMs = this.retryDelayMs(state, retryAfterMs);
       state.lastAttemptAt = lastAttemptAt;
       state.nextPollAt = now + retryDelayMs;
-      if (candidate.provider === "claude" && retryAfterMs !== undefined) {
-        this.logFailure(candidate, error, retryAfterMs);
-        return;
-      }
+      state.consecutiveFailures = (state.consecutiveFailures ?? 0) + 1;
       await this.relay.reportProviderQuota({
         provider: candidate.provider,
         accountKey: candidate.accountKey,
@@ -287,10 +304,37 @@ export class OrchestratorQuotaPoller {
         lastError,
         sourceAgentId: this.sourceAgentId(),
       }).catch((publishError) => this.log(`quota status publish failed: ${errMessage(publishError)}`));
-      this.logFailure(candidate, error, retryAfterMs);
+      this.logFailure(candidate, error, retryAfterMs, retryDelayMs);
     }
   }
 
+  private async handleCandidateFailure(candidate: QuotaCandidate, error: unknown): Promise<void> {
+    const state = this.stateFor(candidate);
+    const now = this.now();
+    const retryAfterMs = quotaRetryAfterMs(error);
+    const retryDelayMs = this.retryDelayMs(state, retryAfterMs);
+    state.lastAttemptAt = now;
+    state.nextPollAt = now + retryDelayMs;
+    state.consecutiveFailures = (state.consecutiveFailures ?? 0) + 1;
+    await this.relay.reportProviderQuota({
+      provider: candidate.provider,
+      accountKey: candidate.accountKey,
+      lastAttemptAt: now,
+      lastError: providerQuotaErrorFromCollectorError(error, retryAfterMs),
+      sourceAgentId: this.sourceAgentId(),
+    }).catch((publishError) => this.log(`quota status publish failed: ${errMessage(publishError)}`));
+    this.logFailure(candidate, error, retryAfterMs, retryDelayMs);
+  }
+
+  private retryDelayMs(state: QuotaPollState, retryAfterMs: number | undefined): number {
+    const baseDelayMs = retryAfterMs ?? QUOTA_FAST_RETRY_MS;
+    const multiplier = 2 ** Math.min(state.consecutiveFailures ?? 0, 10);
+    return Math.min(
+      QUOTA_RETRY_BACKOFF_MAX_MS,
+      Math.max(QUOTA_RETRY_BACKOFF_MIN_MS, Math.round(baseDelayMs * multiplier)),
+    );
+  }
+
   private async ensureLease(candidate: QuotaCandidate, state: QuotaPollState, now: number): Promise<boolean> {
     if (state.leaseToken && state.leaseExpiresAt && state.leaseExpiresAt - now > QUOTA_LEASE_RENEW_MS) return true;
     const result = await this.relay.acquireProviderQuotaLease(this.config.id, {
@@ -353,13 +397,13 @@ export class OrchestratorQuotaPoller {
     (this.options.log ?? ((line) => console.error(`[orchestrator] ${line}`)))(message);
   }
 
-  private logFailure(candidate: QuotaCandidate, error: unknown, retryAfterMs: number | undefined): void {
+  private logFailure(candidate: QuotaCandidate, error: unknown, retryAfterMs: number | undefined, retryDelayMs: number): void {
     const state = this.stateFor(candidate);
-    const key = retryAfterMs !== undefined ? `retry-after:${retryAfterMs}` : errMessage(error);
+    const key = retryAfterMs !== undefined ? `retry-after:${retryAfterMs}:delay:${retryDelayMs}` : `${errMessage(error)}:delay:${retryDelayMs}`;
     const now = this.now();
     if (state.lastLog?.key === key && now - state.lastLog.at < QUOTA_FAILURE_LOG_INTERVAL_MS) return;
     state.lastLog = { key, at: now };
-    const suffix = retryAfterMs !== undefined ? `; retrying in ${Math.round(retryAfterMs / 1000)}s` : "";
+    const suffix = `; retrying in ${Math.round(retryDelayMs / 1000)}s`;
     this.log(`quota refresh failed for ${candidate.provider}/${candidate.accountKey}${suffix}: ${errMessage(error)}`);
   }