agent-relay-orchestrator 0.78.2 → 0.78.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.78.2",
+  "version": "0.78.4",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {
@@ -16,7 +16,7 @@
     "test": "bun test"
   },
   "dependencies": {
-    "agent-relay-sdk": "0.2.54"
+    "agent-relay-sdk": "0.2.56"
   },
   "devDependencies": {
     "@types/bun": "latest",

package/src/quota-poller.ts

@@ -1,22 +1,25 @@
 import { homedir } from "node:os";
-import { existsSync, readdirSync } from "node:fs";
+import { existsSync, readFileSync, readdirSync } from "node:fs";
 import { createServer } from "node:net";
 import { join } from "node:path";
 import {
+  DEFAULT_PROVIDER_QUOTA_CONFIG,
   QUOTA_FAILURE_LOG_INTERVAL_MS,
   QUOTA_FAST_RETRY_MS,
-  QUOTA_POLL_INTERVAL_MS,
   QuotaCollectionError,
   collectClaudeQuotaSample,
   collectCodexQuotaSample,
+  credentialAccountKey,
+  normalizeProviderQuotaConfig,
   providerQuotaErrorFromCollectorError,
   quotaRetryAfterMs,
+  readClaudeOAuthAccessToken,
   resolveStableClaudeQuotaIdentity,
   resolveStableCodexQuotaIdentityFromHome,
   type ProviderQuotaIdentity,
   type ProviderQuotaSample,
 } from "agent-relay-sdk/provider-quota";
-import type { ProviderQuotaLeaseAcquireInput, ProviderQuotaUpdateInput } from "agent-relay-sdk";
+import type { ProviderQuotaConfig, ProviderQuotaConfigMap, ProviderQuotaLeaseAcquireInput, ProviderQuotaUpdateInput } from "agent-relay-sdk";
 import { errMessage } from "agent-relay-sdk";
 import { codexCommandFromEnv, providerHomeRootFromEnv, type OrchestratorConfig } from "./config";
 
@@ -25,6 +28,14 @@ const QUOTA_LEASE_RENEW_MS = 30_000;
 const CODEX_APP_SERVER_CONNECT_ATTEMPTS = 40;
 const CODEX_APP_SERVER_CONNECT_RETRY_MS = 250;
 
+// A provider that is configured but has no usable quota credential on this host
+// is uncollectable by design. Rather than silently omitting it (so the row
+// vanishes from the dashboard), we report a marker record carrying the reason,
+// so the widget can show a muted "unavailable" row.
+// A stable synthetic account key keeps it out of the host:/home: unstable-key prune.
+const PROVIDER_QUOTA_UNAVAILABLE_ACCOUNT_KEY = "unavailable";
+const PROVIDER_QUOTA_UNAVAILABLE_ERROR_TYPE = "unavailable";
+
 type QuotaRelay = {
   acquireProviderQuotaLease(orchestratorId: string, input: ProviderQuotaLeaseAcquireInput): Promise<{
     acquired: boolean;
@@ -33,14 +44,19 @@ type QuotaRelay = {
   }>;
   releaseProviderQuotaLease(orchestratorId: string, input: ProviderQuotaLeaseAcquireInput & { leaseToken: string }): Promise<unknown>;
   reportProviderQuota(input: ProviderQuotaUpdateInput): Promise<unknown>;
+  getProviderQuotaConfig(): Promise<ProviderQuotaConfigMap>;
   connected: boolean;
 };
 
 type QuotaCandidate = ProviderQuotaIdentity & {
+  accessToken?: string;
   appServerUrl?: string;
   codexHome?: string;
 };
 
+type ProviderSkip = { provider: string; reason: string };
+type QuotaDiscovery = { candidates: QuotaCandidate[]; skips: ProviderSkip[] };
+
 type QuotaPollState = {
   leaseToken?: string;
   leaseExpiresAt?: number;
@@ -55,6 +71,10 @@ export class OrchestratorQuotaPoller {
   private inFlight = false;
   private readonly states = new Map<string, QuotaPollState>();
   private readonly logStates = new Map<string, { key: string; at: number }>();
+  // Per-provider quota config (#605), refreshed each tick. Empty until the first
+  // successful fetch — configFor() falls back to defaults so a fresh/empty config
+  // (or an older relay without the endpoint) preserves today's behavior.
+  private quotaConfig: ProviderQuotaConfigMap = {};
 
   constructor(
     private readonly config: OrchestratorConfig,
@@ -91,11 +111,15 @@ export class OrchestratorQuotaPoller {
     if (this.inFlight || !this.active || !this.relay.connected) return;
     this.inFlight = true;
     try {
-      const candidates = await this.discoverCandidates();
+      await this.refreshQuotaConfig();
+      const { candidates, skips } = await this.discoverCandidates();
       await this.releaseRemovedCandidates(candidates);
       for (const candidate of candidates) {
         await this.processCandidate(candidate);
       }
+      for (const skip of skips) {
+        await this.reportSkip(skip);
+      }
     } finally {
       this.inFlight = false;
       this.schedule(this.options.intervalMs ?? QUOTA_LEASE_RENEW_MS);
@@ -112,38 +136,73 @@ export class OrchestratorQuotaPoller {
     }, Math.max(1_000, delayMs));
   }
 
-  private async discoverCandidates(): Promise<QuotaCandidate[]> {
+  // Refresh per-provider quota config (#605). Best-effort: on failure we keep the
+  // last known config (defaults for any unset provider), so a transient relay blip
+  // never silently stops collection.
+  private async refreshQuotaConfig(): Promise<void> {
+    try {
+      this.quotaConfig = await this.relay.getProviderQuotaConfig();
+    } catch {
+      // keep prior config; configFor() defaults any missing provider.
+    }
+  }
+
+  private configFor(provider: string): ProviderQuotaConfig {
+    const stored = this.quotaConfig[provider];
+    return stored ? normalizeProviderQuotaConfig(stored) : { ...DEFAULT_PROVIDER_QUOTA_CONFIG };
+  }
+
+  private async discoverCandidates(): Promise<QuotaDiscovery> {
     const candidates: QuotaCandidate[] = [];
-    if (this.config.providers.includes("claude")) {
-      candidates.push(...await this.discoverClaudeCandidates());
+    const skips: ProviderSkip[] = [];
+    // A disabled provider (#605) is collected from at all: no discovery → no
+    // polling/API calls, leases released by releaseRemovedCandidates, and no
+    // skip-marker row (disabled is intentional, not a credential failure).
+    if (this.config.providers.includes("claude") && this.configFor("claude").enabled) {
+      const found = await this.discoverClaudeCandidates();
+      candidates.push(...found.candidates);
+      if (found.skipReason) skips.push({ provider: "claude", reason: found.skipReason });
     }
-    if (this.config.providers.includes("codex")) {
-      candidates.push(...await this.discoverCodexCandidates());
+    if (this.config.providers.includes("codex") && this.configFor("codex").enabled) {
+      const found = await this.discoverCodexCandidates();
+      candidates.push(...found.candidates);
+      if (found.skipReason) skips.push({ provider: "codex", reason: found.skipReason });
     }
     const deduped = new Map<string, QuotaCandidate>();
     for (const candidate of candidates) {
       deduped.set(candidateStateKey(candidate), candidate);
     }
-    return [...deduped.values()];
+    return { candidates: [...deduped.values()], skips };
   }
 
-  private async discoverClaudeCandidates(): Promise<QuotaCandidate[]> {
-    const paths = [
+  private async discoverClaudeCandidates(): Promise<{ candidates: QuotaCandidate[]; skipReason?: string }> {
+    const credentialsPaths = [
       join(this.config.env.CLAUDE_CONFIG_DIR || process.env.CLAUDE_CONFIG_DIR || join(homedir(), ".claude"), ".credentials.json"),
       ...providerHomeCredentialPaths("claude", ".credentials.json"),
     ];
     const candidates: QuotaCandidate[] = [];
-    for (const credentialsPath of paths) {
+    for (const accessToken of [
+      this.config.env.CLAUDE_CODE_OAUTH_TOKEN,
+      process.env.CLAUDE_CODE_OAUTH_TOKEN,
+      ...claudeSetupTokenEnvPaths(this.config).map((path) => readEnvFileValue(path, "CLAUDE_CODE_OAUTH_TOKEN")),
+    ]) {
+      if (accessToken) candidates.push(claudeBearerCandidate(accessToken));
+    }
+    for (const credentialsPath of credentialsPaths) {
+      const accessToken = await readClaudeOAuthAccessToken(credentialsPath);
+      if (!accessToken) continue;
       const identity = await resolveStableClaudeQuotaIdentity({ credentialsPath });
-      if (identity) candidates.push(identity);
+      candidates.push({ ...(identity ?? claudeBearerCandidate(accessToken)), credentialsPath, accessToken });
     }
     if (candidates.length === 0) {
-      this.logOnce("claude:no-stable-credentials", "quota refresh skipped for claude: no materialized Claude OAuth credentials found");
+      const reason = "no usable Claude bearer token on this host";
+      this.logOnce("claude:no-usable-bearer", `quota refresh skipped for claude: ${reason}`);
+      return { candidates, skipReason: reason };
     }
-    return candidates;
+    return { candidates };
   }
 
-  private async discoverCodexCandidates(): Promise<QuotaCandidate[]> {
+  private async discoverCodexCandidates(): Promise<{ candidates: QuotaCandidate[]; skipReason?: string }> {
     const homes = [
       this.config.env.CODEX_HOME || process.env.CODEX_HOME || join(homedir(), ".codex"),
       ...providerHomeConfigDirs("codex", "auth.json"),
@@ -155,9 +214,25 @@ export class OrchestratorQuotaPoller {
       if (identity) candidates.push({ ...identity, codexHome, ...(appServerUrl ? { appServerUrl } : {}) });
     }
     if (candidates.length === 0) {
-      this.logOnce("codex:no-stable-auth", "quota refresh skipped for codex: no account id found in Codex auth.json");
+      const reason = "no Codex account id found in auth.json";
+      this.logOnce("codex:no-stable-auth", `quota refresh skipped for codex: ${reason}`);
+      return { candidates, skipReason: reason };
     }
-    return candidates;
+    return { candidates };
+  }
+
+  // Publish a marker record for a provider that is configured but uncollectable on
+  // this host, so the dashboard shows a muted "unavailable" row with the reason
+  // instead of dropping the provider entirely. Re-sent each tick to keep the row
+  // fresh (the day-old prune is keyed on updated_at).
+  private async reportSkip(skip: ProviderSkip): Promise<void> {
+    await this.relay.reportProviderQuota({
+      provider: skip.provider,
+      accountKey: PROVIDER_QUOTA_UNAVAILABLE_ACCOUNT_KEY,
+      lastAttemptAt: this.now(),
+      lastError: { type: PROVIDER_QUOTA_UNAVAILABLE_ERROR_TYPE, message: skip.reason },
+      sourceAgentId: this.sourceAgentId(),
+    }).catch((publishError) => this.log(`quota skip publish failed for ${skip.provider}: ${errMessage(publishError)}`));
   }
 
   private async releaseRemovedCandidates(candidates: QuotaCandidate[]): Promise<void> {
@@ -177,6 +252,9 @@ export class OrchestratorQuotaPoller {
     if (!await this.ensureLease(candidate, state, now)) return;
     if (state.nextPollAt !== undefined && state.nextPollAt > now) return;
 
+    // Per-provider cadence (#605): the configured interval governs the gap between
+    // successful polls and the post-failure retry once a first attempt has landed.
+    const pollIntervalMs = this.configFor(candidate.provider).pollIntervalMs;
     const lastAttemptAt = now;
     try {
       const sample = await this.collect(candidate);
@@ -191,13 +269,17 @@ export class OrchestratorQuotaPoller {
       };
       await this.relay.reportProviderQuota(update);
       state.lastAttemptAt = update.lastAttemptAt;
-      state.nextPollAt = now + QUOTA_POLL_INTERVAL_MS;
+      state.nextPollAt = now + pollIntervalMs;
     } catch (error) {
       const retryAfterMs = quotaRetryAfterMs(error);
       const lastError = providerQuotaErrorFromCollectorError(error, retryAfterMs);
-      const retryDelayMs = retryAfterMs ?? (state.lastAttemptAt ? QUOTA_POLL_INTERVAL_MS : QUOTA_FAST_RETRY_MS);
+      const retryDelayMs = retryAfterMs ?? (state.lastAttemptAt ? pollIntervalMs : QUOTA_FAST_RETRY_MS);
       state.lastAttemptAt = lastAttemptAt;
       state.nextPollAt = now + retryDelayMs;
+      if (candidate.provider === "claude" && retryAfterMs !== undefined) {
+        this.logFailure(candidate, error, retryAfterMs);
+        return;
+      }
       await this.relay.reportProviderQuota({
         provider: candidate.provider,
         accountKey: candidate.accountKey,
@@ -232,6 +314,7 @@ export class OrchestratorQuotaPoller {
     if (candidate.provider === "claude") {
       return collectClaudeQuotaSample({
         agentId: this.sourceAgentId(),
+        accessToken: candidate.accessToken,
         credentialsPath: candidate.credentialsPath,
         fetchImpl: this.options.fetchImpl,
       });
@@ -374,6 +457,45 @@ function providerHomeCredentialPaths(provider: "claude", markerFile: string): st
   return providerHomeConfigDirs(provider, markerFile).map((dir) => join(dir, markerFile));
 }
 
+function claudeSetupTokenEnvPaths(config: OrchestratorConfig): string[] {
+  const configDir = config.env.CLAUDE_CONFIG_DIR || process.env.CLAUDE_CONFIG_DIR || join(homedir(), ".claude");
+  return [
+    join(configDir, "setup-token.env"),
+    ...providerHomeConfigDirs("claude", "setup-token.env").map((dir) => join(dir, "setup-token.env")),
+  ];
+}
+
+function claudeBearerCandidate(accessToken: string): QuotaCandidate {
+  return {
+    provider: "claude",
+    accountKey: credentialAccountKey(accessToken),
+    accessToken,
+  };
+}
+
+function readEnvFileValue(path: string, key: string): string | undefined {
+  try {
+    for (const line of readFileSync(path, "utf8").split(/\r?\n/)) {
+      const match = new RegExp(`^\\s*(?:export\\s+)?${key}\\s*=\\s*(.*)\\s*$`).exec(line);
+      if (!match) continue;
+      return cleanEnvValue(match[1] ?? "");
+    }
+  } catch {
+    return undefined;
+  }
+  return undefined;
+}
+
+function cleanEnvValue(value: string): string | undefined {
+  let cleaned = value.trim();
+  if (!cleaned) return undefined;
+  const quote = cleaned[0];
+  if ((quote === "'" || quote === "\"") && cleaned.endsWith(quote)) {
+    cleaned = cleaned.slice(1, -1);
+  }
+  return cleaned.trim() || undefined;
+}
+
 function safeReadDir(path: string): string[] {
   try {
     return readdirSync(path, { withFileTypes: true })

package/src/relay.ts

@@ -2,7 +2,7 @@ import type { OrchestratorConfig } from "./config";
 import type { ProviderProbeCache } from "./provider-probe";
 import { detectSelfSupervision } from "./self-supervision";
 import { GIT_SHA, ORCHESTRATOR_PROTOCOL_VERSION, VERSION, runtimeMetadata } from "./version";
-import type { AgentLifecycle, ProviderQuotaLeaseAcquireInput, ProviderQuotaLeaseAcquireResult, ProviderQuotaUpdateInput, WorkspaceMetadata, WorkspaceMode, ManagedSessionExitDiagnostics as SdkManagedSessionExitDiagnostics } from "agent-relay-sdk";
+import type { AgentLifecycle, ProviderQuotaConfigMap, ProviderQuotaLeaseAcquireInput, ProviderQuotaLeaseAcquireResult, ProviderQuotaUpdateInput, WorkspaceMetadata, WorkspaceMode, ManagedSessionExitDiagnostics as SdkManagedSessionExitDiagnostics } from "agent-relay-sdk";
 import { ReconnectionManager, RelayHttpClient } from "agent-relay-sdk";
 
 export interface RelayClient {
@@ -14,6 +14,7 @@ export interface RelayClient {
   acquireProviderQuotaLease(orchestratorId: string, input: ProviderQuotaLeaseAcquireInput): Promise<ProviderQuotaLeaseAcquireResult>;
   releaseProviderQuotaLease(orchestratorId: string, input: ProviderQuotaLeaseAcquireInput & { leaseToken: string }): Promise<{ released: boolean }>;
   reportProviderQuota(input: ProviderQuotaUpdateInput): Promise<unknown>;
+  getProviderQuotaConfig(): Promise<ProviderQuotaConfigMap>;
   setApiUrl(url: string): void;
   startHeartbeatLoop(): void;
   stopHeartbeatLoop(): void;
@@ -251,6 +252,9 @@ export function createRelayClient(config: OrchestratorConfig, probeCache: Provid
     reportProviderQuota(input: ProviderQuotaUpdateInput): Promise<unknown> {
       return http.upsertProviderQuota(input);
     },
+    getProviderQuotaConfig(): Promise<ProviderQuotaConfigMap> {
+      return http.getProviderQuotaConfig();
+    },
     get connected() { return connected; },
   };
 }