agent-relay-orchestrator 0.78.0 → 0.78.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.78.0",
+  "version": "0.78.1",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {
@@ -16,7 +16,7 @@
     "test": "bun test"
   },
   "dependencies": {
-    "agent-relay-sdk": "0.2.53"
+    "agent-relay-sdk": "0.2.54"
   },
   "devDependencies": {
     "@types/bun": "latest",

package/src/config.ts

@@ -42,6 +42,14 @@ export function bunBinFromEnv(): string | undefined {
   return process.env.AGENT_RELAY_BUN_BIN;
 }
 
+export function codexCommandFromEnv(): string | undefined {
+  return process.env.AGENT_RELAY_CODEX_COMMAND;
+}
+
+export function providerHomeRootFromEnv(): string {
+  return process.env.AGENT_RELAY_PROVIDER_HOME_ROOT || join(homedir(), ".agent-relay", "provider-homes");
+}
+
 export function disableSystemdSupervisor(): boolean {
   return process.env.AGENT_RELAY_DISABLE_SYSTEMD_SUPERVISOR === "1";
 }

package/src/quota-poller.ts

@@ -1,4 +1,6 @@
 import { homedir } from "node:os";
+import { existsSync, readdirSync } from "node:fs";
+import { createServer } from "node:net";
 import { join } from "node:path";
 import {
   QUOTA_FAILURE_LOG_INTERVAL_MS,
@@ -9,17 +11,19 @@ import {
   collectCodexQuotaSample,
   providerQuotaErrorFromCollectorError,
   quotaRetryAfterMs,
-  resolveClaudeQuotaIdentity,
-  resolveCodexQuotaIdentityFromHome,
+  resolveStableClaudeQuotaIdentity,
+  resolveStableCodexQuotaIdentityFromHome,
   type ProviderQuotaIdentity,
   type ProviderQuotaSample,
 } from "agent-relay-sdk/provider-quota";
 import type { ProviderQuotaLeaseAcquireInput, ProviderQuotaUpdateInput } from "agent-relay-sdk";
 import { errMessage } from "agent-relay-sdk";
-import type { OrchestratorConfig } from "./config";
+import { codexCommandFromEnv, providerHomeRootFromEnv, type OrchestratorConfig } from "./config";
 
 const QUOTA_LEASE_TTL_MS = 90_000;
 const QUOTA_LEASE_RENEW_MS = 30_000;
+const CODEX_APP_SERVER_CONNECT_ATTEMPTS = 40;
+const CODEX_APP_SERVER_CONNECT_RETRY_MS = 250;
 
 type QuotaRelay = {
   acquireProviderQuotaLease(orchestratorId: string, input: ProviderQuotaLeaseAcquireInput): Promise<{
@@ -34,6 +38,7 @@ type QuotaRelay = {
 
 type QuotaCandidate = ProviderQuotaIdentity & {
   appServerUrl?: string;
+  codexHome?: string;
 };
 
 type QuotaPollState = {
@@ -49,6 +54,7 @@ export class OrchestratorQuotaPoller {
   private active = false;
   private inFlight = false;
   private readonly states = new Map<string, QuotaPollState>();
+  private readonly logStates = new Map<string, { key: string; at: number }>();
 
   constructor(
     private readonly config: OrchestratorConfig,
@@ -78,6 +84,7 @@ export class OrchestratorQuotaPoller {
       void this.relay.releaseProviderQuotaLease(this.config.id, { provider, accountKey, leaseToken: state.leaseToken }).catch(() => {});
     }
     this.states.clear();
+    this.logStates.clear();
   }
 
   async tick(): Promise<void> {
@@ -108,15 +115,47 @@ export class OrchestratorQuotaPoller {
   private async discoverCandidates(): Promise<QuotaCandidate[]> {
     const candidates: QuotaCandidate[] = [];
     if (this.config.providers.includes("claude")) {
-      const configDir = this.config.env.CLAUDE_CONFIG_DIR || process.env.CLAUDE_CONFIG_DIR || join(homedir(), ".claude");
-      candidates.push(await resolveClaudeQuotaIdentity({ configDir, host: this.config.hostname }));
+      candidates.push(...await this.discoverClaudeCandidates());
     }
     if (this.config.providers.includes("codex")) {
-      const codexHome = this.config.env.CODEX_HOME || process.env.CODEX_HOME || join(homedir(), ".codex");
-      candidates.push({
-        ...(await resolveCodexQuotaIdentityFromHome({ codexHome, host: this.config.hostname })),
-        appServerUrl: this.config.env.CODEX_APP_SERVER_URL || process.env.CODEX_APP_SERVER_URL,
-      });
+      candidates.push(...await this.discoverCodexCandidates());
+    }
+    const deduped = new Map<string, QuotaCandidate>();
+    for (const candidate of candidates) {
+      deduped.set(candidateStateKey(candidate), candidate);
+    }
+    return [...deduped.values()];
+  }
+
+  private async discoverClaudeCandidates(): Promise<QuotaCandidate[]> {
+    const paths = [
+      join(this.config.env.CLAUDE_CONFIG_DIR || process.env.CLAUDE_CONFIG_DIR || join(homedir(), ".claude"), ".credentials.json"),
+      ...providerHomeCredentialPaths("claude", ".credentials.json"),
+    ];
+    const candidates: QuotaCandidate[] = [];
+    for (const credentialsPath of paths) {
+      const identity = await resolveStableClaudeQuotaIdentity({ credentialsPath });
+      if (identity) candidates.push(identity);
+    }
+    if (candidates.length === 0) {
+      this.logOnce("claude:no-stable-credentials", "quota refresh skipped for claude: no materialized Claude OAuth credentials found");
+    }
+    return candidates;
+  }
+
+  private async discoverCodexCandidates(): Promise<QuotaCandidate[]> {
+    const homes = [
+      this.config.env.CODEX_HOME || process.env.CODEX_HOME || join(homedir(), ".codex"),
+      ...providerHomeConfigDirs("codex", "auth.json"),
+    ];
+    const appServerUrl = this.config.env.CODEX_APP_SERVER_URL || process.env.CODEX_APP_SERVER_URL;
+    const candidates: QuotaCandidate[] = [];
+    for (const codexHome of homes) {
+      const identity = await resolveStableCodexQuotaIdentityFromHome({ codexHome });
+      if (identity) candidates.push({ ...identity, codexHome, ...(appServerUrl ? { appServerUrl } : {}) });
+    }
+    if (candidates.length === 0) {
+      this.logOnce("codex:no-stable-auth", "quota refresh skipped for codex: no account id found in Codex auth.json");
     }
     return candidates;
   }
@@ -198,12 +237,13 @@ export class OrchestratorQuotaPoller {
       });
     }
     if (candidate.provider === "codex") {
-      if (!candidate.appServerUrl) throw new QuotaCollectionError("creds_not_ready", "Codex app-server URL is not configured");
       return collectCodexQuotaSample({
         agentId: this.sourceAgentId(),
         rateLimitsRead: () => this.options.codexRateLimitsRead
-          ? this.options.codexRateLimitsRead(candidate.appServerUrl!)
-          : codexRateLimitsRead(candidate.appServerUrl!),
+          ? this.options.codexRateLimitsRead(candidate.appServerUrl ?? "")
+          : candidate.appServerUrl
+            ? codexRateLimitsRead(candidate.appServerUrl)
+            : codexRateLimitsReadFromHome(candidate.codexHome),
       });
     }
     return {};
@@ -239,12 +279,20 @@ export class OrchestratorQuotaPoller {
     const suffix = retryAfterMs !== undefined ? `; retrying in ${Math.round(retryAfterMs / 1000)}s` : "";
     this.log(`quota refresh failed for ${candidate.provider}/${candidate.accountKey}${suffix}: ${errMessage(error)}`);
   }
+
+  private logOnce(key: string, message: string): void {
+    const lastLog = this.logStates.get(key);
+    const now = this.now();
+    if (lastLog && now - lastLog.at < QUOTA_FAILURE_LOG_INTERVAL_MS) return;
+    this.logStates.set(key, { key, at: now });
+    this.log(message);
+  }
 }
 
-async function codexRateLimitsRead(appServerUrl: string): Promise<unknown> {
+async function codexRateLimitsRead(appServerUrl: string, attempts = 1): Promise<unknown> {
   const client = new JsonRpcWebSocket(appServerUrl);
   try {
-    await client.connect();
+    await connectWithRetry(client, attempts);
     await client.request("initialize", {
       clientInfo: { name: "agent-relay-orchestrator", title: "Agent Relay Orchestrator" },
       capabilities: { experimentalApi: true },
@@ -255,6 +303,86 @@ async function codexRateLimitsRead(appServerUrl: string): Promise<unknown> {
   }
 }
 
+async function codexRateLimitsReadFromHome(codexHome: string | undefined): Promise<unknown> {
+  if (!codexHome) throw new QuotaCollectionError("creds_not_ready", "Codex home is not configured");
+  const appServerUrl = await freeLoopbackWsUrl();
+  const command = codexCommandFromEnv() || "codex";
+  const proc = Bun.spawn([command, "app-server", "--listen", appServerUrl], {
+    env: {
+      ...process.env,
+      CODEX_HOME: codexHome,
+      CODEX_APP_SERVER_URL: appServerUrl,
+    },
+    stdin: "ignore",
+    stdout: "ignore",
+    stderr: "ignore",
+  });
+  try {
+    return await codexRateLimitsRead(appServerUrl, CODEX_APP_SERVER_CONNECT_ATTEMPTS);
+  } finally {
+    proc.kill();
+    await Promise.race([proc.exited.catch(() => undefined), Bun.sleep(2_000)]).catch(() => undefined);
+  }
+}
+
+async function connectWithRetry(client: JsonRpcWebSocket, attempts: number): Promise<void> {
+  let lastError: unknown;
+  for (let attempt = 0; attempt < attempts; attempt += 1) {
+    try {
+      await client.connect();
+      return;
+    } catch (error) {
+      lastError = error;
+      await Bun.sleep(CODEX_APP_SERVER_CONNECT_RETRY_MS);
+    }
+  }
+  throw lastError ?? new Error("websocket connect failed");
+}
+
+async function freeLoopbackWsUrl(): Promise<string> {
+  const port = await new Promise<number>((resolve, reject) => {
+    const server = createServer();
+    server.on("error", reject);
+    server.listen(0, "127.0.0.1", () => {
+      const address = server.address();
+      const port = typeof address === "object" && address ? address.port : undefined;
+      server.close(() => port ? resolve(port) : reject(new Error("failed to reserve loopback port")));
+    });
+  });
+  return `ws://127.0.0.1:${port}`;
+}
+
+function providerHomeRoot(): string {
+  return providerHomeRootFromEnv();
+}
+
+function providerHomeConfigDirs(provider: "claude" | "codex", markerFile: string): string[] {
+  const root = join(providerHomeRoot(), provider);
+  const dirs: string[] = [];
+  for (const profile of safeReadDir(root)) {
+    const profileDir = join(root, profile);
+    for (const instance of safeReadDir(profileDir)) {
+      const dir = join(profileDir, instance);
+      if (existsSync(join(dir, markerFile))) dirs.push(dir);
+    }
+  }
+  return dirs;
+}
+
+function providerHomeCredentialPaths(provider: "claude", markerFile: string): string[] {
+  return providerHomeConfigDirs(provider, markerFile).map((dir) => join(dir, markerFile));
+}
+
+function safeReadDir(path: string): string[] {
+  try {
+    return readdirSync(path, { withFileTypes: true })
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => entry.name);
+  } catch {
+    return [];
+  }
+}
+
 class JsonRpcWebSocket {
   private ws!: WebSocket;
   private nextId = 1;