agent-relay-server 0.77.1 → 0.78.1

package/docs/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "Agent Relay API",
-    "version": "0.77.1",
+    "version": "0.78.1",
     "description": "Real-time message bus for inter-agent communication. Agent-first: this spec is designed for machine consumption — agents can self-discover the full API surface via GET /api/spec.",
     "license": {
       "name": "MIT",
@@ -729,6 +729,84 @@
         ]
       }
     },
+    "/api/orchestrators/{id}/provider-quota-leases/acquire": {
+      "post": {
+        "operationId": "postProviderQuotaLeaseAcquireRoute",
+        "summary": "Acquire provider quota poll lease",
+        "tags": [
+          "Orchestrators"
+        ],
+        "description": "Orchestrator-only election endpoint. Grants or renews the single fleet-wide polling lease for one `(provider, accountKey)` when no non-expired holder exists, or returns the current holder and retry delay. The caller must renew with the returned `leaseToken`; another process with the same orchestrator id but no token cannot steal the lease.",
+        "parameters": [
+          {
+            "name": "id",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Success",
+            "content": {
+              "application/json": {}
+            }
+          }
+        },
+        "security": [
+          {
+            "bearerAuth": []
+          },
+          {
+            "tokenHeader": []
+          },
+          {
+            "tokenQuery": []
+          }
+        ]
+      }
+    },
+    "/api/orchestrators/{id}/provider-quota-leases/release": {
+      "post": {
+        "operationId": "postProviderQuotaLeaseReleaseRoute",
+        "summary": "Release provider quota poll lease",
+        "tags": [
+          "Orchestrators"
+        ],
+        "description": "Releases a provider quota poll lease only when `provider`, `accountKey`, orchestrator id, and `leaseToken` all match the current holder.",
+        "parameters": [
+          {
+            "name": "id",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Success",
+            "content": {
+              "application/json": {}
+            }
+          }
+        },
+        "security": [
+          {
+            "bearerAuth": []
+          },
+          {
+            "tokenHeader": []
+          },
+          {
+            "tokenQuery": []
+          }
+        ]
+      }
+    },
     "/api/agents/spawn/directories": {
       "get": {
         "operationId": "getHostDirectories",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-server",
-  "version": "0.77.1",
+  "version": "0.78.1",
   "description": "Lightweight HTTP message relay for inter-agent communication across machines",
   "module": "src/index.ts",
   "type": "module",
@@ -33,7 +33,7 @@
     "CONTRIBUTING.md"
   ],
   "dependencies": {
-    "agent-relay-sdk": "0.2.52",
+    "agent-relay-sdk": "0.2.54",
     "ajv": "^8.20.0"
   },
   "scripts": {

package/src/config-store.ts

@@ -1,6 +1,7 @@
 import { getDb, ValidationError } from "./db";
 import { cleanEnum, cleanString, cleanStringArray } from "./validation";
 import { DEFAULT_CONTEXT_THRESHOLD_CONFIG, validateContextThresholdConfig } from "./context-threshold-config";
+import { DEFAULT_QUOTA_THRESHOLD_CONFIG, validateQuotaThresholdConfig } from "./quota-threshold-config";
 import { DEFAULT_SELF_RESUME_CONFIG, validateSelfResumeConfig } from "./self-resume-config";
 import { TEAM_TEMPLATE_NAMESPACE, validateTeamTemplate } from "./team-template-config";
 import { PROJECT_INSTRUCTION_POLICY_NAMES } from "./project-instructions";
@@ -499,7 +500,7 @@ function validateInsightsConfig(value: unknown): InsightsConfig {
 // Relay-driven lifecycle push notifications (#239 event bus). Default-on; the
 // operator can flip the master switch or individual events off via the generic
 // config route. Push messages wake recipients, so they must be suppressible.
-const NOTIFICATIONS_CONFIG_DEFAULTS: NotificationsConfig = { enabled: true, branchLanded: true, agentReady: true, agentExited: true, agentSpawnFailed: true, contextThreshold: DEFAULT_CONTEXT_THRESHOLD_CONFIG };
+const NOTIFICATIONS_CONFIG_DEFAULTS: NotificationsConfig = { enabled: true, branchLanded: true, agentReady: true, agentExited: true, agentSpawnFailed: true, contextThreshold: DEFAULT_CONTEXT_THRESHOLD_CONFIG, quotaThreshold: DEFAULT_QUOTA_THRESHOLD_CONFIG };
 
 function validateNotificationsConfig(value: unknown): NotificationsConfig {
   if (!isRecord(value)) throw new ValidationError("notifications config value must be an object");
@@ -512,6 +513,7 @@ function validateNotificationsConfig(value: unknown): NotificationsConfig {
     agentExited: bool("agentExited"),
     agentSpawnFailed: bool("agentSpawnFailed"),
     contextThreshold: validateContextThresholdConfig(value.contextThreshold),
+    quotaThreshold: validateQuotaThresholdConfig(value.quotaThreshold),
   };
 }
 

package/src/db/index.ts

@@ -30,3 +30,4 @@ export * from "./projects.ts";
 export * from "./project-entries.ts";
 export * from "./provisioning-registry.ts";
 export * from "./provider-quotas.ts";
+export * from "./provider-quota-leases.ts";

package/src/db/migrations.ts

@@ -658,6 +658,7 @@ export function applyMigrations(): void {
       account_key TEXT NOT NULL,
       quota_state TEXT,
       source_agent_ids TEXT NOT NULL DEFAULT '[]',
+      quota_advisory_state TEXT NOT NULL DEFAULT '{}',
       last_updated_at INTEGER,
       last_attempt_at INTEGER NOT NULL,
       last_error TEXT,
@@ -666,6 +667,10 @@ export function applyMigrations(): void {
       PRIMARY KEY (provider, account_key)
     )
   `);
+  const providerQuotaCols = (getDb().query("PRAGMA table_info(provider_quotas)").all() as Array<{ name: string }>).map((col) => col.name);
+  if (!providerQuotaCols.includes("quota_advisory_state")) {
+    getDb().run("ALTER TABLE provider_quotas ADD COLUMN quota_advisory_state TEXT NOT NULL DEFAULT '{}'");
+  }
   getDb().run(`
     INSERT INTO provider_quotas (
       provider, account_key, quota_state, source_agent_ids, last_updated_at,
@@ -703,6 +708,20 @@ export function applyMigrations(): void {
   getDb().run("CREATE INDEX IF NOT EXISTS idx_quota_snapshots_agent_time ON quota_snapshots(agent_id, captured_at DESC)");
   normalizeProviderQuotaAccountKeys();
 
+  getDb().run(`
+    CREATE TABLE IF NOT EXISTS provider_quota_leases (
+      provider TEXT NOT NULL,
+      account_key TEXT NOT NULL,
+      orchestrator_id TEXT NOT NULL,
+      lease_token TEXT NOT NULL,
+      expires_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL,
+      created_at INTEGER NOT NULL,
+      PRIMARY KEY (provider, account_key)
+    )
+  `);
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_provider_quota_leases_expires ON provider_quota_leases(expires_at)");
+
   getDb().run(`
     CREATE TABLE IF NOT EXISTS memories (
       id TEXT PRIMARY KEY,

package/src/db/provider-quota-leases.ts

@@ -0,0 +1,134 @@
+import { randomUUID } from "node:crypto";
+import { getDb } from "./connection.ts";
+
+export interface ProviderQuotaLease {
+  provider: string;
+  accountKey: string;
+  orchestratorId: string;
+  leaseToken: string;
+  expiresAt: number;
+  updatedAt: number;
+  createdAt: number;
+}
+
+export interface ProviderQuotaLeaseAcquireResult {
+  acquired: boolean;
+  lease?: ProviderQuotaLease;
+  holder?: {
+    orchestratorId: string;
+    expiresAt: number;
+  };
+  retryAfterMs?: number;
+}
+
+type ProviderQuotaLeaseRow = {
+  provider: string;
+  account_key: string;
+  orchestrator_id: string;
+  lease_token: string;
+  expires_at: number;
+  updated_at: number;
+  created_at: number;
+};
+
+function normalizeProvider(value: string): string {
+  return value.trim().toLowerCase();
+}
+
+function cleanAccountKey(value: string): string {
+  return value.trim();
+}
+
+function rowToLease(row: ProviderQuotaLeaseRow): ProviderQuotaLease {
+  return {
+    provider: row.provider,
+    accountKey: row.account_key,
+    orchestratorId: row.orchestrator_id,
+    leaseToken: row.lease_token,
+    expiresAt: row.expires_at,
+    updatedAt: row.updated_at,
+    createdAt: row.created_at,
+  };
+}
+
+export function acquireProviderQuotaLease(input: {
+  provider: string;
+  accountKey: string;
+  orchestratorId: string;
+  leaseToken?: string;
+  ttlMs: number;
+  now?: number;
+}): ProviderQuotaLeaseAcquireResult {
+  const provider = normalizeProvider(input.provider);
+  const accountKey = cleanAccountKey(input.accountKey);
+  const orchestratorId = input.orchestratorId.trim();
+  const ttlMs = Math.max(5_000, input.ttlMs);
+  const now = input.now ?? Date.now();
+  if (!provider) throw new Error("provider required");
+  if (!accountKey) throw new Error("accountKey required");
+  if (!orchestratorId) throw new Error("orchestratorId required");
+
+  const acquire = getDb().transaction(() => {
+    const existing = getDb().query("SELECT * FROM provider_quota_leases WHERE provider = ? AND account_key = ?")
+      .get(provider, accountKey) as ProviderQuotaLeaseRow | undefined;
+    const expired = !existing || existing.expires_at <= now;
+    const sameHolder = existing?.orchestrator_id === orchestratorId && existing.lease_token === input.leaseToken;
+    if (existing && !expired && !sameHolder) {
+      return {
+        acquired: false,
+        holder: { orchestratorId: existing.orchestrator_id, expiresAt: existing.expires_at },
+        retryAfterMs: Math.max(1_000, existing.expires_at - now),
+      } satisfies ProviderQuotaLeaseAcquireResult;
+    }
+
+    const leaseToken = sameHolder ? existing!.lease_token : randomUUID();
+    const createdAt = sameHolder ? existing!.created_at : now;
+    const expiresAt = now + ttlMs;
+    getDb().query(`
+      INSERT INTO provider_quota_leases (
+        provider, account_key, orchestrator_id, lease_token, expires_at, updated_at, created_at
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(provider, account_key) DO UPDATE SET
+        orchestrator_id = excluded.orchestrator_id,
+        lease_token = excluded.lease_token,
+        expires_at = excluded.expires_at,
+        updated_at = excluded.updated_at
+    `).run(provider, accountKey, orchestratorId, leaseToken, expiresAt, now, createdAt);
+    return {
+      acquired: true,
+      lease: {
+        provider,
+        accountKey,
+        orchestratorId,
+        leaseToken,
+        expiresAt,
+        updatedAt: now,
+        createdAt,
+      },
+    } satisfies ProviderQuotaLeaseAcquireResult;
+  });
+
+  return acquire();
+}
+
+export function releaseProviderQuotaLease(input: {
+  provider: string;
+  accountKey: string;
+  orchestratorId: string;
+  leaseToken: string;
+}): boolean {
+  const provider = normalizeProvider(input.provider);
+  const accountKey = cleanAccountKey(input.accountKey);
+  const result = getDb().query(`
+    DELETE FROM provider_quota_leases
+    WHERE provider = ? AND account_key = ? AND orchestrator_id = ? AND lease_token = ?
+  `).run(provider, accountKey, input.orchestratorId.trim(), input.leaseToken);
+  return result.changes > 0;
+}
+
+export function getProviderQuotaLease(provider: string, accountKey: string): ProviderQuotaLease | null {
+  const row = getDb().query("SELECT * FROM provider_quota_leases WHERE provider = ? AND account_key = ?")
+    .get(normalizeProvider(provider), cleanAccountKey(accountKey)) as ProviderQuotaLeaseRow | undefined;
+  return row ? rowToLease(row) : null;
+}

package/src/db/provider-quotas.ts

@@ -6,11 +6,28 @@ import type { AgentCard, ProviderQuotaError, ProviderQuotaMatrix, ProviderQuotaR
 
 const PROVIDER_QUOTA_STALE_AFTER_MS = 10 * 60_000;
 
+export interface ProviderQuotaAdvisoryInput {
+  provider: string;
+  accountKey: string;
+  previousQuota?: QuotaState;
+  quota: QuotaState;
+  sourceAgentIds: string[];
+  previousAdvisoryState?: string | null;
+  now: number;
+}
+
+let providerQuotaAdvisoryHandler: ((input: ProviderQuotaAdvisoryInput) => string) | undefined;
+
+export function setProviderQuotaAdvisoryHandler(handler: ((input: ProviderQuotaAdvisoryInput) => string) | undefined): void {
+  providerQuotaAdvisoryHandler = handler;
+}
+
 type ProviderQuotaRow = {
   provider: string;
   account_key: string;
   quota_state?: string | null;
   source_agent_ids?: string | null;
+  quota_advisory_state?: string | null;
   last_updated_at?: number | null;
   last_attempt_at: number;
   last_error?: string | null;
@@ -42,6 +59,10 @@ function canonicalProviderQuotaAccountKey(provider: string, value: string): stri
     : accountKey;
 }
 
+function unstableProviderQuotaAccountKey(accountKey: string): boolean {
+  return accountKey.startsWith("host:") || accountKey.startsWith("home:");
+}
+
 function sourceAgentHost(sourceAgentId: string | undefined): string | undefined {
   if (!sourceAgentId) return undefined;
   const row = getDb().query("SELECT machine, rig FROM agents WHERE id = ?").get(sourceAgentId) as { machine?: string | null; rig?: string | null } | undefined;
@@ -96,6 +117,13 @@ function newestLastError(rows: ProviderQuotaRow[]): string | null {
   return row?.last_error ?? null;
 }
 
+function newestQuotaAdvisoryState(rows: ProviderQuotaRow[]): string {
+  const row = rows
+    .filter((candidate) => candidate.quota_advisory_state)
+    .sort((a, b) => b.updated_at - a.updated_at)[0];
+  return row?.quota_advisory_state ?? "{}";
+}
+
 function mergeProviderQuotaRows(provider: string, accountKey: string, oldAccountKeys: string[], now: number): number {
   const keys = [...new Set(oldAccountKeys.filter((key) => key && key !== accountKey))];
   if (keys.length === 0) return 0;
@@ -117,15 +145,16 @@ function mergeProviderQuotaRows(provider: string, accountKey: string, oldAccount
   getDb().query(`
     INSERT INTO provider_quotas (
       provider, account_key, quota_state, source_agent_ids, last_updated_at,
-      last_attempt_at, last_error, updated_at, created_at
+      last_attempt_at, last_error, quota_advisory_state, updated_at, created_at
     )
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     ON CONFLICT(provider, account_key) DO UPDATE SET
       quota_state = excluded.quota_state,
       source_agent_ids = excluded.source_agent_ids,
       last_updated_at = excluded.last_updated_at,
       last_attempt_at = excluded.last_attempt_at,
       last_error = excluded.last_error,
+      quota_advisory_state = excluded.quota_advisory_state,
       updated_at = excluded.updated_at
   `).run(
     provider,
@@ -135,6 +164,7 @@ function mergeProviderQuotaRows(provider: string, accountKey: string, oldAccount
     lastUpdatedAt ?? null,
     lastAttemptAt || now,
     newestLastError(rows),
+    newestQuotaAdvisoryState(rows),
     updatedAt,
     createdAt,
   );
@@ -251,6 +281,23 @@ export function upsertProviderQuota(input: ProviderQuotaUpdateInput, now = Date.
   if (input.quota && latestSnapshotFingerprint(provider, accountKey) !== quotaFingerprint(input.quota)) {
     recordProviderQuotaSnapshot({ provider, accountKey, quota: input.quota, sourceAgentId: input.sourceAgentId }, now);
   }
+  if (input.quota) {
+    const quotaAdvisoryState = providerQuotaAdvisoryHandler?.({
+      provider,
+      accountKey,
+      previousQuota: existing?.quota_state ? parseJson<QuotaState>(existing.quota_state, { windows: [], source: "probe", updatedAt: 0 }) : undefined,
+      quota: input.quota,
+      sourceAgentIds,
+      previousAdvisoryState: existing?.quota_advisory_state,
+      now,
+    });
+    if (quotaAdvisoryState !== undefined) {
+      getDb().query("UPDATE provider_quotas SET quota_advisory_state = ? WHERE provider = ? AND account_key = ?").run(quotaAdvisoryState, provider, accountKey);
+    }
+  }
+  if (!unstableProviderQuotaAccountKey(accountKey)) {
+    pruneUnstableProviderQuotaRecords(provider);
+  }
   pruneProviderQuotaRecords(provider, undefined, now - DAY_MS);
 
   return listProviderQuotas({ provider, accountKey, historyLimit: 24, now }).quotas[0]!;
@@ -291,19 +338,22 @@ function pruneProviderQuotaRecords(provider?: string, accountKey?: string, older
   return result.changes;
 }
 
+function pruneUnstableProviderQuotaRecords(provider?: string): number {
+  const result = provider
+    ? getDb().query("DELETE FROM provider_quotas WHERE provider = ? AND (account_key LIKE 'host:%' OR account_key LIKE 'home:%')").run(provider)
+    : getDb().query("DELETE FROM provider_quotas WHERE account_key LIKE 'host:%' OR account_key LIKE 'home:%'").run();
+  return result.changes;
+}
+
 export function normalizeProviderQuotaAccountKeys(now = Date.now()): number {
   let changes = 0;
   for (const row of getDb().query("SELECT * FROM provider_quotas").all() as ProviderQuotaRow[]) {
-    let accountKey = canonicalProviderQuotaAccountKey(row.provider, row.account_key);
-    if (accountKey.startsWith("credential:")) {
-      const sourceAgentId = normalizeSourceAgentIds(row.source_agent_ids)[0];
-      const host = sourceAgentHost(sourceAgentId);
-      if (host) accountKey = `host:${host}`;
-    }
+    const accountKey = canonicalProviderQuotaAccountKey(row.provider, row.account_key);
     if (accountKey !== row.account_key) {
       changes += mergeProviderQuotaRows(row.provider, accountKey, [row.account_key], now);
     }
   }
+  changes += pruneUnstableProviderQuotaRecords();
   changes += pruneProviderQuotaRecords(undefined, undefined, now - DAY_MS);
   return changes;
 }

package/src/db/schema.ts

@@ -307,6 +307,7 @@ export function initDb(path: string = "agent-relay.db"): Database {
       account_key TEXT NOT NULL,
       quota_state TEXT,
       source_agent_ids TEXT NOT NULL DEFAULT '[]',
+      quota_advisory_state TEXT NOT NULL DEFAULT '{}',
       last_updated_at INTEGER,
       last_attempt_at INTEGER NOT NULL,
       last_error TEXT,
@@ -315,6 +316,18 @@ export function initDb(path: string = "agent-relay.db"): Database {
       PRIMARY KEY (provider, account_key)
     );
 
+    CREATE TABLE IF NOT EXISTS provider_quota_leases (
+      provider TEXT NOT NULL,
+      account_key TEXT NOT NULL,
+      orchestrator_id TEXT NOT NULL,
+      lease_token TEXT NOT NULL,
+      expires_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL,
+      created_at INTEGER NOT NULL,
+      PRIMARY KEY (provider, account_key)
+    );
+    CREATE INDEX IF NOT EXISTS idx_provider_quota_leases_expires ON provider_quota_leases(expires_at);
+
     CREATE TABLE IF NOT EXISTS agent_continuations (
       agent_id TEXT PRIMARY KEY REFERENCES agents(id) ON DELETE CASCADE,
       objective TEXT NOT NULL,

package/src/index.ts

@@ -11,6 +11,7 @@ import { getCompactionWatch } from "./compaction-watch";
 import { getConfig, setConfig } from "./config-store";
 import { startConnectorStatusPoller } from "./connectors";
 import { installNotificationSubscriptionRule } from "./notification-subscription-rules";
+import { installQuotaAdvisoryHandler } from "./quota-advisory";
 import { startPlanLiveBindings } from "./services/plan-live-bindings";
 import { resolve, sep } from "path";
 import { gzipSync, brotliCompressSync, constants as zlibConstants } from "node:zlib";
@@ -76,6 +77,7 @@ function startServer(): void {
 
   assertSafeNetworkConfig(HOST);
   initDb(DB_PATH);
+  installQuotaAdvisoryHandler();
   installNotificationSubscriptionRule();
   startPlanLiveBindings();
   getLifecycleManager().start();

package/src/notification-types.ts

@@ -5,6 +5,7 @@ export type NotificationType =
   | "agent.exited"
   | "agent.spawn_failed"
   | "agent.context_threshold"
+  | "agent.quota_threshold"
   | "agent.resume_budget_warning"
   | "agent.resume_budget_exhausted"
   | "agent.transient_land_hold"
@@ -54,6 +55,7 @@ export const DEFAULT_TTL_MS: Record<NotificationType, number | null> = {
   "agent.exited": null,
   "agent.spawn_failed": null,
   "agent.context_threshold": 15 * MINUTES,
+  "agent.quota_threshold": 15 * MINUTES,
   "agent.resume_budget_warning": null,
   "agent.resume_budget_exhausted": null,
   "agent.transient_land_hold": null,

package/src/quota-advisory.ts

@@ -0,0 +1,300 @@
+import { isRecord } from "agent-relay-sdk";
+import { getDb } from "./db/connection.ts";
+import { emitRelayEvent } from "./events";
+import { routeNotification } from "./notification-router";
+import { DEFAULT_QUOTA_THRESHOLD_CONFIG, validateQuotaThresholdConfig } from "./quota-threshold-config";
+import { parseJson } from "./utils";
+import { setProviderQuotaAdvisoryHandler, type ProviderQuotaAdvisoryInput } from "./db/provider-quotas.ts";
+import type { QuotaState } from "./types";
+
+interface QuotaAdvisoryWindowState {
+  lastQuotaAdvisoryBand?: number;
+  lastQuotaAdvisoryAt?: number;
+  lastRemainingPercent?: number;
+  lastResetsAt?: number;
+}
+
+type QuotaAdvisoryState = Record<string, QuotaAdvisoryWindowState>;
+type QuotaAdvisoryStage = "warn" | "critical" | "reset";
+
+type AgentRecipientRow = {
+  id: string;
+  name: string;
+  kind?: string | null;
+  machine?: string | null;
+  rig?: string | null;
+  ready?: number | null;
+  status: string;
+  meta?: string | null;
+};
+
+const RESET_ROUNDING_PERCENT = 99;
+
+export function installQuotaAdvisoryHandler(): void {
+  setProviderQuotaAdvisoryHandler(handleProviderQuotaAdvisory);
+}
+
+function handleProviderQuotaAdvisory(input: ProviderQuotaAdvisoryInput): string {
+  const config = effectiveQuotaNotificationConfig().quotaThreshold;
+  const state = parseJson<QuotaAdvisoryState>(input.previousAdvisoryState ?? "{}", {});
+  const nextState: QuotaAdvisoryState = { ...state };
+
+  for (const window of input.quota.windows) {
+    const previousWindow = input.previousQuota?.windows.find((candidate) => candidate.name === window.name);
+    const remaining = quotaRemaining(window.utilization);
+    const remainingPercent = percent(remaining);
+    const previous = nextState[window.name];
+    const reset = previous?.lastQuotaAdvisoryBand !== undefined && isQuotaReset(remaining, previousWindow, window, config.resetRemaining);
+    if (reset) {
+      notifyQuotaAdvisory({
+        provider: input.provider,
+        accountKey: input.accountKey,
+        windowName: window.name,
+        stage: "reset",
+        band: 100,
+        remainingPercent,
+        resetsAt: window.resetsAt,
+        recipients: quotaAdvisoryRecipients(input.provider, input.accountKey, input.sourceAgentIds, input.now),
+        headroomHint: crossProviderHeadroomHint(input.provider),
+        now: input.now,
+      });
+      delete nextState[window.name];
+      continue;
+    }
+
+    if (remaining > config.warnRemaining) {
+      if (previous) delete nextState[window.name];
+      continue;
+    }
+
+    const band = quotaRemainingBand(remaining, config.bandSize);
+    if (previous?.lastQuotaAdvisoryBand === band) continue;
+    const stage: QuotaAdvisoryStage = remaining <= config.criticalRemaining ? "critical" : "warn";
+    notifyQuotaAdvisory({
+      provider: input.provider,
+      accountKey: input.accountKey,
+      windowName: window.name,
+      stage,
+      band,
+      remainingPercent,
+      resetsAt: window.resetsAt,
+      recipients: quotaAdvisoryRecipients(input.provider, input.accountKey, input.sourceAgentIds, input.now),
+      headroomHint: crossProviderHeadroomHint(input.provider),
+      now: input.now,
+    });
+    nextState[window.name] = {
+      lastQuotaAdvisoryBand: band,
+      lastQuotaAdvisoryAt: input.now,
+      lastRemainingPercent: remainingPercent,
+      ...(window.resetsAt !== undefined ? { lastResetsAt: window.resetsAt } : {}),
+    };
+  }
+
+  return JSON.stringify(nextState);
+}
+
+function notifyQuotaAdvisory(input: {
+  provider: string;
+  accountKey: string;
+  windowName: string;
+  stage: QuotaAdvisoryStage;
+  band: number;
+  remainingPercent: number;
+  resetsAt?: number;
+  recipients: string[];
+  headroomHint: string;
+  now: number;
+}): void {
+  const payload = {
+    kind: "agent.quota_threshold",
+    provider: input.provider,
+    accountKey: input.accountKey,
+    window: input.windowName,
+    stage: input.stage,
+    band: input.band,
+    remainingPercent: input.remainingPercent,
+    ...(input.resetsAt !== undefined ? { resetsAt: input.resetsAt } : {}),
+    recipients: input.recipients,
+  };
+
+  emitRelayEvent({
+    type: "agent.quota_threshold",
+    source: "server",
+    subject: `${input.provider}:${input.accountKey}:${input.windowName}`,
+    data: payload,
+  });
+
+  const notifications = effectiveQuotaNotificationConfig();
+  if (!notifications.enabled || !notifications.quotaThreshold.enabled) return;
+
+  routeNotification({
+    type: "agent.quota_threshold",
+    scope: {},
+    recipients: input.recipients,
+    message: {
+      subject: input.stage === "reset"
+        ? `${windowLabel(input.windowName)} ${providerLabel(input.provider)} quota reset`
+        : `${windowLabel(input.windowName)} ${providerLabel(input.provider)} quota ${input.stage === "critical" ? "critical" : "warning"}`,
+      body: quotaAdvisoryBody(input),
+      payload,
+      replyExpected: false,
+    },
+    emittedAt: input.now,
+  });
+}
+
+function effectiveQuotaNotificationConfig(): { enabled: boolean; quotaThreshold: ReturnType<typeof validateQuotaThresholdConfig> } {
+  const row = getDb().query("SELECT value FROM config WHERE namespace = 'notifications' AND key = 'default'").get() as { value?: string | null } | undefined;
+  const value = parseJson<Record<string, unknown>>(row?.value ?? "{}", {});
+  return {
+    enabled: value.enabled === undefined ? true : value.enabled === true,
+    quotaThreshold: validateQuotaThresholdConfig({ ...DEFAULT_QUOTA_THRESHOLD_CONFIG, ...(isRecord(value.quotaThreshold) ? value.quotaThreshold : {}) }),
+  };
+}
+
+function quotaAdvisoryBody(input: {
+  provider: string;
+  windowName: string;
+  stage: QuotaAdvisoryStage;
+  band: number;
+  remainingPercent: number;
+  resetsAt?: number;
+  headroomHint: string;
+  now: number;
+}): string {
+  const provider = providerLabel(input.provider);
+  const window = windowLabel(input.windowName);
+  if (input.stage === "reset" || input.remainingPercent >= RESET_ROUNDING_PERCENT) {
+    return `${window} ${provider} quota reset to 100%. ${input.headroomHint}`;
+  }
+  const reset = input.resetsAt === undefined ? "reset time unknown" : `resets in ${formatDuration(input.resetsAt - input.now)}`;
+  return `${window} ${provider} quota below ${input.band}% remaining - ${input.remainingPercent}% remaining, ${reset}. ${input.headroomHint}`;
+}
+
+function quotaAdvisoryRecipients(provider: string, accountKey: string, sourceAgentIds: string[], now: number): string[] {
+  const sourceHosts = agentHosts(sourceAgentIds);
+  const accountHost = hostFromAccountKey(accountKey);
+  const candidates = getDb().query(`
+    SELECT id, name, kind, machine, rig, ready, status, meta
+    FROM agents
+    WHERE kind = 'provider'
+      AND ready = 1
+      AND status NOT IN ('offline', 'stale')
+  `).all() as AgentRecipientRow[];
+  return candidates
+    .filter((agent) => isSpawnCapableAgent(agent, now))
+    .filter((agent) => isBoundToQuotaAccount(agent, provider, accountKey, accountHost, sourceHosts, sourceAgentIds))
+    .map((agent) => agent.id);
+}
+
+function isSpawnCapableAgent(agent: AgentRecipientRow, now: number): boolean {
+  const meta = parseJson<Record<string, unknown>>(agent.meta ?? "{}", {});
+  const auth = isRecord(meta.auth) ? meta.auth : undefined;
+  const jti = typeof auth?.jti === "string" ? auth.jti : undefined;
+  if (!jti) return false;
+  const row = getDb().query("SELECT scope, revoked_at, expires_at FROM tokens WHERE jti = ?").get(jti) as { scope: string; revoked_at?: number | null; expires_at?: number | null } | undefined;
+  if (!row || row.revoked_at) return false;
+  const nowSeconds = Math.floor(now / 1000);
+  if (row.expires_at !== null && row.expires_at !== undefined && row.expires_at <= nowSeconds) return false;
+  const scopes = parseJson<string[]>(row.scope, []);
+  return scopes.includes("command:spawn") || scopes.includes("command:*") || scopes.includes("admin:*") || scopes.includes("*");
+}
+
+function isBoundToQuotaAccount(agent: AgentRecipientRow, provider: string, accountKey: string, accountHost: string | undefined, sourceHosts: Set<string>, sourceAgentIds: string[]): boolean {
+  if (sourceAgentIds.includes(agent.id)) return true;
+  const meta = parseJson<Record<string, unknown>>(agent.meta ?? "{}", {});
+  if (stringMeta(meta.provider)?.toLowerCase() === provider && accountKeyMatches(meta, accountKey)) return true;
+  if (accountKeyMatches(meta, accountKey)) return true;
+  const host = stringMeta(agent.machine) ?? stringMeta(agent.rig);
+  if (!host) return false;
+  return host === accountHost || sourceHosts.has(host);
+}
+
+function accountKeyMatches(meta: Record<string, unknown>, accountKey: string): boolean {
+  return [stringMeta(meta.accountKey), stringMeta(meta.accountId), stringMeta(meta.orgId)].includes(accountKey);
+}
+
+function agentHosts(agentIds: string[]): Set<string> {
+  const hosts = new Set<string>();
+  if (agentIds.length === 0) return hosts;
+  const rows = getDb().query(`
+    SELECT machine, rig FROM agents
+    WHERE id IN (${agentIds.map(() => "?").join(",")})
+  `).all(...agentIds) as Array<{ machine?: string | null; rig?: string | null }>;
+  for (const row of rows) {
+    const host = stringMeta(row.machine) ?? stringMeta(row.rig);
+    if (host) hosts.add(host);
+  }
+  return hosts;
+}
+
+function crossProviderHeadroomHint(provider: string): string {
+  const rows = getDb().query(`
+    SELECT provider, quota_state
+    FROM provider_quotas
+    WHERE provider <> ? AND quota_state IS NOT NULL
+  `).all(provider) as Array<{ provider: string; quota_state: string }>;
+  const options = rows.flatMap((row) => {
+    const quota = parseJson<QuotaState>(row.quota_state, { windows: [], source: "probe", updatedAt: 0 });
+    return quota.windows.map((window) => ({
+      provider: row.provider,
+      window: window.name,
+      remainingPercent: percent(quotaRemaining(window.utilization)),
+    }));
+  }).sort((a, b) => b.remainingPercent - a.remainingPercent);
+  const best = options[0];
+  if (!best) return "No separate provider headroom is currently recorded.";
+  return `${providerLabel(best.provider)} pool separate, ${windowLabel(best.window)} at ${best.remainingPercent}% remaining - route new work there.`;
+}
+
+function isQuotaReset(
+  remaining: number,
+  previousWindow: QuotaState["windows"][number] | undefined,
+  window: QuotaState["windows"][number],
+  resetRemaining: number,
+): boolean {
+  if (remaining >= resetRemaining) return true;
+  if (!previousWindow?.resetsAt || !window.resetsAt || window.resetsAt <= previousWindow.resetsAt) return false;
+  return remaining - quotaRemaining(previousWindow.utilization) >= 0.5 && remaining >= 0.8;
+}
+
+function quotaRemainingBand(remaining: number, bandSize: number): number {
+  return Math.min(100, Math.max(0, percent(Math.ceil(remaining / bandSize) * bandSize)));
+}
+
+function quotaRemaining(utilization: number): number {
+  return Math.max(0, Math.min(1, 1 - utilization));
+}
+
+function percent(value: number): number {
+  return Math.round(value * 100);
+}
+
+function hostFromAccountKey(accountKey: string): string | undefined {
+  if (!accountKey.startsWith("host:")) return undefined;
+  const host = accountKey.slice("host:".length);
+  return host && !host.includes(":") ? host : undefined;
+}
+
+function stringMeta(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+
+function providerLabel(provider: string): string {
+  return provider.slice(0, 1).toUpperCase() + provider.slice(1);
+}
+
+function windowLabel(value: string): string {
+  if (value === "five_hour") return "5h";
+  if (value === "seven_day") return "7d";
+  return value.replace(/_/g, " ");
+}
+
+function formatDuration(ms: number): string {
+  if (!Number.isFinite(ms) || ms <= 0) return "now";
+  const minutes = Math.max(1, Math.round(ms / 60_000));
+  if (minutes < 60) return `${minutes}m`;
+  const hours = Math.round(minutes / 60);
+  if (hours < 48) return `${hours}h`;
+  return `${Math.round(hours / 24)}d`;
+}

package/src/quota-threshold-config.ts

@@ -0,0 +1,46 @@
+import { isRecord } from "agent-relay-sdk";
+import { ValidationError } from "./db/connection.ts";
+import type { NotificationsConfig } from "./types";
+
+export const DEFAULT_QUOTA_THRESHOLD_CONFIG: NotificationsConfig["quotaThreshold"] = {
+  enabled: true,
+  warnRemaining: 0.25,
+  criticalRemaining: 0.1,
+  resetRemaining: 0.99,
+  bandSize: 0.05,
+};
+
+export function validateQuotaThresholdConfig(value: unknown): NotificationsConfig["quotaThreshold"] {
+  const input = isRecord(value) ? value : {};
+  const warnRemaining = cleanFraction(input.warnRemaining, "quotaThreshold.warnRemaining", DEFAULT_QUOTA_THRESHOLD_CONFIG.warnRemaining);
+  const criticalRemaining = cleanFraction(input.criticalRemaining, "quotaThreshold.criticalRemaining", DEFAULT_QUOTA_THRESHOLD_CONFIG.criticalRemaining);
+  const resetRemaining = cleanFraction(input.resetRemaining, "quotaThreshold.resetRemaining", DEFAULT_QUOTA_THRESHOLD_CONFIG.resetRemaining);
+  const bandSize = cleanFraction(input.bandSize, "quotaThreshold.bandSize", DEFAULT_QUOTA_THRESHOLD_CONFIG.bandSize);
+  if (bandSize <= 0) throw new ValidationError("quotaThreshold.bandSize must be greater than 0");
+  if (criticalRemaining >= warnRemaining) {
+    throw new ValidationError("quotaThreshold.criticalRemaining must be less than quotaThreshold.warnRemaining");
+  }
+  if (resetRemaining <= warnRemaining) {
+    throw new ValidationError("quotaThreshold.resetRemaining must be greater than quotaThreshold.warnRemaining");
+  }
+  return {
+    enabled: input.enabled === undefined ? DEFAULT_QUOTA_THRESHOLD_CONFIG.enabled : cleanBoolean(input.enabled, "quotaThreshold.enabled"),
+    warnRemaining,
+    criticalRemaining,
+    resetRemaining,
+    bandSize,
+  };
+}
+
+function cleanBoolean(value: unknown, field: string): boolean {
+  if (typeof value !== "boolean") throw new ValidationError(`${field} must be a boolean`);
+  return value;
+}
+
+function cleanFraction(value: unknown, field: string, fallback: number): number {
+  if (value === undefined || value === null) return fallback;
+  if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) {
+    throw new ValidationError(`${field} must be a number between 0 and 1`);
+  }
+  return value;
+}

package/src/routes/index.ts

@@ -5,7 +5,7 @@ import { deleteAgentTerminalSession, postAgentAction, postAgentPermissionDecisio
 import { deleteArtifactById, getArtifactById, getArtifactContent, getArtifacts, headArtifactContent, postArtifact } from "./artifacts";
 import { deleteAutomationById, getAutomationById, getAutomationRuns, getAutomations, patchAutomation, postAutomation, postAutomationRun } from "./automations";
 import { deleteCommandById, deleteProviderModelRoute, getProviderConfigRoute, getProviderConfigsRoute, getProvidersRoute, patchProviderModelRoute, postProviderConfigTestRoute, putProviderConfigRoute, putProviderModelRoute } from "./provider-config";
-import { getProviderQuotasRoute, postProviderQuotaRoute } from "./provider-quotas";
+import { getProviderQuotasRoute, postProviderQuotaLeaseAcquireRoute, postProviderQuotaLeaseReleaseRoute, postProviderQuotaRoute } from "./provider-quotas";
 import { deleteConfigKey, getConfigKey, getConfigKeyHistory, getConfigNamespace, putConfigKey } from "./config";
 import { deleteInboxDraftRoute, getChatHistoryImportsRoute, getInboxStateRoute, patchInboxThreadState, postChatHistoryImportRoute, putInboxDraft } from "./inbox";
 import { deleteMemoryRoute, getMemories, getMemoryBrokerInfo, getMemoryById, getMemoryStats, patchMemory, postMemory, postMemoryInject, postMemorySearch } from "./memory";
@@ -90,6 +90,8 @@ const routes: Route[] = [
   route("POST", "/api/route", postRouteAdvice),
   route("GET", "/api/provider-quotas", getProviderQuotasRoute),
   route("POST", "/api/provider-quotas", postProviderQuotaRoute),
+  route("POST", "/api/orchestrators/:id/provider-quota-leases/acquire", postProviderQuotaLeaseAcquireRoute),
+  route("POST", "/api/orchestrators/:id/provider-quota-leases/release", postProviderQuotaLeaseReleaseRoute),
   route("GET", "/api/agents/spawn/directories", getHostDirectories),
   route("GET", "/api/agents/:id/chat", getAgentChatHistory),
   route("GET", "/api/agents/:id/context-snapshots", getAgentContextSnapshots),

package/src/routes/provider-quotas.ts

@@ -1,8 +1,8 @@
 import { isRecord } from "agent-relay-sdk";
-import { listProviderQuotas, upsertProviderQuota } from "../db";
+import { acquireProviderQuotaLease, getOrchestrator, listProviderQuotas, releaseProviderQuotaLease, upsertProviderQuota } from "../db";
 import { quotaStateFromUnknown, providerQuotaErrorFromUnknown } from "../quota";
 import { cleanString } from "../validation";
-import { error, json, parseBody, parseQueryInt, type Handler } from "./_shared";
+import { authorizeRoute, error, json, parseBody, parseQueryInt, type Handler } from "./_shared";
 import type { ProviderQuotaUpdateInput } from "../types";
 
 function cleanProviderQuotaUpdate(body: unknown): ProviderQuotaUpdateInput | string {
@@ -46,3 +46,57 @@ export const postProviderQuotaRoute: Handler = async (req) => {
   if (typeof input === "string") return error(input);
   return json(upsertProviderQuota(input), 201);
 };
+
+function cleanLeaseBody(body: unknown): {
+  provider: string;
+  accountKey: string;
+  leaseToken?: string;
+  ttlMs: number;
+} | string {
+  if (!isRecord(body)) return "JSON object body required";
+  const provider = cleanString(body.provider, "provider", { required: true, max: 80 })!;
+  const accountKey = cleanString(body.accountKey, "accountKey", { required: true, max: 240 })!;
+  const leaseToken = cleanString(body.leaseToken, "leaseToken", { max: 120 });
+  const ttlMs = typeof body.ttlMs === "number" && Number.isFinite(body.ttlMs)
+    ? Math.max(5_000, Math.min(10 * 60_000, Math.round(body.ttlMs)))
+    : 90_000;
+  return { provider, accountKey, ...(leaseToken ? { leaseToken } : {}), ttlMs };
+}
+
+export const postProviderQuotaLeaseAcquireRoute: Handler = async (req, params) => {
+  const orch = getOrchestrator(params.id!);
+  if (!orch) return error("orchestrator not found", 404);
+  const denied = authorizeRoute(req, { scope: "command:write", resource: { orchestratorId: orch.id } });
+  if (denied) return denied;
+  const parsed = await parseBody<unknown>(req);
+  if (!parsed.ok) return error(parsed.error, parsed.status);
+  const input = cleanLeaseBody(parsed.body);
+  if (typeof input === "string") return error(input);
+  return json(acquireProviderQuotaLease({
+    provider: input.provider,
+    accountKey: input.accountKey,
+    orchestratorId: orch.id,
+    leaseToken: input.leaseToken,
+    ttlMs: input.ttlMs,
+  }));
+};
+
+export const postProviderQuotaLeaseReleaseRoute: Handler = async (req, params) => {
+  const orch = getOrchestrator(params.id!);
+  if (!orch) return error("orchestrator not found", 404);
+  const denied = authorizeRoute(req, { scope: "command:write", resource: { orchestratorId: orch.id } });
+  if (denied) return denied;
+  const parsed = await parseBody<unknown>(req);
+  if (!parsed.ok) return error(parsed.error, parsed.status);
+  const input = cleanLeaseBody(parsed.body);
+  if (typeof input === "string") return error(input);
+  if (!input.leaseToken) return error("leaseToken required");
+  return json({
+    released: releaseProviderQuotaLease({
+      provider: input.provider,
+      accountKey: input.accountKey,
+      orchestratorId: orch.id,
+      leaseToken: input.leaseToken,
+    }),
+  });
+};

package/src/token-db.ts

@@ -67,7 +67,7 @@ const BUILT_IN_PROFILES: Array<Omit<TokenProfile, "createdAt" | "updatedAt">> =
     name: "Orchestrator Host",
     description: "Host supervisor access for spawn, command, log, and terminal operations.",
     role: "orchestrator",
-    scope: ["agent:read", "agent:write", "command:read", "command:write", "task:read", "task:write", "logs:read", "terminal:attach"],
+    scope: ["agent:read", "agent:write", "command:read", "command:write", "task:read", "task:write", "logs:read", "terminal:attach", "quota:write"],
     constraints: { logsRead: true, terminalAttach: true },
     ttlSeconds: 30 * 24 * 60 * 60,
     builtIn: true,