@mcoda/agents 0.1.27 → 0.1.28

package/dist/AgentService/AgentService.d.ts

@@ -1,9 +1,14 @@
 import { Agent, AgentAuthMetadata, AgentHealth, AgentPromptManifest } from "@mcoda/shared";
 import { GlobalRepository } from "@mcoda/db";
 import { AgentAdapter, InvocationRequest, InvocationResult } from "../adapters/AdapterTypes.js";
+interface AgentServiceOptions {
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+}
 export declare class AgentService {
     private repo;
-    constructor(repo: GlobalRepository);
+    private options;
+    constructor(repo: GlobalRepository, options?: AgentServiceOptions);
     static create(): Promise<AgentService>;
     close(): Promise<void>;
     resolveAgent(identifier: string): Promise<Agent>;
@@ -14,6 +19,18 @@ export declare class AgentService {
     private buildAdapterConfig;
     private resolveAdapterType;
     getAdapter(agent: Agent, adapterOverride?: string): Promise<AgentAdapter>;
+    private nowMs;
+    private sleepMs;
+    private sleepUntil;
+    private metric;
+    private estimateWindowResetMs;
+    private estimateResetMsFromWindowTypes;
+    private normalizeLimitKey;
+    private getAgentAvailability;
+    private listEquivalentAgents;
+    private findNextAvailableAgent;
+    private findEarliestResetMs;
+    private persistUsageLimitObservation;
     healthCheck(agentId: string): Promise<AgentHealth>;
     invoke(agentId: string, request: InvocationRequest): Promise<InvocationResult>;
     invokeStream(agentId: string, request: InvocationRequest): Promise<AsyncGenerator<InvocationResult>>;
@@ -21,4 +38,5 @@ export declare class AgentService {
     private recordInvocationFailure;
     private applyDocdexGuidance;
 }
+export {};
 //# sourceMappingURL=AgentService.d.ts.map

package/dist/AgentService/AgentService.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"AgentService.d.ts","sourceRoot":"","sources":["../../src/AgentService/AgentService.ts"],"names":[],"mappings":"AAGA,OAAO,EAAgB,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACzG,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAY7C,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AA6KhG,qBAAa,YAAY;IACX,OAAO,CAAC,IAAI;gBAAJ,IAAI,EAAE,gBAAgB;WAE7B,MAAM,IAAI,OAAO,CAAC,YAAY,CAAC;IAKtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;IAUhD,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IAIrE,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAInD,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;YAIpD,kBAAkB;YAMlB,kBAAkB;IA6BhC,OAAO,CAAC,kBAAkB;IA+BpB,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IA4CzE,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAmBlD,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAyB9E,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;YAyC5F,mBAAmB;YAkBnB,uBAAuB;YAwBvB,mBAAmB;CAYlC"}
+{"version":3,"file":"AgentService.d.ts","sourceRoot":"","sources":["../../src/AgentService/AgentService.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,EACL,iBAAiB,EACjB,WAAW,EACX,mBAAmB,EAKpB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAY7C,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAsLhG,UAAU,mBAAmB;IAC3B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC;AAED,qBAAa,YAAY;IAErB,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,OAAO;gBADP,IAAI,EAAE,gBAAgB,EACtB,OAAO,GAAE,mBAAwB;WAG9B,MAAM,IAAI,OAAO,CAAC,YAAY,CAAC;IAKtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;IAUhD,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IAIrE,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAInD,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;YAIpD,kBAAkB;YAMlB,kBAAkB;IA6BhC,OAAO,CAAC,kBAAkB;IA+BpB,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IA4C/E,OAAO,CAAC,KAAK;YAIC,OAAO;YASP,UAAU;IAQxB,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,qBAAqB;IAI7B,OAAO,CAAC,8BAA8B;IAStC,OAAO,CAAC,iBAAiB;YAOX,oBAAoB;YA6BpB,oBAAoB;YA6CpB,sBAAsB;YAatB,mBAAmB;YAUnB,4BAA4B;IAkCpC,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAmBlD,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2E9E,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;YA0I5F,mBAAmB;YAkBnB,uBAAuB;YAwBvB,mBAAmB;CAYlC"}

package/dist/AgentService/AgentService.js

@@ -1,7 +1,7 @@
 import { promises as fs } from "node:fs";
 import os from "node:os";
 import path from "node:path";
-import { CryptoHelper } from "@mcoda/shared";
+import { CryptoHelper, } from "@mcoda/shared";
 import { GlobalRepository } from "@mcoda/db";
 import { CodexAdapter } from "../adapters/codex/CodexAdapter.js";
 import { GeminiAdapter } from "../adapters/gemini/GeminiAdapter.js";
@@ -14,6 +14,7 @@ import { OpenAiCliAdapter } from "../adapters/openai/OpenAiCliAdapter.js";
 import { ZhipuApiAdapter } from "../adapters/zhipu/ZhipuApiAdapter.js";
 import { QaAdapter } from "../adapters/qa/QaAdapter.js";
 import { ClaudeAdapter } from "../adapters/claude/ClaudeAdapter.js";
+import { parseUsageLimitError } from "./UsageLimitParser.js";
 const CLI_BASED_ADAPTERS = new Set(["codex-cli", "gemini-cli", "openai-cli", "ollama-cli", "codali-cli", "claude-cli"]);
 const LOCAL_ADAPTERS = new Set(["local-model"]);
 const SUPPORTED_ADAPTERS = new Set([
@@ -43,6 +44,14 @@ let docdexGuidanceCache;
 let docdexGuidanceLoaded = false;
 const DOCDEX_JSON_ONLY_MARKERS = [/output json only/i, /return json only/i, /no prose, no analysis/i];
 const HANDOFF_END_MARKERS = [/^\s*END OF FILE\s*$/i, /^\s*\*\*\* End of File\s*$/i];
+const EQUIVALENCE_THRESHOLD = 1.5;
+const MAX_SLEEP_CHUNK_MS = 60 * 60 * 1000;
+const WINDOW_RESET_FALLBACK_MS = {
+    rolling_5h: 5 * 60 * 60 * 1000,
+    daily: 24 * 60 * 60 * 1000,
+    weekly: 7 * 24 * 60 * 60 * 1000,
+    other: 60 * 60 * 1000,
+};
 const isIoEnabled = () => {
     const raw = process.env[IO_ENV];
     if (!raw)
@@ -179,8 +188,9 @@ const normalizeDocdexGuidanceInput = (input, prefix) => {
     return `${prefix}${remainder}`;
 };
 export class AgentService {
-    constructor(repo) {
+    constructor(repo, options = {}) {
         this.repo = repo;
+        this.options = options;
     }
     static async create() {
         const repo = await GlobalRepository.create();
@@ -313,6 +323,180 @@ export class AgentService {
         }
         throw new Error(`Unsupported adapter type: ${adapterType}`);
     }
+    nowMs() {
+        return this.options.now ? this.options.now() : Date.now();
+    }
+    async sleepMs(ms) {
+        if (ms <= 0)
+            return;
+        if (this.options.sleep) {
+            await this.options.sleep(ms);
+            return;
+        }
+        await new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    async sleepUntil(timestampMs) {
+        while (true) {
+            const remaining = timestampMs - this.nowMs();
+            if (remaining <= 0)
+                return;
+            await this.sleepMs(Math.min(remaining, MAX_SLEEP_CHUNK_MS));
+        }
+    }
+    metric(value, fallback = 5) {
+        return Number.isFinite(value) ? Number(value) : fallback;
+    }
+    estimateWindowResetMs(windowType, nowMs) {
+        return nowMs + (WINDOW_RESET_FALLBACK_MS[windowType] ?? WINDOW_RESET_FALLBACK_MS.other);
+    }
+    estimateResetMsFromWindowTypes(windowTypes, nowMs) {
+        const normalized = windowTypes.length ? windowTypes : ["other"];
+        let earliest = Number.POSITIVE_INFINITY;
+        for (const windowType of normalized) {
+            earliest = Math.min(earliest, this.estimateWindowResetMs(windowType, nowMs));
+        }
+        return Number.isFinite(earliest) ? earliest : this.estimateWindowResetMs("other", nowMs);
+    }
+    normalizeLimitKey(agent) {
+        const model = (agent.defaultModel ?? "").trim().toLowerCase();
+        if (model.includes("codex-spark"))
+            return "codex-spark";
+        if (model.includes("codex"))
+            return "codex-main";
+        return model || (agent.slug ?? agent.id).trim().toLowerCase();
+    }
+    async getAgentAvailability(agent, nowMs) {
+        const limits = await this.repo.listAgentUsageLimits(agent.id);
+        if (!limits.length)
+            return { available: true };
+        const limitKey = this.normalizeLimitKey(agent);
+        const relevant = limits.filter((entry) => {
+            if (entry.limitScope === "model") {
+                return entry.limitKey === limitKey;
+            }
+            return true;
+        });
+        if (!relevant.length)
+            return { available: true };
+        let earliestResetMs;
+        for (const entry of relevant) {
+            if (entry.status !== "exhausted")
+                continue;
+            let resetMs = entry.resetAt ? Date.parse(entry.resetAt) : Number.NaN;
+            if (!Number.isFinite(resetMs)) {
+                resetMs = this.estimateWindowResetMs(entry.windowType, nowMs);
+            }
+            if (resetMs > nowMs) {
+                earliestResetMs = earliestResetMs === undefined ? resetMs : Math.min(earliestResetMs, resetMs);
+            }
+        }
+        if (earliestResetMs !== undefined)
+            return { available: false, earliestResetMs };
+        return { available: true };
+    }
+    async listEquivalentAgents(baseAgent) {
+        const baseCapabilities = await this.repo.getAgentCapabilities(baseAgent.id);
+        const allAgents = await this.repo.listAgents();
+        const healthRows = await this.repo.listAgentHealthSummary();
+        const healthByAgentId = new Map(healthRows.map((entry) => [entry.agentId, entry]));
+        const baseRating = this.metric(baseAgent.rating);
+        const baseReasoning = this.metric(baseAgent.reasoningRating, baseRating);
+        const baseComplexity = this.metric(baseAgent.maxComplexity);
+        const candidates = [];
+        for (const candidate of allAgents) {
+            if (candidate.id === baseAgent.id)
+                continue;
+            const health = healthByAgentId.get(candidate.id);
+            if (health?.status === "unreachable")
+                continue;
+            const candidateCapabilities = await this.repo.getAgentCapabilities(candidate.id);
+            const hasAllCapabilities = baseCapabilities.every((capability) => candidateCapabilities.includes(capability));
+            if (!hasAllCapabilities)
+                continue;
+            const candidateRating = this.metric(candidate.rating);
+            const candidateReasoning = this.metric(candidate.reasoningRating, candidateRating);
+            const candidateComplexity = this.metric(candidate.maxComplexity);
+            const ratingDiff = Math.abs(candidateRating - baseRating);
+            const reasoningDiff = Math.abs(candidateReasoning - baseReasoning);
+            const complexityDiff = Math.abs(candidateComplexity - baseComplexity);
+            if (ratingDiff > EQUIVALENCE_THRESHOLD)
+                continue;
+            if (reasoningDiff > EQUIVALENCE_THRESHOLD)
+                continue;
+            if (complexityDiff > EQUIVALENCE_THRESHOLD)
+                continue;
+            candidates.push({
+                agent: candidate,
+                distance: ratingDiff + reasoningDiff + complexityDiff,
+            });
+        }
+        candidates.sort((left, right) => {
+            if (left.distance !== right.distance)
+                return left.distance - right.distance;
+            const leftRating = this.metric(left.agent.rating);
+            const rightRating = this.metric(right.agent.rating);
+            if (rightRating !== leftRating)
+                return rightRating - leftRating;
+            const leftCost = Number.isFinite(left.agent.costPerMillion) ? Number(left.agent.costPerMillion) : Number.POSITIVE_INFINITY;
+            const rightCost = Number.isFinite(right.agent.costPerMillion)
+                ? Number(right.agent.costPerMillion)
+                : Number.POSITIVE_INFINITY;
+            if (leftCost !== rightCost)
+                return leftCost - rightCost;
+            return (left.agent.slug ?? left.agent.id).localeCompare(right.agent.slug ?? right.agent.id);
+        });
+        return candidates.map((entry) => entry.agent);
+    }
+    async findNextAvailableAgent(candidates, attemptedAgentIds, nowMs) {
+        for (const candidate of candidates) {
+            if (attemptedAgentIds.has(candidate.id))
+                continue;
+            const availability = await this.getAgentAvailability(candidate, nowMs);
+            if (availability.available)
+                return candidate;
+        }
+        return undefined;
+    }
+    async findEarliestResetMs(candidates, nowMs) {
+        let earliest;
+        for (const candidate of candidates) {
+            const availability = await this.getAgentAvailability(candidate, nowMs);
+            if (!availability.earliestResetMs)
+                continue;
+            earliest = earliest === undefined ? availability.earliestResetMs : Math.min(earliest, availability.earliestResetMs);
+        }
+        return earliest;
+    }
+    async persistUsageLimitObservation(agent, observation) {
+        const observedAt = new Date(this.nowMs()).toISOString();
+        const limitKey = this.normalizeLimitKey(agent);
+        const windowTypes = observation.windowTypes.length
+            ? observation.windowTypes
+            : ["other"];
+        const fallbackResetMs = this.estimateResetMsFromWindowTypes(windowTypes, this.nowMs());
+        const resolvedResetAt = observation.resetAt ?? new Date(fallbackResetMs).toISOString();
+        const resetAtSource = observation.resetAt
+            ? observation.resetAtSource ?? "unknown"
+            : "estimated_window_fallback";
+        const records = windowTypes.map((windowType) => ({
+            agentId: agent.id,
+            limitScope: "model",
+            limitKey,
+            windowType,
+            status: "exhausted",
+            resetAt: resolvedResetAt,
+            observedAt,
+            source: "invoke_error_parse",
+            details: {
+                message: observation.message,
+                rawText: observation.rawText.slice(0, 4000),
+                resetAtSource,
+                resetAtProvided: Boolean(observation.resetAt),
+                estimatedResetAt: observation.resetAt ? undefined : resolvedResetAt,
+            },
+        }));
+        return this.repo.upsertAgentUsageLimits(records);
+    }
     async healthCheck(agentId) {
         const agent = await this.resolveAgent(agentId);
         try {
@@ -333,71 +517,205 @@ export class AgentService {
         }
     }
     async invoke(agentId, request) {
-        const agent = await this.resolveAgent(agentId);
-        const adapter = await this.getAdapter(agent, request.adapterType);
-        if (!adapter.invoke) {
-            throw new Error("Adapter does not support invoke");
-        }
-        const withDocdex = await this.applyDocdexGuidance(request);
-        const enriched = await this.applyGatewayHandoff(withDocdex);
-        const ioEnabled = isIoEnabled();
-        if (ioEnabled) {
-            renderIoHeader(agent, enriched, "invoke");
-        }
-        try {
-            const result = await adapter.invoke(enriched);
-            if (ioEnabled) {
-                renderIoChunk(result);
-                renderIoEnd();
+        const baseAgent = await this.resolveAgent(agentId);
+        const equivalentAgents = await this.listEquivalentAgents(baseAgent);
+        const attemptedAgentIds = new Set();
+        const failoverEvents = [];
+        let activeAgent = baseAgent;
+        for (;;) {
+            attemptedAgentIds.add(activeAgent.id);
+            const adapter = await this.getAdapter(activeAgent, request.adapterType);
+            if (!adapter.invoke) {
+                throw new Error("Adapter does not support invoke");
             }
-            return result;
-        }
-        catch (error) {
-            await this.recordInvocationFailure(agent, error);
-            throw error;
-        }
-    }
-    async invokeStream(agentId, request) {
-        const agent = await this.resolveAgent(agentId);
-        const adapter = await this.getAdapter(agent, request.adapterType);
-        if (!adapter.invokeStream) {
-            throw new Error("Adapter does not support streaming");
-        }
-        const withDocdex = await this.applyDocdexGuidance(request);
-        const enriched = await this.applyGatewayHandoff(withDocdex);
-        const ioEnabled = isIoEnabled();
-        let generator;
-        try {
-            generator = await adapter.invokeStream(enriched);
-        }
-        catch (error) {
-            await this.recordInvocationFailure(agent, error);
-            throw error;
-        }
-        const recordFailure = (error) => this.recordInvocationFailure(agent, error);
-        async function* wrap() {
-            const streamIo = ioEnabled ? createStreamIoRenderer() : undefined;
+            const withDocdex = await this.applyDocdexGuidance(request);
+            const enriched = await this.applyGatewayHandoff(withDocdex);
+            const ioEnabled = isIoEnabled();
             if (ioEnabled) {
-                renderIoHeader(agent, enriched, "stream");
+                renderIoHeader(activeAgent, enriched, "invoke");
             }
             try {
-                for await (const chunk of generator) {
-                    if (ioEnabled) {
-                        streamIo?.push(chunk);
-                    }
-                    yield chunk;
+                const result = await adapter.invoke(enriched);
+                if (ioEnabled) {
+                    renderIoChunk(result);
+                    renderIoEnd();
                 }
+                if (!failoverEvents.length)
+                    return result;
+                return {
+                    ...result,
+                    metadata: {
+                        ...(result.metadata ?? {}),
+                        failoverEvents,
+                    },
+                };
             }
             catch (error) {
-                await recordFailure(error);
-                throw error;
+                await this.recordInvocationFailure(activeAgent, error);
+                const parsedLimit = parseUsageLimitError(error, this.nowMs());
+                if (!parsedLimit) {
+                    throw error;
+                }
+                await this.persistUsageLimitObservation(activeAgent, parsedLimit);
+                const nowMs = this.nowMs();
+                const nextAgent = await this.findNextAvailableAgent([baseAgent, ...equivalentAgents], attemptedAgentIds, nowMs);
+                if (nextAgent) {
+                    failoverEvents.push({
+                        type: "switch_agent",
+                        at: new Date(nowMs).toISOString(),
+                        fromAgentId: activeAgent.id,
+                        fromAgentSlug: activeAgent.slug,
+                        toAgentId: nextAgent.id,
+                        toAgentSlug: nextAgent.slug,
+                    });
+                    activeAgent = nextAgent;
+                    continue;
+                }
+                const earliestResetMs = await this.findEarliestResetMs([baseAgent, ...equivalentAgents], nowMs);
+                const fallbackResetMs = this.estimateResetMsFromWindowTypes(parsedLimit.windowTypes, nowMs);
+                const waitUntilMs = earliestResetMs && earliestResetMs > nowMs ? earliestResetMs : Math.max(fallbackResetMs, nowMs + 1000);
+                const durationMs = waitUntilMs - nowMs;
+                failoverEvents.push({
+                    type: "sleep_until_reset",
+                    at: new Date(nowMs).toISOString(),
+                    until: new Date(waitUntilMs).toISOString(),
+                    durationMs,
+                });
+                await this.sleepUntil(waitUntilMs);
+                attemptedAgentIds.clear();
+                activeAgent = baseAgent;
             }
-            if (ioEnabled) {
-                streamIo?.flush();
-                renderIoEnd();
+        }
+    }
+    async invokeStream(agentId, request) {
+        const baseAgent = await this.resolveAgent(agentId);
+        const equivalentAgents = await this.listEquivalentAgents(baseAgent);
+        const attemptedAgentIds = new Set();
+        const self = this;
+        async function* run() {
+            let activeAgent = baseAgent;
+            const failoverEvents = [];
+            for (;;) {
+                attemptedAgentIds.add(activeAgent.id);
+                const adapter = await self.getAdapter(activeAgent, request.adapterType);
+                if (!adapter.invokeStream) {
+                    throw new Error("Adapter does not support streaming");
+                }
+                const withDocdex = await self.applyDocdexGuidance(request);
+                const enriched = await self.applyGatewayHandoff(withDocdex);
+                const ioEnabled = isIoEnabled();
+                let generator;
+                try {
+                    generator = await adapter.invokeStream(enriched);
+                }
+                catch (error) {
+                    await self.recordInvocationFailure(activeAgent, error);
+                    const parsedLimit = parseUsageLimitError(error, self.nowMs());
+                    if (!parsedLimit)
+                        throw error;
+                    await self.persistUsageLimitObservation(activeAgent, parsedLimit);
+                    const nowMs = self.nowMs();
+                    const nextAgent = await self.findNextAvailableAgent([baseAgent, ...equivalentAgents], attemptedAgentIds, nowMs);
+                    if (nextAgent) {
+                        failoverEvents.push({
+                            type: "switch_agent",
+                            at: new Date(nowMs).toISOString(),
+                            fromAgentId: activeAgent.id,
+                            fromAgentSlug: activeAgent.slug,
+                            toAgentId: nextAgent.id,
+                            toAgentSlug: nextAgent.slug,
+                        });
+                        activeAgent = nextAgent;
+                        continue;
+                    }
+                    const earliestResetMs = await self.findEarliestResetMs([baseAgent, ...equivalentAgents], nowMs);
+                    const fallbackResetMs = self.estimateResetMsFromWindowTypes(parsedLimit.windowTypes, nowMs);
+                    const waitUntilMs = earliestResetMs && earliestResetMs > nowMs ? earliestResetMs : Math.max(fallbackResetMs, nowMs + 1000);
+                    const durationMs = waitUntilMs - nowMs;
+                    failoverEvents.push({
+                        type: "sleep_until_reset",
+                        at: new Date(nowMs).toISOString(),
+                        until: new Date(waitUntilMs).toISOString(),
+                        durationMs,
+                    });
+                    await self.sleepUntil(waitUntilMs);
+                    attemptedAgentIds.clear();
+                    activeAgent = baseAgent;
+                    continue;
+                }
+                const streamIo = ioEnabled ? createStreamIoRenderer() : undefined;
+                if (ioEnabled) {
+                    renderIoHeader(activeAgent, enriched, "stream");
+                }
+                let emitted = false;
+                try {
+                    for await (const chunk of generator) {
+                        emitted = true;
+                        const chunkWithMetadata = failoverEvents.length > 0
+                            ? {
+                                ...chunk,
+                                metadata: {
+                                    ...(chunk.metadata ?? {}),
+                                    failoverEvents: failoverEvents.map((event) => ({ ...event })),
+                                },
+                            }
+                            : chunk;
+                        if (ioEnabled) {
+                            streamIo?.push(chunkWithMetadata);
+                        }
+                        yield chunkWithMetadata;
+                    }
+                    if (ioEnabled) {
+                        streamIo?.flush();
+                        renderIoEnd();
+                    }
+                    return;
+                }
+                catch (error) {
+                    await self.recordInvocationFailure(activeAgent, error);
+                    const parsedLimit = parseUsageLimitError(error, self.nowMs());
+                    if (!parsedLimit)
+                        throw error;
+                    const nowMs = self.nowMs();
+                    if (emitted) {
+                        failoverEvents.push({
+                            type: "stream_restart_after_limit",
+                            at: new Date(nowMs).toISOString(),
+                            fromAgentId: activeAgent.id,
+                            fromAgentSlug: activeAgent.slug,
+                        });
+                    }
+                    await self.persistUsageLimitObservation(activeAgent, parsedLimit);
+                    const nextAgent = await self.findNextAvailableAgent([baseAgent, ...equivalentAgents], attemptedAgentIds, nowMs);
+                    if (nextAgent) {
+                        failoverEvents.push({
+                            type: "switch_agent",
+                            at: new Date(nowMs).toISOString(),
+                            fromAgentId: activeAgent.id,
+                            fromAgentSlug: activeAgent.slug,
+                            toAgentId: nextAgent.id,
+                            toAgentSlug: nextAgent.slug,
+                        });
+                        activeAgent = nextAgent;
+                        continue;
+                    }
+                    const earliestResetMs = await self.findEarliestResetMs([baseAgent, ...equivalentAgents], nowMs);
+                    const fallbackResetMs = self.estimateResetMsFromWindowTypes(parsedLimit.windowTypes, nowMs);
+                    const waitUntilMs = earliestResetMs && earliestResetMs > nowMs ? earliestResetMs : Math.max(fallbackResetMs, nowMs + 1000);
+                    const durationMs = waitUntilMs - nowMs;
+                    failoverEvents.push({
+                        type: "sleep_until_reset",
+                        at: new Date(nowMs).toISOString(),
+                        until: new Date(waitUntilMs).toISOString(),
+                        durationMs,
+                    });
+                    await self.sleepUntil(waitUntilMs);
+                    attemptedAgentIds.clear();
+                    activeAgent = baseAgent;
+                }
             }
         }
-        return wrap();
+        return run();
     }
     async applyGatewayHandoff(request) {
         if (request.metadata?.command === "gateway-agent") {

package/dist/AgentService/UsageLimitParser.d.ts

@@ -0,0 +1,13 @@
+import { AgentUsageLimitWindowType } from "@mcoda/shared";
+export type UsageLimitResetSource = "header" | "relative" | "absolute";
+export interface ParsedUsageLimitError {
+    isUsageLimit: true;
+    message: string;
+    rawText: string;
+    windowTypes: AgentUsageLimitWindowType[];
+    resetAt?: string;
+    resetAtSource?: UsageLimitResetSource;
+}
+export declare const extractUsageLimitErrorText: (error: unknown) => string;
+export declare const parseUsageLimitError: (error: unknown, nowMs?: number) => ParsedUsageLimitError | null;
+//# sourceMappingURL=UsageLimitParser.d.ts.map

package/dist/AgentService/UsageLimitParser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"UsageLimitParser.d.ts","sourceRoot":"","sources":["../../src/AgentService/UsageLimitParser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAqC1D,MAAM,MAAM,qBAAqB,GAAG,QAAQ,GAAG,UAAU,GAAG,UAAU,CAAC;AAEvE,MAAM,WAAW,qBAAqB;IACpC,YAAY,EAAE,IAAI,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,yBAAyB,EAAE,CAAC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,qBAAqB,CAAC;CACvC;AAkJD,eAAO,MAAM,0BAA0B,GAAI,OAAO,OAAO,KAAG,MAI3D,CAAC;AAEF,eAAO,MAAM,oBAAoB,GAAI,OAAO,OAAO,EAAE,cAAkB,KAAG,qBAAqB,GAAG,IA6BjG,CAAC"}

package/dist/AgentService/UsageLimitParser.js

@@ -0,0 +1,197 @@
+const LIMIT_PATTERNS = [
+    /usage[_\s-]*limit/i,
+    /usage_limit_reached/i,
+    /rate[_\s-]*limit/i,
+    /too many requests/i,
+    /\b429\b/,
+    /quota exceeded/i,
+    /retry after/i,
+];
+const NON_LIMIT_AUTH_PATTERNS = [
+    /invalid[_\s-]*api[_\s-]*key/i,
+    /api key.*invalid/i,
+    /authentication failed/i,
+    /missing api key/i,
+    /unauthorized/i,
+];
+const WINDOW_HINTS = [
+    { pattern: /\b(?:5\s*(?:h|hr|hrs|hour|hours)|five[-\s]?hour)\b/i, windowType: "rolling_5h" },
+    { pattern: /\b(?:daily|24\s*(?:h|hr|hrs|hour|hours))\b/i, windowType: "daily" },
+    { pattern: /\b(?:weekly|7\s*(?:d|day|days))\b/i, windowType: "weekly" },
+];
+const RELATIVE_RESET_PATTERNS = [
+    /retry after\s+([^\n.;]+)/i,
+    /try again in\s+([^\n.;]+)/i,
+    /resets?\s+in\s+([^\n.;]+)/i,
+];
+const ISO_LIKE_PATTERN = /\b(20\d{2}-\d{2}-\d{2}[tT ]\d{2}:\d{2}(?::\d{2})?(?:\.\d{1,3})?(?:Z|[+-]\d{2}:?\d{2}| ?UTC| ?GMT)?)\b/g;
+const EPOCH_RESET_PATTERN = /\bx-ratelimit-reset(?:-at)?\s*[:=]\s*(\d{10,13})\b/i;
+const SHORT_RESET_PATTERN = /\bx-ratelimit-reset-after\s*[:=]\s*(\d+)\b/i;
+const normalizeSpace = (value) => value.replace(/\s+/g, " ").trim();
+const collectText = (value, out, depth = 0) => {
+    if (depth > 4 || value === null || value === undefined)
+        return;
+    if (typeof value === "string") {
+        const trimmed = value.trim();
+        if (trimmed)
+            out.push(trimmed);
+        return;
+    }
+    if (typeof value === "number" || typeof value === "boolean") {
+        out.push(String(value));
+        return;
+    }
+    if (value instanceof Error) {
+        collectText(value.message, out, depth + 1);
+        const extra = value;
+        for (const [key, nested] of Object.entries(extra)) {
+            if (key === "message" || key === "name" || key === "stack")
+                continue;
+            collectText(nested, out, depth + 1);
+        }
+        return;
+    }
+    if (Array.isArray(value)) {
+        for (const item of value)
+            collectText(item, out, depth + 1);
+        return;
+    }
+    if (typeof value === "object") {
+        for (const [key, nested] of Object.entries(value)) {
+            if (/(error|message|reason|stderr|stdout|detail|details|body)/i.test(key)) {
+                collectText(nested, out, depth + 1);
+                continue;
+            }
+            if (depth < 2) {
+                collectText(nested, out, depth + 1);
+            }
+        }
+    }
+};
+const parseDurationMs = (input) => {
+    let totalMs = 0;
+    let matched = false;
+    const pattern = /(\d+)\s*(weeks?|w|days?|d|hours?|hrs?|h|minutes?|mins?|m|seconds?|secs?|s)\b/gi;
+    let match = pattern.exec(input);
+    while (match) {
+        const amount = Number.parseInt(match[1] ?? "", 10);
+        const unit = (match[2] ?? "").toLowerCase();
+        if (Number.isFinite(amount) && amount > 0) {
+            matched = true;
+            if (unit.startsWith("w"))
+                totalMs += amount * 7 * 24 * 60 * 60 * 1000;
+            else if (unit.startsWith("d"))
+                totalMs += amount * 24 * 60 * 60 * 1000;
+            else if (unit.startsWith("h"))
+                totalMs += amount * 60 * 60 * 1000;
+            else if (unit.startsWith("m"))
+                totalMs += amount * 60 * 1000;
+            else if (unit.startsWith("s"))
+                totalMs += amount * 1000;
+        }
+        match = pattern.exec(input);
+    }
+    return matched && totalMs > 0 ? totalMs : undefined;
+};
+const parseIsoLikeDate = (candidate) => {
+    const normalized = candidate.replace(/\bUTC\b/i, "Z").replace(/\bGMT\b/i, "Z").replace(" ", "T");
+    const parsed = Date.parse(normalized);
+    return Number.isFinite(parsed) ? parsed : undefined;
+};
+const resolveResetTimestamp = (text, nowMs) => {
+    const candidates = [];
+    const epochMatch = EPOCH_RESET_PATTERN.exec(text);
+    if (epochMatch?.[1]) {
+        const raw = Number.parseInt(epochMatch[1], 10);
+        if (Number.isFinite(raw) && raw > 0) {
+            candidates.push({
+                timestampMs: raw > 1000000000000 ? raw : raw * 1000,
+                source: "header",
+            });
+        }
+    }
+    const shortMatch = SHORT_RESET_PATTERN.exec(text);
+    if (shortMatch?.[1]) {
+        const seconds = Number.parseInt(shortMatch[1], 10);
+        if (Number.isFinite(seconds) && seconds > 0) {
+            candidates.push({
+                timestampMs: nowMs + seconds * 1000,
+                source: "header",
+            });
+        }
+    }
+    for (const pattern of RELATIVE_RESET_PATTERNS) {
+        const match = pattern.exec(text);
+        const durationText = match?.[1];
+        if (!durationText)
+            continue;
+        const durationMs = parseDurationMs(durationText);
+        if (durationMs && durationMs > 0) {
+            candidates.push({
+                timestampMs: nowMs + durationMs,
+                source: "relative",
+            });
+        }
+    }
+    ISO_LIKE_PATTERN.lastIndex = 0;
+    let isoMatch = ISO_LIKE_PATTERN.exec(text);
+    while (isoMatch) {
+        const ts = parseIsoLikeDate(isoMatch[1] ?? "");
+        if (Number.isFinite(ts)) {
+            candidates.push({
+                timestampMs: ts,
+                source: "absolute",
+            });
+        }
+        isoMatch = ISO_LIKE_PATTERN.exec(text);
+    }
+    const future = candidates.filter((candidate) => Number.isFinite(candidate.timestampMs) && candidate.timestampMs > nowMs);
+    if (!future.length)
+        return undefined;
+    future.sort((left, right) => left.timestampMs - right.timestampMs);
+    return future[0];
+};
+const inferWindowTypes = (text) => {
+    const set = new Set();
+    for (const hint of WINDOW_HINTS) {
+        if (hint.pattern.test(text)) {
+            set.add(hint.windowType);
+        }
+    }
+    if (!set.size) {
+        set.add("other");
+    }
+    return Array.from(set);
+};
+export const extractUsageLimitErrorText = (error) => {
+    const parts = [];
+    collectText(error, parts);
+    return normalizeSpace(parts.join(" "));
+};
+export const parseUsageLimitError = (error, nowMs = Date.now()) => {
+    const message = error instanceof Error ? error.message : String(error ?? "");
+    const rawText = extractUsageLimitErrorText(error);
+    const messageLower = message.trim().toLowerCase();
+    const rawLower = rawText.trim().toLowerCase();
+    const text = normalizeSpace(rawText && rawLower.includes(messageLower) ? rawText : `${message} ${rawText}`);
+    if (!text)
+        return null;
+    const looksLikeLimit = LIMIT_PATTERNS.some((pattern) => pattern.test(text));
+    if (!looksLikeLimit)
+        return null;
+    const looksLikeOnlyNonLimitAuth = NON_LIMIT_AUTH_PATTERNS.some((pattern) => pattern.test(text)) &&
+        !/(usage[_\s-]*limit|rate[_\s-]*limit|too many requests|\b429\b)/i.test(text);
+    if (looksLikeOnlyNonLimitAuth) {
+        return null;
+    }
+    const reset = resolveResetTimestamp(text, nowMs);
+    const windowTypes = inferWindowTypes(text);
+    return {
+        isUsageLimit: true,
+        message,
+        rawText: text,
+        windowTypes,
+        resetAt: reset ? new Date(reset.timestampMs).toISOString() : undefined,
+        resetAtSource: reset?.source,
+    };
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mcoda/agents",
-  "version": "0.1.27",
+  "version": "0.1.28",
   "description": "Agent registry and capabilities for mcoda.",
   "type": "module",
   "main": "dist/index.js",
@@ -30,8 +30,8 @@
     "access": "public"
   },
   "dependencies": {
-    "@mcoda/shared": "0.1.27",
-    "@mcoda/db": "0.1.27"
+    "@mcoda/shared": "0.1.28",
+    "@mcoda/db": "0.1.28"
   },
   "scripts": {
     "build": "tsc -p tsconfig.json",