@mcoda/agents 0.1.28 → 0.1.29

package/dist/AgentService/AgentService.d.ts

@@ -4,6 +4,8 @@ import { AgentAdapter, InvocationRequest, InvocationResult } from "../adapters/A
 interface AgentServiceOptions {
     now?: () => number;
     sleep?: (ms: number) => Promise<void>;
+    checkInternetReachable?: () => Promise<boolean>;
+    connectivityPollIntervalMs?: number;
 }
 export declare class AgentService {
     private repo;
@@ -22,6 +24,10 @@ export declare class AgentService {
     private nowMs;
     private sleepMs;
     private sleepUntil;
+    private getConnectivityPollIntervalMs;
+    private isInternetReachable;
+    private waitForInternetRecovery;
+    private isOfflineCapable;
     private metric;
     private estimateWindowResetMs;
     private estimateResetMsFromWindowTypes;

package/dist/AgentService/AgentService.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"AgentService.d.ts","sourceRoot":"","sources":["../../src/AgentService/AgentService.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,EACL,iBAAiB,EACjB,WAAW,EACX,mBAAmB,EAKpB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAY7C,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAsLhG,UAAU,mBAAmB;IAC3B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC;AAED,qBAAa,YAAY;IAErB,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,OAAO;gBADP,IAAI,EAAE,gBAAgB,EACtB,OAAO,GAAE,mBAAwB;WAG9B,MAAM,IAAI,OAAO,CAAC,YAAY,CAAC;IAKtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;IAUhD,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IAIrE,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAInD,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;YAIpD,kBAAkB;YAMlB,kBAAkB;IA6BhC,OAAO,CAAC,kBAAkB;IA+BpB,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IA4C/E,OAAO,CAAC,KAAK;YAIC,OAAO;YASP,UAAU;IAQxB,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,qBAAqB;IAI7B,OAAO,CAAC,8BAA8B;IAStC,OAAO,CAAC,iBAAiB;YAOX,oBAAoB;YA6BpB,oBAAoB;YA6CpB,sBAAsB;YAatB,mBAAmB;YAUnB,4BAA4B;IAkCpC,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAmBlD,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2E9E,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;YA0I5F,mBAAmB;YAkBnB,uBAAuB;YAwBvB,mBAAmB;CAYlC"}
+{"version":3,"file":"AgentService.d.ts","sourceRoot":"","sources":["../../src/AgentService/AgentService.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,EACL,iBAAiB,EACjB,WAAW,EACX,mBAAmB,EAKpB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAY7C,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AA2LhG,UAAU,mBAAmB;IAC3B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,sBAAsB,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,qBAAa,YAAY;IAErB,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,OAAO;gBADP,IAAI,EAAE,gBAAgB,EACtB,OAAO,GAAE,mBAAwB;WAG9B,MAAM,IAAI,OAAO,CAAC,YAAY,CAAC;IAKtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;IAUhD,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IAIrE,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAInD,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;YAIpD,kBAAkB;YAMlB,kBAAkB;IA6BhC,OAAO,CAAC,kBAAkB;IA+BpB,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IA4C/E,OAAO,CAAC,KAAK;YAIC,OAAO;YASP,UAAU;IAQxB,OAAO,CAAC,6BAA6B;YAOvB,mBAAmB;YAgCnB,uBAAuB;YAiBvB,gBAAgB;IAM9B,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,qBAAqB;IAI7B,OAAO,CAAC,8BAA8B;IAStC,OAAO,CAAC,iBAAiB;YAOX,oBAAoB;YA6BpB,oBAAoB;YA6CpB,sBAAsB;YAkBtB,mBAAmB;YAUnB,4BAA4B;IAkCpC,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAmBlD,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6H9E,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;YAwP5F,mBAAmB;YAkBnB,uBAAuB;YAwBvB,mBAAmB;CAYlC"}

package/dist/AgentService/AgentService.js

@@ -14,9 +14,10 @@ import { OpenAiCliAdapter } from "../adapters/openai/OpenAiCliAdapter.js";
 import { ZhipuApiAdapter } from "../adapters/zhipu/ZhipuApiAdapter.js";
 import { QaAdapter } from "../adapters/qa/QaAdapter.js";
 import { ClaudeAdapter } from "../adapters/claude/ClaudeAdapter.js";
-import { parseUsageLimitError } from "./UsageLimitParser.js";
+import { parseInvocationFailure } from "./InvocationFailureParser.js";
 const CLI_BASED_ADAPTERS = new Set(["codex-cli", "gemini-cli", "openai-cli", "ollama-cli", "codali-cli", "claude-cli"]);
 const LOCAL_ADAPTERS = new Set(["local-model"]);
+const OFFLINE_CAPABLE_ADAPTERS = new Set(["local-model", "ollama-cli", "qa-cli"]);
 const SUPPORTED_ADAPTERS = new Set([
     "openai-api",
     "codex-cli",
@@ -46,6 +47,9 @@ const DOCDEX_JSON_ONLY_MARKERS = [/output json only/i, /return json only/i, /no
 const HANDOFF_END_MARKERS = [/^\s*END OF FILE\s*$/i, /^\s*\*\*\* End of File\s*$/i];
 const EQUIVALENCE_THRESHOLD = 1.5;
 const MAX_SLEEP_CHUNK_MS = 60 * 60 * 1000;
+const DEFAULT_CONNECTIVITY_POLL_INTERVAL_MS = 5000;
+const INTERNET_CHECK_TIMEOUT_MS = 2500;
+const INTERNET_CHECK_TARGETS = ["https://clients3.google.com/generate_204", "https://www.cloudflare.com/cdn-cgi/trace"];
 const WINDOW_RESET_FALLBACK_MS = {
     rolling_5h: 5 * 60 * 60 * 1000,
     daily: 24 * 60 * 60 * 1000,
@@ -343,6 +347,68 @@ export class AgentService {
             await this.sleepMs(Math.min(remaining, MAX_SLEEP_CHUNK_MS));
         }
     }
+    getConnectivityPollIntervalMs() {
+        const configured = this.options.connectivityPollIntervalMs;
+        if (!Number.isFinite(configured))
+            return DEFAULT_CONNECTIVITY_POLL_INTERVAL_MS;
+        const normalized = Math.floor(Number(configured));
+        return normalized > 0 ? normalized : DEFAULT_CONNECTIVITY_POLL_INTERVAL_MS;
+    }
+    async isInternetReachable() {
+        if (this.options.checkInternetReachable) {
+            try {
+                return Boolean(await this.options.checkInternetReachable());
+            }
+            catch {
+                return false;
+            }
+        }
+        const globalFetch = globalThis.fetch;
+        if (typeof globalFetch !== "function")
+            return false;
+        for (const target of INTERNET_CHECK_TARGETS) {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), INTERNET_CHECK_TIMEOUT_MS);
+            try {
+                const response = await globalFetch(target, {
+                    method: "HEAD",
+                    cache: "no-store",
+                    signal: controller.signal,
+                });
+                if (response.status >= 200 && response.status < 500) {
+                    return true;
+                }
+            }
+            catch {
+                // keep probing next endpoint.
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+        }
+        return false;
+    }
+    async waitForInternetRecovery() {
+        const pollMs = this.getConnectivityPollIntervalMs();
+        const startedAtMs = this.nowMs();
+        for (;;) {
+            const reachable = await this.isInternetReachable();
+            if (reachable) {
+                const recoveredAtMs = this.nowMs();
+                return {
+                    startedAtMs,
+                    recoveredAtMs,
+                    durationMs: Math.max(0, recoveredAtMs - startedAtMs),
+                };
+            }
+            await this.sleepMs(pollMs);
+        }
+    }
+    async isOfflineCapable(agent, adapterOverride) {
+        const secret = await this.getDecryptedSecret(agent.id);
+        const adapterType = this.resolveAdapterType(agent, secret, adapterOverride);
+        return OFFLINE_CAPABLE_ADAPTERS.has(adapterType);
+    }
     metric(value, fallback = 5) {
         return Number.isFinite(value) ? Number(value) : fallback;
     }
@@ -447,10 +513,15 @@ export class AgentService {
         });
         return candidates.map((entry) => entry.agent);
     }
-    async findNextAvailableAgent(candidates, attemptedAgentIds, nowMs) {
+    async findNextAvailableAgent(candidates, attemptedAgentIds, nowMs, options) {
         for (const candidate of candidates) {
             if (attemptedAgentIds.has(candidate.id))
                 continue;
+            if (options?.offlineCapableOnly) {
+                const offlineCapable = await this.isOfflineCapable(candidate, options.adapterOverride);
+                if (!offlineCapable)
+                    continue;
+            }
             const availability = await this.getAgentAvailability(candidate, nowMs);
             if (availability.available)
                 return candidate;
@@ -552,38 +623,90 @@ export class AgentService {
             }
             catch (error) {
                 await this.recordInvocationFailure(activeAgent, error);
-                const parsedLimit = parseUsageLimitError(error, this.nowMs());
-                if (!parsedLimit) {
+                const nowMs = this.nowMs();
+                const parsedFailure = parseInvocationFailure(error, nowMs);
+                if (!parsedFailure) {
                     throw error;
                 }
-                await this.persistUsageLimitObservation(activeAgent, parsedLimit);
-                const nowMs = this.nowMs();
-                const nextAgent = await this.findNextAvailableAgent([baseAgent, ...equivalentAgents], attemptedAgentIds, nowMs);
-                if (nextAgent) {
+                const candidates = [baseAgent, ...equivalentAgents];
+                if (parsedFailure.kind === "usage_limit") {
+                    await this.persistUsageLimitObservation(activeAgent, parsedFailure.usageLimit);
+                    const nextAgent = await this.findNextAvailableAgent(candidates, attemptedAgentIds, nowMs);
+                    if (nextAgent) {
+                        failoverEvents.push({
+                            type: "switch_agent",
+                            at: new Date(nowMs).toISOString(),
+                            fromAgentId: activeAgent.id,
+                            fromAgentSlug: activeAgent.slug,
+                            toAgentId: nextAgent.id,
+                            toAgentSlug: nextAgent.slug,
+                            reason: parsedFailure.kind,
+                        });
+                        activeAgent = nextAgent;
+                        continue;
+                    }
+                    const earliestResetMs = await this.findEarliestResetMs(candidates, nowMs);
+                    const fallbackResetMs = this.estimateResetMsFromWindowTypes(parsedFailure.usageLimit.windowTypes, nowMs);
+                    const waitUntilMs = earliestResetMs && earliestResetMs > nowMs ? earliestResetMs : Math.max(fallbackResetMs, nowMs + 1000);
+                    const durationMs = waitUntilMs - nowMs;
                     failoverEvents.push({
-                        type: "switch_agent",
+                        type: "sleep_until_reset",
                         at: new Date(nowMs).toISOString(),
-                        fromAgentId: activeAgent.id,
-                        fromAgentSlug: activeAgent.slug,
-                        toAgentId: nextAgent.id,
-                        toAgentSlug: nextAgent.slug,
+                        until: new Date(waitUntilMs).toISOString(),
+                        durationMs,
                     });
-                    activeAgent = nextAgent;
+                    await this.sleepUntil(waitUntilMs);
+                    attemptedAgentIds.clear();
+                    activeAgent = baseAgent;
                     continue;
                 }
-                const earliestResetMs = await this.findEarliestResetMs([baseAgent, ...equivalentAgents], nowMs);
-                const fallbackResetMs = this.estimateResetMsFromWindowTypes(parsedLimit.windowTypes, nowMs);
-                const waitUntilMs = earliestResetMs && earliestResetMs > nowMs ? earliestResetMs : Math.max(fallbackResetMs, nowMs + 1000);
-                const durationMs = waitUntilMs - nowMs;
+                if (parsedFailure.kind === "connectivity_issue") {
+                    const internetReachable = await this.isInternetReachable();
+                    if (!internetReachable) {
+                        const localFallback = await this.findNextAvailableAgent(candidates, attemptedAgentIds, nowMs, {
+                            offlineCapableOnly: true,
+                            adapterOverride: request.adapterType,
+                        });
+                        if (localFallback) {
+                            failoverEvents.push({
+                                type: "switch_agent",
+                                at: new Date(nowMs).toISOString(),
+                                fromAgentId: activeAgent.id,
+                                fromAgentSlug: activeAgent.slug,
+                                toAgentId: localFallback.id,
+                                toAgentSlug: localFallback.slug,
+                                reason: parsedFailure.kind,
+                            });
+                            activeAgent = localFallback;
+                            continue;
+                        }
+                        const recovery = await this.waitForInternetRecovery();
+                        failoverEvents.push({
+                            type: "wait_for_internet",
+                            at: new Date(recovery.startedAtMs).toISOString(),
+                            until: new Date(recovery.recoveredAtMs).toISOString(),
+                            durationMs: recovery.durationMs,
+                            pollIntervalMs: this.getConnectivityPollIntervalMs(),
+                        });
+                        attemptedAgentIds.clear();
+                        activeAgent = baseAgent;
+                        continue;
+                    }
+                }
+                const nextAgent = await this.findNextAvailableAgent(candidates, attemptedAgentIds, nowMs);
+                if (!nextAgent) {
+                    throw error;
+                }
                 failoverEvents.push({
-                    type: "sleep_until_reset",
+                    type: "switch_agent",
                     at: new Date(nowMs).toISOString(),
-                    until: new Date(waitUntilMs).toISOString(),
-                    durationMs,
+                    fromAgentId: activeAgent.id,
+                    fromAgentSlug: activeAgent.slug,
+                    toAgentId: nextAgent.id,
+                    toAgentSlug: nextAgent.slug,
+                    reason: parsedFailure.kind,
                 });
-                await this.sleepUntil(waitUntilMs);
-                attemptedAgentIds.clear();
-                activeAgent = baseAgent;
+                activeAgent = nextAgent;
             }
         }
     }
@@ -610,37 +733,89 @@ export class AgentService {
                 }
                 catch (error) {
                     await self.recordInvocationFailure(activeAgent, error);
-                    const parsedLimit = parseUsageLimitError(error, self.nowMs());
-                    if (!parsedLimit)
-                        throw error;
-                    await self.persistUsageLimitObservation(activeAgent, parsedLimit);
                     const nowMs = self.nowMs();
-                    const nextAgent = await self.findNextAvailableAgent([baseAgent, ...equivalentAgents], attemptedAgentIds, nowMs);
-                    if (nextAgent) {
+                    const parsedFailure = parseInvocationFailure(error, nowMs);
+                    if (!parsedFailure)
+                        throw error;
+                    const candidates = [baseAgent, ...equivalentAgents];
+                    if (parsedFailure.kind === "usage_limit") {
+                        await self.persistUsageLimitObservation(activeAgent, parsedFailure.usageLimit);
+                        const nextAgent = await self.findNextAvailableAgent(candidates, attemptedAgentIds, nowMs);
+                        if (nextAgent) {
+                            failoverEvents.push({
+                                type: "switch_agent",
+                                at: new Date(nowMs).toISOString(),
+                                fromAgentId: activeAgent.id,
+                                fromAgentSlug: activeAgent.slug,
+                                toAgentId: nextAgent.id,
+                                toAgentSlug: nextAgent.slug,
+                                reason: parsedFailure.kind,
+                            });
+                            activeAgent = nextAgent;
+                            continue;
+                        }
+                        const earliestResetMs = await self.findEarliestResetMs(candidates, nowMs);
+                        const fallbackResetMs = self.estimateResetMsFromWindowTypes(parsedFailure.usageLimit.windowTypes, nowMs);
+                        const waitUntilMs = earliestResetMs && earliestResetMs > nowMs ? earliestResetMs : Math.max(fallbackResetMs, nowMs + 1000);
+                        const durationMs = waitUntilMs - nowMs;
                         failoverEvents.push({
-                            type: "switch_agent",
+                            type: "sleep_until_reset",
                             at: new Date(nowMs).toISOString(),
-                            fromAgentId: activeAgent.id,
-                            fromAgentSlug: activeAgent.slug,
-                            toAgentId: nextAgent.id,
-                            toAgentSlug: nextAgent.slug,
+                            until: new Date(waitUntilMs).toISOString(),
+                            durationMs,
                         });
-                        activeAgent = nextAgent;
+                        await self.sleepUntil(waitUntilMs);
+                        attemptedAgentIds.clear();
+                        activeAgent = baseAgent;
                         continue;
                     }
-                    const earliestResetMs = await self.findEarliestResetMs([baseAgent, ...equivalentAgents], nowMs);
-                    const fallbackResetMs = self.estimateResetMsFromWindowTypes(parsedLimit.windowTypes, nowMs);
-                    const waitUntilMs = earliestResetMs && earliestResetMs > nowMs ? earliestResetMs : Math.max(fallbackResetMs, nowMs + 1000);
-                    const durationMs = waitUntilMs - nowMs;
+                    if (parsedFailure.kind === "connectivity_issue") {
+                        const internetReachable = await self.isInternetReachable();
+                        if (!internetReachable) {
+                            const localFallback = await self.findNextAvailableAgent(candidates, attemptedAgentIds, nowMs, {
+                                offlineCapableOnly: true,
+                                adapterOverride: request.adapterType,
+                            });
+                            if (localFallback) {
+                                failoverEvents.push({
+                                    type: "switch_agent",
+                                    at: new Date(nowMs).toISOString(),
+                                    fromAgentId: activeAgent.id,
+                                    fromAgentSlug: activeAgent.slug,
+                                    toAgentId: localFallback.id,
+                                    toAgentSlug: localFallback.slug,
+                                    reason: parsedFailure.kind,
+                                });
+                                activeAgent = localFallback;
+                                continue;
+                            }
+                            const recovery = await self.waitForInternetRecovery();
+                            failoverEvents.push({
+                                type: "wait_for_internet",
+                                at: new Date(recovery.startedAtMs).toISOString(),
+                                until: new Date(recovery.recoveredAtMs).toISOString(),
+                                durationMs: recovery.durationMs,
+                                pollIntervalMs: self.getConnectivityPollIntervalMs(),
+                            });
+                            attemptedAgentIds.clear();
+                            activeAgent = baseAgent;
+                            continue;
+                        }
+                    }
+                    const nextAgent = await self.findNextAvailableAgent(candidates, attemptedAgentIds, nowMs);
+                    if (!nextAgent) {
+                        throw error;
+                    }
                     failoverEvents.push({
-                        type: "sleep_until_reset",
+                        type: "switch_agent",
                         at: new Date(nowMs).toISOString(),
-                        until: new Date(waitUntilMs).toISOString(),
-                        durationMs,
+                        fromAgentId: activeAgent.id,
+                        fromAgentSlug: activeAgent.slug,
+                        toAgentId: nextAgent.id,
+                        toAgentSlug: nextAgent.slug,
+                        reason: parsedFailure.kind,
                     });
-                    await self.sleepUntil(waitUntilMs);
-                    attemptedAgentIds.clear();
-                    activeAgent = baseAgent;
+                    activeAgent = nextAgent;
                     continue;
                 }
                 const streamIo = ioEnabled ? createStreamIoRenderer() : undefined;
@@ -673,45 +848,106 @@ export class AgentService {
                 }
                 catch (error) {
                     await self.recordInvocationFailure(activeAgent, error);
-                    const parsedLimit = parseUsageLimitError(error, self.nowMs());
-                    if (!parsedLimit)
-                        throw error;
                     const nowMs = self.nowMs();
-                    if (emitted) {
+                    const parsedFailure = parseInvocationFailure(error, nowMs);
+                    if (!parsedFailure)
+                        throw error;
+                    const candidates = [baseAgent, ...equivalentAgents];
+                    if (parsedFailure.kind === "usage_limit") {
+                        if (emitted) {
+                            failoverEvents.push({
+                                type: "stream_restart_after_limit",
+                                at: new Date(nowMs).toISOString(),
+                                fromAgentId: activeAgent.id,
+                                fromAgentSlug: activeAgent.slug,
+                            });
+                        }
+                        await self.persistUsageLimitObservation(activeAgent, parsedFailure.usageLimit);
+                        const nextAgent = await self.findNextAvailableAgent(candidates, attemptedAgentIds, nowMs);
+                        if (nextAgent) {
+                            failoverEvents.push({
+                                type: "switch_agent",
+                                at: new Date(nowMs).toISOString(),
+                                fromAgentId: activeAgent.id,
+                                fromAgentSlug: activeAgent.slug,
+                                toAgentId: nextAgent.id,
+                                toAgentSlug: nextAgent.slug,
+                                reason: parsedFailure.kind,
+                            });
+                            activeAgent = nextAgent;
+                            continue;
+                        }
+                        const earliestResetMs = await self.findEarliestResetMs(candidates, nowMs);
+                        const fallbackResetMs = self.estimateResetMsFromWindowTypes(parsedFailure.usageLimit.windowTypes, nowMs);
+                        const waitUntilMs = earliestResetMs && earliestResetMs > nowMs ? earliestResetMs : Math.max(fallbackResetMs, nowMs + 1000);
+                        const durationMs = waitUntilMs - nowMs;
                         failoverEvents.push({
-                            type: "stream_restart_after_limit",
+                            type: "sleep_until_reset",
                             at: new Date(nowMs).toISOString(),
-                            fromAgentId: activeAgent.id,
-                            fromAgentSlug: activeAgent.slug,
+                            until: new Date(waitUntilMs).toISOString(),
+                            durationMs,
                         });
+                        await self.sleepUntil(waitUntilMs);
+                        attemptedAgentIds.clear();
+                        activeAgent = baseAgent;
+                        continue;
                     }
-                    await self.persistUsageLimitObservation(activeAgent, parsedLimit);
-                    const nextAgent = await self.findNextAvailableAgent([baseAgent, ...equivalentAgents], attemptedAgentIds, nowMs);
-                    if (nextAgent) {
+                    if (emitted) {
                         failoverEvents.push({
-                            type: "switch_agent",
+                            type: "stream_restart_after_failure",
                             at: new Date(nowMs).toISOString(),
                             fromAgentId: activeAgent.id,
                             fromAgentSlug: activeAgent.slug,
-                            toAgentId: nextAgent.id,
-                            toAgentSlug: nextAgent.slug,
+                            reason: parsedFailure.kind,
                         });
-                        activeAgent = nextAgent;
-                        continue;
                     }
-                    const earliestResetMs = await self.findEarliestResetMs([baseAgent, ...equivalentAgents], nowMs);
-                    const fallbackResetMs = self.estimateResetMsFromWindowTypes(parsedLimit.windowTypes, nowMs);
-                    const waitUntilMs = earliestResetMs && earliestResetMs > nowMs ? earliestResetMs : Math.max(fallbackResetMs, nowMs + 1000);
-                    const durationMs = waitUntilMs - nowMs;
+                    if (parsedFailure.kind === "connectivity_issue") {
+                        const internetReachable = await self.isInternetReachable();
+                        if (!internetReachable) {
+                            const localFallback = await self.findNextAvailableAgent(candidates, attemptedAgentIds, nowMs, {
+                                offlineCapableOnly: true,
+                                adapterOverride: request.adapterType,
+                            });
+                            if (localFallback) {
+                                failoverEvents.push({
+                                    type: "switch_agent",
+                                    at: new Date(nowMs).toISOString(),
+                                    fromAgentId: activeAgent.id,
+                                    fromAgentSlug: activeAgent.slug,
+                                    toAgentId: localFallback.id,
+                                    toAgentSlug: localFallback.slug,
+                                    reason: parsedFailure.kind,
+                                });
+                                activeAgent = localFallback;
+                                continue;
+                            }
+                            const recovery = await self.waitForInternetRecovery();
+                            failoverEvents.push({
+                                type: "wait_for_internet",
+                                at: new Date(recovery.startedAtMs).toISOString(),
+                                until: new Date(recovery.recoveredAtMs).toISOString(),
+                                durationMs: recovery.durationMs,
+                                pollIntervalMs: self.getConnectivityPollIntervalMs(),
+                            });
+                            attemptedAgentIds.clear();
+                            activeAgent = baseAgent;
+                            continue;
+                        }
+                    }
+                    const nextAgent = await self.findNextAvailableAgent(candidates, attemptedAgentIds, nowMs);
+                    if (!nextAgent) {
+                        throw error;
+                    }
                     failoverEvents.push({
-                        type: "sleep_until_reset",
+                        type: "switch_agent",
                         at: new Date(nowMs).toISOString(),
-                        until: new Date(waitUntilMs).toISOString(),
-                        durationMs,
+                        fromAgentId: activeAgent.id,
+                        fromAgentSlug: activeAgent.slug,
+                        toAgentId: nextAgent.id,
+                        toAgentSlug: nextAgent.slug,
+                        reason: parsedFailure.kind,
                     });
-                    await self.sleepUntil(waitUntilMs);
-                    attemptedAgentIds.clear();
-                    activeAgent = baseAgent;
+                    activeAgent = nextAgent;
                 }
             }
         }

package/dist/AgentService/InvocationFailureParser.d.ts

@@ -0,0 +1,15 @@
+import { ParsedUsageLimitError } from "./UsageLimitParser.js";
+export type ParsedInvocationFailure = {
+    kind: "usage_limit";
+    usageLimit: ParsedUsageLimitError;
+} | {
+    kind: "connectivity_issue";
+    message: string;
+    rawText: string;
+} | {
+    kind: "technical_issue";
+    message: string;
+    rawText: string;
+};
+export declare const parseInvocationFailure: (error: unknown, nowMs?: number) => ParsedInvocationFailure | null;
+//# sourceMappingURL=InvocationFailureParser.d.ts.map

package/dist/AgentService/InvocationFailureParser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"InvocationFailureParser.d.ts","sourceRoot":"","sources":["../../src/AgentService/InvocationFailureParser.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,qBAAqB,EACtB,MAAM,uBAAuB,CAAC;AAyC/B,MAAM,MAAM,uBAAuB,GAC/B;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,UAAU,EAAE,qBAAqB,CAAA;CAAE,GAC1D;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAChE;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAElE,eAAO,MAAM,sBAAsB,GACjC,OAAO,OAAO,EACd,cAAkB,KACjB,uBAAuB,GAAG,IAwB5B,CAAC"}

package/dist/AgentService/InvocationFailureParser.js

@@ -0,0 +1,57 @@
+import { extractUsageLimitErrorText, parseUsageLimitError, } from "./UsageLimitParser.js";
+const NON_RETRYABLE_AUTH_PATTERNS = [
+    /invalid[_\s-]*api[_\s-]*key/i,
+    /api key.*invalid/i,
+    /authentication failed/i,
+    /missing api key/i,
+    /\bunauthorized\b/i,
+    /\bforbidden\b/i,
+];
+const CONNECTIVITY_PATTERNS = [
+    /\boffline\b/i,
+    /\bnetwork (?:is )?unreachable\b/i,
+    /\binternet (?:is )?(?:down|offline|unavailable)\b/i,
+    /\bfetch failed\b/i,
+    /\bfailed to fetch\b/i,
+    /\bgetaddrinfo\b/i,
+    /\bENOTFOUND\b/i,
+    /\bEAI_AGAIN\b/i,
+    /\bECONNRESET\b/i,
+    /\bECONNREFUSED\b/i,
+    /\bETIMEDOUT\b/i,
+    /\bsocket hang up\b/i,
+    /\bconnection (?:reset|refused|closed|failed)\b/i,
+];
+const TECHNICAL_PATTERNS = [
+    /\b(?:500|501|502|503|504)\b/,
+    /\binternal server error\b/i,
+    /\bbad gateway\b/i,
+    /\bservice unavailable\b/i,
+    /\bgateway timeout\b/i,
+    /\btemporar(?:y|ily) unavailable\b/i,
+    /\boverload(?:ed)?\b/i,
+    /\bupstream .*?(?:error|failed|timeout)\b/i,
+    /\bretry later\b/i,
+];
+const normalizeSpace = (value) => value.replace(/\s+/g, " ").trim();
+export const parseInvocationFailure = (error, nowMs = Date.now()) => {
+    const usageLimit = parseUsageLimitError(error, nowMs);
+    if (usageLimit) {
+        return { kind: "usage_limit", usageLimit };
+    }
+    const message = error instanceof Error ? error.message : String(error ?? "");
+    const rawText = normalizeSpace(extractUsageLimitErrorText(error));
+    const merged = normalizeSpace([message, rawText].filter(Boolean).join(" "));
+    if (!merged)
+        return null;
+    if (NON_RETRYABLE_AUTH_PATTERNS.some((pattern) => pattern.test(merged))) {
+        return null;
+    }
+    if (CONNECTIVITY_PATTERNS.some((pattern) => pattern.test(merged))) {
+        return { kind: "connectivity_issue", message, rawText: merged };
+    }
+    if (TECHNICAL_PATTERNS.some((pattern) => pattern.test(merged))) {
+        return { kind: "technical_issue", message, rawText: merged };
+    }
+    return null;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mcoda/agents",
-  "version": "0.1.28",
+  "version": "0.1.29",
   "description": "Agent registry and capabilities for mcoda.",
   "type": "module",
   "main": "dist/index.js",
@@ -30,8 +30,8 @@
     "access": "public"
   },
   "dependencies": {
-    "@mcoda/shared": "0.1.28",
-    "@mcoda/db": "0.1.28"
+    "@mcoda/shared": "0.1.29",
+    "@mcoda/db": "0.1.29"
   },
   "scripts": {
     "build": "tsc -p tsconfig.json",