@frockbot/plugin-provider-ollama-cloud 0.3.6 → 0.3.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/plugin-provider-ollama-cloud",
-  "version": "0.3.6",
+  "version": "0.3.7",
   "private": false,
   "type": "module",
   "exports": {
@@ -20,19 +20,19 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@frockbot/configuration-core": "0.3.6",
-    "@frockbot/connection-core": "0.3.6",
-    "@frockbot/kernel-agent-loop": "0.3.6",
-    "@frockbot/kernel-contracts": "0.3.6",
-    "@frockbot/plugin-credentials": "0.3.6",
-    "@frockbot/plugin-models": "0.3.6",
-    "@frockbot/plugin-settings": "0.3.6",
-    "@frockbot/plugin-web": "0.3.6",
-    "@frockbot/provider-openai-compatible": "0.3.6",
+    "@frockbot/configuration-core": "0.3.7",
+    "@frockbot/connection-core": "0.3.7",
+    "@frockbot/kernel-agent-loop": "0.3.7",
+    "@frockbot/kernel-contracts": "0.3.7",
+    "@frockbot/plugin-credentials": "0.3.7",
+    "@frockbot/plugin-models": "0.3.7",
+    "@frockbot/plugin-settings": "0.3.7",
+    "@frockbot/plugin-web": "0.3.7",
+    "@frockbot/provider-openai-compatible": "0.3.7",
     "cordis": "4.0.0-rc.8"
   },
   "devDependencies": {
-    "@frockbot/plugin-tools": "0.3.6",
+    "@frockbot/plugin-tools": "0.3.7",
     "@types/bun": "1.3.6",
     "typescript": "^7.0.2"
   },

package/src/client.test.ts

@@ -137,4 +137,35 @@ describe("Ollama Cloud client", () => {
       "Ollama Cloud request failed (401)",
     );
   });
+  test("bounds a provider that accepts the request and never answers", async () => {
+    const client = new OllamaCloudClient({
+      requestTimeoutMs: 20,
+      fetch: (_input, init) =>
+        new Promise((_resolve, reject) => {
+          init?.signal?.addEventListener("abort", () =>
+            reject(init.signal?.reason ?? new Error("aborted")),
+          );
+        }),
+    });
+
+    await expect(client.listModels("account-key")).rejects.toThrow(
+      "Ollama Cloud did not answer within 20ms",
+    );
+  });
+
+  test("bounds an inference probe that never answers", async () => {
+    const client = new OllamaCloudClient({
+      probeTimeoutMs: 20,
+      fetch: (_input, init) =>
+        new Promise((_resolve, reject) => {
+          init?.signal?.addEventListener("abort", () =>
+            reject(init.signal?.reason ?? new Error("aborted")),
+          );
+        }),
+    });
+
+    await expect(
+      client.probeInference("account-key", "glm-5.3-flash:cloud"),
+    ).rejects.toThrow("Ollama Cloud did not answer within 20ms");
+  });
 });

package/src/client.ts

@@ -9,6 +9,16 @@ export type OllamaFetch = (
   init?: RequestInit,
 ) => Promise<Response>;
 
+/**
+ * Every provider call is bounded. A provider that accepts the connection and
+ * then never answers is the failure this defends against: without a deadline
+ * the connect command's promise never settles and the Connection sits in
+ * `authorizing` forever.
+ */
+export const DEFAULT_OLLAMA_REQUEST_TIMEOUT_MS = 15_000;
+/** Inference is slower than a catalog read, so the probe gets its own budget. */
+export const DEFAULT_OLLAMA_PROBE_TIMEOUT_MS = 30_000;
+
 export interface OllamaCloudClientConfig {
   /**
    * Endpoint root, without the `/api` or `/v1` path segment: the Package
@@ -17,6 +27,25 @@ export interface OllamaCloudClientConfig {
    */
   apiBaseUrl?: string;
   fetch?: OllamaFetch;
+  /** Deadline for a catalog or model-detail read. */
+  requestTimeoutMs?: number;
+  /** Deadline for the authenticated inference probe. */
+  probeTimeoutMs?: number;
+}
+
+/**
+ * Bound one provider call: the caller's signal still cancels it, and a deadline
+ * of its own settles it when the provider simply never answers.
+ */
+export function withDeadlineV1(
+  timeoutMs: number,
+  signal?: AbortSignal,
+): { signal: AbortSignal; timedOut: () => boolean } {
+  const deadline = AbortSignal.timeout(timeoutMs);
+  return {
+    signal: signal ? AbortSignal.any([signal, deadline]) : deadline,
+    timedOut: () => deadline.aborted,
+  };
 }
 
 /** The endpoint every Connection uses until its User points it elsewhere. */
@@ -191,8 +220,14 @@ async function mapConcurrent<T, R>(
 export class OllamaCloudClient {
   private readonly apiBaseUrl: string;
   private readonly fetcher: OllamaFetch;
+  private readonly requestTimeoutMs: number;
+  private readonly probeTimeoutMs: number;
 
   constructor(config: OllamaCloudClientConfig = {}) {
+    this.requestTimeoutMs =
+      config.requestTimeoutMs ?? DEFAULT_OLLAMA_REQUEST_TIMEOUT_MS;
+    this.probeTimeoutMs =
+      config.probeTimeoutMs ?? DEFAULT_OLLAMA_PROBE_TIMEOUT_MS;
     this.apiBaseUrl = `${decodeOllamaApiBaseUrl(
       config.apiBaseUrl ?? DEFAULT_OLLAMA_API_BASE_URL,
     )}/api`;
@@ -207,21 +242,45 @@ export class OllamaCloudClient {
     apiKey: string,
     init: RequestInit,
   ): Promise<unknown> {
-    const response = await this.fetcher(`${this.apiBaseUrl}${path}`, {
-      ...init,
-      headers: {
-        authorization: `Bearer ${apiKey}`,
-        "content-type": "application/json",
-        ...init.headers,
-      },
-    });
+    const deadline = withDeadlineV1(
+      this.requestTimeoutMs,
+      init.signal ?? undefined,
+    );
+    let response: Response;
+    try {
+      response = await this.fetcher(`${this.apiBaseUrl}${path}`, {
+        ...init,
+        signal: deadline.signal,
+        headers: {
+          authorization: `Bearer ${apiKey}`,
+          "content-type": "application/json",
+          ...init.headers,
+        },
+      });
+    } catch (error) {
+      throw deadline.timedOut()
+        ? new Error(
+            `Ollama Cloud did not answer within ${this.requestTimeoutMs}ms`,
+          )
+        : error;
+    }
     if (!response.ok) {
       throw new Error(`Ollama Cloud request failed (${response.status})`);
     }
-    return boundedJson(
-      response,
-      path === "/tags" ? MAX_CATALOG_RESPONSE_BYTES : MAX_MODEL_RESPONSE_BYTES,
-    );
+    try {
+      return await boundedJson(
+        response,
+        path === "/tags"
+          ? MAX_CATALOG_RESPONSE_BYTES
+          : MAX_MODEL_RESPONSE_BYTES,
+      );
+    } catch (error) {
+      throw deadline.timedOut()
+        ? new Error(
+            `Ollama Cloud did not answer within ${this.requestTimeoutMs}ms`,
+          )
+        : error;
+    }
   }
 
   async listModels(
@@ -319,20 +378,30 @@ export class OllamaCloudClient {
     signal?: AbortSignal,
   ): Promise<void> {
     const model = modelId(providerModelId);
-    const response = await this.fetcher(`${this.apiBaseUrl}/chat`, {
-      method: "POST",
-      signal,
-      headers: {
-        authorization: `Bearer ${apiKey}`,
-        "content-type": "application/json",
-      },
-      body: JSON.stringify({
-        model,
-        messages: [{ role: "user", content: "hi" }],
-        stream: false,
-        options: { num_predict: PROBE_PREDICTED_TOKENS },
-      }),
-    });
+    const deadline = withDeadlineV1(this.probeTimeoutMs, signal);
+    let response: Response;
+    try {
+      response = await this.fetcher(`${this.apiBaseUrl}/chat`, {
+        method: "POST",
+        signal: deadline.signal,
+        headers: {
+          authorization: `Bearer ${apiKey}`,
+          "content-type": "application/json",
+        },
+        body: JSON.stringify({
+          model,
+          messages: [{ role: "user", content: "hi" }],
+          stream: false,
+          options: { num_predict: PROBE_PREDICTED_TOKENS },
+        }),
+      });
+    } catch (error) {
+      throw deadline.timedOut()
+        ? new Error(
+            `Ollama Cloud did not answer within ${this.probeTimeoutMs}ms`,
+          )
+        : error;
+    }
     if (!response.ok) {
       const reported = await boundedText(response);
       if (response.status === 401 || response.status === 403) {

package/src/runtime.test.ts

@@ -11,7 +11,6 @@ import {
   sealCredentialV1,
   type CredentialLeaseV1,
 } from "@frockbot/connection-core";
-import { OpenAICompatibleHttpError } from "@frockbot/provider-openai-compatible";
 import { Context, Service } from "cordis";
 import {
   createOllamaCloudRuntimePlugin,
@@ -458,61 +457,88 @@ describe("Ollama Cloud runtime Contribution", () => {
     },
   );
 
-  test("requires reconciliation for ambiguous HTTP failures", async () => {
-    const keyringText = serializedKeyring();
-    const envelope = await sealCredentialV1({
-      keyring: parseCredentialKeyringV1(keyringText),
-      context: {
-        accountId: "account-1",
-        connectionId: "connection-1",
-        packageId: "provider-ollama-cloud",
-        credentialGeneration: "generation-1",
-      },
-      plaintext: "account-secret",
-    });
-    const settled: string[] = [];
+  test.each([408, 429, 500, 502])(
+    "settles pre-stream HTTP %i as not started rather than parking the Turn",
+    async (status) => {
+      const keyringText = serializedKeyring();
+      const envelope = await sealCredentialV1({
+        keyring: parseCredentialKeyringV1(keyringText),
+        context: {
+          accountId: "account-1",
+          connectionId: "connection-1",
+          packageId: "provider-ollama-cloud",
+          credentialGeneration: "generation-1",
+        },
+        plaintext: "account-secret",
+      });
+      const settled: string[] = [];
+      const root = new Context();
+      await root.plugin(LlmRegistry);
+      await mountCredentialRuntime(root, keyringText);
+      await root.plugin(
+        createOllamaCloudRuntimePlugin({
+          accountId: "account-1",
+          connectionId: "connection-1",
+          packageId: "provider-ollama-cloud",
+          now: () => Date.parse("2026-08-30T00:00:00.000Z"),
+          leaseCredential: (effectId) =>
+            Promise.resolve({
+              schemaVersion: 1,
+              leaseId: "lease-1",
+              effectId,
+              connectionId: "connection-1",
+              credentialGeneration: "generation-1",
+              expiresAt: "2026-08-30T01:00:00.000Z",
+              envelope,
+            }),
+          settleCredential: (effectId) => {
+            settled.push(effectId);
+            return Promise.resolve();
+          },
+          fetch: () => Promise.resolve(new Response("transient", { status })),
+        }),
+      );
+
+      let failure: unknown;
+      try {
+        for await (const _ of root.llm.stream(
+          request,
+          new AbortController().signal,
+        )) {
+          void _;
+        }
+      } catch (error) {
+        failure = error;
+      }
+
+      expect(failure).toBeInstanceOf(LlmEffectNotStartedError);
+      expect(settled).toEqual([]);
+      await root.fiber.dispose();
+    },
+  );
+
+  test("reports an interrupted response as not retrievable so the run settles", async () => {
     const root = new Context();
     await root.plugin(LlmRegistry);
-    await mountCredentialRuntime(root, keyringText);
+    await mountCredentialRuntime(root, serializedKeyring());
     await root.plugin(
       createOllamaCloudRuntimePlugin({
         accountId: "account-1",
         connectionId: "connection-1",
         packageId: "provider-ollama-cloud",
         now: () => Date.parse("2026-08-30T00:00:00.000Z"),
-        leaseCredential: (effectId) =>
-          Promise.resolve({
-            schemaVersion: 1,
-            leaseId: "lease-1",
-            effectId,
-            connectionId: "connection-1",
-            credentialGeneration: "generation-1",
-            expiresAt: "2026-08-30T01:00:00.000Z",
-            envelope,
-          }),
-        settleCredential: (effectId) => {
-          settled.push(effectId);
-          return Promise.resolve();
-        },
-        fetch: () => Promise.resolve(new Response("timeout", { status: 408 })),
+        leaseCredential: () => Promise.reject(new Error("unused")),
+        settleCredential: () => Promise.resolve(),
+        fetch: () => Promise.reject(new Error("unused")),
       }),
     );
 
-    let failure: unknown;
-    try {
-      for await (const _ of root.llm.stream(
-        request,
-        new AbortController().signal,
-      )) {
-        void _;
-      }
-    } catch (error) {
-      failure = error;
-    }
+    const outcome = await root.llm.reconcile(
+      request,
+      new AbortController().signal,
+    );
 
-    expect(failure).toBeInstanceOf(OpenAICompatibleHttpError);
-    expect(failure).not.toBeInstanceOf(LlmEffectNotStartedError);
-    expect(settled).toEqual([]);
+    expect(outcome.status).toBe("not-retrievable");
     await root.fiber.dispose();
   });
 });

package/src/runtime.ts

@@ -1,6 +1,7 @@
 import {
   LlmEffectNotStartedError,
   type LlmProvider,
+  type LlmReconciliationCapability,
   type NormalizedModelRequest,
 } from "@frockbot/kernel-contracts";
 import { type Agent } from "@frockbot/kernel-agent-loop/agent";
@@ -143,6 +144,19 @@ class OllamaCloudProvider implements LlmProvider {
     }
   }
 
+  /**
+   * Ollama keeps no addressable copy of a completion, so an interrupted stream
+   * can never be read back. Saying so settles the run as a failure with its
+   * partial text intact instead of parking it forever.
+   */
+  readonly reconciliation: LlmReconciliationCapability = {
+    retrieve: async () => ({
+      status: "not-retrievable",
+      reason:
+        "Ollama keeps no durable copy of an interrupted response, so it cannot be recovered",
+    }),
+  };
+
   async *stream(request: NormalizedModelRequest, signal: AbortSignal) {
     await this.authorize(request);
     const authorization = this.authorized.get(request.requestId);
@@ -157,16 +171,24 @@ class OllamaCloudProvider implements LlmProvider {
       providerId: this.id,
       fetch: this.config.fetch,
     });
+    // Every failure raised before the first stream event happened before a
+    // provider effect existed, so it is definitive rather than uncertain. A 429
+    // or a 502 reported as a bare failure would park the run on a retrieval
+    // this Package cannot perform; reported as "not started" it fails cleanly
+    // and can be retried.
+    let started = false;
     try {
-      yield* provider.stream(request, signal);
-    } catch (error) {
-      if (
-        error instanceof OpenAICompatibleHttpError &&
-        (error.status === 401 || error.status === 403 || error.status === 404)
-      ) {
-        throw new LlmEffectNotStartedError(error.message);
+      for await (const event of provider.stream(request, signal)) {
+        started = true;
+        yield event;
       }
-      throw error;
+    } catch (error) {
+      if (started || signal.aborted) throw error;
+      throw new LlmEffectNotStartedError(
+        error instanceof OpenAICompatibleHttpError || error instanceof Error
+          ? error.message
+          : "Ollama Cloud request did not reach the provider",
+      );
     }
   }
 }

package/src/user.test.ts

@@ -11,6 +11,7 @@ import {
 } from "@frockbot/plugin-settings/user";
 import { OllamaCloudClient, type OllamaFetch } from "./client.js";
 import {
+  catalogRetryDelayMsV1,
   createOllamaCloudUserBackendContribution,
   type OllamaUserBackendHost,
 } from "./user.js";
@@ -176,6 +177,18 @@ async function fixture(
   };
 }
 
+describe("catalog retry backoff", () => {
+  test("retries a failed refresh soon and backs off, capped at the interval", () => {
+    // A refresh that failed used to wait the full hour, so a thirty-second
+    // outage cost an hour of stale models.
+    expect(catalogRetryDelayMsV1(1)).toBe(60_000);
+    expect(catalogRetryDelayMsV1(2)).toBe(120_000);
+    expect(catalogRetryDelayMsV1(3)).toBe(240_000);
+    expect(catalogRetryDelayMsV1(20)).toBe(60 * 60 * 1_000);
+    expect(catalogRetryDelayMsV1(0)).toBe(60_000);
+  });
+});
+
 describe("Ollama Cloud User Contribution", () => {
   test("rejects malformed durable account, command, and pending records", async () => {
     const accountFixture = await fixture();
@@ -464,6 +477,45 @@ describe("Ollama Cloud User Contribution", () => {
     ).toMatchObject({ displayName: "Recovered 2" });
   });
 
+  test("abandons a Connection whose provider never answers", async () => {
+    // A provider that accepts the request and never responds: the connect
+    // command must not re-drive forever, and the Connection must not sit in
+    // `authorizing` where the User is shown nothing at all.
+    const { settings, ollama } = await fixture((_input, init) =>
+      Promise.reject(
+        init?.signal?.reason ?? new Error("provider never answered"),
+      ),
+    );
+    const created = await ollama.executeConnection("account-1", {
+      schemaVersion: 1,
+      type: "connection/create-api-key",
+      commandId: "connect-hang",
+      packageId: "provider-ollama-cloud",
+      connectionTypeId: "ollama-cloud-account",
+      label: "Hang",
+      apiKey: "valid-key",
+    });
+
+    for (let attempt = 0; attempt < 5; attempt += 1) {
+      await ollama.alarm().catch(() => undefined);
+    }
+
+    const connection = await settings.getConnection(
+      "account-1",
+      created.connectionId,
+    );
+    expect(connection).toMatchObject({
+      state: "failed",
+      failure: expect.any(String),
+    });
+    expect(connection?.authorization).toMatchObject({
+      credential: { configured: false },
+    });
+    await expect(
+      ollama.lookupConnectionCommand("account-1", "connect-hang"),
+    ).resolves.toMatchObject({ status: "failed" });
+  });
+
   test("keeps the active generation when rotation validation fails", async () => {
     const { settings, ollama, rejectCatalog } = await fixture();
     const created = await ollama.executeConnection("account-1", {

package/src/user.ts

@@ -41,6 +41,13 @@ const MAX_COMMAND_TOMBSTONES = 128;
 const MAX_MANUAL_COMMANDS = MAX_MANUAL_RECEIPTS + MAX_COMMAND_TOMBSTONES;
 const MAX_PENDING_COMMANDS = 64;
 const MAX_PENDING_RECOVERIES_PER_ALARM = 1;
+/**
+ * How many alarm-driven attempts a pending Connection command gets before it is
+ * abandoned. Without a cap an endpoint that accepts the connection and never
+ * answers is re-driven once a minute forever and its Connection sits in
+ * `authorizing` with nothing to show the User.
+ */
+const MAX_PENDING_RECOVERY_ATTEMPTS = 3;
 const MAX_CATALOG_REFRESHES_PER_ALARM = 1;
 const ACCOUNT_KEY = "ollama-connection-account";
 const AUTOMATIC_REFRESH_RECEIPT_PREFIX = "ollama-refresh-receipt:";
@@ -48,6 +55,36 @@ const AUTOMATIC_REFRESH_RECEIPT_PREFIX = "ollama-refresh-receipt:";
 const MUTATION_SEQUENCE_PREFIX = "ollama-mutation-sequence:";
 const MODEL_RESOLUTION_PREFIX = "ollama-model-resolution:";
 const REFRESH_INTERVAL_MS = 60 * 60 * 1_000;
+/**
+ * How soon a catalog refresh that failed is tried again.
+ *
+ * A refresh that failed used to wait the full hour, so an endpoint that was
+ * down for thirty seconds stayed stale for an hour. The delay doubles per
+ * consecutive failure up to the ordinary interval, so a transient outage
+ * recovers quickly and a lasting one does not hammer the provider.
+ */
+const CATALOG_RETRY_BASE_MS = 60_000;
+const MAX_CATALOG_RETRY_ATTEMPTS = 8;
+const CATALOG_RETRY_PREFIX = "ollama-catalog-retry:";
+
+function catalogRetryKey(connectionId: string): string {
+  return `${CATALOG_RETRY_PREFIX}${connectionId}`;
+}
+
+/** Backoff for the nth consecutive catalog failure, capped at the interval. */
+export function catalogRetryDelayMsV1(attempt: number): number {
+  const bounded = Math.min(Math.max(attempt, 1), MAX_CATALOG_RETRY_ATTEMPTS);
+  return Math.min(
+    CATALOG_RETRY_BASE_MS * 2 ** (bounded - 1),
+    REFRESH_INTERVAL_MS,
+  );
+}
+
+function catalogRetryAttempt(value: unknown): number {
+  return typeof value === "number" && Number.isSafeInteger(value) && value > 0
+    ? value
+    : 0;
+}
 const RECOVERY_DELAY_MS = 60_000;
 const MODEL_LEASE_MS = 30 * 60 * 1_000;
 const MAX_CONNECTION_MODELS = 100;
@@ -81,6 +118,8 @@ interface StoredCommand {
   validationFailure?: string;
   validationStatus?: "applied" | "failed";
   providerRetryPolicy?: "safe-metadata-read";
+  /** Alarm-driven resume attempts so far; capped, so a command always settles. */
+  recoveryAttempts?: number;
 }
 
 interface StoredModelResolution {
@@ -379,6 +418,7 @@ function decodeStoredCommand(input: unknown): StoredCommand {
       "validationFailure",
       "validationStatus",
       "providerRetryPolicy",
+      "recoveryAttempts",
     ],
   );
   const operations: StoredCommand["operation"][] = [
@@ -1566,6 +1606,7 @@ export class OllamaCloudUserBackendContribution {
             },
             storage,
           );
+          await storage.delete(catalogRetryKey(record.connectionId));
         } else if (appliesProjection) {
           const settings = await this.host.settings.readSnapshot(storage);
           const current = settings.connections.find(
@@ -1581,6 +1622,15 @@ export class OllamaCloudUserBackendContribution {
             current.state === "ready" &&
             current.modelCatalog
           ) {
+            const attempt =
+              catalogRetryAttempt(
+                await storage.get<unknown>(
+                  catalogRetryKey(record.connectionId),
+                ),
+              ) + 1;
+            await storage.put({
+              [catalogRetryKey(record.connectionId)]: attempt,
+            });
             await this.host.settings.replaceConnection(
               record.accountId,
               record.connectionId,
@@ -1591,7 +1641,7 @@ export class OllamaCloudUserBackendContribution {
                   ...current.modelCatalog,
                   state: "stale",
                   refreshAfter: new Date(
-                    this.now() + REFRESH_INTERVAL_MS,
+                    this.now() + catalogRetryDelayMsV1(attempt),
                   ).toISOString(),
                   failure:
                     outcomeFailure instanceof Error
@@ -1895,6 +1945,56 @@ export class OllamaCloudUserBackendContribution {
     return this.finishRecord(record, terminalStatus);
   }
 
+  /**
+   * Settle a command that has exhausted its attempts. The Connection it created
+   * leaves `authorizing` for `failed` with a reason the User can act on, rather
+   * than staying in a state that only a page reload even renders.
+   */
+  private async abandonPendingCommand(record: StoredCommand): Promise<void> {
+    const failure =
+      "The provider did not respond. The connection attempt was abandoned.";
+    const generation = record.credentialGeneration;
+    if (generation) {
+      await this.host.credentials
+        .discardPending(record.connectionId, generation)
+        .catch(() => undefined);
+    }
+    const current = await this.host.settings.getConnection(
+      record.accountId,
+      record.connectionId,
+    );
+    if (
+      current &&
+      current.packageId === PACKAGE_ID &&
+      current.state === "authorizing"
+    ) {
+      await this.host.settings.replaceConnection(
+        record.accountId,
+        record.connectionId,
+        current.generation,
+        {
+          ...current,
+          state: "failed",
+          authorization: {
+            schemaVersion: 1,
+            kind: "api-key",
+            credential: {
+              schemaVersion: 1,
+              configured: false,
+              source: "api-key",
+              writable: true,
+            },
+          },
+          failure,
+        },
+      );
+    }
+    await this.finishRecord(
+      { ...record, validationFailure: failure },
+      "failed",
+    );
+  }
+
   private async finishRecord(
     record: StoredCommand,
     status: ConnectionCommandReceiptV1["status"],
@@ -2307,7 +2407,17 @@ export class OllamaCloudUserBackendContribution {
       );
       if (recordValue === undefined) continue;
       const record = decodeStoredCommand(recordValue);
-      if (!record.receipt) await this.resumeOnce(record);
+      if (record.receipt) continue;
+      const attempts = (record.recoveryAttempts ?? 0) + 1;
+      if (attempts > MAX_PENDING_RECOVERY_ATTEMPTS) {
+        await this.abandonPendingCommand(record);
+        continue;
+      }
+      // Counted before the attempt, so an attempt that throws still counts and
+      // the command cannot be re-driven indefinitely.
+      const attempted = { ...record, recoveryAttempts: attempts };
+      await this.host.storage.put({ [commandKey(commandId)]: attempted });
+      await this.resumeOnce(attempted);
     }
     const accountValue = await this.host.storage.get<unknown>(ACCOUNT_KEY);
     if (accountValue === undefined) return;