@plane31/blind-relay 0.3.0 → 0.4.0

package/src/client.ts

@@ -235,6 +235,7 @@ export type ReplaceDeviceResult =
 
 export interface DrainOptions {
   readonly catchUp?: SyncOptions["catchUp"];
+  readonly skipMaintenance?: boolean;
   /** Rounds at most. Default 10. */
   readonly maxRounds?: number;
   /** A soft bound in milliseconds across every round: no round starts past it and each round's reading budget is the time left, but a transport call already in flight ends on its own timeout. Default 30,000. */
@@ -254,6 +255,8 @@ export type DrainStop =
   | "authentication"
   /** Nothing more is reported, but a round could not read its inbox fully (discovery or a stream failed), so emptiness is not established. */
   | "incomplete"
+  /** The host asked for a pause; `remaining.retryAfterMs` says how long, and the product schedules the next attempt. */
+  | "rateLimited"
   | "closed";
 
 /** The merged report of every round, with why the loop stopped and what remains. */
@@ -269,6 +272,8 @@ export interface DrainReport extends SyncReport {
     readonly chainsIncomplete: number;
     /** Authors whose current epoch has not reached this device: nothing to do but wait for them. */
     readonly waitingFor: number;
+    /** On `rateLimited`: the longest wait the host asked for in the last round, when it said one. */
+    readonly retryAfterMs?: number;
   };
 }
 
@@ -280,6 +285,12 @@ export interface SyncOptions {
    * at least one page.
    */
   readonly catchUp?: { readonly maxRecords?: number; readonly maxMs?: number };
+  /**
+   * Leave the housekeeping to a later sync: the retention purge, the periodic re-check of the
+   * reset-key list (a stale list is still fetched, since verification needs it) and the
+   * recovery-key re-save. For a short background window; the report says `maintenance: "skipped"`.
+   */
+  readonly skipMaintenance?: boolean;
 }
 
 export const DEFAULT_CATCH_UP = { maxRecords: 500, maxMs: 5_000 } as const;
@@ -291,11 +302,39 @@ export interface SyncReport {
   /** `hasMore`: a stream still has records on the relay that this sync's budget did not reach; sync again soon. */
   inbox: { streams: number; stored: number; keyPending: number; invalid: number; acked: number; hasMore: boolean };
   purged: number;
+  /** Whether this sync did its housekeeping (purge, list re-check, key re-save) or left it to a later one. */
+  maintenance: "done" | "skipped";
   /** Present while account recovery discovers circles through the host: pinned, pending and failed on this sync. */
   bootstrap?: { circles: number; pending: number; failed: number };
+  /** Wall-clock milliseconds per phase and record bytes moved, for diagnostics (section 14: counts and durations only). */
+  timings: SyncTimings;
   errors: SyncError[];
 }
 
+export interface SyncTimings {
+  resume: number;
+  bootstrap: number;
+  metadata: number;
+  preferences: number;
+  epoch: number;
+  outbox: number;
+  inbox: number;
+  maintenance: number;
+  total: number;
+  /** Record bytes handed to the relay. */
+  bytesSent: number;
+  /** Record bytes read from the relay. */
+  bytesReceived: number;
+}
+
+const emptyTimings = (): SyncTimings => ({ resume: 0, bootstrap: 0, metadata: 0, preferences: 0, epoch: 0, outbox: 0, inbox: 0, maintenance: 0, total: 0, bytesSent: 0, bytesReceived: 0 });
+
+/** A monotonic wall clock for durations; the protocol clock (`now`) is not used, since a test drives it by hand. */
+const wall = (): number => {
+  const g = globalThis as { performance?: { now?: () => number } };
+  return typeof g.performance?.now === "function" ? g.performance.now() : Date.now();
+};
+
 /** What `prepareReplacement()` returns: everything the product hands to the host for authorization. All public. */
 export interface ReplacementProposal {
   readonly successionId: Uint8Array;
@@ -333,10 +372,14 @@ export interface SyncError {
   readonly authorIdentityId?: Uint8Array;
   /** On a `bootstrap` entry: the circle that could not be adopted on this sync; it is retried by the next. */
   readonly circleId?: Uint8Array;
+  /** On a `RATE_LIMITED` entry: the wait the host asked for, when it said. */
+  readonly retryAfterMs?: number;
 }
 
 function syncError(operation: string, e: unknown, extra: Partial<Pick<SyncError, "circleId" | "streamId" | "authorIdentityId">> = {}): SyncError {
-  return e instanceof KavachError ? { operation, code: e.code, stage: e.stage, retryable: e.retryable, ...extra } : { operation, code: "UNEXPECTED", stage: operation, retryable: false, ...extra };
+  return e instanceof KavachError
+    ? { operation, code: e.code, stage: e.stage, retryable: e.retryable, ...(e.retryAfterMs === undefined ? {} : { retryAfterMs: e.retryAfterMs }), ...extra }
+    : { operation, code: "UNEXPECTED", stage: operation, retryable: false, ...extra };
 }
 
 const RESET_LIST_RECHECK_MS = 86_400_000n;
@@ -985,6 +1028,8 @@ class ClientImpl implements Client {
       outbox: { sent: 0, accepted: 0, cancelled: 0, unknown: 0, retry: 0 },
       inbox: { streams: 0, stored: 0, keyPending: 0, invalid: 0, acked: 0, hasMore: false },
       purged: 0,
+      maintenance: options.skipMaintenance ? "skipped" : "done",
+      timings: emptyTimings(),
       errors: []
     };
     let rounds = 0;
@@ -1010,7 +1055,7 @@ class ClientImpl implements Client {
         break;
       }
       const roundMs = Math.max(0, Math.min(remainingMs, options.catchUp?.maxMs ?? DEFAULT_CATCH_UP.maxMs));
-      const r = await this.sync({ catchUp: { maxMs: roundMs, ...(options.catchUp?.maxRecords !== undefined ? { maxRecords: options.catchUp.maxRecords } : {}) } });
+      const r = await this.sync({ catchUp: { maxMs: roundMs, ...(options.catchUp?.maxRecords !== undefined ? { maxRecords: options.catchUp.maxRecords } : {}) }, ...(options.skipMaintenance ? { skipMaintenance: true } : {}) });
       rounds++;
       last = r;
       // The round's work counts whether or not it was the last: what it accepted, stored and failed
@@ -1023,6 +1068,7 @@ class ClientImpl implements Client {
       merged.inbox.invalid += r.inbox.invalid;
       merged.inbox.acked += r.inbox.acked;
       merged.purged += r.purged;
+      for (const k of Object.keys(merged.timings) as (keyof SyncTimings)[]) merged.timings[k] += r.timings[k];
       merged.epoch = r.epoch;
       merged.errors.push(...r.errors);
       if (r.bootstrap) merged.bootstrap = r.bootstrap;
@@ -1031,6 +1077,11 @@ class ClientImpl implements Client {
         stop = "closed";
         break;
       }
+      if (r.errors.some((e) => e.code === "RATE_LIMITED")) {
+        // The host asked for a pause: no further round now; the wait goes back to the product.
+        stop = "rateLimited";
+        break;
+      }
       const bootstrapIncomplete = (await this.inspect()).bootstrap?.complete === false;
       const currentChains = await chains();
       const more = r.inbox.hasMore || bootstrapIncomplete || currentChains.size > 0;
@@ -1065,19 +1116,43 @@ class ClientImpl implements Client {
     const bootstrapIncomplete = this.closed ? false : (await this.inspect()).bootstrap?.complete === false;
     const chainsIncomplete = this.closed ? previousChains.size : (await chains()).size;
     const waitingFor = this.closed ? 0 : (await this.getState()).waitingFor.length;
-    const inboxHasMore = !last || inboxUncertain(last) || stop === "closed" ? "unknown" : merged.inbox.hasMore;
-    return { ...merged, rounds, stoppedBecause: stop, remaining: { inboxHasMore, bootstrapIncomplete, chainsIncomplete, waitingFor } };
+    const inboxHasMore = !last || inboxUncertain(last) || stop === "closed" || stop === "rateLimited" ? "unknown" : merged.inbox.hasMore;
+    const waits = (last?.errors ?? []).flatMap((e) => (e.code === "RATE_LIMITED" && e.retryAfterMs !== undefined ? [e.retryAfterMs] : []));
+    const retryAfterMs = waits.length ? Math.max(...waits) : undefined;
+    return { ...merged, rounds, stoppedBecause: stop, remaining: { inboxHasMore, bootstrapIncomplete, chainsIncomplete, waitingFor, ...(retryAfterMs === undefined ? {} : { retryAfterMs }) } };
   }
 
   private async runSync(options: SyncOptions): Promise<SyncReport> {
+    const started = wall();
     const report: SyncReport = {
       metadata: { certificates: 0, snapshots: 0, rotations: 0, quarantined: 0, resets: 0, successions: 0 },
       epoch: "unchanged",
       outbox: { sent: 0, accepted: 0, cancelled: 0, unknown: 0, retry: 0 },
       inbox: { streams: 0, stored: 0, keyPending: 0, invalid: 0, acked: 0, hasMore: false },
       purged: 0,
+      maintenance: options.skipMaintenance ? "skipped" : "done",
+      timings: emptyTimings(),
       errors: []
     };
+    const maintain = !options.skipMaintenance;
+    let mark = started;
+    const phase = (name: keyof Omit<SyncTimings, "total" | "bytesSent" | "bytesReceived">): void => {
+      const now = wall();
+      report.timings[name] += now - mark;
+      mark = now;
+    };
+    const finish = (): SyncReport => {
+      report.timings.total = wall() - started;
+      return report;
+    };
+    try {
+      return await this.runSyncPhases(options, report, maintain, phase);
+    } finally {
+      finish();
+    }
+  }
+
+  private async runSyncPhases(options: SyncOptions, report: SyncReport, maintain: boolean, phase: (name: keyof Omit<SyncTimings, "total" | "bytesSent" | "bytesReceived">) => void): Promise<SyncReport> {
     this.syncBudget = {
       remainingRecords: Math.max(1, Math.floor(options.catchUp?.maxRecords ?? DEFAULT_CATCH_UP.maxRecords)),
       deadline: this.now() + BigInt(Math.max(0, Math.floor(options.catchUp?.maxMs ?? DEFAULT_CATCH_UP.maxMs)))
@@ -1095,6 +1170,7 @@ class ClientImpl implements Client {
         }
       }
     }
+    phase("resume");
     if (!this.account || !this.selfPin) return report;
     const meta = await this.store.read((tx) => tx.get("meta", META));
     if (!meta || meta.state === "RETIRED") return report;
@@ -1119,7 +1195,9 @@ class ClientImpl implements Client {
         return report;
       }
     } else if (meta.state !== "ACTIVE") return report;
-    await this.ensureBackup();
+    phase("resume");
+    if (maintain) await this.ensureBackup();
+    phase("maintenance");
     if (meta.bootstrap && !meta.bootstrap.complete) {
       try {
         report.bootstrap = await this.bootstrap(report, meta.bootstrap);
@@ -1127,16 +1205,19 @@ class ClientImpl implements Client {
         report.errors.push(syncError("bootstrap", e));
       }
     }
+    phase("bootstrap");
     try {
-      await this.syncMetadata(report);
+      await this.syncMetadata(report, maintain);
     } catch (e) {
       report.errors.push(syncError("metadata", e));
     }
+    phase("metadata");
     try {
       await this.pushPreferences(report);
     } catch (e) {
       report.errors.push(syncError("preference", e));
     }
+    phase("preferences");
     const after = await this.store.read((tx) => tx.get("meta", META));
     if (after?.state === "RETIRED" || this.closed) return report;
     try {
@@ -1144,12 +1225,17 @@ class ClientImpl implements Client {
     } catch (e) {
       report.errors.push(syncError("epoch", e));
     }
+    phase("epoch");
     if (this.closed) return report;
     try {
-      report.outbox = await drain(this.outboxDeps());
+      const drained = await drain(this.outboxDeps());
+      report.outbox = { sent: drained.sent, accepted: drained.accepted, cancelled: drained.cancelled, unknown: drained.unknown, retry: drained.retry };
+      report.timings.bytesSent += drained.bytes;
+      if (drained.blocked) report.errors.push(syncError("outbox", drained.blocked)); // the records stay queued
     } catch (e) {
       report.errors.push(syncError("outbox", e));
     }
+    phase("outbox");
     if (this.closed) return report;
     let subs: Awaited<ReturnType<Relay["listSubscriptions"]>> = [];
     try {
@@ -1168,22 +1254,30 @@ class ClientImpl implements Client {
         const known = await this.store.read((tx) => tx.list("certificates").some(([, c]) => c.streamId === toHex(s.streamId)));
         if (!known) continue;
         const r = await syncStream(this.inboxDeps(), s.streamId, s.authorIdentityId, budget);
-        report.inbox.streams++;
+        // What landed counts, whether or not the stream then failed; a failed stream is not synced.
         report.inbox.stored += r.stored;
         report.inbox.keyPending += r.keyPending;
         report.inbox.invalid += r.invalid;
+        report.timings.bytesReceived += r.bytes;
+        if (r.failure !== undefined) {
+          report.errors.push({ ...syncError("inbox", r.failure), streamId: s.streamId, authorIdentityId: s.authorIdentityId });
+          continue;
+        }
+        report.inbox.streams++;
         if (r.acked !== undefined) report.inbox.acked++;
         if (r.hasMore) report.inbox.hasMore = true;
       } catch (e) {
         report.errors.push({ ...syncError("inbox", e), streamId: s.streamId, authorIdentityId: s.authorIdentityId });
       }
     }
-    report.purged = (await purgeHistory(this.store, this.policy, this.now())) + (await purgeOutbox(this.store, this.policy, this.now()));
+    phase("inbox");
+    if (maintain) report.purged = (await purgeHistory(this.store, this.policy, this.now())) + (await purgeOutbox(this.store, this.policy, this.now()));
+    phase("maintenance");
     await this.emitState();
     return report;
   }
 
-  private async syncMetadata(report: SyncReport): Promise<void> {
+  private async syncMetadata(report: SyncReport, maintain = true): Promise<void> {
     const { pin } = this.requireAccount("sync");
     // Own account: has this device been superseded?
     const [me] = await this.transport.directory.resolveAccounts([this.accountName]);
@@ -1205,9 +1299,9 @@ class ClientImpl implements Client {
         return;
       }
     }
-    // Reset-key list when due.
+    // Reset-key list when due: always when stale (verification needs it), periodically unless maintenance is skipped.
     const now = this.now();
-    const due = !this.trust.fresh(now) || now - this.trust.state.timeHighWater >= RESET_LIST_RECHECK_MS;
+    const due = !this.trust.fresh(now) || (maintain && now - this.trust.state.timeHighWater >= RESET_LIST_RECHECK_MS);
     if (due) {
       const wire = await this.transport.directory.getResetKeyList();
       if (wire) {
@@ -1901,7 +1995,7 @@ class ClientImpl implements Client {
   private async approve(circleId: Uint8Array, accountName: string): Promise<{ revision: bigint }> {
     const [resolved] = await this.transport.directory.resolveAccounts([accountName]);
     if (!resolved) throw new KavachError("METADATA_REQUIRED", "approve", "unknown account");
-    const report: SyncReport = { metadata: { certificates: 0, snapshots: 0, rotations: 0, quarantined: 0, resets: 0, successions: 0 }, epoch: "unchanged", outbox: { sent: 0, accepted: 0, cancelled: 0, unknown: 0, retry: 0 }, inbox: { streams: 0, stored: 0, keyPending: 0, invalid: 0, acked: 0, hasMore: false }, purged: 0, errors: [] };
+    const report: SyncReport = { metadata: { certificates: 0, snapshots: 0, rotations: 0, quarantined: 0, resets: 0, successions: 0 }, epoch: "unchanged", outbox: { sent: 0, accepted: 0, cancelled: 0, unknown: 0, retry: 0 }, inbox: { streams: 0, stored: 0, keyPending: 0, invalid: 0, acked: 0, hasMore: false }, purged: 0, maintenance: "done", timings: emptyTimings(), errors: [] };
     await this.acceptMemberCertificate(resolved, report);
     const requester = resolvedAccountId(resolved);
     const pinned = await this.pinFor(requester);

package/src/errors.ts

@@ -52,6 +52,7 @@ export type ErrorCode =
   | "CLOSED"
   | "RECOVERY_REQUIRED"
   | "STORAGE_FAILURE"
+  | "RATE_LIMITED"
   /** The host could not be reached or did not answer in time; the same call may succeed later. A transport raises it, never a plain error. */
   | "NETWORK";
 
@@ -63,6 +64,7 @@ const RETRYABLE: ReadonlySet<ErrorCode> = new Set<ErrorCode>([
   "LOCKED",
   "STORAGE_FAILURE",
   "NETWORK",
+  "RATE_LIMITED",
   "SUCCESSION_PENDING"
 ]);
 
@@ -75,13 +77,16 @@ export class KavachError extends Error {
   readonly code: ErrorCode;
   readonly stage: string;
   readonly retryable: boolean;
+  /** On `RATE_LIMITED`: how long the host asked the client to wait, when it said. */
+  readonly retryAfterMs: number | undefined;
 
-  constructor(code: ErrorCode, stage = "unknown", message?: string) {
+  constructor(code: ErrorCode, stage = "unknown", message?: string, options: { retryAfterMs?: number } = {}) {
     super(message ?? `${code} at ${stage}`);
     this.name = "KavachError";
     this.code = code;
     this.stage = stage;
     this.retryable = RETRYABLE.has(code);
+    this.retryAfterMs = options.retryAfterMs;
   }
 }
 

package/src/inbox.ts

@@ -191,6 +191,10 @@ export interface StreamSyncSummary {
   acked: bigint | undefined;
   /** The relay holds more records past the last page read; the budget stopped the catch-up. */
   hasMore: boolean;
+  /** Record bytes read from the relay. */
+  bytes: number;
+  /** Set when a read failed after the pages counted above: the caller reports it, with what landed. */
+  failure?: unknown;
 }
 
 function subscription(tx: Pick<Tx, "get">, streamId: Uint8Array, authorIdentityId: Uint8Array): SubscriptionRow {
@@ -236,23 +240,29 @@ export interface ReadBudget {
  * records past the last page when the budget stopped the loop.
  */
 export async function syncStream(deps: InboxDeps, streamId: Uint8Array, authorIdentityId: Uint8Array, budget: ReadBudget = { remainingRecords: READ_BATCH, deadline: deps.now() }): Promise<StreamSyncSummary> {
-  const summary: StreamSyncSummary = { read: 0, stored: 0, duplicate: 0, keyPending: 0, invalid: 0, expired: 0, acked: undefined, hasMore: false };
+  const summary: StreamSyncSummary = { read: 0, stored: 0, duplicate: 0, keyPending: 0, invalid: 0, expired: 0, acked: undefined, hasMore: false, bytes: 0 };
   const sub = await deps.store.read((tx) => subscription(tx, streamId, authorIdentityId));
   const unresolved = await deps.store.read((tx) => tx.list("unresolved", toHex(streamId) + ":"));
   let from = unresolved.length > 0 ? unbig(sub.ackPrefix) : unbig(sub.scanPos);
-  for (;;) {
-    // A failed read is the caller's to report: the sync names the stream in its report and goes
-    // on to the next one, rather than counting a stream it never read as synced.
-    const page = await deps.relay.readRecords(streamId, from, READ_BATCH);
-    await processPage(deps, streamId, authorIdentityId, page, summary);
-    budget.remainingRecords -= page.records.length;
-    const caughtUp = page.nextSeq >= page.tail || page.nextSeq <= from;
-    if (caughtUp) break;
-    if (budget.remainingRecords <= 0 || deps.now() >= budget.deadline) {
-      summary.hasMore = true;
-      break;
+  try {
+    for (;;) {
+      // A failed read is the caller's to report: the sync names the stream in its report and goes
+      // on to the next one, rather than counting a stream it never read as synced.
+      const page = await deps.relay.readRecords(streamId, from, READ_BATCH);
+      await processPage(deps, streamId, authorIdentityId, page, summary);
+      budget.remainingRecords -= page.records.length;
+      const caughtUp = page.nextSeq >= page.tail || page.nextSeq <= from;
+      if (caughtUp) break;
+      if (budget.remainingRecords <= 0 || deps.now() >= budget.deadline) {
+        summary.hasMore = true;
+        break;
+      }
+      from = page.nextSeq;
     }
-    from = page.nextSeq;
+  } catch (e) {
+    // The pages that landed before the failure are real work: the summary keeps them and carries
+    // the failure beside them, untouched, for the caller to report.
+    summary.failure = e;
   }
   return summary;
 }
@@ -269,6 +279,7 @@ async function processPage(deps: InboxDeps, streamId: Uint8Array, authorIdentity
   }
   for (const item of page.records) {
     summary.read++;
+    summary.bytes += item.wire.length;
     if (item.deliverySeq <= page.expiredThrough) {
       summary.expired++;
       continue;

package/src/index.ts

@@ -36,7 +36,8 @@ export {
   type ReplacementProposal,
   type SyncError,
   type SyncOptions,
-  type SyncReport
+  type SyncReport,
+  type SyncTimings
 } from "./client.js";
 
 // Errors -------------------------------------------------------------------------------------------

package/src/outbox.ts

@@ -199,6 +199,10 @@ export interface DrainSummary {
   cancelled: number;
   unknown: number;
   retry: number;
+  /** Record bytes handed to the relay by this drain. */
+  bytes: number;
+  /** The error that ended the drain early, when a send failed before or without an answer (NETWORK, RATE_LIMITED); its record stays queued. */
+  blocked?: KavachError;
 }
 
 /** "none": nothing to send, the drain ends. "skipped": a row was cancelled, the next one is tried. */
@@ -211,17 +215,19 @@ type HandoffStep = { kind: "none" } | { kind: "skipped" } | { kind: "sent"; key:
  * revalidates first.
  */
 export async function drain(deps: OutboxDeps, maxRecords = 1000): Promise<DrainSummary> {
-  const summary: DrainSummary = { sent: 0, accepted: 0, cancelled: 0, unknown: 0, retry: 0 };
+  const summary: DrainSummary = { sent: 0, accepted: 0, cancelled: 0, unknown: 0, retry: 0, bytes: 0 };
   for (let i = 0; i < maxRecords; i++) {
     const step = await handoffNext(deps, summary);
     if (step.kind === "none") break;
     if (step.kind === "skipped") continue; // a cancelled row counts against the limit, never ends the drain
     summary.sent++;
+    summary.bytes += step.row.wire.length / 2;
     let outcome: AppendOutcome | undefined;
     try {
       outcome = await step.outcome;
-    } catch {
-      outcome = undefined; // lost response
+    } catch (e) {
+      outcome = undefined; // no answer: the identical bytes are retried, the relay answers from its checkpoint
+      if (e instanceof KavachError) summary.blocked ??= e; // and the reason reaches the report
     }
     const final = await recordOutcome(deps, step.key, outcome);
     if (final === "accepted") summary.accepted++;

package/src/testing/faults.ts

@@ -113,6 +113,10 @@ export interface MaliciousKnobs {
   readonly hold: Map<MutationName, Promise<void>>;
   /** Called when a held mutation has reached its gate (the host has answered; the response is being held), so a test can act at exactly that point. */
   onHeld?: (name: MutationName) => void;
+  /** Mutations whose next call fails with exactly this error before reaching the host (consumed once). */
+  readonly failWith: Map<MutationName, KavachError>;
+  /** Record reads fail once with NETWORK after this many pages have been served. */
+  failReadsAfter?: number | undefined;
   /** Reads that answer "nothing" while present. */
   readonly withhold: Set<"snapshots" | "epochs" | "records" | "requests" | "currentEpoch" | "keyList" | "certificates" | "successions" | "circleList" | "subscriptions">;
   /** Older documents served instead of the current ones. */
@@ -142,6 +146,7 @@ export function emptyKnobs(): MaliciousKnobs {
     loseResponseOf: new Set(),
     refuse: new Set(),
     hold: new Map(),
+    failWith: new Map(),
     withhold: new Set(),
     replay: { snapshots: new Map() },
     fork: { snapshots: new Map(), manifests: new Map(), records: new Map(), certificates: new Map() },
@@ -298,6 +303,11 @@ export class MaliciousTransport implements Transport {
       appendRecord: (recordId, wire) => {
         // Must stay synchronous up to the inner call (T6h).
         this.calls.push("appendRecord");
+        const fail = k.failWith.get("appendRecord");
+        if (fail) {
+          k.failWith.delete("appendRecord");
+          return Promise.reject(fail);
+        }
         const p = r().appendRecord(recordId, wire);
         if (!k.loseResponseOf.has("appendRecord")) return p;
         k.loseResponseOf.delete("appendRecord");
@@ -308,6 +318,13 @@ export class MaliciousTransport implements Transport {
       },
       readRecords: async (streamId, afterSeq, limit): Promise<ReadRecordsResult> => {
         this.calls.push("readRecords");
+        if (k.failReadsAfter !== undefined) {
+          if (k.failReadsAfter <= 0) {
+            k.failReadsAfter = undefined;
+            throw Object.freeze(new KavachError("NETWORK", "transport", "network unreachable")); // frozen, as a transport may hand it over
+          }
+          k.failReadsAfter--;
+        }
         if (k.withhold.has("records")) {
           const probe = await r().readRecords(streamId, afterSeq, limit);
           return { ...probe, records: [], nextSeq: afterSeq };