@gmickel/gno 2.9.0 → 2.9.2

package/src/serve/server.ts

@@ -3,6 +3,8 @@ import type { RequestPeerServer } from "./request-locality";
 import type { ResidentRuntime } from "./resident-runtime";
 import type { ContextHolder } from "./routes/api";
 
+import { markResidentShutdownHandlerInstalled } from "../core/resident-shutdown-handler";
+import { sighupHandlerAllowed } from "../core/sighup-disposition";
 import { getActivePreset } from "../llm/registry";
 import {
   isHttpGatewayLoopbackBind,
@@ -25,6 +27,7 @@ import {
  * @module src/serve/server
  */
 import { withRemoteHostPathRedaction } from "./host-path-redaction";
+import { watchParentLifeline } from "./parent-lifeline";
 import { PDFJS_ASSET_CACHE_CONTROL } from "./pdfjs-assets";
 // HTML import - Bun handles bundling TSX/CSS automatically via routes
 import homepage from "./public/index.html";
@@ -401,7 +404,7 @@ export function loopbackHttpOrigin(host: string, port: number): string {
 
 /**
  * Start the web server.
- * Opens DB once, closes on SIGINT/SIGTERM.
+ * Opens DB once, closes on SIGINT/SIGTERM/SIGHUP.
  */
 export async function startServer(
   options: ServeOptions = {},
@@ -478,9 +481,22 @@ export async function startServer(
 
   process.once("SIGINT", shutdown);
   process.once("SIGTERM", shutdown);
+  // A closed terminal or logout: shut down cleanly, as on SIGTERM, unless
+  // SIGHUP was ignored at start (nohup), which must keep working.
+  const handleSighup = sighupHandlerAllowed();
+  if (handleSighup) process.once("SIGHUP", shutdown);
+  markResidentShutdownHandlerInstalled();
+  // A launcher that owns this process (the desktop shell) closes our stdin
+  // when it exits or is killed; shut down instead of running orphaned.
+  const stopLifeline = watchParentLifeline(() => {
+    console.log("\nParent process gone.");
+    shutdown();
+  });
   const removeShutdownHandlers = (): void => {
     process.off("SIGINT", shutdown);
     process.off("SIGTERM", shutdown);
+    if (handleSighup) process.off("SIGHUP", shutdown);
+    stopLifeline();
   };
 
   // Start server with try/catch for port-in-use etc.
@@ -1778,6 +1794,11 @@ export async function startServer(
     await dependencies.waitForShutdown(shutdownController.signal);
   } else {
     await new Promise<void>((resolve) => {
+      // The lifeline can fire while the listener is still starting.
+      if (shutdownController.signal.aborted) {
+        resolve();
+        return;
+      }
       shutdownController.signal.addEventListener("abort", () => resolve(), {
         once: true,
       });

package/src/serve/watch-reconciliation-shared.ts

@@ -42,9 +42,20 @@ export const WATCHER_MAX_DIRTY_HINTS = 4_096;
 /** Cap on application-write suppression history entries. */
 export const WATCHER_MAX_SUPPRESSION_ENTRIES = 4_096;
 
-/** Bounded retry delay after failed classification/sync. */
+/** Retry delay after the first failed classification/sync of a collection. */
 export const WATCHER_RETRY_BACKOFF_MS = 500;
 
+/** Ceiling for the retry delay while a collection keeps failing. */
+export const WATCHER_MAX_RETRY_BACKOFF_MS = 5 * 60_000;
+
+/** Delay before the retry that follows `failures` consecutive failed flushes. */
+export function watcherRetryDelayMs(failures: number): number {
+  return Math.min(
+    WATCHER_RETRY_BACKOFF_MS * 2 ** Math.max(0, failures - 1),
+    WATCHER_MAX_RETRY_BACKOFF_MS
+  );
+}
+
 /** Retry delay while another writer holds the shared writer lease. */
 export const WATCHER_LEASE_RETRY_MS = 5_000;
 

package/src/serve/watch-reconciliation.ts

@@ -10,8 +10,13 @@
 import type { Collection } from "../config/types";
 import type { SqliteAdapter } from "../store/sqlite/adapter";
 
+import { relPathUnderAnyPrefix } from "../ingestion";
 import { WATCHER_ACTIVE_SOURCE_PATH_MAX } from "../store/types";
 import { fallbackClassifyDirtyHints } from "./watch-reconciliation-fallback";
+import {
+  fallbackFs,
+  inspectNoFollowPresence,
+} from "./watch-reconciliation-fallback-disk";
 import {
   filterEligiblePaths,
   type ClassificationResult,
@@ -28,6 +33,7 @@ export {
   WATCHER_MAX_DIRTY_HINTS,
   WATCHER_MAX_EXACT_PATHS,
   WATCHER_MAX_FLUSH_DELAY_MS,
+  WATCHER_MAX_RETRY_BACKOFF_MS,
   WATCHER_MAX_SUPPRESSION_ENTRIES,
   WATCHER_RETRY_BACKOFF_MS,
   addToCappedSet,
@@ -39,6 +45,7 @@ export {
   mergeSyncPathBatch,
   pruneSuppressionMap,
   successfulChangedPaths,
+  watcherRetryDelayMs,
   widenVanishedExactPaths,
   type ClassificationResult,
   type ExactPathKind,
@@ -46,6 +53,112 @@ export {
   type WatcherEventClassification,
 } from "./watch-reconciliation-shared";
 
+type StoreOnlyRemovals =
+  | { status: "ok"; paths: string[] }
+  | { status: "overflow" }
+  | { status: "error"; cause: unknown; stage: "scan" | "store" };
+
+/**
+ * Store-known active paths under the dirty hints that are gone from disk.
+ * The snapshot only knows what it scanned: a note indexed from its own exact
+ * event after the baseline is in the store but not the snapshot, and when
+ * its directory is then moved or replaced (one directory event) the snapshot
+ * diff alone would leave it active (fn-209).
+ *
+ * A path the fresh snapshot lists as a file or symlink is present. Paths in
+ * subtrees the snapshot left unproven are left alone. Every other path gets
+ * the fallback classifier's no-follow, availability-aware proof: missing or
+ * no longer a regular source is a removal; an inconclusive check is a scan
+ * error, so the watcher keeps the work and retries rather than infer a
+ * delete. The root hint (a watcher event without a filename) uses the
+ * bounded whole-collection inventory.
+ */
+async function storeOnlyRemovals(options: {
+  store: SqliteAdapter;
+  collection: Collection;
+  rootAbs: string;
+  dirtyHints: readonly string[];
+  next: WatcherSnapshot | null;
+  sourcePathMax: number;
+  snapshotOptions?: WatcherSnapshotOptions;
+}): Promise<StoreOnlyRemovals> {
+  const { store, collection, rootAbs, next, sourcePathMax } = options;
+  const hints = [...new Set(options.dirtyHints)].sort();
+  const storePaths: string[] = [];
+  if (hints.includes("")) {
+    if (typeof store.listActiveSourcePaths !== "function") {
+      return { status: "ok", paths: [] };
+    }
+    const listed = await store.listActiveSourcePaths(
+      collection.name,
+      sourcePathMax
+    );
+    if (!listed.ok) {
+      if (listed.error.code === "OVERFLOW") return { status: "overflow" };
+      return {
+        status: "error",
+        cause: new Error(listed.error.message),
+        stage: "store",
+      };
+    }
+    storePaths.push(...listed.value);
+  } else {
+    if (typeof store.listActiveDescendantSourcePaths !== "function") {
+      return { status: "ok", paths: [] };
+    }
+    // A hint under another hint is covered by the outer descendant lookup.
+    const roots = hints.filter(
+      (hint, index) =>
+        !hints.slice(0, index).some((outer) => hint.startsWith(`${outer}/`))
+    );
+    for (const dir of roots) {
+      const listed = await store.listActiveDescendantSourcePaths(
+        collection.name,
+        dir,
+        Math.max(1, sourcePathMax - storePaths.length)
+      );
+      if (!listed.ok) {
+        if (listed.error.code === "OVERFLOW") return { status: "overflow" };
+        // Not a directory path the store can scope (e.g. escapes the root).
+        if (listed.error.code === "INVALID_INPUT") continue;
+        return {
+          status: "error",
+          cause: new Error(listed.error.message),
+          stage: "store",
+        };
+      }
+      storePaths.push(...listed.value);
+      if (storePaths.length > sourcePathMax) return { status: "overflow" };
+    }
+  }
+
+  const unproven = [...(next?.unprovenSubtrees ?? [])];
+  const fs = options.snapshotOptions?.fs ?? fallbackFs();
+  const removals: string[] = [];
+  for (const path of storePaths) {
+    if (relPathUnderAnyPrefix(path, unproven)) continue;
+    const slash = path.lastIndexOf("/");
+    const parent = slash === -1 ? "" : path.slice(0, slash);
+    const kind = next?.directories
+      .get(parent)
+      ?.get(path.slice(slash + 1))?.kind;
+    if (kind === "file" || kind === "symlink") continue;
+    const presence = await inspectNoFollowPresence(
+      rootAbs,
+      path,
+      fs,
+      options.snapshotOptions?.directoryAvailability
+    );
+    if (presence.status === "error") {
+      return { status: "error", cause: presence.cause, stage: "scan" };
+    }
+    if (presence.status === "missing" || !presence.indexable) {
+      removals.push(path);
+    }
+  }
+  return { status: "ok", paths: removals };
+}
+
 /**
  * Snapshot-first classification of dirty hints. On overflow/scan/metadata
  * failure, uses bounded store + disk enumeration without inferring deletes
@@ -94,10 +207,32 @@ export async function classifyDirtyHints(options: {
       snapshotOptions
     );
     if (diff.status === "ok") {
+      const storeRemovals = await storeOnlyRemovals({
+        store,
+        collection,
+        rootAbs,
+        dirtyHints,
+        next: diff.nextSnapshot,
+        sourcePathMax,
+        snapshotOptions,
+      });
+      if (storeRemovals.status === "overflow") {
+        return { status: "full_reconcile", reason: "budget_overflow" };
+      }
+      if (storeRemovals.status === "error") {
+        return {
+          status: "error",
+          cause: storeRemovals.cause,
+          stage: storeRemovals.stage,
+        };
+      }
       return {
         status: "ok",
         candidates: filterEligiblePaths(diff.candidates, collection),
-        removals: filterEligiblePaths(diff.removals, collection),
+        removals: filterEligiblePaths(
+          [...new Set([...diff.removals, ...storeRemovals.paths])],
+          collection
+        ),
         nextSnapshot: diff.nextSnapshot,
         usedFallback: false,
       };

package/src/serve/watch-service-events.ts

@@ -12,8 +12,8 @@ import type { Collection } from "../config/types";
 import {
   WATCHER_FLUSH_DEBOUNCE_MS,
   WATCHER_MAX_FLUSH_DELAY_MS,
-  WATCHER_RETRY_BACKOFF_MS,
   classifyWatcherFilename,
+  watcherRetryDelayMs,
 } from "./watch-reconciliation";
 import {
   applyPendingForceFlags,
@@ -25,6 +25,12 @@ import {
   type PendingForceFlags,
 } from "./watch-service-state";
 
+/**
+ * Why an explicit retry timer is armed: "backoff" after a failed flush, "fixed"
+ * for an explicit delay such as writer-lease contention (R3: stays at 5 s).
+ */
+export type WatcherRetryKind = "backoff" | "fixed";
+
 export interface WatchEventHost {
   disposed: () => boolean;
   findCollection: (collectionName: string) => Collection | undefined;
@@ -45,8 +51,12 @@ export interface WatchQueueHost {
   pendingByCollection: Map<string, CollectionPending>;
   flushDeadlineAt: Map<string, number>;
   timers: Map<string, ReturnType<typeof setTimeout>>;
-  /** Collections with an explicit retry timer; finally must not bypass. */
-  retryScheduled: Set<string>;
+  /** Collections with an explicit retry timer, by kind; finally must not bypass. */
+  retryScheduled: Map<string, WatcherRetryKind>;
+  /** Consecutive failed flushes per collection; cleared by a successful flush. */
+  retryFailures: Map<string, number>;
+  /** Collections with watcher events queued since their last flush took its work. */
+  freshEvents: Set<string>;
   snapshotReady: Map<string, boolean>;
   inFlightSyncs: Set<Promise<void>>;
   runFlush: (collectionName: string) => Promise<void>;
@@ -105,6 +115,7 @@ export function enqueueExactPath(
     collectionName,
     queueExactPath(pending, relPath, host.maxExactPaths)
   );
+  host.freshEvents.add(collectionName);
   scheduleFlush(host, collectionName);
 }
 
@@ -123,6 +134,7 @@ export function enqueueDirtyHint(
     collectionName,
     queueDirtyHint(pending, hint, host.maxDirtyHints)
   );
+  host.freshEvents.add(collectionName);
   scheduleFlush(host, collectionName);
 }
 
@@ -152,9 +164,17 @@ export function scheduleFlush(
   if (host.disposed()) {
     return;
   }
-  // Explicit retry owns the single timer; do not replace it with debounce.
-  if (host.retryScheduled.has(collectionName)) {
-    return;
+  const retryKind = host.retryScheduled.get(collectionName);
+  if (retryKind !== undefined) {
+    // Explicit retry owns the single timer; do not replace it with debounce.
+    if (retryKind === "fixed" || !host.freshEvents.has(collectionName)) {
+      return;
+    }
+    // A fresh event never waits out a failure backoff: drop the retry timer and
+    // flush on the normal debounce. The failure count stays until a flush succeeds.
+    // A fixed (lease) retry keeps its timer so a held lease is not probed per event.
+    host.retryScheduled.delete(collectionName);
+    host.flushDeadlineAt.delete(collectionName);
   }
   const schedule = computeFlushDelay({
     nowMs: host.clock(),
@@ -213,6 +233,8 @@ export function startFlush(host: WatchQueueHost, collectionName: string): void {
  * Re-arm pending work after a file-level failure with retry backoff.
  * At most one retry timer per collection; finally must not startFlush while set.
  * forceFallback/overflow survive until forced classification + sync succeed.
+ * Without an explicit delay, each consecutive failure doubles the wait (capped),
+ * so work that fails the same way every time cannot spin the resident.
  */
 export function requeueAfterFailure(
   host: WatchQueueHost,
@@ -220,7 +242,7 @@ export function requeueAfterFailure(
   exact: string[],
   dirty: string[],
   forceFlags?: PendingForceFlags,
-  delayMs = WATCHER_RETRY_BACKOFF_MS
+  delayMs?: number
 ): void {
   queueWithoutSchedule(host, collectionName, exact, dirty, forceFlags);
   if (host.disposed()) {
@@ -230,7 +252,19 @@ export function requeueAfterFailure(
     // Pending already merged; existing retry timer remains the sole attempt.
     return;
   }
-  host.retryScheduled.add(collectionName);
+  host.retryScheduled.set(
+    collectionName,
+    delayMs === undefined ? "backoff" : "fixed"
+  );
+  let retryDelayMs = delayMs;
+  if (retryDelayMs === undefined) {
+    const failures = (host.retryFailures.get(collectionName) ?? 0) + 1;
+    host.retryFailures.set(collectionName, failures);
+    // Events that arrived while the failed flush ran flush on the debounce.
+    retryDelayMs = host.freshEvents.has(collectionName)
+      ? host.flushDebounceMs
+      : watcherRetryDelayMs(failures);
+  }
   const existingTimer = host.timers.get(collectionName);
   if (existingTimer) {
     clearTimeout(existingTimer);
@@ -242,7 +276,7 @@ export function requeueAfterFailure(
       host.timers.delete(collectionName);
       host.retryScheduled.delete(collectionName);
       startFlush(host, collectionName);
-    }, delayMs)
+    }, retryDelayMs)
   );
 }
 

package/src/serve/watch-service-flush-generation.ts

@@ -7,18 +7,65 @@
 // node:path — Bun has no path utilities
 import { normalize } from "node:path";
 
+import type { CollectionSyncResult } from "../ingestion";
 import type {
   FlushCollectionInput,
   FlushCollectionOutcome,
 } from "./watch-service-flush";
 
-import { defaultSyncService } from "../ingestion";
+import { defaultSyncService, isSourceAvailabilitySkip } from "../ingestion";
 import { hasFileLevelSyncError } from "./watch-reconciliation";
 import {
   contentChangedPaths,
   notifyCompletedSync,
 } from "./watch-service-flush-helpers";
 
+/** Failure codes from the store side of a sync: transient, worth a retry. */
+const STORE_SIDE_FAILURE_CODES = new Set(["QUERY_FAILED", "STORE_ERROR"]);
+const SOURCE_AVAILABILITY_PREFIX = "SOURCE_AVAILABILITY_";
+
+interface ReconcileFailures {
+  /** A collection-level failure (no path): the reconcile itself is incomplete. */
+  unscoped: boolean;
+  /** Every named failed path, from file receipts and walker errors alike. */
+  named: string[];
+  /** Named paths that failed in the store, not on the file's content. */
+  storeSide: string[];
+}
+
+/** Classify the failures of a completed full-collection sync. */
+function classifyReconcileFailures(
+  result: CollectionSyncResult
+): ReconcileFailures {
+  const named = new Set<string>();
+  const storeSide = new Set<string>();
+  let unscoped = false;
+  for (const file of result.files ?? []) {
+    if (file.status !== "error") continue;
+    named.add(file.relPath);
+    if (STORE_SIDE_FAILURE_CODES.has(file.errorCode ?? ""))
+      storeSide.add(file.relPath);
+  }
+  // A collection whose source availability cannot be proven fails the same
+  // way on every attempt until its config or volume changes (a config edit
+  // starts a new reconcile): record it once, like a content failure.
+  let unavailable = false;
+  // Walker failures (TOO_LARGE, PERMISSION...) appear only here.
+  for (const entry of result.errors) {
+    if (isSourceAvailabilitySkip(entry.code)) continue;
+    if (!entry.relPath) {
+      if (entry.code.startsWith(SOURCE_AVAILABILITY_PREFIX)) unavailable = true;
+      else unscoped = true;
+      continue;
+    }
+    named.add(entry.relPath);
+    if (STORE_SIDE_FAILURE_CODES.has(entry.code)) storeSide.add(entry.relPath);
+  }
+  // A failure count with nothing named is collection-level too.
+  if (named.size === 0 && !unavailable) unscoped = true;
+  return { unscoped, named: [...named], storeSide: [...storeSide] };
+}
+
 /**
  * When collection generation advanced during/before flush, run full
  * syncCollection. Failures leave durable generation work and never advance
@@ -96,11 +143,38 @@ export async function runGenerationReconcile(
         contentChangedPaths(result);
 
       if (hasFileLevelSyncError(result)) {
-        // Notify completed sync once; keep durable generation work.
-        notifyCompletedSync(input, operationPaths, result, ownership);
         const error = new Error(
           "One or more paths failed during watcher generation reconcile"
         );
+        const failures = classifyReconcileFailures(result);
+        // The walk completed: the collection is reconciled except for the
+        // named failed paths (fn-211). Requeuing the whole generation
+        // re-walked the collection every backoff step for as long as one file
+        // kept failing. Store-side failures are transient and can leave a
+        // stale document, so only those paths are retried (exact paths,
+        // failure backoff). Content failures (unreadable, corrupt, too
+        // large...) are recorded once, as `gno update` and the daemon's
+        // initial sync do; the next edit to the file retries it. Keep the
+        // whole generation when any failure is collection-level (e.g. the
+        // document inventory failed) or this flush no longer owns the
+        // collection (its requeue would be dropped).
+        if (
+          !failures.unscoped &&
+          input.getCurrentGeneration() === input.ownerGeneration
+        ) {
+          input.invalidateSnapshot(stillCurrent);
+          notifyCompletedSync(input, operationPaths, result, ownership);
+          input.onSyncError(failures.named, error);
+          if (failures.storeSide.length > 0) {
+            input.requeue(failures.storeSide, []);
+            return { status: "failed", error };
+          }
+          completedGeneration = currentGeneration;
+          needsWork = input.getCurrentGeneration() !== completedGeneration;
+          continue;
+        }
+        // Notify completed sync once; keep durable generation work.
+        notifyCompletedSync(input, operationPaths, result, ownership);
         input.onSyncError([], error);
         input.requeueGeneration();
         return { status: "failed", error };

package/src/serve/watch-service-hosts.ts

@@ -8,7 +8,11 @@ import type { FSWatcher } from "node:fs";
 
 import type { Collection } from "../config/types";
 import type { SyncOptions } from "../ingestion";
-import type { WatchEventHost, WatchQueueHost } from "./watch-service-events";
+import type {
+  WatchEventHost,
+  WatchQueueHost,
+  WatcherRetryKind,
+} from "./watch-service-events";
 import type { WatchLifecycleHost } from "./watch-service-lifecycle";
 import type { CollectionPending } from "./watch-service-state";
 import type { WatcherSnapshot } from "./watch-snapshot";
@@ -46,7 +50,9 @@ export interface WatchServiceHostState {
   maxDirtyHints: number;
   flushDeadlineAt: Map<string, number>;
   timers: Map<string, ReturnType<typeof setTimeout>>;
-  retryScheduled: Set<string>;
+  retryScheduled: Map<string, WatcherRetryKind>;
+  retryFailures: Map<string, number>;
+  freshEvents: Set<string>;
   inFlightSyncs: Set<Promise<void>>;
   runFlush: (collectionName: string) => Promise<void>;
 }
@@ -71,6 +77,8 @@ export function buildLifecycleHost(
     snapshotInit: state.snapshotInit,
     syncing: state.syncing,
     pendingByCollection: state.pendingByCollection,
+    retryFailures: state.retryFailures,
+    freshEvents: state.freshEvents,
     clearCollectionRuntimeState: state.clearCollectionRuntimeState,
     beginSnapshotInit: state.beginSnapshotInit,
     watchFactory: state.watchFactory,
@@ -102,6 +110,8 @@ export function buildQueueHost(state: WatchServiceHostState): WatchQueueHost {
     flushDeadlineAt: state.flushDeadlineAt,
     timers: state.timers,
     retryScheduled: state.retryScheduled,
+    retryFailures: state.retryFailures,
+    freshEvents: state.freshEvents,
     snapshotReady: state.snapshotReady,
     inFlightSyncs: state.inFlightSyncs,
     runFlush: state.runFlush,

package/src/serve/watch-service-lifecycle.ts

@@ -38,6 +38,10 @@ export interface WatchLifecycleHost {
    * forceFallback + durable generation reconcile so a new baseline cannot absorb it.
    */
   pendingByCollection: Map<string, CollectionPending>;
+  /** Consecutive failed flushes per collection (failure backoff input). */
+  retryFailures: Map<string, number>;
+  /** Collections whose queued work is new user action, not a failure retry. */
+  freshEvents: Set<string>;
   clearCollectionRuntimeState: (collectionName: string) => void;
   beginSnapshotInit: (collection: Collection) => void;
   watchFactory: (
@@ -186,6 +190,10 @@ export function applyCollectionUpdate(
         pending.forceFallback = true;
       }
       host.pendingByCollection.set(collection.name, pending);
+      // A config edit is user action that may fix the failure cause: it must
+      // not wait out an armed failure backoff, and the count starts over.
+      host.freshEvents.add(collection.name);
+      host.retryFailures.delete(collection.name);
     }
     if (host.watchers.has(collection.name)) {
       host.beginSnapshotInit(collection);

package/src/serve/watch-service-run-flush.ts

@@ -10,7 +10,7 @@ import { normalize } from "node:path";
 import type { Collection } from "../config/types";
 import type { CollectionSyncResult, SyncOptions } from "../ingestion";
 import type { SqliteAdapter } from "../store/sqlite/adapter";
-import type { WatchQueueHost } from "./watch-service-events";
+import type { WatchQueueHost, WatcherRetryKind } from "./watch-service-events";
 import type { CollectionPending } from "./watch-service-state";
 import type { WatcherSnapshot, WatcherSnapshotFs } from "./watch-snapshot";
 
@@ -36,7 +36,7 @@ export interface RunFlushContext {
   pendingByCollection: Map<string, CollectionPending>;
   flushDeadlineAt: Map<string, number>;
   syncing: Set<string>;
-  retryScheduled: Set<string>;
+  retryScheduled: Map<string, WatcherRetryKind>;
   collectionGenerations: Map<string, number>;
   snapshots: Map<string, WatcherSnapshot>;
   snapshotReady: Map<string, boolean>;
@@ -125,6 +125,7 @@ export async function runOwnedCollectionFlush(
   }
 
   // Events that arrived while the lease was being taken join this flush.
+  ctx.queueHost.freshEvents.delete(collectionName);
   const taken = takePending(
     ctx.pendingByCollection.get(collectionName) ?? pending
   );
@@ -148,6 +149,9 @@ export async function runOwnedCollectionFlush(
     );
   };
 
+  // A flush can requeue failed work (e.g. a dirty classification error) and
+  // still settle as synced/idle; that must not reset the failure backoff.
+  let requeuedFailure = false;
   try {
     const outcome = await flushCollectionOnce({
       collection,
@@ -218,6 +222,7 @@ export async function runOwnedCollectionFlush(
         if (!stillOwner()) {
           return;
         }
+        requeuedFailure = true;
         requeueAfterFailure(
           ctx.queueHost,
           collectionName,
@@ -236,6 +241,12 @@ export async function runOwnedCollectionFlush(
         requeueGenerationReconcile(ctx.queueHost, collectionName);
       },
     });
+    if (
+      (outcome.status === "synced" || outcome.status === "idle") &&
+      !requeuedFailure
+    ) {
+      ctx.queueHost.retryFailures.delete(collectionName);
+    }
     if (
       outcome.status === "failed" &&
       outcome.error &&