@faable/deploy-sdk 2.18.0 → 2.18.1

package/dist/informers/Indexer.d.ts

@@ -16,6 +16,10 @@ export declare class Indexer<T extends {
         applied: boolean;
         old?: T;
     };
+    /** Set unconditionally (a relist is authoritative). Returns the previous object. */
+    set(obj: T, rv: string): T | undefined;
+    /** Remove unconditionally (a relist did not list it). */
+    evict(id: string): T | undefined;
     /** Remove unless what is held is newer than the delete. */
     delete(id: string, rv: string): T | undefined;
     private indexObj;

package/dist/informers/Indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Indexer.d.ts","sourceRoot":"","sources":["../../src/informers/Indexer.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,MAAM,EAAE,CAAA;AAE7C,qBAAa,OAAO,CAAC,CAAC,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE;IAC3C,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,QAAQ,CAAgC;IAChD,OAAO,CAAC,OAAO,CAA8C;IAE7D,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI;IAO5C,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS;IAI9B,IAAI,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAIpC,IAAI,IAAI,CAAC,EAAE;IAIX,GAAG,IAAI,MAAM,EAAE;IAIf,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,EAAE;IAMvC,wEAAwE;IACxE,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,GAAG;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,GAAG,CAAC,EAAE,CAAC,CAAA;KAAE;IASzD,2DAA2D;IAC3D,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS;IAQ7C,OAAO,CAAC,QAAQ;IAQhB,OAAO,CAAC,OAAO;CAShB"}
+{"version":3,"file":"Indexer.d.ts","sourceRoot":"","sources":["../../src/informers/Indexer.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,MAAM,EAAE,CAAA;AAE7C,qBAAa,OAAO,CAAC,CAAC,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE;IAC3C,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,QAAQ,CAAgC;IAChD,OAAO,CAAC,OAAO,CAA8C;IAE7D,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI;IAO5C,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS;IAI9B,IAAI,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAIpC,IAAI,IAAI,CAAC,EAAE;IAIX,GAAG,IAAI,MAAM,EAAE;IAIf,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,EAAE;IAMvC,wEAAwE;IACxE,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,GAAG;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,GAAG,CAAC,EAAE,CAAC,CAAA;KAAE;IASzD,oFAAoF;IACpF,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS;IAQtC,yDAAyD;IACzD,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS;IAQhC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS;IAQ7C,OAAO,CAAC,QAAQ;IAQhB,OAAO,CAAC,OAAO;CAShB"}

package/dist/informers/Indexer.js

@@ -40,6 +40,25 @@ export class Indexer {
             this.indexObj(name, fn, obj);
         return { applied: true, old: cur?.obj };
     }
+    /** Set unconditionally (a relist is authoritative). Returns the previous object. */
+    set(obj, rv) {
+        const cur = this.objects.get(obj.id);
+        if (cur)
+            this.unindex(cur.obj);
+        this.objects.set(obj.id, { obj, rv });
+        for (const [name, fn] of this.indexFns)
+            this.indexObj(name, fn, obj);
+        return cur?.obj;
+    }
+    /** Remove unconditionally (a relist did not list it). */
+    evict(id) {
+        const cur = this.objects.get(id);
+        if (!cur)
+            return undefined;
+        this.unindex(cur.obj);
+        this.objects.delete(id);
+        return cur.obj;
+    }
     /** Remove unless what is held is newer than the delete. */
     delete(id, rv) {
         const cur = this.objects.get(id);

package/dist/informers/InformerFactory.d.ts

@@ -1,14 +1,23 @@
 import { Indexer, type IndexFn } from "./Indexer.js";
-import { type WatchClient, type WatchResource } from "./WatchClient.js";
+import { type WatchClient, type WatchEvent, type WatchResource } from "./WatchClient.js";
 export type InformerEvent = "add" | "update" | "delete";
 type Handler<T> = (obj: T, old?: T) => void;
+type Logger = {
+    warn: (o: object, m?: string) => void;
+    info?: (o: object, m?: string) => void;
+};
+/** Thrown into the loop when a watch sends nothing for `idleTimeoutMs`. */
+export declare class WatchIdleError extends Error {
+    constructor(idleMs: number);
+}
 export declare class SharedInformer<T extends {
     id: string;
 }> {
     readonly store: Indexer<T>;
     private handlers;
     readonly resource: WatchResource;
-    constructor(resource: WatchResource);
+    private logger?;
+    constructor(resource: WatchResource, logger?: Logger);
     on(event: InformerEvent, handler: Handler<T>): this;
     addIndex(name: string, fn: IndexFn<T>): this;
     get(id: string): T | undefined;
@@ -16,24 +25,36 @@ export declare class SharedInformer<T extends {
     byIndex(name: string, key: string): T[];
     /** @internal */
     emit(event: InformerEvent, obj: T, old?: T): void;
-    /** @internal Apply an upsert; emits add/update. */
-    apply(obj: T, rv: string): void;
-    /** @internal Apply a delete; emits delete. */
-    remove(id: string, rv: string): void;
-    /** @internal A relist: apply every item, then delete what was not listed. */
+    /** @internal Apply an upsert; emits add/update. false if dropped as stale. */
+    apply(obj: T, rv: string): boolean;
+    /** @internal Apply a delete; emits delete. false if dropped as stale. */
+    remove(id: string, rv: string): boolean;
+    /**
+     * @internal A relist, authoritative like client-go's Replace: the listed
+     * objects become the store regardless of rvs (rvs of another generation
+     * are not comparable). add for new, update for changed, delete for gone.
+     */
     replace(items: {
         rv: string;
         object: T;
-    }[], listRv: string): void;
+    }[]): void;
 }
 export type InformerFactoryOptions = {
     resources: WatchResource[];
-    logger?: {
-        warn: (o: object, m?: string) => void;
-        info?: (o: object, m?: string) => void;
-    };
-    /** Backoff after a failed list or watch: first and max delay. */
+    logger?: Logger;
+    /** Backoff after an attempt without progress: first and max delay (±20 % jitter). */
     backoffMs?: [number, number];
+    /**
+     * A watch that sends no line for this long is aborted and reconnected (the
+     * server sends a BOOKMARK every ≤ 15 s). Default 45 s.
+     */
+    idleTimeoutMs?: number;
+    /**
+     * Every event off the watch, before it is applied. `lagMs` is how long a
+     * MODIFIED/DELETED took from the cache's stream to here (null for
+     * BOOKMARK/ERROR, whose rv is not a change). For metrics; must not throw.
+     */
+    onEvent?: (ev: WatchEvent, lagMs: number | null) => void;
 };
 export declare class InformerFactory {
     private informers;
@@ -42,12 +63,24 @@ export declare class InformerFactory {
     private synced;
     private syncWaiters;
     private rv?;
+    /** The generation of the current lists; events of another one → relist. */
+    private gen;
     private stopped;
     readonly stats: {
         relists: number;
         reconnects: number;
         events: number;
         lastEventAt: number;
+        /** Failed lists/watches, idle watches included (each one retried with backoff). */
+        errors: number;
+        /** When the stores last held a complete list (initial sync or relist). */
+        lastSyncAt: number;
+        /** Events older than what the store held (same generation), not applied. */
+        dropped: number;
+        /** ERROR events other than watch_expired. */
+        watchErrors: number;
+        /** From a 410 / generation change until a relist succeeds. */
+        relistPending: boolean;
     };
     private client;
     private options;
@@ -56,11 +89,24 @@ export declare class InformerFactory {
         id: string;
     } = any>(resource: WatchResource): SharedInformer<T>;
     hasSynced(): boolean;
+    /**
+     * Synced, no relist pending, and something (an event — BOOKMARKs included —
+     * or a list) arrived within `maxQuietMs`.
+     */
+    isFresh(maxQuietMs: number): boolean;
     waitForSync(): Promise<void>;
     start(): void;
     stop(): Promise<void>;
+    /** Fetch every list first; replace only if all of them arrived (K4). */
     private relist;
     private handle;
+    /**
+     * One watch connection, read with an idle watchdog: no line for
+     * `idleTimeoutMs` aborts the request (its signal) and throws. The race
+     * does not trust the client to honour the signal.
+     */
+    private watchOnce;
+    private sleep;
     private run;
 }
 export {};

package/dist/informers/InformerFactory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"InformerFactory.d.ts","sourceRoot":"","sources":["../../src/informers/InformerFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,KAAK,OAAO,EAAE,MAAM,cAAc,CAAA;AAEpD,OAAO,EAEL,KAAK,WAAW,EAEhB,KAAK,aAAa,EACnB,MAAM,kBAAkB,CAAA;AAkBzB,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,CAAA;AACvD,KAAK,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,KAAK,IAAI,CAAA;AAE3C,qBAAa,cAAc,CAAC,CAAC,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE;IAClD,QAAQ,CAAC,KAAK,aAAmB;IACjC,OAAO,CAAC,QAAQ,CAIf;IAED,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAA;gBAEpB,QAAQ,EAAE,aAAa;IAInC,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI;IAKnD,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI;IAK5C,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS;IAI9B,IAAI,IAAI,CAAC,EAAE;IAIX,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,EAAE;IAIvC,gBAAgB;IAChB,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IAW1C,mDAAmD;IACnD,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM;IAKxB,8CAA8C;IAC9C,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM;IAK7B,6EAA6E;IAC7E,OAAO,CAAC,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,CAAC,CAAA;KAAE,EAAE,EAAE,MAAM,EAAE,MAAM;CAQ3D;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,SAAS,EAAE,aAAa,EAAE,CAAA;IAC1B,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;QAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAA;KAAE,CAAA;IAC1F,iEAAiE;IACjE,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC7B,CAAA;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,SAAS,CAAgD;IACjE,OAAO,CAAC,KAAK,CAAC,CAAiB;IAC/B,OAAO,CAAC,IAAI,CAAC,CAAe;IAC5B,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,EAAE,CAAC,CAAQ;IACnB,OAAO,CAAC,OAAO,CAAO;IACtB,QAAQ,CAAC,KAAK;;;;;MAA2D;IAEzE,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,OAAO,CAAwB;gBAE3B,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,sBAAsB;IAMhE,QAAQ,CAAC,CAAC,SAAS;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,aAAa,GAAG,cAAc,CAAC,CAAC,CAAC;IAMpF,SAAS,IAAI,OAAO;IAIpB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B,KAAK,IAAI,IAAI;IAOP,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAMb,MAAM;IAiBpB,OAAO,CAAC,MAAM;YAoBA,GAAG;CAoClB"}
+{"version":3,"file":"InformerFactory.d.ts","sourceRoot":"","sources":["../../src/informers/InformerFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,KAAK,OAAO,EAAE,MAAM,cAAc,CAAA;AAEpD,OAAO,EAGL,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,aAAa,EACnB,MAAM,kBAAkB,CAAA;AAyBzB,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,CAAA;AACvD,KAAK,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,KAAK,IAAI,CAAA;AAC3C,KAAK,MAAM,GAAG;IAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAA;CAAE,CAAA;AAE/F,2EAA2E;AAC3E,qBAAa,cAAe,SAAQ,KAAK;gBAC3B,MAAM,EAAE,MAAM;CAI3B;AAED,qBAAa,cAAc,CAAC,CAAC,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE;IAClD,QAAQ,CAAC,KAAK,aAAmB;IACjC,OAAO,CAAC,QAAQ,CAIf;IAED,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAA;IAChC,OAAO,CAAC,MAAM,CAAC,CAAQ;gBAEX,QAAQ,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,MAAM;IAKpD,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI;IAKnD,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI;IAK5C,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS;IAI9B,IAAI,IAAI,CAAC,EAAE;IAIX,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,EAAE;IAIvC,gBAAgB;IAChB,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IAe1C,8EAA8E;IAC9E,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO;IAMlC,yEAAyE;IACzE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO;IAOvC;;;;OAIG;IACH,OAAO,CAAC,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,CAAC,CAAA;KAAE,EAAE;CAc3C;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,SAAS,EAAE,aAAa,EAAE,CAAA;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,qFAAqF;IACrF,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC5B;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;;;OAIG;IACH,OAAO,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,KAAK,IAAI,CAAA;CACzD,CAAA;AAID,qBAAa,eAAe;IAC1B,OAAO,CAAC,SAAS,CAAgD;IACjE,OAAO,CAAC,KAAK,CAAC,CAAiB;IAC/B,OAAO,CAAC,IAAI,CAAC,CAAe;IAC5B,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,EAAE,CAAC,CAAQ;IACnB,2EAA2E;IAC3E,OAAO,CAAC,GAAG,CAAsB;IACjC,OAAO,CAAC,OAAO,CAAO;IACtB,QAAQ,CAAC,KAAK;;;;;QAKZ,mFAAmF;;QAEnF,0EAA0E;;QAE1E,4EAA4E;;QAE5E,6CAA6C;;QAE7C,8DAA8D;;MAE/D;IAED,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,OAAO,CAAwB;gBAE3B,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,sBAAsB;IAOhE,QAAQ,CAAC,CAAC,SAAS;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,aAAa,GAAG,cAAc,CAAC,CAAC,CAAC;IAMpF,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,OAAO,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAKpC,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B,KAAK,IAAI,IAAI;IAOP,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B,wEAAwE;YAC1D,MAAM;IA2BpB,OAAO,CAAC,MAAM;IAgDd;;;;OAIG;YACW,SAAS;IA+CvB,OAAO,CAAC,KAAK;YAcC,GAAG;CAsClB"}

package/dist/informers/InformerFactory.js

@@ -1,6 +1,13 @@
 import { Indexer } from "./Indexer.js";
-import { compareRv, minRv } from "./rv.js";
+import { compareRv, minRv, rvGeneration, rvTimeMs } from "./rv.js";
 import { WatchExpiredError, } from "./WatchClient.js";
+/** Thrown into the loop when a watch sends nothing for `idleTimeoutMs`. */
+export class WatchIdleError extends Error {
+    constructor(idleMs) {
+        super(`watch idle for ${idleMs} ms: reconnecting`);
+        this.name = "WatchIdleError";
+    }
+}
 export class SharedInformer {
     store = new Indexer();
     handlers = {
@@ -9,8 +16,10 @@ export class SharedInformer {
         delete: [],
     };
     resource;
-    constructor(resource) {
+    logger;
+    constructor(resource, logger) {
         this.resource = resource;
+        this.logger = logger;
     }
     on(event, handler) {
         this.handlers[event].push(handler);
@@ -35,34 +44,51 @@ export class SharedInformer {
             try {
                 h(obj, old);
             }
-            catch {
+            catch (err) {
                 // A handler's failure is the handler's: the store and the other
-                // handlers carry on.
+                // handlers carry on. But it is said.
+                this.logger?.warn({ err, resource: this.resource, event, id: obj?.id }, "informers: handler threw");
             }
         }
     }
-    /** @internal Apply an upsert; emits add/update. */
+    /** @internal Apply an upsert; emits add/update. false if dropped as stale. */
     apply(obj, rv) {
         const { applied, old } = this.store.upsert(obj, rv);
         if (applied)
             this.emit(old ? "update" : "add", obj, old);
+        return applied;
     }
-    /** @internal Apply a delete; emits delete. */
+    /** @internal Apply a delete; emits delete. false if dropped as stale. */
     remove(id, rv) {
+        if (!this.store.get(id))
+            return true;
         const gone = this.store.delete(id, rv);
         if (gone)
             this.emit("delete", gone);
+        return !!gone;
     }
-    /** @internal A relist: apply every item, then delete what was not listed. */
-    replace(items, listRv) {
+    /**
+     * @internal A relist, authoritative like client-go's Replace: the listed
+     * objects become the store regardless of rvs (rvs of another generation
+     * are not comparable). add for new, update for changed, delete for gone.
+     */
+    replace(items) {
         const seen = new Set();
         for (const { rv, object } of items) {
             seen.add(object.id);
-            this.apply(object, rv);
+            const old = this.store.set(object, rv);
+            if (!old)
+                this.emit("add", object);
+            else if (JSON.stringify(old) !== JSON.stringify(object))
+                this.emit("update", object, old);
+        }
+        for (const id of this.store.ids()) {
+            if (seen.has(id))
+                continue;
+            const gone = this.store.evict(id);
+            if (gone)
+                this.emit("delete", gone);
         }
-        for (const id of this.store.ids())
-            if (!seen.has(id))
-                this.remove(id, listRv);
     }
 }
 export class InformerFactory {
@@ -72,15 +98,32 @@ export class InformerFactory {
     synced = false;
     syncWaiters = [];
     rv;
+    /** The generation of the current lists; events of another one → relist. */
+    gen = null;
     stopped = true;
-    stats = { relists: 0, reconnects: 0, events: 0, lastEventAt: 0 };
+    stats = {
+        relists: 0,
+        reconnects: 0,
+        events: 0,
+        lastEventAt: 0,
+        /** Failed lists/watches, idle watches included (each one retried with backoff). */
+        errors: 0,
+        /** When the stores last held a complete list (initial sync or relist). */
+        lastSyncAt: 0,
+        /** Events older than what the store held (same generation), not applied. */
+        dropped: 0,
+        /** ERROR events other than watch_expired. */
+        watchErrors: 0,
+        /** From a 410 / generation change until a relist succeeds. */
+        relistPending: false,
+    };
     client;
     options;
     constructor(client, options) {
         this.client = client;
         this.options = options;
         for (const r of options.resources)
-            this.informers.set(r, new SharedInformer(r));
+            this.informers.set(r, new SharedInformer(r, options.logger));
     }
     informer(resource) {
         const inf = this.informers.get(resource);
@@ -91,6 +134,15 @@ export class InformerFactory {
     hasSynced() {
         return this.synced;
     }
+    /**
+     * Synced, no relist pending, and something (an event — BOOKMARKs included —
+     * or a list) arrived within `maxQuietMs`.
+     */
+    isFresh(maxQuietMs) {
+        if (!this.synced || this.stats.relistPending)
+            return false;
+        return Date.now() - Math.max(this.stats.lastEventAt, this.stats.lastSyncAt) < maxQuietMs;
+    }
     waitForSync() {
         if (this.synced)
             return Promise.resolve();
@@ -108,17 +160,26 @@ export class InformerFactory {
         this.abort?.abort();
         await this.loop?.catch(() => { });
     }
+    /** Fetch every list first; replace only if all of them arrived (K4). */
     async relist() {
-        const rvs = [];
-        for (const [resource, inf] of this.informers) {
-            const { rv, items } = await this.client.list(resource);
-            inf.replace(items, rv);
-            rvs.push(rv);
-        }
+        const lists = await Promise.all([...this.informers].map(async ([resource, inf]) => [inf, await this.client.list(resource)]));
+        const rvs = lists.map(([, l]) => l.rv);
+        const gens = new Set(rvs.map(rvGeneration));
+        if (gens.size > 1)
+            throw new Error(`lists span generations (${rvs.join(", ")}): relisting`);
+        for (const [inf, l] of lists)
+            inf.replace(l.items);
         // Watch from the OLDEST list rv: events already reflected in a newer
         // list are dropped by the per-object rv.
         this.rv = minRv(rvs);
+        this.gen = rvGeneration(this.rv);
         this.stats.relists++;
+        this.stats.lastSyncAt = Date.now();
+        this.stats.relistPending = false;
+        const items = {};
+        for (const [inf, l] of lists)
+            items[inf.resource] = l.items.length;
+        this.options.logger?.info?.({ relists: this.stats.relists, rv: this.rv, items }, "informers: relisted");
         if (!this.synced) {
             this.synced = true;
             for (const w of this.syncWaiters.splice(0))
@@ -128,57 +189,155 @@ export class InformerFactory {
     handle(ev) {
         this.stats.events++;
         this.stats.lastEventAt = Date.now();
+        if (this.options.onEvent) {
+            const t = ev.type === "MODIFIED" || ev.type === "DELETED" ? rvTimeMs(ev.rv) : null;
+            try {
+                this.options.onEvent(ev, t === null ? null : Date.now() - t);
+            }
+            catch {
+                // Metrics must never break the watch.
+            }
+        }
+        if (ev.type === "ERROR") {
+            if (ev.code === "watch_expired")
+                return "relist";
+            this.stats.watchErrors++;
+            this.options.logger?.warn({ code: ev.code, message: ev.message }, "informers: watch ERROR event, reconnecting");
+            return "error";
+        }
+        // Another generation than the lists': the server's stream was reset and
+        // rvs across generations do not compare. Relist.
+        const gen = rvGeneration(ev.rv);
+        if (gen !== null && this.gen !== null && gen !== this.gen) {
+            this.options.logger?.info?.({ rv: ev.rv, generation: this.gen }, "informers: generation changed, relisting");
+            return "relist";
+        }
         switch (ev.type) {
             case "BOOKMARK":
                 break;
             case "MODIFIED":
-                this.informers.get(ev.resource)?.apply(ev.object, ev.objectRv);
+                if (this.informers.get(ev.resource)?.apply(ev.object, ev.objectRv) === false)
+                    this.stats.dropped++;
                 break;
             case "DELETED":
-                this.informers.get(ev.resource)?.remove(ev.object.id, ev.rv);
+                if (this.informers.get(ev.resource)?.remove(ev.object.id, ev.rv) === false)
+                    this.stats.dropped++;
                 break;
-            case "ERROR":
-                return ev.code === "watch_expired" ? "relist" : "ok";
         }
         // Resume point: the stream position, which only moves forward.
-        if ("rv" in ev && ev.rv && (!this.rv || compareRv(ev.rv, this.rv) > 0))
+        if (ev.rv && (!this.rv || compareRv(ev.rv, this.rv) > 0))
             this.rv = ev.rv;
         return "ok";
     }
+    /**
+     * One watch connection, read with an idle watchdog: no line for
+     * `idleTimeoutMs` aborts the request (its signal) and throws. The race
+     * does not trust the client to honour the signal.
+     */
+    async watchOnce(onProgress) {
+        const idleMs = this.options.idleTimeoutMs ?? 45_000;
+        const conn = new AbortController();
+        const outer = this.abort.signal;
+        const onStop = () => conn.abort();
+        // stop() may have run already: an aborted signal never fires again.
+        if (outer.aborted)
+            conn.abort();
+        else
+            outer.addEventListener("abort", onStop, { once: true });
+        const stopped = new Promise((r) => conn.signal.aborted ? r("stopped") : conn.signal.addEventListener("abort", () => r("stopped")));
+        const it = this.client.watch(this.options.resources, this.rv, conn.signal)[Symbol.asyncIterator]();
+        let events = 0;
+        try {
+            for (;;) {
+                let timer;
+                const idle = new Promise((r) => (timer = setTimeout(() => r("idle"), idleMs)));
+                let next;
+                try {
+                    next = await Promise.race([it.next(), idle, stopped]);
+                }
+                finally {
+                    clearTimeout(timer);
+                }
+                if (next === "stopped")
+                    return "closed";
+                if (next === "idle")
+                    throw new WatchIdleError(idleMs);
+                if (next.done) {
+                    this.stats.reconnects++;
+                    this.options.logger?.info?.({ reconnects: this.stats.reconnects, events, rv: this.rv }, "informers: watch closed, reconnecting");
+                    return "closed";
+                }
+                events++;
+                const before = this.rv;
+                const out = this.handle(next.value);
+                if (out !== "ok")
+                    return out;
+                if (this.rv !== before)
+                    onProgress();
+            }
+        }
+        finally {
+            conn.abort();
+            outer.removeEventListener("abort", onStop);
+            // Not awaited: a stalled iterator may never settle.
+            it.return?.()?.catch?.(() => { });
+        }
+    }
+    sleep(ms) {
+        return new Promise((r) => {
+            const timer = setTimeout(done, ms);
+            const signal = this.abort.signal;
+            function done() {
+                clearTimeout(timer);
+                signal.removeEventListener("abort", done);
+                r();
+            }
+            if (signal.aborted)
+                done();
+            else
+                signal.addEventListener("abort", done, { once: true });
+        });
+    }
     async run() {
         const [first, max] = this.options.backoffMs ?? [500, 30_000];
         let backoff = first;
         let needList = true;
         while (!this.stopped) {
+            // Progress = the stream position moved. An attempt without it (error,
+            // empty stream, ERROR event, 410 right after a relist) backs off.
+            let progressed = false;
             try {
                 if (needList) {
                     await this.relist();
                     needList = false;
                 }
-                let relist = false;
-                for await (const ev of this.client.watch(this.options.resources, this.rv, this.abort.signal)) {
-                    if (this.handle(ev) === "relist") {
-                        relist = true;
-                        break;
-                    }
-                    backoff = first;
-                }
-                if (relist)
+                const out = await this.watchOnce(() => (progressed = true));
+                if (out === "relist") {
                     needList = true;
-                else
-                    this.stats.reconnects++;
+                    this.stats.relistPending = true;
+                }
             }
             catch (err) {
                 if (this.stopped)
                     break;
                 if (err instanceof WatchExpiredError) {
                     needList = true;
-                    continue;
+                    this.stats.relistPending = true;
                 }
-                this.options.logger?.warn({ err, backoff }, "informers: list/watch failed, retrying");
-                await new Promise((r) => setTimeout(r, backoff));
-                backoff = Math.min(backoff * 2, max);
+                else {
+                    this.stats.errors++;
+                    this.options.logger?.warn({ err, backoff }, "informers: list/watch failed, retrying");
+                }
+            }
+            if (this.stopped)
+                break;
+            if (progressed) {
+                backoff = first;
+                continue;
             }
+            // ±20 % jitter, so a fleet does not reconnect in step.
+            await this.sleep(backoff * (0.8 + Math.random() * 0.4));
+            backoff = Math.min(backoff * 2, max);
         }
     }
 }

package/dist/informers/index.d.ts

@@ -1,6 +1,6 @@
 export { Indexer, type IndexFn } from "./Indexer.js";
-export { InformerFactory, SharedInformer, type InformerEvent, type InformerFactoryOptions, } from "./InformerFactory.js";
-export { compareRv, minRv } from "./rv.js";
+export { InformerFactory, SharedInformer, WatchIdleError, type InformerEvent, type InformerFactoryOptions, } from "./InformerFactory.js";
+export { compareRv, minRv, rvGeneration, rvTimeMs } from "./rv.js";
 export { WatchExpiredError, type ListResult, type WatchClient, type WatchEvent, type WatchResource, } from "./WatchClient.js";
 export { deployApiWatchClient } from "./deployApiWatchClient.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/informers/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/informers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,KAAK,OAAO,EAAE,MAAM,cAAc,CAAA;AACpD,OAAO,EACL,eAAe,EACf,cAAc,EACd,KAAK,aAAa,EAClB,KAAK,sBAAsB,GAC5B,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EACL,iBAAiB,EACjB,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,aAAa,GACnB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/informers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,KAAK,OAAO,EAAE,MAAM,cAAc,CAAA;AACpD,OAAO,EACL,eAAe,EACf,cAAc,EACd,cAAc,EACd,KAAK,aAAa,EAClB,KAAK,sBAAsB,GAC5B,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAClE,OAAO,EACL,iBAAiB,EACjB,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,aAAa,GACnB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAA"}

package/dist/informers/index.js

@@ -1,5 +1,5 @@
 export { Indexer } from "./Indexer.js";
-export { InformerFactory, SharedInformer, } from "./InformerFactory.js";
-export { compareRv, minRv } from "./rv.js";
+export { InformerFactory, SharedInformer, WatchIdleError, } from "./InformerFactory.js";
+export { compareRv, minRv, rvGeneration, rvTimeMs } from "./rv.js";
 export { WatchExpiredError, } from "./WatchClient.js";
 export { deployApiWatchClient } from "./deployApiWatchClient.js";

package/dist/informers/rv.d.ts

@@ -1,4 +1,15 @@
 /** <0, 0, >0 like a comparator; an unparsable rv sorts first. */
 export declare const compareRv: (a: string, b: string) => number;
+/**
+ * When the change entered the cache's stream (ms since epoch), from the
+ * stream position in an rv. `now - rvTimeMs(rv)` is how long the event took
+ * to reach this process — the informer's freshness. null if unparsable.
+ */
+export declare const rvTimeMs: (rv: string) => number | null;
+/**
+ * The generation of an rv (it changes when the server's stream is reset).
+ * rvs of different generations are not comparable. null if unparsable.
+ */
+export declare const rvGeneration: (rv: string) => number | null;
 export declare const minRv: (rvs: string[]) => string;
 //# sourceMappingURL=rv.d.ts.map

package/dist/informers/rv.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rv.d.ts","sourceRoot":"","sources":["../../src/informers/rv.ts"],"names":[],"mappings":"AASA,iEAAiE;AACjE,eAAO,MAAM,SAAS,MAAO,MAAM,KAAK,MAAM,KAAG,MAKhD,CAAA;AAED,eAAO,MAAM,KAAK,QAAS,MAAM,EAAE,KAAG,MACe,CAAA"}
+{"version":3,"file":"rv.d.ts","sourceRoot":"","sources":["../../src/informers/rv.ts"],"names":[],"mappings":"AASA,iEAAiE;AACjE,eAAO,MAAM,SAAS,MAAO,MAAM,KAAK,MAAM,KAAG,MAKhD,CAAA;AAED;;;;GAIG;AACH,eAAO,MAAM,QAAQ,OAAQ,MAAM,KAAG,MAAM,GAAG,IAG9C,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,YAAY,OAAQ,MAAM,KAAG,MAAM,GAAG,IAGlD,CAAA;AAED,eAAO,MAAM,KAAK,QAAS,MAAM,EAAE,KAAG,MACe,CAAA"}

package/dist/informers/rv.js

@@ -13,4 +13,21 @@ export const compareRv = (a, b) => {
         return pa ? 1 : pb ? -1 : 0;
     return pa[0] - pb[0] || pa[1] - pb[1] || pa[2] - pb[2];
 };
+/**
+ * When the change entered the cache's stream (ms since epoch), from the
+ * stream position in an rv. `now - rvTimeMs(rv)` is how long the event took
+ * to reach this process — the informer's freshness. null if unparsable.
+ */
+export const rvTimeMs = (rv) => {
+    const p = parse(rv);
+    return p ? p[1] : null;
+};
+/**
+ * The generation of an rv (it changes when the server's stream is reset).
+ * rvs of different generations are not comparable. null if unparsable.
+ */
+export const rvGeneration = (rv) => {
+    const p = parse(rv);
+    return p ? p[0] : null;
+};
 export const minRv = (rvs) => rvs.reduce((m, r) => (compareRv(r, m) < 0 ? r : m));

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@faable/deploy-sdk",
   "description": "Codebase for SDK building",
-  "version": "2.18.0",
+  "version": "2.18.1",
   "author": "Marc Pomar <marc@faable.com>",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",