@optimystic/db-p2p 1.0.0-beta.3 → 1.1.0

package/dist/src/repo/service.js

@@ -1,13 +1,14 @@
 import { pipe } from 'it-pipe';
 import { decode as lpDecode, encode as lpEncode } from 'it-length-prefixed';
-import { blockIdsForTransforms } from '@optimystic/db-core';
-import { peersEqual } from '../peer-utils.js';
+import { LruMap, blockIdsForTransforms, routingKeyForBlock } from '@optimystic/db-core';
+import { peerIdFromString } from '@libp2p/peer-id';
 import { encodePeers } from './redirect.js';
 import { MAX_BLOCK_MESSAGE_BYTES } from '../protocol-limits.js';
 import { createLogger } from '../logger.js';
 import { publishableAddrsForPeer } from '../peer-address-book.js';
 import { createInboundStreamAuthorization } from '../inbound-authorization.js';
 import { registerProtocolHandler } from '../network/register-protocol-handler.js';
+import { RESPONSIBILITY_TTL_MS } from './responsibility.js';
 export function repoService(init = {}) {
     return (components) => new RepoService(components, init);
 }
@@ -28,7 +29,7 @@ export class RepoService {
      * The libp2p node, injected post-construction by the node wiring (see
      * libp2p-node-base.ts, mirroring how `networkManager`/`fret` receive theirs).
      * The libp2p `components.libp2p` proxy does NOT reliably resolve from inside a
-     * service at request time, so the redirect path resolves the network manager,
+     * service at request time, so the redirect path resolves the key network,
      * self identity, and connection addrs through this explicitly-set reference.
      */
     libp2pRef;
@@ -40,6 +41,16 @@ export class RepoService {
      * ingress point, instead of keeping it filterable as the single `peer-address-book` namespace.
      */
     addressLog;
+    /**
+     * Per block key, the responsible peer ids the key network last named: one cohort lookup per block per
+     * minute on the inbound path, instead of one per request on top of the coordinator's own.
+     *
+     * NOTE: same TTL as `CoordinatorRepo`'s responsibility cache, deliberately not the same cache — the two
+     * sit on either side of the served-repo boundary. Their verdicts can therefore disagree about a block
+     * only inside the minute after its cohort changes; a request this check wrongly lets through is refused
+     * by the coordinator, and one it wrongly redirects is followed by the client to a member that serves it.
+     */
+    responsibleIds = new LruMap(1000);
     constructor(components, init = {}) {
         this.components = components;
         const computed = init.protocol ?? (init.protocolPrefix ?? '/db-p2p') + '/repo/1.0.0';
@@ -88,10 +99,15 @@ export class RepoService {
         await this.components.registrar.unhandle(this.protocol);
         this.running = false;
     }
-    getNetworkManager() {
-        if (this.components.networkManager)
-            return this.components.networkManager;
-        return this.getLibp2p()?.services?.networkManager;
+    /**
+     * The key network the redirect check asks. Absent only before the node attaches it, which happens after
+     * `start()`; a request in that window is handled locally, where the served-repo proxy has no
+     * coordinator to hand it to yet either.
+     */
+    getKeyNetwork() {
+        if (this.components.keyNetwork)
+            return this.components.keyNetwork;
+        return this.getLibp2p()?.keyNetwork;
     }
     getSelfId() {
         if (this.components.peerId)
@@ -126,7 +142,7 @@ export class RepoService {
      *   - pend   → blockIdsForTransforms(transforms)[0]
      *   - cancel → actionRef.blockIds[0]
      *   - commit → blockIds[0]  (CoordinatorRepo.commit anchors consensus on
-     *     getClusterSize(blockIds[0]) / executeClusterTransaction(blockIds[0]) and guards
+     *     resolveCohort(blockIds[0]) / executeClusterTransaction(blockIds[0]) and guards
      *     with verifyResponsibility(blockIds) — NOT tailId; for a per-block commit batch
      *     whose blockIds[0] !== tailId, keying on tailId redirected the commit to the
      *     collection tail's cluster, which then fails verifyResponsibility for the non-tail block.)
@@ -153,37 +169,58 @@ export class RepoService {
     /**
      * Check if this node should redirect the request for a given key.
      * Returns a RedirectPayload if not responsible, null if should handle locally.
-     * Also attaches cluster info to the message for downstream use.
+     * Also attaches the responsible peer ids to the message for downstream use.
+     *
+     * The responsible set comes from the node's own key network — `findCluster` on the block's routing
+     * key, the same call and the same bytes the writer's transactor and this node's `CoordinatorRepo` use —
+     * so a correctly routed request is never redirected, including on machines shared by several networks,
+     * where FRET's raw cohort would include peers that do not serve this one.
+     *
+     * A lookup that THROWS splits by operation: a `get` is handled locally (a read is best-effort, and the
+     * coordinator flags what it could not confirm); every other operation propagates the error, which
+     * aborts the stream so the writer excludes this peer and re-picks — the posture `CoordinatorRepo` takes
+     * on the same fault.
      */
     async checkRedirect(blockKey, opName, message) {
-        const nm = this.getNetworkManager();
-        if (!nm)
+        const keyNetwork = this.getKeyNetwork();
+        if (!keyNetwork)
             return null;
-        // Pass the RAW encoded block-key bytes to getCluster. getCluster hashes
-        // internally (hashKey == sha256), so the responsible-set coordinate becomes
-        // hashKey(encode(blockKey)) — identical to how the cluster coordinator
-        // derives it (ClusterCoordinator.getClusterForBlock → findCluster(encode(blockId))).
-        // Pre-hashing here would double-hash (hashKey(sha256(encode(blockKey)))), placing
-        // the cohort at an unrelated ring coordinate and redirecting requests the
-        // coordinator legitimately routed to this peer.
-        const key = new TextEncoder().encode(blockKey);
-        const cluster = await nm.getCluster(key);
-        message.cluster = cluster.map((p) => p.toString?.() ?? String(p));
-        const selfId = this.getSelfId();
+        let cluster;
+        try {
+            cluster = await this.responsiblePeerIds(keyNetwork, blockKey);
+        }
+        catch (err) {
+            this.log.error('redirect lookup failed op=%s blockKey=%s - %e', opName, blockKey, err);
+            if (opName === 'get')
+                return null;
+            throw err;
+        }
+        ;
+        message.cluster = [...cluster];
+        const selfId = this.getSelfId()?.toString();
         if (!selfId)
             return null;
-        const isMember = cluster.some((p) => peersEqual(p, selfId));
+        const isMember = cluster.includes(selfId);
         const smallMesh = cluster.length < this.responsibilityK;
         if (!smallMesh && !isMember) {
-            const peers = cluster.filter((p) => !peersEqual(p, selfId));
             this.log('redirect op=%s blockKey=%s cluster=%d', opName, blockKey, cluster.length);
-            return encodePeers(await Promise.all(peers.map(async (pid) => ({
-                id: pid.toString(),
-                addrs: await this.getPeerAddrs(pid)
+            return encodePeers(await Promise.all(cluster.map(async (id) => ({
+                id,
+                addrs: await this.getPeerAddrs(peerIdFromString(id))
             }))));
         }
         return null;
     }
+    /** The block's responsible peer ids, memoized per block key for {@link RESPONSIBILITY_TTL_MS}. A throw is never memoized. */
+    async responsiblePeerIds(keyNetwork, blockKey) {
+        const now = Date.now();
+        const memo = this.responsibleIds.get(blockKey);
+        if (memo && memo.expires > now)
+            return memo.peerIds;
+        const peerIds = Object.keys(await keyNetwork.findCluster(routingKeyForBlock(blockKey)));
+        this.responsibleIds.set(blockKey, { peerIds, expires: now + RESPONSIBILITY_TTL_MS });
+        return peerIds;
+    }
     /**
      * Handle incoming streams on the repo protocol
      */

package/dist/src/repo/service.js.map

@@ -1 +1 @@
-{"version":3,"file":"service.js","sourceRoot":"","sources":["../../../src/repo/service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,MAAM,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAG3E,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAA;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,WAAW,EAAwB,MAAM,eAAe,CAAA;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAA;AAE/D,OAAO,EAAE,YAAY,EAAe,MAAM,cAAc,CAAA;AACxD,OAAO,EAAE,uBAAuB,EAA+C,MAAM,yBAAyB,CAAA;AAC9G,OAAO,EAAE,gCAAgC,EAAwE,MAAM,6BAA6B,CAAA;AACpJ,OAAO,EAAE,uBAAuB,EAAE,MAAM,yCAAyC,CAAA;AA0DjF,MAAM,UAAU,WAAW,CAAC,OAAwB,EAAE;IACrD,OAAO,CAAC,UAAiC,EAAE,EAAE,CAAC,IAAI,WAAW,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;AACjF,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,WAAW;IACN,QAAQ,CAAQ;IAChB,iBAAiB,CAAQ;IACzB,kBAAkB,CAAQ;IAC1B,GAAG,CAAQ;IACX,IAAI,CAAO;IACX,UAAU,CAAuB;IAC1C,OAAO,CAAS;IACxB,2FAA2F;IAC1E,eAAe,CAAQ;IACxC;;;;;;OAMG;IACK,SAAS,CAAoB;IACrC,2FAA2F;IAC1E,aAAa,CAAwC;IACtE;;;;OAIG;IACc,UAAU,CAAY;IAEvC,YAAY,UAAiC,EAAE,OAAwB,EAAE;QACxE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,SAAS,CAAC,GAAG,aAAa,CAAA;QACpF,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAA;QACxB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,IAAI,EAAE,CAAA;QACrD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAA;QACvD,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,SAAS,IAAI,cAAc,CAAC,CAAA;QACzD,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,mBAAmB,EAAE,UAAU,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;QAClF,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAA;QAC3B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAA;QACpB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;QAChD,IAAI,CAAC,aAAa,GAAG,gCAAgC,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA;IAC3H,CAAC;IAEQ,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,sBAAsB,CAAA;IAEtD;;;OAGG;IACH,SAAS,CAAC,MAAc;QACvB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAA;IACxB,CAAC;IAED,gGAAgG;IACxF,SAAS;QAChB,OAAO,IAAI,CAAC,SAAS,IAAK,IAAI,CAAC,UAAkB,CAAC,MAAM,CAAA;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAM;QACP,CAAC;QAED,MAAM,uBAAuB,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAC7G,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;SAC3C,CAAC,CAAA;QAEF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACT,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACnB,OAAM;QACP,CAAC;QAED,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACvD,IAAI,CAAC,OAAO,GAAG,KAAK,CAAA;IACrB,CAAC;IAEO,iBAAiB;QACxB,IAAI,IAAI,CAAC,UAAU,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC,cAAc,CAAA;QACzE,OAAQ,IAAI,CAAC,SAAS,EAAU,EAAE,QAAQ,EAAE,cAAgD,CAAA;IAC7F,CAAC;IAEO,SAAS;QAChB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAA;QACzD,OAAO,IAAI,CAAC,SAAS,EAAE,EAAE,MAA4B,CAAA;IACtD,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,YAAY,CAAC,MAAc;QACxC,IAAI,IAAI,CAAC,UAAU,CAAC,kBAAkB;YAAE,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;QAC/F,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAS,CAAA;QACtC,IAAI,CAAC,MAAM,EAAE,cAAc;YAAE,OAAO,EAAE,CAAA;QACtC,MAAM,KAAK,GAA4B,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;QAC1E,OAAO,MAAM,uBAAuB,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;IAC7E,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,cAAc,CAAC,SAA4C;QAC1D,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;YACxB,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;QAC9D,CAAC;QACD,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YACzB,OAAO,EAAE,QAAQ,EAAE,qBAAqB,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;QACzF,CAAC;QACD,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;QAC9E,CAAC;QACD,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;QACpE,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;IAClD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,CAAC,QAAgB,EAAE,MAAc,EAAE,OAAoB;QACzE,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAA;QACnC,IAAI,CAAC,EAAE;YAAE,OAAO,IAAI,CAAA;QAEpB,wEAAwE;QACxE,4EAA4E;QAC5E,uEAAuE;QACvE,qFAAqF;QACrF,kFAAkF;QAClF,0EAA0E;QAC1E,gDAAgD;QAChD,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAC9C,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CACvC;QAAC,OAAe,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAEnF,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAC/B,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAA;QAExB,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAA;QACnE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,CAAA;QAEvD,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAA;YACnE,IAAI,CAAC,GAAG,CAAC,uCAAuC,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;YACnF,OAAO,WAAW,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAW,EAAE,EAAE,CAAC,CAAC;gBACtE,EAAE,EAAE,GAAG,CAAC,QAAQ,EAAE;gBAClB,KAAK,EAAE,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC;aACnC,CAAC,CAAC,CAAC,CAAC,CAAA;QACN,CAAC;QAED,OAAO,IAAI,CAAA;IACZ,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,MAAc,EAAE,UAAuB;QACnE,MAAM,MAAM,GAAG,UAAU,EAAE,UAAU,CAAA;QAErC,MAAM,aAAa,GAAG,KAAK,SAAS,CAAC,EAAqB,MAAqC;YAC9F,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBAChC,qBAAqB;gBACrB,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;gBACxD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAgB,CAAA;gBAElD,6EAA6E;gBAC7E,6EAA6E;gBAC7E,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;gBACvC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAA;gBAC3D,MAAM,QAAQ,GAAG,QAAQ,KAAK,SAAS;oBACtC,CAAC,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC;oBACrD,CAAC,CAAC,IAAI,CAAA;gBAEP,IAAI,QAAa,CAAA;gBACjB,IAAI,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,QAAQ,CAAA;gBACpB,CAAC;qBAAM,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;oBAC/B,kFAAkF;oBAClF,iFAAiF;oBACjF,2EAA2E;oBAC3E,+EAA+E;oBAC/E,+DAA+D;oBAC/D,kFAAkF;oBAClF,kFAAkF;oBAClF,gFAAgF;oBAChF,wDAAwD;oBACxD,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;gBAClF,CAAC;qBAAM,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;oBAChC,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;gBACpF,CAAC;qBAAM,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;oBAClC,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;gBAClG,CAAC;qBAAM,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;oBAClC,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;gBACxF,CAAC;gBAED,gCAAgC;gBAChC,MAAM,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAA;gBACxD,0EAA0E;gBAC1E,0EAA0E;gBAC1E,4EAA4E;gBAC5E,0CAA0C;gBAC1C,OAAM;YACP,CAAC;QACF,CAAC,CAAA;QAED,KAAK,CAAC,KAAK,IAAI,EAAE;YAChB,IAAI,CAAC;gBACJ,iFAAiF;gBACjF,8DAA8D;gBAC9D,IAAI,IAAI,CAAC,aAAa,IAAI,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;oBAAE,OAAM;gBAC3F,MAAM,SAAS,GAAG,IAAI,CACrB,MAAM,EACN,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,aAAa,EAAE,uBAAuB,EAAE,CAAC,EACxE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EACxB,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAC5B,CAAA;gBACD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBACnB,CAAC;gBACD,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YACrB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mDAAmD,EAAE,MAAM,EAAE,GAAG,CAAC,CAAA;gBAChF,MAAM,CAAC,KAAK,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAClE,CAAC;QACF,CAAC,CAAC,EAAE,CAAA;IACL,CAAC;CACD"}
+{"version":3,"file":"service.js","sourceRoot":"","sources":["../../../src/repo/service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,MAAM,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAG3E,OAAO,EAAE,MAAM,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACvF,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAClD,OAAO,EAAE,WAAW,EAAwB,MAAM,eAAe,CAAA;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAA;AAE/D,OAAO,EAAE,YAAY,EAAe,MAAM,cAAc,CAAA;AACxD,OAAO,EAAE,uBAAuB,EAA+C,MAAM,yBAAyB,CAAA;AAC9G,OAAO,EAAE,gCAAgC,EAAwE,MAAM,6BAA6B,CAAA;AACpJ,OAAO,EAAE,uBAAuB,EAAE,MAAM,yCAAyC,CAAA;AAEjF,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAA;AA4D3D,MAAM,UAAU,WAAW,CAAC,OAAwB,EAAE;IACrD,OAAO,CAAC,UAAiC,EAAE,EAAE,CAAC,IAAI,WAAW,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;AACjF,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,WAAW;IACN,QAAQ,CAAQ;IAChB,iBAAiB,CAAQ;IACzB,kBAAkB,CAAQ;IAC1B,GAAG,CAAQ;IACX,IAAI,CAAO;IACX,UAAU,CAAuB;IAC1C,OAAO,CAAS;IACxB,2FAA2F;IAC1E,eAAe,CAAQ;IACxC;;;;;;OAMG;IACK,SAAS,CAAoB;IACrC,2FAA2F;IAC1E,aAAa,CAAwC;IACtE;;;;OAIG;IACc,UAAU,CAAY;IACvC;;;;;;;;OAQG;IACc,cAAc,GAAG,IAAI,MAAM,CAA0D,IAAI,CAAC,CAAA;IAE3G,YAAY,UAAiC,EAAE,OAAwB,EAAE;QACxE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,SAAS,CAAC,GAAG,aAAa,CAAA;QACpF,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAA;QACxB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,IAAI,EAAE,CAAA;QACrD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAA;QACvD,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,SAAS,IAAI,cAAc,CAAC,CAAA;QACzD,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,mBAAmB,EAAE,UAAU,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;QAClF,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAA;QAC3B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAA;QACpB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;QAChD,IAAI,CAAC,aAAa,GAAG,gCAAgC,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA;IAC3H,CAAC;IAEQ,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,sBAAsB,CAAA;IAEtD;;;OAGG;IACH,SAAS,CAAC,MAAc;QACvB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAA;IACxB,CAAC;IAED,gGAAgG;IACxF,SAAS;QAChB,OAAO,IAAI,CAAC,SAAS,IAAK,IAAI,CAAC,UAAkB,CAAC,MAAM,CAAA;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAM;QACP,CAAC;QAED,MAAM,uBAAuB,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAC7G,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;SAC3C,CAAC,CAAA;QAEF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACT,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACnB,OAAM;QACP,CAAC;QAED,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACvD,IAAI,CAAC,OAAO,GAAG,KAAK,CAAA;IACrB,CAAC;IAED;;;;OAIG;IACK,aAAa;QACpB,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC,UAAU,CAAA;QACjE,OAAQ,IAAI,CAAC,SAAS,EAAqD,EAAE,UAAU,CAAA;IACxF,CAAC;IAEO,SAAS;QAChB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAA;QACzD,OAAO,IAAI,CAAC,SAAS,EAAE,EAAE,MAA4B,CAAA;IACtD,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,YAAY,CAAC,MAAc;QACxC,IAAI,IAAI,CAAC,UAAU,CAAC,kBAAkB;YAAE,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;QAC/F,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAS,CAAA;QACtC,IAAI,CAAC,MAAM,EAAE,cAAc;YAAE,OAAO,EAAE,CAAA;QACtC,MAAM,KAAK,GAA4B,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;QAC1E,OAAO,MAAM,uBAAuB,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;IAC7E,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,cAAc,CAAC,SAA4C;QAC1D,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;YACxB,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;QAC9D,CAAC;QACD,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YACzB,OAAO,EAAE,QAAQ,EAAE,qBAAqB,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;QACzF,CAAC;QACD,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;QAC9E,CAAC;QACD,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;QACpE,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;IAClD,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,aAAa,CAAC,QAAgB,EAAE,MAAc,EAAE,OAAoB;QACzE,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;QACvC,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAA;QAE5B,IAAI,OAA0B,CAAA;QAC9B,IAAI,CAAC;YACJ,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QAC9D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,+CAA+C,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAA;YACtF,IAAI,MAAM,KAAK,KAAK;gBAAE,OAAO,IAAI,CAAA;YACjC,MAAM,GAAG,CAAA;QACV,CAAC;QACD,CAAC;QAAC,OAAgD,CAAC,OAAO,GAAG,CAAC,GAAG,OAAO,CAAC,CAAA;QAEzE,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,CAAA;QAC3C,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAA;QAExB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;QACzC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,CAAA;QAEvD,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC,GAAG,CAAC,uCAAuC,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;YACnF,OAAO,WAAW,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAC,EAAE,EAAC,EAAE,CAAC,CAAC;gBAC7D,EAAE;gBACF,KAAK,EAAE,MAAM,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;aACpD,CAAC,CAAC,CAAC,CAAC,CAAA;QACN,CAAC;QAED,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,6HAA6H;IACrH,KAAK,CAAC,kBAAkB,CAAC,UAAyB,EAAE,QAAgB;QAC3E,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC9C,IAAI,IAAI,IAAI,IAAI,CAAC,OAAO,GAAG,GAAG;YAAE,OAAO,IAAI,CAAC,OAAO,CAAA;QACnD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,WAAW,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;QACvF,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,GAAG,qBAAqB,EAAE,CAAC,CAAA;QACpF,OAAO,OAAO,CAAA;IACf,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,MAAc,EAAE,UAAuB;QACnE,MAAM,MAAM,GAAG,UAAU,EAAE,UAAU,CAAA;QAErC,MAAM,aAAa,GAAG,KAAK,SAAS,CAAC,EAAqB,MAAqC;YAC9F,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBAChC,qBAAqB;gBACrB,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;gBACxD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAgB,CAAA;gBAElD,6EAA6E;gBAC7E,6EAA6E;gBAC7E,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;gBACvC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAA;gBAC3D,MAAM,QAAQ,GAAG,QAAQ,KAAK,SAAS;oBACtC,CAAC,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC;oBACrD,CAAC,CAAC,IAAI,CAAA;gBAEP,IAAI,QAAa,CAAA;gBACjB,IAAI,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,QAAQ,CAAA;gBACpB,CAAC;qBAAM,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;oBAC/B,kFAAkF;oBAClF,iFAAiF;oBACjF,2EAA2E;oBAC3E,+EAA+E;oBAC/E,+DAA+D;oBAC/D,kFAAkF;oBAClF,kFAAkF;oBAClF,gFAAgF;oBAChF,wDAAwD;oBACxD,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;gBAClF,CAAC;qBAAM,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;oBAChC,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;gBACpF,CAAC;qBAAM,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;oBAClC,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;gBAClG,CAAC;qBAAM,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;oBAClC,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;gBACxF,CAAC;gBAED,gCAAgC;gBAChC,MAAM,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAA;gBACxD,0EAA0E;gBAC1E,0EAA0E;gBAC1E,4EAA4E;gBAC5E,0CAA0C;gBAC1C,OAAM;YACP,CAAC;QACF,CAAC,CAAA;QAED,KAAK,CAAC,KAAK,IAAI,EAAE;YAChB,IAAI,CAAC;gBACJ,iFAAiF;gBACjF,8DAA8D;gBAC9D,IAAI,IAAI,CAAC,aAAa,IAAI,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;oBAAE,OAAM;gBAC3F,MAAM,SAAS,GAAG,IAAI,CACrB,MAAM,EACN,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,aAAa,EAAE,uBAAuB,EAAE,CAAC,EACxE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EACxB,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAC5B,CAAA;gBACD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBACnB,CAAC;gBACD,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YACrB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mDAAmD,EAAE,MAAM,EAAE,GAAG,CAAC,CAAA;gBAChF,MAAM,CAAC,KAAK,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAClE,CAAC;QACF,CAAC,CAAC,EAAE,CAAA;IACL,CAAC;CACD"}

package/dist/src/repo/stuck-reservation.d.ts

@@ -0,0 +1,127 @@
+import { type ActionId, type BlockId } from "@optimystic/db-core";
+/**
+ * How many DISTINCT later actions one unchanged holder must refuse on a block before the refusals are
+ * named as a stuck reservation rather than as an ordinary lost race.
+ *
+ * **Why a count of distinct actions and not something else.** Elapsed time answers the wrong question
+ * — a slow writer is not a stuck one, and a holder legitimately keeps its reservation for as long as
+ * its own commit takes. A raw refusal count answers the wrong question too: a single writer retrying
+ * a lost race produces a run of refusals under ONE action id (see {@link StuckReservationWatch.refused}).
+ * What no healthy holder can produce is an unbounded stream of *different* writers all losing to it,
+ * because a healthy holder's reservation lasts one pend-to-commit window.
+ *
+ * **Why 8.** The bound to clear is how many distinct actions can honestly be refused inside one such
+ * window. Measured on the in-process mesh, in the healthy-contention arm of
+ * `test/stuck-reservation-named.spec.ts`: a holder that pends, is raced by other writers, and then
+ * commits refuses **2** distinct actions per episode — the two rivals — and the count resets on every
+ * holder change. `concurrent-diary-append-acknowledgement.spec.ts` races three writers at one diary
+ * and cannot exceed that either, for the same reason: at most (writers - 1) rivals can lose to one
+ * winner. 8 is four times the measured healthy figure, and it is a floor a genuinely stuck block
+ * clears trivially (the field instance refused hundreds).
+ *
+ * **The bound stated exactly.** It is distinct SYNC CYCLES, not distinct writers: one writer that
+ * exhausts a sync's retry budget and is re-driven by its caller mints a fresh id for the next cycle,
+ * so it can contribute more than one. That does not widen the window much — a cycle only ends in
+ * exhaustion after `DefaultMaxAttempts` (10) attempts of backoff, roughly 21s (see the exhaustion
+ * NOTE in `Collection.syncAttempts`), so a lone writer needs a holder to keep the block for upwards
+ * of two and a half minutes before it reaches 8 by itself, which is not a healthy holder.
+ *
+ * **What the margin does NOT cover, stated honestly.** A block with more than 8 distinct writers
+ * racing it inside a single pend-to-commit round trip could reach 8 with a perfectly healthy holder.
+ * That is a diagnostic false positive on a log line and nothing else — this counter never refuses,
+ * expires, or deletes anything (see {@link StuckReservationTracker}) — and the remedy if
+ * a deployment ever hits it is to raise this number, not to add a control path. Raising it costs
+ * detection latency on low-traffic blocks, which need this many distinct write ATTEMPTS before the
+ * condition can be named at all.
+ */
+export declare const STUCK_RESERVATION_DISTINCT_ACTIONS = 8;
+/**
+ * The stuck-reservation wording: written for an operator reading logs, in the same register as
+ * `cohortTooSmallMessage` and `soleHolderMessage` in `coordinator-repo.ts` — what is stuck, what will
+ * and will not clear it, and what to do next.
+ *
+ * The claim is deliberately about the RESERVATION, not about the writer's intent: this node cannot
+ * see whether the holding process is alive, only that it has held the block across enough unrelated
+ * later actions that no retry is going to win. So the line says what is provable (the block accepts
+ * no writes while this record stands, and nothing on the node removes it) and points at the one check
+ * that settles the rest.
+ */
+export declare function stuckReservationMessage(holders: readonly ActionId[], refusedActions: number): string;
+/**
+ * One episode that just crossed the threshold — what the caller logs, verbatim, under its own tag.
+ * The ids an operator needs to grep for and to cancel are kept as data beside the prose so a log
+ * search finds the block and the action without parsing English.
+ */
+export interface StuckReservationEpisode {
+    blockId: BlockId;
+    holdingActionIds: readonly ActionId[];
+    distinctRefusedActions: number;
+    message: string;
+}
+/**
+ * Count pending-conflict refusals against the holder(s) of each block they name, and say ONCE — in
+ * words, at the moment it becomes provable — when a block is wedged behind a reservation that is not
+ * going to clear.
+ *
+ * **Why this needs saying at all.** Every individual refusal is indistinguishable from an ordinary
+ * lost race, which is a normal and healthy event, so the logs of a permanently wedged block read
+ * exactly like the logs of a busy one. Finding the difference otherwise means noticing that the SAME
+ * rival action id keeps appearing across unrelated writers for as long as the process lives — a
+ * pattern nothing points at, and one that cost a downstream project several tickets and weeks to
+ * re-derive from raw traces. The node has the fact in hand at every refusal; this makes it sayable.
+ *
+ * **The signal, and the two things that are NOT the signal.** The discriminator is repetition
+ * against an unchanged holder — see {@link STUCK_RESERVATION_DISTINCT_ACTIONS} for why distinct
+ * refused actions is the right counter and for the measured threshold. Two cheaper-looking tests
+ * were tried and do not work: the members' in-memory reservation table
+ * (`ClusterMember.activeTransactions`) clears the moment a rival's pend reaches consensus, so a
+ * perfectly healthy rival inside its pend-to-commit window is absent from it too and absence there
+ * says nothing; and "the block already passed this pending record's revision" catches a different
+ * orphan class entirely — in the verified instance the wedged block sat at revision 1 while the
+ * orphaned record was for revision 2, still nominally promotable. (That second class is now handled
+ * at its own site — a record the incoming writer has built on no longer refuses at all, see
+ * `isReservationAgainst` — so what reaches this counter is a holder whose change the block has not
+ * taken: one whose slot every later writer still wants, or one past the base every later writer
+ * declares, such as an abandoned action's record on a block no later write touched.)
+ *
+ * **Never a control path.** This classifies and reports; it never refuses, expires, or deletes
+ * anything. Deciding when a durable pending record may be removed is precisely the hard problem
+ * backlog `debt-unpromotable-pending-records-need-a-sweep` exists for — deleting a live reservation
+ * is worse than the leak — and a counter accurate enough for a log line is not evidence enough to
+ * destroy state.
+ *
+ * **Two instances, two vantage points.** The coordinator keeps one (`CoordinatorRepo.noteStuckReservation`),
+ * fed by cohort-wide `held`-answered refusals whose holder its OWN storage corroborates, and every
+ * member keeps one (`ClusterMember.validatePendOperations`), fed by its own `held` votes — so a
+ * reservation only remote members hold is named by those members, where the record actually lives,
+ * rather than going unnamed at a coordinator that cannot see it.
+ *
+ * LRU-bounded: an eviction under more than `capacity` conflicted blocks loses an episode's say-once
+ * flag, so the line can repeat once for that block — bounded duplication, far cheaper than the
+ * unbounded silence it replaces.
+ */
+export declare class StuckReservationTracker {
+    private readonly watches;
+    constructor(capacity?: number);
+    /**
+     * Count one refusal of `refusedActionId` against the rival holder(s) of each block in `rivalsByBlock`.
+     * Returns the highest distinct-refusal count any of these blocks has now reached — for a
+     * classification line to carry, saturating at the threshold once an episode has been reported,
+     * since the ids are dropped at that point — and every episode that crossed the threshold on this
+     * refusal, which the caller must log.
+     */
+    note(rivalsByBlock: ReadonlyMap<BlockId, readonly ActionId[]>, refusedActionId: ActionId): {
+        highest: number;
+        named: StuckReservationEpisode[];
+    };
+    /**
+     * Forget any episode recorded for these blocks, optionally only when `holderActionId` is one of
+     * the actions that episode named. Called from the events the message names as the cures — a write
+     * the block accepted, a cancel for the holding action — and it is LRU hygiene rather than
+     * behaviour: a later wedge is named by a DIFFERENT holder, which {@link note}'s holder comparison
+     * already treats as a new episode whether the old entry is still there or not. Kept because a
+     * settled episode holding an LRU slot can only evict a live one.
+     */
+    forget(blockIds: readonly BlockId[], holderActionId?: ActionId): void;
+}
+//# sourceMappingURL=stuck-reservation.d.ts.map

package/dist/src/repo/stuck-reservation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stuck-reservation.d.ts","sourceRoot":"","sources":["../../../src/repo/stuck-reservation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAU,KAAK,QAAQ,EAAE,KAAK,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAgC1E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,eAAO,MAAM,kCAAkC,IAAI,CAAC;AAUpD;;;;;;;;;;GAUG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,SAAS,QAAQ,EAAE,EAAE,cAAc,EAAE,MAAM,GAAG,MAAM,CAiBpG;AAED;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACvC,OAAO,EAAE,OAAO,CAAC;IACjB,gBAAgB,EAAE,SAAS,QAAQ,EAAE,CAAC;IACtC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,OAAO,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,qBAAa,uBAAuB;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;gBAEpD,QAAQ,SAAO;IAI3B;;;;;;OAMG;IACH,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,OAAO,EAAE,SAAS,QAAQ,EAAE,CAAC,EAAE,eAAe,EAAE,QAAQ,GAAG;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,uBAAuB,EAAE,CAAA;KAAE;IAkChJ;;;;;;;OAOG;IACH,MAAM,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,EAAE,cAAc,CAAC,EAAE,QAAQ,GAAG,IAAI;CAMrE"}

package/dist/src/repo/stuck-reservation.js

@@ -0,0 +1,179 @@
+import { LruMap } from "@optimystic/db-core";
+/**
+ * How many DISTINCT later actions one unchanged holder must refuse on a block before the refusals are
+ * named as a stuck reservation rather than as an ordinary lost race.
+ *
+ * **Why a count of distinct actions and not something else.** Elapsed time answers the wrong question
+ * — a slow writer is not a stuck one, and a holder legitimately keeps its reservation for as long as
+ * its own commit takes. A raw refusal count answers the wrong question too: a single writer retrying
+ * a lost race produces a run of refusals under ONE action id (see {@link StuckReservationWatch.refused}).
+ * What no healthy holder can produce is an unbounded stream of *different* writers all losing to it,
+ * because a healthy holder's reservation lasts one pend-to-commit window.
+ *
+ * **Why 8.** The bound to clear is how many distinct actions can honestly be refused inside one such
+ * window. Measured on the in-process mesh, in the healthy-contention arm of
+ * `test/stuck-reservation-named.spec.ts`: a holder that pends, is raced by other writers, and then
+ * commits refuses **2** distinct actions per episode — the two rivals — and the count resets on every
+ * holder change. `concurrent-diary-append-acknowledgement.spec.ts` races three writers at one diary
+ * and cannot exceed that either, for the same reason: at most (writers - 1) rivals can lose to one
+ * winner. 8 is four times the measured healthy figure, and it is a floor a genuinely stuck block
+ * clears trivially (the field instance refused hundreds).
+ *
+ * **The bound stated exactly.** It is distinct SYNC CYCLES, not distinct writers: one writer that
+ * exhausts a sync's retry budget and is re-driven by its caller mints a fresh id for the next cycle,
+ * so it can contribute more than one. That does not widen the window much — a cycle only ends in
+ * exhaustion after `DefaultMaxAttempts` (10) attempts of backoff, roughly 21s (see the exhaustion
+ * NOTE in `Collection.syncAttempts`), so a lone writer needs a holder to keep the block for upwards
+ * of two and a half minutes before it reaches 8 by itself, which is not a healthy holder.
+ *
+ * **What the margin does NOT cover, stated honestly.** A block with more than 8 distinct writers
+ * racing it inside a single pend-to-commit round trip could reach 8 with a perfectly healthy holder.
+ * That is a diagnostic false positive on a log line and nothing else — this counter never refuses,
+ * expires, or deletes anything (see {@link StuckReservationTracker}) — and the remedy if
+ * a deployment ever hits it is to raise this number, not to add a control path. Raising it costs
+ * detection latency on low-traffic blocks, which need this many distinct write ATTEMPTS before the
+ * condition can be named at all.
+ */
+export const STUCK_RESERVATION_DISTINCT_ACTIONS = 8;
+/** Whether two sorted holder lists name the same reservation — i.e. whether a refusal continues an
+ *  existing episode or starts a new one. Both sides come from the same sort, so this is a plain
+ *  element-wise comparison; a block normally has exactly one holder, since a member's own pend refuses
+ *  a second one (`ClusterMember.validatePendOperations`). */
+function sameHolders(a, b) {
+    return a.length === b.length && a.every((id, i) => id === b[i]);
+}
+/**
+ * The stuck-reservation wording: written for an operator reading logs, in the same register as
+ * `cohortTooSmallMessage` and `soleHolderMessage` in `coordinator-repo.ts` — what is stuck, what will
+ * and will not clear it, and what to do next.
+ *
+ * The claim is deliberately about the RESERVATION, not about the writer's intent: this node cannot
+ * see whether the holding process is alive, only that it has held the block across enough unrelated
+ * later actions that no retry is going to win. So the line says what is provable (the block accepts
+ * no writes while this record stands, and nothing on the node removes it) and points at the one check
+ * that settles the rest.
+ */
+export function stuckReservationMessage(holders, refusedActions) {
+    const held = holders.join(', ');
+    return `This block is WEDGED BEHIND A PENDING WRITE THAT IS NOT COMPLETING, and retrying will never ` +
+        `clear it: action(s) ${held} reserved the block and have now refused ${refusedActions} DISTINCT, ` +
+        `unrelated later actions. Each of those refusals on its own looks exactly like an ordinary ` +
+        `optimistic-concurrency loss, which is normal and healthy — the repetition is what is not. A ` +
+        `healthy rival holds a block only for its own pend-to-commit window and then releases it by ` +
+        `committing or cancelling; a reservation that keeps refusing NEW writers is holding the block ` +
+        `against every writer on every machine, and each of them loses again identically. EXACTLY TWO ` +
+        `THINGS CLEAR IT: a cancel for action(s) ${held} on this block (route it through the cohort so ` +
+        `every member drops the record), or that same action's own commit landing. Nothing on the node ` +
+        `expires it — there is no sweep for abandoned pending records — so until one of those two happens ` +
+        `the block takes NO writes while continuing to serve reads and to look healthy in every other ` +
+        `respect. The usual cause is a writer that went away between a failed or half-applied commit and ` +
+        `the cancel it owed, so check whether whatever ran ${held} still exists before cancelling on its ` +
+        `behalf. This line is a diagnosis and nothing more: this node does not expire, refuse, or delete ` +
+        `the record on the strength of it.`;
+}
+/**
+ * Count pending-conflict refusals against the holder(s) of each block they name, and say ONCE — in
+ * words, at the moment it becomes provable — when a block is wedged behind a reservation that is not
+ * going to clear.
+ *
+ * **Why this needs saying at all.** Every individual refusal is indistinguishable from an ordinary
+ * lost race, which is a normal and healthy event, so the logs of a permanently wedged block read
+ * exactly like the logs of a busy one. Finding the difference otherwise means noticing that the SAME
+ * rival action id keeps appearing across unrelated writers for as long as the process lives — a
+ * pattern nothing points at, and one that cost a downstream project several tickets and weeks to
+ * re-derive from raw traces. The node has the fact in hand at every refusal; this makes it sayable.
+ *
+ * **The signal, and the two things that are NOT the signal.** The discriminator is repetition
+ * against an unchanged holder — see {@link STUCK_RESERVATION_DISTINCT_ACTIONS} for why distinct
+ * refused actions is the right counter and for the measured threshold. Two cheaper-looking tests
+ * were tried and do not work: the members' in-memory reservation table
+ * (`ClusterMember.activeTransactions`) clears the moment a rival's pend reaches consensus, so a
+ * perfectly healthy rival inside its pend-to-commit window is absent from it too and absence there
+ * says nothing; and "the block already passed this pending record's revision" catches a different
+ * orphan class entirely — in the verified instance the wedged block sat at revision 1 while the
+ * orphaned record was for revision 2, still nominally promotable. (That second class is now handled
+ * at its own site — a record the incoming writer has built on no longer refuses at all, see
+ * `isReservationAgainst` — so what reaches this counter is a holder whose change the block has not
+ * taken: one whose slot every later writer still wants, or one past the base every later writer
+ * declares, such as an abandoned action's record on a block no later write touched.)
+ *
+ * **Never a control path.** This classifies and reports; it never refuses, expires, or deletes
+ * anything. Deciding when a durable pending record may be removed is precisely the hard problem
+ * backlog `debt-unpromotable-pending-records-need-a-sweep` exists for — deleting a live reservation
+ * is worse than the leak — and a counter accurate enough for a log line is not evidence enough to
+ * destroy state.
+ *
+ * **Two instances, two vantage points.** The coordinator keeps one (`CoordinatorRepo.noteStuckReservation`),
+ * fed by cohort-wide `held`-answered refusals whose holder its OWN storage corroborates, and every
+ * member keeps one (`ClusterMember.validatePendOperations`), fed by its own `held` votes — so a
+ * reservation only remote members hold is named by those members, where the record actually lives,
+ * rather than going unnamed at a coordinator that cannot see it.
+ *
+ * LRU-bounded: an eviction under more than `capacity` conflicted blocks loses an episode's say-once
+ * flag, so the line can repeat once for that block — bounded duplication, far cheaper than the
+ * unbounded silence it replaces.
+ */
+export class StuckReservationTracker {
+    watches;
+    constructor(capacity = 1000) {
+        this.watches = new LruMap(capacity);
+    }
+    /**
+     * Count one refusal of `refusedActionId` against the rival holder(s) of each block in `rivalsByBlock`.
+     * Returns the highest distinct-refusal count any of these blocks has now reached — for a
+     * classification line to carry, saturating at the threshold once an episode has been reported,
+     * since the ids are dropped at that point — and every episode that crossed the threshold on this
+     * refusal, which the caller must log.
+     */
+    note(rivalsByBlock, refusedActionId) {
+        let highest = 0;
+        const named = [];
+        for (const [blockId, rivals] of rivalsByBlock) {
+            const holders = [...new Set(rivals)].sort();
+            const prior = this.watches.get(blockId);
+            // A different holder set is a DIFFERENT episode — the block changed hands, which is the
+            // healthy cycle — so the count starts over and the new holder gets its own chance to speak.
+            const watch = prior !== undefined && sameHolders(prior.holders, holders)
+                ? prior
+                : { holders, refused: new Set(), reported: false };
+            if (watch !== prior)
+                this.watches.set(blockId, watch);
+            if (watch.reported) {
+                highest = Math.max(highest, STUCK_RESERVATION_DISTINCT_ACTIONS);
+                continue;
+            }
+            watch.refused.add(refusedActionId);
+            highest = Math.max(highest, watch.refused.size);
+            if (watch.refused.size < STUCK_RESERVATION_DISTINCT_ACTIONS)
+                continue;
+            named.push({
+                blockId,
+                holdingActionIds: holders,
+                distinctRefusedActions: watch.refused.size,
+                message: stuckReservationMessage(holders, watch.refused.size)
+            });
+            watch.reported = true;
+            // Said once per episode: from here the flag alone suppresses, and the ids have done their
+            // work (their count is in the line above), so drop them rather than growing a set for the
+            // unbounded remainder of a permanent condition.
+            watch.refused.clear();
+        }
+        return { highest, named };
+    }
+    /**
+     * Forget any episode recorded for these blocks, optionally only when `holderActionId` is one of
+     * the actions that episode named. Called from the events the message names as the cures — a write
+     * the block accepted, a cancel for the holding action — and it is LRU hygiene rather than
+     * behaviour: a later wedge is named by a DIFFERENT holder, which {@link note}'s holder comparison
+     * already treats as a new episode whether the old entry is still there or not. Kept because a
+     * settled episode holding an LRU slot can only evict a live one.
+     */
+    forget(blockIds, holderActionId) {
+        for (const blockId of blockIds) {
+            if (holderActionId !== undefined && !this.watches.peek(blockId)?.holders.includes(holderActionId))
+                continue;
+            this.watches.delete(blockId);
+        }
+    }
+}
+//# sourceMappingURL=stuck-reservation.js.map

package/dist/src/repo/stuck-reservation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stuck-reservation.js","sourceRoot":"","sources":["../../../src/repo/stuck-reservation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAA+B,MAAM,qBAAqB,CAAC;AAgC1E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAC;AAEpD;;;6DAG6D;AAC7D,SAAS,WAAW,CAAC,CAAsB,EAAE,CAAsB;IAClE,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAA4B,EAAE,cAAsB;IAC3F,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,OAAO,8FAA8F;QACpG,uBAAuB,IAAI,4CAA4C,cAAc,aAAa;QAClG,4FAA4F;QAC5F,8FAA8F;QAC9F,6FAA6F;QAC7F,+FAA+F;QAC/F,+FAA+F;QAC/F,2CAA2C,IAAI,iDAAiD;QAChG,gGAAgG;QAChG,mGAAmG;QACnG,+FAA+F;QAC/F,kGAAkG;QAClG,qDAAqD,IAAI,yCAAyC;QAClG,kGAAkG;QAClG,mCAAmC,CAAC;AACtC,CAAC;AAcD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,MAAM,OAAO,uBAAuB;IAClB,OAAO,CAAwC;IAEhE,YAAY,QAAQ,GAAG,IAAI;QAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,MAAM,CAAgC,QAAQ,CAAC,CAAC;IACpE,CAAC;IAED;;;;;;OAMG;IACH,IAAI,CAAC,aAAwD,EAAE,eAAyB;QACvF,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,MAAM,KAAK,GAA8B,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACxC,wFAAwF;YACxF,4FAA4F;YAC5F,MAAM,KAAK,GAA0B,KAAK,KAAK,SAAS,IAAI,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC;gBAC9F,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,GAAG,EAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAC9D,IAAI,KAAK,KAAK,KAAK;gBAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtD,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACpB,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,kCAAkC,CAAC,CAAC;gBAChE,SAAS;YACV,CAAC;YACD,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;YACnC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAChD,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,GAAG,kCAAkC;gBAAE,SAAS;YACtE,KAAK,CAAC,IAAI,CAAC;gBACV,OAAO;gBACP,gBAAgB,EAAE,OAAO;gBACzB,sBAAsB,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI;gBAC1C,OAAO,EAAE,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;aAC7D,CAAC,CAAC;YACH,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;YACtB,0FAA0F;YAC1F,0FAA0F;YAC1F,gDAAgD;YAChD,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,QAA4B,EAAE,cAAyB;QAC7D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,IAAI,cAAc,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;gBAAE,SAAS;YAC5G,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;IACF,CAAC;CACD"}