@kumiai/hub-server 0.12.0 → 0.12.2

package/README.md

@@ -13,6 +13,7 @@ client registry for push fan-out, and an in-memory `HubStore` for development an
   development; it is also what runs the `@kumiai/hub-conformance` suites in this repo.
 - `HubClientRegistry` — the live-connection table.
 - `createRateLimiter`, `DEFAULT_RATE_LIMITS`, `DEFAULT_KEYPACKAGE_FETCH_LIMITS`,
+  `DEFAULT_RECEIVE_WINDOW`, `DEFAULT_RECEIVE_WINDOW_BYTES`, `DEFAULT_RECEIVE_ACK_TIMEOUT`,
   `DEFAULT_HUB_ACCESS_RULES`.
 - `HubStoreErrorEvent`, `HubStoreErrorHook` — the `onStoreError` event and hook types.
 
@@ -74,6 +75,34 @@ than assigned, because `logPosition: undefined` becomes a present key with a fal
 off the wire — and the entire point of the field is that a reader can tell "no place in any log" from
 a place.
 
+## The receive channel is paced by acks
+
+`hub/v1/receive` writes a frame only while the frames in flight on that channel stay under a window:
+`receiveWindow` frames (`DEFAULT_RECEIVE_WINDOW`, 256) and `receiveWindowBytes` bytes
+(`DEFAULT_RECEIVE_WINDOW_BYTES`, 512 KiB, counting the base64 payload plus an envelope estimate; one
+frame always goes through when nothing else is in flight). The backlog drain and live pushes share
+the window, and acks are read from the moment the channel opens, so the drain advances as the client
+acks it.
+
+The transport cannot do this pacing: `writer.ready` on a channel says nothing about the HTTP
+bridge's per-session buffer, which `@enkaku/http-serve` drops whole once it passes
+`maxSessionBufferBytes`. Unpaced, a backlog larger than that buffer, read by a slow client, ended
+the session on every connect before any ack reached the hub, and the next connect redelivered the
+same backlog. Keep the byte window well below the transport's session buffer.
+
+The channel is ordered, so an ack for a frame proves the client has read every frame written before
+it: for the window, an ack releases that frame and all earlier ones. The store still frees only the
+deliveries actually acked. A client can leave frames unacked for good — a topic it subscribed to but
+reads with `hub/v1/topic/fetch`, or one no consumer on the device claims — and those frames must not
+hold the window, or they would come back first on every connect and stall it.
+
+A full window that sees no ack at all for `receiveAckTimeout` ms (`DEFAULT_RECEIVE_ACK_TIMEOUT`,
+15 s; each ack restarts it) is released whole and the drain goes on; the channel is not ended,
+because a window made only of never-acked frames would come back first on the next connect too. A
+client that has stopped reading cannot grow a buffer without bound: the HTTP bridge drops its
+session, and a transport with real backpressure stalls the write until `receiveBufferLimit` tears
+the channel down.
+
 ## Retention: the store refuses, the hub schedules
 
 Two different knobs, easily confused. `createHub`'s `purge.olderThan` (default 7 days) is the **age

package/lib/handlers.d.ts

@@ -166,6 +166,34 @@ export declare const DEFAULT_RECEIVE_BUFFER_LIMIT = 256;
  * maximum window a removed member can keep draining a topic. See the receive handler's `gate`.
  */
 export declare const DEFAULT_RECEIVE_AUTH_CACHE_TTL = 5000;
+/**
+ * Max frames in flight on a receive channel: written, and not yet followed by an ack for that frame
+ * or a later one. The hub writes the next frame only once acks bring the count back under this
+ * window.
+ *
+ * Without it the drain writes a whole backlog at once, and a transport that buffers per session
+ * (the SSE bridge drops a session whose unread buffer passes `maxSessionBufferBytes`) ends the
+ * session before the slow client's acks reach the hub: every connect then redelivers the same
+ * backlog. The window must stay well above the client's ack batch (10 for Kubun's device client)
+ * so batched acks keep the pipe full.
+ */
+export declare const DEFAULT_RECEIVE_WINDOW = 256;
+/**
+ * Max bytes (base64 payload plus an estimate of the frame's envelope) in flight on a receive
+ * channel. Kept well below the HTTP transport's per-session buffer (1 MiB by default in `@enkaku/http-serve`, 8 MiB
+ * on Kubun's relay) so a window of large frames cannot overflow it. A single frame is always let
+ * through when nothing else is in flight, whatever its size.
+ */
+export declare const DEFAULT_RECEIVE_WINDOW_BYTES: number;
+/**
+ * How long (ms) a receive channel with a full window waits without any ack before it releases the
+ * whole window and writes on. Each ack restarts the wait. The channel is NOT ended: a window full
+ * of frames the client will never ack (topics no sink on the device claims) would come back first
+ * on every reconnect and livelock it. Long enough for a busy phone to open the first frames of a
+ * window and flush an ack batch (10 frames or 500 ms on Kubun's device client); short, because it
+ * is the delay a window of never-acked frames costs per connect.
+ */
+export declare const DEFAULT_RECEIVE_ACK_TIMEOUT = 15000;
 export type CreateHandlersParams = {
     registry: HubClientRegistry;
     store: HubStore;
@@ -175,6 +203,19 @@ export type CreateHandlersParams = {
     /** Max frames queued-but-unflushed on a receive channel. See {@link DEFAULT_RECEIVE_BUFFER_LIMIT}
      * for the >= 50-frame-page floor this should respect. Default: 256 */
     receiveBufferLimit?: number;
+    /** Max frames in flight on a receive channel. Default: {@link DEFAULT_RECEIVE_WINDOW} */
+    receiveWindow?: number;
+    /**
+     * Max bytes (payload plus envelope estimate) in flight on a receive channel. Default:
+     * {@link DEFAULT_RECEIVE_WINDOW_BYTES}
+     */
+    receiveWindowBytes?: number;
+    /**
+     * How long (ms) a receive channel with a full window waits for an ack before it releases the
+     * window and writes on.
+     * Default: {@link DEFAULT_RECEIVE_ACK_TIMEOUT}
+     */
+    receiveAckTimeout?: number;
     /**
      * TTL (ms) for the per-(did, topicID) receive delivery-authorization cache. `0` disables reuse
      * (the hook is consulted for every frame). Non-finite or negative values fall back to

package/lib/handlers.js

@@ -1,7 +1,7 @@
 import { HandlerError } from '@enkaku/server';
 import { HUB_ERROR_CODES, hubErrorCodeOf, InvalidPayloadError, KeyPackageFetchLimitError, keyPackageDigest, wakeRecipientKeyProblem } from '@kumiai/hub-protocol';
 import { fromB64, toB64 } from '@sozai/codec';
-import { getReporter } from '@sozai/log';
+import { getLogger, getReporter } from '@sozai/log';
 import { createRateLimiter } from './rateLimit.js';
 function normalizeAuthorizeDecision(decision) {
     return typeof decision === 'boolean' ? {
@@ -29,6 +29,11 @@ const reportStoreError = getReporter([
     'kumiai',
     'hub-server'
 ], '@kumiai/hub-server');
+const receiveLogger = getLogger([
+    'kumiai',
+    'hub-server',
+    'receive'
+]);
 /** What the failed operation was about, for the default log line: a DID for the per-recipient
  * methods, a topic for fan-out, nothing for a store-wide purge. */ function subjectOf(event) {
     switch(event.method){
@@ -91,6 +96,31 @@ export const DEFAULT_KEYPACKAGE_FETCH_LIMITS = {
  * Bounds how long an already-resolved allow is reused before the hook is re-consulted — i.e. the
  * maximum window a removed member can keep draining a topic. See the receive handler's `gate`.
  */ export const DEFAULT_RECEIVE_AUTH_CACHE_TTL = 5000;
+/**
+ * Max frames in flight on a receive channel: written, and not yet followed by an ack for that frame
+ * or a later one. The hub writes the next frame only once acks bring the count back under this
+ * window.
+ *
+ * Without it the drain writes a whole backlog at once, and a transport that buffers per session
+ * (the SSE bridge drops a session whose unread buffer passes `maxSessionBufferBytes`) ends the
+ * session before the slow client's acks reach the hub: every connect then redelivers the same
+ * backlog. The window must stay well above the client's ack batch (10 for Kubun's device client)
+ * so batched acks keep the pipe full.
+ */ export const DEFAULT_RECEIVE_WINDOW = 256;
+/**
+ * Max bytes (base64 payload plus an estimate of the frame's envelope) in flight on a receive
+ * channel. Kept well below the HTTP transport's per-session buffer (1 MiB by default in `@enkaku/http-serve`, 8 MiB
+ * on Kubun's relay) so a window of large frames cannot overflow it. A single frame is always let
+ * through when nothing else is in flight, whatever its size.
+ */ export const DEFAULT_RECEIVE_WINDOW_BYTES = 512 * 1024;
+/**
+ * How long (ms) a receive channel with a full window waits without any ack before it releases the
+ * whole window and writes on. Each ack restarts the wait. The channel is NOT ended: a window full
+ * of frames the client will never ack (topics no sink on the device claims) would come back first
+ * on every reconnect and livelock it. Long enough for a busy phone to open the first frames of a
+ * window and flush an ack batch (10 frames or 500 ms on Kubun's device client); short, because it
+ * is the delay a window of never-acked frames costs per connect.
+ */ export const DEFAULT_RECEIVE_ACK_TIMEOUT = 15_000;
 function getClientDID(ctx) {
     const iss = ctx.message.payload.iss;
     if (typeof iss !== 'string' || iss.length === 0) {
@@ -149,6 +179,9 @@ export function createHandlers(params) {
     const topicLimiter = createRateLimiter(rateLimits.perTopic);
     const receiveBufferLimit = params.receiveBufferLimit ?? DEFAULT_RECEIVE_BUFFER_LIMIT;
     const receiveAuthCacheTTL = resolveReceiveAuthCacheTTL(params.receiveAuthCacheTTL);
+    const receiveWindow = Math.max(1, params.receiveWindow ?? DEFAULT_RECEIVE_WINDOW);
+    const receiveWindowBytes = params.receiveWindowBytes ?? DEFAULT_RECEIVE_WINDOW_BYTES;
+    const receiveAckTimeout = params.receiveAckTimeout ?? DEFAULT_RECEIVE_ACK_TIMEOUT;
     const storeErrorReporter = createStoreErrorReporter(params.onStoreError);
     const fetchLimits = {
         ...DEFAULT_KEYPACKAGE_FETCH_LIMITS,
@@ -494,15 +527,103 @@ export function createHandlers(params) {
             ;
             let tornDown = false;
             let writeChain = Promise.resolve();
+            // Ack window (flow control). `writer.ready` says nothing about the transport's per-session
+            // buffer (the HTTP bridge's writable is shared by every session and resolves at once), so the
+            // hub paces itself on acks instead: at most `receiveWindow` frames and `receiveWindowBytes`
+            // bytes are in flight, i.e. written and not yet known to have reached the client. Denied frames
+            // are never written, so never counted.
+            //
+            // The channel is ordered, so an ack for one frame proves the client has read every frame
+            // written before it: an ack releases that frame and all earlier ones from the window. A
+            // client may leave some frames unacked for good (a topic no sink on the device claims, a sink
+            // torn down mid-session); they stay pending in the store but cannot hold the window. This is
+            // window accounting only — the store still frees exactly the deliveries acked.
+            const inFlight = new Map() // sequenceID -> counted bytes, in write order
+            ;
+            let inFlightBytes = 0;
+            let onWindowChange = null;
+            const hasRoom = (bytes)=>inFlight.size === 0 || inFlight.size < receiveWindow && inFlightBytes + bytes <= receiveWindowBytes;
+            function releaseAll() {
+                inFlight.clear();
+                inFlightBytes = 0;
+            }
+            // Resolves once the frame fits or the channel is torn down. A full window that sees no ack for
+            // `receiveAckTimeout` ms is released whole and the drain goes on: the frames in it may be ones
+            // the client will never ack, and ending the channel would only redeliver them first on the
+            // next connect. A client that has stopped reading is not left to grow a buffer: the HTTP
+            // bridge drops a session that falls `maxSessionBufferBytes` behind, and a transport with real
+            // backpressure stalls `writer.write` until `receiveBufferLimit` tears the channel down.
+            function waitForRoom(bytes) {
+                if (tornDown || hasRoom(bytes)) return Promise.resolve();
+                receiveLogger.debug('receive window full for {recipientDID}: paused with {frames} frames ({bytes} bytes) unacked', {
+                    recipientDID: clientDID,
+                    frames: inFlight.size,
+                    bytes: inFlightBytes
+                });
+                return new Promise((resolve)=>{
+                    let timer;
+                    const arm = ()=>{
+                        clearTimeout(timer);
+                        timer = setTimeout(()=>{
+                            receiveLogger.warn('receive ack timeout for {recipientDID}: released {frames} frames ({bytes} bytes) after {timeoutMs} ms', {
+                                recipientDID: clientDID,
+                                frames: inFlight.size,
+                                bytes: inFlightBytes,
+                                timeoutMs: receiveAckTimeout
+                            });
+                            releaseAll();
+                            notifyWindow();
+                        }, receiveAckTimeout);
+                    };
+                    onWindowChange = ()=>{
+                        if (tornDown || hasRoom(bytes)) {
+                            clearTimeout(timer);
+                            onWindowChange = null;
+                            resolve();
+                        } else {
+                            arm();
+                        }
+                    };
+                    arm();
+                });
+            }
+            function notifyWindow() {
+                onWindowChange?.();
+            }
+            function release(sequenceIDs) {
+                const acked = new Set(sequenceIDs);
+                // The acked frame written last, in write order (an ack batch need not be sorted).
+                let last;
+                for (const id of inFlight.keys()){
+                    if (acked.has(id)) last = id;
+                }
+                if (last === undefined) return;
+                for (const [id, bytes] of inFlight){
+                    inFlight.delete(id);
+                    inFlightBytes -= bytes;
+                    if (id === last) break;
+                }
+                notifyWindow();
+            }
             let resolveDone = ()=>{};
             const done = new Promise((resolve)=>{
                 resolveDone = resolve;
             });
+            function logUnackedFrames() {
+                if (inFlight.size === 0) return;
+                receiveLogger.debug('receive stream ended for {recipientDID} with {frames} frames ({bytes} bytes) unacked', {
+                    recipientDID: clientDID,
+                    frames: inFlight.size,
+                    bytes: inFlightBytes
+                });
+            }
             // Idempotent teardown, shared by every failure and end path. Frames stay pending in the store
             // and redeliver on the next connect (store-and-forward).
             function finish() {
                 if (tornDown) return;
+                logUnackedFrames();
                 tornDown = true;
+                notifyWindow();
                 if (receiveToken != null) registry.releaseReceiveWriter(clientDID, receiveToken);
                 registry.unregisterIfIdle(clientDID);
                 reader.cancel().catch(()=>{});
@@ -522,7 +643,13 @@ export function createHandlers(params) {
                     if (tornDown) return;
                     try {
                         if (!await gate(frame.topicID)) return; // denied: skip, leave pending, no ack
+                        // The payload plus the frame's other fields and the transport's envelope (~256 chars).
+                        const bytes = frame.payload.length + frame.senderDID.length + frame.topicID.length + 256;
+                        await waitForRoom(bytes);
+                        if (tornDown) return;
                         await writer.ready;
+                        inFlight.set(frame.sequenceID, bytes);
+                        inFlightBytes += bytes;
                         await writer.write(frame);
                         if (frame.sequenceID > lastServed) lastServed = frame.sequenceID;
                     } catch  {
@@ -555,6 +682,38 @@ export function createHandlers(params) {
                     pushWrite(frame);
                 }
             };
+            // Ack loop (M1): a store.ack failure must not stop later acks; only a reader error closes.
+            // It runs from connect, alongside the drain: the drain waits on the ack window, so acks for the
+            // backlog must be read while it is still being served.
+            void (async ()=>{
+                while(true){
+                    let result;
+                    try {
+                        result = await reader.read();
+                    } catch  {
+                        break;
+                    }
+                    if (result.done) break;
+                    const ack = result.value?.ack;
+                    if (ack != null) {
+                        // The client has the frames, whatever the store does with the ack.
+                        release(ack);
+                        try {
+                            await store.ack({
+                                recipientDID: clientDID,
+                                sequenceIDs: ack
+                            });
+                        } catch (error) {
+                            // Frame stays pending; the client re-acks next round. Do NOT break.
+                            storeErrorReporter({
+                                method: 'ack',
+                                did: clientDID,
+                                error
+                            });
+                        }
+                    }
+                }
+            })();
             try {
                 // Binding evicts whatever held the lane: a reconnect means the old connection broke and the
                 // server learns last, so the stale writer must give way to the live one. See
@@ -565,6 +724,17 @@ export function createHandlers(params) {
                 evicted?.();
                 // The device is draining; a trailing summary would be noise it has already outrun.
                 params.wake?.dispatcher?.online(clientDID);
+                // Teardown triggers are armed before the drain: the drain waits on the ack window, so it
+                // can last as long as the client takes to ack, and an abort or eviction must end it.
+                // H2: an already-aborted signal never fires 'abort', so run teardown now.
+                if (ctx.signal.aborted) {
+                    finish();
+                } else {
+                    void evictedHere.then(finish);
+                    ctx.signal.addEventListener('abort', finish, {
+                        once: true
+                    });
+                }
                 // Drain the backlog. Await each page so lastServed is exact before the flush.
                 let cursor = after;
                 while(!tornDown){
@@ -597,50 +767,15 @@ export function createHandlers(params) {
                 }
             } catch (error) {
                 // A synchronous drain error (e.g. store.fetch threw): clean up and reject the handler.
+                if (!tornDown) logUnackedFrames();
                 tornDown = true;
+                notifyWindow();
                 if (receiveToken != null) registry.releaseReceiveWriter(clientDID, receiveToken);
                 registry.unregisterIfIdle(clientDID);
                 reader.cancel().catch(()=>{});
                 writer.abort(error).catch(()=>{});
                 throw error;
             }
-            // Ack loop (M1): a store.ack failure must not stop later acks; only a reader error closes.
-            void (async ()=>{
-                while(true){
-                    let result;
-                    try {
-                        result = await reader.read();
-                    } catch  {
-                        break;
-                    }
-                    if (result.done) break;
-                    const ack = result.value?.ack;
-                    if (ack != null) {
-                        try {
-                            await store.ack({
-                                recipientDID: clientDID,
-                                sequenceIDs: ack
-                            });
-                        } catch (error) {
-                            // Frame stays pending; the client re-acks next round. Do NOT break.
-                            storeErrorReporter({
-                                method: 'ack',
-                                did: clientDID,
-                                error
-                            });
-                        }
-                    }
-                }
-            })();
-            // H2: an already-aborted signal never fires 'abort', so run teardown now.
-            if (ctx.signal.aborted) {
-                finish();
-                return done;
-            }
-            void evictedHere.then(finish);
-            ctx.signal.addEventListener('abort', finish, {
-                once: true
-            });
             return done;
         },
         'hub/v1/keypackage/upload': async (ctx)=>{

package/lib/index.d.ts

@@ -1,5 +1,5 @@
 export type { AuthorizeDecision, AuthorizeHook, AuthorizeRequest, CreateHandlersParams, HubRateLimits, HubStoreErrorEvent, HubStoreErrorHook, KeyPackageFetchLimits, } from './handlers.js';
-export { createHandlers, DEFAULT_KEYPACKAGE_FETCH_LIMITS, DEFAULT_RATE_LIMITS, DEFAULT_RECEIVE_BUFFER_LIMIT, } from './handlers.js';
+export { createHandlers, DEFAULT_KEYPACKAGE_FETCH_LIMITS, DEFAULT_RATE_LIMITS, DEFAULT_RECEIVE_ACK_TIMEOUT, DEFAULT_RECEIVE_BUFFER_LIMIT, DEFAULT_RECEIVE_WINDOW, DEFAULT_RECEIVE_WINDOW_BYTES, } from './handlers.js';
 export type { CreateHubParams, HubInstance, HubPurgeOptions, HubReplayOptions } from './hub.js';
 export { createHub, DEFAULT_HUB_ACCESS_RULES } from './hub.js';
 export { createMemoryStore, type MemoryStoreOptions } from './memoryStore.js';

package/lib/index.js

@@ -1,4 +1,4 @@
-export { createHandlers, DEFAULT_KEYPACKAGE_FETCH_LIMITS, DEFAULT_RATE_LIMITS, DEFAULT_RECEIVE_BUFFER_LIMIT } from './handlers.js';
+export { createHandlers, DEFAULT_KEYPACKAGE_FETCH_LIMITS, DEFAULT_RATE_LIMITS, DEFAULT_RECEIVE_ACK_TIMEOUT, DEFAULT_RECEIVE_BUFFER_LIMIT, DEFAULT_RECEIVE_WINDOW, DEFAULT_RECEIVE_WINDOW_BYTES } from './handlers.js';
 export { createHub, DEFAULT_HUB_ACCESS_RULES } from './hub.js';
 export { createMemoryStore } from './memoryStore.js';
 export { createRateLimiter } from './rateLimit.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kumiai/hub-server",
-  "version": "0.12.0",
+  "version": "0.12.2",
   "description": "Hub server for Enkaku E2EE group message routing",
   "keywords": [
     "hub",
@@ -25,18 +25,18 @@
     "lib/*"
   ],
   "dependencies": {
-    "@enkaku/protocol": "^0.22.2",
-    "@enkaku/server": "^0.22.2",
+    "@enkaku/protocol": "^0.22.5",
+    "@enkaku/server": "^0.22.5",
     "@kokuin/capability": "^0.5.2",
-    "@kokuin/token": "^0.6.2",
+    "@kokuin/token": "^0.6.4",
     "@kumiai/hub-protocol": "^0.12.0",
-    "@sozai/codec": "^0.4.3",
+    "@sozai/codec": "^0.4.4",
     "@sozai/event": "^0.1.6",
     "@sozai/log": "^0.4.0"
   },
   "devDependencies": {
-    "@enkaku/client": "^0.22.2",
-    "@enkaku/transport": "^0.22.2",
+    "@enkaku/client": "^0.22.5",
+    "@enkaku/transport": "^0.22.5",
     "@kumiai/hub-conformance": "^0.12.0",
     "@kumiai/hub-wake": "^0.12.0"
   },