@kubun/hub 0.11.0 → 0.12.0

package/lib/create-relay.d.ts

@@ -22,6 +22,9 @@ export type CreateRelayOptions = {
     auth?: HubAuthConfig;
     allowedOrigin?: string | Array<string>;
     retention?: RelayRetentionConfig;
+    defaultRetention?: number;
+    maxRetention?: number;
+    maxDepth?: number;
     observers?: RelayObservers;
 };
 export type Relay = {
@@ -37,6 +40,9 @@ export type RelayInternalOptions = {
     auth?: HubAuthConfig;
     allowedOrigin?: string | Array<string>;
     retention?: RelayRetentionConfig;
+    defaultRetention?: number;
+    maxRetention?: number;
+    maxDepth?: number;
     observers?: RelayObservers;
     ownsKysely: boolean;
 };

package/lib/create-relay.js

@@ -1,8 +1,7 @@
 import { ServerTransport } from '@enkaku/http-serve';
-import { Server } from '@enkaku/server';
 import { resolveAdapter } from '@kubun/db-provider';
 import { resolveAllowedOrigin } from '@kubun/http-util';
-import { createHandlers as createHubHandlers, HubClientRegistry } from '@kumiai/hub-server';
+import { createHub } from '@kumiai/hub-server';
 import { Kysely } from 'kysely';
 import { Migrator } from 'kysely/migration';
 import { createHubStore } from './hub-store.js';
@@ -10,6 +9,8 @@ import { createHubMigrations } from './migrations.js';
 import { decorateHubStore } from './observers.js';
 const DEFAULT_RETENTION_INTERVAL_SECONDS = 3600;
 const DEFAULT_SUBSCRIPTION_RETENTION_SECONDS = 90 * 24 * 60 * 60;
+const DEFAULT_MAX_RETENTION_SECONDS = 30 * 24 * 60 * 60;
+const DEFAULT_MAX_DEPTH = 1000;
 function resolveAccessRules(auth) {
     if (auth == null) return {};
     switch(auth.mode){
@@ -62,10 +63,12 @@ export async function createRelayInternal(options) {
     if (migrationError != null) {
         throw migrationError;
     }
-    const registry = new HubClientRegistry();
     const rawStore = createHubStore({
         kysely,
-        adapter
+        adapter,
+        defaultRetention: options.defaultRetention ?? 0,
+        maxRetention: options.maxRetention ?? DEFAULT_MAX_RETENTION_SECONDS,
+        maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH
     });
     const hubStore = decorateHubStore(rawStore, observers);
     let unsubscribePurge;
@@ -81,24 +84,21 @@ export async function createRelayInternal(options) {
             }
         });
     }
-    const hubHandlers = createHubHandlers({
-        registry,
-        store: hubStore
-    });
     const allowedOrigin = resolveAllowedOrigin(options.allowedOrigin);
     const transport = new ServerTransport({
         allowedOrigin
     });
     const accessRules = resolveAccessRules(options.auth);
-    const serverParams = {
+    // `@kumiai/hub-server` assembles the handlers, registry, and Enkaku server for
+    // its own protocol. Purging is disabled here because this relay runs its own
+    // retention timer below, over the same store, with kubun's clamped windows.
+    const { server: enkakuServer } = createHub({
+        transport,
+        store: hubStore,
         identity: options.identity,
         accessRules,
-        handlers: hubHandlers,
-        transports: [
-            transport
-        ]
-    };
-    const enkakuServer = new Server(serverParams);
+        purge: false
+    });
     if (observers?.onAuthRejected != null) {
         const onAuthRejected = observers.onAuthRejected;
         enkakuServer.events.on('handlerError', (info)=>{
@@ -113,6 +113,9 @@ export async function createRelayInternal(options) {
         });
     }
     let retentionTimer;
+    // The sweep a fired tick is still running, so `dispose` can wait for it.
+    let sweep;
+    let runSweep;
     if (options.retention != null) {
         const { olderThanSeconds } = options.retention;
         const intervalSeconds = options.retention.intervalSeconds ?? DEFAULT_RETENTION_INTERVAL_SECONDS;
@@ -133,7 +136,17 @@ export async function createRelayInternal(options) {
             }
         }
         retentionTimer = setInterval(()=>{
-            void hubStore.purge({
+            // A tick that already fired is live work, not a timer: `purge` is an
+            // unbounded per-topic scan plus two bulk deletes, and on Postgres
+            // destroying the pool underneath one wedges `close()` forever. `dispose`
+            // awaits this handle, so cancelling the timer and draining what it
+            // started are two different things and both are needed.
+            sweep = runSweep?.().finally(()=>{
+                sweep = undefined;
+            });
+        }, intervalMs);
+        runSweep = async ()=>{
+            await hubStore.purge({
                 olderThan: olderThanSeconds
             }).catch((err)=>{
                 if (observers?.onError != null) {
@@ -147,7 +160,7 @@ export async function createRelayInternal(options) {
                     }
                 }
             });
-            void hubStore.purgeSubscribers({
+            await hubStore.purgeSubscribers({
                 olderThan: subscriptionOlderThanSeconds
             }).then((purged)=>{
                 if (observers?.onSubscriptionPurge != null) {
@@ -171,7 +184,7 @@ export async function createRelayInternal(options) {
                     }
                 }
             });
-        }, intervalMs);
+        };
     }
     return {
         fetch: (request)=>transport.fetch(request),
@@ -185,6 +198,9 @@ export async function createRelayInternal(options) {
                 clearInterval(retentionTimer);
                 retentionTimer = undefined;
             }
+            // Drain before the server and the pool go: cancelling the timer stops the
+            // NEXT tick and does nothing about the one already running.
+            await sweep;
             if (unsubscribePurge != null) {
                 unsubscribePurge();
                 unsubscribePurge = undefined;
@@ -208,6 +224,9 @@ export async function createRelay(options) {
         auth: options.auth,
         allowedOrigin: options.allowedOrigin,
         retention: options.retention,
+        defaultRetention: options.defaultRetention,
+        maxRetention: options.maxRetention,
+        maxDepth: options.maxDepth,
         observers: options.observers,
         ownsKysely: true
     });

package/lib/hub-store.d.ts

@@ -1,5 +1,5 @@
 import type { Adapter } from '@kubun/db-adapter';
-import type { HubStore } from '@kumiai/hub-protocol';
+import { type HubStore } from '@kumiai/hub-protocol';
 import type { ColumnType, Kysely } from 'kysely';
 export type HubDatabase = {
     kubun_hub_messages: {
@@ -8,6 +8,7 @@ export type HubDatabase = {
         topic_id: string;
         payload: Uint8Array;
         stored_at: number;
+        retain: string;
     };
     kubun_hub_deliveries: {
         sequence_id: string;
@@ -17,16 +18,41 @@ export type HubDatabase = {
         id: ColumnType<number, number | undefined, never>;
         owner_did: string;
         key_package: string;
+        not_after: ColumnType<number | null, number | null | undefined, never>;
+    };
+    kubun_hub_last_resort_key_packages: {
+        owner_did: string;
+        key_package: string;
     };
     kubun_hub_subscriptions: {
         topic_id: string;
         subscriber_did: string;
         last_active_at: number;
+        retention: ColumnType<number, number | undefined, number>;
+    };
+    kubun_hub_topics: {
+        topic_id: string;
+        head: string;
+    };
+    kubun_hub_sequence: {
+        id: number;
+        counter: ColumnType<number, number, number>;
+    };
+    kubun_hub_publish_ids: {
+        topic_id: string;
+        publish_id: string;
+        sequence_id: string;
+        recorded_at: number;
     };
 };
 export type CreateHubStoreParams = {
     kysely: Kysely<HubDatabase> | Promise<Kysely<HubDatabase>>;
     adapter: Adapter;
+    defaultRetention?: number;
+    maxRetention?: number;
+    maxDepth?: number;
+    maxKeyPackagesPerDID?: number;
+    maxSubscriptionsPerDID?: number;
 };
 export type PurgeSubscriptionsParams = {
     olderThan: number;

package/lib/hub-store.js

@@ -1,6 +1,29 @@
+import { HeadMismatchError, KeyPackageQuotaExceededError, NotSubscribedError, RetentionExceededError, SubscriptionQuotaExceededError } from '@kumiai/hub-protocol';
 import { EventEmitter } from '@sozai/event';
+// Normalize a DB binary column (a Buffer under better-sqlite3 / node-postgres)
+// to a plain Uint8Array, so payload equality holds against a Uint8Array caller.
+function toBytes(value) {
+    return new Uint8Array(value);
+}
+/**
+ * How long an idempotency record outlives its own publish, in seconds.
+ *
+ * A year: a publish repeated after this is treated as new, so the window has to
+ * exceed any replay a real client could still be holding — a parked commit, a
+ * device restored from a backup, a lane retried after an outage. It is not a
+ * retention knob a host tunes; it is the horizon past which "the same publish"
+ * stops being a coherent claim.
+ */ const PUBLISH_ID_RETENTION_SECONDS = 365 * 24 * 60 * 60;
+/** Frames deleted per statement — see {@link deleteFrames}. */ const DELETE_CHUNK_SIZE = 500;
+/** Matches `@kumiai/hub-server`'s own defaults — a kubun relay is not a laxer hub. */ const DEFAULT_MAX_KEY_PACKAGES_PER_DID = 100;
+const DEFAULT_MAX_SUBSCRIPTIONS_PER_DID = 1000;
 export function createHubStore(params) {
     const { adapter } = params;
+    const defaultRetention = params.defaultRetention ?? 0;
+    const maxRetention = params.maxRetention;
+    const maxDepth = params.maxDepth;
+    const maxKeyPackagesPerDID = params.maxKeyPackagesPerDID ?? DEFAULT_MAX_KEY_PACKAGES_PER_DID;
+    const maxSubscriptionsPerDID = params.maxSubscriptionsPerDID ?? DEFAULT_MAX_SUBSCRIPTIONS_PER_DID;
     // Lazy DB resolution -- accepts Kysely or Promise<Kysely> and caches the resolved instance
     let resolvedDB;
     async function db() {
@@ -9,48 +32,157 @@ export function createHubStore(params) {
         }
         return resolvedDB;
     }
-    // Lazy sequence counter -- initialized from DB on first publish() call
-    let sequenceCounter;
-    async function nextSequenceID() {
-        const kysely = await db();
-        if (sequenceCounter == null) {
-            const maxRow = await kysely.selectFrom('kubun_hub_messages').select(kysely.fn.max('sequence_id').as('max_id')).executeTakeFirst();
-            sequenceCounter = maxRow?.max_id != null ? Number.parseInt(maxRow.max_id, 10) : 0;
+    const events = new EventEmitter();
+    /**
+   * Remove frames and the deliveries pointing at them, a chunk at a time.
+   *
+   * SQLite binds one parameter per element of an `in` list and throws "too many
+   * SQL variables" past its limit — 32766 on the build this ships with, 999 on
+   * older ones — so an unchunked sweep fails precisely when the backlog it exists
+   * to clear is large. Deliveries first: a delivery references a frame and cannot
+   * be pushed once its referent is gone.
+   */ async function deleteFrames(executor, sequenceIDs) {
+        for(let index = 0; index < sequenceIDs.length; index += DELETE_CHUNK_SIZE){
+            const chunk = sequenceIDs.slice(index, index + DELETE_CHUNK_SIZE);
+            await executor.deleteFrom('kubun_hub_deliveries').where('sequence_id', 'in', chunk).execute();
+            await executor.deleteFrom('kubun_hub_messages').where('sequence_id', 'in', chunk).execute();
         }
-        sequenceCounter++;
-        return String(sequenceCounter).padStart(12, '0');
     }
-    const events = new EventEmitter();
+    // Mint the next sequenceID inside the accepting transaction, from a persistent
+    // global counter. Fixed-width zero-padded so lexicographic order matches
+    // numeric order (`"10" > "9"`), and strictly increasing.
+    //
+    // One row, so every publish on every topic serializes behind it. That is the
+    // cost of a single total order; it deadlocks nothing, because every writer
+    // takes this row FIRST and the per-topic rows after it, so no two transactions
+    // hold each other's next lock.
+    async function nextSequenceID(trx) {
+        const row = await trx.updateTable('kubun_hub_sequence').set((eb)=>({
+                counter: eb('counter', '+', 1)
+            })).where('id', '=', 0).returning('counter').executeTakeFirstOrThrow();
+        return String(row.counter).padStart(12, '0');
+    }
+    // Depth eviction runs inside the publishing transaction. It counts LOG frames
+    // only and removes the oldest beyond the bound (with their deliveries). Silent:
+    // depth eviction is a synchronous consequence of the caller's own publish.
+    async function evictDepth(trx, topicID) {
+        if (maxDepth == null) return;
+        const logRows = await trx.selectFrom('kubun_hub_messages').select('sequence_id').where('topic_id', '=', topicID).where('retain', '=', 'log').orderBy('sequence_id', 'asc').execute();
+        if (logRows.length <= maxDepth) return;
+        const evict = logRows.slice(0, logRows.length - maxDepth).map((r)=>r.sequence_id);
+        await deleteFrames(trx, evict);
+    }
     async function publish(publishParams) {
-        const sequenceID = await nextSequenceID();
         const kysely = await db();
-        // Recipients = current subscribers minus the sender. Zero recipients (no
-        // subscribers, or only the sender) means there is nothing to deliver, so the
-        // sequence is consumed but no message or delivery rows are stored.
-        const subscribers = await getSubscribers(publishParams.topicID);
-        const recipients = subscribers.filter((did)=>did !== publishParams.senderDID);
-        if (recipients.length === 0) {
-            return sequenceID;
-        }
-        const storedAt = Date.now();
-        // The message row and its delivery rows are written in one transaction: a
-        // partial write (message stored, deliveries failed) would orphan a message
-        // with zero deliveries — never fetched, never acked, never GC'd by `ack`,
-        // reclaimed only by the age-based purge.
-        await kysely.transaction().execute(async (trx)=>{
-            await trx.insertInto('kubun_hub_messages').values({
-                sequence_id: sequenceID,
-                sender_did: publishParams.senderDID,
-                topic_id: publishParams.topicID,
-                payload: adapter.encodeBinary(publishParams.payload),
-                stored_at: storedAt
-            }).execute();
-            await trx.insertInto('kubun_hub_deliveries').values(recipients.map((did)=>({
+        const retain = publishParams.retain ?? 'mailbox';
+        const { senderDID, topicID } = publishParams;
+        // The dedup check, the head compare-and-set, the sequence mint, the append,
+        // and the head advance all happen in ONE transaction, in that order. A
+        // read-then-write CAS split across statements is the exact race the head
+        // exists to eliminate.
+        return await kysely.transaction().execute(async (trx)=>{
+            // 1. Dedup — BEFORE the CAS. A replay carries a stale expectedHead by
+            // construction (the publish it replays is what moved the head), so comparing
+            // first would raise HeadMismatchError and tell the caller its commit was lost
+            // when it landed.
+            if (publishParams.publishID != null) {
+                const existing = await trx.selectFrom('kubun_hub_publish_ids').select('sequence_id').where('topic_id', '=', topicID).where('publish_id', '=', publishParams.publishID).executeTakeFirst();
+                if (existing != null) {
+                    return {
+                        sequenceID: existing.sequence_id,
+                        deduped: true
+                    };
+                }
+            }
+            // 2. Stored head (null when the topic has never had an accepted log publish).
+            const topicRow = await trx.selectFrom('kubun_hub_topics').select('head').where('topic_id', '=', topicID).executeTakeFirst();
+            const currentHead = topicRow?.head ?? null;
+            // 3. Compare-and-set. On mismatch, throw and store NOTHING (the transaction
+            // rolls back — no entry, no delivery, no sequenceID consumed, no event).
+            if (publishParams.expectedHead !== undefined && currentHead !== publishParams.expectedHead) {
+                throw new HeadMismatchError(`head mismatch: expected ${String(publishParams.expectedHead)}, current ${String(currentHead)}`);
+            }
+            // 4. Mint the sequenceID.
+            const sequenceID = await nextSequenceID(trx);
+            // 5. Recipients = current subscribers minus the sender.
+            const subscriberRows = await trx.selectFrom('kubun_hub_subscriptions').select('subscriber_did').where('topic_id', '=', topicID).execute();
+            const recipients = subscriberRows.map((r)=>r.subscriber_did).filter((did)=>did !== senderDID);
+            // 6. Append. A log frame is stored unconditionally (a reader that must read
+            // it may not exist yet). A mailbox frame is delivery-derived: with zero
+            // recipients there is nothing to deliver, so it is not stored — but the
+            // sequenceID is still consumed.
+            const storeMessage = retain === 'log' || recipients.length > 0;
+            if (storeMessage) {
+                await trx.insertInto('kubun_hub_messages').values({
+                    sequence_id: sequenceID,
+                    sender_did: senderDID,
+                    topic_id: topicID,
+                    payload: adapter.encodeBinary(publishParams.payload),
+                    stored_at: Date.now(),
+                    retain
+                }).execute();
+                // Push is untouched by the class: a log frame is delivered to every current
+                // subscriber exactly as a mailbox frame is.
+                if (recipients.length > 0) {
+                    await trx.insertInto('kubun_hub_deliveries').values(recipients.map((did)=>({
+                            sequence_id: sequenceID,
+                            recipient_did: did
+                        }))).execute();
+                }
+            }
+            // 7. Advance the stored head — LOG frames only. A mailbox publish never moves
+            // the head.
+            //
+            // The head advance IS the compare-and-set; the earlier head read is only an
+            // MVCC snapshot, which two transactions racing at the same head both pass. The
+            // arbiter is this CONDITIONAL write, whose row count decides the winner: the
+            // loser matches zero rows, throws, and rolls the whole transaction back.
+            if (retain === 'log') {
+                if (publishParams.expectedHead === undefined) {
+                    // Unconditional log publish: last-writer-wins on the head pointer. The
+                    // frame is already appended; only the pointer races, and the caller did
+                    // not ask for a CAS.
+                    await trx.insertInto('kubun_hub_topics').values({
+                        topic_id: topicID,
+                        head: sequenceID
+                    }).onConflict((oc)=>oc.column('topic_id').doUpdateSet({
+                            head: sequenceID
+                        })).execute();
+                } else if (publishParams.expectedHead === null) {
+                    // Expected "never had a log publish": insert the row, and lose to any
+                    // concurrent transaction that inserted it first (conflict → 0 rows).
+                    const inserted = await trx.insertInto('kubun_hub_topics').values({
+                        topic_id: topicID,
+                        head: sequenceID
+                    }).onConflict((oc)=>oc.column('topic_id').doNothing()).executeTakeFirst();
+                    if (Number(inserted.numInsertedOrUpdatedRows ?? 0n) === 0) {
+                        throw new HeadMismatchError('head mismatch: expected null, a concurrent publish set the head first');
+                    }
+                } else {
+                    // Expected a specific head: advance only if it is still that value.
+                    const updated = await trx.updateTable('kubun_hub_topics').set({
+                        head: sequenceID
+                    }).where('topic_id', '=', topicID).where('head', '=', publishParams.expectedHead).executeTakeFirst();
+                    if (Number(updated.numUpdatedRows ?? 0n) === 0) {
+                        throw new HeadMismatchError(`head mismatch: expected ${publishParams.expectedHead}, a concurrent publish advanced it`);
+                    }
+                }
+                await evictDepth(trx, topicID);
+            }
+            // 8. Record the idempotency key in its own table.
+            if (publishParams.publishID != null) {
+                await trx.insertInto('kubun_hub_publish_ids').values({
+                    topic_id: topicID,
+                    publish_id: publishParams.publishID,
                     sequence_id: sequenceID,
-                    recipient_did: did
-                }))).execute();
+                    recorded_at: Date.now()
+                }).execute();
+            }
+            return {
+                sequenceID,
+                deduped: false
+            };
         });
-        return sequenceID;
     }
     async function fetch(fetchParams) {
         // Process acks first if provided
@@ -65,7 +197,8 @@ export function createHubStore(params) {
             'kubun_hub_messages.sequence_id',
             'kubun_hub_messages.sender_did',
             'kubun_hub_messages.topic_id',
-            'kubun_hub_messages.payload'
+            'kubun_hub_messages.payload',
+            'kubun_hub_messages.retain'
         ]).where('kubun_hub_deliveries.recipient_did', '=', fetchParams.recipientDID).orderBy('kubun_hub_messages.sequence_id', 'asc');
         if (fetchParams.after != null) {
             query = query.where('kubun_hub_messages.sequence_id', '>', fetchParams.after);
@@ -81,11 +214,20 @@ export function createHubStore(params) {
             hasMore = true;
             resultRows = rows.slice(0, limit);
         }
+        // One global sequence serves both classes here, so a log frame's own
+        // sequenceID IS its position in the topic's log. A delivery position is a
+        // different number (it spans every subscribed topic, skips the recipient's own
+        // sends, and empties as it is acked), so a recipient cannot derive the log
+        // position itself. Omitted entirely for a mailbox frame — a falsy placeholder is
+        // a position a cursor would move to, skipping every log frame below it.
         const messages = resultRows.map((row)=>({
                 sequenceID: row.sequence_id,
                 senderDID: row.sender_did,
                 topicID: row.topic_id,
-                payload: row.payload
+                payload: toBytes(row.payload),
+                ...row.retain === 'log' ? {
+                    logPosition: row.sequence_id
+                } : {}
             }));
         const cursor = messages.length > 0 ? messages[messages.length - 1].sequenceID : null;
         return {
@@ -94,68 +236,219 @@ export function createHubStore(params) {
             hasMore
         };
     }
+    async function fetchTopic(fetchParams) {
+        const kysely = await db();
+        const { subscriberDID, topicID } = fetchParams;
+        // Authorization: the caller must be a current subscriber of the topic.
+        const subscription = await kysely.selectFrom('kubun_hub_subscriptions').select('subscriber_did').where('topic_id', '=', topicID).where('subscriber_did', '=', subscriberDID).executeTakeFirst();
+        if (subscription == null) {
+            throw new NotSubscribedError(`${subscriberDID} is not subscribed to ${topicID}`);
+        }
+        // Head is stored state — it outlives every frame.
+        const topicRow = await kysely.selectFrom('kubun_hub_topics').select('head').where('topic_id', '=', topicID).executeTakeFirst();
+        const head = topicRow?.head ?? null;
+        // Oldest is the oldest RETAINED log frame, or null if the log is empty.
+        const oldestRow = await kysely.selectFrom('kubun_hub_messages').select((eb)=>eb.fn.min('sequence_id').as('oldest')).where('topic_id', '=', topicID).where('retain', '=', 'log').executeTakeFirst();
+        const oldest = oldestRow?.oldest ?? null;
+        // The log is the topic's log-class frames and NOTHING else — a mailbox frame
+        // never appears here. `limit` counts log frames (applied after the class
+        // filter), so a page of mailbox frames cannot hand a draining reader an empty
+        // page while log frames still wait.
+        let query = kysely.selectFrom('kubun_hub_messages').select([
+            'sequence_id',
+            'sender_did',
+            'topic_id',
+            'payload'
+        ]).where('topic_id', '=', topicID).where('retain', '=', 'log').orderBy('sequence_id', 'asc');
+        if (fetchParams.after != null) {
+            query = query.where('sequence_id', '>', fetchParams.after);
+        }
+        if (fetchParams.limit != null) {
+            query = query.limit(fetchParams.limit);
+        }
+        const rows = await query.execute();
+        // Every row here is log-class by the filter above, so each carries its position
+        // unconditionally — the same value a push of that frame reports.
+        const messages = rows.map((row)=>({
+                sequenceID: row.sequence_id,
+                senderDID: row.sender_did,
+                topicID: row.topic_id,
+                payload: toBytes(row.payload),
+                logPosition: row.sequence_id
+            }));
+        return {
+            messages,
+            head,
+            oldest
+        };
+    }
     async function ack(ackParams) {
         if (ackParams.sequenceIDs.length === 0) return;
         const kysely = await db();
         // Remove delivery entries for this recipient
         await kysely.deleteFrom('kubun_hub_deliveries').where('recipient_did', '=', ackParams.recipientDID).where('sequence_id', 'in', ackParams.sequenceIDs).execute();
-        // Clean up messages with no remaining deliveries (all recipients acked)
+        // GC fully-acked MAILBOX messages only. A log frame is removed by trim/purge
+        // alone — ack frees a delivery, never a log entry or the head.
         const withDeliveries = await kysely.selectFrom('kubun_hub_deliveries').select('sequence_id').where('sequence_id', 'in', ackParams.sequenceIDs).execute();
         const stillDelivered = new Set(withDeliveries.map((r)=>r.sequence_id));
         const fullyAcked = ackParams.sequenceIDs.filter((id)=>!stillDelivered.has(id));
         if (fullyAcked.length > 0) {
-            await kysely.deleteFrom('kubun_hub_messages').where('sequence_id', 'in', fullyAcked).execute();
+            await kysely.deleteFrom('kubun_hub_messages').where('sequence_id', 'in', fullyAcked).where('retain', '=', 'mailbox').execute();
         }
     }
+    async function trim(trimParams) {
+        const kysely = await db();
+        // Remove LOG frames strictly below the bound, and only those. A mailbox frame
+        // on the same topic is delivery-derived and untouched. The head (topics table)
+        // and the publishID records are never touched.
+        const logRows = await kysely.selectFrom('kubun_hub_messages').select('sequence_id').where('topic_id', '=', trimParams.topicID).where('retain', '=', 'log').where('sequence_id', '<', trimParams.before).execute();
+        const sequenceIDs = logRows.map((r)=>r.sequence_id);
+        if (sequenceIDs.length === 0) return;
+        await deleteFrames(kysely, sequenceIDs);
+    }
     async function purge(purgeParams) {
         const kysely = await db();
-        const threshold = Date.now() - purgeParams.olderThan * 1000;
-        const oldMessages = await kysely.selectFrom('kubun_hub_messages').select('sequence_id').where('stored_at', '<=', threshold).execute();
-        const sequenceIDs = oldMessages.map((m)=>m.sequence_id);
-        if (sequenceIDs.length === 0) return [];
-        await kysely.deleteFrom('kubun_hub_deliveries').where('sequence_id', 'in', sequenceIDs).execute();
-        await kysely.deleteFrom('kubun_hub_messages').where('sequence_id', 'in', sequenceIDs).execute();
+        const now = Date.now();
+        const olderThan = purgeParams.olderThan;
+        // Idempotency records outlive the frames they dedup, on their own clock and
+        // deliberately: the contract is that no message deleter reaches one for a
+        // frame it still guards, NOT that they are kept forever. Every commit
+        // publish writes one, so without this the table is the fastest-growing on a
+        // long-lived hub, holding rows for frames purged years earlier. The window
+        // is far longer than any plausible replay — a publish repeated after it is
+        // accepted as new, which is what a fresh publishID would be anyway.
+        await kysely.deleteFrom('kubun_hub_publish_ids').where('recorded_at', '<=', now - PUBLISH_ID_RETENTION_SECONDS * 1000).execute();
+        // Per topic, the effective age bound is the longest retention any CURRENT
+        // subscriber asked for, floored at the hub default (and at the sweep's
+        // olderThan). Applies to both classes. Never touches the head or a publishID
+        // record for a frame it removes.
+        const topicRows = await kysely.selectFrom('kubun_hub_messages').select('topic_id').distinct().execute();
+        // Every topic's longest CURRENT subscriber retention in one grouped read
+        // rather than one read per topic: the sweep walks every topic the hub holds,
+        // so the per-topic version is a query count that grows with the hub.
+        const retentionRows = await kysely.selectFrom('kubun_hub_subscriptions').select((eb)=>[
+                'topic_id',
+                eb.fn.max('retention').as('maxRetention')
+            ]).groupBy('topic_id').execute();
+        const retentionByTopic = new Map(retentionRows.map((row)=>[
+                row.topic_id,
+                Number(row.maxRetention ?? 0)
+            ]));
+        const removed = [];
+        for (const { topic_id } of topicRows){
+            const subscriberRetention = retentionByTopic.get(topic_id) ?? 0;
+            const effective = Math.max(olderThan, subscriberRetention, defaultRetention);
+            const threshold = now - effective * 1000;
+            const frames = await kysely.selectFrom('kubun_hub_messages').select('sequence_id').where('topic_id', '=', topic_id).where('stored_at', '<=', threshold).execute();
+            for (const frame of frames){
+                removed.push(frame.sequence_id);
+            }
+        }
+        if (removed.length === 0) return [];
+        await deleteFrames(kysely, removed);
         await events.emit('purge', {
-            sequenceIDs
+            sequenceIDs: removed
         });
-        return sequenceIDs;
+        return removed;
     }
-    async function storeKeyPackage(ownerDID, keyPackage) {
+    // `not_after` is seconds; every other timestamp here is milliseconds.
+    function nowSeconds() {
+        return Math.floor(Date.now() / 1000);
+    }
+    async function deleteExpiredKeyPackages(kysely, ownerDID) {
+        await kysely.deleteFrom('kubun_hub_key_packages').where('owner_did', '=', ownerDID).where('not_after', 'is not', null).where('not_after', '<=', nowSeconds()).execute();
+    }
+    async function storeKeyPackage(ownerDID, keyPackage, notAfter) {
         const kysely = await db();
+        // Drop the dead BEFORE charging the cap. The cap rejects rather than evicts, so
+        // nothing else would ever remove them: a pool that filled with expired entries
+        // could never be replenished and its owner would be permanently unaddable.
+        await deleteExpiredKeyPackages(kysely, ownerDID);
+        const row = await kysely.selectFrom('kubun_hub_key_packages').select((eb)=>eb.fn.countAll().as('count')).where('owner_did', '=', ownerDID).executeTakeFirst();
+        if (Number(row?.count ?? 0) >= maxKeyPackagesPerDID) {
+            throw new KeyPackageQuotaExceededError(`DID ${ownerDID} exceeds the maximum of ${maxKeyPackagesPerDID} stored key packages`);
+        }
         await kysely.insertInto('kubun_hub_key_packages').values({
             owner_did: ownerDID,
-            key_package: keyPackage
+            key_package: keyPackage,
+            not_after: notAfter ?? null
         }).execute();
     }
     async function fetchKeyPackages(ownerDID, count) {
         const kysely = await db();
+        // Expired entries are DELETED first, not filtered after the limit: a run of dead
+        // rows would otherwise consume the whole page and return nothing while live
+        // packages waited behind them.
+        await deleteExpiredKeyPackages(kysely, ownerDID);
         const limit = count ?? 1;
         // Atomic delete-returning: a non-atomic select+delete pair would let two
         // concurrent fetches return the same key package twice.
         const rows = await kysely.deleteFrom('kubun_hub_key_packages').where('id', 'in', kysely.selectFrom('kubun_hub_key_packages').select('id').where('owner_did', '=', ownerDID).orderBy('id', 'asc').limit(limit)).returning('key_package').execute();
         return rows.map((r)=>r.key_package);
     }
-    async function subscribe(subscriberDID, topicID) {
+    async function countKeyPackages(ownerDID) {
         const kysely = await db();
+        const row = await kysely.selectFrom('kubun_hub_key_packages').select((eb)=>eb.fn.countAll().as('count')).where('owner_did', '=', ownerDID).where((eb)=>eb.or([
+                eb('not_after', 'is', null),
+                eb('not_after', '>', nowSeconds())
+            ])).executeTakeFirst();
+        return Number(row?.count ?? 0);
+    }
+    async function storeLastResortKeyPackage(ownerDID, keyPackage) {
+        const kysely = await db();
+        await kysely.insertInto('kubun_hub_last_resort_key_packages').values({
+            owner_did: ownerDID,
+            key_package: keyPackage
+        }).onConflict((oc)=>oc.column('owner_did').doUpdateSet({
+                key_package: (eb)=>eb.ref('excluded.key_package')
+            })).execute();
+    }
+    async function fetchLastResortKeyPackage(ownerDID) {
+        const kysely = await db();
+        // Never consumes. Reuse is the point: the package carries MLS's `last_resort`
+        // extension, so serving it twice is the availability floor rather than the
+        // init-key reuse `fetchKeyPackages` exists to prevent.
+        const row = await kysely.selectFrom('kubun_hub_last_resort_key_packages').select('key_package').where('owner_did', '=', ownerDID).executeTakeFirst();
+        return row?.key_package ?? null;
+    }
+    async function subscribe(subscribeParams) {
+        const { subscriberDID, topicID } = subscribeParams;
+        // A retention above the hub maximum is refused, never clamped — a peer that
+        // believed it had asked for more would otherwise be stranded.
+        if (subscribeParams.retention != null && maxRetention != null && subscribeParams.retention > maxRetention) {
+            throw new RetentionExceededError(`requested retention ${subscribeParams.retention} exceeds the hub maximum ${maxRetention}`);
+        }
+        const retention = subscribeParams.retention ?? defaultRetention;
+        const kysely = await db();
+        // Charged on NEW rows only: a re-subscribe to a topic this DID already holds is a
+        // refresh, and refusing it would strand a subscriber at the cap on every reconnect.
+        const held = await kysely.selectFrom('kubun_hub_subscriptions').select('topic_id').where('subscriber_did', '=', subscriberDID).execute();
+        if (!held.some((row)=>row.topic_id === topicID) && held.length >= maxSubscriptionsPerDID) {
+            throw new SubscriptionQuotaExceededError(`DID ${subscriberDID} exceeds the maximum of ${maxSubscriptionsPerDID} subscriptions`);
+        }
         // Upsert on the (topic_id, subscriber_did) primary key so re-subscribing an
-        // existing subscriber refreshes its activity timestamp rather than failing
-        // or duplicating.
+        // existing subscriber refreshes its activity timestamp and retention rather
+        // than failing or duplicating.
         await kysely.insertInto('kubun_hub_subscriptions').values({
             topic_id: topicID,
             subscriber_did: subscriberDID,
-            last_active_at: Date.now()
+            last_active_at: Date.now(),
+            retention
         }).onConflict((oc)=>oc.columns([
                 'topic_id',
                 'subscriber_did'
             ]).doUpdateSet({
-                last_active_at: (eb)=>eb.ref('excluded.last_active_at')
+                last_active_at: (eb)=>eb.ref('excluded.last_active_at'),
+                retention: (eb)=>eb.ref('excluded.retention')
             })).execute();
     }
     async function unsubscribe(subscriberDID, topicID) {
         const kysely = await db();
         await kysely.deleteFrom('kubun_hub_subscriptions').where('topic_id', '=', topicID).where('subscriber_did', '=', subscriberDID).execute();
-        // Drop this subscriber's pending deliveries for the topic, then GC any
-        // message left without a recipient (refcount → 0), mirroring ack cleanup.
+        // unsubscribe is a delivery operation, not a trim: it drops this subscriber's
+        // pending deliveries for the topic and GCs any MAILBOX frame left without a
+        // recipient. A log frame (which trim alone may remove) and the head are never
+        // touched.
         const pending = await kysely.selectFrom('kubun_hub_deliveries').innerJoin('kubun_hub_messages', 'kubun_hub_messages.sequence_id', 'kubun_hub_deliveries.sequence_id').select('kubun_hub_deliveries.sequence_id').where('kubun_hub_deliveries.recipient_did', '=', subscriberDID).where('kubun_hub_messages.topic_id', '=', topicID).execute();
         if (pending.length === 0) return;
         const sequenceIDs = pending.map((r)=>r.sequence_id);
@@ -164,7 +457,7 @@ export function createHubStore(params) {
         const stillDelivered = new Set(remaining.map((r)=>r.sequence_id));
         const orphaned = sequenceIDs.filter((id)=>!stillDelivered.has(id));
         if (orphaned.length > 0) {
-            await kysely.deleteFrom('kubun_hub_messages').where('sequence_id', 'in', orphaned).execute();
+            await kysely.deleteFrom('kubun_hub_messages').where('sequence_id', 'in', orphaned).where('retain', '=', 'mailbox').execute();
         }
     }
     async function getSubscribers(topicID) {
@@ -193,11 +486,16 @@ export function createHubStore(params) {
         events,
         publish,
         fetch,
+        fetchTopic,
         ack,
         purge,
+        trim,
         purgeSubscribers,
         storeKeyPackage,
         fetchKeyPackages,
+        countKeyPackages,
+        storeLastResortKeyPackage,
+        fetchLastResortKeyPackage,
         subscribe,
         unsubscribe,
         getSubscribers

package/lib/migrations.js

@@ -1,10 +1,12 @@
+import { sql } from 'kysely';
 export function createHubMigrations(ctx) {
     const t = ctx.types;
     return {
         '001_create_hub_tables': {
             async up (db) {
                 // Hub messages
-                await db.schema.createTable('kubun_hub_messages').ifNotExists().addColumn('sequence_id', t.text, (col)=>col.notNull().primaryKey()).addColumn('sender_did', t.text, (col)=>col.notNull()).addColumn('topic_id', t.text, (col)=>col.notNull()).addColumn('payload', t.binary, (col)=>col.notNull()).addColumn('stored_at', t.bigint, (col)=>col.notNull()).execute();
+                await db.schema.createTable('kubun_hub_messages').ifNotExists().addColumn('sequence_id', t.text, (col)=>col.notNull().primaryKey()).addColumn('sender_did', t.text, (col)=>col.notNull()).addColumn('topic_id', t.text, (col)=>col.notNull()).addColumn('payload', t.binary, (col)=>col.notNull()).addColumn('stored_at', t.bigint, (col)=>col.notNull())// Retention class: 'mailbox' is delivery-derived, 'log' is topic-retained.
+                .addColumn('retain', t.text, (col)=>col.notNull().defaultTo('mailbox')).execute();
                 // Hub deliveries
                 await db.schema.createTable('kubun_hub_deliveries').ifNotExists().addColumn('sequence_id', t.text, (col)=>col.notNull()).addColumn('recipient_did', t.text, (col)=>col.notNull()).addPrimaryKeyConstraint('pk_hub_deliveries', [
                     'sequence_id',
@@ -12,15 +14,50 @@ export function createHubMigrations(ctx) {
                 ]).execute();
                 await db.schema.createIndex('idx_hub_deliveries_recipient').ifNotExists().on('kubun_hub_deliveries').column('recipient_did').execute();
                 // Hub key packages
-                await db.schema.createTable('kubun_hub_key_packages').ifNotExists().addColumn('id', t.serial, (col)=>col.primaryKey()).addColumn('owner_did', t.text, (col)=>col.notNull()).addColumn('key_package', t.text, (col)=>col.notNull()).execute();
+                await db.schema.createTable('kubun_hub_key_packages').ifNotExists().addColumn('id', t.serial, (col)=>col.primaryKey()).addColumn('owner_did', t.text, (col)=>col.notNull()).addColumn('key_package', t.text, (col)=>col.notNull())// Expiry in SECONDS, unlike every millisecond column here. Null never expires.
+                .addColumn('not_after', t.bigint).execute();
                 await db.schema.createIndex('idx_hub_key_packages_owner').ifNotExists().on('kubun_hub_key_packages').column('owner_did').execute();
+                // The last-resort slot gets its OWN table so no ordinary-pool query can reach
+                // it: it is reusable and uncapped, and both properties invert the pool's. The
+                // primary key is what holds "one slot per owner".
+                await db.schema.createTable('kubun_hub_last_resort_key_packages').ifNotExists().addColumn('owner_did', t.text, (col)=>col.notNull().primaryKey()).addColumn('key_package', t.text, (col)=>col.notNull()).execute();
                 // Hub subscriptions
-                await db.schema.createTable('kubun_hub_subscriptions').ifNotExists().addColumn('topic_id', t.text, (col)=>col.notNull()).addColumn('subscriber_did', t.text, (col)=>col.notNull()).addColumn('last_active_at', t.bigint, (col)=>col.notNull().defaultTo(0)).addPrimaryKeyConstraint('pk_hub_subscriptions', [
+                await db.schema.createTable('kubun_hub_subscriptions').ifNotExists().addColumn('topic_id', t.text, (col)=>col.notNull()).addColumn('subscriber_did', t.text, (col)=>col.notNull()).addColumn('last_active_at', t.bigint, (col)=>col.notNull().defaultTo(0))// Requested retention (seconds). The topic's age bound is the longest any
+                // current subscriber asked for, floored at the hub default.
+                .addColumn('retention', t.bigint, (col)=>col.notNull().defaultTo(0)).addPrimaryKeyConstraint('pk_hub_subscriptions', [
                     'topic_id',
                     'subscriber_did'
                 ]).execute();
+                // Stored per-topic head: the sequenceID of the last accepted log publish.
+                // Stored state, NOT a projection — it outlives every frame, so a trim or
+                // purge that empties the log leaves it standing.
+                await db.schema.createTable('kubun_hub_topics').ifNotExists().addColumn('topic_id', t.text, (col)=>col.notNull().primaryKey()).addColumn('head', t.text, (col)=>col.notNull()).execute();
+                // Persistent, global, monotonic sequence counter. Persisted (not derived
+                // from surviving frames) so trimming or purging a topic to empty never
+                // regresses minting below a head the topic still names. Global so a
+                // sequenceID is unique across topics and the mailbox fetch stays globally
+                // ordered.
+                await db.schema.createTable('kubun_hub_sequence').ifNotExists().addColumn('id', t.bigint, (col)=>col.notNull().primaryKey()).addColumn('counter', t.bigint, (col)=>col.notNull()).execute();
+                await sql`INSERT INTO kubun_hub_sequence (id, counter) VALUES (0, 0)`.execute(db);
+                // Idempotency records: publishID -> sequenceID. Its OWN table, so trim and
+                // purge never remove it; its retention is its own (indefinite). Scoped per
+                // topic lane.
+                await db.schema.createTable('kubun_hub_publish_ids').ifNotExists().addColumn('topic_id', t.text, (col)=>col.notNull()).addColumn('publish_id', t.text, (col)=>col.notNull()).addColumn('sequence_id', t.text, (col)=>col.notNull())// Every commit publish writes a row here and nothing else ever deletes one,
+                // so on a long-lived hub this is the fastest-growing table, outliving frames
+                // purged years earlier. The contract only requires that trim and purge never
+                // take a record while the FRAME it dedups survives; keeping records forever
+                // was never part of it. This column is what lets `purge` sweep them on a
+                // window longer than any plausible replay.
+                .addColumn('recorded_at', t.bigint, (col)=>col.notNull().defaultTo(0)).addPrimaryKeyConstraint('pk_hub_publish_ids', [
+                    'topic_id',
+                    'publish_id'
+                ]).execute();
             },
             async down (db) {
+                await db.schema.dropTable('kubun_hub_publish_ids').ifExists().execute();
+                await db.schema.dropTable('kubun_hub_last_resort_key_packages').ifExists().execute();
+                await db.schema.dropTable('kubun_hub_sequence').ifExists().execute();
+                await db.schema.dropTable('kubun_hub_topics').ifExists().execute();
                 await db.schema.dropTable('kubun_hub_subscriptions').ifExists().execute();
                 await db.schema.dropTable('kubun_hub_key_packages').ifExists().execute();
                 await db.schema.dropTable('kubun_hub_deliveries').ifExists().execute();

package/lib/observers.js

@@ -13,20 +13,25 @@ export function decorateHubStore(store, observers) {
         events: store.events,
         storeKeyPackage: store.storeKeyPackage.bind(store),
         fetchKeyPackages: store.fetchKeyPackages.bind(store),
+        countKeyPackages: store.countKeyPackages.bind(store),
+        storeLastResortKeyPackage: store.storeLastResortKeyPackage.bind(store),
+        fetchLastResortKeyPackage: store.fetchLastResortKeyPackage.bind(store),
         subscribe: store.subscribe.bind(store),
         unsubscribe: store.unsubscribe.bind(store),
         getSubscribers: store.getSubscribers.bind(store),
         purgeSubscribers: store.purgeSubscribers.bind(store),
         purge: store.purge.bind(store),
+        fetchTopic: store.fetchTopic.bind(store),
+        trim: store.trim.bind(store),
         async publish (params) {
             try {
-                const sequenceID = await store.publish(params);
+                const result = await store.publish(params);
                 safeCall(onStore, {
                     senderDID: params.senderDID,
                     topicID: params.topicID,
-                    sequenceID
+                    sequenceID: result.sequenceID
                 });
-                return sequenceID;
+                return result;
             } catch (error) {
                 safeCall(onError, {
                     stage: 'store',

package/package.json

@@ -1,49 +1,51 @@
 {
   "name": "@kubun/hub",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "license": "see LICENSE.md",
+  "sideEffects": false,
   "type": "module",
-  "main": "lib/index.js",
-  "types": "lib/index.d.ts",
   "exports": {
     ".": "./lib/index.js"
   },
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
   "files": [
     "lib/*",
     "LICENSE.md"
   ],
-  "sideEffects": false,
   "dependencies": {
-    "@sozai/event": "^0.1.0",
-    "@enkaku/http-serve": "^0.18.1",
-    "@kumiai/hub-protocol": "^0.1.0",
-    "@kumiai/hub-server": "^0.1.0",
-    "@enkaku/server": "^0.18.1",
-    "@kokuin/token": "^0.1.1",
-    "@hono/node-server": "^2.0.8",
+    "@enkaku/http-serve": "^0.20.0",
+    "@enkaku/server": "^0.20.0",
+    "@hono/node-server": "^2.1.0",
+    "@kokuin/token": "^0.3.0",
+    "@kumiai/hub-protocol": "^0.5.0",
+    "@kumiai/hub-server": "^0.5.0",
+    "@sozai/event": "^0.1.3",
     "get-port": "^7.2.0",
-    "hono": "^4.12.27",
-    "kysely": "^0.29.2",
-    "@kubun/db-adapter": "^0.11.0",
-    "@kubun/http-util": "^0.11.0",
-    "@kubun/db-provider": "^0.11.0"
+    "hono": "^4.13.0",
+    "kysely": "^0.29.4",
+    "@kubun/db-adapter": "^0.12.0",
+    "@kubun/db-provider": "^0.12.0",
+    "@kubun/http-util": "^0.12.0"
   },
   "devDependencies": {
-    "@sozai/async": "^0.1.0",
-    "@enkaku/client": "^0.18.1",
-    "@kumiai/mls": "^0.1.0",
-    "@enkaku/http-fetch": "^0.18.1",
-    "@enkaku/protocol": "^0.18.1",
-    "@enkaku/transport": "^0.18.1",
-    "@testcontainers/postgresql": "^12.0.4",
-    "@kubun/db-better-sqlite": "^0.11.0",
-    "@kubun/db-postgres": "^0.11.0",
-    "@kubun/db-node-sqlite": "^0.11.0"
+    "@enkaku/client": "^0.20.0",
+    "@enkaku/http-fetch": "^0.20.0",
+    "@enkaku/protocol": "^0.20.0",
+    "@enkaku/transport": "^0.20.0",
+    "@kumiai/hub-conformance": "^0.5.0",
+    "@kumiai/mls": "^0.5.0",
+    "@sozai/async": "^0.2.1",
+    "@testcontainers/postgresql": "^12.1.0",
+    "@kubun/db-better-sqlite": "^0.12.0",
+    "@kubun/db-node-sqlite": "^0.12.0",
+    "@kubun/db-postgres": "^0.12.0"
   },
   "scripts": {
+    "build": "pnpm run build:clean && pnpm run build:js && pnpm run build:types",
     "build:clean": "del lib",
     "build:js": "swc src -d ./lib --config-file ../../node_modules/@kigu/dev/swc.json --strip-leading-paths",
     "build:types": "tsc --emitDeclarationOnly --skipLibCheck",
-    "build": "pnpm run build:clean && pnpm run build:js && pnpm run build:types"
+    "test:types": "tsc --noEmit -p tsconfig.test.json"
   }
 }