@kubun/hub 0.17.3 → 0.18.0

package/lib/hub-store.d.ts

@@ -1,4 +1,4 @@
-import type { Adapter } from '@kubun/db-adapter';
+import { type Adapter } from '@kubun/db-adapter';
 import { type HubStore } from '@kumiai/hub-protocol';
 import type { ColumnType, Kysely } from 'kysely';
 export type HubDatabase = {
@@ -33,6 +33,7 @@ export type HubDatabase = {
     kubun_hub_topics: {
         topic_id: string;
         head: string;
+        removed_through: ColumnType<string | null, string | null | undefined, string | null>;
     };
     kubun_hub_sequence: {
         id: number;
@@ -48,6 +49,7 @@ export type HubDatabase = {
 export type CreateHubStoreParams = {
     kysely: Kysely<HubDatabase> | Promise<Kysely<HubDatabase>>;
     adapter: Adapter;
+    now?: () => number;
     defaultRetention?: number;
     maxRetention?: number;
     maxDepth?: number;

package/lib/hub-store.js

@@ -1,3 +1,4 @@
+import { AbstractPostgresAdapter } from '@kubun/db-adapter';
 import { HeadMismatchError, KeyPackageQuotaExceededError, NotSubscribedError, RetentionExceededError, SubscriptionQuotaExceededError } from '@kumiai/hub-protocol';
 import { EventEmitter } from '@sozai/event';
 // Normalize a DB binary column (a Buffer under better-sqlite3 / node-postgres)
@@ -18,6 +19,7 @@ function toBytes(value) {
 const DEFAULT_MAX_SUBSCRIPTIONS_PER_DID = 1000;
 export function createHubStore(params) {
     const { adapter } = params;
+    const now = params.now ?? Date.now;
     const defaultRetention = params.defaultRetention ?? 0;
     const maxRetention = params.maxRetention;
     const maxDepth = params.maxDepth;
@@ -39,11 +41,24 @@ export function createHubStore(params) {
    * variables" past its limit (32766 here, 999 on older builds), so an unchunked
    * sweep fails precisely when the backlog is large. Deliveries first: a delivery
    * references a frame and cannot be pushed once its referent is gone.
-   */ async function deleteFrames(executor, sequenceIDs) {
+   */ async function deleteFrames(trx, sequenceIDs) {
         for(let index = 0; index < sequenceIDs.length; index += DELETE_CHUNK_SIZE){
             const chunk = sequenceIDs.slice(index, index + DELETE_CHUNK_SIZE);
-            await executor.deleteFrom('kubun_hub_deliveries').where('sequence_id', 'in', chunk).execute();
-            await executor.deleteFrom('kubun_hub_messages').where('sequence_id', 'in', chunk).execute();
+            const removedPrefixes = await trx.selectFrom('kubun_hub_messages').select((eb)=>[
+                    'topic_id',
+                    eb.fn.max('sequence_id').as('removed_through')
+                ]).where('sequence_id', 'in', chunk).where('retain', '=', 'log').groupBy('topic_id').orderBy('topic_id').execute();
+            // Take the topic lock before deleting frames, as publication does during depth eviction.
+            for (const { topic_id, removed_through } of removedPrefixes){
+                await trx.updateTable('kubun_hub_topics').set({
+                    removed_through
+                }).where('topic_id', '=', topic_id).where((eb)=>eb.or([
+                        eb('removed_through', 'is', null),
+                        eb('removed_through', '<', removed_through)
+                    ])).execute();
+            }
+            await trx.deleteFrom('kubun_hub_deliveries').where('sequence_id', 'in', chunk).execute();
+            await trx.deleteFrom('kubun_hub_messages').where('sequence_id', 'in', chunk).execute();
         }
     }
     // Mint the next sequenceID inside the accepting transaction, from a persistent
@@ -115,7 +130,7 @@ export function createHubStore(params) {
                     sender_did: senderDID,
                     topic_id: topicID,
                     payload: adapter.encodeBinary(publishParams.payload),
-                    stored_at: Date.now(),
+                    stored_at: now(),
                     retain
                 }).execute();
                 // Push is untouched by the class: a log frame is delivered to every current
@@ -172,7 +187,7 @@ export function createHubStore(params) {
                     topic_id: topicID,
                     publish_id: publishParams.publishID,
                     sequence_id: sequenceID,
-                    recorded_at: Date.now()
+                    recorded_at: now()
                 }).execute();
             }
             return {
@@ -236,48 +251,52 @@ export function createHubStore(params) {
     async function fetchTopic(fetchParams) {
         const kysely = await db();
         const { subscriberDID, topicID } = fetchParams;
-        // Authorization: the caller must be a current subscriber of the topic.
-        const subscription = await kysely.selectFrom('kubun_hub_subscriptions').select('subscriber_did').where('topic_id', '=', topicID).where('subscriber_did', '=', subscriberDID).executeTakeFirst();
-        if (subscription == null) {
-            throw new NotSubscribedError(`${subscriberDID} is not subscribed to ${topicID}`);
-        }
-        // Head is stored state — it outlives every frame.
-        const topicRow = await kysely.selectFrom('kubun_hub_topics').select('head').where('topic_id', '=', topicID).executeTakeFirst();
-        const head = topicRow?.head ?? null;
-        // Oldest is the oldest RETAINED log frame, or null if the log is empty.
-        const oldestRow = await kysely.selectFrom('kubun_hub_messages').select((eb)=>eb.fn.min('sequence_id').as('oldest')).where('topic_id', '=', topicID).where('retain', '=', 'log').executeTakeFirst();
-        const oldest = oldestRow?.oldest ?? null;
-        // The log is the topic's log-class frames and NOTHING else — a mailbox frame
-        // never appears here. `limit` counts log frames (applied after the class
-        // filter), so a page of mailbox frames cannot hand a draining reader an empty
-        // page while log frames still wait.
-        let query = kysely.selectFrom('kubun_hub_messages').select([
-            'sequence_id',
-            'sender_did',
-            'topic_id',
-            'payload'
-        ]).where('topic_id', '=', topicID).where('retain', '=', 'log').orderBy('sequence_id', 'asc');
-        if (fetchParams.after != null) {
-            query = query.where('sequence_id', '>', fetchParams.after);
-        }
-        if (fetchParams.limit != null) {
-            query = query.limit(fetchParams.limit);
-        }
-        const rows = await query.execute();
-        // Every row here is log-class by the filter above, so each carries its position
-        // unconditionally — the same value a push of that frame reports.
-        const messages = rows.map((row)=>({
-                sequenceID: row.sequence_id,
-                senderDID: row.sender_did,
-                topicID: row.topic_id,
-                payload: toBytes(row.payload),
-                logPosition: row.sequence_id
-            }));
-        return {
-            messages,
-            head,
-            oldest
-        };
+        // READ COMMITTED takes a new snapshot per statement and can conceal a concurrent trim.
+        return await kysely.transaction().setIsolationLevel('repeatable read').setAccessMode('read only').execute(async (trx)=>{
+            // Authorization: the caller must be a current subscriber of the topic.
+            const subscription = await trx.selectFrom('kubun_hub_subscriptions').select('subscriber_did').where('topic_id', '=', topicID).where('subscriber_did', '=', subscriberDID).executeTakeFirst();
+            if (subscription == null) {
+                throw new NotSubscribedError(`${subscriberDID} is not subscribed to ${topicID}`);
+            }
+            // Head and the removal watermark outlive every frame.
+            const topicRow = await trx.selectFrom('kubun_hub_topics').select([
+                'head',
+                'removed_through'
+            ]).where('topic_id', '=', topicID).executeTakeFirst();
+            const head = topicRow?.head ?? null;
+            const removedThrough = topicRow?.removed_through ?? null;
+            const gap = removedThrough != null && (fetchParams.after == null || removedThrough > fetchParams.after);
+            // Oldest is the oldest retained log frame, or null if the log is empty.
+            const oldestRow = await trx.selectFrom('kubun_hub_messages').select((eb)=>eb.fn.min('sequence_id').as('oldest')).where('topic_id', '=', topicID).where('retain', '=', 'log').executeTakeFirst();
+            const oldest = oldestRow?.oldest ?? null;
+            // Mailbox frames cannot consume a page's log limit.
+            let query = trx.selectFrom('kubun_hub_messages').select([
+                'sequence_id',
+                'sender_did',
+                'topic_id',
+                'payload'
+            ]).where('topic_id', '=', topicID).where('retain', '=', 'log').orderBy('sequence_id', 'asc');
+            if (fetchParams.after != null) {
+                query = query.where('sequence_id', '>', fetchParams.after);
+            }
+            if (fetchParams.limit != null) {
+                query = query.limit(fetchParams.limit);
+            }
+            const rows = await query.execute();
+            const messages = rows.map((row)=>({
+                    sequenceID: row.sequence_id,
+                    senderDID: row.sender_did,
+                    topicID: row.topic_id,
+                    payload: toBytes(row.payload),
+                    logPosition: row.sequence_id
+                }));
+            return {
+                messages,
+                head,
+                oldest,
+                gap
+            };
+        });
     }
     async function ack(ackParams) {
         if (ackParams.sequenceIDs.length === 0) return;
@@ -296,23 +315,26 @@ export function createHubStore(params) {
     async function trim(trimParams) {
         const kysely = await db();
         // Remove LOG frames strictly below the bound, and only those. A mailbox frame
-        // on the same topic is delivery-derived and untouched. The head (topics table)
-        // and the publishID records are never touched.
-        const logRows = await kysely.selectFrom('kubun_hub_messages').select('sequence_id').where('topic_id', '=', trimParams.topicID).where('retain', '=', 'log').where('sequence_id', '<', trimParams.before).execute();
-        const sequenceIDs = logRows.map((r)=>r.sequence_id);
-        if (sequenceIDs.length === 0) return;
-        await deleteFrames(kysely, sequenceIDs);
+        // on the same topic is delivery-derived and untouched. The stored head and
+        // publishID records are never touched.
+        await kysely.transaction().execute(async (trx)=>{
+            // Match purge's topic-before-frame lock order across every deletion chunk.
+            const topicQuery = trx.selectFrom('kubun_hub_topics').select('head').where('topic_id', '=', trimParams.topicID);
+            await (adapter instanceof AbstractPostgresAdapter ? topicQuery.forUpdate() : topicQuery).executeTakeFirst();
+            const logRows = await trx.selectFrom('kubun_hub_messages').select('sequence_id').where('topic_id', '=', trimParams.topicID).where('retain', '=', 'log').where('sequence_id', '<', trimParams.before).orderBy('sequence_id', 'asc').execute();
+            await deleteFrames(trx, logRows.map((row)=>row.sequence_id));
+        });
     }
     async function purge(purgeParams) {
         const kysely = await db();
-        const now = Date.now();
+        const purgeTime = now();
         const olderThan = purgeParams.olderThan;
         // Idempotency records outlive the frames they dedup, on their own clock: the
         // contract is that no message deleter reaches one for a frame it still
         // guards, NOT that they are kept forever. Every commit publish writes one, so
         // without this the table is the fastest-growing on a long-lived hub. The
         // window far exceeds any plausible replay (see PUBLISH_ID_RETENTION_SECONDS).
-        await kysely.deleteFrom('kubun_hub_publish_ids').where('recorded_at', '<=', now - PUBLISH_ID_RETENTION_SECONDS * 1000).execute();
+        await kysely.deleteFrom('kubun_hub_publish_ids').where('recorded_at', '<=', purgeTime - PUBLISH_ID_RETENTION_SECONDS * 1000).execute();
         // Per topic, the effective age bound is the longest retention any CURRENT
         // subscriber asked for, floored at the hub default (and at the sweep's
         // olderThan). Applies to both classes. Never touches the head or a publishID
@@ -330,25 +352,66 @@ export function createHubStore(params) {
                 Number(row.maxRetention ?? 0)
             ]));
         const removed = [];
-        for (const { topic_id } of topicRows){
-            const subscriberRetention = retentionByTopic.get(topic_id) ?? 0;
-            const effective = Math.max(olderThan, subscriberRetention, defaultRetention);
-            const threshold = now - effective * 1000;
-            const frames = await kysely.selectFrom('kubun_hub_messages').select('sequence_id').where('topic_id', '=', topic_id).where('stored_at', '<=', threshold).execute();
-            for (const frame of frames){
-                removed.push(frame.sequence_id);
+        try {
+            for (const { topic_id } of topicRows){
+                const subscriberRetention = retentionByTopic.get(topic_id) ?? 0;
+                const effective = Math.max(olderThan, subscriberRetention, defaultRetention);
+                const threshold = purgeTime - effective * 1000;
+                // One topic per transaction releases SQLite's connection between topics.
+                const topicRemoved = await kysely.transaction().setIsolationLevel('read committed').execute(async (trx)=>{
+                    // Publication takes this row before committing its head and depth eviction.
+                    // SQLite's single connection already serializes the transaction.
+                    const topicQuery = trx.selectFrom('kubun_hub_topics').select('head').where('topic_id', '=', topic_id);
+                    const topicRow = await (adapter instanceof AbstractPostgresAdapter ? topicQuery.forUpdate() : topicQuery).executeTakeFirst();
+                    // Clock regression must not punch a hole behind the first retained frame.
+                    const retainedLog = await trx.selectFrom('kubun_hub_messages').select((eb)=>eb.fn.min('sequence_id').as('cut')).where('topic_id', '=', topic_id).where('retain', '=', 'log').where('stored_at', '>', threshold).executeTakeFirstOrThrow();
+                    const deletionCandidates = trx.selectFrom('kubun_hub_messages').select('sequence_id').where('topic_id', '=', topic_id).where((eb)=>{
+                        const mailbox = eb.and([
+                            eb('retain', '=', 'mailbox'),
+                            eb('stored_at', '<=', threshold)
+                        ]);
+                        // A mailbox-only topic has no lockable row. Its first log must survive,
+                        // and a null cut cannot admit frames newer than the captured head.
+                        if (topicRow == null) return mailbox;
+                        const log = [
+                            eb('retain', '=', 'log'),
+                            eb('sequence_id', '<=', topicRow.head)
+                        ];
+                        if (retainedLog.cut != null) {
+                            log.push(eb('sequence_id', '<', retainedLog.cut));
+                        }
+                        return eb.or([
+                            eb.and(log),
+                            mailbox
+                        ]);
+                    }).orderBy('sequence_id', 'asc').limit(DELETE_CHUNK_SIZE);
+                    const sequenceIDs = [];
+                    while(true){
+                        const frames = await deletionCandidates.execute();
+                        if (frames.length === 0) break;
+                        const chunk = frames.map((frame)=>frame.sequence_id);
+                        await deleteFrames(trx, chunk);
+                        for (const sequenceID of chunk){
+                            sequenceIDs.push(sequenceID);
+                        }
+                        if (frames.length < DELETE_CHUNK_SIZE) break;
+                    }
+                    return sequenceIDs;
+                });
+                for (const sequenceID of topicRemoved){
+                    removed.push(sequenceID);
+                }
             }
+        } finally{
+            if (removed.length > 0) await events.emit('purge', {
+                sequenceIDs: removed
+            });
         }
-        if (removed.length === 0) return [];
-        await deleteFrames(kysely, removed);
-        await events.emit('purge', {
-            sequenceIDs: removed
-        });
         return removed;
     }
     // `not_after` is seconds; every other timestamp here is milliseconds.
     function nowSeconds() {
-        return Math.floor(Date.now() / 1000);
+        return Math.floor(now() / 1000);
     }
     async function deleteExpiredKeyPackages(kysely, ownerDID) {
         await kysely.deleteFrom('kubun_hub_key_packages').where('owner_did', '=', ownerDID).where('not_after', 'is not', null).where('not_after', '<=', nowSeconds()).execute();
@@ -427,7 +490,7 @@ export function createHubStore(params) {
         await kysely.insertInto('kubun_hub_subscriptions').values({
             topic_id: topicID,
             subscriber_did: subscriberDID,
-            last_active_at: Date.now(),
+            last_active_at: now(),
             retention
         }).onConflict((oc)=>oc.columns([
                 'topic_id',
@@ -464,7 +527,7 @@ export function createHubStore(params) {
         const kysely = await db();
         // Strict `<`: a row refreshed exactly at the boundary survives. Subscription
         // retention runs on a much longer window than message purge.
-        const threshold = Date.now() - purgeParams.olderThan * 1000;
+        const threshold = now() - purgeParams.olderThan * 1000;
         // Atomic delete-and-return so the reported set exactly matches the rows
         // actually removed — a subscriber who rejoins between evaluating and deleting
         // is neither reaped nor over-reported. Mirrors `fetchKeyPackages`.

package/lib/migrations.js

@@ -31,7 +31,7 @@ export function createHubMigrations(ctx) {
                 // Stored per-topic head: the sequenceID of the last accepted log publish.
                 // Stored state, NOT a projection — it outlives every frame, so a trim or
                 // purge that empties the log leaves it standing.
-                await db.schema.createTable('kubun_hub_topics').ifNotExists().addColumn('topic_id', t.text, (col)=>col.notNull().primaryKey()).addColumn('head', t.text, (col)=>col.notNull()).execute();
+                await db.schema.createTable('kubun_hub_topics').ifNotExists().addColumn('topic_id', t.text, (col)=>col.notNull().primaryKey()).addColumn('head', t.text, (col)=>col.notNull()).addColumn('removed_through', t.text).execute();
                 // Persistent, global, monotonic sequence counter. Persisted (not derived
                 // from surviving frames) so trimming or purging a topic to empty never
                 // regresses minting below a head the topic still names. Global so a

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kubun/hub",
-  "version": "0.17.3",
+  "version": "0.18.0",
   "license": "see LICENSE.md",
   "sideEffects": false,
   "type": "module",
@@ -14,32 +14,32 @@
     "LICENSE.md"
   ],
   "dependencies": {
-    "@enkaku/http-serve": "^0.21.0",
-    "@enkaku/server": "^0.21.2",
-    "@hono/node-server": "^2.1.1",
-    "@kokuin/token": "^0.5.0",
-    "@kubun/db-adapter": "^0.17.0",
-    "@kubun/db-provider": "^0.17.0",
-    "@kubun/http-util": "^0.17.0",
-    "@kumiai/hub-protocol": "^0.9.0",
-    "@kumiai/hub-server": "^0.9.0",
-    "@sozai/event": "^0.1.3",
+    "@enkaku/http-serve": "^0.21.5",
+    "@enkaku/server": "^0.21.5",
+    "@hono/node-server": "^2.1.5",
+    "@kokuin/token": "^0.5.1",
+    "@kubun/db-adapter": "^0.18.0",
+    "@kubun/db-provider": "^0.18.0",
+    "@kubun/http-util": "^0.18.0",
+    "@kumiai/hub-protocol": "^0.10.2",
+    "@kumiai/hub-server": "^0.10.4",
+    "@sozai/event": "^0.1.6",
     "get-port": "^7.2.0",
-    "hono": "^4.13.7",
-    "kysely": "^0.29.5"
+    "hono": "^4.13.13",
+    "kysely": "^0.29.7"
   },
   "devDependencies": {
-    "@enkaku/client": "^0.21.2",
-    "@enkaku/http-fetch": "^0.21.0",
-    "@enkaku/protocol": "^0.21.1",
-    "@enkaku/transport": "^0.21.0",
-    "@kubun/db-better-sqlite": "^0.17.0",
-    "@kubun/db-node-sqlite": "^0.17.0",
-    "@kubun/db-postgres": "^0.17.0",
-    "@kumiai/hub-conformance": "^0.9.0",
-    "@kumiai/mls": "^0.9.0",
-    "@sozai/async": "^0.2.1",
-    "@testcontainers/postgresql": "^12.1.0"
+    "@enkaku/client": "^0.21.5",
+    "@enkaku/http-fetch": "^0.21.5",
+    "@enkaku/protocol": "^0.21.5",
+    "@enkaku/transport": "^0.21.5",
+    "@kubun/db-better-sqlite": "^0.18.0",
+    "@kubun/db-node-sqlite": "^0.18.0",
+    "@kubun/db-postgres": "^0.18.0",
+    "@kumiai/hub-conformance": "^0.10.1",
+    "@kumiai/mls": "^0.10.9",
+    "@sozai/async": "^0.4.1",
+    "@testcontainers/postgresql": "^12.2.1"
   },
   "publishConfig": {
     "access": "public"
@@ -49,6 +49,9 @@
     "build:clean": "del lib",
     "build:js": "del 'lib/**/*.js' && swc src -d ./lib --config-file ../../node_modules/@kigu/dev/swc.json --strip-leading-paths",
     "build:types": "del 'lib/**/*.d.ts' 'lib/**/*.d.ts.map' && tsc --emitDeclarationOnly --skipLibCheck",
-    "test:types": "tsc --noEmit -p tsconfig.test.json"
+    "build:types:ci": "tsc --emitDeclarationOnly --declarationMap false",
+    "test": "pnpm run test:types && pnpm run test:unit",
+    "test:types": "tsc --noEmit",
+    "test:unit": "vitest run"
   }
 }