@haathie/pgmb 0.2.0 → 0.2.1

package/{readme.md → README.md}

@@ -124,6 +124,19 @@ await pgmb.registerReliableHandler(
 		retryOpts: {
 			// will retry after 1 minute, then after 5 minutes
 			retriesS: [60, 5 * 60]
+		},
+		// optionally provide a splitBy function to split
+		// event to be processed differently based on some attribute.
+		splitBy(ev) {
+			return Object.values(
+				// group events by their topic
+				ev.items.reduce((acc, item) => {
+					const key = item.topic
+					acc[key] ||= { items: [] }
+					acc[key].items.push(item)
+					return acc
+				}, {})
+			)
 		}
 	},
 	async({ items }, { logger }) => {

package/lib/client.d.ts

@@ -1,12 +1,13 @@
 import { type Logger } from 'pino';
 import { PGMBEventBatcher } from './batcher.ts';
 import type { PgClientLike } from './query-types.ts';
-import type { GetWebhookInfoFn, IEphemeralListener, IEventData, IEventHandler, IReadEvent, Pgmb2ClientOpts, registerReliableHandlerParams, RegisterSubscriptionParams } from './types.ts';
+import type { GetWebhookInfoFn, IEphemeralListener, IEventData, IEventHandler, IReadEvent, IReadNextEventsFn, ISplitFn, Pgmb2ClientOpts, registerReliableHandlerParams, RegisterSubscriptionParams } from './types.ts';
 type IReliableListener<T extends IEventData> = {
     type: 'reliable';
     handler: IEventHandler<T>;
     removeOnEmpty?: boolean;
     extra?: unknown;
+    splitBy?: ISplitFn<T>;
     queue: {
         item: IReadEvent<T>;
         checkpoint: Checkpoint;
@@ -37,12 +38,13 @@ export declare class PgmbClient<T extends IEventData = IEventData> extends PGMBE
     readonly subscriptionMaintenanceMs: number;
     readonly tableMaintenanceMs: number;
     readonly maxActiveCheckpoints: number;
+    readonly readNextEvents: IReadNextEventsFn;
     readonly getWebhookInfo: GetWebhookInfoFn;
-    readonly webhookHandler: IEventHandler;
+    readonly webhookHandler: IEventHandler<T>;
     readonly listeners: {
         [subId: string]: IListenerStore<T>;
     };
-    constructor({ client, groupId, logger, sleepDurationMs, readChunkSize, maxActiveCheckpoints, poll, subscriptionMaintenanceMs, webhookHandlerOpts, getWebhookInfo, tableMaintainanceMs, ...batcherOpts }: Pgmb2ClientOpts);
+    constructor({ client, groupId, logger, sleepDurationMs, readChunkSize, maxActiveCheckpoints, poll, subscriptionMaintenanceMs, webhookHandlerOpts: { splitBy: whSplitBy, ...whHandlerOpts }, getWebhookInfo, tableMaintainanceMs, readNextEvents, ...batcherOpts }: Pgmb2ClientOpts<T>);
     init(): Promise<void>;
     end(): Promise<void>;
     publish(events: T[], client?: PgClientLike): Promise<import("./queries.ts").IWriteEventsResult[]>;
@@ -63,7 +65,7 @@ export declare class PgmbClient<T extends IEventData = IEventData> extends PGMBE
      * to retry failed events by the handler itself, allowing for delayed retries
      * with backoff, and without disrupting the overall event flow.
      */
-    registerReliableHandler({ retryOpts, name, ...opts }: registerReliableHandlerParams, handler: IEventHandler<T>): Promise<{
+    registerReliableHandler({ retryOpts, name, splitBy, ...opts }: registerReliableHandlerParams<T>, handler: IEventHandler<T>): Promise<{
         subscriptionId: string;
         cancel: () => void;
     }>;

package/lib/client.js

@@ -21,9 +21,11 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
     subscriptionMaintenanceMs;
     tableMaintenanceMs;
     maxActiveCheckpoints;
+    readNextEvents;
     getWebhookInfo;
     webhookHandler;
     listeners = {};
+    #webhookHandlerOpts;
     #readClient;
     #endAc = new AbortController();
     #shouldPoll;
@@ -33,11 +35,11 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
     #tableMaintainTask;
     #inMemoryCursor = null;
     #activeCheckpoints = [];
-    constructor({ client, groupId, logger = (0, pino_1.pino)(), sleepDurationMs = 750, readChunkSize = 1000, maxActiveCheckpoints = 10, poll, subscriptionMaintenanceMs = 60 * 1000, webhookHandlerOpts = {}, getWebhookInfo = () => ({}), tableMaintainanceMs = 5 * 60 * 1000, ...batcherOpts }) {
+    constructor({ client, groupId, logger = (0, pino_1.pino)(), sleepDurationMs = 750, readChunkSize = 1000, maxActiveCheckpoints = 10, poll, subscriptionMaintenanceMs = 60 * 1000, webhookHandlerOpts: { splitBy: whSplitBy, ...whHandlerOpts } = {}, getWebhookInfo = () => ({}), tableMaintainanceMs = 5 * 60 * 1000, readNextEvents = queries_ts_1.readNextEvents.run.bind(queries_ts_1.readNextEvents), ...batcherOpts }) {
         super({
             ...batcherOpts,
             logger,
-            publish: (...e) => this.publish(e)
+            publish: (...e) => this.publish(e),
         });
         this.client = client;
         this.logger = logger;
@@ -47,9 +49,11 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         this.#shouldPoll = !!poll;
         this.subscriptionMaintenanceMs = subscriptionMaintenanceMs;
         this.maxActiveCheckpoints = maxActiveCheckpoints;
-        this.webhookHandler = (0, webhook_handler_ts_1.createWebhookHandler)(webhookHandlerOpts);
+        this.webhookHandler = (0, webhook_handler_ts_1.createWebhookHandler)(whHandlerOpts);
+        this.#webhookHandlerOpts = { splitBy: whSplitBy };
         this.getWebhookInfo = getWebhookInfo;
         this.tableMaintenanceMs = tableMaintainanceMs;
+        this.readNextEvents = readNextEvents;
     }
     async init() {
         this.#endAc = new AbortController();
@@ -62,11 +66,9 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         await queries_ts_1.assertGroup.run({ id: this.groupId }, this.client);
         this.logger.debug({ groupId: this.groupId }, 'asserted group exists');
         // clean up expired subscriptions on start
-        const [{ deleted }] = await queries_ts_1.removeExpiredSubscriptions
-            .run({ groupId: this.groupId, activeIds: [] }, this.client);
+        const [{ deleted }] = await queries_ts_1.removeExpiredSubscriptions.run({ groupId: this.groupId, activeIds: [] }, this.client);
         this.logger.debug({ deleted }, 'removed expired subscriptions');
-        this.#readTask
-            = this.#startLoop(this.readChanges.bind(this), this.sleepDurationMs);
+        this.#readTask = this.#startLoop(this.readChanges.bind(this), this.sleepDurationMs);
         if (this.#shouldPoll) {
             this.#pollTask = this.#startLoop(queries_ts_1.pollForEvents.run.bind(queries_ts_1.pollForEvents, undefined, this.client), this.sleepDurationMs);
         }
@@ -74,8 +76,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
             this.#subMaintainTask = this.#startLoop(this.#maintainSubscriptions, this.subscriptionMaintenanceMs);
         }
         if (this.tableMaintenanceMs) {
-            this.#tableMaintainTask = this.#startLoop(queries_ts_1.maintainEventsTable.run
-                .bind(queries_ts_1.maintainEventsTable, undefined, this.client), this.tableMaintenanceMs);
+            this.#tableMaintainTask = this.#startLoop(queries_ts_1.maintainEventsTable.run.bind(queries_ts_1.maintainEventsTable, undefined, this.client), this.tableMaintenanceMs);
         }
     }
     async end() {
@@ -91,7 +92,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
             this.#readTask,
             this.#pollTask,
             this.#subMaintainTask,
-            this.#tableMaintainTask
+            this.#tableMaintainTask,
         ]);
         await this.#unlockAndReleaseReadClient();
         this.#readTask = undefined;
@@ -101,14 +102,13 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
     }
     publish(events, client = this.client) {
         return queries_ts_1.writeEvents.run({
-            topics: events.map(e => e.topic),
-            payloads: events.map(e => e.payload),
-            metadatas: events.map(e => e.metadata || null),
+            topics: events.map((e) => e.topic),
+            payloads: events.map((e) => e.payload),
+            metadatas: events.map((e) => e.metadata || null),
         }, client);
     }
     async assertSubscription(opts, client = this.client) {
-        const [rslt] = await queries_ts_1.assertSubscription
-            .run({ ...opts, groupId: this.groupId }, client);
+        const [rslt] = await queries_ts_1.assertSubscription.run({ ...opts, groupId: this.groupId }, client);
         this.logger.debug({ ...opts, ...rslt }, 'asserted subscription');
         return rslt;
     }
@@ -131,18 +131,23 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
      * to retry failed events by the handler itself, allowing for delayed retries
      * with backoff, and without disrupting the overall event flow.
      */
-    async registerReliableHandler({ retryOpts, name = createListenerId(), ...opts }, handler) {
+    async registerReliableHandler({ retryOpts, name = createListenerId(), splitBy, ...opts }, handler) {
         const { id: subId } = await this.assertSubscription(opts);
         if (retryOpts) {
             handler = (0, retry_handler_ts_1.createRetryHandler)(retryOpts, handler);
         }
         const lts = (this.listeners[subId] ||= { values: {} });
-        (0, assert_1.default)(!lts.values[name], `Handler with id ${name} already registered for subscription ${subId}.`
-            + ' Cancel the existing one or use a different id.');
-        this.listeners[subId].values[name] = { type: 'reliable', handler, queue: [] };
+        (0, assert_1.default)(!lts.values[name], `Handler with id ${name} already registered for subscription ${subId}.` +
+            ' Cancel the existing one or use a different id.');
+        this.listeners[subId].values[name] = {
+            type: 'reliable',
+            handler,
+            splitBy,
+            queue: [],
+        };
         return {
             subscriptionId: subId,
-            cancel: () => this.#removeListener(subId, name)
+            cancel: () => this.#removeListener(subId, name),
         };
     }
     async removeSubscription(subId) {
@@ -153,8 +158,8 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         if (!existingSubs) {
             return;
         }
-        await Promise.allSettled(Object.values(existingSubs).map(e => (e.type === 'fire-and-forget'
-            && e.stream.throw(new Error('subscription removed')))));
+        await Promise.allSettled(Object.values(existingSubs).map((e) => e.type === 'fire-and-forget' &&
+            e.stream.throw(new Error('subscription removed'))));
     }
     #listenForEvents(subId) {
         const lid = createListenerId();
@@ -178,8 +183,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         const activeIds = Object.keys(this.listeners);
         await queries_ts_1.markSubscriptionsActive.run({ ids: activeIds }, this.client);
         this.logger.trace({ activeSubscriptions: activeIds.length }, 'marked subscriptions as active');
-        const [{ deleted }] = await queries_ts_1.removeExpiredSubscriptions
-            .run({ groupId: this.groupId, activeIds }, this.client);
+        const [{ deleted }] = await queries_ts_1.removeExpiredSubscriptions.run({ groupId: this.groupId, activeIds }, this.client);
         this.logger.trace({ deleted }, 'removed expired subscriptions');
     }
     async readChanges() {
@@ -188,10 +192,10 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         }
         const now = Date.now();
         await this.#connectReadClient();
-        const rows = await queries_ts_1.readNextEvents.run({
+        const rows = await this.readNextEvents({
             groupId: this.groupId,
             cursor: this.#inMemoryCursor,
-            chunkSize: this.readChunkSize
+            chunkSize: this.readChunkSize,
         }, this.#readClient || this.client)
             .catch(async (err) => {
             if (err instanceof Error && err.message.includes('connection error')) {
@@ -207,7 +211,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
             }
             return 0;
         }
-        const uqSubIds = Array.from(new Set(rows.flatMap(r => r.subscriptionIds)));
+        const uqSubIds = Array.from(new Set(rows.flatMap((r) => r.subscriptionIds)));
         const webhookSubs = await this.getWebhookInfo(uqSubIds);
         let webhookCount = 0;
         for (const sid in webhookSubs) {
@@ -220,14 +224,18 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
                     queue: [],
                     extra: wh,
                     removeOnEmpty: true,
-                    handler: this.webhookHandler
+                    handler: this.webhookHandler,
+                    ...this.#webhookHandlerOpts
                 };
                 webhookCount++;
             }
         }
-        const { map: subToEventMap, retryEvents, retryItemCount } = await (0, retry_handler_ts_1.normaliseRetryEventsInReadEventMap)(rows, this.client);
+        const { map: subToEventMap, retryEvents, retryItemCount, } = await (0, retry_handler_ts_1.normaliseRetryEventsInReadEventMap)(rows, this.client);
         const subs = Object.entries(subToEventMap);
-        const checkpoint = { activeTasks: 0, nextCursor: rows[0].nextCursor };
+        const checkpoint = {
+            activeTasks: 0,
+            nextCursor: rows[0].nextCursor,
+        };
         for (const [subId, evs] of subs) {
             const listeners = this.listeners[subId]?.values;
             if (!listeners) {
@@ -257,7 +265,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
             activeCheckpoints: this.#activeCheckpoints.length,
             webhookCount,
             retryEvents,
-            retryItemCount
+            retryItemCount,
         }, 'read rows');
         if (!checkpoint.activeTasks && this.#activeCheckpoints.length === 1) {
             await this.#updateCursorFromCompletedCheckpoints();
@@ -272,7 +280,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
     async #enqueueEventInReliableListener(subId, lid, item, checkpoint) {
         const lt = this.listeners[subId]?.values?.[lid];
         (0, assert_1.default)(lt?.type === 'reliable', 'invalid listener type: ' + lt.type);
-        const { handler, queue, removeOnEmpty, extra } = lt;
+        const { handler, queue, removeOnEmpty, extra, splitBy = defaultSplitBy } = lt;
         queue.push({ item, checkpoint });
         checkpoint.activeTasks++;
         if (queue.length > 1) {
@@ -286,7 +294,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
             }
             const logger = this.logger.child({
                 subId,
-                items: item.items.map(i => i.id),
+                items: item.items.map((i) => i.id),
                 extra,
                 retryNumber: item.retry?.retryNumber,
             });
@@ -295,13 +303,20 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
                 queue: queue.length,
             }, 'processing handler queue');
             try {
-                await handler(item, {
-                    client: this.client,
-                    logger,
-                    subscriptionId: subId,
-                    extra,
-                    name: lid,
-                });
+                for (const batch of splitBy(item)) {
+                    await handler(batch, {
+                        client: this.client,
+                        logger: this.logger.child({
+                            subId,
+                            items: batch.items.map(i => i.id),
+                            extra,
+                            retryNumber: item.retry?.retryNumber,
+                        }),
+                        subscriptionId: subId,
+                        extra,
+                        name: lid,
+                    });
+                }
                 checkpoint.activeTasks--;
                 (0, assert_1.default)(checkpoint.activeTasks >= 0, 'internal: checkpoint.activeTasks < 0');
                 if (!checkpoint.activeTasks) {
@@ -313,9 +328,9 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
                 }, 'completed handler task');
             }
             catch (err) {
-                logger.error({ err }, 'error in handler,'
-                    + 'cancelling all active checkpoints'
-                    + '. Restarting from last known good cursor.');
+                logger.error({ err }, 'error in handler,' +
+                    'cancelling all active checkpoints' +
+                    '. Restarting from last known good cursor.');
                 this.#cancelAllActiveCheckpoints();
             }
             finally {
@@ -349,11 +364,11 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         await queries_ts_1.setGroupCursor.run({
             groupId: this.groupId,
             cursor: latestMaxCursor,
-            releaseLock: releaseLock
+            releaseLock: releaseLock,
         }, this.#readClient || this.client);
         this.logger.debug({
             cursor: latestMaxCursor,
-            activeCheckpoints: this.#activeCheckpoints.length
+            activeCheckpoints: this.#activeCheckpoints.length,
         }, 'set cursor');
         // if there are no more active checkpoints,
         // clear in-memory cursor, so in case another process takes
@@ -375,8 +390,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
             return;
         }
         try {
-            await queries_ts_1.releaseGroupLock
-                .run({ groupId: this.groupId }, this.#readClient);
+            await queries_ts_1.releaseGroupLock.run({ groupId: this.groupId }, this.#readClient);
         }
         catch (err) {
             this.logger.error({ err }, 'error releasing read client');
@@ -400,8 +414,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         if (cl !== this.#readClient) {
             return;
         }
-        this.logger
-            .info('dedicated read client disconnected, may have dup event processing');
+        this.logger.info('dedicated read client disconnected, may have dup event processing');
     };
     #releaseReadClient() {
         try {
@@ -430,3 +443,6 @@ exports.PgmbClient = PgmbClient;
 function createListenerId() {
     return Math.random().toString(16).slice(2, 10);
 }
+function defaultSplitBy(e) {
+    return [e];
+}

package/lib/types.d.ts

@@ -1,9 +1,11 @@
+import type { IDatabaseConnection } from '@pgtyped/runtime';
 import type { IncomingMessage } from 'node:http';
 import type { Logger } from 'pino';
 import type { HeaderRecord } from 'undici-types/header.js';
 import type { AbortableAsyncIterator } from './abortable-async-iterator.ts';
-import type { IAssertSubscriptionParams } from './queries.ts';
+import type { IAssertSubscriptionParams, IReadNextEventsParams, IReadNextEventsResult } from './queries.ts';
 import type { PgClientLike } from './query-types.ts';
+export type ISplitFn<T extends IEventData> = (event: IReadEvent<T>) => IReadEvent<T>[];
 export type SerialisedEvent = {
     body: Buffer | string;
     contentType: string;
@@ -17,7 +19,7 @@ export type GetWebhookInfoFn = (subscriptionIds: string[]) => Promise<{
 }> | {
     [id: string]: WebhookInfo[];
 };
-export type PgmbWebhookOpts = {
+export type PgmbWebhookOpts<T extends IEventData> = {
     /**
      * Maximum time to wait for webhook request to complete
      * @default 5 seconds
@@ -29,6 +31,7 @@ export type PgmbWebhookOpts = {
      * If null, a failed handler will fail the event processor. Use carefully.
      */
     retryOpts?: IRetryHandlerOpts | null;
+    splitBy?: ISplitFn<T>;
     jsonifier?: JSONifier;
     serialiseEvent?(ev: IReadEvent): SerialisedEvent;
 };
@@ -64,7 +67,8 @@ export type PGMBEventBatcherOpts<T extends IEventData> = {
      */
     maxBatchSize?: number;
 };
-export type Pgmb2ClientOpts = {
+export type IReadNextEventsFn = (parmas: IReadNextEventsParams, db: IDatabaseConnection) => Promise<IReadNextEventsResult[]>;
+export type Pgmb2ClientOpts<T extends IEventData> = {
     client: PgClientLike;
     /**
      * Globally unique identifier for this Pgmb2Client instance. All subs
@@ -100,22 +104,31 @@ export type Pgmb2ClientOpts = {
      * @default true
      */
     poll?: boolean;
-    webhookHandlerOpts?: Partial<PgmbWebhookOpts>;
+    webhookHandlerOpts?: Partial<PgmbWebhookOpts<T>>;
     getWebhookInfo?: GetWebhookInfoFn;
+    /**
+     * Override the default readNextEvents implementation
+     */
+    readNextEvents?: IReadNextEventsFn;
 } & Pick<PGMBEventBatcherOpts<IEventData>, 'flushIntervalMs' | 'maxBatchSize' | 'shouldLog'>;
 export type IReadEvent<T extends IEventData = IEventData> = {
     items: IEvent<T>[];
     retry?: IRetryEventPayload;
 };
 export type RegisterSubscriptionParams = Omit<IAssertSubscriptionParams, 'groupId'>;
-export type registerReliableHandlerParams = RegisterSubscriptionParams & {
+export type registerReliableHandlerParams<T extends IEventData = IEventData> = RegisterSubscriptionParams & {
     /**
      * Name for the retry handler, used to ensure retries for a particular
      * handler are not mixed with another handler. This name need only be
      * unique for a particular subscription.
-    */
+     */
     name?: string;
     retryOpts?: IRetryHandlerOpts;
+    /**
+     * If provided, will split an incoming event into multiple events
+     * as determined by the function.
+     */
+    splitBy?: ISplitFn<T>;
 };
 export type CreateTopicalSubscriptionOpts<T extends IEventData> = {
     /**
@@ -126,6 +139,9 @@ export type CreateTopicalSubscriptionOpts<T extends IEventData> = {
      * To scale out processing, you can partition the subscriptions.
      * For example, with `current: 0, total: 3`, only messages
      * where `hashtext(e.id) % 3 == 0` will be received by this subscription.
+     * This will result in an approximate even split for all processors, the only
+     * caveat being it requires knowing the number of event processors on this
+     * subscription beforehand.
      */
     partition?: {
         current: number;

package/lib/webhook-handler.d.ts

@@ -1,6 +1,6 @@
-import type { IEventHandler, PgmbWebhookOpts } from './types.ts';
+import type { IEventData, IEventHandler, PgmbWebhookOpts } from './types.ts';
 /**
  * Create a handler that sends events to a webhook URL via HTTP POST.
  * @param url Where to send the webhook requests
  */
-export declare function createWebhookHandler({ timeoutMs, headers, retryOpts, jsonifier, serialiseEvent }: Partial<PgmbWebhookOpts>): IEventHandler;
+export declare function createWebhookHandler<T extends IEventData>({ timeoutMs, headers, retryOpts, jsonifier, serialiseEvent }: Partial<PgmbWebhookOpts<T>>): IEventHandler;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@haathie/pgmb",
-  "version": "0.2.0",
-  "description": "Postgres Message Broker",
+  "version": "0.2.1",
+  "description": "PG message broker, with a type-safe typescript client with built-in webhook & SSE support.",
   "main": "lib/index.js",
   "publishConfig": {
     "registry": "https://registry.npmjs.org",