@haathie/pgmb 0.2.8 → 0.2.10

package/README.md

@@ -32,8 +32,7 @@ Install PGMB by running the following command:
 npm install @haathie/pgmb
 ```
 
-Note: PGMB directly exports typescript files, so if you're using a bundler -- ensure it can handle typescript files. NodeJs (v22+), Deno, Bun can all run typescript files natively -- so about time we utilised this!
-For commonjs compatibility, the compiled JS files are also exported.
+Note: PGMB is an ESM module.
 
 Before using PGMB, you'll need to run the setup script to create the required tables, functions & triggers in your database. You can do this by running:
 

package/lib/abortable-async-iterator.d.ts

@@ -0,0 +1,14 @@
+type AAResult<T> = IteratorResult<T>;
+export declare class AbortableAsyncIterator<T> implements AsyncIterableIterator<T> {
+    #private;
+    readonly signal: AbortSignal;
+    readonly onEnd: () => void;
+    ended: boolean;
+    constructor(signal: AbortSignal, onEnd?: () => void);
+    next(): Promise<AAResult<T>>;
+    enqueue(value: T): void;
+    throw(reason?: unknown): Promise<AAResult<T>>;
+    return(value?: any): Promise<AAResult<T>>;
+    [Symbol.asyncIterator](): this;
+}
+export {};

package/lib/abortable-async-iterator.js

@@ -1,11 +1,5 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.AbortableAsyncIterator = void 0;
-const assert_1 = __importDefault(require("assert"));
-class AbortableAsyncIterator {
+import assert from 'assert';
+export class AbortableAsyncIterator {
     signal;
     onEnd;
     ended = false;
@@ -19,8 +13,8 @@ class AbortableAsyncIterator {
         signal.addEventListener('abort', this.#onAbort);
     }
     async next() {
-        (0, assert_1.default)(!this.ended, 'Iterator has already been completed');
-        (0, assert_1.default)(!this.#locked, 'Concurrent calls to next() are not allowed');
+        assert(!this.ended, 'Iterator has already been completed');
+        assert(!this.#locked, 'Concurrent calls to next() are not allowed');
         let nextItem = this.#queue.shift();
         if (nextItem) {
             return { value: nextItem, done: false };
@@ -39,7 +33,7 @@ class AbortableAsyncIterator {
         return { value: undefined, done: true };
     }
     enqueue(value) {
-        (0, assert_1.default)(!this.ended, 'Iterator has already been completed');
+        assert(!this.ended, 'Iterator has already been completed');
         this.#queue.push(value);
         this.#resolve?.();
     }
@@ -83,4 +77,3 @@ class AbortableAsyncIterator {
         return this;
     }
 }
-exports.AbortableAsyncIterator = AbortableAsyncIterator;

package/lib/batcher.d.ts

@@ -0,0 +1,12 @@
+import type { IEventData, PGMBEventBatcherOpts } from './types.ts';
+export declare class PGMBEventBatcher<T extends IEventData> {
+    #private;
+    constructor({ shouldLog, publish, flushIntervalMs, maxBatchSize, logger }: PGMBEventBatcherOpts<T>);
+    end(): Promise<void>;
+    /**
+     * Enqueue a message to be published, will be flushed to the database
+     * when flush() is called (either manually or via interval)
+     */
+    enqueue(msg: T): void;
+    flush(): Promise<void>;
+}

package/lib/batcher.js

@@ -1,7 +1,4 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PGMBEventBatcher = void 0;
-class PGMBEventBatcher {
+export class PGMBEventBatcher {
     #publish;
     #flushIntervalMs;
     #maxBatchSize;
@@ -68,4 +65,3 @@ class PGMBEventBatcher {
         }
     }
 }
-exports.PGMBEventBatcher = PGMBEventBatcher;

package/lib/client.d.ts

@@ -0,0 +1,77 @@
+import { type Logger } from 'pino';
+import { PGMBEventBatcher } from './batcher.ts';
+import type { PgClientLike } from './query-types.ts';
+import type { GetWebhookInfoFn, IEphemeralListener, IEventData, IEventHandler, IFindEventsFn, IReadEvent, IReadNextEventsFn, ISplitFn, Pgmb2ClientOpts, registerReliableHandlerParams, RegisterSubscriptionParams } from './types.ts';
+type IReliableListener<T extends IEventData> = {
+    type: 'reliable';
+    handler: IEventHandler<T>;
+    removeOnEmpty?: boolean;
+    extra?: unknown;
+    splitBy?: ISplitFn<T>;
+    queue: {
+        item: IReadEvent<T>;
+        checkpoint: Checkpoint;
+    }[];
+};
+type IFireAndForgetListener<T extends IEventData> = {
+    type: 'fire-and-forget';
+    stream: IEphemeralListener<T>;
+};
+type IListener<T extends IEventData> = IFireAndForgetListener<T> | IReliableListener<T>;
+type Checkpoint = {
+    activeTasks: number;
+    nextCursor: string;
+    cancelled?: boolean;
+};
+export type IListenerStore<T extends IEventData> = {
+    values: {
+        [id: string]: IListener<T>;
+    };
+};
+export declare class PgmbClient<T extends IEventData = IEventData> extends PGMBEventBatcher<T> {
+    #private;
+    readonly client: PgClientLike;
+    readonly logger: Logger;
+    readonly groupId: string;
+    readonly readEventsIntervalMs: number;
+    readonly pollEventsIntervalMs: number;
+    readonly readChunkSize: number;
+    readonly subscriptionMaintenanceMs: number;
+    readonly tableMaintenanceMs: number;
+    readonly maxActiveCheckpoints: number;
+    readonly readNextEvents: IReadNextEventsFn;
+    readonly findEvents?: IFindEventsFn;
+    readonly getWebhookInfo: GetWebhookInfoFn;
+    readonly webhookHandler: IEventHandler<T>;
+    readonly listeners: {
+        [subId: string]: IListenerStore<T>;
+    };
+    constructor({ client, groupId, logger, readEventsIntervalMs, readChunkSize, maxActiveCheckpoints, pollEventsIntervalMs, subscriptionMaintenanceMs, tableMaintainanceMs, webhookHandlerOpts: { splitBy: whSplitBy, ...whHandlerOpts }, getWebhookInfo, readNextEvents, findEvents, ...batcherOpts }: Pgmb2ClientOpts<T>);
+    init(): Promise<void>;
+    end(): Promise<void>;
+    publish(events: T[], client?: PgClientLike): Promise<import("./queries.ts").IWriteEventsResult[]>;
+    assertSubscription(opts: RegisterSubscriptionParams, client?: PgClientLike): Promise<import("./queries.ts").IAssertSubscriptionResult>;
+    /**
+     * Registers a fire-and-forget handler, returning an async iterator
+     * that yields events as they arrive. The client does not wait for event
+     * processing acknowledgements. Useful for cases where data is eventually
+     * consistent, or when event delivery isn't critical
+     * (eg. http SSE, websockets).
+     */
+    registerFireAndForgetHandler(opts: RegisterSubscriptionParams): Promise<IEphemeralListener<T>>;
+    /**
+     * Registers a reliable handler for the given subscription params.
+     * If the handler throws an error, client will rollback to the last known
+     * good cursor, and re-deliver events.
+     * To avoid a full redelivery of a batch, a retry strategy can be provided
+     * to retry failed events by the handler itself, allowing for delayed retries
+     * with backoff, and without disrupting the overall event flow.
+     */
+    registerReliableHandler({ retryOpts, name, splitBy, ...opts }: registerReliableHandlerParams<T>, handler: IEventHandler<T>): Promise<{
+        subscriptionId: string;
+        cancel: () => void;
+    }>;
+    removeSubscription(subId: string): Promise<void>;
+    readChanges(): Promise<number>;
+}
+export {};

package/lib/client.js

@@ -1,19 +1,13 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PgmbClient = void 0;
-const assert_1 = __importDefault(require("assert"));
-const pino_1 = require("pino");
-const promises_1 = require("timers/promises");
-const abortable_async_iterator_ts_1 = require("./abortable-async-iterator.js");
-const batcher_ts_1 = require("./batcher.js");
-const queries_ts_1 = require("./queries.js");
-const retry_handler_ts_1 = require("./retry-handler.js");
-const utils_ts_1 = require("./utils.js");
-const webhook_handler_ts_1 = require("./webhook-handler.js");
-class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
+import assert from 'assert';
+import { pino } from 'pino';
+import { setTimeout } from 'timers/promises';
+import { AbortableAsyncIterator } from "./abortable-async-iterator.js";
+import { PGMBEventBatcher } from "./batcher.js";
+import { assertGroup, assertSubscription, deleteSubscriptions, getConfigValue, maintainEventsTable, markSubscriptionsActive, pollForEvents, readNextEvents as defaultReadNextEvents, releaseGroupLock, removeExpiredSubscriptions, setGroupCursor, writeEvents, } from "./queries.js";
+import { createRetryHandler, normaliseRetryEventsInReadEventMap, } from "./retry-handler.js";
+import { getEnvNumber } from "./utils.js";
+import { createWebhookHandler } from "./webhook-handler.js";
+export class PgmbClient extends PGMBEventBatcher {
     client;
     logger;
     groupId;
@@ -37,7 +31,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
     #tableMaintainTask;
     #inMemoryCursor = null;
     #activeCheckpoints = [];
-    constructor({ client, groupId, logger = (0, pino_1.pino)(), readEventsIntervalMs = (0, utils_ts_1.getEnvNumber)('PGMB_READ_EVENTS_INTERVAL_MS', 1000), readChunkSize = (0, utils_ts_1.getEnvNumber)('PGMB_READ_CHUNK_SIZE', 1000), maxActiveCheckpoints = (0, utils_ts_1.getEnvNumber)('PGMB_MAX_ACTIVE_CHECKPOINTS', 10), pollEventsIntervalMs = (0, utils_ts_1.getEnvNumber)('PGMB_POLL_EVENTS_INTERVAL_MS', 1000), subscriptionMaintenanceMs = (0, utils_ts_1.getEnvNumber)('PGMB_SUBSCRIPTION_MAINTENANCE_S', 60) * 1000, tableMaintainanceMs = (0, utils_ts_1.getEnvNumber)('PGMB_TABLE_MAINTENANCE_M', 15) * 60 * 1000, webhookHandlerOpts: { splitBy: whSplitBy, ...whHandlerOpts } = {}, getWebhookInfo = () => ({}), readNextEvents = queries_ts_1.readNextEvents.run.bind(queries_ts_1.readNextEvents), findEvents, ...batcherOpts }) {
+    constructor({ client, groupId, logger = pino(), readEventsIntervalMs = getEnvNumber('PGMB_READ_EVENTS_INTERVAL_MS', 1000), readChunkSize = getEnvNumber('PGMB_READ_CHUNK_SIZE', 1000), maxActiveCheckpoints = getEnvNumber('PGMB_MAX_ACTIVE_CHECKPOINTS', 10), pollEventsIntervalMs = getEnvNumber('PGMB_POLL_EVENTS_INTERVAL_MS', 1000), subscriptionMaintenanceMs = getEnvNumber('PGMB_SUBSCRIPTION_MAINTENANCE_S', 60) * 1000, tableMaintainanceMs = getEnvNumber('PGMB_TABLE_MAINTENANCE_M', 15) * 60 * 1000, webhookHandlerOpts: { splitBy: whSplitBy, ...whHandlerOpts } = {}, getWebhookInfo = () => ({}), readNextEvents = defaultReadNextEvents.run.bind(defaultReadNextEvents), findEvents, ...batcherOpts }) {
         super({
             ...batcherOpts,
             logger,
@@ -51,7 +45,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         this.pollEventsIntervalMs = pollEventsIntervalMs;
         this.subscriptionMaintenanceMs = subscriptionMaintenanceMs;
         this.maxActiveCheckpoints = maxActiveCheckpoints;
-        this.webhookHandler = (0, webhook_handler_ts_1.createWebhookHandler)(whHandlerOpts);
+        this.webhookHandler = createWebhookHandler(whHandlerOpts);
         this.#webhookHandlerOpts = { splitBy: whSplitBy };
         this.getWebhookInfo = getWebhookInfo;
         this.tableMaintenanceMs = tableMaintainanceMs;
@@ -63,25 +57,25 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         if ('connect' in this.client) {
             this.client.on('remove', this.#onPoolClientRemoved);
         }
-        const [pgCronRslt] = await queries_ts_1.getConfigValue
+        const [pgCronRslt] = await getConfigValue
             .run({ key: 'use_pg_cron' }, this.client);
         const isPgCronEnabled = pgCronRslt?.value === 'true';
         if (!isPgCronEnabled) {
             // maintain event table
-            await queries_ts_1.maintainEventsTable.run(undefined, this.client);
+            await maintainEventsTable.run(undefined, this.client);
             this.logger.debug('maintained events table');
             if (this.pollEventsIntervalMs) {
-                this.#pollTask = this.#startLoop(queries_ts_1.pollForEvents.run.bind(queries_ts_1.pollForEvents, undefined, this.client), this.pollEventsIntervalMs);
+                this.#pollTask = this.#startLoop(pollForEvents.run.bind(pollForEvents, undefined, this.client), this.pollEventsIntervalMs);
             }
             if (this.tableMaintenanceMs) {
-                this.#tableMaintainTask = this.#startLoop(queries_ts_1.maintainEventsTable.run
-                    .bind(queries_ts_1.maintainEventsTable, undefined, this.client), this.tableMaintenanceMs);
+                this.#tableMaintainTask = this.#startLoop(maintainEventsTable.run
+                    .bind(maintainEventsTable, undefined, this.client), this.tableMaintenanceMs);
             }
         }
-        await queries_ts_1.assertGroup.run({ id: this.groupId }, this.client);
+        await assertGroup.run({ id: this.groupId }, this.client);
         this.logger.debug({ groupId: this.groupId }, 'asserted group exists');
         // clean up expired subscriptions on start
-        const [{ deleted }] = await queries_ts_1.removeExpiredSubscriptions.run({ groupId: this.groupId, activeIds: [] }, this.client);
+        const [{ deleted }] = await removeExpiredSubscriptions.run({ groupId: this.groupId, activeIds: [] }, this.client);
         this.logger.debug({ deleted }, 'removed expired subscriptions');
         this.#readTask = this.#startLoop(this.readChanges.bind(this), this.readEventsIntervalMs);
         if (this.subscriptionMaintenanceMs) {
@@ -93,7 +87,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         await super.end();
         this.#endAc.abort();
         while (this.#activeCheckpoints.length) {
-            await (0, promises_1.setTimeout)(100);
+            await setTimeout(100);
         }
         for (const id in this.listeners) {
             delete this.listeners[id];
@@ -111,14 +105,14 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         this.#activeCheckpoints = [];
     }
     publish(events, client = this.client) {
-        return queries_ts_1.writeEvents.run({
+        return writeEvents.run({
             topics: events.map((e) => e.topic),
             payloads: events.map((e) => e.payload),
             metadatas: events.map((e) => e.metadata || null),
         }, client);
     }
     async assertSubscription(opts, client = this.client) {
-        const [rslt] = await queries_ts_1.assertSubscription.run({ ...opts, groupId: this.groupId }, client);
+        const [rslt] = await assertSubscription.run({ ...opts, groupId: this.groupId }, client);
         this.logger.debug({ ...opts, ...rslt }, 'asserted subscription');
         return rslt;
     }
@@ -144,10 +138,10 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
     async registerReliableHandler({ retryOpts, name = createListenerId(), splitBy, ...opts }, handler) {
         const { id: subId } = await this.assertSubscription(opts);
         if (retryOpts) {
-            handler = (0, retry_handler_ts_1.createRetryHandler)(retryOpts, handler);
+            handler = createRetryHandler(retryOpts, handler);
         }
         const lts = (this.listeners[subId] ||= { values: {} });
-        (0, assert_1.default)(!lts.values[name], `Handler with id ${name} already registered for subscription ${subId}.` +
+        assert(!lts.values[name], `Handler with id ${name} already registered for subscription ${subId}.` +
             ' Cancel the existing one or use a different id.');
         this.listeners[subId].values[name] = {
             type: 'reliable',
@@ -161,7 +155,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         };
     }
     async removeSubscription(subId) {
-        await queries_ts_1.deleteSubscriptions.run({ ids: [subId] }, this.client);
+        await deleteSubscriptions.run({ ids: [subId] }, this.client);
         this.logger.debug({ subId }, 'deleted subscription');
         const existingSubs = this.listeners[subId]?.values;
         delete this.listeners[subId];
@@ -173,7 +167,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
     }
     #listenForEvents(subId) {
         const lid = createListenerId();
-        const iterator = new abortable_async_iterator_ts_1.AbortableAsyncIterator(this.#endAc.signal, () => this.#removeListener(subId, lid));
+        const iterator = new AbortableAsyncIterator(this.#endAc.signal, () => this.#removeListener(subId, lid));
         const stream = iterator;
         stream.id = subId;
         this.listeners[subId] ||= { values: {} };
@@ -191,9 +185,9 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
     }
     async #maintainSubscriptions() {
         const activeIds = Object.keys(this.listeners);
-        await queries_ts_1.markSubscriptionsActive.run({ ids: activeIds }, this.client);
+        await markSubscriptionsActive.run({ ids: activeIds }, this.client);
         this.logger.trace({ activeSubscriptions: activeIds.length }, 'marked subscriptions as active');
-        const [{ deleted }] = await queries_ts_1.removeExpiredSubscriptions.run({ groupId: this.groupId, activeIds }, this.client);
+        const [{ deleted }] = await removeExpiredSubscriptions.run({ groupId: this.groupId, activeIds }, this.client);
         this.logger.trace({ deleted }, 'removed expired subscriptions');
     }
     async readChanges() {
@@ -240,7 +234,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
                 webhookCount++;
             }
         }
-        const { map: subToEventMap, retryEvents, retryItemCount } = await (0, retry_handler_ts_1.normaliseRetryEventsInReadEventMap)(rows, this.client, this.findEvents);
+        const { map: subToEventMap, retryEvents, retryItemCount } = await normaliseRetryEventsInReadEventMap(rows, this.client, this.findEvents);
         const subs = Object.entries(subToEventMap);
         const checkpoint = {
             activeTasks: 0,
@@ -289,7 +283,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
      */
     async #enqueueEventInReliableListener(subId, lid, item, checkpoint) {
         const lt = this.listeners[subId]?.values?.[lid];
-        (0, assert_1.default)(lt?.type === 'reliable', 'invalid listener type: ' + lt.type);
+        assert(lt?.type === 'reliable', 'invalid listener type: ' + lt.type);
         const { handler, queue, removeOnEmpty, extra, splitBy = defaultSplitBy } = lt;
         queue.push({ item, checkpoint });
         checkpoint.activeTasks++;
@@ -328,7 +322,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
                     });
                 }
                 checkpoint.activeTasks--;
-                (0, assert_1.default)(checkpoint.activeTasks >= 0, 'internal: checkpoint.activeTasks < 0');
+                assert(checkpoint.activeTasks >= 0, 'internal: checkpoint.activeTasks < 0');
                 if (!checkpoint.activeTasks) {
                     await this.#updateCursorFromCompletedCheckpoints();
                 }
@@ -371,7 +365,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
             return;
         }
         const releaseLock = !this.#activeCheckpoints.length;
-        await queries_ts_1.setGroupCursor.run({
+        await setGroupCursor.run({
             groupId: this.groupId,
             cursor: latestMaxCursor,
             releaseLock: releaseLock,
@@ -400,7 +394,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
             return;
         }
         try {
-            await queries_ts_1.releaseGroupLock.run({ groupId: this.groupId }, this.#readClient);
+            await releaseGroupLock.run({ groupId: this.groupId }, this.#readClient);
         }
         catch (err) {
             this.logger.error({ err }, 'error releasing read client');
@@ -437,7 +431,7 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         const signal = this.#endAc.signal;
         while (!signal.aborted) {
             try {
-                await (0, promises_1.setTimeout)(sleepDurationMs, undefined, { signal });
+                await setTimeout(sleepDurationMs, undefined, { signal });
                 await fn.call(this);
             }
             catch (err) {
@@ -449,7 +443,6 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         }
     }
 }
-exports.PgmbClient = PgmbClient;
 function createListenerId() {
     return Math.random().toString(16).slice(2, 10);
 }

package/lib/consts.d.ts

@@ -0,0 +1 @@
+export declare const RETRY_EVENT = "pgmb-retry";

package/lib/consts.js

@@ -1,4 +1 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.RETRY_EVENT = void 0;
-exports.RETRY_EVENT = 'pgmb-retry';
+export const RETRY_EVENT = 'pgmb-retry';

package/lib/index.d.ts

@@ -0,0 +1,6 @@
+export * from './client.ts';
+export type * from './types.ts';
+export type * from './query-types.ts';
+export * from './utils.ts';
+export * from './sse.ts';
+export * from './queries.ts';

package/lib/index.js

@@ -1,20 +1,4 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./client.js"), exports);
-__exportStar(require("./utils.js"), exports);
-__exportStar(require("./sse.js"), exports);
-__exportStar(require("./queries.js"), exports);
+export * from "./client.js";
+export * from "./utils.js";
+export * from "./sse.js";
+export * from "./queries.js";