@haathie/pgmb 0.2.0

package/lib/client.js

@@ -0,0 +1,432 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PgmbClient = void 0;
+const assert_1 = __importDefault(require("assert"));
+const pino_1 = require("pino");
+const promises_1 = require("timers/promises");
+const abortable_async_iterator_ts_1 = require("./abortable-async-iterator.js");
+const batcher_ts_1 = require("./batcher.js");
+const queries_ts_1 = require("./queries.js");
+const retry_handler_ts_1 = require("./retry-handler.js");
+const webhook_handler_ts_1 = require("./webhook-handler.js");
+class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
+    client;
+    logger;
+    groupId;
+    sleepDurationMs;
+    readChunkSize;
+    subscriptionMaintenanceMs;
+    tableMaintenanceMs;
+    maxActiveCheckpoints;
+    getWebhookInfo;
+    webhookHandler;
+    listeners = {};
+    #readClient;
+    #endAc = new AbortController();
+    #shouldPoll;
+    #readTask;
+    #pollTask;
+    #subMaintainTask;
+    #tableMaintainTask;
+    #inMemoryCursor = null;
+    #activeCheckpoints = [];
+    constructor({ client, groupId, logger = (0, pino_1.pino)(), sleepDurationMs = 750, readChunkSize = 1000, maxActiveCheckpoints = 10, poll, subscriptionMaintenanceMs = 60 * 1000, webhookHandlerOpts = {}, getWebhookInfo = () => ({}), tableMaintainanceMs = 5 * 60 * 1000, ...batcherOpts }) {
+        super({
+            ...batcherOpts,
+            logger,
+            publish: (...e) => this.publish(e)
+        });
+        this.client = client;
+        this.logger = logger;
+        this.groupId = groupId;
+        this.sleepDurationMs = sleepDurationMs;
+        this.readChunkSize = readChunkSize;
+        this.#shouldPoll = !!poll;
+        this.subscriptionMaintenanceMs = subscriptionMaintenanceMs;
+        this.maxActiveCheckpoints = maxActiveCheckpoints;
+        this.webhookHandler = (0, webhook_handler_ts_1.createWebhookHandler)(webhookHandlerOpts);
+        this.getWebhookInfo = getWebhookInfo;
+        this.tableMaintenanceMs = tableMaintainanceMs;
+    }
+    async init() {
+        this.#endAc = new AbortController();
+        if ('connect' in this.client) {
+            this.client.on('remove', this.#onPoolClientRemoved);
+        }
+        // maintain event table
+        await queries_ts_1.maintainEventsTable.run(undefined, this.client);
+        this.logger.debug('maintained events table');
+        await queries_ts_1.assertGroup.run({ id: this.groupId }, this.client);
+        this.logger.debug({ groupId: this.groupId }, 'asserted group exists');
+        // clean up expired subscriptions on start
+        const [{ deleted }] = await queries_ts_1.removeExpiredSubscriptions
+            .run({ groupId: this.groupId, activeIds: [] }, this.client);
+        this.logger.debug({ deleted }, 'removed expired subscriptions');
+        this.#readTask
+            = this.#startLoop(this.readChanges.bind(this), this.sleepDurationMs);
+        if (this.#shouldPoll) {
+            this.#pollTask = this.#startLoop(queries_ts_1.pollForEvents.run.bind(queries_ts_1.pollForEvents, undefined, this.client), this.sleepDurationMs);
+        }
+        if (this.subscriptionMaintenanceMs) {
+            this.#subMaintainTask = this.#startLoop(this.#maintainSubscriptions, this.subscriptionMaintenanceMs);
+        }
+        if (this.tableMaintenanceMs) {
+            this.#tableMaintainTask = this.#startLoop(queries_ts_1.maintainEventsTable.run
+                .bind(queries_ts_1.maintainEventsTable, undefined, this.client), this.tableMaintenanceMs);
+        }
+    }
+    async end() {
+        await super.end();
+        this.#endAc.abort();
+        while (this.#activeCheckpoints.length) {
+            await (0, promises_1.setTimeout)(100);
+        }
+        for (const id in this.listeners) {
+            delete this.listeners[id];
+        }
+        await Promise.all([
+            this.#readTask,
+            this.#pollTask,
+            this.#subMaintainTask,
+            this.#tableMaintainTask
+        ]);
+        await this.#unlockAndReleaseReadClient();
+        this.#readTask = undefined;
+        this.#pollTask = undefined;
+        this.#subMaintainTask = undefined;
+        this.#activeCheckpoints = [];
+    }
+    publish(events, client = this.client) {
+        return queries_ts_1.writeEvents.run({
+            topics: events.map(e => e.topic),
+            payloads: events.map(e => e.payload),
+            metadatas: events.map(e => e.metadata || null),
+        }, client);
+    }
+    async assertSubscription(opts, client = this.client) {
+        const [rslt] = await queries_ts_1.assertSubscription
+            .run({ ...opts, groupId: this.groupId }, client);
+        this.logger.debug({ ...opts, ...rslt }, 'asserted subscription');
+        return rslt;
+    }
+    /**
+     * Registers a fire-and-forget handler, returning an async iterator
+     * that yields events as they arrive. The client does not wait for event
+     * processing acknowledgements. Useful for cases where data is eventually
+     * consistent, or when event delivery isn't critical
+     * (eg. http SSE, websockets).
+     */
+    async registerFireAndForgetHandler(opts) {
+        const { id: subId } = await this.assertSubscription(opts);
+        return this.#listenForEvents(subId);
+    }
+    /**
+     * Registers a reliable handler for the given subscription params.
+     * If the handler throws an error, client will rollback to the last known
+     * good cursor, and re-deliver events.
+     * To avoid a full redelivery of a batch, a retry strategy can be provided
+     * to retry failed events by the handler itself, allowing for delayed retries
+     * with backoff, and without disrupting the overall event flow.
+     */
+    async registerReliableHandler({ retryOpts, name = createListenerId(), ...opts }, handler) {
+        const { id: subId } = await this.assertSubscription(opts);
+        if (retryOpts) {
+            handler = (0, retry_handler_ts_1.createRetryHandler)(retryOpts, handler);
+        }
+        const lts = (this.listeners[subId] ||= { values: {} });
+        (0, assert_1.default)(!lts.values[name], `Handler with id ${name} already registered for subscription ${subId}.`
+            + ' Cancel the existing one or use a different id.');
+        this.listeners[subId].values[name] = { type: 'reliable', handler, queue: [] };
+        return {
+            subscriptionId: subId,
+            cancel: () => this.#removeListener(subId, name)
+        };
+    }
+    async removeSubscription(subId) {
+        await queries_ts_1.deleteSubscriptions.run({ ids: [subId] }, this.client);
+        this.logger.debug({ subId }, 'deleted subscription');
+        const existingSubs = this.listeners[subId]?.values;
+        delete this.listeners[subId];
+        if (!existingSubs) {
+            return;
+        }
+        await Promise.allSettled(Object.values(existingSubs).map(e => (e.type === 'fire-and-forget'
+            && e.stream.throw(new Error('subscription removed')))));
+    }
+    #listenForEvents(subId) {
+        const lid = createListenerId();
+        const iterator = new abortable_async_iterator_ts_1.AbortableAsyncIterator(this.#endAc.signal, () => this.#removeListener(subId, lid));
+        const stream = iterator;
+        stream.id = subId;
+        this.listeners[subId] ||= { values: {} };
+        this.listeners[subId].values[lid] = { type: 'fire-and-forget', stream };
+        return stream;
+    }
+    #removeListener(subId, lid) {
+        const existingSubs = this.listeners[subId]?.values;
+        delete existingSubs?.[lid];
+        if (existingSubs && Object.keys(existingSubs).length) {
+            return;
+        }
+        delete this.listeners[subId];
+        this.logger.debug({ subId }, 'removed last subscriber for sub');
+    }
+    async #maintainSubscriptions() {
+        const activeIds = Object.keys(this.listeners);
+        await queries_ts_1.markSubscriptionsActive.run({ ids: activeIds }, this.client);
+        this.logger.trace({ activeSubscriptions: activeIds.length }, 'marked subscriptions as active');
+        const [{ deleted }] = await queries_ts_1.removeExpiredSubscriptions
+            .run({ groupId: this.groupId, activeIds }, this.client);
+        this.logger.trace({ deleted }, 'removed expired subscriptions');
+    }
+    async readChanges() {
+        if (this.#activeCheckpoints.length >= this.maxActiveCheckpoints) {
+            return 0;
+        }
+        const now = Date.now();
+        await this.#connectReadClient();
+        const rows = await queries_ts_1.readNextEvents.run({
+            groupId: this.groupId,
+            cursor: this.#inMemoryCursor,
+            chunkSize: this.readChunkSize
+        }, this.#readClient || this.client)
+            .catch(async (err) => {
+            if (err instanceof Error && err.message.includes('connection error')) {
+                await this.#unlockAndReleaseReadClient();
+            }
+            throw err;
+        });
+        if (!rows.length) {
+            // if nothing is happening and there are no active checkpoints,
+            // we can just let the read client go
+            if (!this.#activeCheckpoints.length) {
+                await this.#unlockAndReleaseReadClient();
+            }
+            return 0;
+        }
+        const uqSubIds = Array.from(new Set(rows.flatMap(r => r.subscriptionIds)));
+        const webhookSubs = await this.getWebhookInfo(uqSubIds);
+        let webhookCount = 0;
+        for (const sid in webhookSubs) {
+            const webhooks = webhookSubs[sid];
+            const lts = (this.listeners[sid] ||= { values: {} });
+            for (const wh of webhooks) {
+                // add reliable listener for each webhook
+                lts.values[wh.id] ||= {
+                    type: 'reliable',
+                    queue: [],
+                    extra: wh,
+                    removeOnEmpty: true,
+                    handler: this.webhookHandler
+                };
+                webhookCount++;
+            }
+        }
+        const { map: subToEventMap, retryEvents, retryItemCount } = await (0, retry_handler_ts_1.normaliseRetryEventsInReadEventMap)(rows, this.client);
+        const subs = Object.entries(subToEventMap);
+        const checkpoint = { activeTasks: 0, nextCursor: rows[0].nextCursor };
+        for (const [subId, evs] of subs) {
+            const listeners = this.listeners[subId]?.values;
+            if (!listeners) {
+                continue;
+            }
+            for (const ev of evs) {
+                for (const lid in listeners) {
+                    if (ev.retry?.handlerName && lid !== ev.retry.handlerName) {
+                        continue;
+                    }
+                    const lt = listeners[lid];
+                    if (lt.type === 'fire-and-forget') {
+                        lt.stream.enqueue(ev);
+                        continue;
+                    }
+                    this.#enqueueEventInReliableListener(subId, lid, ev, checkpoint);
+                }
+            }
+        }
+        this.#activeCheckpoints.push(checkpoint);
+        this.#inMemoryCursor = checkpoint.nextCursor;
+        this.logger.debug({
+            rowsRead: rows.length,
+            subscriptions: subs.length,
+            durationMs: Date.now() - now,
+            checkpoint,
+            activeCheckpoints: this.#activeCheckpoints.length,
+            webhookCount,
+            retryEvents,
+            retryItemCount
+        }, 'read rows');
+        if (!checkpoint.activeTasks && this.#activeCheckpoints.length === 1) {
+            await this.#updateCursorFromCompletedCheckpoints();
+        }
+        return rows.length;
+    }
+    /**
+     * Runs the reliable listener's handler for each item in its queue,
+     * one after the other, till the queue is empty or the client has ended.
+     * Any errors are logged, swallowed, and processing continues.
+     */
+    async #enqueueEventInReliableListener(subId, lid, item, checkpoint) {
+        const lt = this.listeners[subId]?.values?.[lid];
+        (0, assert_1.default)(lt?.type === 'reliable', 'invalid listener type: ' + lt.type);
+        const { handler, queue, removeOnEmpty, extra } = lt;
+        queue.push({ item, checkpoint });
+        checkpoint.activeTasks++;
+        if (queue.length > 1) {
+            return;
+        }
+        while (queue.length) {
+            const { item, checkpoint } = queue[0];
+            if (checkpoint.cancelled) {
+                queue.shift();
+                continue;
+            }
+            const logger = this.logger.child({
+                subId,
+                items: item.items.map(i => i.id),
+                extra,
+                retryNumber: item.retry?.retryNumber,
+            });
+            logger.trace({
+                cpActiveTasks: checkpoint.activeTasks,
+                queue: queue.length,
+            }, 'processing handler queue');
+            try {
+                await handler(item, {
+                    client: this.client,
+                    logger,
+                    subscriptionId: subId,
+                    extra,
+                    name: lid,
+                });
+                checkpoint.activeTasks--;
+                (0, assert_1.default)(checkpoint.activeTasks >= 0, 'internal: checkpoint.activeTasks < 0');
+                if (!checkpoint.activeTasks) {
+                    await this.#updateCursorFromCompletedCheckpoints();
+                }
+                logger.trace({
+                    cpActiveTasks: checkpoint.activeTasks,
+                    queue: queue.length,
+                }, 'completed handler task');
+            }
+            catch (err) {
+                logger.error({ err }, 'error in handler,'
+                    + 'cancelling all active checkpoints'
+                    + '. Restarting from last known good cursor.');
+                this.#cancelAllActiveCheckpoints();
+            }
+            finally {
+                queue.shift();
+            }
+        }
+        if (removeOnEmpty) {
+            return this.#removeListener(subId, lid);
+        }
+    }
+    /**
+     * Goes through all checkpoints, and sets the group cursor to the latest
+     * completed checkpoint. If a checkpoint has active tasks, stops there.
+     * This ensures that we don't accidentally move the cursor forward while
+     * there are still pending tasks for earlier checkpoints.
+     */
+    async #updateCursorFromCompletedCheckpoints() {
+        let latestMaxCursor;
+        while (this.#activeCheckpoints.length) {
+            const cp = this.#activeCheckpoints[0];
+            if (cp.activeTasks > 0) {
+                break;
+            }
+            latestMaxCursor = cp.nextCursor;
+            this.#activeCheckpoints.shift();
+        }
+        if (!latestMaxCursor) {
+            return;
+        }
+        const releaseLock = !this.#activeCheckpoints.length;
+        await queries_ts_1.setGroupCursor.run({
+            groupId: this.groupId,
+            cursor: latestMaxCursor,
+            releaseLock: releaseLock
+        }, this.#readClient || this.client);
+        this.logger.debug({
+            cursor: latestMaxCursor,
+            activeCheckpoints: this.#activeCheckpoints.length
+        }, 'set cursor');
+        // if there are no more active checkpoints,
+        // clear in-memory cursor, so in case another process takes
+        // over, if & when we start reading again, we read from the DB cursor
+        if (releaseLock) {
+            this.#inMemoryCursor = null;
+            this.#releaseReadClient();
+        }
+    }
+    #cancelAllActiveCheckpoints() {
+        for (const cp of this.#activeCheckpoints) {
+            cp.cancelled = true;
+        }
+        this.#activeCheckpoints = [];
+        this.#inMemoryCursor = null;
+    }
+    async #unlockAndReleaseReadClient() {
+        if (!this.#readClient) {
+            return;
+        }
+        try {
+            await queries_ts_1.releaseGroupLock
+                .run({ groupId: this.groupId }, this.#readClient);
+        }
+        catch (err) {
+            this.logger.error({ err }, 'error releasing read client');
+        }
+        finally {
+            this.#releaseReadClient();
+        }
+    }
+    async #connectReadClient() {
+        if (!('connect' in this.client)) {
+            return false;
+        }
+        if (this.#readClient) {
+            return true;
+        }
+        this.#readClient = await this.client.connect();
+        this.logger.trace('acquired dedicated read client');
+        return true;
+    }
+    #onPoolClientRemoved = async (cl) => {
+        if (cl !== this.#readClient) {
+            return;
+        }
+        this.logger
+            .info('dedicated read client disconnected, may have dup event processing');
+    };
+    #releaseReadClient() {
+        try {
+            this.#readClient?.release();
+        }
+        catch { }
+        this.#readClient = undefined;
+    }
+    async #startLoop(fn, sleepDurationMs) {
+        const signal = this.#endAc.signal;
+        while (!signal.aborted) {
+            try {
+                await (0, promises_1.setTimeout)(sleepDurationMs, undefined, { signal });
+                await fn.call(this);
+            }
+            catch (err) {
+                if (err instanceof Error && err.name === 'AbortError') {
+                    return;
+                }
+                this.logger.error({ err, fn: fn.name }, 'error in task');
+            }
+        }
+    }
+}
+exports.PgmbClient = PgmbClient;
+function createListenerId() {
+    return Math.random().toString(16).slice(2, 10);
+}

package/lib/consts.d.ts

@@ -0,0 +1 @@
+export declare const RETRY_EVENT = "pgmb-retry";

package/lib/consts.js

@@ -0,0 +1,4 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RETRY_EVENT = void 0;
+exports.RETRY_EVENT = 'pgmb-retry';

package/lib/index.d.ts

@@ -0,0 +1,5 @@
+export * from './client.ts';
+export type * from './types.ts';
+export type * from './query-types.ts';
+export * from './utils.ts';
+export * from './sse.ts';

package/lib/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./client.js"), exports);
+__exportStar(require("./utils.js"), exports);
+__exportStar(require("./sse.js"), exports);