@haathie/pgmb 0.2.5 → 0.2.7

package/README.md

@@ -32,6 +32,9 @@ Install PGMB by running the following command:
 npm install @haathie/pgmb
 ```
 
+Note: PGMB directly exports typescript files, so if you're using a bundler -- ensure it can handle typescript files. NodeJs (v22+), Deno, Bun can all run typescript files natively -- so about time we utilised this!
+For commonjs compatibility, the compiled JS files are also exported.
+
 Before using PGMB, you'll need to run the setup script to create the required tables, functions & triggers in your database. You can do this by running:
 
 ```sh
@@ -494,6 +497,34 @@ const pgmb = new PgmbClient({
 })
 ```
 
+## Configuring Knobs
+
+PGMB provides a number of configuration options to tune its behaviour (eg. how often to poll for events, read events, expire subscriptions, etc.). These can be configured via relevant env vars too, the names for which can be found [here](src/client.ts#L105)
+``` ts
+// poll every 500ms
+process.env.PGMB_READ_EVENTS_INTERVAL_MS = '500' // default: 1000
+const pgmb = new PgmbClient(opts)
+```
+
+## Production Considerations
+
+PGMB relies on 2 functions that need to be run periodically & only once globally to ensure smooth operation, i.e.
+1. `poll_for_events()` -- finds unread events & assigns them to relevant subscriptions.
+	It's okay if this runs simultaneously in multiple processes, but that can create unnecessary contention on the `unread_events` table, which can bubble up to other tables.
+2. `maintain_events_table()` -- removes old partitions & creates new partitions for the events table. It's also okay if this runs simultaneously in multiple processes, as it has advisory locks to ensure only a single process is maintaining the events table at any time, but running this too frequently can cause unnecessary overhead.
+
+If you're only running a single instance of your service, you can simply run these functions in the same process as your PGMB client (default behaviour).
+However, for larger deployments with multiple instances of your service, it's recommended to run these functions in a separate process to avoid contention, and disable polling & table maintainence:
+``` ts
+const pgmb = new PgmbClient({
+	pollEventsIntervalMs: 0,
+	tableMaintainanceMs: 0,
+	...otherOpts
+})
+```
+
+Something like [pg_cron](https://github.com/citusdata/pg_cron) is a good option.
+
 ## General Notes
 
 - **Does the client automatically reconnect on errors & temporary network issues?**

package/lib/client.js

@@ -11,12 +11,14 @@ const abortable_async_iterator_ts_1 = require("./abortable-async-iterator.js");
 const batcher_ts_1 = require("./batcher.js");
 const queries_ts_1 = require("./queries.js");
 const retry_handler_ts_1 = require("./retry-handler.js");
+const utils_ts_1 = require("./utils.js");
 const webhook_handler_ts_1 = require("./webhook-handler.js");
 class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
     client;
     logger;
     groupId;
-    sleepDurationMs;
+    readEventsIntervalMs;
+    pollEventsIntervalMs;
     readChunkSize;
     subscriptionMaintenanceMs;
     tableMaintenanceMs;
@@ -29,14 +31,13 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
     #webhookHandlerOpts;
     #readClient;
     #endAc = new AbortController();
-    #shouldPoll;
     #readTask;
     #pollTask;
     #subMaintainTask;
     #tableMaintainTask;
     #inMemoryCursor = null;
     #activeCheckpoints = [];
-    constructor({ client, groupId, logger = (0, pino_1.pino)(), sleepDurationMs = 750, readChunkSize = 1000, maxActiveCheckpoints = 10, poll, subscriptionMaintenanceMs = 60 * 1000, webhookHandlerOpts: { splitBy: whSplitBy, ...whHandlerOpts } = {}, getWebhookInfo = () => ({}), tableMaintainanceMs = 5 * 60 * 1000, readNextEvents = queries_ts_1.readNextEvents.run.bind(queries_ts_1.readNextEvents), findEvents, ...batcherOpts }) {
+    constructor({ client, groupId, logger = (0, pino_1.pino)(), readEventsIntervalMs = (0, utils_ts_1.getEnvNumber)('PGMB_READ_EVENTS_INTERVAL_MS', 1000), readChunkSize = (0, utils_ts_1.getEnvNumber)('PGMB_READ_CHUNK_SIZE', 1000), maxActiveCheckpoints = (0, utils_ts_1.getEnvNumber)('PGMB_MAX_ACTIVE_CHECKPOINTS', 10), pollEventsIntervalMs = (0, utils_ts_1.getEnvNumber)('PGMB_POLL_EVENTS_INTERVAL_MS', 1000), subscriptionMaintenanceMs = (0, utils_ts_1.getEnvNumber)('PGMB_SUBSCRIPTION_MAINTENANCE_S', 60) * 1000, tableMaintainanceMs = (0, utils_ts_1.getEnvNumber)('PGMB_TABLE_MAINTENANCE_M', 15) * 60 * 1000, webhookHandlerOpts: { splitBy: whSplitBy, ...whHandlerOpts } = {}, getWebhookInfo = () => ({}), readNextEvents = queries_ts_1.readNextEvents.run.bind(queries_ts_1.readNextEvents), findEvents, ...batcherOpts }) {
         super({
             ...batcherOpts,
             logger,
@@ -45,9 +46,9 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         this.client = client;
         this.logger = logger;
         this.groupId = groupId;
-        this.sleepDurationMs = sleepDurationMs;
+        this.readEventsIntervalMs = readEventsIntervalMs;
         this.readChunkSize = readChunkSize;
-        this.#shouldPoll = !!poll;
+        this.pollEventsIntervalMs = pollEventsIntervalMs;
         this.subscriptionMaintenanceMs = subscriptionMaintenanceMs;
         this.maxActiveCheckpoints = maxActiveCheckpoints;
         this.webhookHandler = (0, webhook_handler_ts_1.createWebhookHandler)(whHandlerOpts);
@@ -70,9 +71,9 @@ class PgmbClient extends batcher_ts_1.PGMBEventBatcher {
         // clean up expired subscriptions on start
         const [{ deleted }] = await queries_ts_1.removeExpiredSubscriptions.run({ groupId: this.groupId, activeIds: [] }, this.client);
         this.logger.debug({ deleted }, 'removed expired subscriptions');
-        this.#readTask = this.#startLoop(this.readChanges.bind(this), this.sleepDurationMs);
-        if (this.#shouldPoll) {
-            this.#pollTask = this.#startLoop(queries_ts_1.pollForEvents.run.bind(queries_ts_1.pollForEvents, undefined, this.client), this.sleepDurationMs);
+        this.#readTask = this.#startLoop(this.readChanges.bind(this), this.readEventsIntervalMs);
+        if (this.pollEventsIntervalMs) {
+            this.#pollTask = this.#startLoop(queries_ts_1.pollForEvents.run.bind(queries_ts_1.pollForEvents, undefined, this.client), this.pollEventsIntervalMs);
         }
         if (this.subscriptionMaintenanceMs) {
             this.#subMaintainTask = this.#startLoop(this.#maintainSubscriptions, this.subscriptionMaintenanceMs);

package/lib/index.js

@@ -17,3 +17,4 @@ Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./client.js"), exports);
 __exportStar(require("./utils.js"), exports);
 __exportStar(require("./sse.js"), exports);
+__exportStar(require("./queries.js"), exports);

package/lib/utils.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.getDateFromMessageId = getDateFromMessageId;
 exports.getCreateDateFromSubscriptionId = getCreateDateFromSubscriptionId;
 exports.createTopicalSubscriptionParams = createTopicalSubscriptionParams;
+exports.getEnvNumber = getEnvNumber;
 const node_assert_1 = __importDefault(require("node:assert"));
 /**
  * Extract the date from a message ID, same as the PG function
@@ -50,3 +51,13 @@ function createTopicalSubscriptionParams({ topics, partition, additionalFilters
         ...rest
     };
 }
+/**
+ * Get an environment variable as a number
+ */
+function getEnvNumber(key, defaultValue = 0) {
+    const num = +(process.env[key] || defaultValue);
+    if (isNaN(num) || !isFinite(num)) {
+        return defaultValue;
+    }
+    return num;
+}

package/package.json

@@ -1,17 +1,23 @@
 {
   "name": "@haathie/pgmb",
-  "version": "0.2.5",
+  "version": "0.2.7",
   "description": "PG message broker, with a type-safe typescript client with built-in webhook & SSE support.",
-  "main": "lib/index.js",
   "publishConfig": {
     "registry": "https://registry.npmjs.org",
     "access": "public"
   },
+  "exports": {
+    ".": {
+      "import": "./src/index.ts",
+      "types": "./src/index.ts",
+      "require": "./lib/index.js"
+    }
+  },
   "repository": "https://github.com/haathie/pgmb",
   "scripts": {
     "test": "TZ=UTC NODE_ENV=test node --env-file ./.env.test --test tests/*.test.ts",
-    "prepare": "tsc",
-    "build": "tsc",
+    "prepare": "npm run build",
+    "build": "tsc -p tsconfig.build.json",
     "lint": "eslint ./ --ext .js,.ts,.jsx,.tsx",
     "lint:fix": "eslint ./ --fix --ext .js,.ts,.jsx,.tsx",
     "benchmark": "TZ=utc node --env-file ./.env.test src/benchmark/run.ts",
@@ -27,13 +33,14 @@
     "@types/pg": "^8.11.14",
     "amqplib": "^0.10.7",
     "chance": "^1.1.12",
-    "eslint": "^8.19.0",
+    "eslint": "^9.0.0",
     "eventsource": "^4.1.0",
     "pg": "^8.16.3",
     "typescript": "^5.0.0"
   },
   "files": [
     "lib",
+    "src",
     "sql"
   ],
   "keywords": [

package/src/abortable-async-iterator.ts

@@ -0,0 +1,98 @@
+import assert from 'assert'
+
+type AAResult<T> = IteratorResult<T>
+
+export class AbortableAsyncIterator<T> implements AsyncIterableIterator<T> {
+	readonly signal: AbortSignal
+	readonly onEnd: () => void
+
+	ended = false
+
+	#resolve: (() => void) | undefined
+	#reject: ((reason?: unknown) => void) | undefined
+	#queue: T[] = []
+	#locked = false
+
+	constructor(signal: AbortSignal, onEnd: () => void = () => {}) {
+		this.signal = signal
+		this.onEnd = onEnd
+		signal.addEventListener('abort', this.#onAbort)
+	}
+
+	async next(): Promise<AAResult<T>> {
+		assert(!this.ended, 'Iterator has already been completed')
+		assert(!this.#locked, 'Concurrent calls to next() are not allowed')
+
+		let nextItem = this.#queue.shift()
+		if(nextItem) {
+			return { value: nextItem, done: false }
+		}
+
+		this.#locked = true
+		try {
+			await this.#setupNextPromise()
+		} finally {
+			this.#locked = false
+		}
+
+		nextItem = this.#queue.shift()
+		if(nextItem) {
+			return { value: nextItem, done: false }
+		}
+
+		return { value: undefined, done: true }
+	}
+
+	enqueue(value: T) {
+		assert(!this.ended, 'Iterator has already been completed')
+		this.#queue.push(value)
+		this.#resolve?.()
+	}
+
+	throw(reason?: unknown): Promise<AAResult<T>> {
+		this.signal.throwIfAborted()
+		this.#reject?.(reason)
+		this.#end()
+		return Promise.resolve({ done: true, value: undefined })
+	}
+
+	return(value?: any): Promise<AAResult<T>> {
+		this.#resolve?.()
+		this.#end()
+		return Promise.resolve({ done: true, value })
+	}
+
+	#setupNextPromise() {
+		return new Promise<void>((resolve, reject) => {
+			this.#resolve = () => {
+				resolve()
+				this.#cleanupTask()
+			}
+
+			this.#reject = err => {
+				reject(err)
+				this.#cleanupTask()
+			}
+		})
+	}
+
+	#cleanupTask() {
+		this.#resolve = undefined
+		this.#reject = undefined
+	}
+
+	#onAbort = (reason: any) => {
+		this.#reject?.(reason)
+		this.#end()
+		this.ended = true
+	}
+
+	#end() {
+		this.signal.removeEventListener('abort', this.#onAbort)
+		this.onEnd()
+	}
+
+	[Symbol.asyncIterator]() {
+		return this
+	}
+}

package/src/batcher.ts

@@ -0,0 +1,90 @@
+import type { IEventData, PGMBEventBatcherOpts } from './types.ts'
+
+type Batch<T> = {
+	messages: T[]
+}
+
+export class PGMBEventBatcher<T extends IEventData> {
+
+	#publish: PGMBEventBatcherOpts<T>['publish']
+	#flushIntervalMs: number | undefined
+	#maxBatchSize: number
+	#currentBatch: Batch<T> = { messages: [] }
+	#flushTimeout: NodeJS.Timeout | undefined
+	#flushTask: Promise<void> | undefined
+	#logger: PGMBEventBatcherOpts<T>['logger']
+	#shouldLog?: PGMBEventBatcherOpts<T>['shouldLog']
+	#batch = 0
+
+	constructor({
+		shouldLog,
+		publish,
+		flushIntervalMs,
+		maxBatchSize = 2500,
+		logger
+	}: PGMBEventBatcherOpts<T>) {
+		this.#publish = publish
+		this.#flushIntervalMs = flushIntervalMs
+		this.#maxBatchSize = maxBatchSize
+		this.#logger = logger
+		this.#shouldLog = shouldLog
+	}
+
+	async end() {
+		clearTimeout(this.#flushTimeout)
+		await this.#flushTask
+		await this.flush()
+	}
+
+	/**
+	 * Enqueue a message to be published, will be flushed to the database
+	 * when flush() is called (either manually or via interval)
+	 */
+	enqueue(msg: T) {
+		this.#currentBatch.messages.push(msg)
+		if(this.#currentBatch.messages.length >= this.#maxBatchSize) {
+			this.flush()
+			return
+		}
+
+		if(this.#flushTimeout || !this.#flushIntervalMs) {
+			return
+		}
+
+		this.#flushTimeout = setTimeout(() => this.flush(), this.#flushIntervalMs)
+	}
+
+	async flush() {
+		if(!this.#currentBatch.messages.length) {
+			return
+		}
+
+		const batch = this.#currentBatch
+		this.#currentBatch = { messages: [] }
+		clearTimeout(this.#flushTimeout)
+		this.#flushTimeout = undefined
+
+		await this.#flushTask
+
+		this.#flushTask = this.#publishBatch(batch)
+		return this.#flushTask
+	}
+
+	async #publishBatch({ messages }: Batch<T>) {
+		const batch = ++this.#batch
+		try {
+			const ids = await this.#publish(...messages)
+			for(const [i, { id }] of ids.entries()) {
+				if(this.#shouldLog && !this.#shouldLog(messages[i])) {
+					continue
+				}
+
+				this.#logger
+					?.info({ batch, id, message: messages[i] }, 'published message')
+			}
+		} catch(err) {
+			this.#logger
+				?.error({ batch, err, msgs: messages }, 'failed to publish messages')
+		}
+	}
+}